Harden ITs

 - increase Postgres max-connection configuration to mitigate Task test issue.
 - Fix the influx analytics test to avoid out of order test fixture.
 - stream line the docker-compose files to reduce repetition.

Author: tzolov

spring-cloud-dataflow-server/README.adoc

@@ -152,11 +152,11 @@ With the help of the `TestProperties` properties one can change the docker-compo
 SCDF, Skipper servers installed, or the versions Stream and Task apps:
 
 ----
-./mvnw clean test-compile failsafe:integration-test -pl spring-cloud-dataflow-server -Pfailsafe
-   -Dtest.docker.compose.paths=../src/docker-compose/docker-compose.yml,../src/docker-compose/docker-compose-influxdb.yml,../src/docker-compose/docker-compose-postgres.yml,../src/docker-compose/docker-compose-rabbitmq.yml
-   -Dtest.docker.compose.stream.apps.uri=https://dataflow.spring.io/rabbitmq-maven-latest
-   -Dtest.docker.compose.dataflow.version=2.8.0-SNAPSHOT
-   -Dtest.docker.compose.skipper.version=2.7.0-SNAPSHOT
+./mvnw clean test-compile failsafe:integration-test -pl spring-cloud-dataflow-server -Pfailsafe \
+   -Dtest.docker.compose.paths=../src/docker-compose/docker-compose.yml,../src/docker-compose/docker-compose-influxdb.yml,../src/docker-compose/docker-compose-postgres.yml,../src/docker-compose/docker-compose-rabbitmq.yml \
+   -Dtest.docker.compose.stream.apps.uri=https://dataflow.spring.io/rabbitmq-maven-latest \
+   -Dtest.docker.compose.dataflow.version=2.8.0-SNAPSHOT \
+   -Dtest.docker.compose.skipper.version=2.7.0-SNAPSHOT \
 ----
 The `test.docker.compose.paths` property accepts comma separated list of docker compose file names and supports `file:`, `classpath:`, and `http:`/`https:` URI schemas. If the schema prefix is not explicitly set, the file is treated as local one.
 

spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/DataFlowIT.java

@@ -77,7 +77,6 @@
 import org.springframework.util.StreamUtils;
 import org.springframework.web.client.RestTemplate;
 
-
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -644,9 +643,9 @@ public void analyticsCounter() {
 
 			Awaitility.await().until(() -> stream.getStatus().equals(DEPLOYED));
 
-			String message1 = "Test message 1";
-			String message2 = "Test message 2 with extension";
-			String message3 = "Test message 2 with double extension";
+			String message1 = "Test message 1"; // length 14
+			String message2 = "Test message 2 with extension";  // length 29
+			String message3 = "Test message 2 with double extension";  // length 36
 
 			String httpAppUrl = runtimeApps.getApplicationInstanceUrl(stream.getName(), "http");
 			httpPost(httpAppUrl, message1);
@@ -684,14 +683,14 @@ public void analyticsCounter() {
 						.inPath("$.results[0].series[0].values[1][0]")
 						.isEqualTo("myinfluxdb");
 
+				List<String> messageLengths = java.util.stream.Stream.of(message1, message2, message3)
+						.map(s -> String.format("\"%s\"", s.length())).collect(Collectors.toList());
+
 				// http://localhost:8086/query?db=myinfluxdb&q=SELECT%20%2A%20FROM%20%22my_http_counter%22
 				String myHttpCounter = httpGet(testProperties.getInfluxUrl() + "/query?db=myinfluxdb&q=SELECT * FROM \"my_http_analytics\"");
-				JsonAssertions.assertThatJson(myHttpCounter).inPath("$.results[0].series[0].values[0][7]")
-						.isEqualTo(String.format("\"%s\"", message1.length()));
-				JsonAssertions.assertThatJson(myHttpCounter).inPath("$.results[0].series[0].values[1][7]")
-						.isEqualTo(String.format("\"%s\"", message2.length()));
-				JsonAssertions.assertThatJson(myHttpCounter).inPath("$.results[0].series[0].values[2][7]")
-						.isEqualTo(String.format("\"%s\"", message3.length()));
+				JsonAssertions.assertThatJson(myHttpCounter).inPath("$.results[0].series[0].values[0][7]").isIn(messageLengths);
+				JsonAssertions.assertThatJson(myHttpCounter).inPath("$.results[0].series[0].values[1][7]").isIn(messageLengths);
+				JsonAssertions.assertThatJson(myHttpCounter).inPath("$.results[0].series[0].values[2][7]").isIn(messageLengths);
 			});
 		}
 	}

spring-cloud-dataflow-server/src/test/resources/docker-compose-it.yml

@@ -1,5 +1,7 @@
 version: '3'
 
+# Used by the CI runs to ensures that that DooD containers use the parent docker configuration and therefore
+# reusing the parent DockerHub credentials.
 services:
   dataflow-server:
     environment:

src/docker-compose/docker-compose-dood.yml

@@ -64,12 +64,17 @@ services:
 
   # Best effort to set default STREAM_APPS_URI and TASK_APPS_URI values. But because those can be overridden by other
   # composition YAMLs, it is highly recommended to set the STREAM_APPS_URI and TASK_APPS_URI explicitly!
-  app-import:
+  app-import-stream:
     command: >
       /bin/sh -c "
         ./wait-for-it.sh -t 360 dataflow-server:9393;
         wget -qO- 'http://dataflow-server:9393/apps' --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/kafka-docker-latest&force=true}';
-        echo 'Docker Stream apps imported'
+        echo 'Docker Stream apps imported'"
+
+  app-import-task:
+    command: >
+      /bin/sh -c "
+        ./wait-for-it.sh -t 360 dataflow-server:9393;
         wget -qO- 'http://dataflow-server:9393/apps' --post-data='uri=${TASK_APPS_URI:-https://dataflow.spring.io/task-docker-latest&force=true}';
         wget -qO- 'http://dataflow-server:9393/apps/task/scenario/0.0.1-SNAPSHOT' --post-data='uri=docker:springcloudtask/scenario-task:0.0.1-SNAPSHOT';
         echo 'Docker Task apps imported'"

src/docker-compose/docker-compose-postgres.yml

@@ -9,6 +9,7 @@ services:
 
   postgres:
     image: postgres:10
+    command: postgres -c 'max_connections=300'
     container_name: dataflow-postgres
     restart: always
     environment:

src/docker-compose/docker-compose-rabbitmq.yml

@@ -20,13 +20,10 @@ services:
     environment:
       - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_RABBITMQ_HOST=rabbitmq
 
-  app-import:
+  app-import-stream:
     command: >
       /bin/sh -c "
         ./wait-for-it.sh -t 360 dataflow-server:9393;
         wget -qO- 'http://dataflow-server:9393/apps' --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/rabbitmq-maven-latest&force=true}';
-        echo 'Stream apps imported'
-        wget -qO- 'http://dataflow-server:9393/apps' --post-data='uri=${TASK_APPS_URI:-https://dataflow.spring.io/task-maven-latest&force=true}';
-        wget -qO- 'http://dataflow-server:9393/apps/task/scenario/0.0.1-SNAPSHOT' --post-data='uri=maven://io.spring:scenario-task:0.0.1-SNAPSHOT';
-        echo 'Task apps imported'"
+        echo 'Stream apps imported'"
 

src/docker-compose/docker-compose.yml

@@ -81,7 +81,7 @@ services:
     volumes:
       - ${HOST_MOUNT_PATH:-.}:${DOCKER_MOUNT_PATH:-/root/scdf}
 
-  app-import:
+  app-import-stream:
     image: springcloud/baseimage:1.0.0
     container_name: dataflow-app-import
     depends_on:
@@ -90,7 +90,16 @@ services:
       /bin/sh -c "
         ./wait-for-it.sh -t 360 dataflow-server:9393;
         wget -qO- 'http://dataflow-server:9393/apps' --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/kafka-maven-latest&force=true}';
-        echo 'Maven Stream apps imported'
+        echo 'Maven Stream apps imported'"
+
+  app-import-task:
+    image: springcloud/baseimage:1.0.0
+    container_name: dataflow-app-import
+    depends_on:
+      - dataflow-server
+    command: >
+      /bin/sh -c "
+        ./wait-for-it.sh -t 360 dataflow-server:9393;
         wget -qO- 'http://dataflow-server:9393/apps' --post-data='uri=${TASK_APPS_URI:-https://dataflow.spring.io/task-maven-latest&force=true}';
         wget -qO- 'http://dataflow-server:9393/apps/task/scenario/0.0.1-SNAPSHOT' --post-data='uri=maven://io.spring:scenario-task:0.0.1-SNAPSHOT';
         echo 'Maven Task apps imported'"

src/templates/docker-compose/docker-compose-dood.yml

@@ -64,12 +64,17 @@ services:
 
   # Best effort to set default STREAM_APPS_URI and TASK_APPS_URI values. But because those can be overridden by other
   # composition YAMLs, it is highly recommended to set the STREAM_APPS_URI and TASK_APPS_URI explicitly!
-  app-import:
+  app-import-stream:
     command: >
       /bin/sh -c "
         ./wait-for-it.sh -t 360 dataflow-server:9393;
         wget -qO- 'http://dataflow-server:9393/apps' --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/kafka-docker-latest&force=true}';
-        echo 'Docker Stream apps imported'
+        echo 'Docker Stream apps imported'"
+
+  app-import-task:
+    command: >
+      /bin/sh -c "
+        ./wait-for-it.sh -t 360 dataflow-server:9393;
         wget -qO- 'http://dataflow-server:9393/apps' --post-data='uri=${TASK_APPS_URI:-https://dataflow.spring.io/task-docker-latest&force=true}';
         wget -qO- 'http://dataflow-server:9393/apps/task/scenario/0.0.1-SNAPSHOT' --post-data='uri=docker:springcloudtask/scenario-task:0.0.1-SNAPSHOT';
         echo 'Docker Task apps imported'"

src/templates/docker-compose/docker-compose-postgres.yml

@@ -9,6 +9,7 @@ services:
 
   postgres:
     image: postgres:10
+    command: postgres -c 'max_connections=300'
     container_name: dataflow-postgres
     restart: always
     environment:

src/templates/docker-compose/docker-compose-rabbitmq.yml

@@ -20,13 +20,10 @@ services:
     environment:
       - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_RABBITMQ_HOST=rabbitmq
 
-  app-import:
+  app-import-stream:
     command: >
       /bin/sh -c "
         ./wait-for-it.sh -t 360 dataflow-server:9393;
         wget -qO- 'http://dataflow-server:9393/apps' --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/rabbitmq-maven-latest&force=true}';
-        echo 'Stream apps imported'
-        wget -qO- 'http://dataflow-server:9393/apps' --post-data='uri=${TASK_APPS_URI:-https://dataflow.spring.io/task-maven-latest&force=true}';
-        wget -qO- 'http://dataflow-server:9393/apps/task/scenario/0.0.1-SNAPSHOT' --post-data='uri=maven://io.spring:scenario-task:0.0.1-SNAPSHOT';
-        echo 'Task apps imported'"
+        echo 'Stream apps imported'"