Update k8s deployment files

src/docker-compose/.env

@@ -1,5 +1,5 @@
 
-DATAFLOW_VERSION=2.11.3-SNAPSHOT
-SKIPPER_VERSION=2.11.3-SNAPSHOT
+DATAFLOW_VERSION=2.11.6-SNAPSHOT
+SKIPPER_VERSION=2.11.6-SNAPSHOT
 COMPOSE_PROJECT_NAME=scdf
 COMPOSE_HTTP_TIMEOUT=300

src/docker-compose/docker-compose-dood.yml

@@ -40,14 +40,8 @@ services:
       - SPRING_CLOUD_DATAFLOW_TASK_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_DOCKER_PORTRANGE_HIGH=61000
 
       # Override the CTR form maven to docker
-      - SPRING_CLOUD_DATAFLOW_TASK_COMPOSEDTASKRUNNER_URI=docker://springcloud/spring-cloud-dataflow-composed-task-runner:${DATAFLOW_VERSION:-2.11.3-SNAPSHOT}${BP_JVM_VERSION:-}
+      - SPRING_CLOUD_DATAFLOW_TASK_COMPOSEDTASKRUNNER_URI=docker://springcloud/spring-cloud-dataflow-composed-task-runner:${DATAFLOW_VERSION:-@project.version@}${BP_JVM_VERSION:-}
       - SPRING_CLOUD_DATAFLOW_SERVER_URI=${DATAFLOW_URI:-http://dataflow-server:9393}
-    entrypoint: >
-      /bin/sh -c "
-         apt-get update && apt-get install --no-install-recommends -y wget &&
-         wget --no-check-certificate -P /tmp/ https://get.docker.com/builds/Linux/x86_64/docker-latest.tgz &&
-         tar -xvf /tmp/docker-latest.tgz --directory /tmp/ &&
-         mv /tmp/docker/docker /usr/local/bin && /cnb/process/web"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
 
@@ -61,12 +55,6 @@ services:
       # in the docker-compose.yaml#skipper-server. Tip: you can change the APPS_PORT_RANGE instead.
       - SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_DOCKER_PORTRANGE_LOW=20200
       - SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_DOCKER_PORTRANGE_HIGH=61000
-    entrypoint: >
-      /bin/sh -c "
-         apt-get update && apt-get install --no-install-recommends -y wget &&
-         wget --no-check-certificate -P /tmp/ https://get.docker.com/builds/Linux/x86_64/docker-latest.tgz &&
-         tar -xvf /tmp/docker-latest.tgz --directory /tmp/ &&
-         mv /tmp/docker/docker /usr/local/bin && /cnb/process/web"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
 
@@ -76,15 +64,13 @@ services:
     command: >
       /bin/sh -c "
         ./wait-for-it.sh -t 360 dataflow-server:9393;
-        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/kafka-docker-latest&force=true}';               
-        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:-2.11.3-SNAPSHOT}' --no-check-certificate --post-data='uri=docker:springcloud/spring-cloud-dataflow-tasklauncher-sink-kafka:${DATAFLOW_VERSION:-2.11.3-SNAPSHOT}${BP_JVM_VERSION:-}';
+        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/kafka-docker-latest&force=true}';
+                wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:-@project.version@}' --no-check-certificate --post-data='uri=docker:springcloud/spring-cloud-dataflow-tasklauncher-sink-kafka:${DATAFLOW_VERSION:-@project.version@}${BP_JVM_VERSION:-}';
         echo 'Docker Stream apps imported'"
 
   app-import-task:
     command: >
       /bin/sh -c "
         ./wait-for-it.sh -t 360 dataflow-server:9393;
         wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate  --post-data='uri=${TASK_APPS_URI:-https://dataflow.spring.io/task-docker-latest&force=true}';
-        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/timestamp3' --no-check-certificate --post-data='bootVersion=3&uri=docker:springcloudtask/timestamp-task:3.0.0';
-        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/timestamp-batch3' --no-check-certificate --post-data='bootVersion=3&uri=docker:springcloudtask/timestamp-batch-task:3.0.0';
         echo 'Docker Task apps imported'"

src/docker-compose/docker-compose-influxdb.yml

@@ -19,13 +19,13 @@ services:
         }
 
   influxdb:
-    image: influxdb:1.8
+    image: influxdb:1.7.7
     container_name: influxdb
     ports:
       - '8086:8086'
 
   grafana:
-    image: springcloud/spring-cloud-dataflow-grafana-influxdb:${DATAFLOW_VERSION:-2.11.3-SNAPSHOT}
+    image: springcloud/spring-cloud-dataflow-grafana-influxdb:${DATAFLOW_VERSION:-@project.version@}
     container_name: grafana
     ports:
       - '3000:3000'

src/docker-compose/docker-compose-postgres.yml

@@ -4,36 +4,34 @@ version: '3'
 # Usage: docker-compose -f ./docker-compose.yml -f ./docker-compose-postgres.yml up
 services:
 
+  mariadb:
+    entrypoint: ["echo", "MariaDB service is disabled"]
+
   postgres:
     image: postgres:14
     command: postgres -c 'max_connections=300'
     container_name: dataflow-postgres
     restart: always
     environment:
-      LANG: en_US.utf8
-      LC_ALL: en_US.utf8
-      JDK_JAVA_OPTIONS: '-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8'
       POSTGRES_DB: dataflow
       POSTGRES_USER: root
       POSTGRES_PASSWORD: rootpw
     expose:
       - 5432
     ports:
-      - '5432:5432'
+      - 5432:5432
 
   dataflow-server:
     environment:
       - SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/dataflow
       - SPRING_DATASOURCE_USERNAME=root
       - SPRING_DATASOURCE_PASSWORD=rootpw
       - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
-    depends_on:
-      - postgres
+
   skipper-server:
     environment:
       - SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/dataflow
       - SPRING_DATASOURCE_USERNAME=root
       - SPRING_DATASOURCE_PASSWORD=rootpw
       - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
-    depends_on:
-      - postgres
+

src/docker-compose/docker-compose-prometheus.yml

@@ -38,13 +38,13 @@ services:
       - micrometer.prometheus-proxy.tcp-port=7001
 
   grafana:
-    image: springcloud/spring-cloud-dataflow-grafana-prometheus:${DATAFLOW_VERSION:-2.11.3-SNAPSHOT}
+    image: springcloud/spring-cloud-dataflow-grafana-prometheus:${DATAFLOW_VERSION:-@project.version@}
     container_name: grafana
     ports:
       - '3000:3000'
 
   prometheus:
-    image: springcloud/spring-cloud-dataflow-prometheus-local:${DATAFLOW_VERSION:-2.11.3-SNAPSHOT}
+    image: springcloud/spring-cloud-dataflow-prometheus-local:${DATAFLOW_VERSION:-@project.version@}
     container_name: prometheus
     ports:
       - '9090:9090'

src/docker-compose/docker-compose-rabbitmq.yml

@@ -4,34 +4,29 @@ version: '3'
 # Usage: docker-compose -f ./docker-compose.yml -f ./docker-compose-rabbitmq.yml up
 services:
 
+  kafka-broker:
+    entrypoint: ["echo", "kafka-broker service is disabled"]
+
+  zookeeper:
+    entrypoint: ["echo", "zookeeper service is disabled"]
+
   rabbitmq:
-    image: rabbitmq:3.8-management
+    image: rabbitmq:3.7
     container_name: dataflow-rabbitmq
-    environment:
-      LANG: en_US.utf8
-      LC_ALL: en_US.utf8
-      JDK_JAVA_OPTIONS: '-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8'
     expose:
       - '5672'
-      - '15672'
 
   dataflow-server:
     environment:
       - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_RABBITMQ_HOST=rabbitmq
-    depends_on:
-      - rabbitmq
-
-  skipper-server:
-    environment:
-      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_RABBITMQ_HOST=rabbitmq
-    depends_on:
-      - rabbitmq
 
   app-import-stream:
     command: >
       /bin/sh -c "
         ./wait-for-it.sh -t 360 dataflow-server:9393;
         wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/rabbitmq-maven-latest&force=true}';
-        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:-2.11.3-SNAPSHOT}' --no-check-certificate --post-data='uri=maven://org.springframework.cloud:spring-cloud-dataflow-tasklauncher-sink-rabbit:${DATAFLOW_VERSION:-2.11.3-SNAPSHOT}';
+        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/ver-log/3.0.1' --no-check-certificate --post-data='uri=maven://org.springframework.cloud.stream.app:log-sink-rabbit:3.0.1';
+        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/ver-log/2.1.5.RELEASE' --no-check-certificate --post-data='uri=maven://org.springframework.cloud.stream.app:log-sink-rabbit:2.1.5.RELEASE';
+        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:[email protected]@}' --no-check-certificate --post-data='uri=maven://org.springframework.cloud:spring-cloud-dataflow-tasklauncher-sink-rabbit:${DATAFLOW_VERSION:[email protected]@}';
         echo 'Stream apps imported'"
 

src/docker-compose/docker-compose-zipkin.yml

@@ -6,7 +6,7 @@ version: '3'
 services:
 
   zipkin-server:
-    image: openzipkin/zipkin:2
+    image: openzipkin/zipkin:2.23.2
     container_name: zipkin-server
     expose:
       - '9411'

src/docker-compose/docker-compose.yml

@@ -1,4 +1,5 @@
 version: '3'
+
 # Configuration environment variables:
 # - DATAFLOW_VERSION and SKIPPER_VERSION specify what DataFlow and Skipper image versions to use.
 # - STREAM_APPS_URI and TASK_APPS_URI are used to specify what Stream and Task applications to pre-register.
@@ -19,25 +20,66 @@ version: '3'
 #                             The deployed stream applications that run in their own docker containers (e.g. docker:// registered apps),
 #                             can be reached on the ports they expose.
 services:
+  mariadb:
+    image: mariadb:10.6
+    container_name: dataflow-mariadb
+    environment:
+      MARIADB_DATABASE: dataflow
+      MARIADB_ROOT_PASSWORD: rootpw
+    expose:
+      - 3306
+
+  kafka-broker:
+    image: confluentinc/cp-kafka:5.5.2
+    container_name: dataflow-kafka
+    expose:
+      - "9092"
+    environment:
+      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka-broker:9092
+      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+      - KAFKA_ADVERTISED_HOST_NAME=kafka-broker
+      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+      - KAFKA_LOG4J_ROOT_LOGLEVEL=ERROR
+      - KAFKA_LOG4J_LOGGERS=org.apache.zookeeper=ERROR,org.apache.kafka=ERROR,kafka=ERROR,kafka.cluster=ERROR,kafka.controller=ERROR,kafka.coordinator=ERROR,kafka.log=ERROR,kafka.server=ERROR,kafka.zookeeper=ERROR,state.change.logger=ERROR
+    depends_on:
+      - zookeeper
+
+  zookeeper:
+    image: confluentinc/cp-zookeeper:5.5.2
+    container_name: dataflow-kafka-zookeeper
+    expose:
+      - "2181"
+    environment:
+      - ZOOKEEPER_CLIENT_PORT=2181
+
   dataflow-server:
     user: root
-    image: springcloud/spring-cloud-dataflow-server:${DATAFLOW_VERSION:-2.11.3-SNAPSHOT}${BP_JVM_VERSION:-}
+    image: springcloud/spring-cloud-dataflow-server:${DATAFLOW_VERSION:-@project.version@}${BP_JVM_VERSION:-}
     container_name: dataflow-server
     ports:
       - "9393:9393"
     environment:
-      - LANG=en_US.utf8
-      - LC_ALL=en_US.utf8
-      - JDK_JAVA_OPTIONS=-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8
+      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS=PLAINTEXT://kafka-broker:9092
+      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_STREAMS_BINDER_BROKERS=PLAINTEXT://kafka-broker:9092
+      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_BINDER_ZKNODES=zookeeper:2181
+      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_STREAMS_BINDER_ZKNODES=zookeeper:2181
+      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_KAFKA_STREAMS_PROPERTIES_METRICS_RECORDING_LEVEL=DEBUG
       # Set CLOSECONTEXTENABLED=true to ensure that the CRT launcher is closed.
       - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_TASK_SPRING_CLOUD_TASK_CLOSECONTEXTENABLED=true
       - SPRING_CLOUD_SKIPPER_CLIENT_SERVER_URI=${SKIPPER_URI:-http://skipper-server:7577}/api
+      - SPRING_DATASOURCE_URL=jdbc:mariadb://mariadb:3306/dataflow
+      - SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.MariaDB106Dialect
+      - SPRING_DATASOURCE_USERNAME=root
+      - SPRING_DATASOURCE_PASSWORD=rootpw
+      - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver
       # (Optionally) authenticate the default Docker Hub access for the App Metadata access.
       - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRY_CONFIGURATIONS_DEFAULT_USER=${METADATA_DEFAULT_DOCKERHUB_USER}
       - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRY_CONFIGURATIONS_DEFAULT_SECRET=${METADATA_DEFAULT_DOCKERHUB_PASSWORD}
+
       - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRYCONFIGURATIONS_DEFAULT_USER=${METADATA_DEFAULT_DOCKERHUB_USER}
       - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRYCONFIGURATIONS_DEFAULT_SECRET=${METADATA_DEFAULT_DOCKERHUB_PASSWORD}
     depends_on:
+      - kafka-broker
       - skipper-server
     restart: always
     volumes:
@@ -48,6 +90,12 @@ services:
     container_name: dataflow-app-import-stream
     depends_on:
       - dataflow-server
+    command: >
+      /bin/sh -c "
+        ./wait-for-it.sh -t 360 dataflow-server:9393;
+        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/kafka-maven-latest&force=true}';
+        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:[email protected]@}' --no-check-certificate --post-data='uri=maven://org.springframework.cloud:spring-cloud-dataflow-tasklauncher-sink-kafka:${DATAFLOW_VERSION:[email protected]@}';
+        echo 'Maven Stream apps imported'"
 
   app-import-task:
     image: springcloud/baseimage:1.0.4
@@ -58,24 +106,23 @@ services:
       /bin/sh -c "
         ./wait-for-it.sh -t 360 dataflow-server:9393;
         wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${TASK_APPS_URI:-https://dataflow.spring.io/task-maven-latest&force=true}';
-        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/timestamp3' --no-check-certificate --post-data='bootVersion=3&uri=maven://uri=maven:io.spring:timestamp-task:3.0.0';
-        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/timestamp-batch3' --no-check-certificate --post-data='bootVersion=3&uri=maven://uri=maven:io.spring:timestamp-batch:3.0.0';
         echo 'Maven Task apps imported'"
 
   skipper-server:
     user: root
-    image: springcloud/spring-cloud-skipper-server:${SKIPPER_VERSION:-2.11.3-SNAPSHOT}${BP_JVM_VERSION:-}
-    container_name: skipper-server
+    image: springcloud/spring-cloud-skipper-server:${SKIPPER_VERSION:-@spring-cloud-skipper.version@}${BP_JVM_VERSION:-}
+    container_name: skipper
     ports:
       - "7577:7577"
       - ${APPS_PORT_RANGE:-20000-20195:20000-20195}
     environment:
-      - LANG=en_US.utf8
-      - LC_ALL=en_US.utf8
-      - JDK_JAVA_OPTIONS=-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8
-      - SERVER_PORT=7577
       - SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_PORTRANGE_LOW=20000
       - SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_PORTRANGE_HIGH=20190
+      - SPRING_DATASOURCE_URL=jdbc:mariadb://mariadb:3306/dataflow
+      - SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.MariaDB106Dialect
+      - SPRING_DATASOURCE_USERNAME=root
+      - SPRING_DATASOURCE_PASSWORD=rootpw
+      - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver
       - LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_SKIPPER_SERVER_DEPLOYER=ERROR
     restart: always
     volumes:

src/kubernetes/grafana/grafana-deployment.yaml

@@ -14,7 +14,7 @@ spec:
         app: grafana
     spec:
       containers:
-        - image: springcloud/spring-cloud-dataflow-grafana-prometheus:2.11.4
+        - image: springcloud/spring-cloud-dataflow-grafana-prometheus:2.11.6-SNAPSHOT
           name: grafana
           env:
             - name: GF_SECURITY_ADMIN_USER
@@ -30,12 +30,9 @@ spec:
           ports:
             - containerPort: 3000
           resources:
-            limits:
-              cpu: 500m
-              memory: 2500Mi
             requests:
-              cpu: 100m
-              memory: 100Mi
+              cpu: 500m
+              memory: 256Mi
           volumeMounts:
             - name: config
               mountPath: "/etc/grafana/provisioning/datasources/datasources.yaml"

src/kubernetes/kafka/kafka-deployment.yaml

@@ -20,7 +20,6 @@ spec:
       containers:
       - name: kafka
         image: confluentinc/cp-kafka:5.5.2
-        imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9092
         env:

src/kubernetes/kafka/kafka-svc.yaml

@@ -6,6 +6,7 @@ metadata:
     app: kafka
     component: kafka-broker
 spec:
+  clusterIP: None
   ports:
   - port: 9092
     name: kafka-port

src/kubernetes/kafka/kafka-zk-deployment.yaml

@@ -19,7 +19,6 @@ spec:
       containers:
       - name: kafka-zk
         image: confluentinc/cp-zookeeper:5.5.2
-        imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 2181
         env: