Update k8s deployment files

Author: scdf-build-bot

src/docker-compose/docker-compose-postgres.yml

@@ -4,6 +4,9 @@ version: '3'
 # Usage: docker-compose -f ./docker-compose.yml -f ./docker-compose-postgres.yml up
 services:
 
+  mariadb:
+    entrypoint: ["echo", "MariaDB service is disabled"]
+
   postgres:
     image: postgres:10
     command: postgres -c 'max_connections=300'
@@ -24,13 +27,11 @@ services:
       - SPRING_DATASOURCE_USERNAME=root
       - SPRING_DATASOURCE_PASSWORD=rootpw
       - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
-    depends_on:
-      - postgres
+
   skipper-server:
     environment:
       - SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/dataflow
       - SPRING_DATASOURCE_USERNAME=root
       - SPRING_DATASOURCE_PASSWORD=rootpw
       - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
-    depends_on:
-      - postgres
+

src/docker-compose/docker-compose-rabbitmq.yml

@@ -19,8 +19,6 @@ services:
   dataflow-server:
     environment:
       - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_RABBITMQ_HOST=rabbitmq
-    depends_on:
-      - rabbitmq
 
   app-import-stream:
     command: >

src/docker-compose/docker-compose.yml

@@ -20,6 +20,38 @@ version: '3'
 #                             The deployed stream applications that run in their own docker containers (e.g. docker:// registered apps),
 #                             can be reached on the ports they expose.
 services:
+  mariadb:
+    image: mariadb:10.4.22
+    container_name: dataflow-mariadb
+    environment:
+      MARIADB_DATABASE: dataflow
+      MARIADB_ROOT_PASSWORD: rootpw
+    expose:
+      - 3306
+
+  kafka-broker:
+    image: confluentinc/cp-kafka:5.5.2
+    container_name: dataflow-kafka
+    expose:
+      - "9092"
+    environment:
+      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka-broker:9092
+      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+      - KAFKA_ADVERTISED_HOST_NAME=kafka-broker
+      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+      - KAFKA_LOG4J_ROOT_LOGLEVEL=ERROR
+      - KAFKA_LOG4J_LOGGERS=org.apache.zookeeper=ERROR,org.apache.kafka=ERROR,kafka=ERROR,kafka.cluster=ERROR,kafka.controller=ERROR,kafka.coordinator=ERROR,kafka.log=ERROR,kafka.server=ERROR,kafka.zookeeper=ERROR,state.change.logger=ERROR
+    depends_on:
+      - zookeeper
+
+  zookeeper:
+    image: confluentinc/cp-zookeeper:5.5.2
+    container_name: dataflow-kafka-zookeeper
+    expose:
+      - "2181"
+    environment:
+      - ZOOKEEPER_CLIENT_PORT=2181
+
   dataflow-server:
     user: root
     image: springcloud/spring-cloud-dataflow-server:${DATAFLOW_VERSION:-2.10.0-SNAPSHOT}${BP_JVM_VERSION:-}
@@ -38,14 +70,25 @@ services:
 
       - SPRING_CLOUD_SKIPPER_CLIENT_SERVER_URI=${SKIPPER_URI:-http://skipper-server:7577}/api
 
+      - SPRING_DATASOURCE_URL=jdbc:mariadb://mariadb:3306/dataflow
+      - SPRING_DATASOURCE_USERNAME=root
+      - SPRING_DATASOURCE_PASSWORD=rootpw
+      - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver
       # (Optionally) authenticate the default Docker Hub access for the App Metadata access.
       - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRY_CONFIGURATIONS_DEFAULT_USER=${METADATA_DEFAULT_DOCKERHUB_USER}
       - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRY_CONFIGURATIONS_DEFAULT_SECRET=${METADATA_DEFAULT_DOCKERHUB_PASSWORD}
 
       - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRYCONFIGURATIONS_DEFAULT_USER=${METADATA_DEFAULT_DOCKERHUB_USER}
       - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRYCONFIGURATIONS_DEFAULT_SECRET=${METADATA_DEFAULT_DOCKERHUB_PASSWORD}
     depends_on:
+      - kafka-broker
       - skipper-server
+    entrypoint: >
+      /bin/sh -c "
+         apt-get update && apt-get install --no-install-recommends -y wget &&
+         wget --no-check-certificate -P /tmp/ https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh &&
+         chmod a+x /tmp/wait-for-it.sh &&
+         /tmp/wait-for-it.sh mariadb:3306 -- /cnb/process/web"
     restart: always
     volumes:
       - ${HOST_MOUNT_PATH:-.}:${DOCKER_MOUNT_PATH:-/home/cnb/scdf}
@@ -85,7 +128,17 @@ services:
     environment:
       - SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_PORTRANGE_LOW=20000
       - SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_PORTRANGE_HIGH=20190
+      - SPRING_DATASOURCE_URL=jdbc:mariadb://mariadb:3306/dataflow
+      - SPRING_DATASOURCE_USERNAME=root
+      - SPRING_DATASOURCE_PASSWORD=rootpw
+      - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver
       - LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_SKIPPER_SERVER_DEPLOYER=ERROR
+    entrypoint: >
+      /bin/sh -c "
+         apt-get update && apt-get install --no-install-recommends -y wget &&
+         wget --no-check-certificate -P /tmp/ https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh &&
+         chmod a+x /tmp/wait-for-it.sh &&
+         /tmp/wait-for-it.sh mariadb:3306 -- /cnb/process/web"
     restart: always
     volumes:
-      - ${HOST_MOUNT_PATH:-.}:${DOCKER_MOUNT_PATH:-/home/cnb/scdf}
+      - ${HOST_MOUNT_PATH:-.}:${DOCKER_MOUNT_PATH:-/home/cnb/scdf}

src/kubernetes/server/server-deployment.yaml

@@ -17,7 +17,7 @@ spec:
       containers:
       - name: scdf-server
         image: springcloud/spring-cloud-dataflow-server:2.10.0-SNAPSHOT
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         volumeMounts:
           - name: config
             mountPath: /workspace/config

src/kubernetes/skipper/skipper-config-kafka.yaml

@@ -16,15 +16,9 @@ data:
                   default:
                     environmentVariables: 'SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS=kafka-broker:9092,SPRING_CLOUD_STREAM_KAFKA_BINDER_ZK_NODES=${KAFKA_ZK_SERVICE_HOST}:${KAFKA_ZK_SERVICE_PORT}'
                     limits:
-                      memory: 2g
-                      cpu: 2.0                    
-                    readinessProbeDelay: 1
-                    readinessProbeTimeout: 5
-                    livenessProbeDelay: 1
-                    livenessProbeTimeout: 2
-                    startupProbeDelay: 20
-                    startupProbeTimeout: 5
-                    startupProbeFailure: 50
+                      memory: 1024Mi
+                      cpu: 500m                    
+                    startupProbeFailures: 30
       datasource:
         url: jdbc:mariadb://${MARIADB_SERVICE_HOST}:${MARIADB_SERVICE_PORT}/skipper
         username: root

src/kubernetes/skipper/skipper-config-rabbit.yaml

@@ -16,15 +16,10 @@ data:
                   default:
                     environmentVariables: 'SPRING_RABBITMQ_HOST=${RABBITMQ_SERVICE_HOST},SPRING_RABBITMQ_PORT=${RABBITMQ_SERVICE_PORT}'
                     limits:
-                      memory: 2g
-                      cpu: 2.0
-                    readinessProbeDelay: 1
-                    readinessProbeTimeout: 5
-                    livenessProbeDelay: 1
-                    livenessProbeTimeout: 2
-                    startupProbeDelay: 20
-                    startupProbeTimeout: 5
-                    startupProbeFailure: 50
+                      memory: 1024Mi
+                      cpu: 500m
+                    readinessProbeDelay: 120
+                    livenessProbeDelay: 90
       datasource:
         url: jdbc:mariadb://${MARIADB_SERVICE_HOST}:${MARIADB_SERVICE_PORT}/skipper
         username: root

src/kubernetes/skipper/skipper-deployment.yaml

@@ -17,7 +17,7 @@ spec:
       containers:
       - name: skipper
         image: springcloud/spring-cloud-skipper-server:2.9.0-SNAPSHOT
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         volumeMounts:
           - name: config
             mountPath: /workspace/config