Skip to content

Commit 8ec25a5

Browse files
scdf-build-botscdf-build-bot
committed
Update k8s deployment files
1 parent 72adc03 commit 8ec25a5

28 files changed

+244
-247
lines changed

‎src/docker-compose/.env

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11

2-
DATAFLOW_VERSION=2.11.3-SNAPSHOT
3-
SKIPPER_VERSION=2.11.3-SNAPSHOT
2+
DATAFLOW_VERSION=2.11.5-SNAPSHOT
3+
SKIPPER_VERSION=2.11.5-SNAPSHOT
44
COMPOSE_PROJECT_NAME=scdf
55
COMPOSE_HTTP_TIMEOUT=300

‎src/docker-compose/docker-compose-dood.yml

+3-17
Original file line numberDiff line numberDiff line change
@@ -40,14 +40,8 @@ services:
4040
- SPRING_CLOUD_DATAFLOW_TASK_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_DOCKER_PORTRANGE_HIGH=61000
4141

4242
# Override the CTR form maven to docker
43-
- SPRING_CLOUD_DATAFLOW_TASK_COMPOSEDTASKRUNNER_URI=docker://springcloud/spring-cloud-dataflow-composed-task-runner:${DATAFLOW_VERSION:-2.11.3-SNAPSHOT}${BP_JVM_VERSION:-}
43+
- SPRING_CLOUD_DATAFLOW_TASK_COMPOSEDTASKRUNNER_URI=docker://springcloud/spring-cloud-dataflow-composed-task-runner:${DATAFLOW_VERSION:-@project.version@}${BP_JVM_VERSION:-}
4444
- SPRING_CLOUD_DATAFLOW_SERVER_URI=${DATAFLOW_URI:-http://dataflow-server:9393}
45-
entrypoint: >
46-
/bin/sh -c "
47-
apt-get update && apt-get install --no-install-recommends -y wget &&
48-
wget --no-check-certificate -P /tmp/ https://get.docker.com/builds/Linux/x86_64/docker-latest.tgz &&
49-
tar -xvf /tmp/docker-latest.tgz --directory /tmp/ &&
50-
mv /tmp/docker/docker /usr/local/bin && /cnb/process/web"
5145
volumes:
5246
- /var/run/docker.sock:/var/run/docker.sock
5347

@@ -61,12 +55,6 @@ services:
6155
# in the docker-compose.yaml#skipper-server. Tip: you can change the APPS_PORT_RANGE instead.
6256
- SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_DOCKER_PORTRANGE_LOW=20200
6357
- SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_DOCKER_PORTRANGE_HIGH=61000
64-
entrypoint: >
65-
/bin/sh -c "
66-
apt-get update && apt-get install --no-install-recommends -y wget &&
67-
wget --no-check-certificate -P /tmp/ https://get.docker.com/builds/Linux/x86_64/docker-latest.tgz &&
68-
tar -xvf /tmp/docker-latest.tgz --directory /tmp/ &&
69-
mv /tmp/docker/docker /usr/local/bin && /cnb/process/web"
7058
volumes:
7159
- /var/run/docker.sock:/var/run/docker.sock
7260

@@ -76,15 +64,13 @@ services:
7664
command: >
7765
/bin/sh -c "
7866
./wait-for-it.sh -t 360 dataflow-server:9393;
79-
wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/kafka-docker-latest&force=true}';
80-
wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:-2.11.3-SNAPSHOT}' --no-check-certificate --post-data='uri=docker:springcloud/spring-cloud-dataflow-tasklauncher-sink-kafka:${DATAFLOW_VERSION:-2.11.3-SNAPSHOT}${BP_JVM_VERSION:-}';
67+
wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/kafka-docker-latest&force=true}';
68+
wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:-@project.version@}' --no-check-certificate --post-data='uri=docker:springcloud/spring-cloud-dataflow-tasklauncher-sink-kafka:${DATAFLOW_VERSION:-@project.version@}${BP_JVM_VERSION:-}';
8169
echo 'Docker Stream apps imported'"
8270
8371
app-import-task:
8472
command: >
8573
/bin/sh -c "
8674
./wait-for-it.sh -t 360 dataflow-server:9393;
8775
wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${TASK_APPS_URI:-https://dataflow.spring.io/task-docker-latest&force=true}';
88-
wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/timestamp3' --no-check-certificate --post-data='bootVersion=3&uri=docker:springcloudtask/timestamp-task:3.0.0';
89-
wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/timestamp-batch3' --no-check-certificate --post-data='bootVersion=3&uri=docker:springcloudtask/timestamp-batch-task:3.0.0';
9076
echo 'Docker Task apps imported'"

‎src/docker-compose/docker-compose-influxdb.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,13 @@ services:
1919
}
2020
2121
influxdb:
22-
image: influxdb:1.8
22+
image: influxdb:1.7.7
2323
container_name: influxdb
2424
ports:
2525
- '8086:8086'
2626

2727
grafana:
28-
image: springcloud/spring-cloud-dataflow-grafana-influxdb:${DATAFLOW_VERSION:-2.11.3-SNAPSHOT}
28+
image: springcloud/spring-cloud-dataflow-grafana-influxdb:${DATAFLOW_VERSION:-@project.version@}
2929
container_name: grafana
3030
ports:
3131
- '3000:3000'

‎src/docker-compose/docker-compose-postgres.yml

+6-8
Original file line numberDiff line numberDiff line change
@@ -4,36 +4,34 @@ version: '3'
44
# Usage: docker-compose -f ./docker-compose.yml -f ./docker-compose-postgres.yml up
55
services:
66

7+
mariadb:
8+
entrypoint: ["echo", "MariaDB service is disabled"]
9+
710
postgres:
811
image: postgres:14
912
command: postgres -c 'max_connections=300'
1013
container_name: dataflow-postgres
1114
restart: always
1215
environment:
13-
LANG: en_US.utf8
14-
LC_ALL: en_US.utf8
15-
JDK_JAVA_OPTIONS: '-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8'
1616
POSTGRES_DB: dataflow
1717
POSTGRES_USER: root
1818
POSTGRES_PASSWORD: rootpw
1919
expose:
2020
- 5432
2121
ports:
22-
- '5432:5432'
22+
- 5432:5432
2323

2424
dataflow-server:
2525
environment:
2626
- SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/dataflow
2727
- SPRING_DATASOURCE_USERNAME=root
2828
- SPRING_DATASOURCE_PASSWORD=rootpw
2929
- SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
30-
depends_on:
31-
- postgres
30+
3231
skipper-server:
3332
environment:
3433
- SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/dataflow
3534
- SPRING_DATASOURCE_USERNAME=root
3635
- SPRING_DATASOURCE_PASSWORD=rootpw
3736
- SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
38-
depends_on:
39-
- postgres
37+

‎src/docker-compose/docker-compose-prometheus.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -38,13 +38,13 @@ services:
3838
- micrometer.prometheus-proxy.tcp-port=7001
3939

4040
grafana:
41-
image: springcloud/spring-cloud-dataflow-grafana-prometheus:${DATAFLOW_VERSION:-2.11.3-SNAPSHOT}
41+
image: springcloud/spring-cloud-dataflow-grafana-prometheus:${DATAFLOW_VERSION:-@project.version@}
4242
container_name: grafana
4343
ports:
4444
- '3000:3000'
4545

4646
prometheus:
47-
image: springcloud/spring-cloud-dataflow-prometheus-local:${DATAFLOW_VERSION:-2.11.3-SNAPSHOT}
47+
image: springcloud/spring-cloud-dataflow-prometheus-local:${DATAFLOW_VERSION:-@project.version@}
4848
container_name: prometheus
4949
ports:
5050
- '9090:9090'

‎src/docker-compose/docker-compose-rabbitmq.yml

+10-15
Original file line numberDiff line numberDiff line change
@@ -4,34 +4,29 @@ version: '3'
44
# Usage: docker-compose -f ./docker-compose.yml -f ./docker-compose-rabbitmq.yml up
55
services:
66

7+
kafka-broker:
8+
entrypoint: ["echo", "kafka-broker service is disabled"]
9+
10+
zookeeper:
11+
entrypoint: ["echo", "zookeeper service is disabled"]
12+
713
rabbitmq:
8-
image: rabbitmq:3.8-management
14+
image: rabbitmq:3.7
915
container_name: dataflow-rabbitmq
10-
environment:
11-
LANG: en_US.utf8
12-
LC_ALL: en_US.utf8
13-
JDK_JAVA_OPTIONS: '-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8'
1416
expose:
1517
- '5672'
16-
- '15672'
1718

1819
dataflow-server:
1920
environment:
2021
- SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_RABBITMQ_HOST=rabbitmq
21-
depends_on:
22-
- rabbitmq
23-
24-
skipper-server:
25-
environment:
26-
- SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_RABBITMQ_HOST=rabbitmq
27-
depends_on:
28-
- rabbitmq
2922

3023
app-import-stream:
3124
command: >
3225
/bin/sh -c "
3326
./wait-for-it.sh -t 360 dataflow-server:9393;
3427
wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/rabbitmq-maven-latest&force=true}';
35-
wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:-2.11.3-SNAPSHOT}' --no-check-certificate --post-data='uri=maven://org.springframework.cloud:spring-cloud-dataflow-tasklauncher-sink-rabbit:${DATAFLOW_VERSION:-2.11.3-SNAPSHOT}';
28+
wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/ver-log/3.0.1' --no-check-certificate --post-data='uri=maven://org.springframework.cloud.stream.app:log-sink-rabbit:3.0.1';
29+
wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/ver-log/2.1.5.RELEASE' --no-check-certificate --post-data='uri=maven://org.springframework.cloud.stream.app:log-sink-rabbit:2.1.5.RELEASE';
30+
wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:[email protected]@}' --no-check-certificate --post-data='uri=maven://org.springframework.cloud:spring-cloud-dataflow-tasklauncher-sink-rabbit:${DATAFLOW_VERSION:[email protected]@}';
3631
echo 'Stream apps imported'"
3732

‎src/docker-compose/docker-compose-zipkin.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ version: '3'
66
services:
77

88
zipkin-server:
9-
image: openzipkin/zipkin:2
9+
image: openzipkin/zipkin:2.23.2
1010
container_name: zipkin-server
1111
expose:
1212
- '9411'

‎src/docker-compose/docker-compose.yml

+59-12
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
version: '3'
2+
23
# Configuration environment variables:
34
# - DATAFLOW_VERSION and SKIPPER_VERSION specify what DataFlow and Skipper image versions to use.
45
# - STREAM_APPS_URI and TASK_APPS_URI are used to specify what Stream and Task applications to pre-register.
@@ -19,25 +20,66 @@ version: '3'
1920
# The deployed stream applications that run in their own docker containers (e.g. docker:// registered apps),
2021
# can be reached on the ports they expose.
2122
services:
23+
mariadb:
24+
image: mariadb:10.6
25+
container_name: dataflow-mariadb
26+
environment:
27+
MARIADB_DATABASE: dataflow
28+
MARIADB_ROOT_PASSWORD: rootpw
29+
expose:
30+
- 3306
31+
32+
kafka-broker:
33+
image: confluentinc/cp-kafka:5.5.2
34+
container_name: dataflow-kafka
35+
expose:
36+
- "9092"
37+
environment:
38+
- KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka-broker:9092
39+
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
40+
- KAFKA_ADVERTISED_HOST_NAME=kafka-broker
41+
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
42+
- KAFKA_LOG4J_ROOT_LOGLEVEL=ERROR
43+
- KAFKA_LOG4J_LOGGERS=org.apache.zookeeper=ERROR,org.apache.kafka=ERROR,kafka=ERROR,kafka.cluster=ERROR,kafka.controller=ERROR,kafka.coordinator=ERROR,kafka.log=ERROR,kafka.server=ERROR,kafka.zookeeper=ERROR,state.change.logger=ERROR
44+
depends_on:
45+
- zookeeper
46+
47+
zookeeper:
48+
image: confluentinc/cp-zookeeper:5.5.2
49+
container_name: dataflow-kafka-zookeeper
50+
expose:
51+
- "2181"
52+
environment:
53+
- ZOOKEEPER_CLIENT_PORT=2181
54+
2255
dataflow-server:
2356
user: root
24-
image: springcloud/spring-cloud-dataflow-server:${DATAFLOW_VERSION:-2.11.3-SNAPSHOT}${BP_JVM_VERSION:-}
57+
image: springcloud/spring-cloud-dataflow-server:${DATAFLOW_VERSION:-@project.version@}${BP_JVM_VERSION:-}
2558
container_name: dataflow-server
2659
ports:
2760
- "9393:9393"
2861
environment:
29-
- LANG=en_US.utf8
30-
- LC_ALL=en_US.utf8
31-
- JDK_JAVA_OPTIONS=-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8
62+
- SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS=PLAINTEXT://kafka-broker:9092
63+
- SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_STREAMS_BINDER_BROKERS=PLAINTEXT://kafka-broker:9092
64+
- SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_BINDER_ZKNODES=zookeeper:2181
65+
- SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_STREAMS_BINDER_ZKNODES=zookeeper:2181
66+
- SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_KAFKA_STREAMS_PROPERTIES_METRICS_RECORDING_LEVEL=DEBUG
3267
# Set CLOSECONTEXTENABLED=true to ensure that the CRT launcher is closed.
3368
- SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_TASK_SPRING_CLOUD_TASK_CLOSECONTEXTENABLED=true
3469
- SPRING_CLOUD_SKIPPER_CLIENT_SERVER_URI=${SKIPPER_URI:-http://skipper-server:7577}/api
70+
- SPRING_DATASOURCE_URL=jdbc:mariadb://mariadb:3306/dataflow
71+
- SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.MariaDB106Dialect
72+
- SPRING_DATASOURCE_USERNAME=root
73+
- SPRING_DATASOURCE_PASSWORD=rootpw
74+
- SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver
3575
# (Optionally) authenticate the default Docker Hub access for the App Metadata access.
3676
- SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRY_CONFIGURATIONS_DEFAULT_USER=${METADATA_DEFAULT_DOCKERHUB_USER}
3777
- SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRY_CONFIGURATIONS_DEFAULT_SECRET=${METADATA_DEFAULT_DOCKERHUB_PASSWORD}
78+
3879
- SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRYCONFIGURATIONS_DEFAULT_USER=${METADATA_DEFAULT_DOCKERHUB_USER}
3980
- SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRYCONFIGURATIONS_DEFAULT_SECRET=${METADATA_DEFAULT_DOCKERHUB_PASSWORD}
4081
depends_on:
82+
- kafka-broker
4183
- skipper-server
4284
restart: always
4385
volumes:
@@ -48,6 +90,12 @@ services:
4890
container_name: dataflow-app-import-stream
4991
depends_on:
5092
- dataflow-server
93+
command: >
94+
/bin/sh -c "
95+
./wait-for-it.sh -t 360 dataflow-server:9393;
96+
wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/kafka-maven-latest&force=true}';
97+
wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:[email protected]@}' --no-check-certificate --post-data='uri=maven://org.springframework.cloud:spring-cloud-dataflow-tasklauncher-sink-kafka:${DATAFLOW_VERSION:[email protected]@}';
98+
echo 'Maven Stream apps imported'"
5199
52100
app-import-task:
53101
image: springcloud/baseimage:1.0.4
@@ -58,24 +106,23 @@ services:
58106
/bin/sh -c "
59107
./wait-for-it.sh -t 360 dataflow-server:9393;
60108
wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${TASK_APPS_URI:-https://dataflow.spring.io/task-maven-latest&force=true}';
61-
wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/timestamp3' --no-check-certificate --post-data='bootVersion=3&uri=maven://uri=maven:io.spring:timestamp-task:3.0.0';
62-
wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/timestamp-batch3' --no-check-certificate --post-data='bootVersion=3&uri=maven://uri=maven:io.spring:timestamp-batch:3.0.0';
63109
echo 'Maven Task apps imported'"
64110
65111
skipper-server:
66112
user: root
67-
image: springcloud/spring-cloud-skipper-server:${SKIPPER_VERSION:-2.11.3-SNAPSHOT}${BP_JVM_VERSION:-}
68-
container_name: skipper-server
113+
image: springcloud/spring-cloud-skipper-server:${SKIPPER_VERSION:-@spring-cloud-skipper.version@}${BP_JVM_VERSION:-}
114+
container_name: skipper
69115
ports:
70116
- "7577:7577"
71117
- ${APPS_PORT_RANGE:-20000-20195:20000-20195}
72118
environment:
73-
- LANG=en_US.utf8
74-
- LC_ALL=en_US.utf8
75-
- JDK_JAVA_OPTIONS=-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8
76-
- SERVER_PORT=7577
77119
- SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_PORTRANGE_LOW=20000
78120
- SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_PORTRANGE_HIGH=20190
121+
- SPRING_DATASOURCE_URL=jdbc:mariadb://mariadb:3306/dataflow
122+
- SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.MariaDB106Dialect
123+
- SPRING_DATASOURCE_USERNAME=root
124+
- SPRING_DATASOURCE_PASSWORD=rootpw
125+
- SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver
79126
- LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_SKIPPER_SERVER_DEPLOYER=ERROR
80127
restart: always
81128
volumes:

‎src/kubernetes/grafana/grafana-deployment.yaml

+3-6
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ spec:
1414
app: grafana
1515
spec:
1616
containers:
17-
- image: springcloud/spring-cloud-dataflow-grafana-prometheus:2.11.4
17+
- image: springcloud/spring-cloud-dataflow-grafana-prometheus:2.11.5-SNAPSHOT
1818
name: grafana
1919
env:
2020
- name: GF_SECURITY_ADMIN_USER
@@ -30,12 +30,9 @@ spec:
3030
ports:
3131
- containerPort: 3000
3232
resources:
33-
limits:
34-
cpu: 500m
35-
memory: 2500Mi
3633
requests:
37-
cpu: 100m
38-
memory: 100Mi
34+
cpu: 500m
35+
memory: 256Mi
3936
volumeMounts:
4037
- name: config
4138
mountPath: "/etc/grafana/provisioning/datasources/datasources.yaml"

‎src/kubernetes/kafka/kafka-deployment.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ spec:
2020
containers:
2121
- name: kafka
2222
image: confluentinc/cp-kafka:5.5.2
23-
imagePullPolicy: IfNotPresent
2423
ports:
2524
- containerPort: 9092
2625
env:

‎src/kubernetes/kafka/kafka-svc.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ metadata:
66
app: kafka
77
component: kafka-broker
88
spec:
9+
clusterIP: None
910
ports:
1011
- port: 9092
1112
name: kafka-port

‎src/kubernetes/kafka/kafka-zk-deployment.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ spec:
1919
containers:
2020
- name: kafka-zk
2121
image: confluentinc/cp-zookeeper:5.5.2
22-
imagePullPolicy: IfNotPresent
2322
ports:
2423
- containerPort: 2181
2524
env:

0 commit comments

Comments
 (0)