From 117fd6630b4c09a4a09bd2fa4488789f6c09d2f2 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 27 Feb 2025 20:52:23 +0100 Subject: [PATCH] chore(demo/nifi-kafka-druid-earthquake-data): Improve initialization jobs --- demos/demos-v2.yaml | 1 + .../create-druid-ingestion-job.yaml | 22 ++++++++- .../create-nifi-ingestion-job.yaml | 23 ++++++++- .../serviceaccount.yaml | 47 +++++++++++++++++++ .../setup-superset.yaml | 23 ++++++++- 5 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml index ff275071..f8c92aed 100644 --- a/demos/demos-v2.yaml +++ b/demos/demos-v2.yaml @@ -64,6 +64,7 @@ demos: - s3 - earthquakes manifests: + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml diff --git a/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml b/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml index 421ba348..2c25c32c 100644 --- a/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml +++ b/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml @@ -6,10 +6,30 @@ metadata: spec: template: spec: + serviceAccountName: demo-serviceaccount + initContainers: + - name: wait-for-druid-coordinator + image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev + command: + - bash + - -euo + - pipefail + - -c + - | + echo 'Waiting for Druid Coordinator to be created' + kubectl wait --for=create pod/druid-coordinator-default-0 + echo 'Waiting for Druid Coordinator to be ready' + kubectl wait --for=condition=Ready pod/druid-coordinator-default-0 --timeout=30m containers: - name: create-druid-ingestion-job image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev - command: ["bash", "-c", "curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor"] + command: + - bash + - -euo + - pipefail + - -c + - | + curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor volumeMounts: - name: ingestion-job-spec mountPath: /tmp/ingestion-job-spec diff --git a/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml b/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml index 19bb7675..efddab0c 100644 --- a/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml +++ b/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml @@ -6,10 +6,31 @@ metadata: spec: template: spec: + serviceAccountName: demo-serviceaccount + initContainers: + - name: wait-for-nifi + image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev + command: + - bash + - -euo + - pipefail + - -c + - | + echo 'Waiting for NiFi to be created' + kubectl wait --for=create pod/nifi-node-default-0 --timeout=30m + echo 'Waiting for NiFi to be ready' + kubectl wait --for=condition=Ready pod/nifi-node-default-0 --timeout=30m containers: - name: create-nifi-ingestion-job image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev - command: ["bash", "-c", "curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/IngestEarthquakesToKafka.xml && python -u /tmp/script/script.py"] + command: + - bash + - -euo + - pipefail + - -c + - | + curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/IngestEarthquakesToKafka.xml + python -u /tmp/script/script.py volumeMounts: - name: script mountPath: /tmp/script diff --git a/demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml b/demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml new file mode 100644 index 00000000..8a2d0b19 --- /dev/null +++ b/demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml @@ -0,0 +1,47 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: demo-serviceaccount + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: demo-clusterrolebinding +subjects: + - kind: ServiceAccount + name: demo-serviceaccount + namespace: default +roleRef: + kind: ClusterRole + name: demo-clusterrole + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: demo-clusterrole +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - pods/exec + verbs: + - create diff --git a/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml b/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml index 15ee14c9..9be84027 100644 --- a/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml +++ b/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml @@ -6,10 +6,31 @@ metadata: spec: template: spec: + serviceAccountName: demo-serviceaccount + initContainers: + - name: wait-for-superset + image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev + command: + - bash + - -euo + - pipefail + - -c + - | + echo 'Waiting for Superset to be created' + kubectl wait --for=create pod/superset-node-default-0 --timeout=30m + echo 'Waiting for Superset to be ready' + kubectl wait --for=condition=Ready pod/superset-node-default-0 --timeout=30m containers: - name: setup-superset image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev - command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/superset-assets.zip && python -u /tmp/script/script.py"] + command: + - bash + - -euo + - pipefail + - -c + - | + curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/superset-assets.zip + python -u /tmp/script/script.py volumeMounts: - name: script mountPath: /tmp/script