Replace Minio with SeaweedFs as an s3 storage for Analytics cluster.

charts/druid/Chart.yaml

@@ -7,5 +7,5 @@ maintainers:
   - name: Gazza7205
 sources:
   - https://github.com/CAAPIM/apim-charts
-version: 1.0.17
+version: 1.0.18
 appVersion: 31.0

charts/druid/templates/_helpers.tpl

@@ -109,4 +109,21 @@ Portal Docops page
 {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 {{- end -}}
+{{- end -}}
+
+
+{{/*
+Return the Master Server peers
+*/}}
+{{- define "seaweedfs.master.servers" -}}
+{{- $peers := list -}}
+{{- $masterFullname := "seaweedfs-s3"  -}}
+{{- $masterHeadlessSvcName := printf "seaweedfs-s3"  -}}
+{{- $clusterDomain := .Values.seaweedfs.clusterDomain -}}
+{{- $masterPort := int .Values.seaweedfs.master.port -}}
+{{- $namespace := .Release.Namespace -}}
+{{- range $i := until (int .Values.seaweedfs.replicaCount) }}
+    {{- $peers = append $peers (printf "%s-%d.%s.%s.svc.%s:%d" $masterFullname $i $masterHeadlessSvcName $namespace $clusterDomain $masterPort) -}}
+{{- end -}}
+{{- print (join "," $peers) -}}
 {{- end -}}

charts/druid/templates/coordinator/coordinator-config.yaml

@@ -11,8 +11,14 @@ data:
   SERVICE_NAME: {{ .Values.coordinator.hostname }}
   ZOOKEEPER_HOST: {{ .Values.zookeeper.hostname }}
   ZOOKEEPER_PORT: {{ .Values.zookeeper.port | quote }}
+{{- if .Values.global.deepStorage.seaweedfs }}
+  MINIO_URL: http://seaweedfs-s3:8333
+  MINIO_HEALTH_URL: http://seaweedfs-s3:8333/status
+  MINIO_BUCKET: {{ .Values.global.deepStorage.analytics.bucketName }}
+{{ else }}
   MINIO_URL: http://minio:9000
   MINIO_BUCKET: {{ .Values.minio.bucketName }}
+{{ end }}
   DATABASE_TYPE: {{ required "Please fill in databaseType in values.yaml" .Values.global.databaseType | quote }}
 {{ if .Values.global.setupDemoDatabase}}
   DATABASE_HOST: {{ .Release.Name }}-mysql-headless
@@ -38,4 +44,8 @@ data:
     druid.coordinator.kill.period=P1D
     druid.coordinator.kill.durationToRetain=PT1H
     druid.coordinator.kill.maxSegments=25
-    druid.storage.bucket={{ .Values.minio.bucketName }}
+{{- if (eq .Values.global.deepStorage.seaweedfs true) }}
+    druid.storage.bucket={{ .Values.global.deepStorage.analytics.bucketName }}
+{{- else }}
+    druid.storage.bucket={{ .Values.minio.bucketName }}
+{{- end }}

charts/druid/templates/coordinator/coordinator-statefulset.yaml

@@ -64,6 +64,18 @@ spec:
           securityContext: {{- toYaml .Values.global.containerSecurityContext | nindent 12 }}
           {{- end }}
           env:
+          {{- if (eq .Values.global.deepStorage.seaweedfs true) }} 
+            - name: MINIO_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.global.deepStorage.auth.secretName }}
+                  key: admin_access_key_id
+            - name: MINIO_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.global.deepStorage.auth.secretName }}
+                  key: admin_secret_access_key
+          {{- else }}
             - name: MINIO_ACCESS_KEY
               valueFrom:
                 secretKeyRef:
@@ -74,6 +86,7 @@ spec:
                 secretKeyRef:
                   name: {{ .Values.minio.auth.secretName }}
                   key: MINIO_SECRET_KEY
+          {{- end }}
             - name: DATABASE_PASSWORD
               valueFrom:
                 secretKeyRef:

charts/druid/templates/historical/historical-config.yaml

@@ -11,8 +11,14 @@ data:
   SERVICE_NAME: historical
   ZOOKEEPER_HOST: {{ .Values.zookeeper.hostname }}
   ZOOKEEPER_PORT: {{ .Values.zookeeper.port | quote }}
+{{- if .Values.global.deepStorage.seaweedfs }}
+  MINIO_URL: http://seaweedfs-s3:8333
+  MINIO_HEALTH_URL: http://seaweedfs-s3:8333/status
+  MINIO_BUCKET: {{ .Values.global.deepStorage.analytics.bucketName }}
+{{ else }}
   MINIO_URL: http://minio:9000
   MINIO_BUCKET: {{ .Values.minio.bucketName }}
+{{ end }}
   CONTAINER_LOG_LEVEL: {{ .Values.historical.loglevel }}
   runtime.properties: |-
     druid.service=druid/historical
@@ -35,4 +41,8 @@ data:
     druid.cache.type=caffeine
     druid.cache.sizeInBytes=10000000
 
-    druid.storage.bucket={{ .Values.minio.bucketName }}
+{{- if (eq .Values.global.deepStorage.seaweedfs true) }}
+    druid.storage.bucket={{ .Values.global.deepStorage.analytics.bucketName }}
+{{- else }}
+    druid.storage.bucket={{ .Values.minio.bucketName }}
+{{- end }}

charts/druid/templates/historical/historical-statefulset.yaml

@@ -67,6 +67,18 @@ spec:
           securityContext: {{- toYaml .Values.global.containerSecurityContext | nindent 12 }}
           {{- end }}
           env:
+          {{- if (eq .Values.global.deepStorage.seaweedfs true) }} 
+            - name: MINIO_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.global.deepStorage.auth.secretName }}
+                  key: admin_access_key_id
+            - name: MINIO_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.global.deepStorage.auth.secretName }}
+                  key: admin_secret_access_key
+          {{- else }}
             - name: MINIO_ACCESS_KEY
               valueFrom:
                 secretKeyRef:
@@ -77,6 +89,7 @@ spec:
                 secretKeyRef:
                   name: {{  .Values.minio.auth.secretName }}
                   key: MINIO_SECRET_KEY
+          {{- end }}
           envFrom:
           - configMapRef:
               name: historical-config

charts/druid/templates/ingestion/ingestion-config.yaml

@@ -15,8 +15,14 @@ data:
   OVERLORD_PORT: {{ .Values.coordinator.port | quote }}
   COORDINATOR_HOST: {{ .Values.coordinator.hostname }}
   COORDINATOR_PORT: {{ .Values.coordinator.port | quote }}
+{{- if .Values.global.deepStorage.seaweedfs }}
+  MINIO_URL: http://seaweedfs-s3:8333
+  MINIO_HEALTH_URL: http://seaweedfs-s3:8333/status
+  MINIO_BUCKET: {{ .Values.global.deepStorage.analytics.bucketName }}
+{{ else }}
   MINIO_URL: http://minio:9000
   MINIO_BUCKET: {{ .Values.minio.bucketName }}
+{{ end }}
   PARTITION_COUNT: "{{ .Values.ingestion.config.partitionCount }}"
   REPLICATION_COUNT: "{{ .Values.kafka.replicaCount }}"
 {{ if .Values.global.setupDemoDatabase}}

charts/druid/templates/ingestion/ingestion-deployment.yaml

@@ -65,6 +65,18 @@ spec:
           securityContext: {{- toYaml .Values.global.containerSecurityContext | nindent 12 }}
           {{- end }}
           env:
+          {{- if (eq .Values.global.deepStorage.seaweedfs true) }} 
+            - name: MINIO_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.global.deepStorage.auth.secretName }}
+                  key: admin_access_key_id
+            - name: MINIO_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.global.deepStorage.auth.secretName }}
+                  key: admin_secret_access_key
+          {{- else }}
             - name: MINIO_ACCESS_KEY
               valueFrom:
                 secretKeyRef:
@@ -75,6 +87,7 @@ spec:
                 secretKeyRef:
                   name: {{ .Values.minio.auth.secretName }}
                   key: MINIO_SECRET_KEY
+          {{- end }}
             - name: DATABASE_PASSWORD
               valueFrom:
                 secretKeyRef:

charts/druid/templates/middlemanager/middlemanager-config.yaml

@@ -11,8 +11,14 @@ data:
   SERVICE_NAME: middleManager
   ZOOKEEPER_HOST: {{ .Values.zookeeper.hostname }}
   ZOOKEEPER_PORT: {{ .Values.zookeeper.port | quote }}
+{{- if .Values.global.deepStorage.seaweedfs }}
+  MINIO_URL: http://seaweedfs-s3:8333
+  MINIO_HEALTH_URL: http://seaweedfs-s3:8333/status
+  MINIO_BUCKET: {{ .Values.minio.bucketName }}
+{{ else }}
   MINIO_URL: http://minio:9000
   MINIO_BUCKET: {{ .Values.minio.bucketName }}
+{{ end }}
   CONTAINER_LOG_LEVEL: {{ .Values.middlemanager.loglevel }}
   runtime.properties: |-
     druid.indexer.runner.startPort={{ .Values.middlemanager.indexer.startPort }}
@@ -37,4 +43,8 @@ data:
     # Hadoop indexing
     druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp
 
+{{- if (eq .Values.global.deepStorage.seaweedfs true) }}
+    druid.storage.bucket={{ .Values.global.deepStorage.bucketName }}
+{{- else }}
     druid.storage.bucket={{ .Values.minio.bucketName }}
+{{- end }}

charts/druid/templates/middlemanager/middlemanager-statefulset.yaml

@@ -64,6 +64,18 @@ spec:
           securityContext: {{- toYaml .Values.global.containerSecurityContext | nindent 12 }}
           {{- end }}
           env:
+          {{- if (eq .Values.global.deepStorage.seaweedfs true) }} 
+            - name: MINIO_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.global.deepStorage.auth.secretName }}
+                  key: admin_access_key_id
+            - name: MINIO_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.global.deepStorage.auth.secretName }}
+                  key: admin_secret_access_key
+          {{- else }}
             - name: MINIO_ACCESS_KEY
               valueFrom:
                 secretKeyRef:
@@ -74,6 +86,7 @@ spec:
                 secretKeyRef:
                   name: {{  .Values.minio.auth.secretName }}
                   key: MINIO_SECRET_KEY
+          {{- end }}
           envFrom:
           - configMapRef:
               name: middlemanager-config

charts/druid/templates/minio/minio-pdb.yaml

@@ -15,4 +15,4 @@ spec:
   selector:
     matchLabels:
       app: minio
-{{ end }}
+{{- end }}

charts/druid/templates/minio/minio-secret.yaml

@@ -31,4 +31,4 @@ data:
 {{- if .Values.minio.s3gateway.secretKey }}
   awsSecretAccessKey: {{ .Values.minio.s3gateway.secretKey | b64enc | quote }}
 {{- end }}
-{{- end }}
+{{- end }}

charts/portal/Chart.lock

@@ -1,7 +1,10 @@
 dependencies:
 - name: druid
   repository: file://../druid
-  version: 1.0.17
+  version: 1.0.18
+- name: seaweedfs
+  repository: file://../seaweedfs
+  version: 1.0.1
 - name: mysql
   repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
   version: 12.2.2
@@ -11,5 +14,5 @@ dependencies:
 - name: ingress-nginx
   repository: https://kubernetes.github.io/ingress-nginx/
   version: 4.12.1
-digest: sha256:a232d427aafce12b14085dbf887219aaf83e563693d86fd6c0985e1d5c800810
-generated: "2025-06-17T23:17:16.7865154+05:30"
+digest: sha256:0023f65c31a891b2c52d96cedcce53c12f711c0d3c157dbe3cc970731b83d077
+generated: "2025-08-31T23:36:41.2457391+05:30"

charts/portal/Chart.yaml

@@ -14,6 +14,10 @@ dependencies:
   version: ^1.0.0
   condition: portal.analytics.enabled
   repository: "file://../druid"
+- name: seaweedfs
+  version: ^1.0.0
+  condition: global.deepStorage.seaweedfs
+  repository: "file://../seaweedfs"
 - name: mysql
   version: 12.2.2
   repository: "https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami"

charts/portal/README.md

@@ -8,6 +8,41 @@ This Chart deploys the Layer7 API Developer Portal on a Kubernetes Cluster using
 
 ## 2.3.20 General Updates
 - This new version of the chart supports API Portal 5.4
+- Added seaweedfs as s3 storage for analytics data
+  - This resolves a race condition that occurs on slower hardware where apim/ingress starts before other dependent services are ready. 
+  - This is not ***enabled by default***.
+    - This only gets added when you install the Chart.
+    - If you wish to enable this and use seaweedfs as deep storage, set global.deepStorage.seaweedfs to true
+    ```
+    global:
+      deepStorage:
+        seaweedfs: false
+        bucketName: api-metrics
+        ...
+        auth:
+          secretName: seaweedfs-s3-secret
+        ...
+    ```
+    - If you are upgrading from previous version and want to copy analytics data from minio to seaweedfs, set seaweedfs.migrateData to true.
+      - A Helm install will ***NOT*** data migration.
+      - A Helm upgrade will run the data migration and bucket name can be customized.
+      - Make Sure that the value seaweedfs.minio.bucketName is same as the druid.minio.bucketName.
+      - The migrated data will be stored in the new bucket global.deepStorage.bucketName which the druid stack starts using is seaweedfs is enabled.
+    ```
+    global:
+      deepStorage:
+        seaweedfs: false
+        enableDataMigration: true
+        bucketName: api-metrics
+        ...
+        auth:
+          secretName: seaweedfs-s3-secret
+        ...
+    seaweedfs:
+      ...
+      minio:
+        bucketName: api-metrics
+    ```
 
 ## 2.3.18 General Updates
 - Switch bitnami/mysql to bitnamilegacy/mysql.