feat: load env from files

* (connector) fix secrets reference * (connector) with their config or use global too * (global) load data from files * (global) allow generate configMaps with custom data * (global) allow initContainers closes #65 closes #93
devops-ia · Dec 8, 2024 · 514a5bf · 514a5bf
1 parent 4f0254a
commit 514a5bf
Show file tree

Hide file tree

Showing 10 changed files with 596 additions and 261 deletions.
diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml
@@ -13,6 +13,10 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: Configure sysctl
+        run: |
+          sudo sysctl -w vm.max_map_count=262144
+
       # default install latest (stable)
       - name: Set up Helm
         uses: azure/setup-helm@v4

diff --git a/charts/opencti/README.md b/charts/opencti/README.md
@@ -80,17 +80,24 @@ helm show values opencti/opencti
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | affinity | object | `{}` | Affinity for pod assignment </br> Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity |
+| args | list | `[]` | Configure args </br> Ref: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/ |
 | autoscaling | object | `{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80}` | Autoscaling with CPU or memory utilization percentage </br> Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/ |
+| command | list | `[]` | Configure command </br> Ref: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/ |
+| configMaps | list | `[]` | ConfigMap values to create configuration files Generate ConfigMap with following name: <release-name>-<name> </br> Ref: https://kubernetes.io/docs/concepts/configuration/configmap/ |
 | connectors | list | `[]` | Connectors </br> Ref: https://github.com/OpenCTI-Platform/connectors/tree/master |
-| connectorsGlobal | object | `{"env":{},"envFromSecrets":{},"volumeMounts":[],"volumes":[]}` | Connectors Globals |
+| connectorsGlobal | object | `{"env":{},"envFromConfigMap":{},"envFromFiles":[],"envFromSecrets":{},"volumeMounts":[],"volumes":[]}` | Connectors global configuration |
 | connectorsGlobal.env | object | `{}` | Additional environment variables on the output connector definition |
-| connectorsGlobal.envFromSecrets | object | `{}` | Secrets from variables |
+| connectorsGlobal.envFromConfigMap | object | `{}` | Variables from configMap |
+| connectorsGlobal.envFromFiles | list | `[]` | Load all variables from files |
+| connectorsGlobal.envFromSecrets | object | `{}` | Variables from secrets |
 | connectorsGlobal.volumeMounts | list | `[]` | Additional volumeMounts on the output connector Deployment definition |
 | connectorsGlobal.volumes | list | `[]` | Additional volumes on the output connector Deployment definition |
 | elasticsearch | object | `{"clusterName":"elastic","coordinating":{"replicaCount":0},"data":{"persistence":{"enabled":false},"replicaCount":1},"enabled":true,"extraEnvVars":[{"name":"ES_JAVA_OPTS","value":"-Xms512M -Xmx512M"}],"ingest":{"enabled":false},"master":{"masterOnly":true,"persistence":{"enabled":false},"replicaCount":1},"sysctlImage":{"enabled":false}}` | ElasticSearch subchart deployment </br> Ref: https://github.com/bitnami/charts/blob/main/bitnami/elasticsearch/values.yaml |
 | elasticsearch.enabled | bool | `true` | Enable or disable ElasticSearch subchart |
 | env | object | `{"APP__ADMIN__EMAIL":"[email protected]","APP__ADMIN__PASSWORD":"ChangeMe","APP__ADMIN__TOKEN":"ChangeMe","APP__BASE_PATH":"/","APP__GRAPHQL__PLAYGROUND__ENABLED":false,"APP__GRAPHQL__PLAYGROUND__FORCE_DISABLED_INTROSPECTION":false,"APP__HEALTH_ACCESS_KEY":"ChangeMe","APP__TELEMETRY__METRICS__ENABLED":true,"ELASTICSEARCH__URL":"http://release-name-elasticsearch:9200","MINIO__ENDPOINT":"release-name-minio:9000","RABBITMQ__HOSTNAME":"release-name-rabbitmq","RABBITMQ__PASSWORD":"ChangeMe","RABBITMQ__PORT":5672,"RABBITMQ__PORT_MANAGEMENT":15672,"RABBITMQ__USERNAME":"user","REDIS__HOSTNAME":"release-name-redis-master","REDIS__MODE":"single","REDIS__PORT":6379}` | Environment variables to configure application </br> Ref: https://docs.openbas.io/latest/deployment/configuration/#platform |
-| envFromSecrets | object | `{}` | Secrets from variables |
+| envFromConfigMap | object | `{}` | Variables from configMap |
+| envFromFiles | list | `[]` | Load all variables from files </br> Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables |
+| envFromSecrets | object | `{}` | Variables from secrets |
 | fullnameOverride | string | `""` | String to fully override opencti.fullname template |
 | global | object | `{"imagePullSecrets":[],"imageRegistry":""}` | Global section contains configuration options that are applied to all services |
 | global.imagePullSecrets | list | `[]` | Specifies the secrets to use for pulling images from private registries Leave empty if no secrets are required E.g. imagePullSecrets:   - name: myRegistryKeySecretName |
@@ -101,6 +108,7 @@ helm show values opencti/opencti
 | image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion |
 | imagePullSecrets | list | `[]` | Global Docker registry secret names as an array |
 | ingress | object | `{"annotations":{},"className":"","enabled":false,"hosts":[{"host":"chart-example.local","paths":[{"path":"/","pathType":"ImplementationSpecific"}]}],"tls":[]}` | Ingress configuration to expose app </br> Ref: https://kubernetes.io/docs/concepts/services-networking/ingress/ |
+| initContainers | list | `[]` | Configure additional containers </br> Ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ |
 | lifecycle | object | `{}` | Configure lifecycle hooks </br> Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ </br> Ref: https://learnk8s.io/graceful-shutdown |
 | livenessProbe | object | `{"enabled":true,"failureThreshold":3,"initialDelaySeconds":180,"periodSeconds":10,"successThreshold":1,"timeoutSeconds":5}` | Configure liveness checker </br> Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#define-startup-probes |
 | livenessProbeCustom | object | `{}` | Custom livenessProbe |
@@ -133,15 +141,15 @@ helm show values opencti/opencti
 | redis.enabled | bool | `true` | Enable or disable Redis subchart |
 | replicaCount | int | `1` | Number of replicas for the service |
 | resources | object | `{}` | The resources limits and requested </br> Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ |
-| secrets | object | `{}` | Secrets values to create credentials and reference by envFromSecrets Generate Secret with following name: `<release-name>-credentials` |
+| secrets | object | `{}` | Secrets values to create credentials and reference by envFromSecrets Generate Secret with following name: `<release-name>-credentials` </br> Ref: https://kubernetes.io/docs/concepts/configuration/secret/ |
 | securityContext | object | `{}` | Defines privilege and access control settings for a Container </br> Ref: https://kubernetes.io/docs/concepts/security/pod-security-standards/ </br> Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ |
 | service | object | `{"port":80,"targetPort":4000,"type":"ClusterIP"}` | Kubernetes service to expose Pod </br> Ref: https://kubernetes.io/docs/concepts/services-networking/service/ |
 | service.port | int | `80` | Kubernetes Service port |
 | service.targetPort | int | `4000` | Pod expose port |
 | service.type | string | `"ClusterIP"` | Kubernetes Service type. Allowed values: NodePort, LoadBalancer or ClusterIP |
 | serviceAccount | object | `{"annotations":{},"automountServiceAccountToken":false,"create":true,"name":""}` | Enable creation of ServiceAccount |
 | serviceAccount.annotations | object | `{}` | Annotations to add to the service account |
-| serviceAccount.automountServiceAccountToken | bool | `false` | Specifies if you don't want the kubelet to automatically mount a ServiceAccount's API credentials |
+| serviceAccount.automountServiceAccountToken | bool | `false` | Specifies if you don't want the kubelet to automatically mount a ServiceAccount API credentials |
 | serviceAccount.create | bool | `true` | Specifies whether a service account should be created |
 | serviceAccount.name | string | `""` | Name of the service account to use. If not set and create is true, a name is generated using the fullname template |
 | serviceMonitor | object | `{"enabled":false,"interval":"30s","metricRelabelings":[],"relabelings":[],"scrapeTimeout":"10s"}` | Enable ServiceMonitor to get metrics </br> Ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor |
@@ -154,16 +162,22 @@ helm show values opencti/opencti
 | topologySpreadConstraints | list | `[]` | Control how Pods are spread across your cluster </br> Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/#example-multiple-topologyspreadconstraints |
 | volumeMounts | list | `[]` | Additional volumeMounts on the output Deployment definition |
 | volumes | list | `[]` | Additional volumes on the output Deployment definition |
-| worker | object | `{"affinity":{},"autoscaling":{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80},"enabled":true,"env":{"WORKER_LOG_LEVEL":"info","WORKER_TELEMETRY_ENABLED":true},"envFromSecrets":{},"image":{"pullPolicy":"IfNotPresent","repository":"opencti/worker","tag":""},"lifecycle":{},"networkPolicy":{"egress":[],"enabled":false,"ingress":[],"policyTypes":[]},"nodeSelector":{},"podDisruptionBudget":{"enabled":false,"maxUnavailable":1,"minAvailable":null},"readyChecker":{"enabled":true,"pullPolicy":"IfNotPresent","repository":"busybox","retries":30,"tag":"latest","timeout":5},"replicaCount":1,"resources":{},"serviceMonitor":{"enabled":false,"interval":"30s","metricRelabelings":[],"relabelings":[],"scrapeTimeout":"10s"},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[],"volumeMounts":[],"volumes":[]}` | OpenCTI worker deployment configuration </br> Ref: https://docs.opencti.io/latest/deployment/overview/#workers |
+| worker | object | `{"affinity":{},"args":[],"autoscaling":{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80},"command":[],"configMaps":[],"enabled":true,"env":{"WORKER_LOG_LEVEL":"info","WORKER_TELEMETRY_ENABLED":true},"envFromConfigMap":{},"envFromFiles":[],"envFromSecrets":{},"image":{"pullPolicy":"IfNotPresent","repository":"opencti/worker","tag":""},"initContainers":[],"lifecycle":{},"networkPolicy":{"egress":[],"enabled":false,"ingress":[],"policyTypes":[]},"nodeSelector":{},"podDisruptionBudget":{"enabled":false,"maxUnavailable":1,"minAvailable":null},"readyChecker":{"enabled":true,"pullPolicy":"IfNotPresent","repository":"busybox","retries":30,"tag":"latest","timeout":5},"replicaCount":1,"resources":{},"serviceMonitor":{"enabled":false,"interval":"30s","metricRelabelings":[],"relabelings":[],"scrapeTimeout":"10s"},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[],"volumeMounts":[],"volumes":[]}` | OpenCTI worker deployment configuration </br> Ref: https://docs.opencti.io/latest/deployment/overview/#workers |
 | worker.affinity | object | `{}` | Affinity for pod assignment </br> Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity |
+| worker.args | list | `[]` | Configure args </br> Ref: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/ |
 | worker.autoscaling | object | `{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80}` | Autoscaling with CPU or memory utilization percentage </br> Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/ |
+| worker.command | list | `[]` | Configure command </br> Ref: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/ |
+| worker.configMaps | list | `[]` | ConfigMap values to create configuration files Generate ConfigMap with following name: <release-name>-<name> </br> Ref: https://kubernetes.io/docs/concepts/configuration/configmap/ |
 | worker.enabled | bool | `true` | Enable or disable worker |
 | worker.env | object | `{"WORKER_LOG_LEVEL":"info","WORKER_TELEMETRY_ENABLED":true}` | Environment variables to configure application </br> Ref: https://docs.opencti.io/latest/deployment/configuration/#platform |
-| worker.envFromSecrets | object | `{}` | Secrets from variables |
+| worker.envFromConfigMap | object | `{}` | Variables from configMap |
+| worker.envFromFiles | list | `[]` | Load all variables from files </br> Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables |
+| worker.envFromSecrets | object | `{}` | Variables from secrets |
 | worker.image | object | `{"pullPolicy":"IfNotPresent","repository":"opencti/worker","tag":""}` | Image registry configuration for the base service |
 | worker.image.pullPolicy | string | `"IfNotPresent"` | Pull policy for the image |
 | worker.image.repository | string | `"opencti/worker"` | Repository of the image |
 | worker.image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion |
+| worker.initContainers | list | `[]` | Configure additional containers </br> Ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ |
 | worker.lifecycle | object | `{}` | Configure lifecycle hooks </br> Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ </br> Ref: https://learnk8s.io/graceful-shutdown |
 | worker.networkPolicy | object | `{"egress":[],"enabled":false,"ingress":[],"policyTypes":[]}` | NetworkPolicy configuration </br> Ref: https://kubernetes.io/docs/concepts/services-networking/network-policies/ |
 | worker.networkPolicy.enabled | bool | `false` | Enable or disable NetworkPolicy |

diff --git a/charts/opencti/ci/ci-common-values.yaml b/charts/opencti/ci/ci-common-values.yaml
@@ -27,6 +27,11 @@ lifecycle:
     exec:
       command: ["sh", "-c", "sleep 10"]
 
+initContainers:
+  - name: my-container
+    image: busybox
+    command: ['sh', '-c', 'echo "Hello, World!"']
+
 terminationGracePeriodSeconds: 40
 
 networkPolicy:
@@ -42,6 +47,13 @@ secrets:
   APP__ADMIN__TOKEN: "b1976749-8a53-4f49-bf04-cafa2a3458c1"
   RABBITMQ__PASSWORD: ChangeMe
 
+configMaps:
+  - name: configmap-name-ci
+    data:
+      my.key: >
+        my-content
+      my_var: my-value
+
 envFromSecrets:
   APP__ADMIN__TOKEN:
     name: opencti-ci-credentials
@@ -53,6 +65,11 @@ envFromSecrets:
     name: opencti-ci-credentials
     key: RABBITMQ__PASSWORD
 
+envFromConfigMap:
+  my_var_worker:
+    name: opencti-ci-configmap-name-ci
+    key: my_var
+
 topologySpreadConstraints:
   - maxSkew: 1
     topologyKey: kubernetes.io/os
@@ -63,14 +80,34 @@ worker:
 
   readyChecker:
     enabled: true
-    retries: 40
+    retries: 60
     timeout: 10
 
   lifecycle:
     preStop:
       exec:
         command: ["sh", "-c", "sleep 10"]
 
+  initContainers:
+    - name: my-container
+      image: busybox
+      command: ['sh', '-c', 'echo "Hello, World!"']
+
+  env:
+    MY_VARIABLE_WORKER_ENV: my_value
+
+  configMaps:
+    - name: configmap-name-worker-ci
+      data:
+        my.key: >
+          my-content
+        my_var_worker: my-value
+
+  envFromConfigMap:
+    my_var_worker:
+      name: opencti-ci-configmap-name-worker-ci
+      key: my_var_worker
+
   terminationGracePeriodSeconds: 40
 
   networkPolicy:
@@ -87,10 +124,18 @@ worker:
       topologyKey: kubernetes.io/os
       whenUnsatisfiable: DoNotSchedule
 
+connectorsGlobal:
+  env:
+    MY_VARIABLE_ENV: my_value
+
 connectors:
 - name: opencti
   enabled: true
   replicas: 1
+  podSecurityContext:
+    runAsUser: 1000
+    runAsGroup: 1000
+    fsGroup: 1000
   serviceMonitor:
     enabled: true
     interval: 30s
@@ -103,7 +148,7 @@ connectors:
     automountServiceAccountToken: true
   readyChecker:
     enabled: true
-    retries: 40
+    retries: 60
     timeout: 10
   lifecycle:
     preStop:
@@ -123,6 +168,9 @@ connectors:
     OPENCTI_TOKEN:
       name: opencti-ci-credentials
       key: APP__ADMIN__TOKEN
+  envFromFiles:
+    - secretRef:
+        name: opencti-ci-credentials
   topologySpreadConstraints:
     - maxSkew: 1
       topologyKey: kubernetes.io/os