feat: allow scaling telemetry-proxy

charts/lumigo-operator/templates/controller-deployment-and-webhooks.yaml

@@ -124,6 +124,19 @@ spec:
   ports:
   {{- .Values.injectorWebhookService.ports | toYaml | nindent 2 }}
 ---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "helm.fullname" . }}-shared-configurations
+  labels:
+  {{- include "helm.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 5Mi
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -226,9 +239,86 @@ spec:
         - name: cert
           mountPath: /tmp/k8s-webhook-server/serving-certs
           readOnly: true
-        - name: namespace-configurations
+        - name: shared-configurations
           mountPath: /lumigo/etc/namespaces/
           readOnly: false
+      - name: kube-rbac-proxy
+        image: {{ .Values.controllerManager.kubeRbacProxy.image.repository }}:{{ .Values.controllerManager.kubeRbacProxy.image.tag | default .Chart.AppVersion }}
+        args:
+        - --secure-listen-address=0.0.0.0:8443
+        - --upstream=http://127.0.0.1:8080/
+        - --logtostderr=true
+        - --v=0
+        env:
+        - name: KUBERNETES_CLUSTER_DOMAIN
+          value: {{ .Values.kubernetesClusterDomain }}
+        ports:
+        - containerPort: 8443
+          name: https
+          protocol: TCP
+        resources: {{- toYaml .Values.controllerManager.kubeRbacProxy.resources | nindent 10 }}
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+      securityContext:
+        runAsNonRoot: true
+        fsGroup: 1234
+      serviceAccountName: lumigo-kubernetes-operator
+      terminationGracePeriodSeconds: 10
+      volumes:
+      - name: cert
+        secret:
+          defaultMode: 420
+          secretName: '{{ include "helm.fullname" . }}-lumigo-injector-webhook-certs'
+      - name: shared-configurations
+        persistentVolumeClaim:
+          claimName: {{ include "helm.fullname" . }}-shared-configurations
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "helm.fullname" . }}-telemetry-proxy
+  labels:
+  {{- include "helm.labels" . | nindent 4 }}
+    app.kubernetes.io/component: telemetry-proxy
+    app.kubernetes.io/created-by: lumigo
+    app.kubernetes.io/part-of: lumigo
+    control-plane: telemetry-proxy
+    lumigo.auto-trace: 'false'
+spec:
+  replicas: {{ .Values.controllerManager.telemetryProxy.replicas }}
+  selector:
+    matchLabels:
+      control-plane: telemetry-proxy
+    {{- include "helm.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+      {{- include "helm.selectorLabels" . | nindent 8 }}
+        control-plane: telemetry-proxy
+        lumigo.auto-trace: 'false'
+      annotations:
+        kubectl.kubernetes.io/default-container: telemetry-proxy
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: kubernetes.io/arch
+                operator: In
+                values:
+                - amd64
+                - arm64
+              - key: kubernetes.io/os
+                operator: In
+                values:
+                - linux
+      containers:
       - name: telemetry-proxy
         image: {{ .Values.controllerManager.telemetryProxy.image.repository }}:{{ .Values.controllerManager.telemetryProxy.image.tag | default .Chart.AppVersion }}
         env:
@@ -261,8 +351,6 @@ spec:
           value: "{{ .Values.clusterCollection.metrics.frequency }}"
         - name: LUMIGO_CLUSTER_AGENT_SERVICE
           value: "{{ include "helm.fullname" . }}-cluster-agent-service.{{ .Release.Namespace }}.svc.cluster.local"
-        - name: LUMIGO_PROM_NODE_EXPORTER_PORT
-          value: "{{ .Values.prometheusNodeExporter.service.nodePort }}"
         - name: LUMIGO_KUBE_STATE_METRICS_SERVICE
           value: "{{ .Release.Name }}-kube-state-metrics.{{ .Release.Namespace }}.svc.cluster.local"
         - name: LUMIGO_KUBE_STATE_METRICS_PORT
@@ -284,6 +372,8 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        - name: LUMIGO_TARGET_ALLOCATOR_ENDPOINT
+          value: "http://{{ include "helm.fullname" . }}-target-allocator.{{ .Release.Namespace }}.svc.cluster.local:8080"
         ports:
         - containerPort: 4318
           name: otlphttp
@@ -294,51 +384,23 @@ spec:
           capabilities:
             drop:
             - ALL
-          # readOnlyRootFilesystem: true
           runAsNonRoot: true
           runAsUser: 1234
         volumeMounts:
         - name: telemetry-proxy-configurations
           mountPath: /lumigo/etc/otelcol/
           readOnly: false
-        - name: namespace-configurations
+        - name: shared-configurations
           mountPath: /lumigo/etc/namespaces/
           readOnly: false
-      - name: kube-rbac-proxy
-        image: {{ .Values.controllerManager.kubeRbacProxy.image.repository }}:{{ .Values.controllerManager.kubeRbacProxy.image.tag | default .Chart.AppVersion }}
-        args:
-        - --secure-listen-address=0.0.0.0:8443
-        - --upstream=http://127.0.0.1:8080/
-        - --logtostderr=true
-        - --v=0
-        env:
-        - name: KUBERNETES_CLUSTER_DOMAIN
-          value: {{ .Values.kubernetesClusterDomain }}
-        # - name: GOMEMLIMIT
-        #   value: {{ .Values.controllerManager.kubeRbacProxy.resources.limits.memory }}
-        ports:
-        - containerPort: 8443
-          name: https
-          protocol: TCP
-        resources: {{- toYaml .Values.controllerManager.kubeRbacProxy.resources | nindent 10 }}
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-            - ALL
-          readOnlyRootFilesystem: true
-          runAsNonRoot: true
       securityContext:
         runAsNonRoot: true
         fsGroup: 1234
       serviceAccountName: lumigo-kubernetes-operator
       terminationGracePeriodSeconds: 10
       volumes:
-      - name: cert
-        secret:
-          defaultMode: 420
-          secretName: '{{ include "helm.fullname" . }}-lumigo-injector-webhook-certs'
-      - name: namespace-configurations
-        emptyDir: {}
+      - name: shared-configurations
+        persistentVolumeClaim:
+          claimName: {{ include "helm.fullname" . }}-shared-configurations
       - name: telemetry-proxy-configurations
         emptyDir: {}

charts/lumigo-operator/templates/target-allocator.yaml

@@ -0,0 +1,157 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "helm.fullname" . }}-target-allocator
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "helm.fullname" . }}-target-allocator
+rules:
+  - apiGroups: [""]
+    resources: ["nodes", "pods", "endpoints", "services"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "helm.fullname" . }}-target-allocator-binding
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "helm.fullname" . }}-target-allocator
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ include "helm.fullname" . }}-target-allocator
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "helm.fullname" . }}-target-allocator-config
+  namespace: {{ .Release.Namespace }}
+data:
+  target-allocator-config.yaml: |
+    allocation_strategy: least-weighted
+    collector_selector:
+      strategy: random
+    config:
+      global:
+        scrape_interval: {{ .Values.clusterCollection.metrics.frequency }}
+      scrape_configs:
+        - job_name: 'k8s-infra-metrics'
+          metrics_path: /metrics
+          scrape_interval: {{ .Values.clusterCollection.metrics.frequency }}
+          scheme: https
+          tls_config:
+            insecure_skip_verify: true
+          relabel_configs:
+            - source_labels: [__address__]
+              regex: (.+)
+              replacement: ${1}:10250
+              target_label: __address__
+          kubernetes_sd_configs:
+            - role: node
+          authorization:
+            credentials_file: "/var/run/secrets/kubernetes.io/serviceaccount/token"
+        - job_name: 'k8s-infra-metrics-cadvisor'
+          metrics_path: /metrics/cadvisor
+          scrape_interval: {{ .Values.clusterCollection.metrics.frequency }}
+          scheme: https
+          tls_config:
+            insecure_skip_verify: true
+          relabel_configs:
+            - source_labels: [__address__]
+              regex: (.+)
+              replacement: ${1}:10250
+              target_label: __address__
+          kubernetes_sd_configs:
+            - role: node
+          authorization:
+            credentials_file: "/var/run/secrets/kubernetes.io/serviceaccount/token"
+        - job_name: 'k8s-infra-metrics-resources'
+          metrics_path: /metrics/resource
+          scrape_interval: {{ .Values.clusterCollection.metrics.frequency }}
+          scheme: https
+          tls_config:
+            insecure_skip_verify: true
+          relabel_configs:
+            - source_labels: [__address__]
+              regex: (.+)
+              replacement: ${1}:10250
+              target_label: __address__
+          kubernetes_sd_configs:
+            - role: node
+          authorization:
+            credentials_file: "/var/run/secrets/kubernetes.io/serviceaccount/token"
+        - job_name: 'prometheus-node-exporter'
+          kubernetes_sd_configs:
+            - role: node
+          relabel_configs:
+            - source_labels: [__meta_kubernetes_node_address_InternalIP]
+              action: replace
+              target_label: __address__
+              # Scrape a custom port
+              replacement: '$1:{{ .Values.prometheusNodeExporter.service.nodePort }}'
+            - source_labels: [__meta_kubernetes_node_name]
+              action: replace
+              target_label: node
+          metrics_path: "/metrics"
+          authorization:
+            credentials_file: "/var/run/secrets/kubernetes.io/serviceaccount/token"
+        - job_name: 'kube-state-metrics'
+          metrics_path: /metrics
+          scrape_interval: 15s
+          static_configs:
+            - targets: ['{{ .Release.Name }}-kube-state-metrics.{{ .Release.Namespace }}.svc.cluster.local:{{ index .Values "kube-state-metrics" "service" "port" }}']
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "helm.fullname" . }}-target-allocator
+  namespace: {{ .Release.Namespace }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ include "helm.fullname" . }}-target-allocator
+  template:
+    metadata:
+      labels:
+        app: {{ include "helm.fullname" . }}-target-allocator
+    spec:
+      serviceAccountName: {{ include "helm.fullname" . }}-target-allocator
+      containers:
+        - name: target-allocator
+          image: ghcr.io/open-telemetry/opentelemetry-operator/target-allocator:0.124.0
+          env:
+            - name: OTELCOL_NAMESPACE
+              value: "{{ .Release.Namespace }}"
+          args:
+            - "--config-file=/conf/target-allocator-config.yaml"
+{{- if .Values.debug.enabled }}
+            - "--zap-log-level=debug"
+{{- end }}
+          ports:
+            - containerPort: 80
+          volumeMounts:
+            - name: config
+              mountPath: /conf
+      volumes:
+        - name: config
+          configMap:
+            name: {{ include "helm.fullname" . }}-target-allocator-config
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "helm.fullname" . }}-target-allocator
+  namespace: {{ .Release.Namespace }}
+spec:
+  selector:
+    app: {{ include "helm.fullname" . }}-target-allocator
+  ports:
+    - protocol: TCP
+      port: 8080
+      targetPort: 8080

charts/lumigo-operator/values.yaml

@@ -46,6 +46,7 @@ controllerManager:
         cpu: 10m
         memory: 64Mi
   telemetryProxy:
+    replicas: 1
     image:
       repository: host.docker.internal:5000/telemetry-proxy
       tag: latest
@@ -56,7 +57,6 @@ controllerManager:
       requests:
         cpu: 10m
         memory: 256Mi
-  replicas: 1
 logFileCollector:
   image:
     repository: otel/opentelemetry-collector-contrib