Scale down cilium-operator before deleting a cluster (only in eni mode)

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- Scale down cilium-operator before deleting a cluster (only in eni mode)
+
 ## [2.6.1] - 2025-02-07
 
 ### Added

helm/cluster-aws/templates/cilium-cleanup/job.yaml

@@ -0,0 +1,71 @@
+{{- if eq (required "global.connectivity.cilium.ipamMode is required" .Values.global.connectivity.cilium.ipamMode) "eni" }}
+#
+# When working in eni mode and ENIs need to be deleted (e.g. on cluster deletion), Cilium first needs to detach
+# the ENIs from the EC2 instance, and then delete them.
+# If cilium-operator gets killed / removed after it detaches an ENI but before it deletes it, the ENI will be orphaned.
+#
+# This is a possible scenario on cluster deletion. To prevent it, this pre-delete hook will scale down cilium-operator before deleting.
+# This way ENIs remain attached to the EC2 instances and get automatically deleted by AWS upon instance termination.
+#
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "resource.default.name" . }}-cilium-cleanup
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    helm.sh/hook: pre-delete
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+    helm.sh/hook-weight: "0"
+  labels:
+    {{- include "labels.common" . | nindent 4 }}
+spec:
+  template:
+    metadata:
+      labels:
+        {{- include "labels.common" . | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "resource.default.name" . }}-cilium-cleanup
+      containers:
+      - name: kubectl
+        image: {{ include "awsContainerImageRegistry" . }}/giantswarm/kubectl:{{ .Values.cluster.providerIntegration.kubernetesVersion }}
+        securityContext:
+          runAsNonRoot: true
+          runAsUser: 1000
+          runAsGroup: 1000
+          allowPrivilegeEscalation: false
+          seccompProfile:
+            type: RuntimeDefault
+          capabilities:
+            drop:
+            - ALL
+        env:
+        - name: CLUSTER
+          value: {{ include "resource.default.name" . }}
+        - name: NAMESPACE
+          value: {{ .Release.Namespace }}
+        command:
+        - /bin/sh
+        args:
+        - -c
+        - |
+          echo "Fetching ${CLUSTER} kubeconfig..."
+          kubectl get secret -n ${NAMESPACE} ${CLUSTER}-kubeconfig -o jsonpath='{.data.value}' | base64 -d > /tmp/${CLUSTER}-kubeconfig
+          if [ ! -f /tmp/${CLUSTER}-kubeconfig ]; then
+            echo "Failed to fetch kubeconfig for cluster ${CLUSTER}. Assuming it is already deleted."
+            exit 0
+          fi
+
+          echo "Scaling down cilium-operator on cluster ${CLUSTER} as it is about to be deleted..."
+          kubectl --kubeconfig /tmp/${CLUSTER}-kubeconfig scale --namespace kube-system deployment cilium-operator --replicas=0
+
+          exit 0 # Ignore previous exit code, we don't want to block app deletion
+        resources:
+          requests:
+            cpu: 10m
+            memory: 64Mi
+          limits:
+            cpu: 100m
+            memory: 256Mi
+      restartPolicy: Never
+  ttlSecondsAfterFinished: 86400 # 24h
+{{- end }}

helm/cluster-aws/templates/cilium-cleanup/role.yaml

@@ -0,0 +1,22 @@
+{{- if eq (required "global.connectivity.cilium.ipamMode is required" .Values.global.connectivity.cilium.ipamMode) "eni" }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "resource.default.name" . }}-cilium-cleanup
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    helm.sh/hook: pre-delete
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed
+    helm.sh/hook-weight: "-1"
+  labels:
+    {{- include "labels.common" . | nindent 4 }}
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  resourceNames:
+  - {{ include "resource.default.name" . }}-kubeconfig
+{{- end }}

helm/cluster-aws/templates/cilium-cleanup/rolebinding.yaml

@@ -0,0 +1,21 @@
+{{- if eq (required "global.connectivity.cilium.ipamMode is required" .Values.global.connectivity.cilium.ipamMode) "eni" }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "resource.default.name" . }}-cilium-cleanup
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    helm.sh/hook: pre-delete
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed
+    helm.sh/hook-weight: "-1"
+  labels:
+    {{- include "labels.common" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "resource.default.name" . }}-cilium-cleanup
+subjects:
+- kind: ServiceAccount
+  name: {{ include "resource.default.name" . }}-cilium-cleanup
+  namespace: {{ .Release.Namespace }}
+{{- end }}

helm/cluster-aws/templates/cilium-cleanup/serviceaccount.yaml

@@ -0,0 +1,13 @@
+{{- if eq (required "global.connectivity.cilium.ipamMode is required" .Values.global.connectivity.cilium.ipamMode) "eni" }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "resource.default.name" . }}-cilium-cleanup
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    helm.sh/hook: pre-delete
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed
+    helm.sh/hook-weight: "-1"
+  labels:
+    {{- include "labels.common" . | nindent 4 }}
+{{- end }}