feat: add podsecuritycontext to expose sysctl stuffs

Signed-off-by: vsoch <[email protected]>
flux-framework · Dec 11, 2024 · 8a23c67 · 8a23c67
1 parent 1f686fe
commit 8a23c67
Show file tree

Hide file tree

Showing 20 changed files with 357 additions and 2 deletions.
diff --git a/api/v1alpha2/minicluster_types.go b/api/v1alpha2/minicluster_types.go
@@ -188,6 +188,10 @@ type PodSpec struct {
 	// +optional
 	ServiceAccountName string `json:"serviceAccountName,omitempty"`
 
+	// PodSecurityContext
+	// +optional
+	SecurityContext PodSecurityContext `json:"securityContext,omitempty"`
+
 	// RuntimeClassName for the pod
 	// +optional
 	RuntimeClassName string `json:"runtimeClassName,omitempty"`
@@ -592,6 +596,13 @@ type SecurityContext struct {
 	AddCapabilities []string `json:"addCapabilities,omitempty"`
 }
 
+type PodSecurityContext struct {
+
+	// Sysctls
+	// +optional
+	Sysctls map[string]string `json:"sysctls,omitempty"`
+}
+
 type LifeCycle struct {
 
 	// +optional

diff --git a/api/v1alpha2/swagger.json b/api/v1alpha2/swagger.json
@@ -742,6 +742,19 @@
         }
       }
     },
+    "PodSecurityContext": {
+      "type": "object",
+      "properties": {
+        "sysctls": {
+          "description": "Sysctls",
+          "type": "object",
+          "additionalProperties": {
+            "type": "string",
+            "default": ""
+          }
+        }
+      }
+    },
     "PodSpec": {
       "description": "PodSpec controlls variables for the cluster pod",
       "type": "object",
@@ -793,6 +806,11 @@
           "description": "Scheduler name for the pod",
           "type": "string"
         },
+        "securityContext": {
+          "description": "PodSecurityContext",
+          "default": {},
+          "$ref": "#/definitions/PodSecurityContext"
+        },
         "serviceAccountName": {
           "description": "Service account name for the pod",
           "type": "string"

diff --git a/api/v1alpha2/zz_generated.deepcopy.go b/api/v1alpha2/zz_generated.deepcopy.go
diff --git a/api/v1alpha2/zz_generated.openapi.go b/api/v1alpha2/zz_generated.openapi.go
diff --git a/chart/templates/minicluster-crd.yaml b/chart/templates/minicluster-crd.yaml
@@ -567,6 +567,15 @@ spec:
                   schedulerName:
                     description: Scheduler name for the pod
                     type: string
+                  securityContext:
+                    description: PodSecurityContext
+                    properties:
+                      sysctls:
+                        additionalProperties:
+                          type: string
+                        description: Sysctls
+                        type: object
+                    type: object
                   serviceAccountName:
                     description: Service account name for the pod
                     type: string

diff --git a/config/crd/bases/flux-framework.org_miniclusters.yaml b/config/crd/bases/flux-framework.org_miniclusters.yaml
@@ -570,6 +570,15 @@ spec:
                   schedulerName:
                     description: Scheduler name for the pod
                     type: string
+                  securityContext:
+                    description: PodSecurityContext
+                    properties:
+                      sysctls:
+                        additionalProperties:
+                          type: string
+                        description: Sysctls
+                        type: object
+                    type: object
                   serviceAccountName:
                     description: Service account name for the pod
                     type: string

diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml
@@ -13,3 +13,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: ghcr.io/flux-framework/flux-operator
+  newTag: test
diff --git a/controllers/flux/containers.go b/controllers/flux/containers.go
@@ -137,6 +137,7 @@ func getContainers(
 				Add: addCaps,
 			},
 		}
+
 		newContainer := corev1.Container{
 
 			// Call this the driver container, number 0

diff --git a/controllers/flux/job.go b/controllers/flux/job.go
@@ -73,6 +73,7 @@ func NewMiniClusterJob(cluster *api.MiniCluster) (*batchv1.Job, error) {
 					RestartPolicy:                corev1.RestartPolicyOnFailure,
 					NodeSelector:                 cluster.Spec.Pod.NodeSelector,
 					SchedulerName:                cluster.Spec.Pod.SchedulerName,
+					SecurityContext:              getPodSecurityContext(cluster),
 				},
 			},
 		},

diff --git a/controllers/flux/pods.go b/controllers/flux/pods.go
@@ -36,6 +36,22 @@ func getPodLabels(cluster *api.MiniCluster) map[string]string {
 	return podLabels
 }
 
+// getPodSecurityContext is shared for the pod and service pods
+func getPodSecurityContext(cluster *api.MiniCluster) *corev1.PodSecurityContext {
+	sysctls := []corev1.Sysctl{}
+	for key, value := range cluster.Spec.Pod.SecurityContext.Sysctls {
+		newSysctls := corev1.Sysctl{
+			Name:  key,
+			Value: value,
+		}
+		sysctls = append(sysctls, newSysctls)
+	}
+	securityContext := &corev1.PodSecurityContext{
+		Sysctls: sysctls,
+	}
+	return securityContext
+}
+
 // ensure service containers are running, currently in one pod
 func (r *MiniClusterReconciler) ensureServicePod(
 	ctx context.Context,
@@ -139,6 +155,7 @@ func (r *MiniClusterReconciler) newServicePod(
 			ServiceAccountName:           cluster.Spec.Pod.ServiceAccountName,
 			AutomountServiceAccountToken: &cluster.Spec.Pod.AutomountServiceAccountToken,
 			NodeSelector:                 cluster.Spec.Pod.NodeSelector,
+			SecurityContext:              getPodSecurityContext(cluster),
 		},
 	}
 

diff --git a/docs/getting_started/custom-resource-definition.md b/docs/getting_started/custom-resource-definition.md
@@ -639,6 +639,39 @@ be adjusted if needed.
 
 Variables and attributes for each pod in the Indexed job.
 
+#### securityContext
+
+Currently, we just support setting sysctls. The following section:
+
+```yaml
+pod:
+  securityContext:
+    sysctls:
+      "net.core.somaxconn": "4096"
+```
+
+Would map to this for the minicluster pod:
+
+```yaml
+pod:
+  securityContext:
+    sysctls:
+    - name: net.core.somaxconn
+      value: "4096"
+```
+
+Note that you need to also make changes to the kubeletConfiguration to allowUnsafeSysctls.
+
+```yaml
+kind: KubeletConfiguration
+apiVersion: kubelet.config.k8s.io/v1beta1
+failSwapOn: false
+featureGates:
+  KubeletInUserNamespace: true
+allowedUnsafeSysctls:
+- "net.core*"
+```
+
 #### labels
 
 To add custom labels for your pods (in the indexed job), add a set of key value pairs (strings) to a "labels" section:

diff --git a/examples/dist/flux-operator-arm.yaml b/examples/dist/flux-operator-arm.yaml
@@ -576,6 +576,15 @@ spec:
                   schedulerName:
                     description: Scheduler name for the pod
                     type: string
+                  securityContext:
+                    description: PodSecurityContext
+                    properties:
+                      sysctls:
+                        additionalProperties:
+                          type: string
+                        description: Sysctls
+                        type: object
+                    type: object
                   serviceAccountName:
                     description: Service account name for the pod
                     type: string

diff --git a/examples/dist/flux-operator.yaml b/examples/dist/flux-operator.yaml
@@ -576,6 +576,15 @@ spec:
                   schedulerName:
                     description: Scheduler name for the pod
                     type: string
+                  securityContext:
+                    description: PodSecurityContext
+                    properties:
+                      sysctls:
+                        additionalProperties:
+                          type: string
+                        description: Sysctls
+                        type: object
+                    type: object
                   serviceAccountName:
                     description: Service account name for the pod
                     type: string

diff --git a/sdk/python/v1alpha2/docs/PodSecurityContext.md b/sdk/python/v1alpha2/docs/PodSecurityContext.md
@@ -0,0 +1,29 @@
+# PodSecurityContext
+
+
+## Properties
+
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**sysctls** | **Dict[str, str]** | Sysctls | [optional] 
+
+## Example
+
+```python
+from fluxoperator.models.pod_security_context import PodSecurityContext
+
+# TODO update the JSON string below
+json = "{}"
+# create an instance of PodSecurityContext from a JSON string
+pod_security_context_instance = PodSecurityContext.from_json(json)
+# print the JSON string representation of the object
+print(PodSecurityContext.to_json())
+
+# convert the object into a dict
+pod_security_context_dict = pod_security_context_instance.to_dict()
+# create an instance of PodSecurityContext from a dict
+pod_security_context_from_dict = PodSecurityContext.from_dict(pod_security_context_dict)
+```
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+
diff --git a/sdk/python/v1alpha2/docs/PodSpec.md b/sdk/python/v1alpha2/docs/PodSpec.md
@@ -14,6 +14,7 @@ Name | Type | Description | Notes
 **restart_policy** | **str** | Restart Policy | [optional] 
 **runtime_class_name** | **str** | RuntimeClassName for the pod | [optional] 
 **scheduler_name** | **str** | Scheduler name for the pod | [optional] 
+**security_context** | [**PodSecurityContext**](PodSecurityContext.md) |  | [optional] 
 **service_account_name** | **str** | Service account name for the pod | [optional] 
 
 ## Example

diff --git a/sdk/python/v1alpha2/fluxoperator/__init__.py b/sdk/python/v1alpha2/fluxoperator/__init__.py
@@ -49,6 +49,7 @@
 from fluxoperator.models.mini_cluster_status import MiniClusterStatus
 from fluxoperator.models.mini_cluster_user import MiniClusterUser
 from fluxoperator.models.network import Network
+from fluxoperator.models.pod_security_context import PodSecurityContext
 from fluxoperator.models.pod_spec import PodSpec
 from fluxoperator.models.secret import Secret
 from fluxoperator.models.security_context import SecurityContext
diff --git a/sdk/python/v1alpha2/fluxoperator/models/__init__.py b/sdk/python/v1alpha2/fluxoperator/models/__init__.py
@@ -33,6 +33,7 @@
 from fluxoperator.models.mini_cluster_status import MiniClusterStatus
 from fluxoperator.models.mini_cluster_user import MiniClusterUser
 from fluxoperator.models.network import Network
+from fluxoperator.models.pod_security_context import PodSecurityContext
 from fluxoperator.models.pod_spec import PodSpec
 from fluxoperator.models.secret import Secret
 from fluxoperator.models.security_context import SecurityContext
-Original file line number
+Diff line change
@@ Expand Up / @@ -137,6 +137,7 @@ func getContainers( @@
     				Add: addCaps,
     			},
     		}
     		newContainer := corev1.Container{
     			// Call this the driver container, number 0
@@ Expand Down @@