Fix customizer RBAC for run.ai resources

Signed-off-by: Shiva Krishna, Merla <[email protected]>

Author: shivamerla

api/apps/v1alpha1/nemo_customizer_types.go

@@ -866,6 +866,11 @@ func (n *NemoCustomizer) GetRoleParams() *rendertypes.RoleParams {
 			Resources: []string{"pods", "persistentvolumeclaims", "services", "configmaps"},
 			Verbs:     []string{"create", "get", "list", "watch", "delete"},
 		},
+		{
+			APIGroups: []string{""},
+			Resources: []string{"events"},
+			Verbs:     []string{"create", "get", "list", "watch"},
+		},
 		{
 			APIGroups: []string{"nvidia.com"},
 			Resources: []string{"nemotrainingjobs", "nemotrainingjobs/status", "nemoentityhandlers"},
@@ -896,8 +901,18 @@ func (n *NemoCustomizer) GetRoleParams() *rendertypes.RoleParams {
 		},
 	}
 
+	runAIRules := []rbacv1.PolicyRule{
+		{
+			APIGroups: []string{"run.ai"},
+			Resources: []string{"trainingworkloads", "runaijobs"},
+			Verbs:     []string{"create", "get", "list", "watch", "update", "delete", "patch"},
+		},
+	}
+
 	if n.Spec.Scheduler.Type == SchedulerTypeVolcano {
 		params.Rules = append(params.Rules, volcanoRules...)
+	} else if n.Spec.Scheduler.Type == SchedulerTypeRunAI {
+		params.Rules = append(params.Rules, runAIRules...)
 	}
 
 	return params

bundle/manifests/k8s-nim-operator.clusterserviceversion.yaml

@@ -744,9 +744,12 @@ spec:
               resources:
               - events
               verbs:
+              - get
               - create
               - patch
               - update
+              - list
+              - watch
       clusterPermissions:
         - serviceAccountName: k8s-nim-operator
           rules:
@@ -763,9 +766,12 @@ spec:
               resources:
               - events
               verbs:
+              - get
               - create
               - patch
               - update
+              - list
+              - watch
             - apiGroups:
                 - ''
               resources:
@@ -1206,6 +1212,18 @@ spec:
               - get
               - list
               - watch
+            - apiGroups:
+              - run.ai
+              resources:
+              - trainingworkloads
+              - runaijobs
+              verbs:
+              - get
+              - list
+              - watch
+              - delete
+              - patch
+              - update
             - apiGroups:
               - nvidia.com
               resources:

config/rbac/role.yaml

@@ -30,8 +30,11 @@ rules:
   - events
   verbs:
   - create
+  - get
+  - list
   - patch
   - update
+  - watch
 - apiGroups:
   - ""
   resources:
@@ -221,6 +224,19 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - run.ai
+  resources:
+  - runaijobs
+  - trainingworkloads
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - scheduling.incubator.k8s.io
   - scheduling.volcano.sh

deployments/helm/k8s-nim-operator/templates/manager-rbac.yaml

@@ -32,6 +32,8 @@ rules:
   resources:
   - events
   verbs:
+  - get
+  - list
   - create
   - update
   - patch
@@ -486,7 +488,18 @@ rules:
   - get
   - list
   - watch
-
+- apiGroups:
+  - run.ai
+  resources:
+  - trainingworkloads
+  - runaijobs
+  verbs:
+  - get
+  - list
+  - watch
+  - delete
+  - patch
+  - update
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1

internal/controller/nemocustomizer_controller.go

@@ -90,6 +90,8 @@ func NewNemoCustomizerReconciler(client client.Client, scheme *runtime.Scheme, u
 // +kubebuilder:rbac:groups=apps.nvidia.com,resources=nemocustomizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=apps.nvidia.com,resources=nemocustomizers/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=apps.nvidia.com,resources=nemocustomizers/finalizers,verbs=update
+// +kubebuilder:rbac:groups=run.ai,resources=trainingworkloads;runaijobs,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups="",resources=events,verbs=get;list;watch;create
 // +kubebuilder:rbac:groups=nvidia.com,resources=nemotrainingjobs;nemotrainingjobs/status;nemoentityhandlers,verbs=create;get;list;watch;update;delete;patch
 // +kubebuilder:rbac:groups=batch.volcano.sh,resources=jobs;jobs/status,verbs=get;list;watch
 // +kubebuilder:rbac:groups=nodeinfo.volcano.sh,resources=numatopologies,verbs=get;list;watch
@@ -644,10 +646,10 @@ func (r *NemoCustomizerReconciler) addTrainingConfig(ctx context.Context, cfg ma
 	trainingCfg["env"] = n.Spec.Training.Env
 	trainingCfg["training_networking"] = n.Spec.Training.NetworkConfig
 	trainingCfg["workspace_dir"] = n.Spec.Training.WorkspacePVC.MountPath
-	trainingCfg["use_run_ai_executor"] = "false"
+	trainingCfg["use_run_ai_executor"] = false
 
 	if n.Spec.Scheduler.Type == appsv1alpha1.SchedulerTypeRunAI {
-		trainingCfg["use_run_ai_executor"] = "true"
+		trainingCfg["use_run_ai_executor"] = true
 	}
 	if n.Spec.Training.TTLSecondsAfterFinished != nil {
 		trainingCfg["ttl_seconds_after_finished"] = *n.Spec.Training.TTLSecondsAfterFinished