Make k8s job backoff limit configurable for RayJob (#2091)

docs/reference/api.md

@@ -160,6 +160,7 @@ _Appears in:_
 | `entrypointNumCpus` _float_ | EntrypointNumCpus specifies the number of cpus to reserve for the entrypoint command. |
 | `entrypointNumGpus` _float_ | EntrypointNumGpus specifies the number of gpus to reserve for the entrypoint command. |
 | `entrypointResources` _string_ | EntrypointResources specifies the custom resources and quantities to reserve for the entrypoint command. |
+| `submitterConfig` _[SubmitterConfig](#submitterconfig)_ | Configurations of submitter k8s job. |
 
 
 
@@ -214,6 +215,20 @@ _Appears in:_
 | `workersToDelete` _string array_ | WorkersToDelete workers to be deleted |
 
 
+#### SubmitterConfig
+
+
+
+
+
+_Appears in:_
+- [RayJobSpec](#rayjobspec)
+
+| Field | Description |
+| --- | --- |
+| `backoffLimit` _integer_ | BackoffLimit of the submitter k8s job. |
+
+
 #### UpscalingMode
 
 _Underlying type:_ _string_

ray-operator/apis/ray/v1/rayjob_types.go

@@ -59,6 +59,11 @@ const (
 	HTTPMode   JobSubmissionMode = "HTTPMode"   // Submit job via HTTP request
 )
 
+type SubmitterConfig struct {
+	// BackoffLimit of the submitter k8s job.
+	BackoffLimit *int32 `json:"backoffLimit,omitempty"`
+}
+
 // RayJobSpec defines the desired state of RayJob
 type RayJobSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
@@ -104,6 +109,8 @@ type RayJobSpec struct {
 	// EntrypointResources specifies the custom resources and quantities to reserve for the
 	// entrypoint command.
 	EntrypointResources string `json:"entrypointResources,omitempty"`
+	// Configurations of submitter k8s job.
+	SubmitterConfig *SubmitterConfig `json:"submitterConfig,omitempty"`
 }
 
 // RayJobStatus defines the observed state of RayJob

ray-operator/controllers/ray/rayjob_controller.go

@@ -422,6 +422,10 @@ func (r *RayJobReconciler) getSubmitterTemplate(ctx context.Context, rayJobInsta
 // createNewK8sJob creates a new Kubernetes Job. It returns an error.
 func (r *RayJobReconciler) createNewK8sJob(ctx context.Context, rayJobInstance *rayv1.RayJob, submitterTemplate corev1.PodTemplateSpec) error {
 	logger := ctrl.LoggerFrom(ctx)
+	submitterBackoffLimit := pointer.Int32(2)
+	if rayJobInstance.Spec.SubmitterConfig != nil && rayJobInstance.Spec.SubmitterConfig.BackoffLimit != nil {
+		submitterBackoffLimit = rayJobInstance.Spec.SubmitterConfig.BackoffLimit
+	}
 	job := &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      rayJobInstance.Name,
@@ -437,7 +441,7 @@ func (r *RayJobReconciler) createNewK8sJob(ctx context.Context, rayJobInstance *
 			// is attempted 3 times at the maximum, but still mitigates the case of unrecoverable
 			// application-level errors, where the maximum number of retries is reached, and the job
 			// completion time increases with no benefits, but wasted resource cycles.
-			BackoffLimit: pointer.Int32(2),
+			BackoffLimit: submitterBackoffLimit,
 			Template:     submitterTemplate,
 		},
 	}