feat: add RayCluster.status.readyWorkerReplicas

A worker Pod is ready if it has a PodCondition with type == Ready and status ==
True. See https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-conditions

Author: davidxia

helm-chart/kuberay-operator/crds/ray.io_rayclusters.yaml

@@ -121,6 +121,8 @@ type RayClusterStatus struct {
 	// Important: Run "make" to regenerate code after modifying this file
 	// Status reflects the status of the cluster
 	State ClusterState `json:"state,omitempty"`
+	// ReadyWorkerReplicas indicates how many worker replicas are ready in the cluster
+	ReadyWorkerReplicas int32 `json:"readyWorkerReplicas,omitempty"`
 	// AvailableWorkerReplicas indicates how many replicas are available in the cluster
 	AvailableWorkerReplicas int32 `json:"availableWorkerReplicas,omitempty"`
 	// DesiredWorkerReplicas indicates overall desired replicas claimed by the user at the cluster level.

helm-chart/kuberay-operator/crds/ray.io_rayjobs.yaml

@@ -119,6 +119,8 @@ type RayClusterStatus struct {
 	// Important: Run "make" to regenerate code after modifying this file
 	// Status reflects the status of the cluster
 	State ClusterState `json:"state,omitempty"`
+	// ReadyWorkerReplicas indicates how many worker replicas are ready in the cluster
+	ReadyWorkerReplicas int32 `json:"readyWorkerReplicas,omitempty"`
 	// AvailableWorkerReplicas indicates how many replicas are available in the cluster
 	AvailableWorkerReplicas int32 `json:"availableWorkerReplicas,omitempty"`
 	// DesiredWorkerReplicas indicates overall desired replicas claimed by the user at the cluster level.

helm-chart/kuberay-operator/crds/ray.io_rayservices.yaml

@@ -400,13 +400,22 @@ func (r *RayClusterReconciler) inconsistentRayClusterStatus(ctx context.Context,
 			oldStatus.State, newStatus.State, oldStatus.Reason, newStatus.Reason))
 		return true
 	}
-	if oldStatus.AvailableWorkerReplicas != newStatus.AvailableWorkerReplicas || oldStatus.DesiredWorkerReplicas != newStatus.DesiredWorkerReplicas ||
-		oldStatus.MinWorkerReplicas != newStatus.MinWorkerReplicas || oldStatus.MaxWorkerReplicas != newStatus.MaxWorkerReplicas {
+	if oldStatus.ReadyWorkerReplicas != newStatus.ReadyWorkerReplicas ||
+		oldStatus.AvailableWorkerReplicas != newStatus.AvailableWorkerReplicas ||
+		oldStatus.DesiredWorkerReplicas != newStatus.DesiredWorkerReplicas ||
+		oldStatus.MinWorkerReplicas != newStatus.MinWorkerReplicas ||
+		oldStatus.MaxWorkerReplicas != newStatus.MaxWorkerReplicas {
 		logger.Info("inconsistentRayClusterStatus", "detect inconsistency", fmt.Sprintf(
-			"old AvailableWorkerReplicas: %d, new AvailableWorkerReplicas: %d, old DesiredWorkerReplicas: %d, new DesiredWorkerReplicas: %d, "+
-				"old MinWorkerReplicas: %d, new MinWorkerReplicas: %d, old MaxWorkerReplicas: %d, new MaxWorkerReplicas: %d",
-			oldStatus.AvailableWorkerReplicas, newStatus.AvailableWorkerReplicas, oldStatus.DesiredWorkerReplicas, newStatus.DesiredWorkerReplicas,
-			oldStatus.MinWorkerReplicas, newStatus.MinWorkerReplicas, oldStatus.MaxWorkerReplicas, newStatus.MaxWorkerReplicas))
+			"old ReadyWorkerReplicas: %d, new ReadyWorkerReplicas: %d, "+
+				"old AvailableWorkerReplicas: %d, new AvailableWorkerReplicas: %d, "+
+				"old DesiredWorkerReplicas: %d, new DesiredWorkerReplicas: %d, "+
+				"old MinWorkerReplicas: %d, new MinWorkerReplicas: %d, "+
+				"old MaxWorkerReplicas: %d, new MaxWorkerReplicas: %d",
+			oldStatus.ReadyWorkerReplicas, newStatus.ReadyWorkerReplicas,
+			oldStatus.AvailableWorkerReplicas, newStatus.AvailableWorkerReplicas,
+			oldStatus.DesiredWorkerReplicas, newStatus.DesiredWorkerReplicas,
+			oldStatus.MinWorkerReplicas, newStatus.MinWorkerReplicas,
+			oldStatus.MaxWorkerReplicas, newStatus.MaxWorkerReplicas))
 		return true
 	}
 	if !reflect.DeepEqual(oldStatus.Endpoints, newStatus.Endpoints) || !reflect.DeepEqual(oldStatus.Head, newStatus.Head) {
@@ -1222,6 +1231,7 @@ func (r *RayClusterReconciler) calculateStatus(ctx context.Context, instance *ra
 		return nil, err
 	}
 
+	newInstance.Status.ReadyWorkerReplicas = utils.CalculateReadyReplicas(runtimePods)
 	newInstance.Status.AvailableWorkerReplicas = utils.CalculateAvailableReplicas(runtimePods)
 	newInstance.Status.DesiredWorkerReplicas = utils.CalculateDesiredReplicas(ctx, newInstance)
 	newInstance.Status.MinWorkerReplicas = utils.CalculateMinReplicas(newInstance)

ray-operator/apis/ray/v1/raycluster_types.go

@@ -1530,6 +1530,7 @@ func TestInconsistentRayClusterStatus(t *testing.T) {
 	timeNow := metav1.Now()
 	oldStatus := rayv1.RayClusterStatus{
 		State:                   rayv1.Ready,
+		ReadyWorkerReplicas:     1,
 		AvailableWorkerReplicas: 1,
 		DesiredWorkerReplicas:   1,
 		MinWorkerReplicas:       1,
@@ -1564,42 +1565,47 @@ func TestInconsistentRayClusterStatus(t *testing.T) {
 	newStatus.Reason = "new reason"
 	assert.True(t, r.inconsistentRayClusterStatus(ctx, oldStatus, *newStatus))
 
-	// Case 3: `AvailableWorkerReplicas` is different => return true
+	// Case 3: `ReadyWorkerReplicas` is different => return true
+	newStatus = oldStatus.DeepCopy()
+	newStatus.ReadyWorkerReplicas = oldStatus.ReadyWorkerReplicas + 1
+	assert.True(t, r.inconsistentRayClusterStatus(ctx, oldStatus, *newStatus))
+
+	// Case 4: `AvailableWorkerReplicas` is different => return true
 	newStatus = oldStatus.DeepCopy()
 	newStatus.AvailableWorkerReplicas = oldStatus.AvailableWorkerReplicas + 1
 	assert.True(t, r.inconsistentRayClusterStatus(ctx, oldStatus, *newStatus))
 
-	// Case 4: `DesiredWorkerReplicas` is different => return true
+	// Case 5: `DesiredWorkerReplicas` is different => return true
 	newStatus = oldStatus.DeepCopy()
 	newStatus.DesiredWorkerReplicas = oldStatus.DesiredWorkerReplicas + 1
 	assert.True(t, r.inconsistentRayClusterStatus(ctx, oldStatus, *newStatus))
 
-	// Case 5: `MinWorkerReplicas` is different => return true
+	// Case 6: `MinWorkerReplicas` is different => return true
 	newStatus = oldStatus.DeepCopy()
 	newStatus.MinWorkerReplicas = oldStatus.MinWorkerReplicas + 1
 	assert.True(t, r.inconsistentRayClusterStatus(ctx, oldStatus, *newStatus))
 
-	// Case 6: `MaxWorkerReplicas` is different => return true
+	// Case 7: `MaxWorkerReplicas` is different => return true
 	newStatus = oldStatus.DeepCopy()
 	newStatus.MaxWorkerReplicas = oldStatus.MaxWorkerReplicas + 1
 	assert.True(t, r.inconsistentRayClusterStatus(ctx, oldStatus, *newStatus))
 
-	// Case 7: `Endpoints` is different => return true
+	// Case 8: `Endpoints` is different => return true
 	newStatus = oldStatus.DeepCopy()
 	newStatus.Endpoints["fakeEndpoint"] = "10009"
 	assert.True(t, r.inconsistentRayClusterStatus(ctx, oldStatus, *newStatus))
 
-	// Case 8: `Head` is different => return true
+	// Case 9: `Head` is different => return true
 	newStatus = oldStatus.DeepCopy()
 	newStatus.Head.PodIP = "test head pod ip"
 	assert.True(t, r.inconsistentRayClusterStatus(ctx, oldStatus, *newStatus))
 
-	// Case 9: `LastUpdateTime` is different => return false
+	// Case 10: `LastUpdateTime` is different => return false
 	newStatus = oldStatus.DeepCopy()
 	newStatus.LastUpdateTime = &metav1.Time{Time: timeNow.Add(time.Hour)}
 	assert.False(t, r.inconsistentRayClusterStatus(ctx, oldStatus, *newStatus))
 
-	// Case 10: `ObservedGeneration` is different => return false
+	// Case 11: `ObservedGeneration` is different => return false
 	newStatus = oldStatus.DeepCopy()
 	newStatus.ObservedGeneration = oldStatus.ObservedGeneration + 1
 	assert.False(t, r.inconsistentRayClusterStatus(ctx, oldStatus, *newStatus))