From c5ebe47f22ea178d9ac0d57019ceac00f025ec4a Mon Sep 17 00:00:00 2001 From: Martin Schuppert Date: Wed, 9 Apr 2025 16:11:36 +0200 Subject: [PATCH] Use deployment.IsReady() to validate status Changes to use the common IsReady() func to validate a deployment is fully up as requested and e.g. no update rollout in progress. During a minor update this has seen to already report/mark the condition ready, even the deployment is still in progress, or the replacement pod failed. Jira: OSPRH-14472 Depends-On: https://github.com/openstack-k8s-operators/lib-common/pull/616 Signed-off-by: Martin Schuppert --- api/go.mod | 2 +- api/go.sum | 4 ++-- controllers/heatapi_controller.go | 20 +++++++++++++++++--- controllers/heatcfnapi_controller.go | 22 +++++++++++++++++++--- controllers/heatengine_controller.go | 22 +++++++++++++++++++--- go.mod | 2 +- go.sum | 4 ++-- 7 files changed, 61 insertions(+), 15 deletions(-) diff --git a/api/go.mod b/api/go.mod index e83d85dd..85bb2c92 100644 --- a/api/go.mod +++ b/api/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20250403063905-eb287d52f38d - github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250402133843-5a4c5f4fb4f1 + github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250408123225-0d9e9b82c41b k8s.io/api v0.29.15 k8s.io/apimachinery v0.29.15 sigs.k8s.io/controller-runtime v0.17.6 diff --git a/api/go.sum b/api/go.sum index d9ded0db..c03807f2 100644 --- a/api/go.sum +++ b/api/go.sum @@ -75,8 +75,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20250403063905-eb287d52f38d h1:/C+ysubV00VYrqGFhQlDeQ5tUtnhIWPwQUc8MOfCEBA= github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20250403063905-eb287d52f38d/go.mod h1:IY5zp1GRhacGMvjZMUZQ0LlxWDaogIRb/4H3by1Pivk= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250402133843-5a4c5f4fb4f1 h1:hO90JhfinKysbdrWCJugTmJbkrs1d9tR7ypg3yOD12E= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250402133843-5a4c5f4fb4f1/go.mod h1:A9Ohw5Q90YOGhcDF4ZHkMr5RArz3phoBu9+ibbYDKG4= +github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250408123225-0d9e9b82c41b h1:T+N6xOT2NP+hVp2K1xl/NV3uheVHu38CcBuW+8uOBYw= +github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250408123225-0d9e9b82c41b/go.mod h1:A9Ohw5Q90YOGhcDF4ZHkMr5RArz3phoBu9+ibbYDKG4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/controllers/heatapi_controller.go b/controllers/heatapi_controller.go index b3246d46..e01e4c48 100644 --- a/controllers/heatapi_controller.go +++ b/controllers/heatapi_controller.go @@ -827,10 +827,24 @@ func (r *HeatAPIReconciler) reconcileNormal(ctx context.Context, instance *heatv // Update the ReadyCount and evaluate the readiness only when the last // ObservedGeneration is seen - if depl.GetDeployment().Generation == depl.GetDeployment().Status.ObservedGeneration { - instance.Status.ReadyCount = depl.GetDeployment().Status.ReadyReplicas - if instance.Status.ReadyCount > 0 { + deploy := depl.GetDeployment() + if deploy.Generation == deploy.Status.ObservedGeneration { + instance.Status.ReadyCount = deploy.Status.ReadyReplicas + + // Mark the Deployment as Ready only if the number of Replicas is equals + // to the Deployed instances (ReadyCount), and the the Status.Replicas + // match Status.ReadyReplicas. If a deployment update is in progress, + // Replicas > ReadyReplicas. + // In addition, make sure the controller sees the last Generation + // by comparing it with the ObservedGeneration. + if deployment.IsReady(deploy) { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + } else { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) } } // create Deployment - end diff --git a/controllers/heatcfnapi_controller.go b/controllers/heatcfnapi_controller.go index f2dc863c..35ccaec2 100644 --- a/controllers/heatcfnapi_controller.go +++ b/controllers/heatcfnapi_controller.go @@ -826,10 +826,26 @@ func (r *HeatCfnAPIReconciler) reconcileNormal(ctx context.Context, instance *he condition.DeploymentReadyRunningMessage)) return ctrlResult, nil } - if depl.GetDeployment().Generation == depl.GetDeployment().Status.ObservedGeneration { - instance.Status.ReadyCount = depl.GetDeployment().Status.ReadyReplicas - if instance.Status.ReadyCount > 0 { + // Update the ReadyCount and evaluate the readiness only when the last + // ObservedGeneration is seen + deploy := depl.GetDeployment() + if deploy.Generation == deploy.Status.ObservedGeneration { + instance.Status.ReadyCount = deploy.Status.ReadyReplicas + + // Mark the Deployment as Ready only if the number of Replicas is equals + // to the Deployed instances (ReadyCount), and the the Status.Replicas + // match Status.ReadyReplicas. If a deployment update is in progress, + // Replicas > ReadyReplicas. + // In addition, make sure the controller sees the last Generation + // by comparing it with the ObservedGeneration. + if deployment.IsReady(deploy) { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + } else { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) } } // create Deployment - end diff --git a/controllers/heatengine_controller.go b/controllers/heatengine_controller.go index b0c09416..65fe1983 100644 --- a/controllers/heatengine_controller.go +++ b/controllers/heatengine_controller.go @@ -543,10 +543,26 @@ func (r *HeatEngineReconciler) reconcileNormal( return ctrlResult, nil } - if depl.GetDeployment().Generation == depl.GetDeployment().Status.ObservedGeneration { - instance.Status.ReadyCount = depl.GetDeployment().Status.ReadyReplicas - if instance.Status.ReadyCount > 0 { + // Update the ReadyCount and evaluate the readiness only when the last + // ObservedGeneration is seen + deploy := depl.GetDeployment() + if deploy.Generation == deploy.Status.ObservedGeneration { + instance.Status.ReadyCount = deploy.Status.ReadyReplicas + + // Mark the Deployment as Ready only if the number of Replicas is equals + // to the Deployed instances (ReadyCount), and the the Status.Replicas + // match Status.ReadyReplicas. If a deployment update is in progress, + // Replicas > ReadyReplicas. + // In addition, make sure the controller sees the last Generation + // by comparing it with the ObservedGeneration. + if deployment.IsReady(deploy) { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + } else { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) } } diff --git a/go.mod b/go.mod index c42a8d4f..352fd9c7 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/openstack-k8s-operators/heat-operator/api v0.3.1-0.20240214134649-6643d1b09d49 github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20250403063905-eb287d52f38d github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20250406092234-10f5f7e5b5a9 - github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250402133843-5a4c5f4fb4f1 + github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250408123225-0d9e9b82c41b github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20250402133843-5a4c5f4fb4f1 github.com/openstack-k8s-operators/lib-common/modules/test v0.6.1-0.20250402133843-5a4c5f4fb4f1 github.com/openstack-k8s-operators/mariadb-operator/api v0.6.0 diff --git a/go.sum b/go.sum index fe165e44..b6a654b7 100644 --- a/go.sum +++ b/go.sum @@ -82,8 +82,8 @@ github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20250403063905-e github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20250403063905-eb287d52f38d/go.mod h1:IY5zp1GRhacGMvjZMUZQ0LlxWDaogIRb/4H3by1Pivk= github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20250406092234-10f5f7e5b5a9 h1:Q1yMkuwIOCtGQJDsKYAbspJU5b6emRvoKw5VyPXfiak= github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20250406092234-10f5f7e5b5a9/go.mod h1:klMsoleakNm0dfNR9ePkL5pNZQsIyx4WMLaHDKIkTt4= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250402133843-5a4c5f4fb4f1 h1:hO90JhfinKysbdrWCJugTmJbkrs1d9tR7ypg3yOD12E= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250402133843-5a4c5f4fb4f1/go.mod h1:A9Ohw5Q90YOGhcDF4ZHkMr5RArz3phoBu9+ibbYDKG4= +github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250408123225-0d9e9b82c41b h1:T+N6xOT2NP+hVp2K1xl/NV3uheVHu38CcBuW+8uOBYw= +github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250408123225-0d9e9b82c41b/go.mod h1:A9Ohw5Q90YOGhcDF4ZHkMr5RArz3phoBu9+ibbYDKG4= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20250402133843-5a4c5f4fb4f1 h1:QlwUTGaUrl0Z6ozC7u0UNdgB3L6k/b60jsq2u8w482k= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20250402133843-5a4c5f4fb4f1/go.mod h1:fesgTbs2j30Fhw2hebXkPgbeAIqG0Yk2oaeOklAInZg= github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20250402133843-5a4c5f4fb4f1 h1:KcltUDbUA0sjtf6bV60L7GDpC0pmokhtdK3579yytS4=