From 7e6387e98f9354fa073e01bcde7938e5cf116bea Mon Sep 17 00:00:00 2001 From: jwcesign Date: Wed, 25 Oct 2023 16:08:56 +0800 Subject: [PATCH] feat: add resoruce id label Signed-off-by: jwcesign --- go.mod | 2 +- .../policy/v1alpha1/well_known_constants.go | 14 ++ .../work/v1alpha2/well_known_constants.go | 17 +++ pkg/controllers/binding/common.go | 16 ++- pkg/controllers/binding/common_test.go | 19 +-- .../execution/execution_controller.go | 2 +- .../hpa_replicas_syncer_predicate.go | 4 +- .../dependencies_distributor.go | 36 +++-- pkg/detector/detector.go | 123 ++++++++++++++---- pkg/detector/preemption.go | 6 +- pkg/util/helper/work.go | 9 +- pkg/util/label.go | 11 +- pkg/util/label_test.go | 98 +++++++------- 13 files changed, 244 insertions(+), 113 deletions(-) diff --git a/go.mod b/go.mod index b101317ff051..277236cfa20a 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 + github.com/google/uuid v1.3.0 github.com/kr/pretty v0.3.1 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f github.com/olekukonko/tablewriter v0.0.5 @@ -100,7 +101,6 @@ require ( github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect diff --git a/pkg/apis/policy/v1alpha1/well_known_constants.go b/pkg/apis/policy/v1alpha1/well_known_constants.go index 144f5d868a19..c982e3d315e6 100644 --- a/pkg/apis/policy/v1alpha1/well_known_constants.go +++ b/pkg/apis/policy/v1alpha1/well_known_constants.go @@ -1,6 +1,20 @@ package v1alpha1 const ( + // PropagationPolicyPermanentIDLabel is the identifier of a PropagationPolicy object. + // Karmada generates a unique identifier, such as metadata.UUID, for each PropagationPolicy object. + // This identifier will be used as a label selector to locate corresponding resources, such as ResourceBinding. + // The reason for generating a new unique identifier instead of simply using metadata.UUID is because: + // In backup scenarios, when applying the backup resource manifest in a new cluster, the UUID may change. + PropagationPolicyPermanentIDLabel = "propagationpolicy.karmada.io/permanent-id" + + // ClusterPropagationPolicyPermanentIDLabel is the identifier of a ClusterPropagationPolicy object. + // Karmada generates a unique identifier, such as metadata.UUID, for each ClusterPropagationPolicy object. + // This identifier will be used as a label selector to locate corresponding resources, such as ResourceBinding. + // The reason for generating a new unique identifier instead of simply using metadata.UUID is because: + // In backup scenarios, when applying the backup resource manifest in a new cluster, the UUID may change. + ClusterPropagationPolicyPermanentIDLabel = "clusterpropagationpolicy.karmada.io/permanent-id" + // PropagationPolicyUIDLabel is the uid of PropagationPolicy object. PropagationPolicyUIDLabel = "propagationpolicy.karmada.io/uid" diff --git a/pkg/apis/work/v1alpha2/well_known_constants.go b/pkg/apis/work/v1alpha2/well_known_constants.go index 596208d6b5cd..8a7cec1b9056 100644 --- a/pkg/apis/work/v1alpha2/well_known_constants.go +++ b/pkg/apis/work/v1alpha2/well_known_constants.go @@ -1,6 +1,23 @@ package v1alpha2 const ( + // ResourceBindingPermanentIDLabel is the identifier of a ResourceBinding object. + // Karmada generates a unique identifier, such as metadata.UUID, for each ResourceBinding object. + // This identifier will be used as a label selector to locate corresponding resources, such as Work. + // The reason for generating a new unique identifier instead of simply using metadata.UUID is because: + // In backup scenarios, when applying the backup resource manifest in a new cluster, the UUID may change. + ResourceBindingPermanentIDLabel = "resourcebinding.karmada.io/permanent-id" + + // ClusterResourceBindingPermanentIDLabel is the identifier of a ClusterResourceBinding object. + // Karmada generates a unique identifier, such as metadata.UUID, for each ClusterResourceBinding object. + // This identifier will be used as a label selector to locate corresponding resources, such as Work. + // The reason for generating a new unique identifier instead of simply using metadata.UUID is because: + // In backup scenarios, when applying the backup resource manifest in a new cluster, the UUID may change. + ClusterResourceBindingPermanentIDLabel = "clusterresourcebinding.karmada.io/permanent-id" + + // WorkPermanentIDLabel is the ID of Work object. + WorkPermanentIDLabel = "work.karmada.io/permanent-id" + // ResourceBindingUIDLabel is the UID of ResourceBinding object. ResourceBindingUIDLabel = "resourcebinding.karmada.io/uid" diff --git a/pkg/controllers/binding/common.go b/pkg/controllers/binding/common.go index 95fbf861eb45..445c2e66eca5 100644 --- a/pkg/controllers/binding/common.go +++ b/pkg/controllers/binding/common.go @@ -143,15 +143,23 @@ func mergeLabel(workload *unstructured.Unstructured, workNamespace string, bindi util.MergeLabel(workload, workv1alpha1.WorkNameLabel, names.GenerateWorkName(workload.GetKind(), workload.GetName(), workload.GetNamespace())) util.MergeLabel(workload, util.ManagedByKarmadaLabel, util.ManagedByKarmadaLabelValue) if scope == apiextensionsv1.NamespaceScoped { + util.RemoveLabels(workload, workv1alpha2.ResourceBindingUIDLabel) + + bindingID := util.GetLabelValue(binding.GetLabels(), workv1alpha2.ResourceBindingPermanentIDLabel) util.MergeLabel(workload, workv1alpha2.ResourceBindingReferenceKey, names.GenerateBindingReferenceKey(binding.GetNamespace(), binding.GetName())) - util.MergeLabel(workload, workv1alpha2.ResourceBindingUIDLabel, string(binding.GetUID())) + util.MergeLabel(workload, workv1alpha2.ResourceBindingPermanentIDLabel, bindingID) + workLabel[workv1alpha2.ResourceBindingReferenceKey] = names.GenerateBindingReferenceKey(binding.GetNamespace(), binding.GetName()) - workLabel[workv1alpha2.ResourceBindingUIDLabel] = string(binding.GetUID()) + workLabel[workv1alpha2.ResourceBindingPermanentIDLabel] = bindingID } else { + util.RemoveLabels(workload, workv1alpha2.ClusterResourceBindingUIDLabel) + + bindingID := util.GetLabelValue(binding.GetLabels(), workv1alpha2.ClusterResourceBindingPermanentIDLabel) util.MergeLabel(workload, workv1alpha2.ClusterResourceBindingReferenceKey, names.GenerateBindingReferenceKey("", binding.GetName())) - util.MergeLabel(workload, workv1alpha2.ClusterResourceBindingUIDLabel, string(binding.GetUID())) + util.MergeLabel(workload, workv1alpha2.ClusterResourceBindingPermanentIDLabel, bindingID) + workLabel[workv1alpha2.ClusterResourceBindingReferenceKey] = names.GenerateBindingReferenceKey("", binding.GetName()) - workLabel[workv1alpha2.ClusterResourceBindingUIDLabel] = string(binding.GetUID()) + workLabel[workv1alpha2.ClusterResourceBindingPermanentIDLabel] = bindingID } return workLabel } diff --git a/pkg/controllers/binding/common_test.go b/pkg/controllers/binding/common_test.go index 79b3cb6d8f81..b20f0f4cdf63 100644 --- a/pkg/controllers/binding/common_test.go +++ b/pkg/controllers/binding/common_test.go @@ -7,7 +7,6 @@ import ( v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/types" policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" @@ -88,7 +87,7 @@ func Test_mergeTargetClusters(t *testing.T) { func Test_mergeLabel(t *testing.T) { namespace := "fake-ns" bindingName := "fake-bindingName" - rbUID := "93162d3c-ee8e-4995-9034-05f4d5d2c2b9" + rbID := "93162d3c-ee8e-4995-9034-05f4d5d2c2b9" tests := []struct { name string @@ -115,13 +114,15 @@ func Test_mergeLabel(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: bindingName, Namespace: namespace, - UID: types.UID(rbUID), + Labels: map[string]string{ + workv1alpha2.ResourceBindingPermanentIDLabel: rbID, + }, }, }, scope: v1.NamespaceScoped, want: map[string]string{ - workv1alpha2.ResourceBindingUIDLabel: rbUID, - workv1alpha2.ResourceBindingReferenceKey: names.GenerateBindingReferenceKey(namespace, bindingName), + workv1alpha2.ResourceBindingPermanentIDLabel: rbID, + workv1alpha2.ResourceBindingReferenceKey: names.GenerateBindingReferenceKey(namespace, bindingName), }, }, { @@ -138,13 +139,15 @@ func Test_mergeLabel(t *testing.T) { binding: &workv1alpha2.ClusterResourceBinding{ ObjectMeta: metav1.ObjectMeta{ Name: bindingName, - UID: types.UID(rbUID), + Labels: map[string]string{ + workv1alpha2.ClusterResourceBindingPermanentIDLabel: rbID, + }, }, }, scope: v1.ClusterScoped, want: map[string]string{ - workv1alpha2.ClusterResourceBindingUIDLabel: rbUID, - workv1alpha2.ClusterResourceBindingReferenceKey: names.GenerateBindingReferenceKey("", bindingName), + workv1alpha2.ClusterResourceBindingPermanentIDLabel: rbID, + workv1alpha2.ClusterResourceBindingReferenceKey: names.GenerateBindingReferenceKey("", bindingName), }, }, } diff --git a/pkg/controllers/execution/execution_controller.go b/pkg/controllers/execution/execution_controller.go index da95b326150a..2d44feb54aec 100644 --- a/pkg/controllers/execution/execution_controller.go +++ b/pkg/controllers/execution/execution_controller.go @@ -190,12 +190,12 @@ func (c *Controller) syncToClusters(clusterName string, work *workv1alpha1.Work) for _, manifest := range work.Spec.Workload.Manifests { workload := &unstructured.Unstructured{} err := workload.UnmarshalJSON(manifest.Raw) - util.MergeLabel(workload, workv1alpha2.WorkUIDLabel, string(work.UID)) if err != nil { klog.Errorf("Failed to unmarshal workload, error is: %v", err) errs = append(errs, err) continue } + util.MergeLabel(workload, workv1alpha2.WorkPermanentIDLabel, util.GetLabelValue(work.Labels, workv1alpha2.WorkPermanentIDLabel)) if err = c.tryCreateOrUpdateWorkload(clusterName, workload); err != nil { klog.Errorf("Failed to create or update resource(%v/%v) in the given member cluster %s, err is %v", workload.GetNamespace(), workload.GetName(), clusterName, err) diff --git a/pkg/controllers/hpareplicassyncer/hpa_replicas_syncer_predicate.go b/pkg/controllers/hpareplicassyncer/hpa_replicas_syncer_predicate.go index ce86ac876eea..e90871967c4e 100644 --- a/pkg/controllers/hpareplicassyncer/hpa_replicas_syncer_predicate.go +++ b/pkg/controllers/hpareplicassyncer/hpa_replicas_syncer_predicate.go @@ -71,7 +71,7 @@ func (r *HPAReplicasSyncer) Generic(e event.GenericEvent) bool { } func hasBeenPropagated(hpa *autoscalingv2.HorizontalPodAutoscaler) bool { - _, ppExist := hpa.GetLabels()[policyv1alpha1.PropagationPolicyUIDLabel] - _, cppExist := hpa.GetLabels()[policyv1alpha1.ClusterPropagationPolicyUIDLabel] + _, ppExist := hpa.GetLabels()[policyv1alpha1.PropagationPolicyNameLabel] + _, cppExist := hpa.GetLabels()[policyv1alpha1.ClusterPropagationPolicyLabel] return ppExist || cppExist } diff --git a/pkg/dependenciesdistributor/dependencies_distributor.go b/pkg/dependenciesdistributor/dependencies_distributor.go index 7683a15fe8f0..0840844e1b30 100644 --- a/pkg/dependenciesdistributor/dependencies_distributor.go +++ b/pkg/dependenciesdistributor/dependencies_distributor.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" + "github.com/google/uuid" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -45,6 +46,9 @@ import ( ) const ( + // bindingDependedIdLabelKey is the resoruce id of the independent binding which the attached binding depends on. + bindingDependedIdLabelKey = "resourcebinding.karmada.io/depended-id" + // bindingDependedByLabelKeyPrefix is the prefix to a label key specifying an attached binding referred by which independent binding. // the key is in the label of an attached binding which should be unique, because resource like secret can be referred by multiple deployments. bindingDependedByLabelKeyPrefix = "resourcebinding.karmada.io/depended-by-" @@ -498,19 +502,14 @@ func (d *DependenciesDistributor) removeScheduleResultFromAttachedBindings(bindi } func (d *DependenciesDistributor) createOrUpdateAttachedBinding(attachedBinding *workv1alpha2.ResourceBinding) error { - if err := d.Client.Create(context.TODO(), attachedBinding); err != nil { - if !apierrors.IsAlreadyExists(err) { - klog.Infof("Failed to create resource binding(%s/%s): %v", attachedBinding.Namespace, attachedBinding.Name, err) - return err + existBinding := &workv1alpha2.ResourceBinding{} + key := client.ObjectKeyFromObject(attachedBinding) + err := d.Client.Get(context.TODO(), key, existBinding) + if err == nil { + if util.GetLabelValue(existBinding.Labels, workv1alpha2.ResourceBindingPermanentIDLabel) == "" { + existBinding.Labels = util.DedupeAndMergeLabels(existBinding.Labels, + map[string]string{workv1alpha2.ResourceBindingPermanentIDLabel: uuid.New().String()}) } - - existBinding := &workv1alpha2.ResourceBinding{} - key := client.ObjectKeyFromObject(attachedBinding) - if err := d.Client.Get(context.TODO(), key, existBinding); err != nil { - klog.Infof("Failed to get resource binding(%s/%s): %v", attachedBinding.Namespace, attachedBinding.Name, err) - return err - } - existBinding.Spec.RequiredBy = mergeBindingSnapshot(existBinding.Spec.RequiredBy, attachedBinding.Spec.RequiredBy) existBinding.Labels = util.DedupeAndMergeLabels(existBinding.Labels, attachedBinding.Labels) existBinding.Spec.Resource = attachedBinding.Spec.Resource @@ -520,6 +519,17 @@ func (d *DependenciesDistributor) createOrUpdateAttachedBinding(attachedBinding return err } } + + if !apierrors.IsNotFound(err) { + klog.Infof("Failed to get resource binding(%s/%s): %v", attachedBinding.Namespace, attachedBinding.Name, err) + return err + } + + attachedBinding.Labels = util.DedupeAndMergeLabels(attachedBinding.Labels, + map[string]string{workv1alpha2.ResourceBindingPermanentIDLabel: uuid.New().String()}) + if err := d.Client.Create(context.TODO(), attachedBinding); err != nil { + return err + } return nil } @@ -630,6 +640,8 @@ func buildAttachedBinding(binding *workv1alpha2.ResourceBinding, object *unstruc Clusters: binding.Spec.Clusters, }) + policyID := util.GetLabelValue(binding.Labels, workv1alpha2.ResourceBindingPermanentIDLabel) + dependedLabels = util.DedupeAndMergeLabels(dependedLabels, map[string]string{bindingDependedIdLabelKey: policyID}) return &workv1alpha2.ResourceBinding{ ObjectMeta: metav1.ObjectMeta{ Name: names.GenerateBindingName(object.GetKind(), object.GetName()), diff --git a/pkg/detector/detector.go b/pkg/detector/detector.go index 62d40fd4f08e..4657cfafb1d7 100644 --- a/pkg/detector/detector.go +++ b/pkg/detector/detector.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "github.com/google/uuid" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -388,15 +389,16 @@ func (d *ResourceDetector) ApplyPolicy(object *unstructured.Unstructured, object } }() - if err := d.ClaimPolicyForObject(object, policy.Namespace, policy.Name, string(policy.UID)); err != nil { + policyID, err := d.ClaimPolicyForObject(object, policy) + if err != nil { klog.Errorf("Failed to claim policy(%s) for object: %s", policy.Name, object) return err } policyLabels := map[string]string{ - policyv1alpha1.PropagationPolicyNamespaceLabel: policy.GetNamespace(), - policyv1alpha1.PropagationPolicyNameLabel: policy.GetName(), - policyv1alpha1.PropagationPolicyUIDLabel: string(policy.UID), + policyv1alpha1.PropagationPolicyNamespaceLabel: policy.GetNamespace(), + policyv1alpha1.PropagationPolicyNameLabel: policy.GetName(), + policyv1alpha1.PropagationPolicyPermanentIDLabel: policyID, } policyAnnotations := map[string]string{ policyv1alpha1.PropagationPolicyNamespaceAnnotation: policy.GetNamespace(), @@ -417,6 +419,14 @@ func (d *ResourceDetector) ApplyPolicy(object *unstructured.Unstructured, object return fmt.Errorf("failed to update binding due to different owner reference UID, will " + "try again later after binding is garbage collected, see https://github.com/karmada-io/karmada/issues/2090") } + + // TODO: Delete following two lines in release-1.9 + delete(bindingCopy.Labels, workv1alpha2.ResourceBindingUIDLabel) + delete(bindingCopy.Labels, policyv1alpha1.PropagationPolicyUIDLabel) + if util.GetLabelValue(bindingCopy.Labels, workv1alpha2.ResourceBindingPermanentIDLabel) == "" { + bindingCopy.Labels = util.DedupeAndMergeLabels(bindingCopy.Labels, + map[string]string{workv1alpha2.ResourceBindingPermanentIDLabel: uuid.New().String()}) + } // Just update necessary fields, especially avoid modifying Spec.Clusters which is scheduling result, if already exists. bindingCopy.Annotations = util.DedupeAndMergeAnnotations(bindingCopy.Annotations, binding.Annotations) bindingCopy.Labels = util.DedupeAndMergeLabels(bindingCopy.Labels, binding.Labels) @@ -469,14 +479,15 @@ func (d *ResourceDetector) ApplyClusterPolicy(object *unstructured.Unstructured, } }() - if err := d.ClaimClusterPolicyForObject(object, policy.Name, string(policy.UID)); err != nil { + policyID, err := d.ClaimClusterPolicyForObject(object, policy) + if err != nil { klog.Errorf("Failed to claim cluster policy(%s) for object: %s", policy.Name, object) return err } policyLabels := map[string]string{ - policyv1alpha1.ClusterPropagationPolicyLabel: policy.GetName(), - policyv1alpha1.ClusterPropagationPolicyUIDLabel: string(policy.UID), + policyv1alpha1.ClusterPropagationPolicyLabel: policy.GetName(), + policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel: policyID, } policyAnnotations := map[string]string{ policyv1alpha1.ClusterPropagationPolicyAnnotation: policy.GetName(), @@ -500,6 +511,14 @@ func (d *ResourceDetector) ApplyClusterPolicy(object *unstructured.Unstructured, return fmt.Errorf("failed to update binding due to different owner reference UID, will " + "try again later after binding is garbage collected, see https://github.com/karmada-io/karmada/issues/2090") } + + // TODO: Delete following two lines in release-1.9 + delete(bindingCopy.Labels, workv1alpha2.ResourceBindingUIDLabel) + delete(bindingCopy.Labels, policyv1alpha1.ClusterPropagationPolicyUIDLabel) + if util.GetLabelValue(bindingCopy.Labels, workv1alpha2.ResourceBindingPermanentIDLabel) == "" { + bindingCopy.Labels = util.DedupeAndMergeLabels(bindingCopy.Labels, + map[string]string{workv1alpha2.ResourceBindingPermanentIDLabel: uuid.New().String()}) + } // Just update necessary fields, especially avoid modifying Spec.Clusters which is scheduling result, if already exists. bindingCopy.Annotations = util.DedupeAndMergeAnnotations(bindingCopy.Annotations, binding.Annotations) bindingCopy.Labels = util.DedupeAndMergeLabels(bindingCopy.Labels, binding.Labels) @@ -547,6 +566,14 @@ func (d *ResourceDetector) ApplyClusterPolicy(object *unstructured.Unstructured, return fmt.Errorf("failed to update binding due to different owner reference UID, will " + "try again later after binding is garbage collected, see https://github.com/karmada-io/karmada/issues/2090") } + + // TODO: delete following two lines in release-1.9 + delete(bindingCopy.Labels, workv1alpha2.ClusterResourceBindingUIDLabel) + delete(bindingCopy.Labels, policyv1alpha1.ClusterPropagationPolicyUIDLabel) + if util.GetLabelValue(bindingCopy.Labels, workv1alpha2.ClusterResourceBindingPermanentIDLabel) == "" { + bindingCopy.Labels = util.DedupeAndMergeLabels(bindingCopy.Labels, + map[string]string{workv1alpha2.ClusterResourceBindingPermanentIDLabel: uuid.New().String()}) + } // Just update necessary fields, especially avoid modifying Spec.Clusters which is scheduling result, if already exists. bindingCopy.Annotations = util.DedupeAndMergeAnnotations(bindingCopy.Annotations, binding.Annotations) bindingCopy.Labels = util.DedupeAndMergeLabels(bindingCopy.Labels, binding.Labels) @@ -614,51 +641,95 @@ func (d *ResourceDetector) GetUnstructuredObject(objectKey keys.ClusterWideKey) return unstructuredObj, nil } +func (d *ResourceDetector) getPropagationPolicyID(policy *policyv1alpha1.PropagationPolicy) (string, error) { + id := util.GetLabelValue(policy.GetLabels(), policyv1alpha1.PropagationPolicyPermanentIDLabel) + if id == "" { + id = uuid.New().String() + policy.Labels = util.DedupeAndMergeLabels(policy.Labels, map[string]string{policyv1alpha1.PropagationPolicyPermanentIDLabel: id}) + if err := d.Client.Update(context.TODO(), policy); err != nil { + return id, err + } + } + + return id, nil +} + // ClaimPolicyForObject set policy identifier which the object associated with. -func (d *ResourceDetector) ClaimPolicyForObject(object *unstructured.Unstructured, policyNamespace, policyName, policyUID string) error { +func (d *ResourceDetector) ClaimPolicyForObject(object *unstructured.Unstructured, policy *policyv1alpha1.PropagationPolicy) (string, error) { + policyID, err := d.getPropagationPolicyID(policy) + if err != nil { + klog.Errorf("Get PropagationPolicy(%s/%s) ID error:%v", policy.Namespace, policy.Name, err) + return "", err + } + objLabels := object.GetLabels() if objLabels == nil { objLabels = make(map[string]string) } else if len(objLabels) > 0 { // object has been claimed, don't need to claim again if !excludeClusterPolicy(objLabels) && - objLabels[policyv1alpha1.PropagationPolicyNamespaceLabel] == policyNamespace && - objLabels[policyv1alpha1.PropagationPolicyNameLabel] == policyName { - return nil + objLabels[policyv1alpha1.PropagationPolicyPermanentIDLabel] == policyID { + return policyID, nil } } - objLabels[policyv1alpha1.PropagationPolicyNamespaceLabel] = policyNamespace - objLabels[policyv1alpha1.PropagationPolicyNameLabel] = policyName - objLabels[policyv1alpha1.PropagationPolicyUIDLabel] = policyUID + // Delete following line when release-1.9 + delete(objLabels, policyv1alpha1.PropagationPolicyUIDLabel) + objLabels[policyv1alpha1.PropagationPolicyNamespaceLabel] = policy.Namespace + objLabels[policyv1alpha1.PropagationPolicyNameLabel] = policy.Name + objLabels[policyv1alpha1.PropagationPolicyPermanentIDLabel] = policyID objectAnnotations := object.GetAnnotations() if objectAnnotations == nil { objectAnnotations = make(map[string]string) } - objectAnnotations[policyv1alpha1.PropagationPolicyNamespaceAnnotation] = policyNamespace - objectAnnotations[policyv1alpha1.PropagationPolicyNameAnnotation] = policyName + objectAnnotations[policyv1alpha1.PropagationPolicyNamespaceAnnotation] = policy.Namespace + objectAnnotations[policyv1alpha1.PropagationPolicyNameAnnotation] = policy.Name objectCopy := object.DeepCopy() objectCopy.SetLabels(objLabels) objectCopy.SetAnnotations(objectAnnotations) - return d.Client.Update(context.TODO(), objectCopy) + return policyID, d.Client.Update(context.TODO(), objectCopy) } -// ClaimClusterPolicyForObject set cluster identifier which the object associated with. -func (d *ResourceDetector) ClaimClusterPolicyForObject(object *unstructured.Unstructured, policyName, policyUID string) error { - claimedName := util.GetLabelValue(object.GetLabels(), policyv1alpha1.ClusterPropagationPolicyLabel) +func (d *ResourceDetector) getClusterPropagationPolicyID(policy *policyv1alpha1.ClusterPropagationPolicy) (string, error) { + id := util.GetLabelValue(policy.GetLabels(), policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel) + if id == "" { + id = uuid.New().String() + policy.Labels = util.DedupeAndMergeLabels(policy.Labels, map[string]string{policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel: id}) + if err := d.Client.Update(context.TODO(), policy); err != nil { + return "", err + } + } + + return id, nil +} + +// ClaimClusterPolicyForObject set cluster identifier which the object associated with +func (d *ResourceDetector) ClaimClusterPolicyForObject(object *unstructured.Unstructured, policy *policyv1alpha1.ClusterPropagationPolicy) (string, error) { + policyID, err := d.getClusterPropagationPolicyID(policy) + if err != nil { + klog.Errorf("Get ClusterPropagationPolicy(%s) ID error:%v", policy.Name, err) + return "", err + } + + claimedID := util.GetLabelValue(object.GetLabels(), policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel) // object has been claimed, don't need to claim again - if claimedName == policyName { - return nil + if claimedID == policyID { + return policyID, nil } objectCopy := object.DeepCopy() - util.MergeLabel(objectCopy, policyv1alpha1.ClusterPropagationPolicyLabel, policyName) - util.MergeLabel(objectCopy, policyv1alpha1.ClusterPropagationPolicyUIDLabel, policyUID) + // TODO: delete following 3 lines in release-1.9 + labels := objectCopy.GetLabels() + delete(labels, policyv1alpha1.ClusterPropagationPolicyUIDLabel) + objectCopy.SetLabels(labels) + + util.MergeLabel(objectCopy, policyv1alpha1.ClusterPropagationPolicyLabel, policy.Name) + util.MergeLabel(objectCopy, policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel, policyID) - util.MergeAnnotation(objectCopy, policyv1alpha1.ClusterPropagationPolicyAnnotation, policyName) - return d.Client.Update(context.TODO(), objectCopy) + util.MergeAnnotation(objectCopy, policyv1alpha1.ClusterPropagationPolicyAnnotation, policy.Name) + return policyID, d.Client.Update(context.TODO(), objectCopy) } // BuildResourceBinding builds a desired ResourceBinding for object. diff --git a/pkg/detector/preemption.go b/pkg/detector/preemption.go index 0991db8110e6..d49634f52b8f 100755 --- a/pkg/detector/preemption.go +++ b/pkg/detector/preemption.go @@ -119,7 +119,7 @@ func (d *ResourceDetector) preemptPropagationPolicy(resourceTemplate *unstructur "Propagation policy(%s/%s) preempted propagation policy(%s/%s) successfully", policy.Namespace, policy.Name, claimedPolicyNamespace, claimedPolicyName) }() - if err = d.ClaimPolicyForObject(resourceTemplate, policy.Namespace, policy.Name, string(policy.UID)); err != nil { + if _, err = d.ClaimPolicyForObject(resourceTemplate, policy); err != nil { klog.Errorf("Failed to claim new propagation policy(%s/%s) on resource template(%s, kind=%s, %s): %v.", policy.Namespace, policy.Name, resourceTemplate.GetAPIVersion(), resourceTemplate.GetKind(), names.NamespacedKey(resourceTemplate.GetNamespace(), resourceTemplate.GetName()), err) return err @@ -147,7 +147,7 @@ func (d *ResourceDetector) preemptClusterPropagationPolicyDirectly(resourceTempl "Propagation policy(%s/%s) preempted cluster propagation policy(%s) successfully", policy.Namespace, policy.Name, claimedPolicyName) }() - if err = d.ClaimPolicyForObject(resourceTemplate, policy.Namespace, policy.Name, string(policy.UID)); err != nil { + if _, err = d.ClaimPolicyForObject(resourceTemplate, policy); err != nil { klog.Errorf("Failed to claim new propagation policy(%s/%s) on resource template(%s, kind=%s, %s) directly: %v.", policy.Namespace, policy.Name, resourceTemplate.GetAPIVersion(), resourceTemplate.GetKind(), names.NamespacedKey(resourceTemplate.GetNamespace(), resourceTemplate.GetName()), err) return err @@ -197,7 +197,7 @@ func (d *ResourceDetector) preemptClusterPropagationPolicy(resourceTemplate *uns "Cluster propagation policy(%s) preempted cluster propagation policy(%s) successfully", policy.Name, claimedPolicyName) }() - if err = d.ClaimClusterPolicyForObject(resourceTemplate, policy.Name, string(policy.UID)); err != nil { + if _, err = d.ClaimClusterPolicyForObject(resourceTemplate, policy); err != nil { klog.Errorf("Failed to claim new cluster propagation policy(%s) on resource template(%s, kind=%s, %s): %v.", policy.Name, resourceTemplate.GetAPIVersion(), resourceTemplate.GetKind(), names.NamespacedKey(resourceTemplate.GetNamespace(), resourceTemplate.GetName()), err) return err diff --git a/pkg/util/helper/work.go b/pkg/util/helper/work.go index 4ec095934784..96a32d16e4e8 100644 --- a/pkg/util/helper/work.go +++ b/pkg/util/helper/work.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/google/uuid" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -29,7 +30,6 @@ func CreateOrUpdateWork(client client.Client, workMeta metav1.ObjectMeta, resour } util.MergeAnnotation(workload, workv1alpha2.ResourceTemplateUIDAnnotation, string(workload.GetUID())) util.RecordManagedAnnotations(workload) - util.RecordManagedLabels(workload) workloadJSON, err := workload.MarshalJSON() if err != nil { klog.Errorf("Failed to marshal workload(%s/%s), Error: %v", workload.GetNamespace(), workload.GetName(), err) @@ -58,9 +58,16 @@ func CreateOrUpdateWork(client client.Client, workMeta metav1.ObjectMeta, resour if !runtimeObject.DeletionTimestamp.IsZero() { return fmt.Errorf("work %s/%s is being deleted", runtimeObject.GetNamespace(), runtimeObject.GetName()) } + + // TODO: Delete following one line in release-1.9 + delete(runtimeObject.Labels, workv1alpha2.WorkUIDLabel) runtimeObject.Spec = work.Spec runtimeObject.Labels = work.Labels runtimeObject.Annotations = work.Annotations + if util.GetLabelValue(runtimeObject.Labels, workv1alpha2.WorkPermanentIDLabel) == "" { + runtimeObject.Labels = util.DedupeAndMergeLabels(runtimeObject.Labels, map[string]string{workv1alpha2.WorkPermanentIDLabel: uuid.New().String()}) + } + util.RecordManagedLabels(runtimeObject) return nil }) if err != nil { diff --git a/pkg/util/label.go b/pkg/util/label.go index 8e9c025ae644..07ccccb94f2f 100644 --- a/pkg/util/label.go +++ b/pkg/util/label.go @@ -7,6 +7,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/sets" + workv1alpha1 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha1" workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" ) @@ -42,6 +43,8 @@ func RetainLabels(desired *unstructured.Unstructured, observed *unstructured.Uns } labels[key] = value } + // TODO: Delete following one line in release-1.9 + delete(labels, workv1alpha2.WorkUIDLabel) if len(labels) > 0 { desired.SetLabels(labels) } @@ -92,18 +95,18 @@ func getDeletedLabelKeys(desired, observed *unstructured.Unstructured) sets.Set[ // RecordManagedLabels sets or updates the annotation(resourcetemplate.karmada.io/managed-labels) // to record the label keys. -func RecordManagedLabels(object *unstructured.Unstructured) { - annotations := object.GetAnnotations() +func RecordManagedLabels(w *workv1alpha1.Work) { + annotations := w.GetAnnotations() if annotations == nil { annotations = make(map[string]string, 1) } var managedKeys []string // record labels. - labels := object.GetLabels() + labels := w.GetLabels() for key := range labels { managedKeys = append(managedKeys, key) } sort.Strings(managedKeys) annotations[workv1alpha2.ManagedLabels] = strings.Join(managedKeys, ",") - object.SetAnnotations(annotations) + w.SetAnnotations(annotations) } diff --git a/pkg/util/label_test.go b/pkg/util/label_test.go index e3ad3351763d..8be0f7b04c69 100644 --- a/pkg/util/label_test.go +++ b/pkg/util/label_test.go @@ -4,8 +4,10 @@ import ( "reflect" "testing" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + workv1alpha1 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha1" workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" ) @@ -510,71 +512,65 @@ func TestRetainLabels(t *testing.T) { func TestRecordManagedLabels(t *testing.T) { tests := []struct { name string - object *unstructured.Unstructured - expected *unstructured.Unstructured + object *workv1alpha1.Work + expected *workv1alpha1.Work }{ { name: "nil label", - object: &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": map[string]interface{}{ - "name": "demo-deployment-1", - }, - "spec": map[string]interface{}{ - "replicas": 2, - }, + object: &workv1alpha1.Work{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "work.karmada.io/v1alpha1", + Kind: "Work", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "demo-work-1", + Namespace: "cluster1-ns", + Labels: nil, }, }, - expected: &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": map[string]interface{}{ - "name": "demo-deployment-1", - "annotations": map[string]interface{}{ - workv1alpha2.ManagedLabels: "", - }, - }, - "spec": map[string]interface{}{ - "replicas": 2, + expected: &workv1alpha1.Work{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "work.karmada.io/v1alpha1", + Kind: "Work", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "demo-work-1", + Namespace: "cluster1-ns", + Annotations: map[string]string{ + workv1alpha2.ManagedLabels: "", }, + Labels: nil, }, }, }, { name: "object has has labels", - object: &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": map[string]interface{}{ - "name": "demo-deployment-1", - "labels": map[string]interface{}{ - "foo": "foo", - }, - }, - "spec": map[string]interface{}{ - "replicas": 2, + object: &workv1alpha1.Work{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "work.karmada.io/v1alpha1", + Kind: "Work", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "demo-work-1", + Namespace: "cluster1-ns", + Labels: map[string]string{ + "foo": "bar", }, }, }, - expected: &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": map[string]interface{}{ - "name": "demo-deployment-1", - "annotations": map[string]interface{}{ - workv1alpha2.ManagedLabels: "foo", - }, - "labels": map[string]interface{}{ - "foo": "foo", - }, - }, - "spec": map[string]interface{}{ - "replicas": 2, + expected: &workv1alpha1.Work{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "work.karmada.io/v1alpha1", + Kind: "Work", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "demo-work-1", + Namespace: "cluster1-ns", + Annotations: map[string]string{ + workv1alpha2.ManagedLabels: "foo", + }, + Labels: map[string]string{ + "foo": "bar", }, }, },