[feature] PVC template (#162)

* basic pvc template support

* support PVC Retention policy

* add pvc template unit test

* add pvc utils unit test

* add file header lisences

* supprt pvc with replace update

* rephrase some typo

* pvc compatible with replace update + scale in

* support retain pvc when update pvc template

* fix:re-mount the old pvc when pvc template rollback

* sort inported packages, correct new files Copyright 2024

* pass caller's context

* remove redundant DeepCopy()

* validating webhook: denied delete pvc while pending

* validating webhook: denied delete pvc (must be mounted) while pending

* rename files

* fix: support persistent ownerReference for owned pvcs

* fix: use latest available version

* fix:reclaim pvcs in 1 reconcile

* type

* import files

* always adopt pvcs

* add some comment

Author: ColdsteelRail

Makefile

@@ -45,7 +45,7 @@ help: ## Display this help.
 
 .PHONY: manifests
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
-	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+	$(CONTROLLER_GEN) rbac:roleName=manager-role crd:generateEmbeddedObjectMeta=true webhook paths="./..." output:crd:artifacts:config=config/crd/bases
 
 .PHONY: generate
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.

apis/apps/v1alpha1/well_known_labels.go

@@ -46,6 +46,9 @@ const (
 	PodReplacePairOriginName = "collaset.kusionstack.io/replace-pair-origin-name" // used to indicate the original Pod name for replacement.
 
 	PodReplacePairNewId = "collaset.kusionstack.io/replace-pair-new-id" // used to indicate the new created Pod instance ID for replacement.
+
+	PvcTemplateHashLabelKey = "collaset.kusionstack.io/pvc-template-hash" // used to attach hash of pvc template to pvc resource
+
 )
 
 const (

config/crd/bases/apps.kusionstack.io_collasets.yaml

@@ -255,6 +255,23 @@ spec:
                       type: string
                     metadata:
                       description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata'
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        finalizers:
+                          items:
+                            type: string
+                          type: array
+                        labels:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        name:
+                          type: string
+                        namespace:
+                          type: string
                       type: object
                     spec:
                       description: 'Spec defines the desired characteristics of a

config/crd/bases/apps.kusionstack.io_poddecorations.yaml

@@ -4213,6 +4213,23 @@ spec:
                                     that will be copied into the PVC when creating
                                     it. No other fields are allowed and will be rejected
                                     during validation.
+                                  properties:
+                                    annotations:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                    finalizers:
+                                      items:
+                                        type: string
+                                      type: array
+                                    labels:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                    name:
+                                      type: string
+                                    namespace:
+                                      type: string
                                   type: object
                                 spec:
                                   description: The specification for the PersistentVolumeClaim.

config/rbac/role.yaml

@@ -131,6 +131,18 @@ rules:
   - create
   - patch
   - update
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - ""
   resources:

config/webhook/webhook.yaml

@@ -86,6 +86,31 @@ webhooks:
           operator: In
           values:
             - 'true'
+  - name: validating-pvc.apps.kusionstack.io
+    sideEffects: NoneOnDryRun
+    admissionReviewVersions: ["v1", "v1beta1"]
+    clientConfig:
+      service:
+        namespace: kusionstack-system
+        name: controller-manager
+        path: /validating-generic
+    failurePolicy: Fail
+    rules:
+      - apiGroups:
+          - "*"
+        apiVersions:
+          - v1
+        operations:
+          - DELETE
+        resources:
+          - persistentvolumeclaims
+        scope: '*'
+    objectSelector:
+      matchExpressions:
+        - key: kusionstack.io/control
+          operator: In
+          values:
+            - 'true'
   - name: validating-generic.apps.kusionstack.io
     sideEffects: NoneOnDryRun
     admissionReviewVersions: ["v1", "v1beta1"]

pkg/controllers/collaset/collaset_controller.go

@@ -34,6 +34,7 @@ import (
 	appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1"
 	"kusionstack.io/operating/pkg/controllers/collaset/podcontext"
 	"kusionstack.io/operating/pkg/controllers/collaset/podcontrol"
+	"kusionstack.io/operating/pkg/controllers/collaset/pvccontrol"
 	"kusionstack.io/operating/pkg/controllers/collaset/synccontrol"
 	"kusionstack.io/operating/pkg/controllers/collaset/utils"
 	collasetutils "kusionstack.io/operating/pkg/controllers/collaset/utils"
@@ -71,7 +72,7 @@ func NewReconciler(mgr ctrl.Manager) reconcile.Reconciler {
 	return &CollaSetReconciler{
 		ReconcilerMixin: mixin,
 		revisionManager: revision.NewRevisionManager(mixin.Client, mixin.Scheme, NewRevisionOwnerAdapter(podcontrol.NewRealPodControl(mixin.Client, mixin.Scheme))),
-		syncControl:     synccontrol.NewRealSyncControl(mixin.Client, mixin.Logger, podcontrol.NewRealPodControl(mixin.Client, mixin.Scheme), mixin.Recorder),
+		syncControl:     synccontrol.NewRealSyncControl(mixin.Client, mixin.Logger, podcontrol.NewRealPodControl(mixin.Client, mixin.Scheme), pvccontrol.NewRealPvcControl(mixin.Client, mixin.Scheme), mixin.Recorder),
 	}
 }
 
@@ -112,6 +113,7 @@ func AddToMgr(mgr ctrl.Manager, r reconcile.Reconciler) error {
 // +kubebuilder:rbac:groups=apps.kusionstack.io,resources=resourcecontexts,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=apps.kusionstack.io,resources=resourcecontexts/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=apps.kusionstack.io,resources=resourcecontexts/finalizers,verbs=update
+// +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=apps,resources=controllerrevisions,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=events,verbs=create;update;patch
@@ -140,6 +142,10 @@ func (r *CollaSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 	}
 
 	if instance.DeletionTimestamp != nil {
+		if err := r.ensureReclaimPvcs(ctx, instance); err != nil {
+			// reclaim pvcs before remove finalizers
+			return ctrl.Result{}, err
+		}
 		if controllerutil.ContainsFinalizer(instance, preReclaimFinalizer) {
 			// reclaim owner IDs in ResourceContext
 			if err := r.reclaimResourceContext(instance); err != nil {
@@ -325,3 +331,23 @@ func requeueResult(requeueTime *time.Duration) reconcile.Result {
 	}
 	return reconcile.Result{}
 }
+
+func (r *CollaSetReconciler) ensureReclaimPvcs(ctx context.Context, cls *appsv1alpha1.CollaSet) error {
+	var needReclaimPvcs []*corev1.PersistentVolumeClaim
+	pvcControl := pvccontrol.NewRealPvcControl(r.Client, r.Scheme)
+	pvcs, err := pvcControl.GetFilteredPvcs(ctx, cls)
+	if err != nil {
+		return err
+	}
+	// reclaim pvcs according to whenDelete retention policy
+	for i := range pvcs {
+		owned := pvcs[i].OwnerReferences != nil && len(pvcs[i].OwnerReferences) > 0
+		if owned && collasetutils.PvcPolicyWhenDelete(cls) == appsv1alpha1.RetainPersistentVolumeClaimRetentionPolicyType {
+			needReclaimPvcs = append(needReclaimPvcs, pvcs[i])
+		}
+	}
+	if len(needReclaimPvcs) > 0 {
+		_, err = pvcControl.ReleasePvcsOwnerRef(cls, needReclaimPvcs)
+	}
+	return err
+}