K8SPXC-1476: Disable PVC resize by default (#1846)

* K8SPXC-1476: Disable PVC resize by default

* bump k8s version

* fix versions in manifests

---------

Co-authored-by: Viacheslav Sarzhan <[email protected]>

Author: egegunes

Jenkinsfile

@@ -14,7 +14,7 @@ void createCluster(String CLUSTER_SUFFIX) {
                 gcloud config set project $GCP_PROJECT
                 gcloud container clusters list --filter $CLUSTER_NAME-${CLUSTER_SUFFIX} --zone $region --format='csv[no-heading](name)' | xargs gcloud container clusters delete --zone $region --quiet || true
 
-                gcloud container clusters create --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX} --cluster-version=1.27 --machine-type=n1-standard-4 --preemptible --disk-size 30 --num-nodes=\$NODES_NUM --network=jenkins-vpc --subnetwork=jenkins-${CLUSTER_SUFFIX} --no-enable-autoupgrade --cluster-ipv4-cidr=/21 --labels delete-cluster-after-hours=6 && \
+                gcloud container clusters create --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX} --cluster-version=1.28 --machine-type=n1-standard-4 --preemptible --disk-size 30 --num-nodes=\$NODES_NUM --network=jenkins-vpc --subnetwork=jenkins-${CLUSTER_SUFFIX} --no-enable-autoupgrade --cluster-ipv4-cidr=/21 --labels delete-cluster-after-hours=6 && \
                 kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user jenkins@"$GCP_PROJECT".iam.gserviceaccount.com || ret_val=\$?
                 if [ \${ret_val} -eq 0 ]; then break; fi
                 ret_num=\$((ret_num + 1))

config/crd/bases/pxc.percona.com_perconaxtradbclusters.yaml

@@ -1074,6 +1074,8 @@ spec:
                 type: string
               enableCRValidationWebhook:
                 type: boolean
+              enableVolumeExpansion:
+                type: boolean
               haproxy:
                 properties:
                   affinity:

deploy/bundle.yaml

@@ -1974,6 +1974,8 @@ spec:
                 type: string
               enableCRValidationWebhook:
                 type: boolean
+              enableVolumeExpansion:
+                type: boolean
               haproxy:
                 properties:
                   affinity:
@@ -11308,7 +11310,7 @@ spec:
           value: percona-xtradb-cluster-operator
         - name: DISABLE_TELEMETRY
           value: "false"
-        image: percona/percona-xtradb-cluster-operator:1.15.0
+        image: perconalab/percona-xtradb-cluster-operator:1.15.0
         imagePullPolicy: Always
         livenessProbe:
           failureThreshold: 3

deploy/cr-minimal.yaml

@@ -3,7 +3,7 @@ kind: PerconaXtraDBCluster
 metadata:
   name: minimal-cluster
 spec:
-  crVersion: 1.15.0
+  crVersion: 1.15.1
   secretsName: minimal-cluster-secrets
   unsafeFlags:
     tls: true
@@ -28,4 +28,4 @@ spec:
     image: percona/haproxy:2.8.5
   logcollector:
     enabled: true
-    image: percona/percona-xtradb-cluster-operator:1.15.0-logcollector-fluentbit3.1.4
+    image: percona/percona-xtradb-cluster-operator:1.15.1-logcollector-fluentbit3.1.4

deploy/cr.yaml

@@ -10,7 +10,8 @@ metadata:
 #  annotations:
 #    percona.com/issue-vault-token: "true"
 spec:
-  crVersion: 1.15.0
+  crVersion: 1.15.1
+  #  enableVolumeExpansion: false
 #  ignoreAnnotations:
 #    - iam.amazonaws.com/role
 #  ignoreLabels:

deploy/crd.yaml

@@ -1974,6 +1974,8 @@ spec:
                 type: string
               enableCRValidationWebhook:
                 type: boolean
+              enableVolumeExpansion:
+                type: boolean
               haproxy:
                 properties:
                   affinity:

deploy/cw-bundle.yaml

@@ -1974,6 +1974,8 @@ spec:
                 type: string
               enableCRValidationWebhook:
                 type: boolean
+              enableVolumeExpansion:
+                type: boolean
               haproxy:
                 properties:
                   affinity:
@@ -11318,7 +11320,7 @@ spec:
           value: percona-xtradb-cluster-operator
         - name: DISABLE_TELEMETRY
           value: "false"
-        image: percona/percona-xtradb-cluster-operator:1.15.0
+        image: perconalab/percona-xtradb-cluster-operator:1.15.0
         imagePullPolicy: Always
         resources:
           limits:

deploy/cw-operator.yaml

@@ -42,7 +42,7 @@ spec:
           value: percona-xtradb-cluster-operator
         - name: DISABLE_TELEMETRY
           value: "false"
-        image: percona/percona-xtradb-cluster-operator:1.15.0
+        image: perconalab/percona-xtradb-cluster-operator:1.15.0
         imagePullPolicy: Always
         resources:
           limits:

deploy/operator.yaml

@@ -45,7 +45,7 @@ spec:
           value: percona-xtradb-cluster-operator
         - name: DISABLE_TELEMETRY
           value: "false"
-        image: percona/percona-xtradb-cluster-operator:1.15.0
+        image: perconalab/percona-xtradb-cluster-operator:1.15.0
         imagePullPolicy: Always
         livenessProbe:
           failureThreshold: 3

e2e-tests/pvc-resize/run

@@ -64,6 +64,43 @@ function wait_cluster_status() {
 	echo "pxc/${cluster} status is ${expected}"
 }
 
+function wait_all_pvc_resize() {
+	local expected_size=$1
+	local max_retry=${2:-120}
+	local sleep_time=${3:-5}
+
+	for pvc in $(kubectl_bin get pvc -l app.kubernetes.io/component=pxc -o name); do
+		if ! wait_pvc_resize "$pvc" "$expected_size" "$max_retry" "$sleep_time"; then
+			return 1
+		fi
+	done
+	return 0
+}
+
+function wait_pvc_resize() {
+	local pvc=$1
+	local expected_size=$2
+	local max_retry=${3:-120}
+	local sleep_time=${4:-5}
+
+	local retry=0
+	echo "Waiting for $pvc to be resized"
+	until [[ $(kubectl_bin get "$pvc" -o jsonpath='{.status.capacity.storage}') == "$expected_size" ]]; do
+		if [[ $retry -ge $max_retry ]]; then
+			echo
+			echo "$pvc was not resized, max retries exceeded"
+			return 1
+		fi
+		echo -n "."
+		sleep "$sleep_time"
+
+		retry=$((retry + 1))
+	done
+	echo
+	echo "${pvc} was resized"
+	return 0
+}
+
 set_debug
 
 if [ "$EKS" == 1 -o -n "$OPENSHIFT" ]; then
@@ -89,24 +126,17 @@ desc "test scaling"
 patch_pvc_request "${cluster}" "3G"
 wait_cluster_consistency "$cluster" 3 2
 
-for pvc in $(kubectl_bin get pvc -l app.kubernetes.io/component=pxc -o name); do
-	retry=0
-	echo -n "Waiting for ${pvc} to be resized"
-	until [[ $(kubectl_bin get ${pvc} -o jsonpath={.status.capacity.storage}) == "3Gi" ]]; do
-		if [[ $retry -ge 60 ]]; then
-			echo
-			echo "pvc/${pvc} was not resized, max retries exceeded"
-			exit 1
-		fi
+if wait_all_pvc_resize "3Gi" 120 1; then
+	echo "PVC was resized, but resize.expansion is disabled"
+	exit 1
+fi
 
-		echo -n "."
-		sleep 5
+echo "Enabling PVC resize"
+kubectl_bin patch pxc "${cluster}" --type=json -p='[{"op": "add", "path": "/spec/enableVolumeExpansion", "value":true }]'
+sleep 10
 
-		retry=$((retry + 1))
-	done
-	echo
-	echo "${pvc} was resized"
-done
+wait_cluster_consistency "${cluster}" 3 2
+wait_all_pvc_resize "3Gi"
 
 if [ "$EKS" == 1 -o -n "$OPENSHIFT" ]; then
 	# EKS rate limits PVC expansion for the same EBS volume (1 expand operation in every 6 hours),

pkg/apis/pxc/v1/pxc_types.go

@@ -47,6 +47,7 @@ type PerconaXtraDBClusterSpec struct {
 	UpgradeOptions         UpgradeOptions                       `json:"upgradeOptions,omitempty"`
 	AllowUnsafeConfig      bool                                 `json:"allowUnsafeConfigurations,omitempty"`
 	Unsafe                 UnsafeFlags                          `json:"unsafeFlags,omitempty"`
+	VolumeExpansionEnabled bool                                 `json:"enableVolumeExpansion,omitempty"`
 
 	// Deprecated, should be removed in the future. Use InitContainer.Image instead
 	InitImage string `json:"initImage,omitempty"`

pkg/controller/pxc/volumes.go

@@ -232,6 +232,12 @@ func (r *ReconcilePerconaXtraDBCluster) reconcilePersistentVolumes(ctx context.C
 		return nil
 	}
 
+	if !cr.Spec.VolumeExpansionEnabled {
+		// If expansion is disabled we should keep the old value
+		cr.Spec.PXC.VolumeSpec.PersistentVolumeClaim.Resources.Requests[corev1.ResourceStorage] = configured
+		return nil
+	}
+
 	err = k8s.AnnotateObject(ctx, r.client, cr, map[string]string{pxcv1.AnnotationPVCResizeInProgress: metav1.Now().Format(time.RFC3339)})
 	if err != nil {
 		return errors.Wrap(err, "annotate pxc")