diff --git a/README.md b/README.md index e22b676..8f1d087 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,10 @@ pvmigrate --source-sc "source" --dest-sc "destination" --preflight-validation-on | --pod-ready-timeout | Integer | | 60 | length of time to wait (in seconds) for validation pod(s) to go into Ready phase | | --delete-pv-timeout | Integer | | 300 | length of time to wait (in seconds) for backing PV to be removed when the temporary PVC is deleted | +## Annotations + +`kurl.sh/pvcmigrate-destinationaccessmode` - Modifies the access mode of the PVC during migration. Valid options are - `[ReadWriteOnce, ReadWriteMany, ReadOnlyMany]` + ## Process In order, it: @@ -50,7 +54,9 @@ In order, it: 1. Validates that both the `source` and `dest` StorageClasses exist 2. Finds PVs using the `source` StorageClass 3. Finds PVCs corresponding to the above PVs -4. Creates new PVCs for each existing PVC, but using the `dest` StorageClass +4. Creates new PVCs for each existing PVC + * Uses the `dest` StorageClass for the new PVCs + * Uses the access mode set in the annotation: `kurl.sh/pvcmigrate-destinationaccessmode` if specified on a source PVC 5. For each PVC: * Finds all pods mounting the existing PVC * Finds all StatefulSets and Deployments controlling those pods and adds an annotation with the original scale diff --git a/pkg/migrate/migrate.go b/pkg/migrate/migrate.go index 6b13df5..9fecf15 100644 --- a/pkg/migrate/migrate.go +++ b/pkg/migrate/migrate.go @@ -20,12 +20,13 @@ import ( ) const ( - baseAnnotation = "kurl.sh/pvcmigrate" - scaleAnnotation = baseAnnotation + "-scale" - kindAnnotation = baseAnnotation + "-kind" - sourceNsAnnotation = baseAnnotation + "-sourcens" - sourcePVCAnnotation = baseAnnotation + "-sourcepvc" - desiredReclaimAnnotation = baseAnnotation + "-reclaim" + baseAnnotation = "kurl.sh/pvcmigrate" + scaleAnnotation = baseAnnotation + "-scale" + kindAnnotation = baseAnnotation + "-kind" + sourceNsAnnotation = baseAnnotation + "-sourcens" + sourcePVCAnnotation = baseAnnotation + "-sourcepvc" + desiredReclaimAnnotation = baseAnnotation + "-reclaim" + DesiredAccessModeAnnotation = baseAnnotation + "-destinationaccessmode" ) // IsDefaultStorageClassAnnotation - this is also exported by https://github.com/kubernetes/kubernetes/blob/v1.21.3/pkg/apis/storage/v1/util/helpers.go#L25 @@ -511,6 +512,12 @@ func getPVCs(ctx context.Context, w *log.Logger, clientset k8sclient.Interface, } } + // Set destination access mode based on annotations + destAccessModes, err := GetDestAccessModes(*nsPvc.claim) + if err != nil { + return nil, nil, fmt.Errorf("failed to get destination access mode for PVC %s in %s: %w", nsPvc.claim.Name, ns, err) + } + // if it doesn't already exist, create it newPVC, err := clientset.CoreV1().PersistentVolumeClaims(ns).Create(ctx, &corev1.PersistentVolumeClaim{ TypeMeta: nsPvc.claim.TypeMeta, @@ -529,7 +536,7 @@ func getPVCs(ctx context.Context, w *log.Logger, clientset k8sclient.Interface, corev1.ResourceStorage: desiredPvStorage, }, }, - AccessModes: nsPvc.claim.Spec.AccessModes, + AccessModes: destAccessModes, }, }, metav1.CreateOptions{}) if err != nil { @@ -1031,6 +1038,12 @@ func swapPVs(ctx context.Context, w *log.Logger, clientset k8sclient.Interface, return fmt.Errorf("failed to remove claimrefs from PV %s: %w", migratedPVC.Spec.VolumeName, err) } + // Set destination access mode based on annotations + destAccessModes, err := GetDestAccessModes(*originalPVC) + if err != nil { + return fmt.Errorf("failed to get destination access mode for PVC %s in %s: %w", originalPVC.Name, ns, err) + } + // create new PVC with the old name/annotations/settings, and the new PV w.Printf("Creating new PVC %s with migrated-to PV %s\n", originalPVC.Name, migratedPVC.Spec.VolumeName) newPVC := corev1.PersistentVolumeClaim{ @@ -1044,7 +1057,7 @@ func swapPVs(ctx context.Context, w *log.Logger, clientset k8sclient.Interface, Labels: originalPVC.Labels, // copy labels, don't copy annotations }, Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: originalPVC.Spec.AccessModes, + AccessModes: destAccessModes, Resources: originalPVC.Spec.Resources, VolumeMode: originalPVC.Spec.VolumeMode, @@ -1160,3 +1173,21 @@ func readLineWithTimeout(reader LineReader, timeout time.Duration) ([]byte, erro return message.line, message.err } } + +func GetDestAccessModes(srcPVC corev1.PersistentVolumeClaim) ([]corev1.PersistentVolumeAccessMode, error) { + // default to the source PVCs access mode if DesiredAccessModeAnnotation is not set + destAccessMode := srcPVC.Spec.AccessModes + if accessMode := srcPVC.Annotations[DesiredAccessModeAnnotation]; accessMode != "" { + switch accessMode { + case "ReadWriteOnce": + destAccessMode = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + case "ReadOnlyMany": + destAccessMode = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + case "ReadWriteMany": + destAccessMode = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + default: + return nil, fmt.Errorf("invalid access mode '%s' used in annotation '%s' for PVC '%s' in namespace '%s'", accessMode, DesiredAccessModeAnnotation, srcPVC.Name, srcPVC.Namespace) + } + } + return destAccessMode, nil +} diff --git a/pkg/migrate/migrate_test.go b/pkg/migrate/migrate_test.go index 2a8b9b9..51db8ad 100644 --- a/pkg/migrate/migrate_test.go +++ b/pkg/migrate/migrate_test.go @@ -1564,6 +1564,262 @@ func Test_swapPVs(t *testing.T) { }, }, }, + { + name: "swap one PVC and change destination access mode to ReadWriteOnce", + ns: "testns", + pvcName: "sourcepvc", + backgroundFunc: func(ctx context.Context, logger *log.Logger, k k8sclient.Interface) { + // watch for the statefulset to be scaled down, and then delete the pod + for { + select { + case <-time.After(time.Second / 100): + // check statefulset, maybe delete pod + pvcs, err := k.CoreV1().PersistentVolumeClaims("testns").List(ctx, metav1.ListOptions{}) + if err != nil { + logger.Printf("got listing PVCs: %s", err.Error()) + continue + } + + for _, pvc := range pvcs.Items { + if pvc.Spec.VolumeName != "" { + logger.Printf("setting pv %s claim ref to pvc %s", pvc.Spec.VolumeName, pvc.Name) + err := mutatePV(ctx, logger, k, pvc.Spec.VolumeName, + func(volume *corev1.PersistentVolume) (*corev1.PersistentVolume, error) { + volume.Spec.ClaimRef = &corev1.ObjectReference{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + Namespace: "testns", + Name: pvc.Name, + } + return volume, nil + }, + func(volume *corev1.PersistentVolume) bool { + return true + }, + ) + if err != nil { + logger.Printf("error mutating PV: %s", err) + } + } + } + case <-ctx.Done(): + return + } + } + }, + resources: []runtime.Object{ + // One PVC with kurl.sh/pvcmigrate-destinationaccessmode annotation set to ReadWriteMany + &corev1.PersistentVolumeClaim{ + TypeMeta: metav1.TypeMeta{ + Kind: "PersistentVolumeClaim", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sourcepvc", + Namespace: "testns", + Annotations: map[string]string{ + "kurl.sh/pvcmigrate-destinationaccessmode": "ReadWriteMany", + }, + Labels: map[string]string{ + "testlabel": "sourcepvc", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + StorageClassName: &sourceScName, + VolumeName: "source-pv", + }, + Status: corev1.PersistentVolumeClaimStatus{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Capacity: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + Phase: corev1.ClaimBound, + }, + }, + &corev1.PersistentVolumeClaim{ + TypeMeta: metav1.TypeMeta{ + Kind: "PersistentVolumeClaim", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sourcepvc-pvcmigrate", + Namespace: "testns", + Annotations: map[string]string{ + "testannotation": "sourcepvc-pvcmigrate", + }, + Labels: map[string]string{ + "testlabel": "sourcepvc-pvcmigrate", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany, + }, + Resources: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + StorageClassName: &destScName, + VolumeName: "dest-pv", + }, + Status: corev1.PersistentVolumeClaimStatus{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany, + }, + Capacity: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + Phase: corev1.ClaimBound, + }, + }, + // One PV bound by PVC with kurl.sh/pvcmigrate-destinationaccessmode annotation set + &corev1.PersistentVolume{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "PersistentVolume", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "source-pv", + Labels: map[string]string{ + "testlabel": "source-pv", + }, + Annotations: map[string]string{ + "testannotation": "source-pv", + }, + }, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Capacity: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + ClaimRef: &corev1.ObjectReference{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + Namespace: "testns", + Name: "sourcepvc", + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete, + StorageClassName: sourceScName, + }, + Status: corev1.PersistentVolumeStatus{ + Phase: corev1.VolumeBound, + }, + }, + &corev1.PersistentVolume{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "PersistentVolume", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "dest-pv", + Labels: map[string]string{ + "testlabel": "dest-pv", + }, + Annotations: map[string]string{ + "testannotation": "dest-pv", + }, + }, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany, + }, + Capacity: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + ClaimRef: &corev1.ObjectReference{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + Namespace: "testns", + Name: "sourcepvc-pvcmigrate", + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete, + StorageClassName: sourceScName, + }, + Status: corev1.PersistentVolumeStatus{ + Phase: corev1.VolumeBound, + }, + }, + }, + wantPVs: []corev1.PersistentVolume{ + { + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "PersistentVolume", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "dest-pv", + Labels: map[string]string{ + "testlabel": "dest-pv", + }, + Annotations: map[string]string{ + desiredReclaimAnnotation: "Delete", + sourceNsAnnotation: "testns", + sourcePVCAnnotation: "sourcepvc", + "testannotation": "dest-pv", + }, + }, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany, + }, + Capacity: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete, + StorageClassName: sourceScName, + ClaimRef: &corev1.ObjectReference{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + Namespace: "testns", + Name: "sourcepvc", + }, + }, + Status: corev1.PersistentVolumeStatus{ + Phase: corev1.VolumeBound, + }, + }, + }, + wantPVCs: []corev1.PersistentVolumeClaim{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "PersistentVolumeClaim", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sourcepvc", + Namespace: "testns", + Labels: map[string]string{ + "testlabel": "sourcepvc", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany, + }, + Resources: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + StorageClassName: &destScName, + VolumeName: "dest-pv", + }, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -3299,3 +3555,143 @@ func Test_readLineWithTimeout(t *testing.T) { }) } } + +func Test_GetDestAccessModes(t *testing.T) { + scName := "scName" + for _, tt := range []struct { + name string + srcPVC corev1.PersistentVolumeClaim + wantAccessModes []corev1.PersistentVolumeAccessMode + wantErr bool + }{ + { + name: "destination access mode is RWX", + srcPVC: corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-to-migrate", + Namespace: "ns1", + Annotations: map[string]string{ + "kurl.sh/pvcmigrate-destinationaccessmode": "ReadWriteMany", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "pv2", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + StorageClassName: &scName, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + }, + }, + wantAccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + wantErr: false, + }, + { + name: "destination access mode is RWO", + srcPVC: corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-to-migrate", + Namespace: "ns1", + Annotations: map[string]string{ + "kurl.sh/pvcmigrate-destinationaccessmode": "ReadWriteOnce", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "pv2", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + StorageClassName: &scName, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + }, + }, + wantAccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + wantErr: false, + }, + { + name: "destination access mode is invalid", + srcPVC: corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-to-migrate", + Namespace: "ns1", + Annotations: map[string]string{ + "kurl.sh/pvcmigrate-destinationaccessmode": "ReadWriteInifity", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "pv2", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + StorageClassName: &scName, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + }, + }, + wantAccessModes: nil, + wantErr: true, + }, + { + name: "destination access mode is empty", + srcPVC: corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-to-migrate", + Namespace: "ns1", + Annotations: map[string]string{ + "kurl.sh/pvcmigrate-destinationaccessmode": "", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "pv2", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + StorageClassName: &scName, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + }, + }, + wantAccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + wantErr: false, + }, + { + name: "destination access mode annotation isn't set", + srcPVC: corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-to-migrate", + Namespace: "ns1", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "pv2", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + StorageClassName: &scName, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany}, + }, + }, + wantAccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany}, + wantErr: false, + }, + } { + t.Run(tt.name, func(t *testing.T) { + req := require.New(t) + accessModes, err := GetDestAccessModes(tt.srcPVC) + if tt.wantErr { + req.Error(err) + return + } else { + req.NoError(err) + } + req.Equal(tt.wantAccessModes, accessModes) + }) + } +} diff --git a/pkg/preflight/validate.go b/pkg/preflight/validate.go index 43c2bc2..c062191 100644 --- a/pkg/preflight/validate.go +++ b/pkg/preflight/validate.go @@ -183,7 +183,12 @@ func buildTmpPVCConsumerPod(pvcName, namespace, image string) *corev1.Pod { } // buildTmpPVC creates a temporary PVC requesting for 1Mi of storage for a provided storage class name. -func buildTmpPVC(pvc corev1.PersistentVolumeClaim, sc string) *corev1.PersistentVolumeClaim { +func buildTmpPVC(pvc corev1.PersistentVolumeClaim, sc string) (*corev1.PersistentVolumeClaim, error) { + destAccessModes, err := migrate.GetDestAccessModes(pvc) + if err != nil { + return nil, fmt.Errorf("failed to get destination access mode for PVC %s in %s: %w", pvc.Name, pvc.Namespace, err) + } + return &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: k8sutil.NewPrefixedName(pvcNamePrefix, pvc.Name), @@ -191,14 +196,14 @@ func buildTmpPVC(pvc corev1.PersistentVolumeClaim, sc string) *corev1.Persistent }, Spec: corev1.PersistentVolumeClaimSpec{ StorageClassName: &sc, - AccessModes: pvc.Spec.AccessModes, + AccessModes: destAccessModes, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceStorage: resource.MustParse("1Mi"), }, }, }, - } + }, nil } // checkVolumeAccessModes checks if the access modes of a pv are supported by the @@ -207,7 +212,10 @@ func checkVolumeAccessModes(ctx context.Context, l *log.Logger, client k8sclient var err error // create temp pvc for storage class - tmpPVCSpec := buildTmpPVC(pvc, dstSC) + tmpPVCSpec, err := buildTmpPVC(pvc, dstSC) + if err != nil { + return nil, fmt.Errorf("failed to create temporary pvc spec for %s: %w", pvc.Name, err) + } tmpPVC, err := client.CoreV1().PersistentVolumeClaims(tmpPVCSpec.Namespace).Create( ctx, tmpPVCSpec, metav1.CreateOptions{}) if err != nil { diff --git a/pkg/preflight/validate_test.go b/pkg/preflight/validate_test.go index 4fc2be2..6d0d42c 100644 --- a/pkg/preflight/validate_test.go +++ b/pkg/preflight/validate_test.go @@ -485,10 +485,45 @@ func Test_buildTmpPVC(t *testing.T) { }, dstStorageClass: "dstSc", }, + { + name: "change access mode of tmp PVC if destinationaccessmode annotation is set", + input: &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-name", + Namespace: "default", + Annotations: map[string]string{ + "kurl.sh/pvcmigrate-destinationaccessmode": "ReadWriteMany", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: pointer.String("dstSc"), + AccessModes: []corev1.PersistentVolumeAccessMode{"ReadWriteOnce"}, + }, + }, + expectedPVC: &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pf-pvc-pvc-name", + Namespace: "default", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: pointer.String("dstSc"), + AccessModes: []corev1.PersistentVolumeAccessMode{"ReadWriteMany"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Mi"), + }, + }, + }, + }, + dstStorageClass: "dstSc", + }, } { t.Run(tt.name, func(t *testing.T) { req := require.New(t) - pvc := buildTmpPVC(*tt.input, tt.dstStorageClass) + pvc, err := buildTmpPVC(*tt.input, tt.dstStorageClass) + if err != nil { + req.NoError(err) + } req.Equal(tt.expectedPVC, pvc) }) }