Merge pull request #624 from red-hat-storage/sync_us--master

Syncing latest changes from upstream master for rook

Documentation/Storage-Configuration/ceph-teardown.md

@@ -200,6 +200,7 @@ Once the cleanup job is completed successfully, Rook will remove the finalizers
 
 This cleanup is supported only for the following custom resources:
 
-| Custom Resource                                | Ceph Resources to be cleaned up |
-| --------                                       | ------- |
-| CephFilesystemSubVolumeGroup                   | CSI stored RADOS OMAP details for pvc/volumesnapshots, subvolume snapshots, subvolume clones, subvolumes |
+| Custom Resource                      | Ceph Resources to be cleaned up |
+| --------                             | ------- |
+| CephFilesystemSubVolumeGroup         | CSI stored RADOS OMAP details for pvc/volumesnapshots, subvolume snapshots, subvolume clones, subvolumes |
+| CephBlockPoolRadosNamespace          | Images and snapshots in the RADOS namespace|

cmd/rook/ceph/cleanup.go

@@ -56,6 +56,12 @@ var cleanUpSubVolumeGroupCmd = &cobra.Command{
 	Short: "Starts the cleanup process of a CephFilesystemSubVolumeGroup",
 }
 
+var cleanUpRadosNamespaceCmd = &cobra.Command{
+	// the subcommand matches CRD kind of the custom resource to be cleaned up
+	Use:   "CephBlockPoolRadosNamespace",
+	Short: "Starts the cleanup process for a CephBlockPoolRadosNamespace",
+}
+
 func init() {
 	cleanUpCmd.Flags().StringVar(&dataDirHostPath, "data-dir-host-path", "", "dataDirHostPath on the node")
 	cleanUpCmd.Flags().StringVar(&namespaceDir, "namespace-dir", "", "dataDirHostPath on the node")
@@ -68,10 +74,11 @@ func init() {
 
 	flags.SetFlagsFromEnv(cleanUpSubVolumeGroupCmd.Flags(), rook.RookEnvVarPrefix)
 
-	cleanUpCmd.AddCommand(cleanUpHostCmd, cleanUpSubVolumeGroupCmd)
+	cleanUpCmd.AddCommand(cleanUpHostCmd, cleanUpSubVolumeGroupCmd, cleanUpRadosNamespaceCmd)
 
 	cleanUpHostCmd.RunE = startHostCleanUp
 	cleanUpSubVolumeGroupCmd.RunE = startSubVolumeGroupCleanUp
+	cleanUpRadosNamespaceCmd.RunE = startRadosNamespaceCleanup
 }
 
 func startHostCleanUp(cmd *cobra.Command, args []string) error {
@@ -140,3 +147,30 @@ func startSubVolumeGroupCleanUp(cmd *cobra.Command, args []string) error {
 
 	return nil
 }
+
+func startRadosNamespaceCleanup(cmd *cobra.Command, args []string) error {
+	rook.SetLogLevel()
+	rook.LogStartupInfo(cleanUpRadosNamespaceCmd.Flags())
+
+	ctx := cmd.Context()
+	context := createContext()
+	namespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
+	clusterInfo := client.AdminClusterInfo(ctx, namespace, "")
+
+	poolName := os.Getenv(opcontroller.CephBlockPoolNameEnv)
+	if poolName == "" {
+		rook.TerminateFatal(fmt.Errorf("cephblockpool name is not available in the pod environment variables"))
+	}
+
+	radosNamespace := os.Getenv(opcontroller.CephBlockPoolRadosNamespaceEnv)
+	if radosNamespace == "" {
+		rook.TerminateFatal(fmt.Errorf("cephblockpool radosNamespace is not available in the pod environment variables"))
+	}
+
+	err := cleanup.RadosNamespaceCleanup(context, clusterInfo, poolName, radosNamespace)
+	if err != nil {
+		rook.TerminateFatal(fmt.Errorf("failed to cleanup cephBlockPoolRadosNamespace %q resources in the pool %q. %v", radosNamespace, poolName, err))
+	}
+
+	return nil
+}

pkg/daemon/ceph/cleanup/radosnamespace.go

@@ -0,0 +1,71 @@
+/*
+Copyright 2024 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cleanup
+
+import (
+	"github.com/pkg/errors"
+	"github.com/rook/rook/pkg/clusterd"
+	"github.com/rook/rook/pkg/daemon/ceph/client"
+	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
+)
+
+func RadosNamespaceCleanup(context *clusterd.Context, clusterInfo *client.ClusterInfo, poolName, radosNamespace string) error {
+	logger.Infof("starting clean up of CephBlockPoolRadosNamespace %q resources in cephblockpool %q", radosNamespace, poolName)
+
+	images, err := cephclient.ListImagesInRadosNamespace(context, clusterInfo, poolName, radosNamespace)
+	if err != nil {
+		return errors.Wrapf(err, "failed to list images in cephblockpool %q in rados namespace %q", poolName, radosNamespace)
+	}
+
+	var retErr error
+	for _, image := range images {
+		snaps, err := cephclient.ListSnapshotsInRadosNamespace(context, clusterInfo, poolName, image.Name, radosNamespace)
+		if err != nil {
+			retErr = errors.Wrapf(err, "failed to list snapshots for the image %q in cephblockpool %q in rados namespace %q.", image.Name, poolName, radosNamespace)
+			logger.Error(retErr)
+		}
+
+		for _, snap := range snaps {
+			err := cephclient.DeleteSnapshotInRadosNamespace(context, clusterInfo, poolName, image.Name, snap.Name, radosNamespace)
+			if err != nil {
+				retErr = errors.Wrapf(err, "failed to delete snapshot %q of the image %q in cephblockpool %q in rados namespace %q.", snap.Name, image.Name, poolName, radosNamespace)
+				logger.Error(retErr)
+			} else {
+				logger.Infof("successfully deleted snapshot %q of image %q in cephblockpool %q in rados namespace %q", snap.Name, image.Name, poolName, radosNamespace)
+			}
+		}
+
+		err = cephclient.MoveImageToTrashInRadosNamespace(context, clusterInfo, poolName, image.Name, radosNamespace)
+		if err != nil {
+			retErr = errors.Wrapf(err, "failed to move image %q to trash in cephblockpool %q in rados namespace %q.", image.Name, poolName, radosNamespace)
+			logger.Error(retErr)
+		}
+		err = cephclient.DeleteImageFromTrashInRadosNamespace(context, clusterInfo, poolName, image.ID, radosNamespace)
+		if err != nil {
+			retErr = errors.Wrapf(err, "failed to add task to remove image %q from trash in cephblockpool %q in rados namespace %q.", image.Name, poolName, radosNamespace)
+			logger.Error(retErr)
+		}
+	}
+
+	if retErr != nil {
+		logger.Errorf("failed to clean up CephBlockPoolRadosNamespace %q resources in cephblockpool %q", radosNamespace, poolName)
+		return retErr
+	}
+
+	logger.Infof("successfully cleaned up CephBlockPoolRadosNamespace %q resources in cephblockpool %q", radosNamespace, poolName)
+	return nil
+}

pkg/daemon/ceph/cleanup/radosnamespace_test.go

@@ -0,0 +1,92 @@
+/*
+Copyright 2024 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cleanup
+
+import (
+	"testing"
+
+	"github.com/pkg/errors"
+	"github.com/rook/rook/pkg/clusterd"
+	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
+	exectest "github.com/rook/rook/pkg/util/exec/test"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	mockImageLSResponse   = `[{"image":"csi-vol-136268e8-5386-4453-a6bd-9dca381d187d","id":"16e35cfa56a7","size":1073741824,"format":2}]`
+	mockSnapshotsResponse = `[{"id":5,"name":"snap1","size":1073741824,"protected":"false","timestamp":"Fri Apr 12 13:39:28 2024"}]`
+)
+
+func TestRadosNamespace(t *testing.T) {
+	clusterInfo := cephclient.AdminTestClusterInfo("mycluster")
+	poolName := "test-pool"
+	radosNamespace := "test-namespace"
+
+	t.Run("no images in rados namespace", func(t *testing.T) {
+		executor := &exectest.MockExecutor{}
+		executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
+			logger.Infof("Command: %s %v", command, args)
+			if args[0] == "ls" && args[1] == "-l" {
+				assert.Equal(t, poolName, args[2])
+				return "", nil
+			}
+			return "", errors.New("unknown command")
+		}
+		context := &clusterd.Context{Executor: executor}
+		err := RadosNamespaceCleanup(context, clusterInfo, poolName, radosNamespace)
+		assert.NoError(t, err)
+	})
+
+	t.Run("images with snapshots available in rados namespace", func(t *testing.T) {
+		executor := &exectest.MockExecutor{}
+		executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
+			logger.Infof("Command: %s %v", command, args)
+			// list all subvolumes in subvolumegroup
+			if args[0] == "ls" && args[1] == "-l" {
+				assert.Equal(t, poolName, args[2])
+				return mockImageLSResponse, nil
+			}
+			if args[0] == "snap" && args[1] == "ls" {
+				assert.Equal(t, "test-pool/csi-vol-136268e8-5386-4453-a6bd-9dca381d187d", args[2])
+				assert.Equal(t, "--namespace", args[3])
+				assert.Equal(t, radosNamespace, args[4])
+				return mockSnapshotsResponse, nil
+			}
+			if args[0] == "snap" && args[1] == "rm" {
+				assert.Equal(t, "test-pool/csi-vol-136268e8-5386-4453-a6bd-9dca381d187d@snap1", args[2])
+				assert.Equal(t, "--namespace", args[3])
+				assert.Equal(t, radosNamespace, args[4])
+				return "", nil
+			}
+			if args[0] == "trash" && args[1] == "mv" {
+				assert.Equal(t, "test-pool/csi-vol-136268e8-5386-4453-a6bd-9dca381d187d", args[2])
+				assert.Equal(t, "--namespace", args[3])
+				assert.Equal(t, radosNamespace, args[4])
+				return "", nil
+			}
+			if args[0] == "rbd" && args[1] == "task" && args[2] == "add" && args[3] == "trash" {
+				// pool-name/rados-namespace/image-id
+				assert.Equal(t, "test-pool/test-namespace/16e35cfa56a7", args[5])
+				return "", nil
+			}
+			return "", errors.New("unknown command")
+		}
+		context := &clusterd.Context{Executor: executor}
+		err := RadosNamespaceCleanup(context, clusterInfo, poolName, radosNamespace)
+		assert.NoError(t, err)
+	})
+}

pkg/daemon/ceph/client/image.go

@@ -35,14 +35,28 @@ const (
 )
 
 type CephBlockImage struct {
+	ID       string `json:"id"`
 	Name     string `json:"image"`
 	Size     uint64 `json:"size"`
 	Format   int    `json:"format"`
 	InfoName string `json:"name"`
 }
 
-func ListImages(context *clusterd.Context, clusterInfo *ClusterInfo, poolName string) ([]CephBlockImage, error) {
+type CephBlockImageSnapshot struct {
+	Name string `json:"name"`
+}
+
+// ListImagesInPool returns a list of images created in a cephblockpool
+func ListImagesInPool(context *clusterd.Context, clusterInfo *ClusterInfo, poolName string) ([]CephBlockImage, error) {
+	return ListImagesInRadosNamespace(context, clusterInfo, poolName, "")
+}
+
+// ListImagesInRadosNamespace returns a list if images created in a cephblockpool rados namespace
+func ListImagesInRadosNamespace(context *clusterd.Context, clusterInfo *ClusterInfo, poolName, namespace string) ([]CephBlockImage, error) {
 	args := []string{"ls", "-l", poolName}
+	if namespace != "" {
+		args = append(args, "--namespace", namespace)
+	}
 	cmd := NewRBDCommand(context, clusterInfo, args)
 	cmd.JsonOutput = true
 	buf, err := cmd.Run()
@@ -68,6 +82,56 @@ func ListImages(context *clusterd.Context, clusterInfo *ClusterInfo, poolName st
 	return images, nil
 }
 
+// ListSnapshotsInRadosNamespace lists all the snapshots created for an image in a cephblockpool in a given rados namespace
+func ListSnapshotsInRadosNamespace(context *clusterd.Context, clusterInfo *ClusterInfo, poolName, imageName, namespace string) ([]CephBlockImageSnapshot, error) {
+	snapshots := []CephBlockImageSnapshot{}
+	args := []string{"snap", "ls", getImageSpec(imageName, poolName), "--namespace", namespace}
+	cmd := NewRBDCommand(context, clusterInfo, args)
+	cmd.JsonOutput = true
+	buf, err := cmd.Run()
+	if err != nil {
+		return snapshots, errors.Wrapf(err, "failed to list snapshots of image %q in cephblockpool %q", imageName, poolName)
+	}
+
+	if err = json.Unmarshal(buf, &snapshots); err != nil {
+		return snapshots, errors.Wrapf(err, "unmarshal failed, raw buffer response: %s", string(buf))
+	}
+	return snapshots, nil
+}
+
+// DeleteSnapshotInRadosNamespace deletes a image snapshot created in block pool in a given rados namespace
+func DeleteSnapshotInRadosNamespace(context *clusterd.Context, clusterInfo *ClusterInfo, poolName, imageName, snapshot, namespace string) error {
+	args := []string{"snap", "rm", getImageSnapshotSpec(poolName, imageName, snapshot), "--namespace", namespace}
+	cmd := NewRBDCommand(context, clusterInfo, args)
+	_, err := cmd.Run()
+	if err != nil {
+		return errors.Wrapf(err, "failed to delete snapshot %q of image %q in cephblockpool %q", snapshot, imageName, poolName)
+	}
+	return nil
+}
+
+// MoveImageToTrashInRadosNamespace moves the cephblockpool image to trash in the rados namespace
+func MoveImageToTrashInRadosNamespace(context *clusterd.Context, clusterInfo *ClusterInfo, poolName, imageName, namespace string) error {
+	args := []string{"trash", "mv", getImageSpec(imageName, poolName), "--namespace", namespace}
+	cmd := NewRBDCommand(context, clusterInfo, args)
+	_, err := cmd.Run()
+	if err != nil {
+		return errors.Wrapf(err, "failed to move image %q in cephblockpool %q to trash", imageName, poolName)
+	}
+	return nil
+}
+
+// DeleteImageFromTrashInRadosNamespace deletes the image from trash in the rados namespace
+func DeleteImageFromTrashInRadosNamespace(context *clusterd.Context, clusterInfo *ClusterInfo, poolName, imageID, namespace string) error {
+	args := []string{"rbd", "task", "add", "trash", "remove", getImageSpecInRadosNamespace(poolName, namespace, imageID)}
+	cmd := NewCephCommand(context, clusterInfo, args)
+	_, err := cmd.Run()
+	if err != nil {
+		return errors.Wrapf(err, "failed to delete image %q in cephblockpool %q from trash", imageID, poolName)
+	}
+	return nil
+}
+
 // CreateImage creates a block storage image.
 // If dataPoolName is not empty, the image will use poolName as the metadata pool and the dataPoolname for data.
 // If size is zero an empty image will be created. Otherwise, an image will be
@@ -116,16 +180,22 @@ func CreateImage(context *clusterd.Context, clusterInfo *ClusterInfo, name, pool
 	return &CephBlockImage{Name: name, Size: newSizeBytes}, nil
 }
 
-func DeleteImage(context *clusterd.Context, clusterInfo *ClusterInfo, name, poolName string) error {
+func DeleteImageInPool(context *clusterd.Context, clusterInfo *ClusterInfo, name, poolName string) error {
+	return DeleteImageInRadosNamespace(context, clusterInfo, name, poolName, "")
+}
+
+func DeleteImageInRadosNamespace(context *clusterd.Context, clusterInfo *ClusterInfo, name, poolName, namespace string) error {
 	logger.Infof("deleting rbd image %q from pool %q", name, poolName)
 	imageSpec := getImageSpec(name, poolName)
 	args := []string{"rm", imageSpec}
+	if namespace != "" {
+		args = append(args, "--namespace", namespace)
+	}
 	buf, err := NewRBDCommand(context, clusterInfo, args).Run()
 	if err != nil {
 		return errors.Wrapf(err, "failed to delete image %s in pool %s, output: %s",
 			name, poolName, string(buf))
 	}
-
 	return nil
 }
 
@@ -196,3 +266,10 @@ func UnMapImage(context *clusterd.Context, clusterInfo *ClusterInfo, imageName,
 func getImageSpec(name, poolName string) string {
 	return fmt.Sprintf("%s/%s", poolName, name)
 }
+
+func getImageSpecInRadosNamespace(poolName, namespace, imageID string) string {
+	return fmt.Sprintf("%s/%s/%s", poolName, namespace, imageID)
+}
+func getImageSnapshotSpec(poolName, imageName, snapshot string) string {
+	return fmt.Sprintf("%s/%s@%s", poolName, imageName, snapshot)
+}

pkg/daemon/ceph/client/image_test.go

@@ -187,15 +187,15 @@ func TestListImageLogLevelInfo(t *testing.T) {
 	}
 
 	clusterInfo := AdminTestClusterInfo("mycluster")
-	images, err = ListImages(context, clusterInfo, "pool1")
+	images, err = ListImagesInPool(context, clusterInfo, "pool1")
 	assert.Nil(t, err)
 	assert.NotNil(t, images)
 	assert.True(t, len(images) == 3)
 	assert.True(t, listCalled)
 	listCalled = false
 
 	emptyListResult = true
-	images, err = ListImages(context, clusterInfo, "pool1")
+	images, err = ListImagesInPool(context, clusterInfo, "pool1")
 	assert.Nil(t, err)
 	assert.NotNil(t, images)
 	assert.True(t, len(images) == 0)
@@ -251,15 +251,15 @@ func TestListImageLogLevelDebug(t *testing.T) {
 	}
 
 	clusterInfo := AdminTestClusterInfo("mycluster")
-	images, err = ListImages(context, clusterInfo, "pool1")
+	images, err = ListImagesInPool(context, clusterInfo, "pool1")
 	assert.Nil(t, err)
 	assert.NotNil(t, images)
 	assert.True(t, len(images) == 3)
 	assert.True(t, listCalled)
 	listCalled = false
 
 	emptyListResult = true
-	images, err = ListImages(context, clusterInfo, "pool1")
+	images, err = ListImagesInPool(context, clusterInfo, "pool1")
 	assert.Nil(t, err)
 	assert.NotNil(t, images)
 	assert.True(t, len(images) == 0)