From 4e5237449b052f4de0a4aa02dc879140d635f0d3 Mon Sep 17 00:00:00 2001 From: Mohamed Awnallah Date: Sun, 24 Nov 2024 18:02:23 +0200 Subject: [PATCH] pkg/karmadactl: unit test unjoin In this commit, we unit test unjoining cluster by validating the unjoin cluster operation on arguments passed and by verifying the deletion of all resources on the unjoining cluster in case of cluster kubeconfig passed. Signed-off-by: Mohamed Awnallah --- pkg/karmadactl/unjoin/unjoin.go | 25 ++- pkg/karmadactl/unjoin/unjoin_test.go | 302 +++++++++++++++++++++++++++ 2 files changed, 320 insertions(+), 7 deletions(-) create mode 100644 pkg/karmadactl/unjoin/unjoin_test.go diff --git a/pkg/karmadactl/unjoin/unjoin.go b/pkg/karmadactl/unjoin/unjoin.go index e1bad4c4029b..7876c871fa07 100644 --- a/pkg/karmadactl/unjoin/unjoin.go +++ b/pkg/karmadactl/unjoin/unjoin.go @@ -17,6 +17,7 @@ limitations under the License. package unjoin import ( + "errors" "fmt" "time" @@ -128,10 +129,10 @@ func (j *CommandUnjoinOption) Complete(args []string) error { // Validate ensures that command unjoin options are valid. func (j *CommandUnjoinOption) Validate(args []string) error { if len(args) > 1 { - return fmt.Errorf("only the cluster name is allowed as an argument") + return errors.New("only the cluster name is allowed as an argument") } if len(j.ClusterName) == 0 { - return fmt.Errorf("cluster name is required") + return errors.New("cluster name is required") } if j.Wait <= 0 { return fmt.Errorf(" --wait %v must be a positive duration, e.g. 1m0s ", j.Wait) @@ -178,10 +179,20 @@ func (j *CommandUnjoinOption) Run(f cmdutil.Factory) error { return j.RunUnJoinCluster(controlPlaneRestConfig, clusterConfig) } +var controlPlaneKarmadaClientBuilder = func(controlPlaneRestConfig *rest.Config) karmadaclientset.Interface { + return karmadaclientset.NewForConfigOrDie(controlPlaneRestConfig) +} +var controlPlaneKubeClientBuilder = func(controlPlaneRestConfig *rest.Config) kubeclient.Interface { + return kubeclient.NewForConfigOrDie(controlPlaneRestConfig) +} +var clusterKubeClientBuilder = func(clusterConfig *rest.Config) kubeclient.Interface { + return kubeclient.NewForConfigOrDie(clusterConfig) +} + // RunUnJoinCluster unJoin the cluster from karmada. func (j *CommandUnjoinOption) RunUnJoinCluster(controlPlaneRestConfig, clusterConfig *rest.Config) error { - controlPlaneKarmadaClient := karmadaclientset.NewForConfigOrDie(controlPlaneRestConfig) - controlPlaneKubeClient := kubeclient.NewForConfigOrDie(controlPlaneRestConfig) + controlPlaneKarmadaClient := controlPlaneKarmadaClientBuilder(controlPlaneRestConfig) + controlPlaneKubeClient := controlPlaneKubeClientBuilder(controlPlaneRestConfig) // delete the cluster object in host cluster that associates the unjoining cluster err := cmdutil.DeleteClusterObject(controlPlaneKubeClient, controlPlaneKarmadaClient, j.ClusterName, j.Wait, j.DryRun, j.forceDeletion) @@ -193,7 +204,7 @@ func (j *CommandUnjoinOption) RunUnJoinCluster(controlPlaneRestConfig, clusterCo // Attempt to delete the cluster role, cluster rolebindings and service account from the unjoining cluster // if user provides the kubeconfig of cluster if clusterConfig != nil { - clusterKubeClient := kubeclient.NewForConfigOrDie(clusterConfig) + clusterKubeClient := clusterKubeClientBuilder(clusterConfig) klog.V(1).Infof("Unjoining cluster config. endpoint: %s", clusterConfig.Host) @@ -212,7 +223,7 @@ func (j *CommandUnjoinOption) RunUnJoinCluster(controlPlaneRestConfig, clusterCo } // delete namespace from unjoining cluster - err = deleteNamespaceFromUnjoinCluster(clusterKubeClient, j.ClusterNamespace, j.ClusterName, j.forceDeletion, j.DryRun) + err = deleteNamespace(clusterKubeClient, j.ClusterNamespace, j.ClusterName, j.forceDeletion, j.DryRun) if err != nil { klog.Errorf("Failed to delete namespace in unjoining cluster %q: %v", j.ClusterName, err) return err @@ -270,7 +281,7 @@ func deleteServiceAccount(clusterKubeClient kubeclient.Interface, namespace, unj } // deleteNSFromUnjoinCluster deletes the namespace from the unjoining cluster. -func deleteNamespaceFromUnjoinCluster(clusterKubeClient kubeclient.Interface, namespace, unjoiningClusterName string, forceDeletion, dryRun bool) error { +func deleteNamespace(clusterKubeClient kubeclient.Interface, namespace, unjoiningClusterName string, forceDeletion, dryRun bool) error { if dryRun { return nil } diff --git a/pkg/karmadactl/unjoin/unjoin_test.go b/pkg/karmadactl/unjoin/unjoin_test.go new file mode 100644 index 000000000000..fb4d315c8d1e --- /dev/null +++ b/pkg/karmadactl/unjoin/unjoin_test.go @@ -0,0 +1,302 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package unjoin + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubeclient "k8s.io/client-go/kubernetes" + fakeclientset "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" + coretesting "k8s.io/client-go/testing" + + clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1" + karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" + fakekarmadaclient "github.com/karmada-io/karmada/pkg/generated/clientset/versioned/fake" + "github.com/karmada-io/karmada/pkg/karmadactl/options" + "github.com/karmada-io/karmada/pkg/util" + "github.com/karmada-io/karmada/pkg/util/names" +) + +func TestValidate(t *testing.T) { + tests := []struct { + name string + unjoinOpts *CommandUnjoinOption + args []string + wantErr bool + errMsg string + }{ + { + name: "Validate_WithMoreThanOneArg_OnlyTheClusterNameIsRequired", + unjoinOpts: &CommandUnjoinOption{}, + args: []string{"cluster2", "cluster3"}, + wantErr: true, + errMsg: "only the cluster name is allowed as an argument", + }, + { + name: "Validate_WithoutClusterNameToJoinWith_ClusterNameIsRequired", + unjoinOpts: &CommandUnjoinOption{ClusterName: ""}, + args: []string{"cluster2"}, + wantErr: true, + errMsg: "cluster name is required", + }, + { + name: "Validate_WithNegativeWaitValue_WaitValueMustBePositiveDuration", + unjoinOpts: &CommandUnjoinOption{ + ClusterName: "cluster1", + Wait: -1 * time.Hour, + }, + args: []string{"cluster2"}, + wantErr: true, + errMsg: "must be a positive duration", + }, + { + name: "Validate_ValidateCommandUnjoinOptions_Validated", + unjoinOpts: &CommandUnjoinOption{ + ClusterName: "cluster1", + Wait: 2 * time.Minute, + }, + args: []string{"cluster2"}, + wantErr: false, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.unjoinOpts.Validate(test.args) + if err == nil && test.wantErr { + t.Fatal("expected an error, but got none") + } + if err != nil && !test.wantErr { + t.Errorf("unexpected error, got: %v", err) + } + if err != nil && test.wantErr && !strings.Contains(err.Error(), test.errMsg) { + t.Errorf("expected error message %s to be in %s", test.errMsg, err.Error()) + } + }) + } +} + +func TestRunUnJoinCluster(t *testing.T) { + tests := []struct { + name string + unjoinOpts *CommandUnjoinOption + controlPlaneRestConfig, clusterConfig *rest.Config + controlKubeClient, clusterKubeClient kubeclient.Interface + karmadaClient karmadaclientset.Interface + prep func(controlKubeClient kubeclient.Interface, clusterKubeClient kubeclient.Interface, karmadaClient karmadaclientset.Interface, opts *CommandUnjoinOption) error + verify func(controlKubeClient kubeclient.Interface, clusterKubeClient kubeclient.Interface, karmadaClient karmadaclientset.Interface, opts *CommandUnjoinOption) error + wantErr bool + errMsg string + }{ + { + name: "RunUnJoinCluster_DeleteClusterObject_FailedToDeleteClusterObject", + unjoinOpts: &CommandUnjoinOption{ClusterName: "member1"}, + controlPlaneRestConfig: &rest.Config{}, + clusterConfig: &rest.Config{}, + karmadaClient: fakekarmadaclient.NewSimpleClientset(), + prep: func(_ kubeclient.Interface, _ kubeclient.Interface, karmadaClient karmadaclientset.Interface, _ *CommandUnjoinOption) error { + karmadaClient.(*fakekarmadaclient.Clientset).Fake.PrependReactor("delete", "clusters", func(coretesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("unexpected error: encountered a network issue while deleting the clusters") + }) + controlPlaneKarmadaClientBuilder = func(*rest.Config) karmadaclientset.Interface { + return karmadaClient + } + return nil + }, + verify: func(kubeclient.Interface, kubeclient.Interface, karmadaclientset.Interface, *CommandUnjoinOption) error { + return nil + }, + wantErr: true, + errMsg: "encountered a network issue while deleting the clusters", + }, + { + name: "RunUnJoinCluster_UnjoinCluster_UnjoinedTheCluster", + unjoinOpts: &CommandUnjoinOption{ + ClusterName: "member1", + ClusterNamespace: options.DefaultKarmadaClusterNamespace, + forceDeletion: false, + Wait: time.Minute, + }, + controlKubeClient: fakeclientset.NewClientset(), + karmadaClient: fakekarmadaclient.NewSimpleClientset(), + clusterKubeClient: fakeclientset.NewClientset(), + clusterConfig: &rest.Config{}, + prep: func(controlKubeClient, clusterKubeClient kubeclient.Interface, karmadaClient karmadaclientset.Interface, opts *CommandUnjoinOption) error { + return prepUnjoinCluster(opts, controlKubeClient, clusterKubeClient, karmadaClient) + }, + verify: func(_ kubeclient.Interface, clusterKubeClient kubeclient.Interface, karmadaClient karmadaclientset.Interface, opts *CommandUnjoinOption) error { + if err := verifyClusterObjectDeleted(karmadaClient, opts); err != nil { + return err + } + if err := verifyRBACResourcesDeleted(clusterKubeClient, opts.ClusterName); err != nil { + return err + } + if err := verifyServiceAccountDeleted(clusterKubeClient, opts.ClusterName, opts.ClusterNamespace); err != nil { + return err + } + if err := verifyNamespaceDeleted(clusterKubeClient, opts.ClusterName, opts.ClusterNamespace); err != nil { + return err + } + return nil + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if err := test.prep(test.controlKubeClient, test.clusterKubeClient, test.karmadaClient, test.unjoinOpts); err != nil { + t.Fatalf("failed to prep test environment, got error: %v", err) + } + err := test.unjoinOpts.RunUnJoinCluster(test.controlPlaneRestConfig, test.clusterConfig) + if err == nil && test.wantErr { + t.Fatal("expected an error, but got none") + } + if err != nil && !test.wantErr { + t.Errorf("unexpected error, got: %v", err) + } + if err != nil && test.wantErr && !strings.Contains(err.Error(), test.errMsg) { + t.Errorf("expected error message %s to be in %s", test.errMsg, err.Error()) + } + if err := test.verify(test.controlKubeClient, test.clusterKubeClient, test.karmadaClient, test.unjoinOpts); err != nil { + t.Errorf("failed to verify unjoining the cluster %s, got error: %v", test.unjoinOpts.ClusterName, err) + } + }) + } +} + +func prepUnjoinCluster(opts *CommandUnjoinOption, controlKubeClient, clusterKubeClient kubeclient.Interface, karmadaClient karmadaclientset.Interface) error { + // Create cluster object on karmada client. + cluster := &clusterv1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: opts.ClusterName, + }, + } + if _, err := karmadaClient.ClusterV1alpha1().Clusters().Create(context.TODO(), cluster, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("failed to create cluster %s, got error: %v", opts.ClusterName, err) + } + if err := createNamespace(clusterKubeClient, cluster.GetName(), opts.ClusterNamespace); err != nil { + return err + } + if err := createServiceAccount(clusterKubeClient, cluster.GetName(), opts.ClusterNamespace); err != nil { + return err + } + if err := createRBACResources(clusterKubeClient, cluster.GetName()); err != nil { + return err + } + controlPlaneKarmadaClientBuilder = func(*rest.Config) karmadaclientset.Interface { + return karmadaClient + } + controlPlaneKubeClientBuilder = func(*rest.Config) kubeclient.Interface { + return controlKubeClient + } + clusterKubeClientBuilder = func(*rest.Config) kubeclient.Interface { + return clusterKubeClient + } + return nil +} + +func createRBACResources(clusterKubeClient kubeclient.Interface, unjoiningClusterName string) error { + clusterRole := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: names.GenerateRoleName(unjoiningClusterName), + }, + } + if _, err := util.CreateClusterRole(clusterKubeClient, clusterRole); err != nil { + return fmt.Errorf("failed to create cluster role %s in unjoining cluster %s, got error: %v", clusterRole.GetName(), unjoiningClusterName, err) + } + + serviceAccountName := names.GenerateServiceAccountName(unjoiningClusterName) + clusterRoleBinding := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: names.GenerateRoleName(serviceAccountName), + }, + RoleRef: rbacv1.RoleRef{Name: clusterRole.GetName()}, + } + if _, err := util.CreateClusterRoleBinding(clusterKubeClient, clusterRoleBinding); err != nil { + return fmt.Errorf("failed to create cluster role binding %s in unjoining cluster %s, got error: %v", clusterRoleBinding.GetName(), unjoiningClusterName, err) + } + + return nil +} + +func createServiceAccount(clusterKubeClient kubeclient.Interface, unjoiningClusterName, namespace string) error { + serviceAccountObj := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: names.GenerateServiceAccountName(unjoiningClusterName), + Namespace: namespace, + }, + } + if _, err := clusterKubeClient.CoreV1().ServiceAccounts(namespace).Create(context.TODO(), serviceAccountObj, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("failed to create service account %s in unjoining cluster %s, got error: %v", serviceAccountObj.GetName(), unjoiningClusterName, err) + } + return nil +} + +func createNamespace(clusterKubeClient kubeclient.Interface, unjoiningClusterName, namespace string) error { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, + } + if _, err := clusterKubeClient.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("failed to create namespace %s in unjoining cluster %s, got error: %v", namespace, unjoiningClusterName, err) + } + return nil +} + +func verifyClusterObjectDeleted(karmadaClient karmadaclientset.Interface, opts *CommandUnjoinOption) error { + if _, err := karmadaClient.ClusterV1alpha1().Clusters().Get(context.TODO(), opts.ClusterName, metav1.GetOptions{}); err == nil { + return fmt.Errorf("expected cluster %s to be deleted, but still it is found", opts.ClusterName) + } + return nil +} + +func verifyRBACResourcesDeleted(clusterKubeClient kubeclient.Interface, unjoiningClusterName string) error { + serviceAccountName := names.GenerateServiceAccountName(unjoiningClusterName) + clusterRoleName := names.GenerateRoleName(serviceAccountName) + clusterRoleBindingName := clusterRoleName + if err := clusterKubeClient.RbacV1().ClusterRoleBindings().Delete(context.TODO(), clusterRoleBindingName, metav1.DeleteOptions{}); err == nil { + return fmt.Errorf("expected cluster role binding %s in unjoining cluster %s to be deleted, but it is still found", clusterRoleBindingName, unjoiningClusterName) + } + if err := clusterKubeClient.RbacV1().ClusterRoles().Delete(context.TODO(), clusterRoleName, metav1.DeleteOptions{}); err == nil { + return fmt.Errorf("expected cluster role name %s in unjoining cluster %s to be deleted, but it is still found", clusterRoleName, unjoiningClusterName) + } + return nil +} + +func verifyServiceAccountDeleted(clusterKubeClient kubeclient.Interface, unjoiningClusterName, namespace string) error { + serviceAccountName := names.GenerateServiceAccountName(unjoiningClusterName) + if err := clusterKubeClient.CoreV1().ServiceAccounts(namespace).Delete(context.TODO(), serviceAccountName, metav1.DeleteOptions{}); err == nil { + return fmt.Errorf("expected service account %s in unjoining cluster %s to be deleted, but it is still found", serviceAccountName, unjoiningClusterName) + } + return nil +} + +func verifyNamespaceDeleted(clusterKubeClient kubeclient.Interface, unjoiningClusterName, namespace string) error { + if err := clusterKubeClient.CoreV1().Namespaces().Delete(context.TODO(), namespace, metav1.DeleteOptions{}); err == nil { + return fmt.Errorf("expected namespace %s in unjoining cluster %s to be deleted, but it is still found", namespace, unjoiningClusterName) + } + return nil +}