Skip to content

Commit

Permalink
Merge pull request #4363 from jwcesign/jw-1
Browse files Browse the repository at this point in the history 
e2e: Fix the occasional failure of mcs e2e
  • Loading branch information
@karmada-bot
karmada-bot authored Dec 5, 2023
2 parents f10872b + 0beb460 commit d9f47af
Showing 1 changed file with 179 additions and 169 deletions.
348 changes: 179 additions & 169 deletions test/e2e/search_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,17 +74,6 @@ var _ = ginkgo.Describe("[karmada-search] karmada search testing", ginkgo.Ordere
var pathNSDeploymentsFmt = pathPrefix + "apis/apps/v1/namespaces/%s/deployments"
// var pathWithLabel = pathPrefix + "apis/apps/v1/namespaces/kube-system/deployments?labelSelector=k8s-app=kube-dns"

// var pollTimeout = 30 * time.Second
var searchObject = func(path, target string, exists bool) {
gomega.Eventually(func(g gomega.Gomega) (bool, error) {
res := karmadaClient.SearchV1alpha1().RESTClient().Get().AbsPath(path).Do(context.TODO())
g.Expect(res.Error()).ShouldNot(gomega.HaveOccurred())
raw, err := res.Raw()
g.Expect(err).ShouldNot(gomega.HaveOccurred())
return strings.Contains(string(raw), target), nil
}, pollTimeout, pollInterval).Should(gomega.Equal(exists))
}

ginkgo.BeforeAll(func() {
// get clusters' name
pushModeClusters := framework.ClusterNamesWithSyncMode(clusterv1alpha1.Push)
Expand Down Expand Up @@ -733,193 +722,214 @@ var _ = ginkgo.Describe("[karmada-search] karmada search testing", ginkgo.Ordere
})
})
})
})

// test when cluster joined, updated, karmada will reconcile search cache
ginkgo.Describe("reconcile ResourceRegistry when clusters joined, updated", ginkgo.Ordered, func() {
ginkgo.Context("when cluster joined", func() {
var (
clusterName string
homeDir string
kubeConfigPath string
controlPlane string
clusterContext string
f cmdutil.Factory

rrName string
rr *searchv1alpha1.ResourceRegistry
resourceSelector searchv1alpha1.ResourceSelector
)
// test when cluster joined, updated, karmada will reconcile search cache
var _ = framework.SerialDescribe("reconcile ResourceRegistry when clusters joined, updated", func() {
var member1 = "member1"
var member2 = "member2"

ginkgo.BeforeAll(func() {
clusterName = "member-e2e-" + rand.String(RandomStrLength)
homeDir = os.Getenv("HOME")
kubeConfigPath = fmt.Sprintf("%s/.kube/%s.config", homeDir, clusterName)
controlPlane = fmt.Sprintf("%s-control-plane", clusterName)
clusterContext = fmt.Sprintf("kind-%s", clusterName)
defaultConfigFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag().WithDiscoveryBurst(300).WithDiscoveryQPS(50.0)
defaultConfigFlags.Context = &karmadaContext
f = cmdutil.NewFactory(defaultConfigFlags)

// create a cluster
ginkgo.By(fmt.Sprintf("Creating cluster: %s", clusterName), func() {
err := createCluster(clusterName, kubeConfigPath, controlPlane, clusterContext)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
})
var member1PodName = "etcd-member1-control-plane"
var member2PodName = "etcd-member2-control-plane"
var existsDeploymentName = "coredns"

// create a ResourceRegistry with a non joined cluster name
rrName = resourceRegistryPrefix + rand.String(RandomStrLength)
resourceSelector = searchv1alpha1.ResourceSelector{
APIVersion: "apps/v1",
Kind: "Deployment",
}
rr = &searchv1alpha1.ResourceRegistry{
ObjectMeta: metav1.ObjectMeta{
Name: rrName,
var pathPrefix = "/apis/search.karmada.io/v1alpha1/search/cache/"
var pathAllPods = pathPrefix + "api/v1/pods"
var pathAllDeployments = pathPrefix + "apis/apps/v1/deployments"

ginkgo.Context("when cluster joined", func() {
var (
clusterName string
homeDir string
kubeConfigPath string
controlPlane string
clusterContext string
f cmdutil.Factory

rrName string
rr *searchv1alpha1.ResourceRegistry
resourceSelector searchv1alpha1.ResourceSelector
)

ginkgo.BeforeEach(func() {
clusterName = "member-e2e-" + rand.String(RandomStrLength)
homeDir = os.Getenv("HOME")
kubeConfigPath = fmt.Sprintf("%s/.kube/%s.config", homeDir, clusterName)
controlPlane = fmt.Sprintf("%s-control-plane", clusterName)
clusterContext = fmt.Sprintf("kind-%s", clusterName)
defaultConfigFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag().WithDiscoveryBurst(300).WithDiscoveryQPS(50.0)
defaultConfigFlags.Context = &karmadaContext
f = cmdutil.NewFactory(defaultConfigFlags)

// create a cluster
ginkgo.By(fmt.Sprintf("Creating cluster: %s", clusterName), func() {
err := createCluster(clusterName, kubeConfigPath, controlPlane, clusterContext)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
})

// create a ResourceRegistry with a non joined cluster name
rrName = resourceRegistryPrefix + rand.String(RandomStrLength)
resourceSelector = searchv1alpha1.ResourceSelector{
APIVersion: "apps/v1",
Kind: "Deployment",
}
rr = &searchv1alpha1.ResourceRegistry{
ObjectMeta: metav1.ObjectMeta{
Name: rrName,
},
Spec: searchv1alpha1.ResourceRegistrySpec{
TargetCluster: policyv1alpha1.ClusterAffinity{
ClusterNames: []string{clusterName},
},
Spec: searchv1alpha1.ResourceRegistrySpec{
TargetCluster: policyv1alpha1.ClusterAffinity{
ClusterNames: []string{clusterName},
},
ResourceSelectors: []searchv1alpha1.ResourceSelector{
resourceSelector,
},
ResourceSelectors: []searchv1alpha1.ResourceSelector{
resourceSelector,
},
}
// create a ResourceRegistry with a non joined cluster name
ginkgo.By(fmt.Sprintf("create ResourceRegistry %s with a non joined cluster name %v and resource selector %v", rrName, clusterName, resourceSelector), func() {
framework.CreateResourceRegistry(karmadaClient, rr)
})
},
}
// create a ResourceRegistry with a non joined cluster name
ginkgo.By(fmt.Sprintf("create ResourceRegistry %s with a non joined cluster name %v and resource selector %v", rrName, clusterName, resourceSelector), func() {
framework.CreateResourceRegistry(karmadaClient, rr)
})
})

ginkgo.AfterAll(func() {
framework.RemoveResourceRegistry(karmadaClient, rrName)
ginkgo.AfterEach(func() {
framework.RemoveResourceRegistry(karmadaClient, rrName)

ginkgo.By(fmt.Sprintf("Deleting clusters: %s", clusterName), func() {
err := deleteCluster(clusterName, kubeConfigPath)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
_ = os.Remove(kubeConfigPath)
})
ginkgo.By(fmt.Sprintf("Deleting clusters: %s", clusterName), func() {
err := deleteCluster(clusterName, kubeConfigPath)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
_ = os.Remove(kubeConfigPath)
})
})

ginkgo.It("[member clusters joined] could reconcile ResourceRegistry", func() {
// search cache should not have the deployment
searchObject(pathAllDeployments, existsDeploymentName, false)
// join the cluster
ginkgo.By(fmt.Sprintf("Joining cluster: %s", clusterName), func() {
opts := join.CommandJoinOption{
DryRun: false,
ClusterNamespace: "karmada-cluster",
ClusterName: clusterName,
ClusterContext: clusterContext,
ClusterKubeConfig: kubeConfigPath,
}
err := opts.Run(f)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
})
// search cache should have the deployment
searchObject(pathAllDeployments, existsDeploymentName, true)

ginkgo.By(fmt.Sprintf("Unjoinning cluster: %s", clusterName), func() {
opts := unjoin.CommandUnjoinOption{
DryRun: false,
ClusterNamespace: "karmada-cluster",
ClusterName: clusterName,
ClusterContext: clusterContext,
ClusterKubeConfig: kubeConfigPath,
Wait: 5 * options.DefaultKarmadactlCommandDuration,
}
err := opts.Run(f)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
})
ginkgo.It("[member clusters joined] could reconcile ResourceRegistry", func() {
// search cache should not have the deployment
searchObject(pathAllDeployments, existsDeploymentName, false)
// join the cluster
ginkgo.By(fmt.Sprintf("Joining cluster: %s", clusterName), func() {
opts := join.CommandJoinOption{
DryRun: false,
ClusterNamespace: "karmada-cluster",
ClusterName: clusterName,
ClusterContext: clusterContext,
ClusterKubeConfig: kubeConfigPath,
}
err := opts.Run(f)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
})
// search cache should have the deployment
searchObject(pathAllDeployments, existsDeploymentName, true)

ginkgo.By(fmt.Sprintf("Unjoinning cluster: %s", clusterName), func() {
opts := unjoin.CommandUnjoinOption{
DryRun: false,
ClusterNamespace: "karmada-cluster",
ClusterName: clusterName,
ClusterContext: clusterContext,
ClusterKubeConfig: kubeConfigPath,
Wait: 5 * options.DefaultKarmadactlCommandDuration,
}
err := opts.Run(f)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
})
})
})

ginkgo.Context("when cluster updated", func() {
var (
rrName string
rr *searchv1alpha1.ResourceRegistry
labelKey string
labelValue string
labelSelector *metav1.LabelSelector
resourceSelector searchv1alpha1.ResourceSelector
)
ginkgo.Context("when cluster updated", func() {
var (
rrName string
rr *searchv1alpha1.ResourceRegistry
labelKey string
labelValue string
labelSelector *metav1.LabelSelector
resourceSelector searchv1alpha1.ResourceSelector
)

ginkgo.BeforeAll(func() {
// create a ResourceRegistry with label selector
rrName = resourceRegistryPrefix + rand.String(RandomStrLength)
labelKey = "karmada-cluster.k8s.some.com/managed"
labelValue = "true"
labelSelector = &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: labelKey,
Operator: metav1.LabelSelectorOpExists,
},
ginkgo.BeforeEach(func() {
// create a ResourceRegistry with label selector
rrName = resourceRegistryPrefix + rand.String(RandomStrLength)
labelKey = "karmada-cluster.k8s.some.com/managed"
labelValue = "true"
labelSelector = &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: labelKey,
Operator: metav1.LabelSelectorOpExists,
},
}
resourceSelector = searchv1alpha1.ResourceSelector{
APIVersion: "v1",
Kind: "Pod",
}
rr = &searchv1alpha1.ResourceRegistry{
ObjectMeta: metav1.ObjectMeta{
Name: rrName,
},
}
resourceSelector = searchv1alpha1.ResourceSelector{
APIVersion: "v1",
Kind: "Pod",
}
rr = &searchv1alpha1.ResourceRegistry{
ObjectMeta: metav1.ObjectMeta{
Name: rrName,
},
Spec: searchv1alpha1.ResourceRegistrySpec{
TargetCluster: policyv1alpha1.ClusterAffinity{
LabelSelector: labelSelector,
},
Spec: searchv1alpha1.ResourceRegistrySpec{
TargetCluster: policyv1alpha1.ClusterAffinity{
LabelSelector: labelSelector,
},
ResourceSelectors: []searchv1alpha1.ResourceSelector{
resourceSelector,
},
ResourceSelectors: []searchv1alpha1.ResourceSelector{
resourceSelector,
},
}
ginkgo.By(fmt.Sprintf("create ResourceRegistry %s with label selector %v and resource selector %v", rrName, labelSelector, resourceSelector), func() {
framework.CreateResourceRegistry(karmadaClient, rr)
})
})

ginkgo.AfterAll(func() {
framework.RemoveResourceRegistry(karmadaClient, rrName)
},
}
ginkgo.By(fmt.Sprintf("create ResourceRegistry %s with label selector %v and resource selector %v", rrName, labelSelector, resourceSelector), func() {
framework.CreateResourceRegistry(karmadaClient, rr)
})
})

ginkgo.It("[member clusters updated, deleted label] could reconcile ResourceRegistry", func() {
// search the pod in the ResourceRegistry will fail
searchObject(pathAllPods, member1PodName, false)
ginkgo.AfterEach(func() {
framework.RemoveResourceRegistry(karmadaClient, rrName)
})

// add the label to the member1 cluster
ginkgo.By(fmt.Sprintf("add label %s=%s to cluster %s", labelKey, labelValue, member1), func() {
framework.UpdateClusterLabels(karmadaClient, member1, map[string]string{labelKey: labelValue})
})
ginkgo.It("[member clusters updated, deleted label] could reconcile ResourceRegistry", func() {
// search the pod in the ResourceRegistry will fail
searchObject(pathAllPods, member1PodName, false)

// search the pod in the ResourceRegistry will success
searchObject(pathAllPods, member1PodName, true)
// add the label to the member1 cluster
ginkgo.By(fmt.Sprintf("add label %s=%s to cluster %s", labelKey, labelValue, member1), func() {
framework.UpdateClusterLabels(karmadaClient, member1, map[string]string{labelKey: labelValue})
})

// add the label to the member2 cluster
ginkgo.By(fmt.Sprintf("add label %s=%s to cluster %s", labelKey, labelValue, member2), func() {
framework.UpdateClusterLabels(karmadaClient, member2, map[string]string{labelKey: labelValue})
})
// search the pod in the ResourceRegistry will success
searchObject(pathAllPods, member1PodName, true)

// search the pod in the ResourceRegistry will success
searchObject(pathAllPods, member2PodName, true)
// add the label to the member2 cluster
ginkgo.By(fmt.Sprintf("add label %s=%s to cluster %s", labelKey, labelValue, member2), func() {
framework.UpdateClusterLabels(karmadaClient, member2, map[string]string{labelKey: labelValue})
})

// delete the label of the member2 cluster
ginkgo.By(fmt.Sprintf("delete label %s=%s to cluster %s", labelKey, labelValue, member2), func() {
framework.DeleteClusterLabels(karmadaClient, member2, map[string]string{labelKey: labelValue})
})
// search the pod in the ResourceRegistry will success
searchObject(pathAllPods, member2PodName, true)

// search the pod in the ResourceRegistry will fail
searchObject(pathAllPods, member2PodName, false)
// delete the label of the member2 cluster
ginkgo.By(fmt.Sprintf("delete label %s=%s to cluster %s", labelKey, labelValue, member2), func() {
framework.DeleteClusterLabels(karmadaClient, member2, map[string]string{labelKey: labelValue})
})

// delete the label of the member1 cluster
ginkgo.By(fmt.Sprintf("delete label %s=%s to cluster %s", labelKey, labelValue, member1), func() {
framework.DeleteClusterLabels(karmadaClient, member1, map[string]string{labelKey: labelValue})
// search the pod in the ResourceRegistry will fail
searchObject(pathAllPods, member2PodName, false)

})
// delete the label of the member1 cluster
ginkgo.By(fmt.Sprintf("delete label %s=%s to cluster %s", labelKey, labelValue, member1), func() {
framework.DeleteClusterLabels(karmadaClient, member1, map[string]string{labelKey: labelValue})

// search the pod in the ResourceRegistry will fail
searchObject(pathAllPods, member1PodName, false)
})

// search the pod in the ResourceRegistry will fail
searchObject(pathAllPods, member1PodName, false)
})
})
})

func searchObject(path, target string, exists bool) {
gomega.Eventually(func(g gomega.Gomega) (bool, error) {
res := karmadaClient.SearchV1alpha1().RESTClient().Get().AbsPath(path).Do(context.TODO())
g.Expect(res.Error()).ShouldNot(gomega.HaveOccurred())
raw, err := res.Raw()
g.Expect(err).ShouldNot(gomega.HaveOccurred())
return strings.Contains(string(raw), target), nil
}, pollTimeout, pollInterval).Should(gomega.Equal(exists))
}

0 comments on commit d9f47af

Please sign in to comment.