Skip to content

Commit 1ba15a0

Browse files
vishesh92vishesh92
committed
Move CKS integration to a separate controller
1 parent 3f13fb5 commit 1ba15a0

10 files changed

+456
-71
lines changed

‎controllers/cks_cluster_controller.go

+122
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controllers
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"strings"
23+
24+
ctrl "sigs.k8s.io/controller-runtime"
25+
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
26+
27+
infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3"
28+
csCtrlrUtils "sigs.k8s.io/cluster-api-provider-cloudstack/controllers/utils"
29+
)
30+
31+
const CksClusterFinalizer = "ckscluster.infrastructure.cluster.x-k8s.io"
32+
33+
// RBAC permissions for CloudStackCluster.
34+
// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackclusters,verbs=get;list;watch;create;update;patch;delete
35+
// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackclusters/status,verbs=create;get;update;patch
36+
// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackclusters/finalizers,verbs=update
37+
38+
// CksClusterReconciliationRunner is a ReconciliationRunner with extensions specific to CloudStackClusters.
39+
// The runner does the actual reconciliation.
40+
type CksClusterReconciliationRunner struct {
41+
*csCtrlrUtils.ReconciliationRunner
42+
FailureDomains *infrav1.CloudStackFailureDomainList
43+
ReconciliationSubject *infrav1.CloudStackCluster
44+
}
45+
46+
// CksClusterReconciler is the k8s controller manager's interface to reconcile a CloudStackCluster.
47+
// This is primarily to adapt to k8s.
48+
type CksClusterReconciler struct {
49+
csCtrlrUtils.ReconcilerBase
50+
}
51+
52+
// Initialize a new CloudStackCluster reconciliation runner with concrete types and initialized member fields.
53+
func NewCksClusterReconciliationRunner() *CksClusterReconciliationRunner {
54+
// Set concrete type and init pointers.
55+
runner := &CksClusterReconciliationRunner{ReconciliationSubject: &infrav1.CloudStackCluster{}}
56+
runner.FailureDomains = &infrav1.CloudStackFailureDomainList{}
57+
// Setup the base runner. Initializes pointers and links reconciliation methods.
58+
runner.ReconciliationRunner = csCtrlrUtils.NewRunner(runner, runner.ReconciliationSubject, "CKSClusterController")
59+
runner.CSCluster = runner.ReconciliationSubject
60+
return runner
61+
}
62+
63+
// Reconcile is the method k8s will call upon a reconciliation request.
64+
func (reconciler *CksClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (retRes ctrl.Result, retErr error) {
65+
r := NewCksClusterReconciliationRunner()
66+
r.UsingBaseReconciler(reconciler.ReconcilerBase).ForRequest(req).WithRequestCtx(ctx)
67+
r.WithAdditionalCommonStages(r.GetFailureDomains(r.FailureDomains))
68+
return r.RunBaseReconciliationStages()
69+
}
70+
71+
// Reconcile actually reconciles the CloudStackCluster.
72+
func (r *CksClusterReconciliationRunner) Reconcile() (res ctrl.Result, reterr error) {
73+
if r.CSCluster.Spec.SyncWithACS {
74+
// Prevent premature deletion.
75+
controllerutil.AddFinalizer(r.ReconciliationSubject, CksClusterFinalizer)
76+
77+
if len(r.FailureDomains.Items) == 0 {
78+
return r.RequeueWithMessage("No failure domains found")
79+
}
80+
81+
res, err := r.AsFailureDomainUser(&r.FailureDomains.Items[0].Spec)()
82+
if r.ShouldReturn(res, err) {
83+
return res, err
84+
}
85+
86+
r.Log.Info("Creating entry with CKS")
87+
err = r.CSUser.GetOrCreateUnmanagedCluster(r.CAPICluster, r.ReconciliationSubject, &r.FailureDomains.Items[0].Spec)
88+
if err != nil {
89+
if strings.Contains(err.Error(), "Kubernetes Service plugin is disabled") {
90+
r.Log.Info("Kubernetes Service plugin is disabled on CloudStack. Skipping ExternalManaged kubernetes cluster creation")
91+
return ctrl.Result{}, nil
92+
}
93+
r.Log.Info(fmt.Sprintf("Failed creating ExternalManaged kubernetes cluster on CloudStack. Error: %s", err.Error()))
94+
return r.RequeueWithMessage(fmt.Sprintf("Syncing VMs with CloudStack failed. error: %s", err.Error()))
95+
}
96+
}
97+
return ctrl.Result{}, nil
98+
}
99+
100+
// ReconcileDelete cleans up resources used by the cluster and finally removes the CloudStackCluster's finalizers.
101+
func (r *CksClusterReconciliationRunner) ReconcileDelete() (ctrl.Result, error) {
102+
res, err := r.AsFailureDomainUser(&r.FailureDomains.Items[0].Spec)()
103+
if r.ShouldReturn(res, err) {
104+
return res, err
105+
}
106+
err = r.CSUser.DeleteUnmanagedCluster(r.ReconciliationSubject)
107+
if err != nil {
108+
if strings.Contains(err.Error(), " not found") {
109+
return ctrl.Result{}, nil
110+
}
111+
return r.RequeueWithMessage(fmt.Sprintf("Deleting unmanaged kubernetes cluster on CloudStack failed. error: %s", err.Error()))
112+
}
113+
controllerutil.RemoveFinalizer(r.ReconciliationSubject, CksClusterFinalizer)
114+
return ctrl.Result{}, nil
115+
}
116+
117+
// SetupWithManager sets up the controller with the Manager.
118+
func (reconciler *CksClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
119+
return ctrl.NewControllerManagedBy(mgr).
120+
For(&infrav1.CloudStackCluster{}).
121+
Complete(reconciler)
122+
}

‎controllers/cks_cluster_controller_test.go

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controllers_test
18+
19+
import (
20+
"github.com/golang/mock/gomock"
21+
. "github.com/onsi/ginkgo/v2"
22+
. "github.com/onsi/gomega"
23+
infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3"
24+
"sigs.k8s.io/cluster-api-provider-cloudstack/controllers"
25+
"sigs.k8s.io/cluster-api-provider-cloudstack/pkg/cloud"
26+
dummies "sigs.k8s.io/cluster-api-provider-cloudstack/test/dummies/v1beta3"
27+
"sigs.k8s.io/controller-runtime/pkg/client"
28+
"sigs.k8s.io/controller-runtime/pkg/controller"
29+
)
30+
31+
var _ = Describe("CksCloudStackClusterReconciler", func() {
32+
Context("With k8s like test environment.", func() {
33+
BeforeEach(func() {
34+
dummies.SetDummyVars()
35+
SetupTestEnvironment()
36+
Ω(ClusterReconciler.SetupWithManager(ctx, k8sManager, controller.Options{})).Should(Succeed()) // Register CloudStack ClusterReconciler.
37+
Ω(FailureDomainReconciler.SetupWithManager(k8sManager, controller.Options{})).Should(Succeed()) // Register CloudStack FailureDomainReconciler.
38+
Ω(CksClusterReconciler.SetupWithManager(k8sManager)).Should(Succeed()) // Register CloudStack Cks ClusterReconciler.
39+
mockCloudClient.EXPECT().GetOrCreateUnmanagedCluster(gomock.Any(), gomock.Any(), gomock.Any()).Do(func(_, arg1, _ interface{}) {
40+
arg1.(*infrav1.CloudStackCluster).Status.CloudStackClusterID = "cluster-id-123"
41+
}).MinTimes(1).Return(nil)
42+
mockCloudClient.EXPECT().ResolveZone(gomock.Any()).AnyTimes()
43+
mockCloudClient.EXPECT().ResolveNetworkForZone(gomock.Any()).AnyTimes().Do(
44+
func(arg1 interface{}) {
45+
arg1.(*infrav1.CloudStackZoneSpec).Network.ID = "SomeID"
46+
arg1.(*infrav1.CloudStackZoneSpec).Network.Type = cloud.NetworkTypeShared
47+
}).MinTimes(1)
48+
})
49+
50+
It("Should create a cluster in CKS.", func() {
51+
Eventually(func() string {
52+
key := client.ObjectKeyFromObject(dummies.CSCluster)
53+
if err := k8sClient.Get(ctx, key, dummies.CSCluster); err != nil {
54+
return ""
55+
}
56+
return dummies.CSCluster.Status.CloudStackClusterID
57+
}, timeout).WithPolling(pollInterval).Should(Equal("cluster-id-123"))
58+
59+
})
60+
61+
})
62+
63+
Context("Without a k8s test environment.", func() {
64+
It("Should create a reconciliation runner with a Cloudstack Cluster as the reconciliation subject.", func() {
65+
reconRunner := controllers.NewCksClusterReconciliationRunner()
66+
Ω(reconRunner.ReconciliationSubject).ShouldNot(BeNil())
67+
})
68+
})
69+
})

‎controllers/cks_machine_controller.go

+108
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controllers
18+
19+
import (
20+
"context"
21+
"fmt"
22+
23+
ctrl "sigs.k8s.io/controller-runtime"
24+
25+
infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3"
26+
csCtrlrUtils "sigs.k8s.io/cluster-api-provider-cloudstack/controllers/utils"
27+
)
28+
29+
// RBAC permissions for CloudStackCluster.
30+
// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackmachines,verbs=get;list;watch
31+
// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackmachines/status,verbs=get;update;patch
32+
// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackmachines/finalizers,verbs=update
33+
34+
// CksMachineReconciliationRunner is a ReconciliationRunner with extensions specific to CloudStackClusters.
35+
// The runner does the actual reconciliation.
36+
type CksMachineReconciliationRunner struct {
37+
*csCtrlrUtils.ReconciliationRunner
38+
FailureDomain *infrav1.CloudStackFailureDomain
39+
ReconciliationSubject *infrav1.CloudStackMachine
40+
}
41+
42+
// CksMachineReconciler is the k8s controller manager's interface to reconcile a CloudStackCluster.
43+
// This is primarily to adapt to k8s.
44+
type CksMachineReconciler struct {
45+
csCtrlrUtils.ReconcilerBase
46+
}
47+
48+
// Initialize a new CloudStackCluster reconciliation runner with concrete types and initialized member fields.
49+
func NewCksMachineReconciliationRunner() *CksMachineReconciliationRunner {
50+
// Set concrete type and init pointers.
51+
runner := &CksMachineReconciliationRunner{ReconciliationSubject: &infrav1.CloudStackMachine{}}
52+
runner.FailureDomain = &infrav1.CloudStackFailureDomain{}
53+
// Setup the base runner. Initializes pointers and links reconciliation methods.
54+
runner.ReconciliationRunner = csCtrlrUtils.NewRunner(runner, runner.ReconciliationSubject, "CKSMachineController")
55+
return runner
56+
}
57+
58+
// Reconcile is the method k8s will call upon a reconciliation request.
59+
func (reconciler *CksMachineReconciler) Reconcile(ctx context.Context, req ctrl.Request) (retRes ctrl.Result, retErr error) {
60+
r := NewCksMachineReconciliationRunner()
61+
r.UsingBaseReconciler(reconciler.ReconcilerBase).ForRequest(req).WithRequestCtx(ctx)
62+
r.WithAdditionalCommonStages(
63+
r.GetFailureDomainByName(func() string { return r.ReconciliationSubject.Spec.FailureDomainName }, r.FailureDomain),
64+
r.AsFailureDomainUser(&r.FailureDomain.Spec))
65+
return r.RunBaseReconciliationStages()
66+
}
67+
68+
// Reconcile actually reconciles the CloudStackCluster.
69+
func (r *CksMachineReconciliationRunner) Reconcile() (res ctrl.Result, reterr error) {
70+
if r.CSCluster.Spec.SyncWithACS {
71+
if r.CSCluster.Status.CloudStackClusterID == "" {
72+
return r.RequeueWithMessage("CloudStackClusterID is not set")
73+
}
74+
75+
if r.ReconciliationSubject.Spec.InstanceID == nil || *r.ReconciliationSubject.Spec.InstanceID == "" {
76+
return r.RequeueWithMessage("InstanceID is not set")
77+
}
78+
79+
res, err := r.AsFailureDomainUser(&r.FailureDomain.Spec)()
80+
if r.ShouldReturn(res, err) {
81+
return res, err
82+
}
83+
r.Log.Info("Assigning VM to CKS")
84+
err = r.CSUser.AddVMToUnmanagedCluster(r.CSCluster, r.ReconciliationSubject)
85+
if err != nil {
86+
return r.RequeueWithMessage(fmt.Sprintf("Adding VM to CloudStack Unmanaged kubernetes failed. error: %s", err.Error()))
87+
}
88+
}
89+
return ctrl.Result{}, nil
90+
}
91+
92+
// ReconcileDelete cleans up resources used by the cluster and finally removes the CloudStackCluster's finalizers.
93+
func (r *CksMachineReconciliationRunner) ReconcileDelete() (ctrl.Result, error) {
94+
if r.ReconciliationSubject.Spec.InstanceID != nil && *r.ReconciliationSubject.Spec.InstanceID != "" {
95+
err := r.CSUser.RemoveVMFromUnmanagedCluster(r.CSCluster, r.ReconciliationSubject)
96+
if err != nil {
97+
return r.RequeueWithMessage(fmt.Sprintf("Removing VM from CloudStack Unmanaged kubernetes failed. error: %s", err.Error()))
98+
}
99+
}
100+
return ctrl.Result{}, nil
101+
}
102+
103+
// SetupWithManager sets up the controller with the Manager.
104+
func (reconciler *CksMachineReconciler) SetupWithManager(mgr ctrl.Manager) error {
105+
return ctrl.NewControllerManagedBy(mgr).
106+
For(&infrav1.CloudStackMachine{}).
107+
Complete(reconciler)
108+
}

0 commit comments

Comments
 (0)