From 737d8e6e5be97c3a15699b01d3f40dfb961d4ef3 Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Thu, 14 Mar 2024 14:29:34 +0530 Subject: [PATCH] node selector template (#95) * refactor e2e struct * Revert "refactor e2e struct" This reverts commit 92712243e48ee5703902a094319e9b7bdf061212. * simplify lifecycle test wrapper * add expected status over lifetime * add new taint and label for test minikube * add nodeselector and toleration to status * use expected status from lifetime status input * test with different taints and labels * write a little loop to check tolerations * update helper script and makefile with new tolerations * use nodeselector template * merge template and given * add gvisor test * run copied test * rename tests * naming * propogate nodeselector and tolerations to the helm chart * update status update for nodeselector and tolerations * ensure test namespaces are deleted --- .../nodeselector-pod-toleration.yaml | 6 +- .../nodeselector-toleration.yaml | 8 +- Makefile | 8 +- ....uffizzi.com_customresourcedefinition.yaml | 28 +++++++ hack/minikube-patch-pod-tolerations.sh | 2 +- src/api/v1alpha1/uffizzicluster_types.go | 25 +++--- src/api/v1alpha1/zz_generated.deepcopy.go | 14 ++++ .../bases/uffizzi.com_uffizziclusters.yaml | 45 ++++++++++ src/controllers/uffizzicluster/helm.go | 3 +- src/pkg/helm/build/vcluster/build.go | 82 +++++++++++++++++-- src/pkg/helm/types/vcluster/tolerations.go | 36 ++++++++ src/pkg/helm/types/vcluster/vcluster.go | 6 -- src/test/e2e/suite_test.go | 33 ++++++-- src/test/e2e/ucluster_lifecycle_test.go | 34 ++++++-- .../e2e/uffizzicluster_controller_test.go | 67 +++++++++------ 15 files changed, 320 insertions(+), 77 deletions(-) create mode 100644 src/pkg/helm/types/vcluster/tolerations.go diff --git a/.github/kubectl-patch/nodeselector-pod-toleration.yaml b/.github/kubectl-patch/nodeselector-pod-toleration.yaml index f9b04fef..fb65b742 100644 --- a/.github/kubectl-patch/nodeselector-pod-toleration.yaml +++ b/.github/kubectl-patch/nodeselector-pod-toleration.yaml @@ -1,8 +1,8 @@ spec: tolerations: - - key: "testkey" + - key: "sandbox.gke.io/runtime" operator: "Equal" - value: "testvalue" + value: "gvisor" effect: "NoSchedule" nodeSelector: - testkey: "testvalue" + sandbox.gke.io/runtime: "gvisor" diff --git a/.github/kubectl-patch/nodeselector-toleration.yaml b/.github/kubectl-patch/nodeselector-toleration.yaml index 51bb76d7..d4b69275 100644 --- a/.github/kubectl-patch/nodeselector-toleration.yaml +++ b/.github/kubectl-patch/nodeselector-toleration.yaml @@ -1,10 +1,10 @@ spec: template: spec: - nodeSelector: - testkey: "testvalue" tolerations: - - key: "testkey" + - key: "sandbox.gke.io/runtime" operator: "Equal" - value: "testvalue" + value: "gvisor" effect: "NoSchedule" + nodeSelector: + sandbox.gke.io/runtime: "gvisor" diff --git a/Makefile b/Makefile index abf64832..5486d919 100644 --- a/Makefile +++ b/Makefile @@ -117,7 +117,7 @@ install-fluxcd-controllers: install-flux-prereq ## Install the fluxcd controller .PHONE: install-fluxcd-controllers-with-toleration install-fluxcd-controllers-with-toleration: install-flux-prereq ## Install the fluxcd controllers with toleration. - flux install --namespace=flux-system --components="source-controller,helm-controller" --toleration-keys="testkey" --network-policy=false --insecure-skip-tls-verify + flux install --namespace=flux-system --components="source-controller,helm-controller" --toleration-keys="sandbox.gke.io/runtime" --network-policy=false --insecure-skip-tls-verify .PHONY: start-test-k3d start-test-k3d: ## Start a k3d cluster for testing. @@ -137,10 +137,10 @@ stop-test-minikube: ## Stop the minikube cluster for testing. .PHONY: start-test-minikube-tainted start-test-minikube-tainted: ## Start a minikube cluster with a tainted node for testing. - minikube start --addons default-storageclass,storage-provisioner,hostpat --driver=docker + minikube start --addons default-storageclass,storage-provisioner --driver=docker sh ./hack/minikube-patch-pod-tolerations.sh - kubectl taint nodes minikube testkey=testvalue:NoSchedule || true - kubectl label nodes minikube testkey=testvalue || true + kubectl taint nodes minikube sandbox.gke.io/runtime=gvisor:NoSchedule || true + kubectl label nodes minikube sandbox.gke.io/runtime=gvisor || true $(MAKE) install-fluxcd-controllers-with-toleration sh ./hack/minikube-patch-workload-tolerations.sh diff --git a/chart/templates/uffizziclusters.uffizzi.com_customresourcedefinition.yaml b/chart/templates/uffizziclusters.uffizzi.com_customresourcedefinition.yaml index 23a235b1..800a9d8f 100644 --- a/chart/templates/uffizziclusters.uffizzi.com_customresourcedefinition.yaml +++ b/chart/templates/uffizziclusters.uffizzi.com_customresourcedefinition.yaml @@ -152,6 +152,8 @@ spec: additionalProperties: type: string type: object + nodeSelectorTemplate: + type: string resourceQuota: description: UffizziClusterResourceQuota defines the resource quota which defines the quota of resources a namespace has access to properties: @@ -325,6 +327,32 @@ spec: lastAwakeTime: format: date-time type: string + nodeSelector: + additionalProperties: + type: string + type: object + tolerations: + items: + description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . + properties: + effect: + description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array type: object type: object served: true diff --git a/hack/minikube-patch-pod-tolerations.sh b/hack/minikube-patch-pod-tolerations.sh index 24cbd8ca..15eda6ea 100755 --- a/hack/minikube-patch-pod-tolerations.sh +++ b/hack/minikube-patch-pod-tolerations.sh @@ -1,5 +1,5 @@ kubectl get pods --all-namespaces -o jsonpath="{range .items[*]}{.metadata.namespace}{' '}{.metadata.name}{'\n'}{end}" | while read -r line; do namespace=$(echo "$line" | cut -d' ' -f1) pod=$(echo "$line" | cut -d' ' -f2) - kubectl patch pod "$pod" -n "$namespace" --type='json' -p='[{"op": "add", "path": "/spec/tolerations", "value": [{"key": "testkey", "operator": "Equal", "value": "testvalue", "effect": "NoSchedule"}]}]' + kubectl patch pod "$pod" -n "$namespace" --type='json' -p='[{"op": "add", "path": "/spec/tolerations", "value": [{"key": "sandbox.gke.io/runtime", "operator": "Equal", "value": "gvisor", "effect": "NoSchedule"}]}]' || true done \ No newline at end of file diff --git a/src/api/v1alpha1/uffizzicluster_types.go b/src/api/v1alpha1/uffizzicluster_types.go index bc488e71..afbec6cd 100644 --- a/src/api/v1alpha1/uffizzicluster_types.go +++ b/src/api/v1alpha1/uffizzicluster_types.go @@ -151,17 +151,18 @@ type UffizziClusterStorage struct { type UffizziClusterSpec struct { //+kubebuilder:default:="k3s" //+kubebuilder:validation:Enum=k3s;k8s - Distro string `json:"distro,omitempty"` - NodeSelector map[string]string `json:"nodeSelector,omitempty"` - Toleration []v1.Toleration `json:"tolerations,omitempty"` - APIServer UffizziClusterAPIServer `json:"apiServer,omitempty"` - Ingress UffizziClusterIngress `json:"ingress,omitempty"` - Helm []HelmChart `json:"helm,omitempty"` - Manifests *string `json:"manifests,omitempty"` - ResourceQuota *UffizziClusterResourceQuota `json:"resourceQuota,omitempty"` - LimitRange *UffizziClusterLimitRange `json:"limitRange,omitempty"` - Sleep bool `json:"sleep,omitempty"` - Storage *UffizziClusterStorage `json:"storage,omitempty"` + Distro string `json:"distro,omitempty"` + NodeSelectorTemplate string `json:"nodeSelectorTemplate,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + Toleration []v1.Toleration `json:"tolerations,omitempty"` + APIServer UffizziClusterAPIServer `json:"apiServer,omitempty"` + Ingress UffizziClusterIngress `json:"ingress,omitempty"` + Helm []HelmChart `json:"helm,omitempty"` + Manifests *string `json:"manifests,omitempty"` + ResourceQuota *UffizziClusterResourceQuota `json:"resourceQuota,omitempty"` + LimitRange *UffizziClusterLimitRange `json:"limitRange,omitempty"` + Sleep bool `json:"sleep,omitempty"` + Storage *UffizziClusterStorage `json:"storage,omitempty"` //+kubebuilder:default:="sqlite" //+kubebuilder:validation:Enum=etcd;sqlite ExternalDatastore string `json:"externalDatastore,omitempty"` @@ -176,6 +177,8 @@ type UffizziClusterStatus struct { LastAppliedConfiguration *string `json:"lastAppliedConfiguration,omitempty"` LastAppliedHelmReleaseSpec *string `json:"lastAppliedHelmReleaseSpec,omitempty"` LastAwakeTime metav1.Time `json:"lastAwakeTime,omitempty"` + Tolerations []v1.Toleration `json:"tolerations,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` } // VClusterKubeConfig is the KubeConfig SecretReference of the related VCluster diff --git a/src/api/v1alpha1/zz_generated.deepcopy.go b/src/api/v1alpha1/zz_generated.deepcopy.go index 6ddbceaf..ee45ebb3 100644 --- a/src/api/v1alpha1/zz_generated.deepcopy.go +++ b/src/api/v1alpha1/zz_generated.deepcopy.go @@ -393,6 +393,20 @@ func (in *UffizziClusterStatus) DeepCopyInto(out *UffizziClusterStatus) { **out = **in } in.LastAwakeTime.DeepCopyInto(&out.LastAwakeTime) + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UffizziClusterStatus. diff --git a/src/config/crd/bases/uffizzi.com_uffizziclusters.yaml b/src/config/crd/bases/uffizzi.com_uffizziclusters.yaml index 57c91498..bb37b6f6 100644 --- a/src/config/crd/bases/uffizzi.com_uffizziclusters.yaml +++ b/src/config/crd/bases/uffizzi.com_uffizziclusters.yaml @@ -158,6 +158,8 @@ spec: additionalProperties: type: string type: object + nodeSelectorTemplate: + type: string resourceQuota: description: UffizziClusterResourceQuota defines the resource quota which defines the quota of resources a namespace has access to @@ -380,6 +382,49 @@ spec: lastAwakeTime: format: date-time type: string + nodeSelector: + additionalProperties: + type: string + type: object + tolerations: + items: + description: The pod this Toleration is attached to tolerates any + taint that matches the triple using the matching + operator . + properties: + effect: + description: Effect indicates the taint effect to match. Empty + means match all taint effects. When specified, allowed values + are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies + to. Empty means match all taint keys. If the key is empty, + operator must be Exists; this combination means to match all + values and all keys. + type: string + operator: + description: Operator represents a key's relationship to the + value. Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod + can tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of time + the toleration (which must be of effect NoExecute, otherwise + this field is ignored) tolerates the taint. By default, it + is not set, which means tolerate the taint forever (do not + evict). Zero and negative values will be treated as 0 (evict + immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches + to. If the operator is Exists, the value should be empty, + otherwise just a regular string. + type: string + type: object + type: array type: object type: object served: true diff --git a/src/controllers/uffizzicluster/helm.go b/src/controllers/uffizzicluster/helm.go index 9107d2c5..04180b4d 100644 --- a/src/controllers/uffizzicluster/helm.go +++ b/src/controllers/uffizzicluster/helm.go @@ -27,6 +27,8 @@ func (r *UffizziClusterReconciler) deleteLoftHelmRepo(ctx context.Context, req c } func (r *UffizziClusterReconciler) upsertVClusterK3SHelmRelease(update bool, ctx context.Context, uCluster *uclusteruffizzicomv1alpha1.UffizziCluster) (*fluxhelmv2beta1.HelmRelease, error) { + patch := client.MergeFrom(uCluster.DeepCopy()) + vclusterK3sHelmValues, helmReleaseName := vcluster.BuildK3SHelmValues(uCluster) helmValuesJSONObj, err := build.HelmValuesToJSON(vclusterK3sHelmValues) if err != nil { @@ -77,7 +79,6 @@ func (r *UffizziClusterReconciler) upsertVClusterK3SHelmRelease(update bool, ctx return nil, errors.Wrap(err, "failed to create HelmRelease") } uCluster.Status.LastAppliedHelmReleaseSpec = &newHelmReleaseSpec - patch := client.MergeFrom(uCluster.DeepCopy()) if err := r.Status().Patch(ctx, uCluster, patch); err != nil { return nil, errors.Wrap(err, "Failed to update the default UffizziCluster lastAppliedHelmReleaseSpec") } diff --git a/src/pkg/helm/build/vcluster/build.go b/src/pkg/helm/build/vcluster/build.go index d8edd383..628ed681 100644 --- a/src/pkg/helm/build/vcluster/build.go +++ b/src/pkg/helm/build/vcluster/build.go @@ -10,11 +10,32 @@ import ( ) func BuildK3SHelmValues(uCluster *v1alpha1.UffizziCluster) (vcluster.K3S, string) { - helmReleaseName, vclusterIngressHostname, outKubeConfigServerArgValue := configStrings(uCluster) + var ( + helmReleaseName, vclusterIngressHostname, outKubeConfigServerArgValue = configStrings(uCluster) + nodeSelector = vcluster.NodeSelector{} + tolerations = []v1.Toleration{} + ) + + if uCluster.Spec.NodeSelectorTemplate == constants.GVISOR { + nodeSelector = vcluster.GvisorNodeSelector + tolerations = []v1.Toleration{vcluster.GvisorToleration.ToV1()} + } + + if len(uCluster.Spec.NodeSelector) > 0 { + // merge nodeSelector and uCluster.Spec.NodeSelector + for k, v := range uCluster.Spec.NodeSelector { + map[string]string(nodeSelector)[k] = v + } + } + + if len(uCluster.Spec.Toleration) > 0 { + // merge tolerations and uCluster.Spec.Toleration + tolerations = append(tolerations, uCluster.Spec.Toleration...) + } vclusterK3sHelmValues := vcluster.K3S{ VCluster: k3SAPIServer(uCluster), - Common: common(helmReleaseName, vclusterIngressHostname, uCluster.Spec.NodeSelector, uCluster.Spec.Toleration), + Common: common(helmReleaseName, vclusterIngressHostname, nodeSelector, tolerations), } if uCluster.Spec.ExternalDatastore == constants.ETCD { @@ -94,13 +115,21 @@ func BuildK3SHelmValues(uCluster *v1alpha1.UffizziCluster) (vcluster.K3S, string ) } - for _, t := range uCluster.Spec.Toleration { + for _, t := range tolerations { vclusterK3sHelmValues.Syncer.ExtraArgs = append(vclusterK3sHelmValues.Syncer.ExtraArgs, "--enforce-toleration="+vcluster.Toleration(t).Notation()) + if uCluster.Status.Tolerations == nil { + uCluster.Status.Tolerations = []v1.Toleration{} + } + uCluster.Status.Tolerations = append(uCluster.Status.Tolerations, t) } - if len(uCluster.Spec.NodeSelector) > 0 { - for k, v := range uCluster.Spec.NodeSelector { + if len(nodeSelector) > 0 { + for k, v := range nodeSelector { vclusterK3sHelmValues.Syncer.ExtraArgs = append(vclusterK3sHelmValues.Syncer.ExtraArgs, "--node-selector="+k+"="+v) + if uCluster.Status.NodeSelector == nil { + uCluster.Status.NodeSelector = make(map[string]string) + } + uCluster.Status.NodeSelector[k] = v } vclusterK3sHelmValues.Syncer.ExtraArgs = append(vclusterK3sHelmValues.Syncer.ExtraArgs, "--enforce-node-selector") } @@ -131,11 +160,31 @@ func BuildK3SHelmValues(uCluster *v1alpha1.UffizziCluster) (vcluster.K3S, string } func BuildK8SHelmValues(uCluster *v1alpha1.UffizziCluster) (vcluster.K8S, string) { - helmReleaseName, vclusterIngressHostname, outKubeConfigServerArgValue := configStrings(uCluster) + var ( + helmReleaseName, vclusterIngressHostname, outKubeConfigServerArgValue = configStrings(uCluster) + nodeSelector = vcluster.NodeSelector{} + tolerations = []v1.Toleration{} + ) + if uCluster.Spec.NodeSelectorTemplate == constants.GVISOR { + nodeSelector = vcluster.GvisorNodeSelector + tolerations = []v1.Toleration{vcluster.GvisorToleration.ToV1()} + } + + if len(uCluster.Spec.NodeSelector) > 0 { + // merge nodeSelector and uCluster.Spec.NodeSelector + for k, v := range uCluster.Spec.NodeSelector { + map[string]string(nodeSelector)[k] = v + } + } + + if len(uCluster.Spec.Toleration) > 0 { + // merge tolerations and uCluster.Spec.Toleration + tolerations = append(tolerations, uCluster.Spec.Toleration...) + } vclusterHelmValues := vcluster.K8S{ APIServer: k8SAPIServer(), - Common: common(helmReleaseName, vclusterIngressHostname, uCluster.Spec.NodeSelector, uCluster.Spec.Toleration), + Common: common(helmReleaseName, vclusterIngressHostname, nodeSelector, tolerations), } if uCluster.Spec.APIServer.Image != "" { @@ -210,6 +259,25 @@ func BuildK8SHelmValues(uCluster *v1alpha1.UffizziCluster) (vcluster.K8S, string ) } + for _, t := range tolerations { + vclusterHelmValues.Syncer.ExtraArgs = append(vclusterHelmValues.Syncer.ExtraArgs, "--enforce-toleration="+vcluster.Toleration(t).Notation()) + if uCluster.Status.Tolerations == nil { + uCluster.Status.Tolerations = []v1.Toleration{} + } + uCluster.Status.Tolerations = append(uCluster.Status.Tolerations, t) + } + + if len(nodeSelector) > 0 { + for k, v := range nodeSelector { + vclusterHelmValues.Syncer.ExtraArgs = append(vclusterHelmValues.Syncer.ExtraArgs, "--node-selector="+k+"="+v) + if uCluster.Status.NodeSelector == nil { + uCluster.Status.NodeSelector = make(map[string]string) + } + uCluster.Status.NodeSelector[k] = v + } + vclusterHelmValues.Syncer.ExtraArgs = append(vclusterHelmValues.Syncer.ExtraArgs, "--enforce-node-selector") + } + if len(uCluster.Spec.Helm) > 0 { vclusterHelmValues.Init.Helm = uCluster.Spec.Helm } diff --git a/src/pkg/helm/types/vcluster/tolerations.go b/src/pkg/helm/types/vcluster/tolerations.go new file mode 100644 index 00000000..a4b9f884 --- /dev/null +++ b/src/pkg/helm/types/vcluster/tolerations.go @@ -0,0 +1,36 @@ +package vcluster + +import v1 "k8s.io/api/core/v1" + +type Toleration v1.Toleration +type NodeSelector map[string]string + +func (t Toleration) Notation() string { + if t.Value == "" { + return t.Key + ":" + string(t.Effect) + } + return t.Key + "=" + t.Value + ":" + string(t.Effect) +} + +func (t Toleration) ToV1() v1.Toleration { + return v1.Toleration(t) +} + +func (n NodeSelector) Notation() string { + for k, v := range n { + return k + "=" + v + } + return "" +} + +var ( + GvisorToleration = Toleration{ + Key: "sandbox.gke.io/runtime", + Operator: v1.TolerationOpEqual, + Value: "gvisor", + Effect: v1.TaintEffectNoSchedule, + } + GvisorNodeSelector = NodeSelector{ + "sandbox.gke.io/runtime": "gvisor", + } +) diff --git a/src/pkg/helm/types/vcluster/vcluster.go b/src/pkg/helm/types/vcluster/vcluster.go index 43a42520..73fa1683 100644 --- a/src/pkg/helm/types/vcluster/vcluster.go +++ b/src/pkg/helm/types/vcluster/vcluster.go @@ -204,9 +204,3 @@ type Storage struct { Persistence bool `json:"persistence"` Size string `json:"size"` } - -type Toleration v1.Toleration - -func (t Toleration) Notation() string { - return t.Key + "=" + t.Value + ":" + string(t.Effect) -} diff --git a/src/test/e2e/suite_test.go b/src/test/e2e/suite_test.go index d218d4f6..0f5e47c4 100644 --- a/src/test/e2e/suite_test.go +++ b/src/test/e2e/suite_test.go @@ -22,7 +22,6 @@ import ( "github.com/UffizziCloud/uffizzi-cluster-operator/src/controllers/etcd" "github.com/UffizziCloud/uffizzi-cluster-operator/src/controllers/uffizzicluster" "github.com/UffizziCloud/uffizzi-cluster-operator/src/pkg/utils/exec" - "github.com/UffizziCloud/uffizzi-cluster-operator/src/test/util/resources" fluxhelmv2beta1 "github.com/fluxcd/helm-controller/api/v2beta1" fluxsourcev1beta2 "github.com/fluxcd/source-controller/api/v1beta2" . "github.com/onsi/ginkgo/v2" @@ -63,15 +62,33 @@ type UffizziClusterE2E struct { } type TestDefinition struct { - Name string - Spec uffizziv1alpha1.UffizziClusterSpec + Name string + Spec uffizziv1alpha1.UffizziClusterSpec + ExpectedStatus ExpectedStatusOverLifetime } -func (td *TestDefinition) ExecLifecycleTest(ctx context.Context) { - ns := resources.CreateTestNamespace(td.Name) - uc := resources.CreateTestUffizziCluster(td.Name, ns.Name) - uc.Spec = td.Spec - wrapUffizziClusterLifecycleTest(ctx, ns, uc) +type ExpectedStatusOverLifetime struct { + Initializing uffizziv1alpha1.UffizziClusterStatus + Ready uffizziv1alpha1.UffizziClusterStatus + Sleeping uffizziv1alpha1.UffizziClusterStatus + Awoken uffizziv1alpha1.UffizziClusterStatus +} + +func initExpectedStatusOverLifetime() ExpectedStatusOverLifetime { + return ExpectedStatusOverLifetime{ + Initializing: uffizziv1alpha1.UffizziClusterStatus{ + Conditions: uffizzicluster.GetAllInitializingConditions(), + }, + Ready: uffizziv1alpha1.UffizziClusterStatus{ + Conditions: uffizzicluster.GetAllReadyConditions(), + }, + Sleeping: uffizziv1alpha1.UffizziClusterStatus{ + Conditions: uffizzicluster.GetAllSleepConditions(), + }, + Awoken: uffizziv1alpha1.UffizziClusterStatus{ + Conditions: uffizzicluster.GetAllAwokenConditions(), + }, + } } func TestAPIs(t *testing.T) { diff --git a/src/test/e2e/ucluster_lifecycle_test.go b/src/test/e2e/ucluster_lifecycle_test.go index 08683a21..bc1a88ba 100644 --- a/src/test/e2e/ucluster_lifecycle_test.go +++ b/src/test/e2e/ucluster_lifecycle_test.go @@ -2,8 +2,6 @@ package e2e import ( context "context" - "github.com/UffizziCloud/uffizzi-cluster-operator/src/api/v1alpha1" - "github.com/UffizziCloud/uffizzi-cluster-operator/src/controllers/uffizzicluster" "github.com/UffizziCloud/uffizzi-cluster-operator/src/pkg/constants" "github.com/UffizziCloud/uffizzi-cluster-operator/src/test/util/conditions" "github.com/UffizziCloud/uffizzi-cluster-operator/src/test/util/resources" @@ -14,7 +12,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func wrapUffizziClusterLifecycleTest(ctx context.Context, ns *v1.Namespace, uc *v1alpha1.UffizziCluster) { +func (td *TestDefinition) Run(ctx context.Context) { + // init + ns := resources.CreateTestNamespace(td.Name) + uc := resources.CreateTestUffizziCluster(td.Name, ns.Name) + uc.Spec = td.Spec + var ( timeout = "10m" pollingTimeout = "100ms" @@ -40,6 +43,7 @@ func wrapUffizziClusterLifecycleTest(ctx context.Context, ns *v1.Namespace, uc * }) }) + // Initializing and Ready Context("When creating UffizziCluster", func() { It("Should create a UffizziCluster", func() { // @@ -106,7 +110,7 @@ func wrapUffizziClusterLifecycleTest(ctx context.Context, ns *v1.Namespace, uc * if err := k8sClient.Get(ctx, uffizziClusterNSN, uc); err != nil { return false } - expectedConditions = uffizzicluster.GetAllInitializingConditions() + expectedConditions = td.ExpectedStatus.Initializing.Conditions return containsAllConditionsQ()(expectedConditions, uc.Status.Conditions) }, timeout, pollingTimeout).Should(shouldBeTrueQ()) @@ -121,14 +125,27 @@ func wrapUffizziClusterLifecycleTest(ctx context.Context, ns *v1.Namespace, uc * if err := k8sClient.Get(ctx, uffizziClusterNSN, uc); err != nil { return false } - expectedConditions = uffizzicluster.GetAllReadyConditions() + expectedConditions = td.ExpectedStatus.Ready.Conditions return containsAllConditionsQ()(expectedConditions, uc.Status.Conditions) }, timeout, pollingTimeout).Should(shouldBeTrueQ()) + By("Check if UffizziCluster has the correct tolerations in the Status") + for i, t := range td.ExpectedStatus.Ready.Tolerations { + for _, ucTol := range uc.Status.Tolerations { + if t.Key == ucTol.Key && t.Value == ucTol.Value { + break + } + if i == len(td.ExpectedStatus.Ready.Tolerations)-1 { + Fail("Tolerations do not match") + } + } + } + //GinkgoWriter.Printf(conditions.CreateConditionsCmpDiff(expectedConditions, uc.Status.Conditions)) }) }) + // Sleep Context("When putting a cluster to sleep", func() { It("Should put the cluster to sleep", func() { By("By putting the UffizziCluster to sleep") @@ -137,7 +154,7 @@ func wrapUffizziClusterLifecycleTest(ctx context.Context, ns *v1.Namespace, uc * }) It("Should be in a Sleep State", func() { - expectedConditions := uffizzicluster.GetAllSleepConditions() + expectedConditions := td.ExpectedStatus.Sleeping.Conditions uffizziClusterNSN := resources.CreateNamespacedName(uc.Name, ns.Name) By("Check if UffizziCluster has the correct Sleep conditions") Eventually(func() bool { @@ -151,6 +168,7 @@ func wrapUffizziClusterLifecycleTest(ctx context.Context, ns *v1.Namespace, uc * }) }) + // Awoken Context("When waking a cluster up", func() { It("Should wake the cluster up", func() { By("By waking the UffizziCluster up") @@ -159,9 +177,9 @@ func wrapUffizziClusterLifecycleTest(ctx context.Context, ns *v1.Namespace, uc * }) It("Should be Awoken", func() { - expectedConditions := uffizzicluster.GetAllAwokenConditions() + expectedConditions := td.ExpectedStatus.Awoken.Conditions uffizziClusterNSN := resources.CreateNamespacedName(uc.Name, ns.Name) - By("Check if UffizziCluster has the correct Sleep conditions") + By("Check if UffizziCluster has the correct Awoken conditions") Eventually(func() bool { if err := k8sClient.Get(ctx, uffizziClusterNSN, uc); err != nil { return false diff --git a/src/test/e2e/uffizzicluster_controller_test.go b/src/test/e2e/uffizzicluster_controller_test.go index 35104aa0..eb736b54 100644 --- a/src/test/e2e/uffizzicluster_controller_test.go +++ b/src/test/e2e/uffizzicluster_controller_test.go @@ -4,11 +4,12 @@ import ( "context" "github.com/UffizziCloud/uffizzi-cluster-operator/src/api/v1alpha1" "github.com/UffizziCloud/uffizzi-cluster-operator/src/pkg/constants" + "github.com/UffizziCloud/uffizzi-cluster-operator/src/pkg/helm/types/vcluster" . "github.com/onsi/ginkgo/v2" v1 "k8s.io/api/core/v1" ) -var _ = Describe("Basic Vanilla K3S UffizziCluster Lifecycle", func() { +var _ = Describe("k3s", func() { BeforeEach(func() { if e2e.IsTainted { Skip("Skipping test because cluster is tainted") @@ -16,13 +17,14 @@ var _ = Describe("Basic Vanilla K3S UffizziCluster Lifecycle", func() { }) ctx := context.Background() testUffizziCluster := TestDefinition{ - Name: "basic-k3s-test", - Spec: v1alpha1.UffizziClusterSpec{}, + Name: "basic-k3s-test", + Spec: v1alpha1.UffizziClusterSpec{}, + ExpectedStatus: initExpectedStatusOverLifetime(), } - testUffizziCluster.ExecLifecycleTest(ctx) + testUffizziCluster.Run(ctx) }) -var _ = Describe("Basic Vanilla K8S UffizziCluster Lifecycle", func() { +var _ = Describe("k8s", func() { BeforeEach(func() { if e2e.IsTainted { Skip("Skipping test because cluster is tainted") @@ -30,15 +32,16 @@ var _ = Describe("Basic Vanilla K8S UffizziCluster Lifecycle", func() { }) ctx := context.Background() testUffizziCluster := TestDefinition{ - Name: "basic-k8s-test", + Name: "k8s", Spec: v1alpha1.UffizziClusterSpec{ Distro: "k8s", }, + ExpectedStatus: initExpectedStatusOverLifetime(), } - testUffizziCluster.ExecLifecycleTest(ctx) + testUffizziCluster.Run(ctx) }) -var _ = Describe("Basic K3S UffizziCluster with ETCD Lifecycle", func() { +var _ = Describe("k3s: w/ etcd", func() { BeforeEach(func() { if e2e.IsTainted { Skip("Skipping test because cluster is tainted") @@ -46,38 +49,54 @@ var _ = Describe("Basic K3S UffizziCluster with ETCD Lifecycle", func() { }) ctx := context.Background() testUffizziCluster := TestDefinition{ - Name: "k3s-etcd-test", + Name: "k3s-etcd", Spec: v1alpha1.UffizziClusterSpec{ ExternalDatastore: constants.ETCD, }, + ExpectedStatus: initExpectedStatusOverLifetime(), } - testUffizziCluster.ExecLifecycleTest(ctx) + testUffizziCluster.Run(ctx) }) // Test against cluster with tainted nodes - good for testing node affinities -var _ = Describe("UffizziCluster NodeSelector and Tolerations", func() { +var _ = Describe("k3s: explicit nodeselector and toleration", func() { BeforeEach(func() { if !e2e.IsTainted { Skip("Skipping test because cluster is not tainted") } }) ctx := context.Background() + tolerations := append([]v1.Toleration{}, vcluster.GvisorToleration.ToV1()) testUffizziCluster := TestDefinition{ - Name: "k3s-nodeselector-toleration-test", + Name: "k3s-nds-tlrtn", Spec: v1alpha1.UffizziClusterSpec{ - NodeSelector: map[string]string{ - "testkey": "testvalue", - }, - Toleration: []v1.Toleration{ - { - Key: "testkey", - Operator: "Equal", - Value: "testvalue", - Effect: "NoSchedule", - }, - }, + NodeSelector: vcluster.GvisorNodeSelector, + Toleration: tolerations, }, + ExpectedStatus: initExpectedStatusOverLifetime(), } - testUffizziCluster.ExecLifecycleTest(ctx) + testUffizziCluster.ExpectedStatus.Ready.NodeSelector = vcluster.GvisorNodeSelector + testUffizziCluster.ExpectedStatus.Ready.Tolerations = tolerations + testUffizziCluster.Run(ctx) +}) + +var _ = Describe("k3s: nodeselector template - gvisor", func() { + BeforeEach(func() { + if !e2e.IsTainted { + Skip("Skipping test because cluster is not tainted") + } + }) + ctx := context.Background() + tolerations := append([]v1.Toleration{}, vcluster.GvisorToleration.ToV1()) + testUffizziCluster := TestDefinition{ + Name: "k3s-nds-template-gvisor", + Spec: v1alpha1.UffizziClusterSpec{ + NodeSelectorTemplate: constants.GVISOR, + }, + ExpectedStatus: initExpectedStatusOverLifetime(), + } + testUffizziCluster.ExpectedStatus.Ready.NodeSelector = vcluster.GvisorNodeSelector + testUffizziCluster.ExpectedStatus.Ready.Tolerations = tolerations + testUffizziCluster.Run(ctx) })