[feat] DNS based LoadBalancing for the CAPL cluster API Server (#370)

DNS based LoadBalancing for the CAPL cluster API Server

Author: amold1

.gitignore

@@ -9,3 +9,4 @@ docs/book
 release/*
 templates/cluster-template*.yaml
 infrastructure-*-linode/*
+.envrc

.golangci.yml

@@ -20,7 +20,7 @@ linters-settings:
     # [deprecated] comma-separated list of pairs of the form pkg:regex
     # the regex is used to ignore names within pkg. (default "fmt:.*").
     # see https://github.com/kisielk/errcheck#the-deprecated-method for details
-    ignore: fmt:.*,io/ioutil:^Read.*
+    exclude-functions: fmt:.*,io/ioutil:^Read.*
 
   gci:
     sections:
@@ -49,7 +49,7 @@ linters-settings:
 
   cyclop:
     max-complexity: 15
-    
+
   maligned:
     # print struct with more effective memory layout or not, false by default
     suggest-new: true
@@ -194,7 +194,6 @@ linters:
     - unused
   fast: false
 
-
 issues:
   # Excluding configuration per-path and per-linter
   exclude-rules:
@@ -208,7 +207,7 @@ issues:
         - gosec
         - exportloopref
         - unparam
-    
+
     # Ease some gocritic warnings on test files.
     - path: _test\.go
       text: "(unnamedResult|exitAfterDefer)"
@@ -234,31 +233,31 @@ issues:
     # rather than using a pointer.
     - text: "(hugeParam|rangeValCopy):"
       linters:
-      - gocritic
+        - gocritic
 
     # This "TestMain should call os.Exit to set exit code" warning is not clever
     # enough to notice that we call a helper method that calls os.Exit.
     - text: "SA3000:"
       linters:
-      - staticcheck
+        - staticcheck
 
     - text: "k8s.io/api/core/v1"
       linters:
-      - goimports
+        - goimports
 
     # This is a "potential hardcoded credentials" warning. It's triggered by
     # any variable with 'secret' in the same, and thus hits a lot of false
     # positives in Kubernetes land where a Secret is an object type.
     - text: "G101:"
       linters:
-      - gosec
-      - gas
+        - gosec
+        - gas
 
     # This is an 'errors unhandled' warning that duplicates errcheck.
     - text: "G104:"
       linters:
-      - gosec
-      - gas
+        - gosec
+        - gas
 
   # Independently from option `exclude` we use default exclude patterns,
   # it can be disabled by this option. To list all
@@ -281,4 +280,4 @@ issues:
   max-same-issues: 0
 
   exclude-files:
-  - "zz_generated\\..+\\.go$"
+    - "zz_generated\\..+\\.go$"

Makefile

@@ -121,7 +121,7 @@ gosec: ## Run gosec against code.
 
 .PHONY: lint
 lint: ## Run lint against code.
-	docker run --rm -w /workdir -v $(PWD):/workdir golangci/golangci-lint:v1.57.2 golangci-lint run -c .golangci.yml --fix
+	docker run --rm -w /workdir -v $(PWD):/workdir golangci/golangci-lint:v1.59.1 golangci-lint run -c .golangci.yml --fix
 
 .PHONY: nilcheck
 nilcheck: nilaway ## Run nil check against code.
@@ -152,7 +152,6 @@ e2etest: generate local-release local-deploy chainsaw
 	GIT_REF=$(GIT_REF) $(CHAINSAW) test ./e2e --selector $(E2E_SELECTOR) $(E2E_FLAGS)
 
 local-deploy: kind ctlptl tilt kustomize clusterctl
-	@echo -n "LINODE_TOKEN=$(LINODE_TOKEN)" > config/default/.env.linode
 	$(CTLPTL) apply -f .tilt/ctlptl-config.yaml
 	$(TILT) ci -f Tiltfile
 
@@ -204,7 +203,6 @@ endif
 
 .PHONY: tilt-cluster
 tilt-cluster: ctlptl tilt kind clusterctl
-	@echo -n "LINODE_TOKEN=$(LINODE_TOKEN)" > config/default/.env.linode
 	$(CTLPTL) apply -f .tilt/ctlptl-config.yaml
 	$(TILT) up --stream
 

Tiltfile

@@ -128,6 +128,7 @@ manager_yaml = decode_yaml_stream(kustomize("config/default"))
 for resource in manager_yaml:
     if resource["metadata"]["name"] == "capl-manager-credentials":
         resource["stringData"]["apiToken"] = os.getenv("LINODE_TOKEN")
+        resource["stringData"]["dnsToken"] = os.getenv("LINODE_DNS_TOKEN")
     if (
         resource["kind"] == "CustomResourceDefinition"
         and resource["spec"]["group"] == "infrastructure.cluster.x-k8s.io"

api/v1alpha2/linodecluster_types.go

@@ -104,9 +104,29 @@ func (lm *LinodeCluster) SetConditions(conditions clusterv1.Conditions) {
 // NetworkSpec encapsulates Linode networking resources.
 type NetworkSpec struct {
 	// LoadBalancerType is the type of load balancer to use, defaults to NodeBalancer if not otherwise set
-	// +kubebuilder:validation:Enum=NodeBalancer
+	// +kubebuilder:validation:Enum=NodeBalancer;dns
 	// +optional
 	LoadBalancerType string `json:"loadBalancerType,omitempty"`
+	// DNSProvider is provider who manages the domain
+	// Ignored if the LoadBalancerType is set to anything other than dns
+	// If not set, defaults linode dns
+	// +optional
+	DNSProvider int `json:"dnsProvider,omitempty"`
+	// DNSRootDomain is the root domain used to create a DNS entry for the control-plane endpoint
+	// Ignored if the LoadBalancerType is set to anything other than dns
+	// +optional
+	DNSRootDomain string `json:"dnsRootDomain,omitempty"`
+	// DNSUniqueIdentifier is the unique identifier for the DNS. This let clusters with the same name have unique
+	// DNS record
+	// Ignored if the LoadBalancerType is set to anything other than dns
+	// If not set, CAPL will create a unique identifier for you
+	// +optional
+	DNSUniqueIdentifier string `json:"dnsUniqueIdentifier,omitempty"`
+	// DNSTTLSec is the TTL for the domain record
+	// Ignored if the LoadBalancerType is set to anything other than dns
+	// If not set, defaults to 30
+	// +optional
+	DNSTTLSec int `json:"dnsTTLsec,omitempty"`
 	// apiserverLoadBalancerPort used by the api server. It must be valid ports range (1-65535).
 	// If omitted, default value is 6443.
 	// +kubebuilder:validation:Minimum=1

clients/clients.go

@@ -15,6 +15,7 @@ type LinodeClient interface {
 	LinodeInstanceClient
 	LinodeVPCClient
 	LinodeObjectStorageClient
+	LinodeDNSClient
 }
 
 // LinodeInstanceClient defines the methods that interact with Linode's Instance service.
@@ -64,6 +65,15 @@ type LinodeObjectStorageClient interface {
 	DeleteObjectStorageKey(ctx context.Context, keyID int) error
 }
 
+// LinodeDNSClient defines the methods that interact with Linode's Domains service.
+type LinodeDNSClient interface {
+	CreateDomainRecord(ctx context.Context, domainID int, recordReq linodego.DomainRecordCreateOptions) (*linodego.DomainRecord, error)
+	UpdateDomainRecord(ctx context.Context, domainID int, domainRecordID int, recordReq linodego.DomainRecordUpdateOptions) (*linodego.DomainRecord, error)
+	ListDomainRecords(ctx context.Context, domainID int, opts *linodego.ListOptions) ([]linodego.DomainRecord, error)
+	ListDomains(ctx context.Context, opts *linodego.ListOptions) ([]linodego.Domain, error)
+	DeleteDomainRecord(ctx context.Context, domainID int, domainRecordID int) error
+}
+
 type K8sClient interface {
 	client.Client
 }

cloud/scope/cluster.go

@@ -57,11 +57,12 @@ func NewClusterScope(ctx context.Context, apiKey string, params ClusterScopePara
 
 	// Override the controller credentials with ones from the Cluster's Secret reference (if supplied).
 	if params.LinodeCluster.Spec.CredentialsRef != nil {
-		data, err := getCredentialDataFromRef(ctx, params.Client, *params.LinodeCluster.Spec.CredentialsRef, params.LinodeCluster.GetNamespace())
+		// TODO: This key is hard-coded (for now) to match the externally-managed `manager-credentials` Secret.
+		apiToken, err := getCredentialDataFromRef(ctx, params.Client, *params.LinodeCluster.Spec.CredentialsRef, params.LinodeCluster.GetNamespace(), "apiToken")
 		if err != nil {
 			return nil, fmt.Errorf("credentials from secret ref: %w", err)
 		}
-		apiKey = string(data)
+		apiKey = string(apiToken)
 	}
 	linodeClient, err := CreateLinodeClient(apiKey, defaultClientTimeout)
 	if err != nil {

cloud/scope/common.go

@@ -63,16 +63,14 @@ func CreateLinodeClient(apiKey string, timeout time.Duration, opts ...Option) (L
 	), nil
 }
 
-func getCredentialDataFromRef(ctx context.Context, crClient K8sClient, credentialsRef corev1.SecretReference, defaultNamespace string) ([]byte, error) {
+func getCredentialDataFromRef(ctx context.Context, crClient K8sClient, credentialsRef corev1.SecretReference, defaultNamespace, key string) ([]byte, error) {
 	credSecret, err := getCredentials(ctx, crClient, credentialsRef, defaultNamespace)
 	if err != nil {
 		return nil, err
 	}
-
-	// TODO: This key is hard-coded (for now) to match the externally-managed `manager-credentials` Secret.
-	rawData, ok := credSecret.Data["apiToken"]
+	rawData, ok := credSecret.Data[key]
 	if !ok {
-		return nil, fmt.Errorf("no apiToken key in credentials secret %s/%s", credentialsRef.Namespace, credentialsRef.Name)
+		return nil, fmt.Errorf("no %s key in credentials secret %s/%s", key, credentialsRef.Namespace, credentialsRef.Name)
 	}
 
 	return rawData, nil

cloud/scope/common_test.go

@@ -174,7 +174,7 @@ func TestGetCredentialDataFromRef(t *testing.T) {
 			mockClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(testCase.args.funcBehavior)
 
 			// Call getCredentialDataFromRef using the mock client
-			got, err := getCredentialDataFromRef(context.Background(), mockClient, testCase.args.providedCredentialsRef, "default")
+			got, err := getCredentialDataFromRef(context.Background(), mockClient, testCase.args.providedCredentialsRef, "default", "apiToken")
 
 			// Check that the function returned the expected result
 			if testCase.expectedError != "" {

cloud/scope/machine.go

@@ -26,13 +26,14 @@ type MachineScopeParams struct {
 }
 
 type MachineScope struct {
-	Client        K8sClient
-	PatchHelper   *patch.Helper
-	Cluster       *clusterv1.Cluster
-	Machine       *clusterv1.Machine
-	LinodeClient  LinodeClient
-	LinodeCluster *infrav1alpha2.LinodeCluster
-	LinodeMachine *infrav1alpha1.LinodeMachine
+	Client              K8sClient
+	PatchHelper         *patch.Helper
+	Cluster             *clusterv1.Cluster
+	Machine             *clusterv1.Machine
+	LinodeClient        LinodeClient
+	LinodeDomainsClient LinodeClient
+	LinodeCluster       *infrav1alpha2.LinodeCluster
+	LinodeMachine       *infrav1alpha1.LinodeMachine
 }
 
 func validateMachineScopeParams(params MachineScopeParams) error {
@@ -52,7 +53,7 @@ func validateMachineScopeParams(params MachineScopeParams) error {
 	return nil
 }
 
-func NewMachineScope(ctx context.Context, apiKey string, params MachineScopeParams) (*MachineScope, error) {
+func NewMachineScope(ctx context.Context, apiKey, dnsKey string, params MachineScopeParams) (*MachineScope, error) {
 	if err := validateMachineScopeParams(params); err != nil {
 		return nil, err
 	}
@@ -78,32 +79,47 @@ func NewMachineScope(ctx context.Context, apiKey string, params MachineScopePara
 	}
 
 	if credentialRef != nil {
-		data, err := getCredentialDataFromRef(ctx, params.Client, *credentialRef, defaultNamespace)
+		// TODO: This key is hard-coded (for now) to match the externally-managed `manager-credentials` Secret.
+		apiToken, err := getCredentialDataFromRef(ctx, params.Client, *credentialRef, defaultNamespace, "apiToken")
 		if err != nil {
 			return nil, fmt.Errorf("credentials from secret ref: %w", err)
 		}
-		apiKey = string(data)
+		apiKey = string(apiToken)
+
+		dnsToken, err := getCredentialDataFromRef(ctx, params.Client, *credentialRef, defaultNamespace, "dnsToken")
+		if err != nil || len(dnsToken) == 0 {
+			dnsToken = apiToken
+		}
+		dnsKey = string(dnsToken)
 	}
+
 	linodeClient, err := CreateLinodeClient(apiKey, defaultClientTimeout,
 		WithRetryCount(0),
 	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create linode client: %w", err)
 	}
+	linodeDomainsClient, err := CreateLinodeClient(dnsKey, defaultClientTimeout,
+		WithRetryCount(0),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create linode client: %w", err)
+	}
 
 	helper, err := patch.NewHelper(params.LinodeMachine, params.Client)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init patch helper: %w", err)
 	}
 
 	return &MachineScope{
-		Client:        params.Client,
-		PatchHelper:   helper,
-		Cluster:       params.Cluster,
-		Machine:       params.Machine,
-		LinodeClient:  linodeClient,
-		LinodeCluster: params.LinodeCluster,
-		LinodeMachine: params.LinodeMachine,
+		Client:              params.Client,
+		PatchHelper:         helper,
+		Cluster:             params.Cluster,
+		Machine:             params.Machine,
+		LinodeClient:        linodeClient,
+		LinodeDomainsClient: linodeDomainsClient,
+		LinodeCluster:       params.LinodeCluster,
+		LinodeMachine:       params.LinodeMachine,
 	}, nil
 }