Add custom authenticator support (#295)

* add custom authenticator support and example yaml

* fix minor toolchain issues and update DEVELOPER_GUIDE.md

operator/docs/DEVELOPER_GUIDE.md

@@ -6,8 +6,10 @@ If you are developing KIT operator, finish the installation steps listed in the
 
 ### Prerequisites
 
- - Go version (1.16 or higher)
- - [Ko version](https://github.com/google/ko#install) (v0.8.2 or higher)
+ - Go version 1.16-1.17 (NOTE: 1.18+ fails to install pkg/operator toolchain). [instructions](https://gist.github.com/BigOokie/d5817e88f01e0d452ed585a1590f5aeb)
+ - [Ko version](https://github.com/google/ko#install) (v0.8.2 - 0.11.2, the
+   latest 0.12+ is broken right now)
+ - Run `make toolchain`
 
 ### Create a [Private ECR repository](https://docs.aws.amazon.com/AmazonECR/latest/userguide/repository-create.html) to push controller and webhook image for kit-operator
 
@@ -35,4 +37,4 @@ To delete KIT from Kubernetes cluster
 ```bash
 make delete
 kubectl delete namespace kit
-```
+```

operator/docs/examples/custom-authenticator.yaml

@@ -0,0 +1,37 @@
+# To create this cluster, run:
+# * export KUBECONFIG=<management-cluster>
+# * export GUEST_CLUSTER_NAME="foobar"
+# * envsubst < custom-authenticator.yaml | kubectl --kubeconfig $KUBECONFIG apply -f -
+# * k certificate approve $(k get csr | grep "Pending" | awk '{print $1}')
+
+apiVersion: kit.k8s.sh/v1alpha1
+kind: ControlPlane
+metadata:
+  name: $GUEST_CLUSTER_NAME
+spec:
+  master:
+    apiServer:
+      replicas: 3
+    authenticator:
+      spec:
+        containers:
+        - name: aws-iam-authenticator
+          image: public.ecr.aws/eks-distro/kubernetes-sigs/aws-iam-authenticator:v0.5.9-eks-1-19-22
+          securityContext:
+            runAsUser: 10000
+            runAsGroup: 10000
+          args:
+          - --backend-mode=MountedFile,EKSConfigMap
+          # TODO: scope down permissions
+          # There are 3 kubeconfigish flags for the authenticator:
+          # * --kubeconfig -> this is configuring one direction communication from authenticator to kube-apiserver
+          # * --generate-kubeconfig -> this is configuring communication from kube-apiserver to authenticator (this is the token file passed to kube-apiserver).
+          # * --kubeconfig-pregenerated -> a boolean flag if we don't want the --kubeconfig flag to generate a new kubeconfig.
+          - --kubeconfig=/var/aws-iam-authenticator/auth-to-k8s-kubeconfig/config
+          volumeMounts:
+          - mountPath: /var/aws-iam-authenticator/auth-to-k8s-kubeconfig/
+            name: auth-to-k8s-kubeconfig
+        volumes:
+        - secret:
+            secretName: $GUEST_CLUSTER_NAME-kube-admin-config
+          name: auth-to-k8s-kubeconfig

operator/hack/codegen.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -ex
 
 controller-gen crd \
     object:headerFile="hack/boilerplate.go.txt" \
@@ -21,4 +22,4 @@ yq eval 'del(.. | select(has("initContainers")).initContainers)' -i config/kit.k
 
 
 mv config/kit.k8s.sh_controlplanes.yaml charts/kit-operator/crds/control-plane-crd.yaml
-mv config/kit.k8s.sh_dataplanes.yaml charts/kit-operator/crds/data-plane-crd.yaml
+mv config/kit.k8s.sh_dataplanes.yaml charts/kit-operator/crds/data-plane-crd.yaml

operator/hack/toolchain.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -eu -o pipefail
+set -eux -o pipefail
 
 main() {
     tools
@@ -11,7 +11,7 @@ tools() {
     go install github.com/ahmetb/[email protected]
     go install github.com/fzipp/gocyclo/cmd/[email protected]
     go install github.com/golangci/golangci-lint/cmd/[email protected]
-    go install github.com/google/ko@v0.10.0
+    go install github.com/google/ko@v0.11.2
     go install github.com/mikefarah/yq/[email protected]
     go install github.com/mitchellh/[email protected]
     go install github.com/onsi/ginkgo/[email protected]
@@ -32,4 +32,4 @@ kubebuilder() {
     find $KUBEBUILDER_ASSETS
 }
 
-main "$@"
+main "$@"

operator/pkg/apis/controlplane/v1alpha1/controlplane.go

@@ -19,6 +19,9 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+// NOTE: if you make changes to this file, run `make codegen` to update the
+// appropriate crds and yamls.
+
 // ControlPlane is the Schema for the ControlPlanes API
 // +kubebuilder:object:root=true
 // +kubebuilder:resource:shortName=cp
@@ -61,6 +64,7 @@ type MasterSpec struct {
 	Scheduler         *Component `json:"scheduler,omitempty"`
 	ControllerManager *Component `json:"controllerManager,omitempty"`
 	APIServer         *Component `json:"apiServer,omitempty"`
+	Authenticator     *Component `json:"authenticator,omitempty"`
 }
 
 // Component provides a generic way to pass in args and images to master and etcd

operator/pkg/controllers/master/authenticator.go

@@ -22,6 +22,7 @@ import (
 	"github.com/awslabs/kubernetes-iteration-toolkit/operator/pkg/awsprovider/iam"
 	"github.com/awslabs/kubernetes-iteration-toolkit/operator/pkg/components/iamauthenticator"
 	"github.com/awslabs/kubernetes-iteration-toolkit/operator/pkg/utils/object"
+	"github.com/awslabs/kubernetes-iteration-toolkit/operator/pkg/utils/patch"
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -45,6 +46,24 @@ func (c *Controller) reconcileAuthenticator(ctx context.Context, controlPlane *v
 }
 
 func (c *Controller) ensureDaemonSet(ctx context.Context, controlPlane *v1alpha1.ControlPlane) error {
+	authenticatorPodTemplateSpec := iamauthenticator.PodSpec(controlPlane.ClusterName(), func(template v1.PodTemplateSpec) v1.PodTemplateSpec {
+		template.Spec.NodeSelector = APIServerLabels(controlPlane.ClusterName())
+		template.Spec.Volumes = append(template.Spec.Volumes, v1.Volume{Name: "config",
+			VolumeSource: v1.VolumeSource{ConfigMap: &v1.ConfigMapVolumeSource{
+				LocalObjectReference: v1.LocalObjectReference{Name: iamauthenticator.AuthenticatorConfigMapName(controlPlane.ClusterName())},
+			}},
+		})
+		return template
+	})
+
+	if controlPlane.Spec.Master.Authenticator != nil {
+		var err error
+		authenticatorPodTemplateSpec.Spec, err = patch.PodSpec(&authenticatorPodTemplateSpec.Spec, controlPlane.Spec.Master.Authenticator.Spec)
+		if err != nil {
+			return fmt.Errorf("patch authenticator pod spec, %w", err)
+		}
+	}
+
 	return c.kubeClient.EnsurePatch(ctx, &appsv1.DaemonSet{}, object.WithOwner(controlPlane, &appsv1.DaemonSet{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      fmt.Sprintf("%s-authenticator", controlPlane.ClusterName()),
@@ -54,15 +73,7 @@ func (c *Controller) ensureDaemonSet(ctx context.Context, controlPlane *v1alpha1
 		Spec: appsv1.DaemonSetSpec{
 			UpdateStrategy: appsv1.DaemonSetUpdateStrategy{Type: appsv1.RollingUpdateDaemonSetStrategyType},
 			Selector:       &metav1.LabelSelector{MatchLabels: iamauthenticator.Labels(controlPlane.ClusterName())},
-			Template: iamauthenticator.PodSpec(controlPlane.ClusterName(), func(template v1.PodTemplateSpec) v1.PodTemplateSpec {
-				template.Spec.NodeSelector = APIServerLabels(controlPlane.ClusterName())
-				template.Spec.Volumes = append(template.Spec.Volumes, v1.Volume{Name: "config",
-					VolumeSource: v1.VolumeSource{ConfigMap: &v1.ConfigMapVolumeSource{
-						LocalObjectReference: v1.LocalObjectReference{Name: iamauthenticator.AuthenticatorConfigMapName(controlPlane.ClusterName())},
-					}},
-				})
-				return template
-			}),
+			Template:       authenticatorPodTemplateSpec,
 		},
 	}))
 }

operator/pkg/errors/errors.go

@@ -43,7 +43,7 @@ func IsDNSLookUpNoSuchHost(err error) bool {
 
 func IsNetIOTimeOut(err error) bool {
 	netErr := net.Error(nil)
-	return errors.As(err, &netErr) && netErr.Temporary() && netErr.Timeout()
+	return errors.As(err, &netErr) && netErr.Timeout()
 }
 
 func IsConnectionRefused(err error) bool {