From a2b67b226f4177bbf18b55eeb0c4175d830d7d8a Mon Sep 17 00:00:00 2001
From: antoinetran <antoinetran@users.noreply.github.com>
Date: Fri, 31 Jan 2025 11:42:03 +0100
Subject: [PATCH] doc 06-enable-service-accounts

---
 .../guides/06-enable-service-accounts.mdx     | 373 ++++++++++++++++++
 1 file changed, 373 insertions(+)
 create mode 100644 docs/docs/guides/06-enable-service-accounts.mdx

diff --git a/docs/docs/guides/06-enable-service-accounts.mdx b/docs/docs/guides/06-enable-service-accounts.mdx
new file mode 100644
index 00000000..46c60b75
--- /dev/null
+++ b/docs/docs/guides/06-enable-service-accounts.mdx
@@ -0,0 +1,373 @@
+---
+sidebar_position: 6
+---
+import ThemedImage from '@theme/ThemedImage';
+import useBaseUrl from '@docusaurus/useBaseUrl';
+
+# Enable service accounts
+
+By default, InterLink does not translate Kubernetes Service Accounts from Pod into Plugin. That means workload that needs to interact with Kubernetes API like Argo will not work.
+
+However after following deployment and configuration in this guide, InterLink will give means for Plugin for that. A test has been done with InterLink Slurm Plugin and Argo.
+
+There are two parts on this guide: how to deploy means to access Kubernetes API service from external cluster network, and how to configure InterLink so that Plugin can access it.
+
+The prerequisite of this guide are:
+- provide an external way to access Kubernetes API service (out of scope of InterLink, but an example is written below)
+- provide certificates (that can be self-signed), and its CA root certificate (also out of scope of InterLink, but an example is written below)
+
+## Provide an external way to access Kubernetes API service
+
+By default, InterLink Plugin containers cannot access Kubernetes internal network. Thus they cannot access Kubernetes API service (`kubernetes.default.svc.cluster.local`). Kubernetes offers ways to access internal services (see https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types):
+- LoadBalancer
+- NodePort
+- ...
+
+Because this is highly dependent of the Kubernetes cluster infrastructure, this deployment is out of scope of InterLink automatic deployment. However, below are examples of how and what to deploy, using ingress. This requires a bit of Ingress knowledge as prerequisite.
+
+### Design of ingress use
+
+The Kubernetes cluster can already contain an Ingress resource to let external access to web services. However it is a best-practice to separate internal (meaning flow between Kubernetes and Plugin infrastructure, like Slurm machines) and external flows (meaning flow between Kubernetes and the world).
+
+This guide will thus deploy as an example another ingress separate from the already existing ingress, if it exist. This also works if there is no ingress yet.
+
+Here is a diagram (generated with https://asciiflow.com) with Interlink Slurm Plugin as example. This Kubernetes cluster is deployed next to a Slurm cluster, but it can also be deployed on cloud.
+
+Because Ingress can only serve services in the same namespace, and because Kubernetes API is in default namespace, different from the Ingress "ingress-nginx-internal", a reverse-proxy NGINX HTTPS is instantiated to make Kubernetes API available to Ingress.
+
+```
+       ┌───────────────────────────┐                               
+       │                           │                               
+       │   Kubernetes cluster      │                               
+       │┌────────┐   ┌───────────┐ │                               
+       ││K8S API ├───┤Nginx HTTPS│ │                               
+       │└────────┘   └─────┬─────┘ │                               
+       │                   │       │               Slurm cluster   
+       └─┬─────────────────┼───────┘                     │         
+         │Ingress          │Ingress            ──────────┴─────────
+         │TCP 80, 443      │TCP 6080, 60443      Slurm network     
+   ┌─────┴────┐        ┌───┴──────┐            ──────────┬─────────
+   │ External │        │ Internal │                      │         
+   │ Firewall │        │ Firewall │                      │         
+   └─────┬────┘        └───┬──────┘       ┌────────┐     │         
+         │                 └──────────────┤ Router ├─────┘         
+   xxxxxxxxx xxxx                         └────────┘               
+   x       xxx  xxx                                                
+  xx              x                                                
+ xx  Internet     x                                                
+x            xxxxx                                                 
+x   x           xx                                                 
+xxxxxxx       xxx                                                  
+      xxxxxxxxx                                                    
+```
+
+### NGINX HTTPS
+
+This deploys a reverse-proxy.
+
+```
+cat <<EOF | tee ./nginx-values.yaml
+serverBlock: |-
+  server {
+    listen 0.0.0.0:60443;
+
+    real_ip_header X-Forwarded-For;
+    set_real_ip_from 0.0.0.0/0;
+
+    resolver kube-dns.kube-system.svc.cluster.local valid=30s;
+    set $upstream https://kubernetes.default.svc.cluster.local:443;
+
+    location / {
+      proxy_pass $upstream;
+    }
+  }
+service:
+  # Avoid LoadBalancer type, to avoid external IP.
+  type: "ClusterIP"
+  ports:
+    http: 60443
+
+containerPorts:
+  http: 60443
+EOF
+helm upgrade --install --create-namespace -n ingress-nginx-internal --version 16.0.3 my-nginx-kubernetes-api bitnami/nginx --values ./nginx-values.yaml
+```
+
+#### KIND cluster
+
+Deploy KIND specific ingress-nginx ingress:
+
+```
+helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
+# Replace by the target Chart helm version, see https://artifacthub.io/packages/helm/ingress-nginx/ingress-nginx
+helmIngressNginx=4.11.3
+helm upgrade --install my-ingress-nginx ingress-nginx/ingress-nginx --create-namespace -n ingress-nginx-internal --version "${helmIngressNginx}" \
+  --values https://raw.githubusercontent.com/kubernetes/ingress-nginx/helm-chart-"${helmIngressNginx}"/hack/manifest-templates/provider/kind/values.yaml \
+  --set controller.allowSnippetAnnotations=true \
+  --set controller.hostPort.ports.http=6080 --set controller.hostPort.ports.https=60443 --set controller.ingressClassResource=nginx-internal
+```
+
+Then in KIND map the network ports like this:
+```
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  # ipv6 not needed.
+  ipFamily: ipv4
+kubeadmConfigPatches:
+- |
+  kind: KubeletConfiguration
+  # See https://github.com/kubernetes-sigs/kind/issues/3359
+  localStorageCapacityIsolation: true
+#name: default
+nodes:
+# one node hosting a control plane
+- role: control-plane
+  labels:
+    ingress-ready: "true"
+  # For ingress: https://kind.sigs.k8s.io/docs/user/ingress/
+# kubeadmConfigPatches:
+# - |
+#   kind: InitConfiguration
+#   nodeRegistration:
+#     kubeletExtraArgs:
+#       node-labels: "ingress-ready=true"
+  extraPortMappings:
+  - containerPort: 6080
+    hostPort: 6080
+    protocol: TCP
+  - containerPort: 60443
+    hostPort: 60443
+    protocol: TCP
+- role: worker
+```
+
+#### Cloud cluster
+
+Deploy ingress and uses the Cloud Provider specific Loadbalancer (GCP, AWS, etc.). One can deploy ingress-nginx like this:
+```
+helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
+# Replace by the target Chart helm version, see https://artifacthub.io/packages/helm/ingress-nginx/ingress-nginx
+helmIngressNginx=4.11.3
+helm upgrade --install my-ingress-nginx ingress-nginx/ingress-nginx --create-namespace -n ingress-nginx-internal --version "${helmIngressNginx}" \
+  --set controller.hostPort.ports.http=6080 --set controller.hostPort.ports.https=60443 --set controller.ingressClassResource=nginx-internal
+```
+
+#### Baremetal cluster
+
+Deploy the MetalLB loadbalancer (see https://metallb.io/).
+This has some network prerequisites (like network in promiscuous mode, and dedicated IPs pools).
+
+## Provide HTTPS certificate and its CA certificate
+
+Kubernetes API service can only be accessed using encrypted HTTPS flow. Kubelet is normally responsible for providing `ca.crt` file that matches the Kubernetes API URL that it provides to containers: for example
+
+```
+KUBERNETES_PORT=tcp://10.96.0.1:443
+KUBERNETES_SERVICE_PORT=443
+KUBERNETES_PORT_443_TCP_ADDR=10.96.0.1
+KUBERNETES_PORT_443_TCP_PORT=443
+KUBERNETES_PORT_443_TCP_PROTO=tcp
+KUBERNETES_PORT_443_TCP=tcp://10.96.0.1:443
+KUBERNETES_SERVICE_PORT_HTTPS=443
+KUBERNETES_SERVICE_HOST=10.96.0.1
+```
+
+The former section describe how to access Kubernetes API externally, so an appropriate certificate needs to be generated. In this page, one convenient way to generate is to use certmanager.
+
+It is possible to use any third-party service (like Let's Encrypt) to generate certificates signed by a public root CA, and thus InterLink Virtual-Kubelet would not need to distribute a `ca.crt` because root CA certificates are part of the OS of container image (if they are up to date).
+
+However, it is better and more convenient to generate certificates from an internal CA that has a self-signed certificate. Here is an example of how to deploy a certmanager:
+
+```
+certManagerChartVersion="v1.14.5"
+kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/${certManagerChartVersion}/cert-manager.crds.yaml
+helm repo add jetstack https://charts.jetstack.io
+helm upgrade -i -n cert-manager cert-manager --create-namespace jetstack/cert-manager --wait --version "${certManagerChartVersion}"
+
+cat <<EOF | kubectl apply -n cert-manager -f -
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: selfsigned-root-issuer
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: ingress-nginx-internal-root-ca
+spec:
+  isCA: true
+  commonName: ingress-nginx-internal-root-ca
+  secretName: ingress-nginx-internal-root-ca-secret
+  # To avoid rotation issue, 100 years = 100 * 365 * 24 h = 876000h
+  duration: "876000h"
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: selfsigned-root-issuer
+    kind: ClusterIssuer
+    group: cert-manager.io
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: selfsigned-issuer
+spec:
+  ca:
+    secretName: ingress-nginx-internal-root-ca-secret
+ #selfSigned: {}
+EOF
+```
+
+### Configure Ingress
+
+Given the previous sections, the final step to provide access to Kubernetes API service, with a HTTPS certificate is to provide an ingress resource. Here is an example for KIND cluster, to be adapter for other types:
+
+```
+apiVersion: v1
+items:
+- apiVersion: networking.k8s.io/v1
+  kind: Ingress
+  metadata:
+    annotations:
+      acme.cert-manager.io/http01-edit-in-place: "true"
+      # To avoid rotation issue, 100 years = 100 * 365 * 24 h = 876000h
+      cert-manager.io/duration: 876000h
+      cert-manager.io/issuer: selfsigned-issuer
+    name: default
+    namespace: ingress-nginx-internal
+  spec:
+    # Must match the ingress controller ingress class name.
+    ingressClassName: nginx-internal
+    rules:
+    - http:
+        paths:
+        - backend:
+            service:
+              name: my-nginx-kubernetes-api
+              port:
+                number: 60443
+          path: /api
+          pathType: Prefix
+    - http:
+        paths:
+        - backend:
+            service:
+              name: my-nginx-kubernetes-api
+              port:
+                number: 60443
+          path: /apis
+          pathType: Prefix
+    tls:
+    - hosts:
+      - A.B.C.D # IP of the machine hosting KIND
+      secretName: web-ssl-kubernetes-api
+  status:
+    loadBalancer:
+      ingress:
+      - hostname: localhost
+```
+
+Please note that the certificates are generated for 100 years in order to not having rotation issue.
+
+## Configure InterLink Helm Chart
+
+When deploying following [DeployInterLink](./01-deploy-interlink), at helm install, please add Kubernetes API configuration as follow:
+```
+# Contains the certificate of the service automatically created by certmanager, than contains, for this example, the IP address in SAN field.
+kubernetesApiCaCrt="$(kubectl get secret -n ingress-nginx-internal web-ssl-kubernetes-api -o jsonpath='{.data.ca\.crt}'|base64 -d)"
+helm upgrade --install \
+  --create-namespace \
+  -n interlink \
+  ...
+  --set "interlink.kubernetesApiAddr=A.B.C.D" \
+  --set "interlink.kubernetesApiPort=60443" \
+  --set "interlink.kubernetesApiCaCrt=${kubernetesApiCaCrt}"
+```
+
+## Test your setup
+
+Please apply this:
+```
+cat <<EOF | kubectl apply -f -
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: helloworld
+spec:
+  template: # create pods using pod definition in this template
+    metadata:
+      labels:
+        app: helloworld
+      annotations:
+        slurm-job.vk.io/image-root: "docker://"
+    spec:
+      nodeSelector:
+        # The name of the virtual node HERE
+        kubernetes.io/hostname: interlink-slurm-node
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 1
+            preference:
+              matchExpressions:
+              - key: "kubernetes.io/hostname"
+                operator: In
+                values:
+                - interlink-slurm-node
+      tolerations:
+      - key: virtual-node.interlink/no-schedule
+        operator: Exists
+      restartPolicy: Never
+      containers:
+      - name: helloworld
+        image: docker.io/bitnami/kubectl
+        command:
+          - "kubectl"
+          - "auth"
+          - "can-i"
+          - "--list"
+        tty: true
+        resources:
+          requests:
+            cpu: "50m"
+            memory: "100Mi"
+          limits:
+            cpu: "1500m"
+            memory: "2000Mi"
+EOF
+kubectl logs job/helloworld -f
+```
+
+The expected output should be describe the RBAC related to the default serviceAccount, similar to this
+```
+Resources                                       Non-Resource URLs                      Resource Names   Verbs
+selfsubjectreviews.authentication.k8s.io        []                                     []               [create]
+selfsubjectaccessreviews.authorization.k8s.io   []                                     []               [create]
+selfsubjectrulesreviews.authorization.k8s.io    []                                     []               [create]
+                                                [/.well-known/openid-configuration/]   []               [get]
+                                                [/.well-known/openid-configuration]    []               [get]
+                                                [/api/*]                               []               [get]
+                                                [/api]                                 []               [get]
+                                                [/apis/*]                              []               [get]
+                                                [/apis]                                []               [get]
+                                                [/healthz]                             []               [get]
+                                                [/healthz]                             []               [get]
+                                                [/livez]                               []               [get]
+                                                [/livez]                               []               [get]
+                                                [/openapi/*]                           []               [get]
+                                                [/openapi]                             []               [get]
+                                                [/openid/v1/jwks/]                     []               [get]
+                                                [/openid/v1/jwks]                      []               [get]
+                                                [/readyz]                              []               [get]
+                                                [/readyz]                              []               [get]
+                                                [/version/]                            []               [get]
+                                                [/version/]                            []               [get]
+                                                [/version]                             []               [get]
+                                                [/version]                             []               [get]
+```
+