Fix issue45 - support for serviceAccount in slurm jobs

.github/workflows/build_images.yaml

@@ -29,6 +29,18 @@ jobs:
       - name: Get Repo Owner
         id: get_repo_owner
         run: echo ::set-output name=repo_owner::$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')
+
+      # See https://docs.docker.com/build/ci/github-actions/cache/ for cache to speed go build
+      - name: Go Build Cache for Docker
+        uses: actions/cache@v4
+        with:
+          path: go-build-cache
+          key: ${{ runner.os }}-go-build-cache-${{ hashFiles('**/go.sum') }}
+      - name: Inject go-build-cache
+        uses: reproducible-containers/buildkit-cache-dance@4b2444fec0c0fb9dbf175a96c094720a692ef810 # v2.1.4
+        with:
+          cache-source: go-build-cache
+
       - name: Build container base image vk
         uses: docker/build-push-action@v5
         with:

ci/manifests/service-account.yaml

@@ -33,6 +33,13 @@ rules:
   - get
   - list
   - watch
+# For https://kubernetes.io/docs/reference/kubernetes-api/authentication-resources/token-request-v1/
+- apiGroups: [""]
+  resources: ["serviceaccounts/token"]
+  verbs:
+  - create
+  - get
+  - list
 - apiGroups:
   - ""
   resources:

docker/Dockerfile.interlink

@@ -13,8 +13,8 @@ RUN mkdir -p $GOMODCACHE && mkdir -p $GOCACHE
 ARG VERSION
 RUN bash -c "KUBELET_VERSION=${VERSION} ./cmd/virtual-kubelet/set-version.sh"
 
-RUN go mod tidy
-RUN CGO_ENABLED=0 GOOS=linux go build -o bin/interlink cmd/interlink/main.go
+RUN --mount=type=cache,target=/go/pkg/mod bash -c "time go mod tidy"
+RUN --mount=type=cache,target=/go/build-cache bash -c "time CGO_ENABLED=0 GOOS=linux go build -o bin/interlink cmd/interlink/main.go"
 
 # Deploy the application binary into a lean image
 FROM gcr.io/distroless/base-debian11:latest AS build-release-stage

docker/Dockerfile.vk

@@ -15,8 +15,8 @@ RUN mkdir -p $GOMODCACHE && mkdir -p $GOCACHE
 
 
 RUN bash -c "KUBELET_VERSION=${VERSION} ./cmd/virtual-kubelet/set-version.sh"
-RUN go mod tidy
-RUN CGO_ENABLED=0 GOOS=linux go build -o bin/vk cmd/virtual-kubelet/main.go
+RUN --mount=type=cache,target=/go/pkg/mod bash -c "time go mod tidy"
+RUN --mount=type=cache,target=/go/build-cache bash -c "time CGO_ENABLED=0 GOOS=linux go build -o bin/vk cmd/virtual-kubelet/main.go"
 
 # Deploy the application binary into a lean image
 FROM ubuntu:22.04 AS build-release-stage

example/interlink-docker/vk/service-account.yaml

@@ -33,6 +33,13 @@ rules:
   - get
   - list
   - watch
+# For https://kubernetes.io/docs/reference/kubernetes-api/authentication-resources/token-request-v1/
+- apiGroups: [""]
+  resources: ["serviceaccounts/token"]
+  verbs:
+  - create
+  - get
+  - list
 - apiGroups:
   - ""
   resources:

pkg/interlink/api/create.go

@@ -64,6 +64,17 @@ func (h *InterLinkHandler) CreateHandler(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
+	if log.G(h.Ctx).Logger.IsLevelEnabled(log.DebugLevel) {
+		// For debugging purpose only.
+		allContainers := pod.Pod.Spec.InitContainers
+		allContainers = append(allContainers, pod.Pod.Spec.Containers...)
+		for _, container := range allContainers {
+			for _, envVar := range container.Env {
+				log.G(h.Ctx).Debug("InterLink VK environment variable to pod ", pod.Pod.Name, " container: ", container.Name, " env: ", envVar.Name, " value: ", envVar.Value)
+			}
+		}
+	}
+
 	retrievedData = append(retrievedData, data)
 
 	if retrievedData != nil {

pkg/interlink/api/func.go

@@ -2,7 +2,7 @@
 
 import (
 	"context"
-	"path/filepath"
+	"strings"
 	"sync"
 	"time"
 
@@ -33,17 +33,17 @@
 
 	for _, container := range pod.Pod.Spec.InitContainers {
 		startContainer := time.Now().UnixMicro()
 		log.G(ctx).Info("- Retrieving Secrets and ConfigMaps for the Docker Sidecar. InitContainer: " + container.Name)
 		log.G(ctx).Debug(container.VolumeMounts)
-		data, InterlinkIP := retrieveData(ctx, config, pod, container)
-		if InterlinkIP != nil {
-			log.G(ctx).Error(InterlinkIP)
-			return types.RetrievedPodData{}, InterlinkIP
+		data, err := retrieveData(ctx, config, pod, container)
+		if err != nil {
+			log.G(ctx).Error(err)
+			return types.RetrievedPodData{}, err
 		}
 		retrievedData.Containers = append(retrievedData.Containers, data)
 
 		durationContainer := time.Now().UnixMicro() - startContainer
 		span.AddEvent("Init Container "+container.Name, trace.WithAttributes(
 			attribute.Int64("initcontainer.getdata.duration", durationContainer),
 			attribute.String("pod.name", pod.Pod.Name)))
 	}
@@ -73,42 +73,78 @@
 // retrieveData retrieves ConfigMaps, Secrets and EmptyDirs.
 // The config is needed to specify the EmptyDirs mounting point.
 // It returns the retrieved data in a variable of type commonIL.RetrievedContainer and the first encountered error.
-func retrieveData(ctx context.Context, config types.Config, pod types.PodCreateRequests, container v1.Container) (types.RetrievedContainer, error) {
+func retrieveData(ctx context.Context, _ types.Config, pod types.PodCreateRequests, container v1.Container) (types.RetrievedContainer, error) {
 	retrievedData := types.RetrievedContainer{}
+	retrievedData.Name = container.Name
 	for _, mountVar := range container.VolumeMounts {
-		log.G(ctx).Debug("-- Retrieving data for mountpoint " + mountVar.Name)
+		log.G(ctx).Debug("-- Retrieving data for mountpoint ", mountVar.Name)
 
+	loopVolumes:
 		for _, vol := range pod.Pod.Spec.Volumes {
 			if vol.Name == mountVar.Name {
 				switch {
 				case vol.ConfigMap != nil:
-
-					log.G(ctx).Info("--- Retrieving ConfigMap " + vol.ConfigMap.Name)
-					retrievedData.Name = container.Name
+					log.G(ctx).Info("--- Retrieving ConfigMap ", vol.ConfigMap.Name)
 					for _, cfgMap := range pod.ConfigMaps {
 						if cfgMap.Name == vol.ConfigMap.Name {
-							retrievedData.Name = container.Name
+							log.G(ctx).Debug("configMap found! Name: ", cfgMap.Name)
 							retrievedData.ConfigMaps = append(retrievedData.ConfigMaps, cfgMap)
+							break loopVolumes
+						}
+					}
+					// This should not happen, error. Building error context.
+					var configMapsKeys []string
+					for _, cfgMap := range pod.ConfigMaps {
+						configMapsKeys = append(configMapsKeys, cfgMap.Name)
+					}
+					log.G(ctx).Errorf("could not find in retrievedData the matching object for volume: %s (pod: %s container: %s configMap: %s) retrievedData keys: %s", vol.Name,
+						pod.Pod.Name, container.Name, vol.ConfigMap.Name, strings.Join(configMapsKeys, ","))
+
+				case vol.Projected != nil:
+					log.G(ctx).Info("--- Retrieving ProjectedVolume ", vol.Name)
+					for _, projectedVolumeMap := range pod.ProjectedVolumeMaps {
+						log.G(ctx).Debug("Comparing projectedVolumeMap.Name: ", projectedVolumeMap.Name, " with vol.Name: ", vol.Name)
+						if projectedVolumeMap.Name == vol.Name {
+							log.G(ctx).Debug("projectedVolumeMap found! Name: ", projectedVolumeMap.Name)
+
+							retrievedData.ProjectedVolumeMaps = append(retrievedData.ProjectedVolumeMaps, projectedVolumeMap)
+							break loopVolumes
 						}
 					}
+					// This should not happen, error. Building error context.
+					var projectedVolumeMapsKeys []string
+					for _, projectedVolumeMap := range pod.ProjectedVolumeMaps {
+						projectedVolumeMapsKeys = append(projectedVolumeMapsKeys, projectedVolumeMap.Name)
+					}
+					log.G(ctx).Errorf("could not find in retrievedData the matching object for  volume: %s (pod: %s container: %s projectedVolumeMap) retrievedData keys: %s",
+						vol.Name, pod.Pod.Name, container.Name, strings.Join(projectedVolumeMapsKeys, ","))
 
 				case vol.Secret != nil:
-
-					log.G(ctx).Info("--- Retrieving Secret " + vol.Secret.SecretName)
-					retrievedData.Name = container.Name
+					log.G(ctx).Info("--- Retrieving Secret ", vol.Secret.SecretName)
 					for _, secret := range pod.Secrets {
 						if secret.Name == vol.Secret.SecretName {
-							retrievedData.Name = container.Name
+							log.G(ctx).Debug("secret found! Name: ", secret.Name)
 							retrievedData.Secrets = append(retrievedData.Secrets, secret)
+							break loopVolumes
 						}
 					}
+					// This should not happen, error. Building error context.
+					var secretKeys []string
+					for _, secret := range pod.Secrets {
+						secretKeys = append(secretKeys, secret.Name)
+					}
+					log.G(ctx).Errorf("could not find in retrievedData the matching object for volume: %s (pod: %s container: %s secret: %s) retrievedData keys: %s",
+						pod.Pod.Name, container.Name, vol.Name, vol.Secret.SecretName, strings.Join(secretKeys, ","))
 
 				case vol.EmptyDir != nil:
-					edPath := filepath.Join(config.DataRootFolder, pod.Pod.Namespace+"-"+string(pod.Pod.UID)+"/"+"emptyDirs/"+vol.Name)
+					// Deprecated: EmptyDirs is useless at VK level. It should be moved to plugin level.
+					// edPath := filepath.Join(config.DataRootFolder, pod.Pod.Namespace+"-"+string(pod.Pod.UID), "emptyDirs", vol.Name)
+					// retrievedData.EmptyDirs = append(retrievedData.EmptyDirs, edPath)
 
-					retrievedData.Name = container.Name
-					retrievedData.EmptyDirs = append(retrievedData.EmptyDirs, edPath)
+				default:
+					log.G(ctx).Warning("ignoring unsupported volume type for ", mountVar.Name)
 				}
+
 			}
 		}
 	}

pkg/interlink/types.go

@@ -11,6 +11,9 @@ type PodCreateRequests struct {
 	Pod        v1.Pod         `json:"pod"`
 	ConfigMaps []v1.ConfigMap `json:"configmaps"`
 	Secrets    []v1.Secret    `json:"secrets"`
+	// The projected volumes are those created by ServiceAccounts (in K8S >= 1.24). They are automatically added in the pod from kubelet code.
+	// Here the configmap will hold the files name (as key) and content (as value).
+	ProjectedVolumeMaps []v1.ConfigMap `json:"projectedvolumesmaps"`
 }
 
 // PodStatus is a simplified v1.Pod struct, holding only necessary variables to uniquely identify a job/service in the sidecar. It is used to request
@@ -31,10 +34,14 @@ type CreateStruct struct {
 
 // RetrievedContainer is used in InterLink to rearrange data structure in a suitable way for the sidecar
 type RetrievedContainer struct {
-	Name       string         `json:"name"`
-	ConfigMaps []v1.ConfigMap `json:"configMaps"`
-	Secrets    []v1.Secret    `json:"secrets"`
-	EmptyDirs  []string       `json:"emptyDirs"`
+	Name                string         `json:"name"`
+	ConfigMaps          []v1.ConfigMap `json:"configMaps"`
+	ProjectedVolumeMaps []v1.ConfigMap `json:"projectedvolumemaps"`
+	Secrets             []v1.Secret    `json:"secrets"`
+	// Deprecated: EmptyDirs should be built on plugin side.
+	// Currently, it holds the DATA_ROOT_DIR/emptydirs/volumeName, but this should be a plugin choice instead,
+	// like it currently is for ConfigMaps, ProjectedVolumeMaps, Secrets.
+	EmptyDirs []string `json:"emptyDirs"`
 }
 
 // RetrievedPoData is used in InterLink to rearrange data structure in a suitable way for the sidecar

pkg/virtualkubelet/config.go

@@ -2,21 +2,24 @@ package virtualkubelet
 
 // Config holds the whole configuration
 type Config struct {
-	InterlinkURL      string `yaml:"InterlinkURL"`
-	Interlinkport     string `yaml:"InterlinkPort"`
-	VKConfigPath      string `yaml:"VKConfigPath"`
-	VKTokenFile       string `yaml:"VKTokenFile"`
-	ServiceAccount    string `yaml:"ServiceAccount"`
-	Namespace         string `yaml:"Namespace"`
-	PodIP             string `yaml:"PodIP"`
-	VerboseLogging    bool   `yaml:"VerboseLogging"`
-	ErrorsOnlyLogging bool   `yaml:"ErrorsOnlyLogging"`
-	HTTP              HTTP   `yaml:"HTTP"`
-	KubeletHTTP       HTTP   `yaml:"KubeletHTTP"`
-	CPU               string `yaml:"CPU,omitempty"`
-	Memory            string `yaml:"Memory,omitempty"`
-	Pods              string `yaml:"Pods,omitempty"`
-	GPU               string `yaml:"nvidia.com/gpu,omitempty"`
+	InterlinkURL       string `yaml:"InterlinkURL"`
+	Interlinkport      string `yaml:"InterlinkPort"`
+	KubernetesAPIAddr  string `yaml:"KubernetesApiAddr"`
+	KubernetesAPIPort  string `yaml:"KubernetesApiPort"`
+	KubernetesAPICaCrt string `yaml:"KubernetesApiCaCrt"`
+	VKConfigPath       string `yaml:"VKConfigPath"`
+	VKTokenFile        string `yaml:"VKTokenFile"`
+	ServiceAccount     string `yaml:"ServiceAccount"`
+	Namespace          string `yaml:"Namespace"`
+	PodIP              string `yaml:"PodIP"`
+	VerboseLogging     bool   `yaml:"VerboseLogging"`
+	ErrorsOnlyLogging  bool   `yaml:"ErrorsOnlyLogging"`
+	HTTP               HTTP   `yaml:"HTTP"`
+	KubeletHTTP        HTTP   `yaml:"KubeletHTTP"`
+	CPU                string `yaml:"CPU,omitempty"`
+	Memory             string `yaml:"Memory,omitempty"`
+	Pods               string `yaml:"Pods,omitempty"`
+	GPU                string `yaml:"nvidia.com/gpu,omitempty"`
 }
 
 type HTTP struct {