Support multi-arch images with docker-buildx

Author: empovit

Dockerfile.controller

@@ -1,13 +1,10 @@
-ARG GOLANG_VERSION=1.22.2
-FROM nvidia/cuda:12.4.1-base-ubuntu22.04 as build
+ARG CUDA_VERSION=12.4.1
+ARG BASE_DIST=ubi8
+FROM nvcr.io/nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST} AS build
 
-RUN apt-get update && \
-    apt-get install -y wget make git gcc \
-    && \
-    rm -rf /var/lib/apt/lists/*
+ARG GOLANG_VERSION=1.22.2
+RUN yum install -y wget make git gcc
 
-ARG GOLANG_VERSION=x.x.x
-#TODO: Remove arch discovery
 RUN set -eux; \
     \
     arch="$(uname -m)"; \
@@ -17,7 +14,7 @@ RUN set -eux; \
         aarch64) ARCH='arm64' ;; \
         *) echo "unsupported architecture" ; exit 1 ;; \
     esac; \
-       wget -nv -O - https://storage.googleapis.com/golang/go1.22.2.linux-amd64.tar.gz \
+       wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
     | tar -C /usr/local -xz
 
 ENV GOPATH /go
@@ -43,11 +40,12 @@ COPY internal/controller/instaslice_controller.go internal/controller/instaslice
 # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
 RUN go build -o bin/manager cmd/controller/main.go
 
-FROM nvidia/cuda:12.4.1-base-ubuntu22.04
+ARG CUDA_VERSION=12.4.1
+ARG BASE_DIST=ubi8
+FROM nvcr.io/nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
 
 # Remove CUDA libs(compat etc) in favor of libs installed by the NVIDIA driver
-RUN rm -f cuda-*.deb
-RUN apt-get --purge -y autoremove cuda-*
+RUN dnf remove -y cuda-*
 
 ENV NVIDIA_DISABLE_REQUIRE="true"
 ENV NVIDIA_VISIBLE_DEVICES=all
@@ -60,9 +58,8 @@ COPY --from=build /workspace/bin/manager .
 # Install / upgrade packages here that are required to resolve CVEs
 ARG CVE_UPDATES
 RUN if [ -n "${CVE_UPDATES}" ]; then \
-        rm -f /etc/apt/sources.list.d/cuda.list && \
-        apt-get update && apt-get upgrade -y ${CVE_UPDATES} && \
-        rm -rf /var/lib/apt/lists/*; \
+        yum update -y ${CVE_UPDATES} && \
+        rm -rf /var/cache/yum/*; \
     fi
 
 ENTRYPOINT ["/manager"]

Dockerfile.daemonset

@@ -1,13 +1,10 @@
-ARG GOLANG_VERSION=1.22.2
-FROM nvidia/cuda:12.4.1-base-ubuntu22.04 as build
+ARG CUDA_VERSION=12.4.1
+ARG BASE_DIST=ubi8
+FROM nvcr.io/nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST} AS build
 
-RUN apt-get update && \
-    apt-get install -y wget make git gcc \
-    && \
-    rm -rf /var/lib/apt/lists/*
+ARG GOLANG_VERSION=1.22.2
+RUN yum install -y wget make git gcc
 
-ARG GOLANG_VERSION=x.x.x
-#TODO: Remove arch discovery
 RUN set -eux; \
     \
     arch="$(uname -m)"; \
@@ -17,7 +14,7 @@ RUN set -eux; \
         aarch64) ARCH='arm64' ;; \
         *) echo "unsupported architecture" ; exit 1 ;; \
     esac; \
-       wget -nv -O - https://storage.googleapis.com/golang/go1.22.2.linux-amd64.tar.gz \
+    wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
     | tar -C /usr/local -xz
 
 ENV GOPATH /go
@@ -43,27 +40,26 @@ COPY internal/controller/instaslice_daemonset.go internal/controller/instaslice_
 # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
 RUN go build -o bin/daemonset cmd/daemonset/main.go
 
-FROM nvidia/cuda:12.4.1-base-ubuntu22.04
+ARG CUDA_VERSION=12.4.1
+ARG BASE_DIST=ubi8
+FROM nvcr.io/nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
 
 # Remove CUDA libs(compat etc) in favor of libs installed by the NVIDIA driver
-RUN rm -f cuda-*.deb
-RUN apt-get --purge -y autoremove cuda-*
+RUN dnf remove -y cuda-*
 
 ENV NVIDIA_DISABLE_REQUIRE="true"
 ENV NVIDIA_VISIBLE_DEVICES=all
 ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
 
-
 WORKDIR /
 
 COPY --from=build /workspace/bin/daemonset .
 
 # Install / upgrade packages here that are required to resolve CVEs
 ARG CVE_UPDATES
 RUN if [ -n "${CVE_UPDATES}" ]; then \
-        rm -f /etc/apt/sources.list.d/cuda.list && \
-        apt-get update && apt-get upgrade -y ${CVE_UPDATES} && \
-        rm -rf /var/lib/apt/lists/*; \
+        yum update -y ${CVE_UPDATES} && \
+        rm -rf /var/cache/yum/*; \
     fi
 
 ENTRYPOINT ["/daemonset"]

Makefile

@@ -1,6 +1,7 @@
 # Image URL to use all building/pushing image targets
 IMG ?= asm582/instaslicev2-controller:latest
 IMG_DMST ?= asm582/instaslicev2-daemonset:latest
+
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.29.0
 
@@ -17,6 +18,13 @@ endif
 # tools. (i.e. podman)
 CONTAINER_TOOL ?= docker
 
+ifeq ($(CONTAINER_TOOL),podman)
+MULTI_ARCH_OPTION=--manifest
+else
+MULTI_ARCH_OPTION=--push --provenance=false --tag
+endif
+
+
 # Setting SHELL to bash allows bash commands to be executed by recipes.
 # Options are set to exit when a recipe line exits non-zero or a piped command fails.
 SHELL = /usr/bin/env bash -o pipefail
@@ -93,11 +101,11 @@ build: manifests generate fmt vet ## Build manager binary.
 	go build -o bin/daemonset cmd/daemonset/main.go
 .PHONY: run-controller
 run-controller: manifests generate fmt vet ## Run a controller from your host.
-	sudo -E go run ./cmd/controller/main.go 
+	sudo -E go run ./cmd/controller/main.go
 
 .PHONY: run-daemonset
 run-daemonset: manifests generate fmt vet ## Run a controller from your host.
-	sudo -E go run ./cmd/daemonset/main.go 
+	sudo -E go run ./cmd/daemonset/main.go
 
 # If you wish to build the manager image targeting other platforms you can use the --platform flag.
 # (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.
@@ -113,21 +121,23 @@ docker-push: ## Push docker image with the manager.
 	$(CONTAINER_TOOL) push ${IMG_DMST}
 
 # PLATFORMS defines the target platforms for the manager image be built to provide support to multiple
-# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
-# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/
-# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/
-# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=<myregistry/image:<tag>> then the export will fail)
-# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option.
-PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le
+# architectures. Make sure that base image in the Dockerfile/Containerfile is itself multi-platform, and includes
+# the requested plaforms. Unlike "docker buildx", for multi-platform images podman requires creating a manifest.
+PLATFORMS ?= linux/arm64,linux/amd64
 .PHONY: docker-buildx
-docker-buildx: ## Build and push docker image for the manager for cross-platform support
-	# copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile
-	sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross
-	- $(CONTAINER_TOOL) buildx create --name project-v3-builder
-	$(CONTAINER_TOOL) buildx use project-v3-builder
-	- $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross .
-	- $(CONTAINER_TOOL) buildx rm project-v3-builder
-	rm Dockerfile.cross
+docker-buildx: ## Build and push docker images with multi-platform support
+	if [ "$(CONTAINER_TOOL)" == "podman" ]; then \
+	  $(CONTAINER_TOOL) manifest rm ${IMG} || true; \
+	  $(CONTAINER_TOOL) manifest create ${IMG}; \
+	  $(CONTAINER_TOOL) manifest rm ${IMG_DMST} || true; \
+	  $(CONTAINER_TOOL) manifest create ${IMG_DMST}; \
+	fi
+	DOCKER_BUILDKIT=1 $(CONTAINER_TOOL) buildx build --platform=$(PLATFORMS) $(MULTI_ARCH_OPTION) ${IMG} -f Dockerfile.controller .
+	DOCKER_BUILDKIT=1 $(CONTAINER_TOOL) buildx build --platform=$(PLATFORMS) $(MULTI_ARCH_OPTION) ${IMG_DMST} -f Dockerfile.daemonset .
+	if [ "$(CONTAINER_TOOL)" == "podman" ]; then \
+	  $(CONTAINER_TOOL) manifest push ${IMG}; \
+	  $(CONTAINER_TOOL) manifest push ${IMG_DMST}; \
+	fi
 
 .PHONY: build-installer
 build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment.

README.md

@@ -9,6 +9,7 @@ Experimental InstaSlice works with GPU operator to create mig slices on demand.
 ### Prerequisites
 - [Go](https://go.dev/doc/install) v1.22.0+
 - [Docker](https://docs.docker.com/get-docker/) v17.03+
+- [Docker buildx plugin](https://github.com/docker/buildx) for building cross-platform images.
 - [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) v1.11.3+.
 - Access to a [KinD](https://kind.sigs.k8s.io/docs/user/quick-start/) cluster.
 
@@ -42,7 +43,7 @@ Experimental InstaSlice works with GPU operator to create mig slices on demand.
 |==================+==================================+===========+=======================|
 |  No MIG devices found                                                                   |
 +-----------------------------------------------------------------------------------------+
-                                                                                         
+
 +-----------------------------------------------------------------------------------------+
 | Processes:                                                                              |
 |  GPU   GI   CI        PID   Type   Process name                              GPU Memory |
@@ -77,7 +78,7 @@ nvidia-operator-validator-kh6jf                                   1/1     Runnin
 
 ```sh
 (base) openstack@netsres62:~/asmalvan/instaslice2$ nvidia-smi
-Thu Apr 25 10:08:24 2024       
+Thu Apr 25 10:08:24 2024
 +-----------------------------------------------------------------------------------------+
 | NVIDIA-SMI 550.54.14              Driver Version: 550.54.14      CUDA Version: 12.4     |
 |-----------------------------------------+------------------------+----------------------+
@@ -125,15 +126,15 @@ Thu Apr 25 10:08:24 2024
 |  1   10   0   3  |              12MiB /  4864MiB    | 14      0 |  1   0    0    0    0 |
 |                  |                 0MiB /  8191MiB  |           |                       |
 +------------------+----------------------------------+-----------+-----------------------+
-                                                                                         
+
 +-----------------------------------------------------------------------------------------+
 | Processes:                                                                              |
 |  GPU   GI   CI        PID   Type   Process name                              GPU Memory |
 |        ID   ID                                                               Usage      |
 |=========================================================================================|
 |  No running processes found                                                             |
 +-----------------------------------------------------------------------------------------+
-(base) openstack@netsres62:~/asmalvan/instaslice2$ 
+(base) openstack@netsres62:~/asmalvan/instaslice2$
 ```
 
 
@@ -239,7 +240,7 @@ Done
 |  1    2   0   0  |              37MiB / 19968MiB    | 42      0 |  3   0    2    0    0 |
 |                  |                 0MiB / 32767MiB  |           |                       |
 +------------------+----------------------------------+-----------+-----------------------+
-                                                                                         
+
 +-----------------------------------------------------------------------------------------+
 | Processes:                                                                              |
 |  GPU   GI   CI        PID   Type   Process name                              GPU Memory |
@@ -285,7 +286,7 @@ kubectl delete pod cuda-vectoradd-5
 |  1    2   0   0  |              37MiB / 19968MiB    | 42      0 |  3   0    2    0    0 |
 |                  |                 0MiB / 32767MiB  |           |                       |
 +------------------+----------------------------------+-----------+-----------------------+
-                                                                                         
+
 +-----------------------------------------------------------------------------------------+
 | Processes:                                                                              |
 |  GPU   GI   CI        PID   Type   Process name                              GPU Memory |
@@ -300,7 +301,20 @@ kubectl delete pod cuda-vectoradd-5
 
 **All in one command**
 
-make docker-build && make docker-push && make deploy 
+make docker-build && make docker-push && make deploy
+
+Cross-platform or multi-arch images can be built and pushed using
+`make docker-buildx`. When using Docker as your container tool, make
+sure to create a builder instance. Refer to
+[Multi-platform images](https://docs.docker.com/build/building/multi-platform/)
+for documentation on building mutli-platform images with Docker.
+
+You can change the destination platform(s) by
+setting `PLATFORMS`, e.g.
+
+```sh
+PLATFORMS=linux/arm64,linux/amd64 make docker-buildx
+```
 
 **Build and push your image to the location specified by `IMG`:**