Add e2e test for CUDA Minor Version Compatibility

Signed-off-by: Christopher Desiniotis <cdesiniotis@nvidia.com>

Author: cdesiniotis

.github/workflows/e2e.yaml

@@ -49,8 +49,9 @@ jobs:
           # Production Branch. EOL March 2027
           - 595
         exclude:
-          - ispr: true
-            driver_branch: 580
+          # temporary change to test CUDA minor version compatibility with the 580 branch
+          #- ispr: true
+          #  driver_branch: 580
           - ispr: true
             driver_branch: 595
     steps:

tests/e2e/infra/driver-branch-580.yaml

@@ -29,4 +29,8 @@ spec:
     install: true
     source: package
     package:
-      branch: "580"
+      # We pin the version here to allow us to test CUDA Minor Version Compatibility.
+      # If we deploy a CUDA 13.0.3+ container (associated with NVIDIA Linux Driver
+      # version 580.126.20), NVIDIA Container Toolkit should use the CUDA compat
+      # libraries in the container instead of the 580.105.08 libraries on the host.
+      version: "580.105.08"

tests/e2e/nvidia-container-toolkit_test.go

@@ -224,6 +224,50 @@ var _ = Describe("docker", Ordered, ContinueOnFailure, func() {
 		})
 	})
 
+	When("Testing CUDA Minor Version compatibility", Ordered, func() {
+		BeforeAll(func(ctx context.Context) {
+			/*
+			 * CUDA 13.0.3, NVIDIA Linux Driver 580.126.20
+			 */
+			_, _, err := runner.Run("docker pull nvcr.io/nvidia/cuda:13.0.3-base-ubi8")
+			Expect(err).ToNot(HaveOccurred())
+
+			compatOutput, _, err := runner.Run("docker run --rm -i -e NVIDIA_VISIBLE_DEVICES=void nvcr.io/nvidia/cuda:13.0.3-base-ubi8 bash -c \"ls /usr/local/cuda/compat/libcuda.*.*\"")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(compatOutput).ToNot(BeEmpty())
+
+			compatDriverVersion := strings.TrimPrefix(filepath.Base(compatOutput), "libcuda.so.")
+			compatMajor := strings.SplitN(compatDriverVersion, ".", 2)[0]
+
+			if hostDriverMajor != compatMajor {
+				GinkgoLogr.Info("CUDA Minor Version Compatibility tests require the host driver branch to equal the compat driver branch", "hostDriverVersion", hostDriverVersion, "compatDriverVersion", compatDriverVersion)
+				Skip("CUDA Minor Version Compatibility tests require the host driver branch to equal the compat driver branch")
+			}
+		})
+
+		It("should work with the nvidia runtime in legacy mode", func(ctx context.Context) {
+			ldconfigOut, _, err := runner.Run("docker run --rm -i -e NVIDIA_DISABLE_REQUIRE=true --runtime=nvidia --gpus all nvcr.io/nvidia/cuda:13.0.3-base-ubi8 bash -c \"ldconfig -p | grep libcuda.so.1\"")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ldconfigOut).To(ContainSubstring("/usr/local/cuda-13.0/compat/"))
+		})
+
+		It("should work with the nvidia runtime in CDI mode", func(ctx context.Context) {
+			ldconfigOut, _, err := runner.Run("docker run --rm -i -e NVIDIA_DISABLE_REQUIRE=true  --runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=runtime.nvidia.com/gpu=all nvcr.io/nvidia/cuda:13.0.3-base-ubi8 bash -c \"ldconfig -p | grep libcuda.so.1\"")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ldconfigOut).To(ContainSubstring("/usr/local/cuda-13.0/compat/"))
+		})
+
+		It("should create a single ld.so.conf.d config file", func(ctx context.Context) {
+			lsout, _, err := runner.Run("docker run --rm -i -e NVIDIA_DISABLE_REQUIRE=true --runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=runtime.nvidia.com/gpu=all nvcr.io/nvidia/cuda:13.0.3-base-ubi8 bash -c \"ls -l /etc/ld.so.conf.d/00-compat-*.conf\"")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(lsout).To(WithTransform(
+				func(s string) []string {
+					return strings.Split(strings.TrimSpace(s), "\n")
+				}, HaveLen(1),
+			))
+		})
+	})
+
 	When("Disabling device node creation", Ordered, func() {
 		BeforeAll(func(ctx context.Context) {
 			_, _, err := runner.Run("docker pull ubuntu")