fix: Create NVIDIA control device nodes in nvidia-cdi-refresh.service

Signed-off-by: Seungmin Kim <8457324+ehfd@users.noreply.github.com>

Author: ehfd

deployments/systemd/nvidia-cdi-refresh.env

@@ -18,3 +18,12 @@
 # For example, to change the for the generated CDI specification update and
 # uncomment the following line:
 # NVIDIA_CTK_CDI_OUTPUT_FILE_PATH=/var/run/cdi/nvidia.yaml
+
+# The service also runs
+#   nvidia-ctk system create-device-nodes --control-devices --load-kernel-modules
+# before generating the CDI specification. Its driver and device roots can be
+# overridden by uncommenting the following lines (nvidia-ctk cdi generate uses
+# the separate NVIDIA_CTK_DRIVER_ROOT and NVIDIA_CTK_DEV_ROOT variables):
+#
+# NVIDIA_DRIVER_ROOT=/
+# NVIDIA_DEV_ROOT=/

deployments/systemd/nvidia-cdi-refresh.service

@@ -18,6 +18,9 @@ ConditionPathExists=|/usr/bin/nvidia-smi
 ConditionPathExists=|/usr/sbin/nvidia-smi
 ConditionPathExists=|/usr/lib/wsl/lib/nvidia-smi
 ConditionPathExists=/usr/bin/nvidia-ctk
+# Order container engines after this service so that the first container
+# started at boot does not race the device node creation and CDI refresh.
+Before=docker.service containerd.service crio.service
 # Limit the number of successive restarts to 5 in 10 seconds.
 StartLimitBurst=5
 StartLimitIntervalSec=10s
@@ -29,6 +32,11 @@ Environment=NVIDIA_CTK_CDI_OUTPUT_FILE_PATH=/var/run/cdi/nvidia.yaml
 EnvironmentFile=-/etc/nvidia-container-toolkit/nvidia-cdi-refresh.env
 ExecCondition=/bin/sh -c '/usr/bin/grep -qE "/(nvidia|nvidia-current)[.]ko" /lib/modules/%v/modules.dep || [ -e /dev/dxg ]'
 ExecStart=/bin/sh -c '/usr/bin/nvidia-smi -L || /usr/sbin/nvidia-smi -L || /usr/lib/wsl/lib/nvidia-smi -L'
+# Create any missing NVIDIA control device nodes (loading the kernel modules
+# they require) before generating the CDI specification; nothing else is
+# guaranteed to create them. Best-effort, and skipped on WSL where /dev/dxg is
+# used instead.
+ExecStart=-/bin/sh -c '[ -e /dev/dxg ] || /usr/bin/nvidia-ctk system create-device-nodes --control-devices --load-kernel-modules'
 ExecStart=/usr/bin/nvidia-ctk cdi generate
 CapabilityBoundingSet=CAP_SYS_MODULE CAP_SYS_ADMIN CAP_MKNOD
 # We set the service to restart on failure to ensure that a CDI spec is

deployments/udev/99-nvidia-cdi-refresh.rules

@@ -0,0 +1,20 @@
+# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The NVIDIA control device nodes (/dev/nvidiactl, /dev/nvidia-modeset, and
+# /dev/nvidia-uvm*) are created on demand by userspace and are not provided by
+# devtmpfs, so loading the NVIDIA kernel module does not guarantee that they
+# exist. Trigger nvidia-cdi-refresh.service on module load to create the
+# missing nodes and refresh the CDI specification.
+ACTION=="add", SUBSYSTEM=="module", KERNEL=="nvidia", TAG+="systemd", ENV{SYSTEMD_WANTS}+="nvidia-cdi-refresh.service"

docker/Dockerfile.debian

@@ -56,6 +56,7 @@ RUN make PREFIX=${DIST_DIR} cmds
 WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 COPY deployments/systemd/ .
+COPY deployments/udev/ .
 
 RUN dch --create --package="${PKG_NAME}" \
         --newversion "${REVISION}" \

docker/Dockerfile.opensuse-leap

@@ -48,6 +48,7 @@ RUN make PREFIX=${DIST_DIR} cmds
 WORKDIR $DIST_DIR/..
 COPY packaging/rpm .
 COPY deployments/systemd/ ${DIST_DIR}/
+COPY deployments/udev/ ${DIST_DIR}/
 
 CMD arch=$(uname -m) && \
     rpmbuild --clean --target=$arch -bb \

docker/Dockerfile.rpm-yum

@@ -76,6 +76,7 @@ RUN make PREFIX=${DIST_DIR} cmds
 WORKDIR $DIST_DIR/..
 COPY packaging/rpm .
 COPY deployments/systemd/ ${DIST_DIR}/
+COPY deployments/udev/ ${DIST_DIR}/
 
 CMD arch=$(uname -m) && \
     rpmbuild --clean --target=$arch -bb \

docker/Dockerfile.ubuntu

@@ -54,6 +54,7 @@ RUN make PREFIX=${DIST_DIR} cmds
 WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 COPY deployments/systemd/ .
+COPY deployments/udev/ .
 
 RUN dch --create --package="${PKG_NAME}" \
         --newversion "${REVISION}" \

packaging/debian/nvidia-container-toolkit-base.install

@@ -4,3 +4,4 @@ nvidia-cdi-hook /usr/bin
 nvidia-cdi-refresh.service /lib/systemd/system/
 nvidia-cdi-refresh.path /lib/systemd/system/
 nvidia-cdi-refresh.env /etc/nvidia-container-toolkit/
+99-nvidia-cdi-refresh.rules /lib/udev/rules.d/

packaging/debian/nvidia-container-toolkit-base.postinst

@@ -5,6 +5,12 @@ set -e
 case "$1" in
     configure)
         /usr/bin/nvidia-ctk --quiet config --config-file=/etc/nvidia-container-runtime/config.toml --in-place
+
+        # Reload udev rules so that the nvidia-cdi-refresh rules take effect
+        # without a reboot.
+        if command -v udevadm >/dev/null 2>&1; then
+            udevadm control --reload-rules >/dev/null 2>&1 || true
+        fi
     ;;
 
     abort-upgrade|abort-remove|abort-deconfigure)

packaging/debian/rules

@@ -16,3 +16,4 @@ override_dh_fixperms:
 	chmod 755 debian/$(shell dh_listpackages)/usr/bin/nvidia-cdi-hook || true
 	chmod 644 debian/$(shell dh_listpackages)/lib/systemd/system/nvidia-cdi-refresh.service || true
 	chmod 644 debian/$(shell dh_listpackages)/lib/systemd/system/nvidia-cdi-refresh.path || true
+	chmod 644 debian/$(shell dh_listpackages)/lib/udev/rules.d/99-nvidia-cdi-refresh.rules || true