Skip to content

Commit 0389e8e

Browse files
ehfdehfd
committed
fix: Order container engines after nvidia-cdi-refresh.service
Signed-off-by: Seungmin Kim <8457324+ehfd@users.noreply.github.com>
1 parent d34b304 commit 0389e8e

5 files changed

Lines changed: 61 additions & 1 deletion

File tree

‎deployments/systemd/10-container-engines.conf‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# Copyright (c) NVIDIA CORPORATION. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# Order the container engines after this service so that the first container
16+
# started at boot does not race device node creation and CDI specification
17+
# generation.
18+
#
19+
# This is shipped as a drop-in rather than as part of the unit so that it can be
20+
# removed without overriding the whole unit: an empty file of the same name in
21+
# /etc/systemd/system/nvidia-cdi-refresh.service.d/ takes precedence over this
22+
# one and cancels it. Assigning an empty Before= in a drop-in does not reset the
23+
# list, so an in-unit ordering could not be undone that way.
24+
[Unit]
25+
Before=docker.service containerd.service crio.service
26+
27+
[Service]
28+
# The unit is Type=oneshot, for which systemd defaults to
29+
# TimeoutStartSec=infinity. Now that the container engines are ordered after it,
30+
# bound the start so that a hung nvidia-smi cannot delay them indefinitely; on
31+
# expiry the engines start and the CDI specification is refreshed on the next
32+
# trigger. The value matches systemd's DefaultTimeoutStartSec.
33+
TimeoutStartSec=90s

‎packaging/debian/nvidia-container-toolkit-base.install‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@ nvidia-cdi-refresh.service /lib/systemd/system/
55
nvidia-cdi-refresh.path /lib/systemd/system/
66
nvidia-cdi-refresh.env /etc/nvidia-container-toolkit/
77
99-nvidia-cdi-refresh.rules /lib/udev/rules.d/
8+
10-container-engines.conf /lib/systemd/system/nvidia-cdi-refresh.service.d/

‎packaging/debian/rules‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,4 @@ override_dh_fixperms:
1717
chmod 644 debian/$(shell dh_listpackages)/lib/systemd/system/nvidia-cdi-refresh.service || true
1818
chmod 644 debian/$(shell dh_listpackages)/lib/systemd/system/nvidia-cdi-refresh.path || true
1919
chmod 644 debian/$(shell dh_listpackages)/lib/udev/rules.d/99-nvidia-cdi-refresh.rules || true
20+
chmod 644 debian/$(shell dh_listpackages)/lib/systemd/system/nvidia-cdi-refresh.service.d/10-container-engines.conf || true

‎packaging/rpm/SPECS/nvidia-container-toolkit.spec‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ Source8: nvidia-cdi-refresh.path
2222
Source9: nvidia-cdi-refresh.env
2323
Source10: 90-nvidia-container-toolkit.preset
2424
Source11: 99-nvidia-cdi-refresh.rules
25+
Source12: 10-container-engines.conf
2526

2627
%if 0%{?rhel} == 7 || 0%{?amzn} == 2
2728
BuildRequires: systemd
@@ -43,13 +44,14 @@ Requires: nvidia-container-toolkit-base == %{version}-%{release}
4344
Provides tools and utilities to enable GPU support in containers.
4445

4546
%prep
46-
cp %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} %{SOURCE5} %{SOURCE6} %{SOURCE7} %{SOURCE8} %{SOURCE9} %{SOURCE10} %{SOURCE11} .
47+
cp %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} %{SOURCE5} %{SOURCE6} %{SOURCE7} %{SOURCE8} %{SOURCE9} %{SOURCE10} %{SOURCE11} %{SOURCE12} .
4748

4849
%install
4950
mkdir -p %{buildroot}%{_bindir}
5051
mkdir -p %{buildroot}%{_unitdir}
5152
mkdir -p %{buildroot}%{_presetdir}
5253
mkdir -p %{buildroot}%{_udevrulesdir}
54+
mkdir -p %{buildroot}%{_unitdir}/nvidia-cdi-refresh.service.d
5355
mkdir -p %{buildroot}%{_sysconfdir}/nvidia-container-toolkit
5456

5557
install -m 755 -t %{buildroot}%{_bindir} nvidia-container-runtime-hook
@@ -62,6 +64,7 @@ install -m 644 -t %{buildroot}%{_unitdir} nvidia-cdi-refresh.service
6264
install -m 644 -t %{buildroot}%{_unitdir} nvidia-cdi-refresh.path
6365
install -m 644 -t %{buildroot}%{_presetdir} 90-nvidia-container-toolkit.preset
6466
install -m 644 -t %{buildroot}%{_udevrulesdir} 99-nvidia-cdi-refresh.rules
67+
install -m 644 -t %{buildroot}%{_unitdir}/nvidia-cdi-refresh.service.d 10-container-engines.conf
6568
install -m 644 -t %{buildroot}%{_sysconfdir}/nvidia-container-toolkit nvidia-cdi-refresh.env
6669

6770
%post
@@ -162,6 +165,7 @@ fi
162165
%{_unitdir}/nvidia-cdi-refresh.path
163166
%{_presetdir}/90-nvidia-container-toolkit.preset
164167
%{_udevrulesdir}/99-nvidia-cdi-refresh.rules
168+
%{_unitdir}/nvidia-cdi-refresh.service.d/10-container-engines.conf
165169
%config(noreplace) %{_sysconfdir}/nvidia-container-toolkit/nvidia-cdi-refresh.env
166170

167171
# The OPERATOR EXTENSIONS package consists of components that are required to enable GPU support in Kubernetes.

‎tests/e2e/nvidia-cdi-refresh_test.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,17 @@ EOF
100100
fi
101101
`
102102

103+
nvidiaCdiRefreshOrderingDropInInstalledTemplate = `
104+
if [ ! -f /lib/systemd/system/nvidia-cdi-refresh.service.d/10-container-engines.conf ]; then
105+
echo "10-container-engines.conf is not installed"
106+
exit 1
107+
fi
108+
if ! systemctl show nvidia-cdi-refresh.service -p Before | grep -q docker.service; then
109+
echo "nvidia-cdi-refresh.service is not ordered before docker.service"
110+
exit 1
111+
fi
112+
`
113+
103114
nvidiaCdiRefreshFileExistsTemplate = `
104115
# is /var/run/cdi/nvidia.yaml exists? and exit with 0 if it does not exist
105116
if [ ! -f /var/run/cdi/nvidia.yaml ]; then
@@ -221,6 +232,11 @@ var _ = Describe("nvidia-cdi-refresh", Ordered, ContinueOnFailure, Label("system
221232
Expect(err).ToNot(HaveOccurred())
222233
})
223234

235+
It("should install the container engine ordering drop-in", func(ctx context.Context) {
236+
_, _, err := systemdRunner.Run(nvidiaCdiRefreshOrderingDropInInstalledTemplate)
237+
Expect(err).ToNot(HaveOccurred())
238+
})
239+
224240
It("should generate the nvidia.yaml file", func(ctx context.Context) {
225241
_, _, err := systemdRunner.Run(nvidiaCdiRefreshFileExistsTemplate)
226242
Expect(err).ToNot(HaveOccurred())
@@ -275,6 +291,11 @@ var _ = Describe("nvidia-cdi-refresh", Ordered, ContinueOnFailure, Label("system
275291
Expect(err).ToNot(HaveOccurred())
276292
})
277293

294+
It("should install the container engine ordering drop-in", func(ctx context.Context) {
295+
_, _, err := systemdRunner.Run(nvidiaCdiRefreshOrderingDropInInstalledTemplate)
296+
Expect(err).ToNot(HaveOccurred())
297+
})
298+
278299
It("should generate the nvidia.yaml file", func(ctx context.Context) {
279300
_, _, err := systemdRunner.Run(nvidiaCdiRefreshFileExistsTemplate)
280301
Expect(err).ToNot(HaveOccurred())

0 commit comments

Comments
 (0)