Skip to content

Commit 04deced

Browse files
andyatmiamiandyatmiami
committed
fix(automation): updates to support no chained builds
With the removal of chained builds, we needed to adapt some CI scripts to "play nice". - An 'all-images' target was introduced to the Makefile to allow the discovery of the full set of targets that correspond to image builds. - `gha_pr_changed_files.py` was updated to: - account for the fact a given Makefile target will no longer ever include mutiple `#\(MACHINE-PARSED LINE\)#\` in its output - A given build directory can now potentially contain multiple `Dockerfile` files with various extensions that require being analyzed - `gen_gha_matrix_jobs.py` was updated to: - actually invoke `make` (or `gmake`) to get the list of dependencies listed for `all-images` target - while a `subprocess` call was introduced (to reliably parse the actual `Makefile` targets that now include `Make` variables), this actually simplified the overall `Makefile` parsing logic - `make_tests.py` saw the removal of `test_make_building_only_specified_images` test as it is irrelevant without changed builds - removal of `BUILD_DEPENDENT_IMAGES` variable from `Makefile` will come in a future PR as it is also _defunct_ - `has_tests.py` updated to: - account for the fact a given Makefile target will no longer ever include mutiple `#\(MACHINE-PARSED LINE\)#\` in its output - i.e. there is only a single directory involved for a given `Makefile` `target` when dealing with `gha_pr_changed_files`
1 parent 5007663 commit 04deced

File tree

6 files changed

+171
-298
lines changed

6 files changed

+171
-298
lines changed

Diff for: ‎.github/workflows/build-notebooks.yaml

+22-131
Original file line numberDiff line numberDiff line change
@@ -15,19 +15,8 @@
1515
]
1616
},
1717
"jobs": {
18-
"base-ubi9-python-3_11": {
19-
"needs": [],
20-
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
21-
"with": {
22-
"target": "base-ubi9-python-3.11",
23-
"github": "${{ toJSON(github) }}"
24-
},
25-
"secrets": "inherit"
26-
},
2718
"jupyter-minimal-ubi9-python-3_11": {
28-
"needs": [
29-
"base-ubi9-python-3_11"
30-
],
19+
"needs": [],
3120
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
3221
"with": {
3322
"target": "jupyter-minimal-ubi9-python-3.11",
@@ -36,75 +25,43 @@
3625
"secrets": "inherit"
3726
},
3827
"jupyter-datascience-ubi9-python-3_11": {
39-
"needs": [
40-
"jupyter-minimal-ubi9-python-3_11"
41-
],
28+
"needs": [],
4229
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
4330
"with": {
4431
"target": "jupyter-datascience-ubi9-python-3.11",
4532
"github": "${{ toJSON(github) }}"
4633
},
4734
"secrets": "inherit"
4835
},
49-
"cuda-ubi9-python-3_11": {
50-
"needs": [
51-
"base-ubi9-python-3_11"
52-
],
53-
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
54-
"with": {
55-
"target": "cuda-ubi9-python-3.11",
56-
"github": "${{ toJSON(github) }}"
57-
},
58-
"secrets": "inherit"
59-
},
6036
"cuda-jupyter-minimal-ubi9-python-3_11": {
61-
"needs": [
62-
"cuda-ubi9-python-3_11"
63-
],
37+
"needs": [],
6438
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
6539
"with": {
6640
"target": "cuda-jupyter-minimal-ubi9-python-3.11",
6741
"github": "${{ toJSON(github) }}"
6842
},
6943
"secrets": "inherit"
7044
},
71-
"cuda-jupyter-datascience-ubi9-python-3_11": {
72-
"needs": [
73-
"cuda-jupyter-minimal-ubi9-python-3_11"
74-
],
75-
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
76-
"with": {
77-
"target": "cuda-jupyter-datascience-ubi9-python-3.11",
78-
"github": "${{ toJSON(github) }}"
79-
},
80-
"secrets": "inherit"
81-
},
8245
"cuda-jupyter-tensorflow-ubi9-python-3_11": {
83-
"needs": [
84-
"cuda-jupyter-datascience-ubi9-python-3_11"
85-
],
46+
"needs": [],
8647
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
8748
"with": {
8849
"target": "cuda-jupyter-tensorflow-ubi9-python-3.11",
8950
"github": "${{ toJSON(github) }}"
9051
},
9152
"secrets": "inherit"
9253
},
93-
"jupyter-pytorch-ubi9-python-3_11": {
94-
"needs": [
95-
"cuda-jupyter-datascience-ubi9-python-3_11"
96-
],
54+
"cuda-jupyter-pytorch-ubi9-python-3_11": {
55+
"needs": [],
9756
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
9857
"with": {
99-
"target": "jupyter-pytorch-ubi9-python-3.11",
58+
"target": "cuda-jupyter-pytorch-ubi9-python-3.11",
10059
"github": "${{ toJSON(github) }}"
10160
},
10261
"secrets": "inherit"
10362
},
10463
"jupyter-trustyai-ubi9-python-3_11": {
105-
"needs": [
106-
"jupyter-datascience-ubi9-python-3_11"
107-
],
64+
"needs": [],
10865
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
10966
"with": {
11067
"target": "jupyter-trustyai-ubi9-python-3.11",
@@ -113,9 +70,7 @@
11370
"secrets": "inherit"
11471
},
11572
"runtime-minimal-ubi9-python-3_11": {
116-
"needs": [
117-
"base-ubi9-python-3_11"
118-
],
73+
"needs": [],
11974
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
12075
"with": {
12176
"target": "runtime-minimal-ubi9-python-3.11",
@@ -124,31 +79,25 @@
12479
"secrets": "inherit"
12580
},
12681
"runtime-datascience-ubi9-python-3_11": {
127-
"needs": [
128-
"base-ubi9-python-3_11"
129-
],
82+
"needs": [],
13083
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
13184
"with": {
13285
"target": "runtime-datascience-ubi9-python-3.11",
13386
"github": "${{ toJSON(github) }}"
13487
},
13588
"secrets": "inherit"
13689
},
137-
"runtime-pytorch-ubi9-python-3_11": {
138-
"needs": [
139-
"base-ubi9-python-3_11"
140-
],
90+
"runtime-cuda-pytorch-ubi9-python-3_11": {
91+
"needs": [],
14192
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
14293
"with": {
143-
"target": "runtime-pytorch-ubi9-python-3.11",
94+
"target": "runtime-cuda-pytorch-ubi9-python-3.11",
14495
"github": "${{ toJSON(github) }}"
14596
},
14697
"secrets": "inherit"
14798
},
14899
"runtime-cuda-tensorflow-ubi9-python-3_11": {
149-
"needs": [
150-
"cuda-ubi9-python-3_11"
151-
],
100+
"needs": [],
152101
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
153102
"with": {
154103
"target": "runtime-cuda-tensorflow-ubi9-python-3.11",
@@ -157,40 +106,16 @@
157106
"secrets": "inherit"
158107
},
159108
"codeserver-ubi9-python-3_11": {
160-
"needs": [
161-
"base-ubi9-python-3_11"
162-
],
163-
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
164-
"with": {
165-
"target": "codeserver-ubi9-python-3.11",
166-
"github": "${{ toJSON(github) }}"
167-
},
168-
"secrets": "inherit"
169-
},
170-
"base-c9s-python-3_11": {
171109
"needs": [],
172110
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
173111
"with": {
174-
"target": "base-c9s-python-3.11",
175-
"github": "${{ toJSON(github) }}"
176-
},
177-
"secrets": "inherit"
178-
},
179-
"cuda-c9s-python-3_11": {
180-
"needs": [
181-
"base-c9s-python-3_11"
182-
],
183-
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
184-
"with": {
185-
"target": "cuda-c9s-python-3.11",
112+
"target": "codeserver-ubi9-python-3.11",
186113
"github": "${{ toJSON(github) }}"
187114
},
188115
"secrets": "inherit"
189116
},
190117
"rstudio-c9s-python-3_11": {
191-
"needs": [
192-
"base-c9s-python-3_11"
193-
],
118+
"needs": [],
194119
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
195120
"with": {
196121
"target": "rstudio-c9s-python-3.11",
@@ -199,53 +124,25 @@
199124
"secrets": "inherit"
200125
},
201126
"cuda-rstudio-c9s-python-3_11": {
202-
"needs": [
203-
"cuda-c9s-python-3_11"
204-
],
127+
"needs": [],
205128
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
206129
"with": {
207130
"target": "cuda-rstudio-c9s-python-3.11",
208131
"github": "${{ toJSON(github) }}"
209132
},
210133
"secrets": "inherit"
211134
},
212-
"rocm-ubi9-python-3_11": {
213-
"needs": [
214-
"base-ubi9-python-3_11"
215-
],
216-
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
217-
"with": {
218-
"target": "rocm-ubi9-python-3.11",
219-
"github": "${{ toJSON(github) }}"
220-
},
221-
"secrets": "inherit"
222-
},
223135
"rocm-jupyter-minimal-ubi9-python-3_11": {
224-
"needs": [
225-
"rocm-ubi9-python-3_11"
226-
],
136+
"needs": [],
227137
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
228138
"with": {
229139
"target": "rocm-jupyter-minimal-ubi9-python-3.11",
230140
"github": "${{ toJSON(github) }}"
231141
},
232142
"secrets": "inherit"
233143
},
234-
"rocm-jupyter-datascience-ubi9-python-3_11": {
235-
"needs": [
236-
"rocm-jupyter-minimal-ubi9-python-3_11"
237-
],
238-
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
239-
"with": {
240-
"target": "rocm-jupyter-datascience-ubi9-python-3.11",
241-
"github": "${{ toJSON(github) }}"
242-
},
243-
"secrets": "inherit"
244-
},
245144
"rocm-jupyter-tensorflow-ubi9-python-3_11": {
246-
"needs": [
247-
"rocm-jupyter-datascience-ubi9-python-3_11"
248-
],
145+
"needs": [],
249146
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
250147
"with": {
251148
"target": "rocm-jupyter-tensorflow-ubi9-python-3.11",
@@ -254,9 +151,7 @@
254151
"secrets": "inherit"
255152
},
256153
"rocm-jupyter-pytorch-ubi9-python-3_11": {
257-
"needs": [
258-
"rocm-jupyter-datascience-ubi9-python-3_11"
259-
],
154+
"needs": [],
260155
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
261156
"with": {
262157
"target": "rocm-jupyter-pytorch-ubi9-python-3.11",
@@ -265,9 +160,7 @@
265160
"secrets": "inherit"
266161
},
267162
"rocm-runtime-pytorch-ubi9-python-3_11": {
268-
"needs": [
269-
"rocm-ubi9-python-3_11"
270-
],
163+
"needs": [],
271164
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
272165
"with": {
273166
"target": "rocm-runtime-pytorch-ubi9-python-3.11",
@@ -276,9 +169,7 @@
276169
"secrets": "inherit"
277170
},
278171
"rocm-runtime-tensorflow-ubi9-python-3_11": {
279-
"needs": [
280-
"rocm-ubi9-python-3_11"
281-
],
172+
"needs": [],
282173
"uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
283174
"with": {
284175
"target": "rocm-runtime-tensorflow-ubi9-python-3.11",

Diff for: ‎Makefile

+22-3
Original file line numberDiff line numberDiff line change
@@ -370,9 +370,7 @@ validate-rstudio-image: bin/kubectl
370370
# Default Python version
371371
PYTHON_VERSION ?= 3.11
372372
ROOT_DIR := $(shell pwd)
373-
BASE_DIRS := base/c9s-python-$(PYTHON_VERSION) \
374-
base/ubi9-python-$(PYTHON_VERSION) \
375-
jupyter/minimal/ubi9-python-$(PYTHON_VERSION) \
373+
BASE_DIRS := jupyter/minimal/ubi9-python-$(PYTHON_VERSION) \
376374
jupyter/datascience/ubi9-python-$(PYTHON_VERSION) \
377375
jupyter/pytorch/ubi9-python-$(PYTHON_VERSION) \
378376
jupyter/tensorflow/ubi9-python-$(PYTHON_VERSION) \
@@ -429,3 +427,24 @@ refresh-pipfilelock-files:
429427
.PHONY: scan-image-vulnerabilities
430428
scan-image-vulnerabilities:
431429
python ci/security-scan/quay_security_analysis.py
430+
431+
# This is used primarly for gen_gha_matrix_jobs.py to we know the set of all possible images we may want to build
432+
.PHONY: all-images
433+
all-images: jupyter-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION) \
434+
jupyter-datascience-ubi9-python-$(RELEASE_PYTHON_VERSION) \
435+
cuda-jupyter-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION) \
436+
cuda-jupyter-tensorflow-ubi9-python-$(RELEASE_PYTHON_VERSION) \
437+
cuda-jupyter-pytorch-ubi9-python-$(RELEASE_PYTHON_VERSION) \
438+
jupyter-trustyai-ubi9-python-$(RELEASE_PYTHON_VERSION) \
439+
runtime-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION) \
440+
runtime-datascience-ubi9-python-$(RELEASE_PYTHON_VERSION) \
441+
runtime-cuda-pytorch-ubi9-python-$(RELEASE_PYTHON_VERSION) \
442+
runtime-cuda-tensorflow-ubi9-python-$(RELEASE_PYTHON_VERSION) \
443+
codeserver-ubi9-python-$(RELEASE_PYTHON_VERSION) \
444+
rstudio-c9s-python-$(RELEASE_PYTHON_VERSION) \
445+
cuda-rstudio-c9s-python-$(RELEASE_PYTHON_VERSION) \
446+
rocm-jupyter-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION) \
447+
rocm-jupyter-tensorflow-ubi9-python-$(RELEASE_PYTHON_VERSION) \
448+
rocm-jupyter-pytorch-ubi9-python-$(RELEASE_PYTHON_VERSION) \
449+
rocm-runtime-pytorch-ubi9-python-$(RELEASE_PYTHON_VERSION) \
450+
rocm-runtime-tensorflow-ubi9-python-$(RELEASE_PYTHON_VERSION)

0 commit comments

Comments
 (0)