Skip to content

Commit 5007663

Browse files
andyatmiamiandyatmiami
committed
feat(containers): Remove reliance on "chained builds" architecture for image builds
Commit contains following changes: - Refactors all `Dockerfile`s to to be "self-contained" - not rely on any "build chains" for internal images - please note in the final form of this PR - many intermediate stages will be combined... this current form makes it easier to understand how the `Dockerfile`s present in the "build chain" are being combined - long term - we'd expect the following stages: `base`, `base-<accelerator>` (if applicable), `<final>` - `Dockerfile` file now has a file extension/suffix that indicates CPU or accelerator - `LABEL` directives now should be consistent/accurate - this probably needs a little more attention - `base/` directory removed as its no longer relevant/required - `rocm/` directory removed as its no longer relevant/required - `wheel` + `setuptools` explicitly added to `runtime-` images `Pipfile` - requirements.txt also regenerated as a result - Makefile updated to remove pre-reqs on container image targets - Makefile updated to handle support better across releases - _read: MOWR variables_ - `pytorch` `Makefile` targets now contain `cuda-` prefix - `rocm` + `cuda` `datascience` `Makefile` targets removed as we don't ship those variants - with no chained builds - all datascience-dependent images contain their own required `Dockerfile` logic! - `ENV` directive in `Dockerfile` now properly uses `=` (vs. whitespace) - change in `buildinputs` to pull file paths from the terminal layer - this change particularly needs refined - but its "crude yet effective" in its current form Related-to: https://issues.redhat.com/browse/RHOAIENG-19048
1 parent 8381f4c commit 5007663

File tree

59 files changed

+2601
-1047
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+2601
-1047
lines changed

‎Makefile

+95-139
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ endif
2020

2121
IMAGE_REGISTRY ?= quay.io/opendatahub/workbench-images
2222
RELEASE ?= 2024b
23+
RELEASE_PYTHON_VERSION ?= 3.11
2324
# additional user-specified caching parameters for $(CONTAINER_ENGINE) build
2425
CONTAINER_BUILD_CACHE_ARGS ?= --no-cache
2526
# whether to build all dependent images or just the one specified
@@ -37,6 +38,12 @@ endif
3738
DATE ?= $(shell date +'%Y%m%d')
3839
WHERE_WHICH ?= which
3940

41+
ifeq ($(RELEASE), 2025a)
42+
RELEASE_PYTHON_VERSION = 3.12
43+
else ifeq ($(RELEASE), 2024a)
44+
RELEASE_PYTHON_VERSION = 3.9
45+
endif
46+
4047

4148
# linux/amd64 or darwin/arm64
4249
OS_ARCH=$(shell go env GOOS)/$(shell go env GOARCH)
@@ -62,23 +69,20 @@ else
6269
CONTAINER_ENGINE := podman
6370
endif
6471

65-
# Build function for the notebok image:
72+
# Build function for the notebook image:
6673
# ARG 1: Image tag name.
67-
# ARG 2: Path of image context we want to build.
68-
# ARG 3: Base image tag name (optional).
74+
# ARG 2: Path of Dockerfile we want to build.
6975
define build_image
7076
$(eval IMAGE_NAME := $(IMAGE_REGISTRY):$(1)-$(IMAGE_TAG))
77+
$(eval BUILD_ARGS :=)
78+
7179
$(info # Building $(IMAGE_NAME) image...)
72-
$(if $(3),
73-
$(eval BASE_IMAGE_NAME := $(IMAGE_REGISTRY):$(3)-$(IMAGE_TAG))
74-
$(eval BUILD_ARGS := --build-arg BASE_IMAGE=$(BASE_IMAGE_NAME)),
75-
$(eval BUILD_ARGS :=)
76-
)
77-
$(ROOT_DIR)/scripts/sandbox.py --dockerfile '$(2)/Dockerfile' -- \
78-
$(CONTAINER_ENGINE) build $(CONTAINER_BUILD_CACHE_ARGS) --tag $(IMAGE_NAME) --file '$(2)/Dockerfile' $(BUILD_ARGS) {}\;
80+
81+
$(ROOT_DIR)/scripts/sandbox.py --dockerfile '$(2)' -- \
82+
$(CONTAINER_ENGINE) build $(CONTAINER_BUILD_CACHE_ARGS) --tag $(IMAGE_NAME) --file '$(2)' $(BUILD_ARGS) {}\;
7983
endef
8084

81-
# Push function for the notebok image:
85+
# Push function for the notebook image:
8286
# ARG 1: Path of image context we want to build.
8387
define push_image
8488
$(eval IMAGE_NAME := $(IMAGE_REGISTRY):$(subst /,-,$(1))-$(IMAGE_TAG))
@@ -88,16 +92,16 @@ endef
8892

8993
# Build and push the notebook images:
9094
# ARG 1: Image tag name.
91-
# ARG 2: Path of image context we want to build.
92-
# ARG 3: Base image tag name (optional).
95+
# ARG 2: Path of Dockerfile we want to build.
9396
#
9497
# BUILD_DEPENDENT_IMAGES: only build images that were explicitly given as a goal on command line
9598
# PUSH_IMAGES: allows skipping podman push
9699
define image
97-
$(info #*# Image build directory: <$(2)> #(MACHINE-PARSED LINE)#*#...)
100+
$(eval BUILD_DIRECTORY := $(shell echo $(2) | sed 's/\/Dockerfile.*//'))
101+
$(info #*# Image build directory: <$(BUILD_DIRECTORY)> #(MACHINE-PARSED LINE)#*#...)
98102

99103
$(if $(or $(BUILD_DEPENDENT_IMAGES:no=), $(filter $@,$(MAKECMDGOALS))),
100-
$(call build_image,$(1),$(2),$(3))
104+
$(call build_image,$(1),$(2))
101105
102106
$(if $(PUSH_IMAGES:no=),
103107
$(call push_image,$(1))
@@ -117,130 +121,82 @@ bin/buildinputs: scripts/buildinputs/buildinputs.go scripts/buildinputs/go.mod s
117121
$(info Building a Go helper for Dockerfile dependency analysis...)
118122
go build -C "scripts/buildinputs" -o "$(ROOT_DIR)/$@" ./...
119123

120-
####################################### Buildchain for Python 3.11 using ubi9 #####################################
121-
122-
# Build and push base-ubi9-python-3.11 image to the registry
123-
.PHONY: base-ubi9-python-3.11
124-
base-ubi9-python-3.11:
125-
$(call image,$@,base/ubi9-python-3.11)
126-
127-
# Build and push jupyter-minimal-ubi9-python-3.11 image to the registry
128-
.PHONY: jupyter-minimal-ubi9-python-3.11
129-
jupyter-minimal-ubi9-python-3.11: base-ubi9-python-3.11
130-
$(call image,$@,jupyter/minimal/ubi9-python-3.11,$<)
131-
132-
# Build and push jupyter-datascience-ubi9-python-3.11 image to the registry
133-
.PHONY: jupyter-datascience-ubi9-python-3.11
134-
jupyter-datascience-ubi9-python-3.11: jupyter-minimal-ubi9-python-3.11
135-
$(call image,$@,jupyter/datascience/ubi9-python-3.11,$<)
136-
137-
# Build and push cuda-ubi9-python-3.11 image to the registry
138-
.PHONY: cuda-ubi9-python-3.11
139-
cuda-ubi9-python-3.11: base-ubi9-python-3.11
140-
$(call image,$@,cuda/ubi9-python-3.11,$<)
141-
142-
# Build and push cuda-jupyter-minimal-ubi9-python-3.11 image to the registry
143-
.PHONY: cuda-jupyter-minimal-ubi9-python-3.11
144-
cuda-jupyter-minimal-ubi9-python-3.11: cuda-ubi9-python-3.11
145-
$(call image,$@,jupyter/minimal/ubi9-python-3.11,$<)
146-
147-
# Build and push cuda-jupyter-datascience-ubi9-python-3.11 image to the registry
148-
.PHONY: cuda-jupyter-datascience-ubi9-python-3.11
149-
cuda-jupyter-datascience-ubi9-python-3.11: cuda-jupyter-minimal-ubi9-python-3.11
150-
$(call image,$@,jupyter/datascience/ubi9-python-3.11,$<)
151-
152-
# Build and push cuda-jupyter-tensorflow-ubi9-python-3.11 image to the registry
153-
.PHONY: cuda-jupyter-tensorflow-ubi9-python-3.11
154-
cuda-jupyter-tensorflow-ubi9-python-3.11: cuda-jupyter-datascience-ubi9-python-3.11
155-
$(call image,$@,jupyter/tensorflow/ubi9-python-3.11,$<)
156-
157-
# Build and push jupyter-pytorch-ubi9-python-3.11 image to the registry
158-
.PHONY: jupyter-pytorch-ubi9-python-3.11
159-
jupyter-pytorch-ubi9-python-3.11: cuda-jupyter-datascience-ubi9-python-3.11
160-
$(call image,$@,jupyter/pytorch/ubi9-python-3.11,$<)
161-
162-
# Build and push jupyter-trustyai-ubi9-python-3.11 image to the registry
163-
.PHONY: jupyter-trustyai-ubi9-python-3.11
164-
jupyter-trustyai-ubi9-python-3.11: jupyter-datascience-ubi9-python-3.11
165-
$(call image,$@,jupyter/trustyai/ubi9-python-3.11,$<)
166-
167-
# Build and push runtime-minimal-ubi9-python-3.11 image to the registry
168-
.PHONY: runtime-minimal-ubi9-python-3.11
169-
runtime-minimal-ubi9-python-3.11: base-ubi9-python-3.11
170-
$(call image,$@,runtimes/minimal/ubi9-python-3.11,$<)
171-
172-
# Build and push runtime-datascience-ubi9-python-3.11 image to the registry
173-
.PHONY: runtime-datascience-ubi9-python-3.11
174-
runtime-datascience-ubi9-python-3.11: base-ubi9-python-3.11
175-
$(call image,$@,runtimes/datascience/ubi9-python-3.11,$<)
176-
177-
# Build and push runtime-pytorch-ubi9-python-3.11 image to the registry
178-
.PHONY: runtime-pytorch-ubi9-python-3.11
179-
runtime-pytorch-ubi9-python-3.11: base-ubi9-python-3.11
180-
$(call image,$@,runtimes/pytorch/ubi9-python-3.11,$<)
181-
182-
# Build and push runtime-cuda-tensorflow-ubi9-python-3.11 image to the registry
183-
.PHONY: runtime-cuda-tensorflow-ubi9-python-3.11
184-
runtime-cuda-tensorflow-ubi9-python-3.11: cuda-ubi9-python-3.11
185-
$(call image,$@,runtimes/tensorflow/ubi9-python-3.11,$<)
186-
187-
.PHONY: codeserver-ubi9-python-3.11
188-
codeserver-ubi9-python-3.11: base-ubi9-python-3.11
189-
$(call image,$@,codeserver/ubi9-python-3.11,$<)
190-
191-
####################################### Buildchain for Python 3.11 using C9S #######################################
192-
193-
# Build and push base-c9s-python-3.11 image to the registry
194-
.PHONY: base-c9s-python-3.11
195-
base-c9s-python-3.11:
196-
$(call image,$@,base/c9s-python-3.11)
197-
198-
.PHONY: cuda-c9s-python-3.11
199-
cuda-c9s-python-3.11: base-c9s-python-3.11
200-
$(call image,$@,cuda/c9s-python-3.11,$<)
201-
202-
.PHONY: rstudio-c9s-python-3.11
203-
rstudio-c9s-python-3.11: base-c9s-python-3.11
204-
$(call image,$@,rstudio/c9s-python-3.11,$<)
205-
206-
.PHONY: cuda-rstudio-c9s-python-3.11
207-
cuda-rstudio-c9s-python-3.11: cuda-c9s-python-3.11
208-
$(call image,$@,rstudio/c9s-python-3.11,$<)
209-
210-
####################################### Buildchain for AMD Python 3.11 using UBI9 #######################################
211-
.PHONY: rocm-ubi9-python-3.11
212-
rocm-ubi9-python-3.11: base-ubi9-python-3.11
213-
$(call image,$@,rocm/ubi9-python-3.11,$<)
214-
215-
# We are only using rocm-ubi9 base image here onwards
216-
.PHONY: rocm-jupyter-minimal-ubi9-python-3.11
217-
rocm-jupyter-minimal-ubi9-python-3.11: rocm-ubi9-python-3.11
218-
$(call image,$@,jupyter/minimal/ubi9-python-3.11,$<)
219-
220-
# Build and push rocm-jupyter-datascience-ubi9-python-3.11 image to the registry
221-
.PHONY: rocm-jupyter-datascience-ubi9-python-3.11
222-
rocm-jupyter-datascience-ubi9-python-3.11: rocm-jupyter-minimal-ubi9-python-3.11
223-
$(call image,$@,jupyter/datascience/ubi9-python-3.11,$<)
224-
225-
# Build and push rocm-jupyter-tensorflow-ubi9-python-3.11 image to the registry
226-
.PHONY: rocm-jupyter-tensorflow-ubi9-python-3.11
227-
rocm-jupyter-tensorflow-ubi9-python-3.11: rocm-jupyter-datascience-ubi9-python-3.11
228-
$(call image,$@,jupyter/rocm/tensorflow/ubi9-python-3.11,$<)
229-
230-
# Build and push rocm-jupyter-pytorch-ubi9-python-3.11 image to the registry
231-
.PHONY: rocm-jupyter-pytorch-ubi9-python-3.11
232-
rocm-jupyter-pytorch-ubi9-python-3.11: rocm-jupyter-datascience-ubi9-python-3.11
233-
$(call image,$@,jupyter/rocm/pytorch/ubi9-python-3.11,$<)
234-
235-
# Build and push rocm-jupyter-runtime-pytorch-ubi9-python-3.11 image to the registry
236-
.PHONY: rocm-runtime-pytorch-ubi9-python-3.11
237-
rocm-runtime-pytorch-ubi9-python-3.11: rocm-ubi9-python-3.11
238-
$(call image,$@,runtimes/rocm-pytorch/ubi9-python-3.11,$<)
239-
240-
# Build and push rocm-jupyter-runtime-tensorflow-ubi9-python-3.11 image to the registry
241-
.PHONY: rocm-runtime-tensorflow-ubi9-python-3.11
242-
rocm-runtime-tensorflow-ubi9-python-3.11: rocm-ubi9-python-3.11
243-
$(call image,$@,runtimes/rocm-tensorflow/ubi9-python-3.11,$<)
124+
####################################### Buildchain for Python using ubi9 #####################################
125+
126+
.PHONY: jupyter-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION)
127+
jupyter-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION):
128+
$(call image,$@,jupyter/minimal/ubi9-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.cpu)
129+
130+
.PHONY: jupyter-datascience-ubi9-python-$(RELEASE_PYTHON_VERSION)
131+
jupyter-datascience-ubi9-python-$(RELEASE_PYTHON_VERSION):
132+
$(call image,$@,jupyter/datascience/ubi9-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.cpu)
133+
134+
.PHONY: cuda-jupyter-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION)
135+
cuda-jupyter-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION):
136+
$(call image,$@,jupyter/minimal/ubi9-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.cuda)
137+
138+
.PHONY: cuda-jupyter-tensorflow-ubi9-python-$(RELEASE_PYTHON_VERSION)
139+
cuda-jupyter-tensorflow-ubi9-python-$(RELEASE_PYTHON_VERSION):
140+
$(call image,$@,jupyter/tensorflow/ubi9-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.cuda)
141+
142+
.PHONY: cuda-jupyter-pytorch-ubi9-python-$(RELEASE_PYTHON_VERSION)
143+
cuda-jupyter-pytorch-ubi9-python-$(RELEASE_PYTHON_VERSION):
144+
$(call image,$@,jupyter/pytorch/ubi9-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.cuda)
145+
146+
.PHONY: jupyter-trustyai-ubi9-python-$(RELEASE_PYTHON_VERSION)
147+
jupyter-trustyai-ubi9-python-$(RELEASE_PYTHON_VERSION):
148+
$(call image,$@,jupyter/trustyai/ubi9-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.cpu)
149+
150+
.PHONY: runtime-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION)
151+
runtime-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION):
152+
$(call image,$@,runtimes/minimal/ubi9-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.cpu)
153+
154+
.PHONY: runtime-datascience-ubi9-python-$(RELEASE_PYTHON_VERSION)
155+
runtime-datascience-ubi9-python-$(RELEASE_PYTHON_VERSION):
156+
$(call image,$@,runtimes/datascience/ubi9-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.cpu)
157+
158+
.PHONY: runtime-cuda-pytorch-ubi9-python-$(RELEASE_PYTHON_VERSION)
159+
runtime-cuda-pytorch-ubi9-python-$(RELEASE_PYTHON_VERSION):
160+
$(call image,$@,runtimes/pytorch/ubi9-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.cuda)
161+
162+
.PHONY: runtime-cuda-tensorflow-ubi9-python-$(RELEASE_PYTHON_VERSION)
163+
runtime-cuda-tensorflow-ubi9-python-$(RELEASE_PYTHON_VERSION):
164+
$(call image,$@,runtimes/tensorflow/ubi9-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.cuda)
165+
166+
.PHONY: codeserver-ubi9-python-$(RELEASE_PYTHON_VERSION)
167+
codeserver-ubi9-python-$(RELEASE_PYTHON_VERSION):
168+
$(call image,$@,codeserver/ubi9-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.cpu)
169+
170+
####################################### Buildchain for Python using C9S #######################################
171+
172+
.PHONY: rstudio-c9s-python-$(RELEASE_PYTHON_VERSION)
173+
rstudio-c9s-python-$(RELEASE_PYTHON_VERSION):
174+
$(call image,$@,rstudio/c9s-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.cpu)
175+
176+
.PHONY: cuda-rstudio-c9s-python-$(RELEASE_PYTHON_VERSION)
177+
cuda-rstudio-c9s-python-$(RELEASE_PYTHON_VERSION):
178+
$(call image,$@,rstudio/c9s-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.cuda)
179+
180+
####################################### Buildchain for AMD Python using UBI9 #######################################
181+
.PHONY: rocm-jupyter-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION)
182+
rocm-jupyter-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION):
183+
$(call image,$@,jupyter/minimal/ubi9-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.rocm)
184+
185+
.PHONY: rocm-jupyter-tensorflow-ubi9-python-$(RELEASE_PYTHON_VERSION)
186+
rocm-jupyter-tensorflow-ubi9-python-$(RELEASE_PYTHON_VERSION):
187+
$(call image,$@,jupyter/rocm/tensorflow/ubi9-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.rocm)
188+
189+
.PHONY: rocm-jupyter-pytorch-ubi9-python-$(RELEASE_PYTHON_VERSION)
190+
rocm-jupyter-pytorch-ubi9-python-$(RELEASE_PYTHON_VERSION):
191+
$(call image,$@,jupyter/rocm/pytorch/ubi9-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.rocm)
192+
193+
.PHONY: rocm-runtime-pytorch-ubi9-python-$(RELEASE_PYTHON_VERSION)
194+
rocm-runtime-pytorch-ubi9-python-$(RELEASE_PYTHON_VERSION):
195+
$(call image,$@,runtimes/rocm-pytorch/ubi9-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.rocm)
196+
197+
.PHONY: rocm-runtime-tensorflow-ubi9-python-$(RELEASE_PYTHON_VERSION)
198+
rocm-runtime-tensorflow-ubi9-python-$(RELEASE_PYTHON_VERSION):
199+
$(call image,$@,runtimes/rocm-tensorflow/ubi9-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.rocm)
244200

245201
####################################### Deployments #######################################
246202

‎base/c9s-python-3.11/Dockerfile

-42
This file was deleted.

‎base/ubi9-python-3.11/Dockerfile

-44
This file was deleted.

‎base/ubi9-python-3.11/Pipfile

-14
This file was deleted.

0 commit comments

Comments
 (0)