Skip to content

Commit 4c10ca9

Browse files
hcho3hcho3
authored
Address review comments in dmlc/xgboost#11046 (#2)
* x- prefix for anchors in ci_container.yml * Use latest Miniforge; specify versions as build args * Make PYTHON_VERSION build arg * Fix gpu_build_r_rockylinux8 * Build containers weekly
1 parent b359ae6 commit 4c10ca9

14 files changed

+125
-113
lines changed

‎.github/workflows/containers.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ on:
1414
- main
1515
pull_request:
1616
schedule:
17-
- cron: "0 7 * * *" # Run once daily
17+
- cron: "0 7 * * 1" # Run once weekly
1818

1919
concurrency:
2020
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

‎containers/ci_container.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# Each container will be built using the definition from
44
# containers/dockerfile/Dockerfile.CONTAINER_DEF
55

6-
rapids_versions:
6+
x-rapids_versions:
77
stable: &rapids_version "24.10"
88
dev: &dev_rapids_version "24.12"
99

‎containers/dockerfile/Dockerfile.aarch64‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,23 @@
11
FROM quay.io/pypa/manylinux_2_28_aarch64
2+
ARG MINIFORGE_VERSION=24.9.2-0
23

3-
SHELL ["/bin/bash", "-c"] # Use Bash as shell
4+
SHELL ["/bin/bash", "-c"]
5+
6+
ENV PATH=/opt/miniforge/bin:$PATH
7+
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
8+
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
9+
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
10+
ENV GOSU_VERSION=1.10
411

5-
# Install all basic requirements
612
RUN \
713
dnf -y update && \
814
dnf -y install dnf-plugins-core && \
915
dnf config-manager --set-enabled powertools && \
1016
dnf install -y tar unzip wget xz git which ninja-build gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ && \
11-
# Python
12-
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-aarch64.sh && \
17+
# Miniforge
18+
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/$MINIFORGE_VERSION/Miniforge3-$MINIFORGE_VERSION-Linux-aarch64.sh && \
1319
bash conda.sh -b -p /opt/miniforge
1420

15-
ENV PATH=/opt/miniforge/bin:$PATH
16-
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
17-
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
18-
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
19-
ENV GOSU_VERSION=1.10
20-
2121
# Create new Conda environment
2222
COPY conda_env/aarch64_test.yml /scripts/
2323
RUN mamba create -n aarch64_test && \

‎containers/dockerfile/Dockerfile.clang_tidy‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,15 @@ ARG CUDA_VERSION=notset
22
FROM nvidia/cuda:$CUDA_VERSION-devel-ubuntu22.04
33
ARG CUDA_VERSION
44

5-
# Environment
65
ENV DEBIAN_FRONTEND=noninteractive
6+
ENV GOSU_VERSION=1.10
77

8-
# Install all basic requirements
98
RUN \
109
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub && \
1110
apt-get update && \
1211
apt-get install -y wget git python3 python3-pip software-properties-common \
1312
apt-transport-https ca-certificates gnupg-agent && \
14-
apt-get install -y ninja-build
13+
apt-get install -y ninja-build libgtest-dev libgmock-dev
1514

1615
# Install clang-tidy: https://apt.llvm.org/
1716
RUN \
@@ -23,18 +22,11 @@ RUN \
2322
apt-get install -y clang-tidy-19 clang-19 libomp-19-dev
2423

2524
# Set default clang-tidy version
26-
RUN \
27-
update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-19 100 && \
25+
RUN update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-19 100 && \
2826
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-19 100
2927

30-
RUN \
31-
apt-get install libgtest-dev libgmock-dev -y
32-
3328
# Install Python packages
34-
RUN \
35-
pip3 install cmake
36-
37-
ENV GOSU_VERSION=1.10
29+
RUN pip3 install cmake
3830

3931
# Install lightweight sudo (not bound to TTY)
4032
RUN set -ex; \

‎containers/dockerfile/Dockerfile.cpu‎

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,27 @@
11
FROM ubuntu:22.04
2+
ARG MINIFORGE_VERSION=24.9.2-0
23

3-
# Environment
4-
ENV DEBIAN_FRONTEND=noninteractive
54
SHELL ["/bin/bash", "-c"]
65

6+
ENV DEBIAN_FRONTEND=noninteractive
7+
ENV PATH=/opt/miniforge/bin:$PATH
8+
ENV CC=gcc-10
9+
ENV CXX=g++-10
10+
ENV CPP=cpp-10
11+
ENV GOSU_VERSION=1.10
12+
ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
13+
714
# Install all basic requirements
815
RUN \
916
apt-get update && \
1017
apt-get install -y software-properties-common && \
1118
add-apt-repository ppa:ubuntu-toolchain-r/test && \
1219
apt-get update && \
1320
apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libidn12 cmake ninja-build gcc-10 g++-10 openjdk-8-jdk-headless && \
14-
# Python
15-
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
21+
# Miniforge
22+
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/$MINIFORGE_VERSION/Miniforge3-$MINIFORGE_VERSION-Linux-x86_64.sh && \
1623
bash conda.sh -b -p /opt/miniforge
1724

18-
ENV PATH=/opt/miniforge/bin:$PATH
19-
ENV CC=gcc-10
20-
ENV CXX=g++-10
21-
ENV CPP=cpp-10
22-
23-
ENV GOSU_VERSION=1.10
24-
ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
25-
2625
# Install gRPC
2726
# Patch Abseil to apply https://github.com/abseil/abseil-cpp/issues/1629
2827
RUN git clone -b v1.65.4 https://github.com/grpc/grpc.git \

‎containers/dockerfile/Dockerfile.gpu‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,29 +5,32 @@ ARG RAPIDS_VERSION
55
# Should be first 4 digits (e.g. 24.06)
66
ARG NCCL_VERSION
77
ARG RAPIDSAI_CONDA_CHANNEL="rapidsai"
8+
ARG MINIFORGE_VERSION=24.9.2-0
9+
ARG PYTHON_VERSION=3.10
810

9-
# Environment
10-
ENV DEBIAN_FRONTEND=noninteractive
1111
SHELL ["/bin/bash", "-c"]
1212

13+
ENV DEBIAN_FRONTEND=noninteractive
14+
ENV PATH=/opt/miniforge/bin:$PATH
15+
ENV GOSU_VERSION=1.10
16+
ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
17+
1318
# Install all basic requirements
1419
RUN \
1520
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub && \
1621
apt-get update && \
1722
apt-get install -y wget unzip bzip2 libgomp1 build-essential openjdk-8-jdk-headless && \
1823
apt-get install libnccl2 libnccl-dev -y --allow-change-held-packages && \
19-
# Python
20-
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
24+
# Miniforge
25+
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/$MINIFORGE_VERSION/Miniforge3-$MINIFORGE_VERSION-Linux-x86_64.sh && \
2126
bash conda.sh -b -p /opt/miniforge
2227

23-
ENV PATH=/opt/miniforge/bin:$PATH
24-
2528
# Create new Conda environment with cuDF, Dask, and cuPy
2629
RUN \
2730
export NCCL_SHORT_VER=$(echo "$NCCL_VERSION" | cut -d "-" -f 1) && \
2831
export CUDA_SHORT_VER=$(echo "$CUDA_VERSION" | grep -o -E '[0-9]+\.[0-9]') && \
2932
mamba create -y -n gpu_test -c ${RAPIDSAI_CONDA_CHANNEL} -c conda-forge -c nvidia \
30-
python=3.10 "cudf=$RAPIDS_VERSION.*" "rmm=$RAPIDS_VERSION.*" cuda-version=$CUDA_SHORT_VER \
33+
python=$PYTHON_VERSION "cudf=$RAPIDS_VERSION.*" "rmm=$RAPIDS_VERSION.*" cuda-version=$CUDA_SHORT_VER \
3134
"nccl>=${NCCL_SHORT_VER}" \
3235
"dask<=2024.10.0" \
3336
"distributed<=2024.10.0" \
@@ -37,9 +40,6 @@ RUN \
3740
"pyspark>=3.4.0" cloudpickle cuda-python && \
3841
mamba clean --all --yes
3942

40-
ENV GOSU_VERSION=1.10
41-
ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
42-
4343
# Install lightweight sudo (not bound to TTY)
4444
RUN set -ex; \
4545
wget -nv -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \

‎containers/dockerfile/Dockerfile.gpu_build_r_rockylinux8‎

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,19 @@ ARG CUDA_VERSION=notset
22
FROM nvcr.io/nvidia/cuda:$CUDA_VERSION-devel-rockylinux8
33
ARG CUDA_VERSION
44
ARG R_VERSION
5+
ARG MINIFORGE_VERSION=24.9.2-0
6+
ARG CMAKE_VERSION=3.31.2
7+
8+
SHELL ["/bin/bash", "-c"]
9+
10+
ENV PATH=/usr/local/ninja:/opt/software/packages/bin:$PATH
11+
ENV LD_LIBRARY_PATH=/opt/software/packages/lib:/opt/R/$R_VERSION/lib64:$LD_LIBRARY_PATH
12+
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
13+
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
14+
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
15+
ENV F77=/opt/rh/gcc-toolset-10/root/usr/bin/gfortran
16+
ENV FC=/opt/rh/gcc-toolset-10/root/usr/bin/gfortran
17+
ENV GOSU_VERSION=1.10
518

619
# Install all basic requirements
720
RUN \
@@ -15,15 +28,13 @@ RUN \
1528
pcre2-devel libcurl-devel texlive-* \
1629
gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ \
1730
gcc-toolset-10-gcc-gfortran gcc-toolset-10-libquadmath-devel \
18-
gcc-toolset-10-runtime gcc-toolset-10-libstdc++-devel
19-
20-
ENV PATH=/opt/miniforge/bin:/usr/local/ninja:/opt/software/packages/bin:/opt/R/$R_VERSION/bin:$PATH
21-
ENV LD_LIBRARY_PATH=/opt/software/packages/lib:/opt/R/$R_VERSION/lib64:$LD_LIBRARY_PATH
22-
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
23-
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
24-
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
25-
ENV F77=/opt/rh/gcc-toolset-10/root/usr/bin/gfortran
26-
ENV FC=/opt/rh/gcc-toolset-10/root/usr/bin/gfortran
31+
gcc-toolset-10-runtime gcc-toolset-10-libstdc++-devel && \
32+
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/$MINIFORGE_VERSION/Miniforge3-$MINIFORGE_VERSION-Linux-x86_64.sh && \
33+
bash conda.sh -b -p /opt/miniforge && \
34+
/opt/miniforge/bin/python -m pip install awscli && \
35+
# CMake
36+
wget -nv -O cmake.sh https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh && \
37+
bash cmake.sh --skip-license --prefix=/usr
2738

2839
RUN \
2940
wget -nv -nc https://cran.r-project.org/src/base/R-4/R-$R_VERSION.tar.gz && \
@@ -33,16 +44,7 @@ RUN \
3344
make -j$(nproc) && \
3445
make install
3546

36-
run \
37-
# Python
38-
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
39-
bash conda.sh -b -p /opt/miniforge && \
40-
/opt/miniforge/bin/python -m pip install auditwheel awscli && \
41-
# CMake
42-
wget -nv -nc https://cmake.org/files/v3.29/cmake-3.29.5-linux-x86_64.sh --no-check-certificate && \
43-
bash cmake-3.29.5-linux-x86_64.sh --skip-license --prefix=/usr
44-
45-
ENV GOSU_VERSION=1.10
47+
ENV PATH=/opt/miniforge/bin:$PATH
4648

4749
# Install lightweight sudo (not bound to TTY)
4850
RUN set -ex; \

‎containers/dockerfile/Dockerfile.gpu_build_rockylinux8‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,17 @@ FROM nvcr.io/nvidia/cuda:$CUDA_VERSION-devel-rockylinux8
33
ARG CUDA_VERSION
44
ARG NCCL_VERSION
55
ARG RAPIDS_VERSION
6+
ARG MINIFORGE_VERSION=24.9.2-0
7+
ARG CMAKE_VERSION=3.31.2
8+
9+
SHELL ["/bin/bash", "-c"]
10+
11+
ENV PATH=/opt/miniforge/bin:/usr/local/ninja:$PATH
12+
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
13+
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
14+
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
15+
ENV CUDAHOSTCXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
16+
ENV GOSU_VERSION=1.10
617

718
# Install all basic requirements
819
RUN \
@@ -12,13 +23,13 @@ RUN \
1223
dnf -y install dnf-plugins-core && \
1324
dnf config-manager --set-enabled powertools && \
1425
dnf install -y tar unzip wget xz git which ninja-build gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ && \
15-
# Python
16-
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
26+
# Miniforge
27+
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/$MINIFORGE_VERSION/Miniforge3-$MINIFORGE_VERSION-Linux-x86_64.sh && \
1728
bash conda.sh -b -p /opt/miniforge && \
1829
/opt/miniforge/bin/python -m pip install awscli && \
1930
# CMake
20-
wget -nv -nc https://cmake.org/files/v3.29/cmake-3.29.5-linux-x86_64.sh --no-check-certificate && \
21-
bash cmake-3.29.5-linux-x86_64.sh --skip-license --prefix=/usr
31+
wget -nv -O cmake.sh https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh && \
32+
bash cmake.sh --skip-license --prefix=/usr
2233

2334
# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
2435
RUN \
@@ -28,14 +39,6 @@ RUN \
2839
dnf -y update && \
2940
dnf install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT}
3041

31-
ENV PATH=/opt/miniforge/bin:/usr/local/ninja:$PATH
32-
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
33-
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
34-
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
35-
ENV CUDAHOSTCXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
36-
37-
ENV GOSU_VERSION=1.10
38-
3942
# Install gRPC
4043
# Patch Abseil to apply https://github.com/abseil/abseil-cpp/issues/1629
4144
RUN git clone -b v1.65.4 https://github.com/grpc/grpc.git \

‎containers/dockerfile/Dockerfile.i386‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
FROM i386/debian:sid
22

3-
ENV DEBIAN_FRONTEND=noninteractive
43
SHELL ["/bin/bash", "-c"]
54

6-
RUN \
7-
apt-get update && \
5+
ENV DEBIAN_FRONTEND=noninteractive
6+
7+
RUN apt-get update && \
88
apt-get install -y tar unzip wget git build-essential ninja-build cmake

‎containers/dockerfile/Dockerfile.jvm‎

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,37 @@
11
FROM rockylinux:8
2+
ARG MINIFORGE_VERSION=24.9.2-0
3+
ARG CMAKE_VERSION=3.31.2
4+
ARG MAVEN_VERSION=3.9.9
5+
6+
SHELL ["/bin/bash", "-c"]
7+
8+
ENV PATH=/opt/miniforge/bin:/opt/maven/bin:$PATH
9+
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
10+
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
11+
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
12+
ENV GOSU_VERSION=1.10
213

314
# Install all basic requirements
4-
RUN \
5-
dnf -y update && \
15+
RUN dnf -y update && \
616
dnf -y install dnf-plugins-core && \
717
dnf config-manager --set-enabled powertools && \
818
dnf install -y tar unzip make bzip2 wget xz git which ninja-build java-1.8.0-openjdk-devel \
919
gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ \
1020
gcc-toolset-10-runtime gcc-toolset-10-libstdc++-devel && \
11-
# Python
12-
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
21+
# Miniforge
22+
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/$MINIFORGE_VERSION/Miniforge3-$MINIFORGE_VERSION-Linux-x86_64.sh && \
1323
bash conda.sh -b -p /opt/miniforge && \
1424
# CMake
15-
wget -nv -nc https://cmake.org/files/v3.29/cmake-3.29.5-linux-x86_64.sh --no-check-certificate && \
16-
bash cmake-3.29.5-linux-x86_64.sh --skip-license --prefix=/usr && \
25+
wget -nv -O cmake.sh https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh && \
26+
bash cmake.sh --skip-license --prefix=/usr && \
1727
# Maven
18-
wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.9.7/binaries/apache-maven-3.9.7-bin.tar.gz && \
19-
tar xvf apache-maven-3.9.7-bin.tar.gz -C /opt && \
20-
ln -s /opt/apache-maven-3.9.7/ /opt/maven
21-
22-
ENV PATH=/opt/miniforge/bin:/opt/maven/bin:$PATH
23-
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
24-
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
25-
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
28+
wget -nv -O apache-maven.tar.gz https://dlcdn.apache.org/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz && \
29+
tar xvf apache-maven.tar.gz -C /opt && \
30+
ln -s /opt/apache-maven-${MAVEN_VERSION}/ /opt/maven
2631

2732
# Install Python packages
2833
RUN pip install numpy pytest scipy scikit-learn wheel kubernetes awscli
2934

30-
ENV GOSU_VERSION=1.10
31-
3235
# Install lightweight sudo (not bound to TTY)
3336
RUN set -ex; \
3437
wget -nv -nc -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \

0 commit comments

Comments
 (0)