From ce5b70b7ca8ca07a17e9fb32738bca82a10b604a Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 15 Jan 2025 22:18:36 +0000 Subject: [PATCH 1/6] Update gcc to 12.4 to support avx512_fp16 in k-NN faiss lib Signed-off-by: Peter Zhu --- .../build.al2.opensearch.x64.arm64.dockerfile | 32 ++++++++++++------- ...ch-dashboards.x64.arm64.ppc64le.dockerfile | 2 +- ...x8.opensearch.x64.arm64.ppc64le.dockerfile | 2 +- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile index 4d1f2136f4..143d46074b 100644 --- a/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile @@ -23,7 +23,7 @@ USER 0 RUN yum clean all && \ amazon-linux-extras install epel -y && \ yum update -y && \ - yum install -y which curl git gnupg2 tar net-tools procps-ng python3 python3-devel python3-pip zip unzip jq pigz + yum install -y which curl git gnupg2 tar net-tools procps-ng python3 python3-devel python3-pip zip unzip jq pigz # Create user group RUN groupadd -g 1000 $CONTAINER_USER && \ @@ -100,19 +100,28 @@ RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \ pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.32.17 # Upgrade gcc -RUN yum install -y gcc10* && \ - mv -v /usr/bin/gcc /usr/bin/gcc7-gcc && \ - mv -v /usr/bin/g++ /usr/bin/gcc7-g++ && \ - mv -v /usr/bin/gfortran /usr/bin/gcc7-gfortran && \ - update-alternatives --install /usr/bin/gcc gcc $(which gcc10-gcc) 1 && \ - update-alternatives --install /usr/bin/g++ g++ $(which gcc10-g++) 1 && \ - update-alternatives --install /usr/bin/gfortran gfortran $(which gcc10-gfortran) 1 +RUN curl -SL https://mirrors.ocf.berkeley.edu/gnu/gcc/gcc-12.4.0/gcc-12.4.0.tar.gz -o gcc12.tgz && \ + tar -xzf gcc12.tgz && cd gcc-12.4.0 && \ + ./contrib/download_prerequisites && \ + mkdir build && cd build && \ + ../configure --enable-languages=all --prefix=/usr --disable-multilib --disable-bootstrap && \ + make && make install && gcc --version && g++ --version && gfortran --version && \ + cd ../../ && rm -rf gcc12.tgz gcc-12.4.0 + +# Upgrade binutils +RUN yum install -y texinfo && curl -SLO https://sourceware.org/pub/binutils/snapshots/binutils-2.42.90.tar.xz && \ + tar -xf binutils-2.42.90.tar.xz && cd binutils-2.42.90 && \ + mkdir build && cd build && \ + ../configure --prefix=/usr && \ + make && make install && ld --version && \ + cd ../../ && rm -rf binutils-2.42.90.tar.xz binutils-2.42.90 + ENV FC=gfortran ENV CXX=g++ # Add k-NN Library dependencies RUN yum repolist && yum install lapack -y -RUN git clone -b v0.3.27 --single-branch https://github.com/xianyi/OpenBLAS.git && \ +RUN git clone -b v0.3.27 --single-branch https://github.com/OpenMathLib/OpenBLAS.git && \ cd OpenBLAS && \ if [ "$(uname -m)" = "x86_64" ]; then \ echo "Machine is x86_64. Adding DYNAMIC_ARCH=1 to openblas make command."; \ @@ -120,7 +129,8 @@ RUN git clone -b v0.3.27 --single-branch https://github.com/xianyi/OpenBLAS.git else \ make USE_OPENMP=1 FC=gfortran; \ fi && \ - make PREFIX=/usr/local install + make PREFIX=/usr/local install && \ + cd ../ && rm -rf OpenBLAS ENV LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" RUN pip3 install cmake==3.26.4 @@ -144,6 +154,6 @@ USER $CONTAINER_USER WORKDIR $CONTAINER_USER_HOME # Install fpm for opensearch dashboards core -RUN gem install dotenv -v 2.8.1 && gem install public_suffix -v 5.1.1 && gem install fpm -v 1.14.2 +RUN gem install dotenv -v 2.8.1 && gem install public_suffix -v 5.1.1 && gem install rchardet -v 1.8.0 && gem install fpm -v 1.14.2 ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH RUN fpm -v diff --git a/docker/ci/dockerfiles/current/build.almalinux8.opensearch-dashboards.x64.arm64.ppc64le.dockerfile b/docker/ci/dockerfiles/current/build.almalinux8.opensearch-dashboards.x64.arm64.ppc64le.dockerfile index a8ca5ed47d..c0eb3bed31 100644 --- a/docker/ci/dockerfiles/current/build.almalinux8.opensearch-dashboards.x64.arm64.ppc64le.dockerfile +++ b/docker/ci/dockerfiles/current/build.almalinux8.opensearch-dashboards.x64.arm64.ppc64le.dockerfile @@ -58,7 +58,7 @@ RUN curl -sSL https://rvm.io/mpapis.asc | gpg2 --import - && \ SHELL ["/bin/bash", "-lc"] # Install ruby / rpm / fpm related dependencies -RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && dnf install -y rpm-build createrepo && dnf clean all +RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && gem install rchardet -v 1.8.0 && dnf install -y rpm-build createrepo && dnf clean all ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin ENV RVM_HOME=/usr/local/rvm/bin diff --git a/docker/ci/dockerfiles/current/build.almalinux8.opensearch.x64.arm64.ppc64le.dockerfile b/docker/ci/dockerfiles/current/build.almalinux8.opensearch.x64.arm64.ppc64le.dockerfile index 4648f784f9..62d9aa915c 100644 --- a/docker/ci/dockerfiles/current/build.almalinux8.opensearch.x64.arm64.ppc64le.dockerfile +++ b/docker/ci/dockerfiles/current/build.almalinux8.opensearch.x64.arm64.ppc64le.dockerfile @@ -102,6 +102,6 @@ USER $CONTAINER_USER WORKDIR $CONTAINER_USER_HOME # Install fpm for opensearch dashboards core -RUN gem install dotenv -v 2.8.1 && gem install public_suffix -v 5.1.1 && gem install fpm -v 1.14.2 +RUN gem install dotenv -v 2.8.1 && gem install public_suffix -v 5.1.1 && gem install rchardet -v 1.8.0&& gem install fpm -v 1.14.2 ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH RUN fpm -v From 0677373d2b1443605c8e5af6133b0b84e9a14032 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 15 Jan 2025 22:21:50 +0000 Subject: [PATCH 2/6] More tweaks Signed-off-by: Peter Zhu --- .../current/build.al2.opensearch.x64.arm64.dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile index 143d46074b..aded083a10 100644 --- a/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile @@ -23,7 +23,7 @@ USER 0 RUN yum clean all && \ amazon-linux-extras install epel -y && \ yum update -y && \ - yum install -y which curl git gnupg2 tar net-tools procps-ng python3 python3-devel python3-pip zip unzip jq pigz + yum install -y which curl git gnupg2 tar net-tools procps-ng python3 python3-devel python3-pip zip unzip jq pigz # Create user group RUN groupadd -g 1000 $CONTAINER_USER && \ @@ -120,7 +120,7 @@ ENV FC=gfortran ENV CXX=g++ # Add k-NN Library dependencies -RUN yum repolist && yum install lapack -y +RUN yum repolist && yum install lapack -y && yum clean all && rm -rf /var/cache/yum/* RUN git clone -b v0.3.27 --single-branch https://github.com/OpenMathLib/OpenBLAS.git && \ cd OpenBLAS && \ if [ "$(uname -m)" = "x86_64" ]; then \ From 167a5e44fdbdd8c9c76fce5b7808a05f3f54a53e Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 22 Jan 2025 15:28:13 -0500 Subject: [PATCH 3/6] Support compilation arm64 Signed-off-by: Peter Zhu --- .../current/build.al2.opensearch.x64.arm64.dockerfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile index aded083a10..cd9a1aeaca 100644 --- a/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile @@ -67,8 +67,8 @@ RUN curl -sSL https://rvm.io/mpapis.asc | gpg2 --import - && \ SHELL ["/bin/bash", "-lc"] CMD ["/bin/bash", "-l"] -# Install ruby / rpm / fpm related dependencies -RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && yum install -y rpm-build createrepo && yum clean all +# Install ruby / rpm / fpm / openssl / gcc / binutils related dependencies +RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && yum install -y rpm-build createrepo texinfo && yum clean all ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin ENV RVM_HOME=/usr/local/rvm/bin @@ -100,8 +100,10 @@ RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \ pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.32.17 # Upgrade gcc -RUN curl -SL https://mirrors.ocf.berkeley.edu/gnu/gcc/gcc-12.4.0/gcc-12.4.0.tar.gz -o gcc12.tgz && \ +RUN curl -SL https://ci.opensearch.org/ci/dbc/tools/gcc/gcc-12.4.0.tar.gz -o gcc12.tgz && \ tar -xzf gcc12.tgz && cd gcc-12.4.0 && \ + sed -i 's@base_url=.*@base_url=https://ci.opensearch.org/ci/dbc/tools/gcc/@g' ./contrib/download_prerequisites && \ + cat ./contrib/download_prerequisites | grep "base_url=" && \ ./contrib/download_prerequisites && \ mkdir build && cd build && \ ../configure --enable-languages=all --prefix=/usr --disable-multilib --disable-bootstrap && \ @@ -109,7 +111,7 @@ RUN curl -SL https://mirrors.ocf.berkeley.edu/gnu/gcc/gcc-12.4.0/gcc-12.4.0.tar. cd ../../ && rm -rf gcc12.tgz gcc-12.4.0 # Upgrade binutils -RUN yum install -y texinfo && curl -SLO https://sourceware.org/pub/binutils/snapshots/binutils-2.42.90.tar.xz && \ +RUN curl -SLO https://ci.opensearch.org/ci/dbc/tools/gcc/binutils-2.42.90.tar.xz && \ tar -xf binutils-2.42.90.tar.xz && cd binutils-2.42.90 && \ mkdir build && cd build && \ ../configure --prefix=/usr && \ From 52315b2efd45f2de744bb017f933ad62bf4eaef3 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 22 Jan 2025 16:50:47 -0500 Subject: [PATCH 4/6] Support node20 on the AL2 images Signed-off-by: Peter Zhu --- .../build.al2.opensearch.x64.arm64.dockerfile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile index cd9a1aeaca..0899958412 100644 --- a/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile @@ -143,13 +143,16 @@ RUN pip3 install cmake==3.26.4 # GitHub enforce nodejs 20 official build in runner 2.317.0 of their actions and CentOS7/AL2 would fail due to having older glibc versions # Until https://github.com/actions/runner/pull/3128 is merged or AL2 is deprecated (2025/06) this is a quick fix with unofficial builds support glibc 2.17 # With changes done similar to this PR (https://github.com/opensearch-project/job-scheduler/pull/702) alongside the image here -# Only linux x64 is supported in unofficial build until https://github.com/nodejs/unofficial-builds/pull/91 is merged +# Only linux x64 glibc217 is supported in unofficial build until https://github.com/nodejs/unofficial-builds/pull/91 is merged for pre-compiled arm64 binaries +# The linux arm64 glibc226 tarball here is directly compiled from the source code on AL2 host for the time being RUN if [ `uname -m` = "x86_64" ]; then \ - curl -SL https://unofficial-builds.nodejs.org/download/release/v20.10.0/node-v20.10.0-linux-x64-glibc-217.tar.xz -o /node20.tar.xz; \ - mkdir /node_al2; \ - tar -xf /node20.tar.xz --strip-components 1 -C /node_al2; \ - rm -v /node20.tar.xz; \ - fi + curl -SL https://ci.opensearch.org/ci/dbc/tools/node/node-v20.18.0-linux-x64-glibc-217.tar.xz -o /node20.tar.xz; \ + else; \ + curl -SL https://ci.opensearch.org/ci/dbc/tools/node/node-v20.18.0-linux-arm64-glibc-226.tar.xz -o /node20.tar.xz; \ + fi; \ + mkdir /node_al2 && \ + tar -xf /node20.tar.xz --strip-components 1 -C /node_al2 && \ + rm -v /node20.tar.xz # Change User USER $CONTAINER_USER From f57df1229c68d7528b96a674c8f1517d05097f69 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 22 Jan 2025 16:52:21 -0500 Subject: [PATCH 5/6] Support node20 on the AL2 images Signed-off-by: Peter Zhu --- .../current/build.al2.opensearch.x64.arm64.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile index 0899958412..d6eeccef37 100644 --- a/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile @@ -147,7 +147,7 @@ RUN pip3 install cmake==3.26.4 # The linux arm64 glibc226 tarball here is directly compiled from the source code on AL2 host for the time being RUN if [ `uname -m` = "x86_64" ]; then \ curl -SL https://ci.opensearch.org/ci/dbc/tools/node/node-v20.18.0-linux-x64-glibc-217.tar.xz -o /node20.tar.xz; \ - else; \ + else \ curl -SL https://ci.opensearch.org/ci/dbc/tools/node/node-v20.18.0-linux-arm64-glibc-226.tar.xz -o /node20.tar.xz; \ fi; \ mkdir /node_al2 && \ From e676b85b1cdc360ab3a9aa783a8e85a37e409eda Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 22 Jan 2025 17:06:36 -0500 Subject: [PATCH 6/6] remove debug checks Signed-off-by: Peter Zhu --- .../current/build.al2.opensearch.x64.arm64.dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile index d6eeccef37..815086959b 100644 --- a/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile @@ -103,7 +103,6 @@ RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \ RUN curl -SL https://ci.opensearch.org/ci/dbc/tools/gcc/gcc-12.4.0.tar.gz -o gcc12.tgz && \ tar -xzf gcc12.tgz && cd gcc-12.4.0 && \ sed -i 's@base_url=.*@base_url=https://ci.opensearch.org/ci/dbc/tools/gcc/@g' ./contrib/download_prerequisites && \ - cat ./contrib/download_prerequisites | grep "base_url=" && \ ./contrib/download_prerequisites && \ mkdir build && cd build && \ ../configure --enable-languages=all --prefix=/usr --disable-multilib --disable-bootstrap && \