Build GPU ARM64 binaries

soedinglab · Jan 25, 2025 · 1668032 · 1668032
1 parent 0bda833
commit 1668032
Show file tree

Hide file tree

Showing 2 changed files with 83 additions and 3 deletions.
diff --git a/.github/workflows/Dockerfile.GPU-buster-cross-sbsa b/.github/workflows/Dockerfile.GPU-buster-cross-sbsa
@@ -0,0 +1,58 @@
+FROM debian:buster
+ARG VER="12-6"
+RUN dpkg --add-architecture arm64
+RUN apt-get update
+RUN apt-get -y install -o APT::Immediate-Configure=false \
+        git wget vim ninja-build crossbuild-essential-arm64 zlib1g-dev:arm64 libbz2-dev:arm64 software-properties-common
+
+ENV CC=/usr/bin/aarch64-linux-gnu-gcc
+ENV CXX=/usr/bin/aarch64-linux-gnu-g++
+ENV CUDAHOSTCXX=/usr/bin/aarch64-linux-gnu-g++
+ENV CUDACXX=/usr/local/cuda/bin/nvcc
+ENV LIBGCC=/usr/lib/gcc-cross/aarch64-linux-gnu/8
+ENV CUDAFLAGS="-t 0 --split-compile=0"
+
+RUN wget https://github.com/Kitware/CMake/releases/download/v3.31.0/cmake-3.31.0-linux-x86_64.sh; \
+    chmod +x cmake-3.31.0-linux-x86_64.sh; \
+    ./cmake-3.31.0-linux-x86_64.sh --skip-license --prefix=/usr/local;
+
+RUN wget -O cuda-x64.deb  https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.1-1_all.deb
+RUN wget -O cuda-sbsa.deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/cross-linux-sbsa/cuda-keyring_1.1-1_all.deb
+RUN dpkg -i cuda-x64.deb && dpkg -i cuda-sbsa.deb
+RUN apt-get update
+RUN apt-get install -y cuda-nvcc-${VER} cuda-nvcc-cross-sbsa-${VER} cuda-cudart-cross-sbsa-${VER}
+
+WORKDIR /work
+ADD . .
+RUN echo "set(CMAKE_SYSTEM_NAME Linux)" > /work/toolchain.cmake; \
+    echo "set(CMAKE_SYSTEM_PROCESSOR aarch64)" >> /work/toolchain.cmake; \
+    echo "set(CMAKE_C_COMPILER /usr/bin/aarch64-linux-gnu-gcc)" >> /work/toolchain.cmake; \
+    echo "set(CMAKE_CXX_COMPILER /usr/bin/aarch64-linux-gnu-g++)" >> /work/toolchain.cmake;
+
+RUN mkdir -p /work/build && cd /work/build; \
+    if [ -e "${LIBGCC}/libgomp.so" ]; then \
+        mv -f -- "${LIBGCC}/libgomp.so" "${LIBGCC}/libgomp.so.disabled"; \
+    fi; \
+    /usr/local/bin/cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DHAVE_TESTS=1 -DENABLE_WERROR=1 -DHAVE_ARM8=1 \
+        -DCMAKE_TOOLCHAIN_FILE=/work/toolchain.cmake \
+        -DOpenMP_C_FLAGS="-fopenmp -I${LIBGCC} -L${LIBGCC}" -DOpenMP_C_LIB_NAMES=gomp -DOpenMP_CXX_FLAGS="-fopenmp -I${LIBGCC} -L${LIBGCC}" -DOpenMP_CXX_LIB_NAMES=gomp -DOpenMP_gomp_LIBRARY="${LIBGCC}/libgomp.a" \
+        -DATOMIC_LIB_OVERRIDE="${LIBGCC}/libatomic.a" \
+        -DCMAKE_POLICY_DEFAULT_CMP0074=NEW -DCMAKE_POLICY_DEFAULT_CMP0144=NEW \
+        -DFORCE_STATIC_DEPS=1 -DENABLE_CUDA=1 -DCMAKE_CUDA_ARCHITECTURES="75-real;80-real;86-real;89-real;90" ..; \
+    cmake --build . -j$(nproc --all) -v;
+
+RUN if readelf -a /work/build/src/mmseqs | grep -i "Shared library" | grep -P -v "(linux-vdso|ld-linux-aarch64|libc|libm|libdl|librt|libpthread).so" | grep -q .; then \
+        echo "Error: unwanted libraries found"; \
+        readelf -a /work/build/src/mmseqs | grep -i "Shared library"; \
+        exit 1; \
+    fi; \
+    if readelf -Ws /work/build/src/mmseqs | grep -q GLIBC_PRIVATE; then \
+        echo "Error: binary contains private glibc symbols"; \
+        readelf -Ws /work/build/src/mmseqs; \
+        exit 1; \
+    fi; \
+    LIBC_V=$(readelf -V /work/build/src/mmseqs | awk '$3 ~ /^GLIBC_/ { print $3 }' | sort -V | tail -n1); \
+    if awk -v ver="$LIBC_V" 'BEGIN { split(ver, v, /[_.]/); if ((v[2] > 2) || (v[2] == 2 && v[3] > 28)) exit 0; exit 1; }'; then \
+        echo "Error: glibc too new"; \
+        exit 1; \
+    fi;
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -134,13 +134,21 @@ jobs:
     pool:
       vmImage: 'Ubuntu-20.04'
     timeoutInMinutes: 120
+    strategy:
+      matrix:
+        x86_64:
+          ARCH: x86_64
+          DOCKER: .github/workflows/Dockerfile.GPU-manylinux2014
+        aarch64:
+          ARCH: aarch64
+          DOCKER: .github/workflows/Dockerfile.GPU-buster-cross-sbsa
     steps:
       - checkout: self
         submodules: false
       - script: |
           mkdir -p ${BUILD_SOURCESDIRECTORY}/output
           docker build -t manylinux-builder \
-            -f .github/workflows/Dockerfile.GPU-manylinux2014 .
+            -f ${DOCKER} .
           docker run --rm \
             -v ${BUILD_SOURCESDIRECTORY}/output:/output \
             manylinux-builder \
@@ -149,7 +157,7 @@ jobs:
       - task: PublishPipelineArtifact@0
         inputs:
           targetPath: $(Build.SourcesDirectory)/output/mmseqs
-          artifactName: mmseqs-linux-gpu
+          artifactName: mmseqs-linux-gpu-$(ARCH)
 
   - job: build_ubuntu_cross
     displayName: Ubuntu Cross-Compile
@@ -333,7 +341,7 @@ jobs:
           archiveType: tar
       - task: DownloadPipelineArtifact@1
         inputs:
-          artifactName: mmseqs-linux-gpu
+          artifactName: mmseqs-linux-gpu-x86_64
           targetPath: $(Build.SourcesDirectory)/mmseqs/bin
       - script:
           chmod +x "${BUILD_SOURCESDIRECTORY}/mmseqs/bin/mmseqs"
@@ -343,6 +351,18 @@ jobs:
           archiveFile: $(Build.SourcesDirectory)/mmseqs-linux-gpu.tar.gz
           includeRootFolder: true
           archiveType: tar
+      - task: DownloadPipelineArtifact@1
+        inputs:
+          artifactName: mmseqs-linux-gpu-aarch64
+          targetPath: $(Build.SourcesDirectory)/mmseqs/bin
+      - script:
+          chmod +x "${BUILD_SOURCESDIRECTORY}/mmseqs/bin/mmseqs"
+      - task: ArchiveFiles@2
+        inputs:
+          rootFolderOrFile: $(Build.SourcesDirectory)/mmseqs
+          archiveFile: $(Build.SourcesDirectory)/mmseqs-linux-gpu-arm64.tar.gz
+          includeRootFolder: true
+          archiveType: tar
       - task: DownloadPipelineArtifact@1
         inputs:
           artifactName: mmseqs-linux-POWER9
@@ -413,6 +433,7 @@ jobs:
             mmseqs-linux-sse41.tar.gz \
             mmseqs-linux-avx2.tar.gz \
             mmseqs-linux-gpu.tar.gz \
+            mmseqs-linux-gpu-arm64.tar.gz \
             mmseqs-linux-arm64.tar.gz \
             mmseqs-linux-ppc64le-power9.tar.gz \
             mmseqs-win64.zip
@@ -423,6 +444,7 @@ jobs:
             -F file[][email protected] -F signature[][email protected] \
             -F file[][email protected] -F signature[][email protected] \
             -F file[][email protected] -F signature[][email protected] \
+            -F file[][email protected] -F signature[][email protected] \
             -F file[][email protected] -F signature[][email protected] \
             -F file[][email protected] -F signature[][email protected] \
             -F file[][email protected]  -F signature[][email protected] \