Merge branch 'master' of https://github.com/soedinglab/mmseqs2

Author: martin-steinegger

.cirrus.yml

@@ -39,7 +39,7 @@ task:
         CXX: g++-4.9
   compile_script: |
     mkdir build && cd build
-    cmake -DCMAKE_BUILD_TYPE=Release -DHAVE_TESTS=1 -DENABLE_WERROR=1 -DHAVE_SSE4_1=1 -DREQUIRE_OPENMP=0 ..
+    cmake -DCMAKE_BUILD_TYPE=Release -DHAVE_TESTS=1 -DENABLE_WERROR=0 -DHAVE_SSE4_1=1 -DREQUIRE_OPENMP=0 ..
     make -j $(nproc --all)
   test_script: MMSEQS_NUM_THREADS=4 ./util/regression/run_regression.sh ./build/src/mmseqs SCRATCH SEARCH
 

.github/workflows/docker.yml

@@ -11,9 +11,11 @@ on:
       tag:
         required: true
         type: string
+        description: "Docker tag"
       latest:
         default: false
         type: boolean
+        description: "Mark as latest"
 
 
 env:
@@ -45,7 +47,7 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
@@ -77,4 +79,63 @@ jobs:
             ${{ steps.dispatch_tag.outputs.tag }}
             ${{ steps.dispatch_tag.outputs.latest }}
           labels: ${{ steps.meta.outputs.labels }}
+  build-and-push-gpu-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # - name: Set up QEMU
+      #   uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          flavor: |
+            suffix=-cuda12,onlatest=true
+
+      - name: Tag for workflow_dispatch
+        id: dispatch_tag
+        run: |
+          if [ x"$TAG" != x"" ];then
+            echo "::set-output name=tag::${FULL_TAG}"
+          fi
+          if [ x"$LATEST" = x"true" ]; then
+            echo "::set-output name=latest::${LATEST_TAG}"
+          fi
+        env:
+          FULL_TAG: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.event.inputs.tag }}
+          LATEST_TAG: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          TAG: ${{ github.event.inputs.tag }}
+          LATEST: ${{ github.event.inputs.latest }}
 
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          platforms: linux/amd64
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          build-args: |
+            GPU=1
+          tags: |
+            ${{ steps.meta.outputs.tags }}
+            ${{ steps.dispatch_tag.outputs.tag }}
+            ${{ steps.dispatch_tag.outputs.latest }}
+          labels: ${{ steps.meta.outputs.labels }}

.gitignore

@@ -30,6 +30,7 @@
 src/workflow/time_test
 
 build/
+build-*/
 .idea/
 cmake-build-*/
 BenchmarkingDatas/

CMakeLists.txt

@@ -19,6 +19,13 @@ set(HAVE_S390X 0 CACHE BOOL "Have s390x architecture")
 set(NATIVE_ARCH 1 CACHE BOOL "Assume native architecture for SIMD. Use one of the HAVE_* options or set CMAKE_CXX_FLAGS to the appropriate flags if you disable this.")
 set(USE_SYSTEM_ZSTD 0 CACHE BOOL "Use zstd provided by system instead of bundled version")
 
+set(ENABLE_CUDA 0 CACHE BOOL "Enable CUDA")
+
+set(PREFER_STATIC 0 CACHE BOOL "Prefer finding .a libs")
+if (PREFER_STATIC)
+    set(CMAKE_FIND_LIBRARY_SUFFIXES ".a" ".so")
+endif()
+
 if (HAVE_SANITIZER)
     include(FindUBSan)
     include(FindASan)
@@ -214,16 +221,20 @@ include_directories(lib/simde)
 
 include_directories(lib)
 include_directories(lib/simd)
-include_directories(lib/gzstream)
 include_directories(lib/alp)
-include_directories(lib/cacode)
 include_directories(lib/ksw2)
 include_directories(lib/xxhash)
 if (NOT DISABLE_IPS4O)
     include_directories(lib/ips4o)
 endif ()
 
-add_subdirectory(lib/cacode)
+# libmarv
+if (ENABLE_CUDA)
+    set(LIBRARY_ONLY 1 CACHE INTERNAL "" FORCE)
+    include_directories(lib/libmarv/src)
+    add_subdirectory(lib/libmarv/src EXCLUDE_FROM_ALL)
+endif ()
+
 add_subdirectory(lib/alp)
 add_subdirectory(lib/ksw2)
 add_subdirectory(data)

Dockerfile

@@ -1,15 +1,21 @@
 ARG APP=mmseqs
-FROM --platform=$BUILDPLATFORM debian:stable-slim as builder
+FROM --platform=$BUILDPLATFORM debian:bookworm-slim AS builder
 ARG TARGETARCH
 ARG APP
+ARG GPU
 
 RUN dpkg --add-architecture $TARGETARCH \
     && apt-get update \
     && apt-get install -y \
-      build-essential cmake xxd git \
+      build-essential cmake xxd git wget \
       zlib1g-dev libbz2-dev libatomic1 \
-      crossbuild-essential-$TARGETARCH zlib1g-dev:$TARGETARCH libbz2-dev:$TARGETARCH \
-    && rm -rf /var/lib/apt/lists/*
+      crossbuild-essential-$TARGETARCH zlib1g-dev:$TARGETARCH libbz2-dev:$TARGETARCH; \
+    if [ "$GPU" = "1" ]; then \
+      wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb; \
+      dpkg -i cuda-keyring_1.1-1_all.deb; \
+      apt-get update && apt-get install -y cuda-nvcc-12-6 ninja-build; \
+    fi; \
+    rm -rf /var/lib/apt/lists/*;
 
 WORKDIR /opt/build
 ADD . .
@@ -22,33 +28,51 @@ RUN if [ "$TARGETARCH" = "arm64" ]; then \
       mv src/${APP} /opt/build/${APP}_arch; \
       touch /opt/build/${APP}_sse2 /opt/build/${APP}_sse41 /opt/build/${APP}_avx2; \
     else \
-      mkdir -p build_sse2/src && mkdir -p build_sse41/src && mkdir -p build_avx2/src; \
-      cd /opt/build/build_sse2; \
-      cmake -DHAVE_SSE2=1 -DHAVE_MPI=0 -DHAVE_TESTS=0 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. ..; \
-      make -j $(nproc --all); \
-      mv src/${APP} /opt/build/${APP}_sse2; \
-      cd /opt/build/build_sse41; \
-      cmake -DHAVE_SSE4_1=1 -DHAVE_MPI=0 -DHAVE_TESTS=0 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. ..; \
-      make -j $(nproc --all); \
-      mv src/${APP} /opt/build/${APP}_sse41; \
-      cd /opt/build/build_avx2; \
-      cmake -DHAVE_AVX2=1 -DHAVE_MPI=0 -DHAVE_TESTS=0 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. ..; \
-      make -j $(nproc --all); \
-      mv src/${APP} /opt/build/${APP}_avx2; \
-      touch /opt/build/${APP}_arch; \
+      if [ "$GPU" = "1" ]; then \
+        export CUDACXX=/usr/local/cuda/bin/nvcc; \
+        mkdir -p build_avx2/src; \
+        cd /opt/build/build_avx2; \
+        cmake -GNinja -DHAVE_AVX2=1 -DHAVE_MPI=0 -DHAVE_TESTS=0 \
+        -DPREFER_STATIC=1 -DBUILD_SHARED_LIBS=OFF \
+        -DCMAKE_EXE_LINKER_FLAGS="-static-libgcc -static-libstdc++" \
+        -DENABLE_CUDA=1 -DCMAKE_CUDA_ARCHITECTURES="75-real;80-real;86-real;89-real;90" \
+        -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. ..; \
+        cmake --build . -j$(nproc --all); \
+        mv src/${APP} /opt/build/${APP}_avx2; \
+        touch /opt/build/${APP}_arch /opt/build/${APP}_sse41 /opt/build/${APP}_sse2; \
+      else \
+        mkdir -p build_sse2/src && mkdir -p build_sse41/src && mkdir -p build_avx2/src; \
+        cd /opt/build/build_sse2; \
+        cmake -DHAVE_SSE2=1 -DHAVE_MPI=0 -DHAVE_TESTS=0 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. ..; \
+        make -j $(nproc --all); \
+        mv src/${APP} /opt/build/${APP}_sse2; \
+        cd /opt/build/build_sse41; \
+        cmake -DHAVE_SSE4_1=1 -DHAVE_MPI=0 -DHAVE_TESTS=0 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. ..; \
+        make -j $(nproc --all); \
+        mv src/${APP} /opt/build/${APP}_sse41; \
+        cd /opt/build/build_avx2; \
+        cmake -DHAVE_AVX2=1 -DHAVE_MPI=0 -DHAVE_TESTS=0 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. ..; \
+        make -j $(nproc --all); \
+        mv src/${APP} /opt/build/${APP}_avx2; \
+        touch /opt/build/${APP}_arch; \
+      fi; \
     fi
 
-FROM debian:stable-slim
+FROM debian:bookworm-slim
 ARG TARGETARCH
 ARG APP
-
-RUN apt-get update && apt-get install -y \
-      gawk bash grep libstdc++6 libgomp1 libatomic1 zlib1g libbz2-1.0 wget tar \
-    && rm -rf /var/lib/apt/lists/*
+ARG GPU
 
 COPY --from=builder /opt/build/${APP}_arch /opt/build/${APP}_sse2 /opt/build/${APP}_sse41 /opt/build/${APP}_avx2 /usr/local/bin/
 ADD util/${APP}_wrapper.sh /usr/local/bin/entrypoint
-RUN if [ "$TARGETARCH" = "arm64" ]; then rm -f /usr/local/bin/entrypoint; ln -s /usr/local/bin/${APP}_arch /usr/local/bin/entrypoint; fi
 
-ENTRYPOINT ["/usr/local/bin/entrypoint"]
+RUN apt-get update && apt-get install -y \
+      gawk bash grep libstdc++6 libgomp1 libatomic1 zlib1g libbz2-1.0 wget tar aria2 \
+    && rm -rf /var/lib/apt/lists/*; \
+    if [ "$TARGETARCH" = "arm64" ]; then \
+      rm -f /usr/local/bin/entrypoint; ln -s /usr/local/bin/${APP}_arch /usr/local/bin/entrypoint; \
+    elif [ "$GPU" = "1" ]; then \
+      rm -f /usr/local/bin/entrypoint; ln -s /usr/local/bin/${APP}_avx2 /usr/local/bin/entrypoint; \
+    fi
 
+ENTRYPOINT ["/usr/local/bin/entrypoint"]