Swap to Spack and matrix-based GHA for more reliability

Author: Matthew-Whitlock

.github/Dockerfile

@@ -1,81 +1,49 @@
-version: "3.9"
-
-x-fenix: &fenix
-  build: &fenix-build
-    context: ./
-    dockerfile: .github/Dockerfile
-    args:
-      OPENMPI_REPO: open-mpi/ompi
-      OPENMPI_VERS_PREFIX: tags
-      OPENMPI_VERS: v5.0.0rc10
-    #Caches should be manually scoped, or they'll conflict.
-    x-bake:
-      cache-from:
-        - type=gha,scope=default
-      cache-to:
-        - type=gha,scope=default,mode=max
-
 services:
-  #fenix_ompi_5rc10:
-  # <<: *fenix
-  # image: "fenix:ompi_5rc10"
-  # build: 
-  #   <<: *fenix-build
-  #   x-bake:
-  #     cache-from:
-  #       - type=gha,scope=ompi_5rc10
-  #     cache-to:
-  #       - type=gha,scope=ompi_5rc10,mode=max
-
-  fenix_ompi_5:
-    <<: *fenix
-    image: "fenix:ompi_5"
+  bootstrap:
+    image: "bootstrap"
     build:
-      <<: *fenix-build
+      dockerfile_inline: |
+        FROM spack/ubuntu-jammy:0.22.2
+        VOLUME /configs
+        ARG OMPI_VERSION
+        ENV OMPI_VERSION=$${OMPI_VERSION}
+        CMD cp /configs/spack.yaml . && \
+            spack -e . add openmpi@$${OMPI_VERSION} && \
+            spack -e . containerize >/configs/spack.Dockerfile
       args:
-        - OPENMPI_VERS_PREFIX=heads
-        - OPENMPI_VERS=v5.0.x
-      x-bake:
-        cache-from:
-          - type=gha,scope=ompi_5
-        cache-to:
-          - type=gha,scope=ompi_5,mode=max
-  
-  fenix_ompi_main:
-    <<: *fenix
-    image: "fenix:ompi_main"
+        OMPI_VERSION: main
+      no_cache: true
+    pull_policy: build
+    volumes:
+      - .github/:/configs
+
+  env:
+    image: "ghcr.io/sandialabs/fenix/env:main"
     build:
-      <<: *fenix-build
-      args:
-        - OPENMPI_VERS_PREFIX=heads
-        - OPENMPI_VERS=main
-      x-bake:
-        cache-from:
-          - type=gha,scope=ompi_main
-        cache-to:
-          - type=gha,scope=ompi_main,mode=max
-  
-  fenix_icldisco_latest:
-    <<: *fenix
-    image: "fenix:icldisco_latest"
+      # Generated by running the bootstrap image
+      dockerfile: .github/spack.Dockerfile
+
+  fenix:
+    image: "fenix"
     build:
-      <<: *fenix-build
+      dockerfile_inline: |
+        ARG OMPI_VERSION main
+        FROM ghcr.io/sandialabs/fenix/env:$${OMPI_VERSION}
+        COPY . /fenix
+        RUN . /opt/spack-environment/activate.sh && \
+            mkdir -p /fenix/build && \
+            cd /fenix/build && \
+            cmake /fenix \
+                -DCMAKE_BUILD_TYPE=Release \
+                -DCMAKE_C_COMPILER=mpicc \
+                -DFENIX_EXAMPLES=ON \
+                -DFENIX_TESTS=ON \
+                -DMPIEXEC_PREFLAGS="--allow-run-as-root;--map-by;:oversubscribe" && \
+            make -j
+        
+        WORKDIR /fenix/build
+        ENTRYPOINT ["/entrypoint.sh"]
+        CMD ["ctest", "--output-on-failure", "--timeout", "60"]
       args:
-        - OPENMPI_REPO=icldisco/ompi
-        - OPENMPI_VERS_PREFIX=heads
-        - OPENMPI_VERS=ulfm/latest
-      x-bake:
-        cache-from:
-          - type=gha,scope=icldisco_latest
-        cache-to:
-          - type=gha,scope=icldisco_latest,mode=max
-  
-  #fenix_icldisco_experimental:
-  #  <<: *fenix
-  #  image: fenix/icldisco
-  #  build:
-  #    <<: *fenix-build
-  #   args:
-  #     - OPENMPI_REPO=icldisco/ompi
-  #     - OPENMPI_VERS_PREFIX=heads
-  #     - OPENMPI_VERS=ulfm/experimental
+        OMPI_VERSION: main
+    pull_policy: build

.github/docker-compose.yml

@@ -0,0 +1,31 @@
+spack:
+  packages:
+    openmpi:
+      variants: +internal-hwloc +internal-libevent +internal-pmix
+  concretizer:
+    unify: true
+    reuse: true
+
+  container:
+    format: docker
+    strip: false
+    images:
+      os: ubuntu:22.04
+      spack: 0.22.2
+    os_packages:
+      build:
+        - build-essential
+        - autotools-dev
+        - pkg-config
+        - python3
+        - m4
+        - autoconf
+        - automake
+        - flex
+        - git
+        - zlib1g-dev
+        - libperl-dev
+        - numactl
+      final:
+        - build-essential
+        - cmake

.github/spack.yaml

@@ -11,21 +11,65 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ompi_version:
+          - main
+          - 5.0.3
+
     steps:
-      - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
-      - name: Build
-        uses: docker/bake-action@master
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to GHCR container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Bake the bootstrap docker image
+        uses: docker/bake-action@v5
         with:
-          files: |
-            .github/docker-compose.yml
-          load: true
-      - name: Test open-mpi v5.0.x
-        if: success() || failure()
-        run: docker run fenix:ompi_5
-      - name: Test open-mpi main
-        if: success() || failure()
-        run: docker run fenix:ompi_main
-      - name: Test icldisco latest
-        if: success() || failure()
-        run: docker run fenix:icldisco_latest
+          files: .github/docker-compose.yml
+          targets: bootstrap
+          workdir: .
+          set: |
+            *.output=type=docker,name=bootstrap
+            *.cache-from=type=gha,scope=bootstrap/${{ matrix.ompi_version }}
+            *.cache-to=type=gha,mode=max,scope=bootstrap/${{ matrix.ompi_version }}
+            *.args.OMPI_VERSION=${{ matrix.ompi_version }}
+      
+      - name: Bootstrap the environment Dockerfile
+        run: docker run -v ${GITHUB_WORKSPACE}/.github:/configs bootstrap
+
+      - name: Build the environment
+        uses: docker/bake-action@v5
+        with:
+          files: .github/docker-compose.yml
+          targets: env
+          workdir: .
+          pull: true
+          set: |
+            *.cache-from=type=gha,scope=env/${{ matrix.ompi_version }}
+            *.cache-to=type=gha,mode=max,scope=env/${{ matrix.ompi_version }}
+            env.tags=ghcr.io/sandialabs/fenix/env:${{ matrix.ompi_version }}
+            env.output=type=registry,name=ghcr.io/sandialabs/fenix/env:${{ matrix.ompi_version }}
+
+      - name: Build Fenix
+        uses: docker/bake-action@v5
+        with:
+          source: .
+          files: .github/docker-compose.yml
+          targets: fenix
+          workdir: .
+          set: |
+            *.output=type=docker,name=fenix
+            *.args.OMPI_VERSION=${{ matrix.ompi_version }}
+
+      - name: Test Fenix
+        run: docker run fenix