diff --git a/.github/workflows/base-glibc-busybox-bash.yaml b/.github/workflows/base-glibc-busybox-bash.yaml index 9f76c16d..300702a8 100644 --- a/.github/workflows/base-glibc-busybox-bash.yaml +++ b/.github/workflows/base-glibc-busybox-bash.yaml @@ -19,7 +19,7 @@ jobs: # The base image is not intended to change often and should be used with # version tags or checksum IDs, but not via "latest". MAJOR_VERSION: 3 - MINOR_VERSION: 0 + MINOR_VERSION: 1 IMAGE_NAME: base-glibc-busybox-bash BUSYBOX_VERSION: '1.36.1' DEBIAN_VERSION: '12.2' @@ -28,7 +28,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: arm64 @@ -44,15 +44,19 @@ jobs: ${{ env.MAJOR_VERSION }}.${{ env.MINOR_VERSION }} latest ' + # Adds image and tags to outputs which can be used in later steps. printf %s\\n \ "image=${image_name}" \ "tags=$( echo ${tags} )" \ >> $GITHUB_OUTPUT + # Create manifest (which is considered arch-independent) for tag in ${tags} ; do buildah manifest create "${image_name}:${tag}" done + # --iidfile prints the built image ID to the specified file. This is + # used so we can refer to the image in later steps. iidfile="$( mktemp )" buildah bud \ --iidfile="${iidfile}" \ @@ -71,6 +75,7 @@ jobs: image_id="$( cat "${iidfile}" )" rm "${iidfile}" + # Extract various package info and version info to store as labels container="$( buildah from "${image_id}" )" run() { buildah run "${container}" "${@}" ; } deb_list="$( run cat /.deb.lst | tr '\n' '|' | sed 's/|$//' )" @@ -80,6 +85,7 @@ jobs: bash="$( run bash --version | sed '1!d' )" buildah rm "${container}" + # Store package/version info as labels for the image container="$( buildah from "${image_id}" )" buildah config \ --label=glibc="${glibc}" \ @@ -89,8 +95,11 @@ jobs: --label=pkg-list="${pkg_list}" \ "${container}" + # Store the new image (now with labels) image_id="$( buildah commit "${container}" )" buildah rm "${container}" + + # image tag includes arch; then added to manifest which does not include arch for tag in ${tags} ; do buildah tag \ "${image_id}" \ @@ -104,6 +113,8 @@ jobs: - name: Test run: | image='${{ steps.build.outputs.image }}' + + # Extract image ids from manifest to test. ids="$( for tag in ${{ steps.build.outputs.tags }} ; do buildah manifest inspect "${image}:${tag}" \ @@ -114,6 +125,8 @@ jobs: done done )" + + # See Dockerfile.test for actual tests run ids="$( printf %s "${ids}" | sort -u )" for id in ${ids} ; do podman history "${id}" @@ -127,7 +140,7 @@ jobs: - name: Check Tags run: | # FIX upstream: Quay.io does not support immutable images currently. - # => Try to use the REST API to check for duplicate tags. + # => Try to use the REST API to check for duplicate tags and exit if they exist response="$( curl -sL \ 'https://quay.io/api/v1/repository/bioconda/${{ steps.build.outputs.image }}/tag/' diff --git a/.github/workflows/base-glibc-debian-bash.yaml b/.github/workflows/base-glibc-debian-bash.yaml index 331f44ea..29abe884 100644 --- a/.github/workflows/base-glibc-debian-bash.yaml +++ b/.github/workflows/base-glibc-debian-bash.yaml @@ -19,7 +19,7 @@ jobs: # The base image is not intended to change often and should be used with # version tags or checksum IDs, but not via "latest". MAJOR_VERSION: 3 - MINOR_VERSION: 0 + MINOR_VERSION: 1 IMAGE_NAME: base-glibc-debian-bash DEBIAN_VERSION: '12.2' @@ -27,7 +27,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: arm64 @@ -43,15 +43,19 @@ jobs: ${{ env.MAJOR_VERSION }}.${{ env.MINOR_VERSION }} latest ' + # Adds image and tags to outputs which can be used in later steps. printf %s\\n \ "image=${image_name}" \ "tags=$( echo ${tags} )" \ >> $GITHUB_OUTPUT + # Create manifest (which is considered arch-independent) for tag in ${tags} ; do buildah manifest create "${image_name}:${tag}" done + # --iidfile prints the built image ID to the specified file. This is + # used so we can refer to the image in later steps. for arch in amd64 arm64 ; do iidfile="$( mktemp )" buildah bud \ @@ -61,6 +65,7 @@ jobs: image_id="$( cat "${iidfile}" )" rm "${iidfile}" + # Extract various package info and version info to store as labels container="$( buildah from "${image_id}" )" run() { buildah run "${container}" "${@}" ; } deb_list="$( run cat /.deb.lst | tr '\n' '|' | sed 's/|$//' )" @@ -70,6 +75,7 @@ jobs: bash="$( run bash --version | sed '1!d' )" buildah rm "${container}" + # Store package/version info as labels for the image container="$( buildah from "${image_id}" )" buildah config \ --label=glibc="${glibc}" \ @@ -79,8 +85,11 @@ jobs: --label=pkg-list="${pkg_list}" \ "${container}" + # Store the new image (now with labels) image_id="$( buildah commit "${container}" )" buildah rm "${container}" + + # image tag includes arch; then added to manifest which does not include arch for tag in ${tags} ; do buildah tag \ "${image_id}" \ @@ -94,6 +103,8 @@ jobs: - name: Test run: | image='${{ steps.build.outputs.image }}' + + # Extract image ids from manifest to test. ids="$( for tag in ${{ steps.build.outputs.tags }} ; do buildah manifest inspect "${image}:${tag}" \ @@ -104,6 +115,8 @@ jobs: done done )" + + # See Dockerfile.test for actual tests run ids="$( printf %s "${ids}" | sort -u )" for id in ${ids} ; do podman history "${id}" @@ -117,7 +130,7 @@ jobs: - name: Check Tags run: | # FIX upstream: Quay.io does not support immutable images currently. - # => Try to use the REST API to check for duplicate tags. + # => Try to use the REST API to check for duplicate tags and exit if they exist response="$( curl -sL \ 'https://quay.io/api/v1/repository/bioconda/${{ steps.build.outputs.image }}/tag/' diff --git a/.github/workflows/bioconda-utils-build-env-cos7.yml b/.github/workflows/bioconda-utils-build-env-cos7.yml new file mode 100644 index 00000000..a180973d --- /dev/null +++ b/.github/workflows/bioconda-utils-build-env-cos7.yml @@ -0,0 +1,212 @@ +name: 'Build & Push: bioconda-utils-build-env-cos7' +on: + push: + branches: + - test-push + paths: + - images/bioconda-utils-build-env-cos7/** + - .github/workflows/bioconda-utils-build-env-cos7.yml + pull_request: + paths: + - images/bioconda-utils-build-env-cos7/** + - .github/workflows/bioconda-utils-build-env-cos7.yml + +jobs: + build: + name: Build & Push + runs-on: ubuntu-22.04 + env: + # The base image is not intended to change often and should be used with + # version tags or checksum IDs, but not via "latest". + MAJOR_VERSION: 3 + MINOR_VERSION: 1 + IMAGE_NAME: bioconda-utils-build-env-cos7 + + steps: + - name: Checkout bioconda-containers + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Checkout bioconda-utils + uses: actions/checkout@v4 + with: + fetch-depth: 0 + repository: 'bioconda/bioconda-utils' + path: 'images/bioconda-utils-build-env-cos7/bioconda-utils' + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + + - name: Build + id: build + run: | + set -xeu + cd 'images/${{ env.IMAGE_NAME }}' + + image_name='${{ env.IMAGE_NAME }}' + tags=' + ${{ env.MAJOR_VERSION }} + ${{ env.MAJOR_VERSION }}.${{ env.MINOR_VERSION }} + latest + ' + # Adds image and tags to outputs which can be used in later steps. + printf %s\\n \ + "image=${image_name}" \ + "tags=$( echo ${tags} )" \ + >> $GITHUB_OUTPUT + + # Create manifest (which is considered arch-independent) + for tag in ${tags} ; do + buildah manifest create "${image_name}:${tag}" + done + + # Due to different nomenclature used by conda-forge and buildah, we + # need to map archs to base images. + for arch_and_image in \ + "amd64=quay.io/condaforge/linux-anvil-cos7-x86_64" \ + "arm64=quay.io/condaforge/linux-anvil-aarch64"; + do + # Unpack archs and base images + arch=$(echo $arch_and_image | cut -f1 -d "=") + base_image=$(echo $arch_and_image | cut -f2 -d "=") + + # --iidfile prints the built image ID to the specified file. This is + # used so we can refer to the image in later steps. + iidfile="$( mktemp )" + buildah bud \ + --arch="${arch}" \ + --iidfile="${iidfile}" \ + --build-arg=base_image="$base_image" + image_id="$( cat "${iidfile}" )" + rm "${iidfile}" + + # Extract various package info and version info to store as labels + container="$( buildah from "${image_id}" )" + run() { buildah run "${container}" "${@}" ; } + deb_list="$( run cat /.deb.lst | tr '\n' '|' | sed 's/|$//' )" + pkg_list="$( run cat /.pkg.lst | tr '\n' '|' | sed 's/|$//' )" + glibc="$( run sh -c 'exec "$( find -xdev -name libc.so.6 -print -quit )"' | sed '1!d' )" + bash="$( run bash --version | sed '1!d' )" + bioconda_utils="$( run sh -c '. /opt/conda/etc/profile.d/conda.sh && conda activate base && bioconda-utils --version' | rev | cut -f1 -d " " | rev )" + buildah rm "${container}" + + # Store package/version info as labels for the image + container="$( buildah from "${image_id}" )" + buildah config \ + --label=glibc="${glibc}" \ + --label=bash="${bash}" \ + --label=deb-list="${deb_list}" \ + --label=pkg-list="${pkg_list}" \ + --label=bioconda-utils="${bioconda_utils}" \ + "${container}" + + # Store the new image (now with labels) + image_id="$( buildah commit "${container}" )" + buildah rm "${container}" + + # image tag includes arch; then added to manifest which does not include arch + for tag in ${tags} ; do + buildah tag \ + "${image_id}" \ + "${image_name}:${tag}-${arch}" + buildah manifest add \ + "${image_name}:${tag}" \ + "${image_id}" + done + done + + - name: Test + run: | + image='${{ steps.build.outputs.image }}' + + # Extract image ids from manifest to test. + ids="$( + for tag in ${{ steps.build.outputs.tags }} ; do + buildah manifest inspect "${image}:${tag}" \ + | jq -r '.manifests[]|.digest' \ + | while read id ; do + buildah images --format '{{.ID}}{{.Digest}}' \ + | sed -n "s/${id}//p" + done + done + )" + + # See Dockerfile.test for actual tests run + ids="$( printf %s "${ids}" | sort -u )" + for id in ${ids} ; do + podman history "${id}" + buildah bud \ + --build-arg=base="${id}" \ + --file=Dockerfile.test \ + "images/${image}" + done + buildah rmi --prune || true + + - name: Check Tags + run: | + # FIX upstream: Quay.io does not support immutable images currently. + # => Try to use the REST API to check for duplicate tags and exit if they exist + response="$( + curl -sL \ + 'https://quay.io/api/v1/repository/bioconda/${{ steps.build.outputs.image }}/tag/' + )" + + existing_tags="$( + printf %s "${response}" \ + | jq -r '.tags[]|select(.end_ts == null or .end_ts >= now)|.name' + )" \ + || { + printf %s\\n \ + 'Could not get list of image tags.' \ + 'Does the repository exist on Quay.io?' \ + 'Quay.io REST API response was:' \ + "${response}" + exit 1 + } + for tag in ${{ steps.build.outputs.tags }} ; do + case "${tag}" in + latest | '${{ env.MAJOR_VERSION }}' ) ;; + * ) + if printf %s "${existing_tags}" | grep -qxF "${tag}" ; then + printf 'Tag %s already exists!\n' "${tag}" + exit 1 + fi + esac + done + + - if: ${{ github.ref == 'refs/heads/test-push' }} + name: Push + uses: redhat-actions/push-to-registry@v2 + with: + image: ${{ steps.build.outputs.image }} + tags: ${{ steps.build.outputs.tags }} + registry: ${{ secrets.QUAY_BIOCONDA_REPO }} + username: ${{ secrets.QUAY_BIOCONDA_USERNAME }} + password: ${{ secrets.QUAY_BIOCONDA_TOKEN }} + + - if: ${{ github.ref == 'refs/heads/test-push' }} + name: Test Pushed + run: | + image='${{ env.IMAGE_NAME }}' + ids="$( + for tag in ${{ steps.build.outputs.tags }} ; do + buildah manifest inspect "${image}:${tag}" \ + | jq -r '.manifests[]|.digest' \ + | while read id ; do + buildah images --format '{{.ID}}{{.Digest}}' \ + | sed -n "s/${id}//p" + done + done + )" + ids="$( printf %s "${ids}" | sort -u )" + for id in ${ids} ; do + podman history "${id}" + buildah bud \ + --build-arg=base="${id}" \ + --file=Dockerfile.test \ + "images/${image}" + done + buildah rmi --prune || true diff --git a/.github/workflows/build-env-image.yml b/.github/workflows/build-env-image.yml deleted file mode 100644 index 1bad2d18..00000000 --- a/.github/workflows/build-env-image.yml +++ /dev/null @@ -1,117 +0,0 @@ -name: Build image -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - pull_request: - paths: - - 'images/build-env/**' - - '.github/workflows/build-env-image.yml' - push: - branches: - - 'main' - -jobs: - build: - name: Build image - ${{ matrix.image }} - runs-on: ubuntu-22.04 - strategy: - matrix: - include: - - arch: arm64 - image: bioconda/bioconda-utils-build-env-cos7-aarch64 - base_image: quay.io/condaforge/linux-anvil-aarch64 - - arch: amd64 - image: bioconda/bioconda-utils-build-env-cos7-x86_64 - base_image: quay.io/condaforge/linux-anvil-cos7-x86_64 - steps: - - name: Checkout bioconda-containers - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Checkout bioconda-utils - uses: actions/checkout@v4 - with: - fetch-depth: 0 - repository: 'bioconda/bioconda-utils' - path: 'bioconda-utils' - - - id: get-tag - run: | - tag=${{ github.event.release && github.event.release.tag_name || github.sha }} - printf %s "tag=${tag#v}" >> $GITHUB_OUTPUT - - - name: Install qemu dependency - if: ${{ matrix.arch == 'arm64' }} - uses: docker/setup-qemu-action@v3 - with: - platforms: arm64 - - - name: Build image - id: buildah-build - uses: redhat-actions/buildah-build@v2 - with: - image: ${{ matrix.image }} - arch: ${{ matrix.arch }} - build-args: | - BASE_IMAGE=${{ matrix.base_image }} - tags: >- - latest - ${{ steps.get-tag.outputs.tag }} - dockerfiles: | - ./images/build-env/Dockerfile - - - name: Test built image - run: | - image='${{ steps.buildah-build.outputs.image }}' - for tag in ${{ steps.buildah-build.outputs.tags }} ; do - podman run --rm "${image}:${tag}" bioconda-utils --version - done - - - name: Push To Quay - if: github.ref == 'refs/heads/main' && github.repository == 'bioconda/bioconda-containers' - uses: redhat-actions/push-to-registry@v2 - with: - image: ${{ steps.buildah-build.outputs.image }} - tags: ${{ steps.buildah-build.outputs.tags }} - registry: ${{ secrets.QUAY_BIOCONDA_REPO }} - username: ${{ secrets.QUAY_BIOCONDA_USERNAME }} - password: ${{ secrets.QUAY_BIOCONDA_TOKEN }} - - build-manifest: - needs: [build] - if: github.ref == 'refs/heads/main' && github.repository == 'bioconda/bioconda-containers' - name: quay.io/bioconda/${{ matrix.cfg.DOCKER_MANIFEST }}:${{ matrix.cfg.DOCKER_TAG }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - cfg: - - DOCKER_MANIFEST: bioconda-utils-build-env-cos7 - DOCKER_TAG: "latest" - DOCKER_IMAGES: "quay.io/<>/bioconda-utils-build-env-cos7:<>,quay.io/<>/bioconda-utils-build-env-cos7-aarch64:<>" - - steps: - - uses: actions/checkout@v4 - - - name: Interpolate placeholders - id: interpolate - run: | - set -x - INTERPOLATED=`echo "${{ matrix.cfg.DOCKER_IMAGES }}" | sed "s#<>#${{ secrets.QUAY_BIOCONDA_USERNAME }}#g" | sed "s#<>#${{ matrix.cfg.DOCKER_TAG }}#g"` - echo "DOCKER_IMAGES=${INTERPOLATED}" >> "$GITHUB_OUTPUT" - - - name: Login to Quay.io registry - uses: docker/login-action@v2 - with: - registry: ${{ secrets.QUAY_BIOCONDA_REPO }} - username: ${{ secrets.QUAY_BIOCONDA_USERNAME }} - password: ${{ secrets.QUAY_BIOCONDA_TOKEN }} - - - name: Push Docker manifest list for quay.io/bioconda - uses: Noelware/docker-manifest-action@v0.3.0 - with: - inputs: quay.io/${{ secrets.QUAY_BIOCONDA_USERNAME }}/${{ matrix.cfg.DOCKER_MANIFEST }}:${{ matrix.cfg.DOCKER_TAG }} - images: ${{ steps.interpolate.outputs.DOCKER_IMAGES }} - push: true diff --git a/.github/workflows/create-env.yaml b/.github/workflows/create-env.yaml index 21162b41..811cb643 100644 --- a/.github/workflows/create-env.yaml +++ b/.github/workflows/create-env.yaml @@ -17,14 +17,14 @@ jobs: runs-on: ubuntu-22.04 env: MAJOR_VERSION: 3 - MINOR_VERSION: 0 + MINOR_VERSION: 1 IMAGE_NAME: create-env steps: - uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: arm64 diff --git a/images/build-env/Dockerfile b/images/bioconda-utils-build-env-cos7/Dockerfile similarity index 90% rename from images/build-env/Dockerfile rename to images/bioconda-utils-build-env-cos7/Dockerfile index 6573054b..f90b0a69 100644 --- a/images/build-env/Dockerfile +++ b/images/bioconda-utils-build-env-cos7/Dockerfile @@ -1,9 +1,6 @@ -# Specify the base image to support multi-arch images, such as -# - 'quay.io/condaforge/linux-anvil-aarch64' for Linux aarch64 -# - 'quay.io/condaforge/linux-anvil-cos7-x86_64' for Linux x86_64 -ARG BASE_IMAGE=quay.io/condaforge/linux-anvil-cos7-x86_64 +ARG base_image -FROM ${BASE_IMAGE} as base +FROM ${base_image} as base # Copy over C.UTF-8 locale from our base image to make it consistently available during build. COPY --from=quay.io/bioconda/base-glibc-busybox-bash /usr/lib/locale/C.utf8 /usr/lib/locale/C.utf8 diff --git a/images/bioconda-utils-build-env-cos7/Dockerfile.test b/images/bioconda-utils-build-env-cos7/Dockerfile.test new file mode 100644 index 00000000..1c9b9830 --- /dev/null +++ b/images/bioconda-utils-build-env-cos7/Dockerfile.test @@ -0,0 +1,6 @@ +ARG base +FROM "${base}" +RUN . /opt/conda/etc/profile.d/conda.sh \ + && conda activate base \ + && bioconda-utils --version \ + && bioconda-utils --help