Error wrap 7.22 #1684
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Test | |
| # This workflow is responsible for building Pelican's container images and | |
| # running the test suite. Much of its length and complexity is a consequence | |
| # of trying to keep build times short by taking advantage of caching. | |
| # | |
| # The intended behavior is that all container images are always built. | |
| # However: | |
| # | |
| # - For pull requests: We never push images to a registry. In fact, | |
| # we can't, because the credentials aren't available. | |
| # | |
| # - Pushes to main: We push only pelican-test and pelican-dev. | |
| # | |
| # - Pushes to semvar tags: We push pelican-test and all the "server" images. | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| # Run only on release tags for v7.0.0 and up. | |
| - v[7-9]\.[0-9]+\.[0-9]+ | |
| - v[7-9]\.[0-9]+\.[0-9]+-rc\.[0-9]+ | |
| - v[1-9][0-9]+\.[0-9]+\.[0-9]+ | |
| - v[1-9][0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+ | |
| repository_dispatch: | |
| types: | |
| - dispatch-build | |
| workflow_dispatch: | |
| jobs: | |
| set-build-parameters: | |
| outputs: | |
| # | |
| # While some of these outputs are intended to be treated as booleans, | |
| # they are actually strings ('true', 'false'). In any conditionals, test | |
| # their values using "== 'true'" or "== 'false'". | |
| # | |
| platforms: ${{ steps.parameters.outputs.PLATFORMS }} | |
| push-dev: ${{ steps.parameters.outputs.PUSH_DEV }} | |
| push-server: ${{ steps.parameters.outputs.PUSH_SERVER }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Determine build parameters | |
| id: parameters | |
| run: | | |
| PUSH_DEV=${{ github.repository == 'PelicanPlatform/pelican' && github.ref == 'refs/heads/main' }} | |
| PUSH_SERVER=${{ github.repository == 'PelicanPlatform/pelican' && startsWith(github.ref, 'refs/tags/') }} | |
| if ${PUSH_DEV} || ${PUSH_SERVER}; then | |
| PLATFORMS=linux/amd64,linux/arm64 | |
| else | |
| # If we're not pushing images to a registry, then save resources. | |
| PLATFORMS=linux/amd64 | |
| fi | |
| echo "PLATFORMS=${PLATFORMS}" >> $GITHUB_OUTPUT | |
| echo "PUSH_DEV=${PUSH_DEV}" >> $GITHUB_OUTPUT | |
| echo "PUSH_SERVER=${PUSH_SERVER}" >> $GITHUB_OUTPUT | |
| set-tags: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| TIMESTAMP: ${{ steps.make_timestamp.outputs.TIMESTAMP }} | |
| IS_LATEST: ${{ steps.is_latest.outputs.IS_LATEST }} | |
| GITHUB_TAG: ${{ steps.determine_tags.outputs.GITHUB_TAG }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Make timestamp tag | |
| id: make_timestamp | |
| run: echo "TIMESTAMP=$(date +%Y%m%d-%H%M)" >> $GITHUB_OUTPUT | |
| - name: Determine whether to tag this build with "latest" | |
| id: is_latest | |
| run: | | |
| git fetch --tags | |
| tags=$(git tag -l 'v*.*.*' | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -V) | |
| highest_tag=$(echo "${tags}" | tail -n1) | |
| echo "Highest version tag is ${highest_tag}" | |
| if [[ "${GITHUB_REF##*/}" == "${highest_tag}" ]]; then | |
| echo "IS_LATEST=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "IS_LATEST=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Determine the tags that triggered this build | |
| id: determine_tags | |
| run: | | |
| # Check if we're working with a tagged version | |
| if [ -z "${{ inputs.tag }}" ] | |
| then | |
| # Use regex to check for a semver tag match. | |
| if [[ ${GITHUB_REF##*/} =~ v[0-9]+\.[0-9]+\.[0-9]+ ]] | |
| then | |
| GITHUB_TAG=${GITHUB_REF##*/} | |
| else | |
| GITHUB_TAG="latest-dev" | |
| fi | |
| else | |
| GITHUB_TAG=${{ inputs.tag }} | |
| fi | |
| echo "Master SHA:" | |
| echo $(git rev-parse $GITHUB_REF_NAME) | |
| echo "Current SHA:" | |
| echo $(git rev-parse HEAD) | |
| echo "Computed tags:" | |
| echo $GITHUB_TAG | |
| echo "GITHUB_TAG=$GITHUB_TAG" >> $GITHUB_OUTPUT | |
| cache-image-layers: | |
| needs: [set-build-parameters] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Free disk space | |
| shell: bash | |
| run: | | |
| # Use `df` to quickly check how much space is available. | |
| echo "::group::Before" | |
| df -h | |
| echo "::endgroup::" | |
| sudo rm -rf /usr/local/lib/android | |
| echo "::group::After" | |
| df -h | |
| echo "::endgroup::" | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: amd64,arm64 | |
| # NOTE (brianaydemir): Do not try to be clever and/or ambitious here. | |
| # Keep the cache small by restricting it to the current GitHub Action | |
| # run. We need to be able to save it for future jobs. | |
| - name: Create a cache for Docker Buildx | |
| uses: actions/cache@v4 | |
| with: | |
| path: /tmp/.base-buildx-cache | |
| key: base-buildx-${{ github.sha }}-${{ github.run_id }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| buildkitd-flags: --debug | |
| - name: Save the GitHub workspace | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: ${{ github.workspace }} | |
| key: github-workspace-${{ github.sha }}-${{ github.run_id }} | |
| - name: Create and cache image layers | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: ./images/Dockerfile | |
| platforms: ${{ needs.set-build-parameters.outputs.platforms }} | |
| target: origin | |
| cache-from: | | |
| type=registry,ref=hub.opensciencegrid.org/pelican_platform/pelican-dev:buildcache | |
| type=local,src=/tmp/.base-buildx-cache | |
| cache-to: type=local,dest=/tmp/.base-buildx-cache,mode=max | |
| build-server-images: | |
| needs: [set-build-parameters, cache-image-layers, set-tags] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| image: | |
| - origin | |
| - cache | |
| - director | |
| - registry | |
| - osdf-origin | |
| - osdf-cache | |
| - osdf-director | |
| - osdf-registry | |
| # For release candidates, this is where we build and push the testing container. | |
| - pelican-test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: amd64,arm64 | |
| - name: Generate tag list | |
| id: generate-tag-list | |
| env: | |
| GITHUB_TAG: ${{ needs.set-tags.outputs.GITHUB_TAG }} | |
| IS_LATEST: ${{ needs.set-tags.outputs.IS_LATEST }} | |
| run: | | |
| docker_repo="pelican_platform" | |
| image_name=${{ matrix.image }} | |
| tag_list=() | |
| for registry in hub.opensciencegrid.org; do | |
| for image_tag in "$GITHUB_TAG"; do | |
| tag_list+=("$registry/$docker_repo/$image_name":"$image_tag") | |
| done | |
| done | |
| if [[ "$IS_LATEST" == "true" ]]; then | |
| tag_list+=("$registry/$docker_repo/$image_name:latest") | |
| fi | |
| # This causes the tag_list array to be comma-separated below, | |
| # which is required for build-push-action | |
| IFS=, | |
| echo "taglist=${tag_list[*]}" >> $GITHUB_OUTPUT | |
| - name: Restore the cache for Docker Buildx | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: /tmp/.base-buildx-cache | |
| key: base-buildx-${{ github.sha }}-${{ github.run_id }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| buildkitd-flags: --debug | |
| - name: Log in to OSG Harbor | |
| uses: docker/login-action@v3 | |
| if: ${{ needs.set-build-parameters.outputs.push-server == 'true' }} | |
| with: | |
| registry: hub.opensciencegrid.org | |
| username: ${{ secrets.PELICAN_HARBOR_ROBOT_USER }} | |
| password: ${{ secrets.PELICAN_HARBOR_ROBOT_PASSWORD }} | |
| - name: Restore the GitHub workspace | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ${{ github.workspace }} | |
| key: github-workspace-${{ github.sha }}-${{ github.run_id }} | |
| # | |
| # NOTE (brianaydemir): If we've somehow managed to get the GitHub | |
| # workspace cache evicted, then there are likely serious problems | |
| # with how we're triggering and using GitHub actions. Make noise! | |
| # | |
| fail-on-cache-miss: true | |
| - name: Build and push Docker images | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: ./images/Dockerfile | |
| platforms: ${{ needs.set-build-parameters.outputs.platforms }} | |
| target: ${{ matrix.image }} | |
| push: ${{ needs.set-build-parameters.outputs.push-server == 'true' }} | |
| tags: "${{ steps.generate-tag-list.outputs.taglist }}" | |
| cache-from: | | |
| type=registry,ref=hub.opensciencegrid.org/pelican_platform/pelican-dev:buildcache | |
| type=local,src=/tmp/.base-buildx-cache | |
| build-devtest-images: | |
| needs: [set-build-parameters, cache-image-layers] | |
| runs-on: ubuntu-latest | |
| steps: | |
| # NOTE (brianaydemir): If it wasn't for the potential expense of | |
| # loading the Docker Buildx cache, we could implement this job using | |
| # a matrix and eliminate near-identical steps. | |
| - name: Determine image tags (pelican-test) | |
| id: pelican-test-tags | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: hub.opensciencegrid.org/pelican_platform/pelican-test | |
| tags: | | |
| type=raw,value=latest-itb | |
| type=raw,value=sha-{{sha}} | |
| - name: Determine image tags (pelican-dev) | |
| id: pelican-dev-tags | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: hub.opensciencegrid.org/pelican_platform/pelican-dev | |
| tags: | | |
| type=raw,value=latest-itb | |
| type=raw,value=sha-{{sha}} | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: amd64,arm64 | |
| - name: Restore the cache for Docker Buildx | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: /tmp/.base-buildx-cache | |
| key: base-buildx-${{ github.sha }}-${{ github.run_id }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| buildkitd-flags: --debug | |
| - name: Log in to OSG Harbor | |
| uses: docker/login-action@v3 | |
| if: ${{ needs.set-build-parameters.outputs.push-dev == 'true' }} | |
| with: | |
| registry: hub.opensciencegrid.org | |
| username: ${{ secrets.PELICAN_HARBOR_ROBOT_USER }} | |
| password: ${{ secrets.PELICAN_HARBOR_ROBOT_PASSWORD }} | |
| - name: Restore the GitHub workspace | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ${{ github.workspace }} | |
| key: github-workspace-${{ github.sha }}-${{ github.run_id }} | |
| # | |
| # NOTE (brianaydemir): If we've somehow managed to get the GitHub | |
| # workspace cache evicted, then there are likely serious problems | |
| # with how we're triggering and using GitHub actions. Make noise! | |
| # | |
| fail-on-cache-miss: true | |
| - name: Build and push Docker images (pelican-test) | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: ./images/Dockerfile | |
| platforms: ${{ needs.set-build-parameters.outputs.platforms }} | |
| target: pelican-test | |
| push: ${{ needs.set-build-parameters.outputs.push-dev == 'true' }} | |
| tags: ${{ steps.pelican-test-tags.outputs.tags }} | |
| cache-from: | | |
| type=registry,ref=hub.opensciencegrid.org/pelican_platform/pelican-dev:buildcache | |
| type=local,src=/tmp/.base-buildx-cache | |
| - name: Build and push Docker images (pelican-dev) | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: ./images/Dockerfile | |
| platforms: ${{ needs.set-build-parameters.outputs.platforms }} | |
| target: pelican-dev | |
| push: ${{ needs.set-build-parameters.outputs.push-dev == 'true' }} | |
| tags: ${{ steps.pelican-dev-tags.outputs.tags }} | |
| cache-from: | | |
| type=registry,ref=hub.opensciencegrid.org/pelican_platform/pelican-dev:buildcache | |
| type=local,src=/tmp/.base-buildx-cache | |
| cache-to: type=registry,ref=hub.opensciencegrid.org/pelican_platform/pelican-dev:buildcache,mode=max,image-manifest=true,oci-mediatypes=true,ignore-error=true | |
| # The macOS and Windows tests do not depend on any of the images being | |
| # built above. Thus, there is a separate workflow for running the tests: | |
| # test-macos-windows.yaml. | |
| # | |
| # The Linux tests are less straightforward because we need to decide | |
| # which container image to run the tests in, which might require waiting | |
| # for the image to be built. | |
| # | |
| # For a pull request, we cannot build and push a newer image. Thus, there | |
| # is a separate workflow for running the tests: test-linux-pr.yaml. | |
| test-linux-push-to-main: | |
| needs: [set-build-parameters, build-devtest-images] | |
| if: ${{ needs.set-build-parameters.outputs.push-dev == 'true' }} | |
| uses: ./.github/workflows/test-linux.yml | |
| with: | |
| image: hub.opensciencegrid.org/pelican_platform/pelican-test:latest-itb | |
| test-linux-push-to-tag: | |
| needs: [set-build-parameters, set-tags, build-server-images] | |
| if: ${{ needs.set-build-parameters.outputs.push-server == 'true' }} | |
| uses: ./.github/workflows/test-linux.yml | |
| with: | |
| image: hub.opensciencegrid.org/pelican_platform/pelican-test:${{ needs.set-tags.outputs.GITHUB_TAG }} |