From 684d684c5948b633c359bb12863f656df1e9d18d Mon Sep 17 00:00:00 2001 From: Matthew Sevey <15232757+MSevey@users.noreply.github.com> Date: Fri, 28 Jun 2024 11:07:38 -0400 Subject: [PATCH] feat: update the docker build workflow based on updates from celestia (#12) * feat: update the docker build workflow based on updates from celestia * ci: add basic docker test for docker workflow --- .../workflows/dockerfile_workflow_test.yaml | 33 +++ .../reusable_dockerfile_pipeline.yml | 263 +++++++++++++----- docker-action-test/Dockerfile | 20 ++ docker-action-test/go.mod | 3 + docker-action-test/main.go | 7 + 5 files changed, 258 insertions(+), 68 deletions(-) create mode 100644 .github/workflows/dockerfile_workflow_test.yaml create mode 100644 docker-action-test/Dockerfile create mode 100644 docker-action-test/go.mod create mode 100644 docker-action-test/main.go diff --git a/.github/workflows/dockerfile_workflow_test.yaml b/.github/workflows/dockerfile_workflow_test.yaml new file mode 100644 index 0000000..78c8d3b --- /dev/null +++ b/.github/workflows/dockerfile_workflow_test.yaml @@ -0,0 +1,33 @@ +# +# This workflow is used to test the `reusable_dockerfile_pipeline` action used +# to build and push the Docker image to the container registries. +# +# The reason this workflow targets the develop branch is so that we can test the +# action in the PR. If we targeted main, we would need to merge changes into main +# before being able to test them. +# +name: Build Using Reusable Workflow +on: [push, pull_request] +jobs: + # reusable-build tests calling the reusable_dockerfile_pipeline while + # providing a custom packageName + reusable-build: + permissions: + contents: write + packages: write + uses: rollkit/.github/.github/workflows/reusable_dockerfile_pipeline.yml@develop + with: + dockerfile: docker-action-test/Dockerfile + packageName: docker-test + secrets: inherit + + # reusable-build-defaults tests calling the reusable_dockerfile_pipeline with + # the defaults + reusable-build-defaults: + permissions: + contents: write + packages: write + uses: rollkit/.github/.github/workflows/reusable_dockerfile_pipeline.yml@develop + with: + dockerfile: docker-action-test/Dockerfile + secrets: inherit diff --git a/.github/workflows/reusable_dockerfile_pipeline.yml b/.github/workflows/reusable_dockerfile_pipeline.yml index d337585..12a223a 100644 --- a/.github/workflows/reusable_dockerfile_pipeline.yml +++ b/.github/workflows/reusable_dockerfile_pipeline.yml @@ -12,10 +12,14 @@ on: required: false type: string description: "You can specify a different package name." - default: "${{ github.repository }}" + dockerContext: + required: false + type: string + description: "The docker context" + default: "." env: - REGISTRY: ghcr.io + GITHUB_REG: ghcr.io MAINTAINER: ${{ github.repository_owner }} DESCRIPTION: "${{ github.repository_owner }} repository ${{ github.repository }}" @@ -23,50 +27,135 @@ jobs: prepare-env: runs-on: "ubuntu-latest" outputs: + repo_owner: ${{ steps.setting_env.outputs.repo_owner }} output_short_sha: ${{ steps.setting_env.outputs.short_sha }} output_image_name: ${{ steps.setting_env.outputs.image_name }} + build_for_pr: ${{ steps.setting_logic.outputs.build_for_pr }} + build_for_merge: ${{ steps.setting_logic.outputs.build_for_merge }} + not_a_fork: ${{ steps.setting_logic.outputs.not_a_fork }} steps: - name: Checkout - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: Add vars to ENV id: setting_env run: | + # Extract both the repository owner and repository name + # According to docs, github.repository is in the format of owner/repo, not just repo + # https://docs.github.com/en/actions/learn-github-actions/contexts#github-context + REPO_OWNER=$(echo "${{ github.repository }}" | awk -F'/' '{print $1}' | tr '[:upper:]' '[:lower:]') + REPO_NAME=$(echo "${{ github.repository }}" | awk -F'/' '{print $2}' | tr '[:upper:]' '[:lower:]') + + # Check repo name for .github use to test this workflow + if [[ $REPO_NAME == ".github" ]]; then + # Remove the leading . to avoid failing the character check + REPO_NAME="github" + fi + + # Log variables for debugging + echo "Repository Owner: $REPO_OWNER" + echo "Repository Name: $REPO_NAME" + echo "INPUT PACKAGE NAME: ${{ inputs.packageName }}" + + # Set environment variables and outputs + echo "REPO_OWNER=$REPO_OWNER" >> $GITHUB_ENV + echo "repo_owner=$REPO_OWNER" >> "$GITHUB_OUTPUT" echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_ENV echo "short_sha=`echo ${GITHUB_SHA} | cut -c1-8`" >> "$GITHUB_OUTPUT" - # yamllint disable - echo "IMAGE_NAME=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV - echo "image_name=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_OUTPUT" - # here we validate if we have specified a different package name in - # the inputs, if so, we change the package to it. - if [[ ${{ inputs.packageName }} != ${{ github.repository}} ]];then - # validate the input package name characters - if [[ ! "${{ inputs.packageName }}" =~ ^[A-Za-z0-9\-]+$ ]]; then - echo "------------------------------------------------------------" - echo "ERROR: Package name not valid! => [ ${{ inputs.packageName }} ]" - echo "ONLY can use: A-Za-z0-9\-" - echo "------------------------------------------------------------" - exit 1 - fi - echo "IMAGE_NAME=$(echo ${{ github.repository_owner }}/${{ inputs.packageName }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV - echo "image_name=$(echo ${{ github.repository_owner }}/${{ inputs.packageName }} | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_OUTPUT" + + if [[ "${{ inputs.packageName }}" == "${{ github.repository }}" ]]; then + # If a user submitted package name that has the naming containing + # both the repository owner and repository name, we fail + # e.g: inputs.packageName = "rollkit/rollkit" is not allowed, just submit rollkit + echo "------------------------------------------------------------" + echo "ERROR: Package name not valid! => [ ${{ inputs.packageName }} ]" + echo "Don't use the repository owner and repository name in the package name." + echo "------------------------------------------------------------" + exit 1 fi + + # Set the default package name to the repository name + PACKAGE_NAME=$REPO_NAME + + # If there is a user submitted package name, use it + if [[ -n "${{ inputs.packageName }}" ]]; then + PACKAGE_NAME=$(echo "${{ inputs.packageName }}" | tr '[:upper:]' '[:lower:]') + fi + + # validate the package name characters + if [[ ! $PACKAGE_NAME =~ ^[A-Za-z0-9\-]+$ ]]; then + echo "------------------------------------------------------------" + echo "ERROR: Package name not valid! => [ $PACKAGE_NAME ]" + echo "ONLY can use: A-Za-z0-9\-" + echo "------------------------------------------------------------" + exit 1 + fi + + # Log the package name for debugging + echo "PACKAGE_NAME: $PACKAGE_NAME" + + # Set environment variables and outputs + echo "IMAGE_NAME=$PACKAGE_NAME" >> $GITHUB_ENV + echo "image_name=$PACKAGE_NAME" >> "$GITHUB_OUTPUT" + + # The key logic that we want to determine is whether or not we are working + # on a fork and if this is a pull request or merge to main. + # + # We care about forks because of github's security policies that prevent + # forks from pushing images. So we only want to build images on forks + # + # The distinction between pull requests and merges to main is that on pull + # requests we want a single image available quickly for testing. On merges + # to main we want all the images built and are ok waiting longer to ensure + # there are not bugs. + - name: Add logic to ENV + id: setting_logic + run: | + # yamllint disable + echo "build_for_pr=${{ github.event_name == 'pull_request' }}" >> "$GITHUB_OUTPUT" + echo "build_for_merge=${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v') }}" >> "$GITHUB_OUTPUT" + echo "not_a_fork=${{ github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name }}" >> "$GITHUB_OUTPUT" # yamllint enable - docker-security: + # Log the key inputs to the logic as well a the outputs. We check that + # build_for_pr and build_for_merge are never equal when they are true as that + # would indicate a bug. If they are both false, this is ok, as this is the + # case on pushing commits to a PR. + logic-check: needs: prepare-env runs-on: "ubuntu-latest" + steps: + - name: Log logic + run: | + echo "github.event_name: ${{ github.event_name }}" + echo "github.ref: ${{ github.ref }}" + echo "head repo: ${{ github.event.pull_request.head.repo.full_name }}" + echo "base repo: ${{ github.event.pull_request.base.repo.full_name }}" + echo "build_for_pr: ${{ needs.prepare-env.outputs.build_for_pr }}" + echo "build_for_merge: ${{ needs.prepare-env.outputs.build_for_merge }}" + echo "not_a_fork: ${{ needs.prepare-env.outputs.not_a_fork }}" + - name: Check logic + if: | + (needs.prepare-env.outputs.build_for_pr == needs.prepare-env.outputs.build_for_merge) + && needs.prepare-env.outputs.build_for_pr != 'false' + run: | + echo "Failing step due to build_for_pr == build_for_merge" + exit 1 + + docker-security: + needs: ["prepare-env", "logic-check"] + runs-on: "ubuntu-latest" steps: - name: Checkout - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - - name: Build and Push - uses: docker/build-push-action@v4 + - name: Build + uses: docker/build-push-action@v5 env: OUTPUT_SHORT_SHA: ${{ needs.prepare-env.outputs.output_short_sha }} OUTPUT_IMAGE_NAME: ${{ needs.prepare-env.outputs.output_image_name }} with: - context: . + context: ${{ inputs.dockerContext}} push: false platforms: linux/amd64 # we're building the container before the scan, use the short sha tag @@ -91,37 +180,76 @@ jobs: severity: "CRITICAL,HIGH" docker-build: + name: docker-build (${{ matrix.registry.name }}; ${{ matrix.registry.registry-url }}/${{ matrix.registry.registry-owner }}/${{ needs.prepare-env.outputs.output_image_name }}) runs-on: "ubuntu-latest" # wait until the jobs are finished. - needs: ["prepare-env", "docker-security"] + needs: ["prepare-env", "logic-check", "docker-security"] + # We only want to run this step if one of the build flags is true. We don't + # run if both logic flags are false. This is the case for push events on PR + # commits. The logic-check job protects us from the case of both build flags + # being equal to true. + if: | + needs.prepare-env.outputs.build_for_pr == 'true' + || needs.prepare-env.outputs.build_for_merge == 'true' permissions: contents: write packages: write - + strategy: + matrix: + # run-on-pr is used to skip running registries that are expected to fail + # due to github permission issues with org wide secrets. + registry: + # - name: DockerHub + # user-secret: DOCKERHUB_USERNAME + # token-secret: DOCKERHUB_TOKEN + # registry-url: docker.io + # registry-owner: rollkitorg + # run-on-pr: "false" + - name: GHCR + user-secret: ${{ github.repository_owner }} + token-secret: GITHUB_TOKEN + registry-url: ghcr.io + registry-owner: ${{ needs.prepare-env.outputs.repo_owner }} + run-on-pr: "true" + # - name: ScaleWay + # user-secret: SCALEWAY_USERNAME + # token-secret: SCW_SECRET_KEY + # registry-url: rg.fr-par.scw.cloud + # registry-owner: rollkitorg + # run-on-pr: "false" + fail-fast: false steps: + - name: Check run conditions + id: run_check + # We only want to run when the registry is able to run on PR or if it is a merge event + run: echo "run=${{ matrix.registry.run-on-pr == needs.prepare-env.outputs.build_for_pr || needs.prepare-env.outputs.build_for_merge == 'true'}}" >> "$GITHUB_OUTPUT" + - name: Checkout - uses: "actions/checkout@v3" + if: ${{ steps.run_check.outputs.run == 'true'}} + uses: "actions/checkout@v4" - - name: Login to GHCR - uses: docker/login-action@v2 + - name: Login to ${{ matrix.registry.name }} + if: ${{ steps.run_check.outputs.run == 'true'}} + uses: docker/login-action@v3 with: - registry: ${{ env.REGISTRY }} - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} + registry: ${{ matrix.registry.registry-url }} + username: ${{ matrix.registry.registry-url == env.GITHUB_REG && matrix.registry.user-secret || secrets[matrix.registry.user-secret] }} + password: ${{ secrets[matrix.registry.token-secret] }} - name: Extract Docker Metadata + if: ${{ steps.run_check.outputs.run == 'true'}} id: meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 env: OUTPUT_SHORT_SHA: ${{ needs.prepare-env.outputs.output_short_sha }} OUTPUT_IMAGE_NAME: ${{ needs.prepare-env.outputs.output_image_name }} with: - images: ${{ env.REGISTRY }}/${{ env.OUTPUT_IMAGE_NAME }} + images: ${{ matrix.registry.registry-url }}/${{ matrix.registry.registry-owner }}/${{ env.OUTPUT_IMAGE_NAME }} # yamllint disable labels: | maintainer=${{ env.MAINTAINER }} commitUrl=https://github.com/${{ github.repository }}/commit/${{ github.sha }} - dockerPull=docker pull ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.OUTPUT_SHORT_SHA }} + dockerPull=docker pull ${{ matrix.registry.registry-url }}/${{ matrix.registry.registry-owner }}/${{ env.OUTPUT_IMAGE_NAME }}:${{ env.OUTPUT_SHORT_SHA }} org.opencontainers.image.description=${{ env.DESCRIPTION }} tags: | # output minimal (short sha) @@ -132,53 +260,52 @@ jobs: type=ref,enable=true,prefix=pr-,suffix=,event=pr # yamllint enable + - name: Set up QEMU + if: ${{ steps.run_check.outputs.run == 'true'}} + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + if: ${{ steps.run_check.outputs.run == 'true'}} + uses: docker/setup-buildx-action@v3 - # Build amd64 images always, and publish when it is not a fork. The Github - # security model prevents forks from pushing to the registry so we can - # only push if the branch/PR is not generated from a fork. Even though - # forks can't push, we still want to try and build the image to catch - # bugs. For testing purposes we only need an amd64 image. - - name: Build and Push Docker Image amd64 - uses: docker/build-push-action@v4 + # Build and Publish images on main, master, and versioned branches. + # + # The reason we split out these steps into 2 is for better handling of + # forks when building amd64 images and to enable faster availability of + # the amd64 image since building the arm64 image takes significantly + # longer. + - name: "Merge on Main Trigger: Build and Push All Docker Images" + if: ${{ needs.prepare-env.outputs.build_for_merge == 'true' && steps.run_check.outputs.run == 'true'}} + uses: docker/build-push-action@v5 env: OUTPUT_SHORT_SHA: ${{ needs.prepare-env.outputs.output_short_sha }} OUTPUT_IMAGE_NAME: ${{ needs.prepare-env.outputs.output_image_name }} with: - context: . - platforms: linux/amd64 - # Only push if the head and base repos match, meaning it is not a fork - # yamllint disable - push: ${{ github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name }} - # yamllint enable + context: ${{ inputs.dockerContext}} + platforms: linux/arm64,linux/amd64 + provenance: false + push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} file: ${{ inputs.dockerfile }} - # Build and Publish images on main, master, and versioned branches. - # - # NOTES: - # This step overrides the tag from the previous step. It will re-use - # the cached image that was built and only build the remaining images. - # - # The reason we split out these steps into 2 is for better handling of - # forks when building amd64 images and to enable faster availability of - # the amd64 image since building the arm64 image takes significantly - # longer. - - name: Build and Push Docker Images - uses: docker/build-push-action@v4 - # yamllint disable - # only run when the branch is main, master or starts with v* - if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v') }} - # yamllint enable + # Build amd64 images always, and publish when it is not a fork. The Github + # security model prevents forks from pushing to the registry so we can + # only push if the branch/PR is not generated from a fork. Even though + # forks can't push, we still want to try and build the image to catch + # bugs. For testing purposes we only need an amd64 image. + - name: "Pull Request Trigger: Build and Push amd64 Docker Image" + if: ${{ needs.prepare-env.outputs.build_for_pr == 'true' && steps.run_check.outputs.run == 'true'}} + uses: docker/build-push-action@v5 env: OUTPUT_SHORT_SHA: ${{ needs.prepare-env.outputs.output_short_sha }} OUTPUT_IMAGE_NAME: ${{ needs.prepare-env.outputs.output_image_name }} with: - context: . - platforms: linux/amd64,linux/arm64 - push: true + context: ${{ inputs.dockerContext}} + platforms: linux/amd64 + provenance: false + # Only push if the head and base repos match, meaning it is not a fork + push: ${{ needs.prepare-env.outputs.not_a_fork == 'true' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} file: ${{ inputs.dockerfile }} diff --git a/docker-action-test/Dockerfile b/docker-action-test/Dockerfile new file mode 100644 index 0000000..38c3768 --- /dev/null +++ b/docker-action-test/Dockerfile @@ -0,0 +1,20 @@ +FROM --platform=$BUILDPLATFORM docker.io/golang:1.21-alpine3.18 as builder + +ARG TARGETOS +ARG TARGETARCH + +ENV CGO_ENABLED=0 +ENV GO111MODULE=on + +ADD ./docker-action-test /app +WORKDIR /app + +RUN uname -a &&\ + CGO_ENABLED=${CGO_ENABLED} GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o main . +######## Start a new stage from scratch ####### +FROM docker.io/alpine:3.18.4 + +RUN apk update && apk add --no-cache bash curl jq +COPY --from=builder /app/main . +# Command to run the executable +CMD ["./main"] diff --git a/docker-action-test/go.mod b/docker-action-test/go.mod new file mode 100644 index 0000000..184f4ae --- /dev/null +++ b/docker-action-test/go.mod @@ -0,0 +1,3 @@ +module github.com/celestiaorg/.celestia-github/docker-action-test + +go 1.21 diff --git a/docker-action-test/main.go b/docker-action-test/main.go new file mode 100644 index 0000000..7534a75 --- /dev/null +++ b/docker-action-test/main.go @@ -0,0 +1,7 @@ +package main + +import "fmt" + +func main() { + fmt.Println("Docker CICD! ⚙️ ") +}