From e417ecb076ca02c8b0a7ebb83558c3246eac9fc7 Mon Sep 17 00:00:00 2001 From: Nikita Belonogov Date: Wed, 31 Jan 2024 23:54:27 +0700 Subject: [PATCH] ci: PLATE-837: Change Docker build to common approach --- .github/workflows/build.yml | 67 ------------ .github/workflows/cicd_pipeline.yml | 151 ++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 67 deletions(-) delete mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/cicd_pipeline.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index ef814bea2..000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: "Build and Push" - -on: - push: - branches: - - master - pull_request: - -jobs: - build-image: - name: Build image - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - matrix: - # Allow list of models - include: - - backend_dir_name: segment_anything_model - backend_tag_name: sam-v0 - - backend_dir_name: llm_interactive - backend_tag_name: llm-v8 - - backend_dir_name: the_simplest_backend - backend_tag_name: simplebackend-v0 - env: - IMAGE_NAME: heartexlabs/label-studio-ml-backend - examples_dir: label_studio_ml/examples - backend_dir_name: ${{ matrix.backend_dir_name }} - backend_tag_name: ${{ matrix.backend_tag_name }} - steps: - - uses: actions/checkout@v4 - with: - ref: "${{ env.GITHUB_SHA }}" - fetch-depth: 0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 - - - name: Check for Changes in Directory - id: check_changes - shell: bash - run: | - changes=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }} -- "${{ env.examples_dir }}/${{ env.backend_dir_name }}" || true) - if [ -z "$changes" ]; then - echo "No changes in directory ${{ matrix.dir }}. Exiting job." - echo "skip=true" >> $GITHUB_OUTPUT - else - echo "Changes detected in directory ${{ matrix.dir }}." - echo "skip=false" >> $GITHUB_OUTPUT - fi - - - name: Login to DockerHub - if: ${{ steps.check_changes.outputs.skip != 'true' && !github.event.pull_request.head.repo.fork }} - uses: docker/login-action@v3.0.0 - with: - username: ${{ vars.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Push Docker image - if: steps.check_changes.outputs.skip != 'true' - uses: docker/build-push-action@v5.1.0 - id: docker_build_and_push - with: - context: ${{ env.examples_dir }}/${{ env.backend_dir_name }} - push: true - tags: ${{ env.IMAGE_NAME }}:${{ env.backend_tag_name }} - cache-from: type=gha - cache-to: type=gha,mode=max diff --git a/.github/workflows/cicd_pipeline.yml b/.github/workflows/cicd_pipeline.yml new file mode 100644 index 000000000..165fd0699 --- /dev/null +++ b/.github/workflows/cicd_pipeline.yml @@ -0,0 +1,151 @@ +name: "CI/CD Pipeline" + +on: + push: + branches: + - master + pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + +jobs: + changed_files: + name: "Changed files" + runs-on: ubuntu-latest + outputs: + changes: ${{ steps.changes.outputs.changes }} + pretty_branch_name: ${{ steps.version.outputs.pretty_branch_name }} + image_version: ${{ steps.version.outputs.image_version }} + steps: + - uses: hmarr/debug-action@v2.1.0 + + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + + - uses: dorny/paths-filter@v2 + id: changes + with: + filters: | + segment_anything_model: + - 'label_studio_ml/examples/segment_anything_model/**' + - '.github/workflows/cicd_pipeline.yml' + llm_interactive: + - 'label_studio_ml/examples/llm_interactive/**' + - '.github/workflows/cicd_pipeline.yml' + the_simplest_backend: + - 'label_studio_ml/examples/the_simplest_backend/**' + - '.github/workflows/cicd_pipeline.yml' + + - name: Calculate version + id: version + env: + BRANCH_NAME: ${{ github.event.pull_request.head.ref || github.ref_name }} + run: | + set -x + pretty_branch_name="$(echo -n "${BRANCH_NAME#refs/heads/}" | sed 's#/#-#g' | sed 's#_#-#g'| sed 's#\.#-#g' | tr '[:upper:]' '[:lower:]')" + echo "pretty_branch_name=$pretty_branch_name" >> $GITHUB_OUTPUT + current_time="$(date +'%Y%m%d.%H%M%S')" + branch="-${pretty_branch_name}" + short_sha="$(git rev-parse --short HEAD)" + long_sha="$(git rev-parse HEAD)" + echo "sha=$long_sha" >> $GITHUB_OUTPUT + short_sha_length="$(echo $short_sha | awk '{print length}')" + current_time_length="$(echo $current_time | awk '{print length}')" + version="${current_time}$(echo $branch | cut -c1-$((50 - short_sha_length - current_time_length)))-${short_sha}" + echo "image_version=$version" >> $GITHUB_OUTPUT + + + build: + name: "Build" + runs-on: ubuntu-latest + needs: + - changed_files + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + image: ${{ fromJSON(needs.changed_files.outputs.changes) }} + env: + examples_dir: "label_studio_ml/examples" + steps: + - uses: hmarr/debug-action@v2.1.0 + + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + + - name: Login to DockerHub + uses: docker/login-action@v3.0.0 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Calculate Docker Context + id: calculate-docker-context + uses: actions/github-script@v7 + env: + NAME: "${{ matrix.image }}" + IMAGE_NAME: "${{ vars.DOCKERHUB_ORG }}/label-studio-ml-backend" + TAGS: "${{ needs.changed_files.outputs.image_version }},${{ needs.changed_files.outputs.pretty_branch_name }}" + TAG_PREFIX: "${{ matrix.image }}-" + CONTEXT: "${{ env.examples_dir }}/${{ matrix.image }}" + FILE: "${{ env.examples_dir }}/${{ matrix.image }}/Dockerfile" + with: + script: | + const name = process.env.NAME; + const raw_tags_input = process.env.TAGS; + const image_name = process.env.IMAGE_NAME; + let properties = { + "context": process.env.CONTEXT, + "file": process.env.FILE, + "tag_prefix": process.env.TAG_PREFIX, + }; + const overrides = { + "segment_anything_model": { + "tag_prefix": "sam-", + }, + "llm_interactive": { + "tag_prefix": "llm-", + }, + "the_simplest_backend": { + "tag_prefix": "simplebackend-", + }, + } + const override = overrides[name] || {}; + properties = { ...properties, ...override }; + + const tags = raw_tags_input + .split(',') + .map(x => x.trim()) + .map(x => `${image_name}:${properties.tag_prefix}${x}`) + .join(','); + core.notice(`tags='${tags}'`) + core.setOutput("tags", tags); + + core.setOutput("context", properties.context); + core.setOutput("file", properties.file); + + + - name: Push Docker image + uses: docker/build-push-action@v5.1.0 + with: + context: "${{ steps.calculate-docker-context.outputs.context }}" + file: "${{ steps.calculate-docker-context.outputs.file }}" + push: true + tags: "${{ steps.calculate-docker-context.outputs.tags }}" + cache-from: type=gha + cache-to: type=gha,mode=max