build-and-deploy.yml

on:
  push:
    branches:
      - master
      - staging
      - release
  pull_request:
  repository_dispatch:
    types: build-and-deploy
  workflow_dispatch:
  workflow_call:

concurrency:
  group: cli-${{ github.workflow }}-${{ github.ref_type }}-${{ github.event.pull_request.number || github.ref || github.run_id }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash -euxo pipefail {0}

env:
  VERBOSE: 1

jobs:
  build-and-deploy-datasets:
    runs-on: ubuntu-20.04

    environment:
      name: ${{ github.ref }}

    env:
      VERBOSE: 0
      PYTHONUNBUFFERED: 1
      GH_TOKEN: ${{ github.token }}

    steps:
      - name: "Checkout code"
        uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Number of commits to fetch. 0 indicates all history for all branches and tags.
          fetch-tags: true # Whether to fetch tags, even if fetch-depth > 0.
          ref: ${{ github.event.pull_request.head.ref || github.ref }}

      - name: "Install system dependencies"
        run: |
          sudo apt-get install brotli pigz parallel python3 rename --yes -qq >/dev/null

      - name: "Install Python dependencies"
        run: |
          pip3 install -r requirements.txt

      - name: "DEBUG"
        run: |
          echo ${{ github.event.pull_request.head.repo.full_name }}
          echo ${{ github.repository }}
          echo ${{ github.ref }}

      - name: "Rebuild datasets (fork)"
        if: ${{ github.event.pull_request.head.repo.full_name != github.repository }}
        run: |
          ./scripts/rebuild --input-dir 'data/' --output-dir 'data_output/' --no-pull

      - name: "Rebuild, commit and push datasets"
        if: ${{ github.event.pull_request.head.repo.full_name == github.repository && github.ref != 'refs/heads/release' }}
        run: |
          git config --global user.email "${{ secrets.BOT_GIT_USER_EMAIL }}"
          git config --global user.name "${{ secrets.BOT_GIT_USER_NAME }}"

          ./scripts/rebuild --input-dir 'data/' --output-dir 'data_output/' --push --repo="${GITHUB_REPOSITORY}"

      - name: "Rebuild, commit, push and make a release"
        if: ${{ github.event.pull_request.head.repo.full_name == github.repository && github.ref == 'refs/heads/release' }}
        run: |
          git config --global user.email "${{ secrets.BOT_GIT_USER_EMAIL }}"
          git config --global user.name "${{ secrets.BOT_GIT_USER_NAME }}"

          ./scripts/rebuild --input-dir 'data/' --output-dir 'data_output/' --release --repo="${GITHUB_REPOSITORY}"

      - name: "Install awscli"
        if: ${{ github.event.pull_request.head.repo.full_name == github.repository && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/release') }}
        run: |
          pushd /tmp >/dev/null
            curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
            unzip -oqq awscliv2.zip
            sudo ./aws/install --update
          popd >/dev/null
          aws --version

      - name: "Deploy dataset server"
        if: ${{ github.event.pull_request.head.repo.full_name == github.repository && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/release') }}
        env:
          DATA_AWS_S3_BUCKET: ${{ secrets.DATA_AWS_S3_BUCKET }}
          DATA_AWS_CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.DATA_AWS_CLOUDFRONT_DISTRIBUTION_ID }}
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: us-east-2
        run: |
          ./scripts/upload 'data_output/'

      - name: "Upload build artifacts: dataset server"
        uses: actions/upload-artifact@v4
        with:
          name: server
          path: ./data_output/*

      - name: "Upload build artifacts: zip archives"
        uses: actions/upload-artifact@v4
        with:
          name: zips
          path: ./data_temp/*