main.yml

# Copyright (C) 2021 The Authors of projectnessie
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Nessie / Iceberg

on:
  push:
    branches: [ main ]
  pull_request:
  schedule:
    # Run every 12 hours on week days
    - cron:  '0 4/12 * * 1-5'

jobs:
  latest:
    name: Nessie main/Iceberg main
    runs-on: ubuntu-latest
    env:
      GIT_FETCH_DEPTH: 0
      GIT_FETCH_OPTION: ''
      NESSIE_DIR: included-builds/nessie
      NESSIE_MAIN_REPOSITORY: projectnessie/nessie
      NESSIE_MAIN_BRANCH: main
      NESSIE_PATCH_REPOSITORY: ''
      NESSIE_PATCH_BRANCH: ''
      ICEBERG_DIR: included-builds/iceberg
      ICEBERG_MAIN_REPOSITORY: apache/iceberg
      ICEBERG_MAIN_BRANCH: main
      ICEBERG_PATCH_REPOSITORY: snazy/iceberg
      ICEBERG_PATCH_BRANCH: iceberg-nesqueit
      SPARK_LOCAL_IP: localhost

    steps:
      - name: Prepare Git
        run: |
          git config --global user.email "integrations-testing@projectnessie.org"
          git config --global user.name "Integrations Testing [Bot]"

      - name: Checkout Integrations Testing repo
        uses: actions/checkout@v4

      - name: Free disk space
        uses: ./.github/actions/free-disk-space

      - name: Setup runner
        uses: ./.github/actions/setup-runner
        with:
          more-memory: 'true'

      - name: Checkout and patch Nessie
        uses: ./.github/actions/patch-git
        with:
          name: Nessie
          local-dir: ${{env.NESSIE_DIR}}
          main-repository: ${{env.NESSIE_MAIN_REPOSITORY}}
          main-branch: ${{env.NESSIE_MAIN_BRANCH}}
          patch-repository: ${{env.NESSIE_PATCH_REPOSITORY}}
          patch-branch: ${{env.NESSIE_PATCH_BRANCH}}
          work-branch: nessie_integtest

      - name: Checkout and patch Iceberg
        uses: ./.github/actions/patch-git
        with:
          name: Iceberg
          local-dir: ${{env.ICEBERG_DIR}}
          main-repository: ${{env.ICEBERG_MAIN_REPOSITORY}}
          main-branch: ${{env.ICEBERG_MAIN_BRANCH}}
          patch-repository: ${{env.ICEBERG_PATCH_REPOSITORY}}
          patch-branch: ${{env.ICEBERG_PATCH_BRANCH}}
          work-branch: iceberg_integtest

      # Setup Gradle properties, heap requirements are for the "Integration test w/ Nessie".
      - name: Setup gradle.properties
        run: |
          mkdir -p ~/.gradle
          echo "org.gradle.jvmargs=-Xms2g -Xmx4g -XX:MaxMetaspaceSize=768m -Dfile.encoding=UTF-8" >> ~/.gradle/gradle.properties
          echo "org.gradle.vfs.watch=false" >> ~/.gradle/gradle.properties

      - name: Set up JDK
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          # Java 17 required for Nessie build, Java 11 required for Flink & Presto
          java-version: |
            11
            17

      - name: Spotless
        uses: gradle/gradle-build-action@v2
        with:
          arguments: spotlessCheck --scan

      - name: Checkstyle
        uses: gradle/gradle-build-action@v2
        with:
          arguments: checkstyleMain checkstyleTest --scan

      - name: Iceberg Nessie test
        uses: gradle/gradle-build-action@v2
        with:
          arguments: :iceberg:iceberg-nessie:test --scan

      - name: Nessie Spark 3.2 / 2.12 Extensions test
        uses: gradle/gradle-build-action@v2
        with:
          arguments: :nessie:nessie-iceberg:nessie-spark-extensions-3.2_2.12:test :nessie:nessie-iceberg:nessie-spark-extensions-3.2_2.12:intTest --scan

      - name: Nessie Spark 3.3 / 2.12 Extensions test
        uses: gradle/gradle-build-action@v2
        with:
          arguments: :nessie:nessie-iceberg:nessie-spark-extensions-3.3_2.12:test :nessie:nessie-iceberg:nessie-spark-extensions-3.3_2.12:intTest --scan

      - name: Nessie Spark 3.4 / 2.12 Extensions test
        uses: gradle/gradle-build-action@v2
        with:
          arguments: :nessie:nessie-iceberg:nessie-spark-extensions-3.4_2.12:test :nessie:nessie-iceberg:nessie-spark-extensions-3.4_2.12:intTest --scan

      - name: Nessie Spark 3.5 / 2.13 Extensions test
        uses: gradle/gradle-build-action@v2
        with:
          arguments: :nessie:nessie-iceberg:nessie-spark-extensions-3.5_2.13:test :nessie:nessie-iceberg:nessie-spark-extensions-3.5_2.13:intTest --scan

      - name: Build before publish
        uses: gradle/gradle-build-action@v2
        with:
          arguments: jar testClasses javadoc --scan

      # TODO re-enable once Iceberg/main references a Nessie release w/ Nessie PR #6197,
      #   Gradle's Maven publishing fails with the group-ID relocation.
      #
      #- name: Publish Nessie + Iceberg to local Maven repo
      #  uses: gradle/gradle-build-action@v2
      #  with:
      #    arguments: publishLocal --scan

      - name: Gather locally published versions
        run: |
          NESSIE_VERSION="$(cat included-builds/nessie/version.txt)"
          ICEBERG_VERSION="$(cat included-builds/iceberg/build/iceberg-build.properties | grep '^git.build.version=' | cut -d= -f2)"
          echo "NESSIE_VERSION=${NESSIE_VERSION}" >> ${GITHUB_ENV}
          echo "ICEBERG_VERSION=${ICEBERG_VERSION}" >> ${GITHUB_ENV}
          cat <<! >> $GITHUB_STEP_SUMMARY
          ## Published versions
          | Published Nessie version | Published Iceberg version |
          | ------------------------ | ------------------------- |
          | ${NESSIE_VERSION}        | ${ICEBERG_VERSION}        |
          !

      # Iceberg does not support builds with more than one Scala versions, so we have to explicitly
      # Specify the Scala version here - and run `intTest` for each Scala version.
      # But note: Flink does not provide any Scala 2.13 artifacts at the moment, so when selecting
      # Scala 2.13 with a build using Iceberg from source, all Flink related code will be skipped.

      - name: Show Gradle projects for Scala 2.12
        uses: gradle/gradle-build-action@v2
        with:
          arguments: projects -DscalaVersion=2.12

      - name: Tools & Integrations tests / Scala 2.12
        uses: gradle/gradle-build-action@v2
        with:
          arguments: intTest -DscalaVersion=2.12 --scan

      - name: Show Gradle projects for Scala 2.13
        uses: gradle/gradle-build-action@v2
        with:
          arguments: projects -DscalaVersion=2.13

      - name: Tools & Integrations tests / Scala 2.13
        uses: gradle/gradle-build-action@v2
        with:
          arguments: intTest -DscalaVersion=2.13 --scan

      #- name: Checkout Presto repo
      #  uses: actions/checkout@v3
      #  with:
      #    repository: ${{env.PRESTO_MAIN_REPOSITORY}}
      #    ref: ${{env.PRESTO_MAIN_BRANCH}}
      #    path: ${{env.PRESTO_DIR}}
      #
      #- name: Cache local Maven repository
      #  uses: actions/cache@v3
      #  with:
      #    path: |
      #      ~/.m2/repository
      #      !~/.m2/repository/org/projectnessie
      #      !~/.m2/repository/org/apache/iceberg
      #      ~/.m2/wrapper
      #    key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml', '**/maven-wrapper.properties') }}
      #    restore-keys: |
      #      ${{ runner.os }}-maven-
      #
      #- name: Build Presto w/ latest Nessie+Iceberg
      #  working-directory: included-builds/presto
      #  run: |
      #    ./mvnw \
      #      --no-transfer-progress \
      #      install -DskipTests \
      #      -Ddep.iceberg.version=${ICEBERG_VERSION} \
      #      -Ddep.nessie.version=${NESSIE_VERSION}

  iceberg140_nessie0710:
    name: Nessie 0.71.0 / Iceberg 1.4.0
    runs-on: ubuntu-latest
    env:
      SPARK_LOCAL_IP: localhost

    steps:
      - name: Checkout Integrations Testing repo
        uses: actions/checkout@v4

      - name: Free disk space
        uses: ./.github/actions/free-disk-space

      - name: Setup gradle.properties
        run: |
          mkdir -p ~/.gradle
          echo "org.gradle.jvmargs=-Xmx1024m -XX:MaxMetaspaceSize=768m -Dfile.encoding=UTF-8" >> ~/.gradle/gradle.properties
          echo "org.gradle.vfs.watch=false" >> ~/.gradle/gradle.properties

      - name: Set up JDK
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: 11

      - name: Show Gradle projects
        uses: gradle/gradle-build-action@v2
        with:
          cache-read-only: true
          arguments: projects -Dnessie.versionNessie=0.71.0 -Dnessie.versionIceberg=1.4.0

      - name: Tools & Integrations tests
        uses: gradle/gradle-build-action@v2
        with:
          arguments: intTest -Dnessie.versionNessie=0.71.0 -Dnessie.versionIceberg=1.4.0 --scan

  iceberg131_nessie0651:
    name: Nessie 0.65.1 / Iceberg 1.3.1
    runs-on: ubuntu-latest
    env:
      SPARK_LOCAL_IP: localhost

    steps:
      - name: Checkout Integrations Testing repo
        uses: actions/checkout@v4

      - name: Free disk space
        uses: ./.github/actions/free-disk-space

      - name: Setup gradle.properties
        run: |
          mkdir -p ~/.gradle
          echo "org.gradle.jvmargs=-Xmx1024m -XX:MaxMetaspaceSize=768m -Dfile.encoding=UTF-8" >> ~/.gradle/gradle.properties
          echo "org.gradle.vfs.watch=false" >> ~/.gradle/gradle.properties

      - name: Set up JDK
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: 11

      - name: Show Gradle projects
        uses: gradle/gradle-build-action@v2
        with:
          cache-read-only: true
          arguments: projects -Dnessie.versionNessie=0.65.1 -Dnessie.versionIceberg=1.3.1

      - name: Tools & Integrations tests
        uses: gradle/gradle-build-action@v2
        with:
          arguments: intTest -Dnessie.versionNessie=0.65.1 -Dnessie.versionIceberg=1.3.1 --scan

  iceberg131_nessie0601:
    name: Nessie 0.60.1 / Iceberg 1.3.1
    runs-on: ubuntu-latest
    env:
      SPARK_LOCAL_IP: localhost

    steps:
      - name: Checkout Integrations Testing repo
        uses: actions/checkout@v4

      - name: Free disk space
        uses: ./.github/actions/free-disk-space

      - name: Setup gradle.properties
        run: |
          mkdir -p ~/.gradle
          echo "org.gradle.jvmargs=-Xmx1024m -XX:MaxMetaspaceSize=768m -Dfile.encoding=UTF-8" >> ~/.gradle/gradle.properties
          echo "org.gradle.vfs.watch=false" >> ~/.gradle/gradle.properties

      - name: Set up JDK
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: 11

      - name: Show Gradle projects
        uses: gradle/gradle-build-action@v2
        with:
          cache-read-only: true
          arguments: projects -Dnessie.versionNessie=0.60.1 -Dnessie.versionIceberg=1.3.1

      - name: Tools & Integrations tests
        uses: gradle/gradle-build-action@v2
        with:
          arguments: intTest -Dnessie.versionNessie=0.60.1 -Dnessie.versionIceberg=1.3.1 --scan