venice/.github/workflows/VeniceCI-CompatibilityTests.yml at 6e69e74cc4dbfa28e66edd743f9ec4fd593b05fd · linkedin/venice · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
# GitHub Actions workflow for running compatibility tests: Avro and Pulsar Venice integration tests

name:  TestsDeCompatibilite

on: [push, pull_request, workflow_dispatch]

jobs:

  AvroCompatibilityTests:
    strategy:
      fail-fast: false
      matrix:
        jdk: [17]
    runs-on: ubuntu-latest
    permissions:
      contents: read
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
      - uses: ./.github/actions/gradle-test-setup
        with:
          java-version: ${{ matrix.jdk }}
      - name: Run Avro Compatibility Tests
        run: ./gradlew -DmaxParallelForks=2 --parallel --build-cache :internal:venice-avro-compatibility-test:test --continue
      - name: Package Build Artifacts
        if: success() || failure()
        shell: bash
        run: |
          mkdir ${{ github.job }}-artifacts
          find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list
          rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts
          tar -zcvf ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz ${{ github.job }}-artifacts
      - name: Upload Build Artifacts
        if: success() || failure()
        uses: actions/upload-artifact@v6
        with:
          name: ${{ github.job }}
          path: ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz
          retention-days: 30

  DuckVinciIntegrationTests:
    strategy:
      fail-fast: false
      matrix:
        jdk: [17]
    runs-on: ubuntu-latest
    permissions:
      contents: read
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
      - uses: ./.github/actions/gradle-test-setup
        with:
          java-version: ${{ matrix.jdk }}
      - name: Run DuckDB Integration Tests
        run: ./gradlew -DforkEvery=1 -DmaxParallelForks=1 --parallel --build-cache :integrations:venice-duckdb:integrationTest --continue
      - name: Package Build Artifacts
        if: success() || failure()
        shell: bash
        run: |
          mkdir ${{ github.job }}-artifacts
          find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list
          rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts
          tar -zcvf ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz ${{ github.job }}-artifacts
      - name: Upload Build Artifacts
        if: success() || failure()
        uses: actions/upload-artifact@v6
        with:
          name: ${{ github.job }}
          path: ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz
          retention-days: 30

  PulsarVeniceIntegrationTests:
    strategy:
      fail-fast: false
      matrix:
        jdk: [17]
    runs-on: ubuntu-latest
    permissions:
      contents: read
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
      - uses: ./.github/actions/gradle-test-setup
        with:
          java-version: ${{ matrix.jdk }}
      - name: Build NAR, shadow jars, and Docker images
        shell: bash
        run: |
          # Single Gradle invocation: NAR for Pulsar connector + shadow jars for Venice Docker images
          ./gradlew --parallel --build-cache --no-daemon \
            :integrations:venice-pulsar:nar \
            :services:venice-controller:shadowJar \
            :services:venice-server:shadowJar \
            :services:venice-router:shadowJar \
            :clients:venice-admin-tool:shadowJar \
            :clients:venice-thin-client:shadowJar \
            :clients:venice-push-job:shadowJar \
            :clients:venice-client:shadowJar

          # Build Pulsar and Venice Docker images in parallel
          docker build --tag=pulsar/venice-test:latest-dev -f tests/docker-images/pulsar-sink/Dockerfile . &
          pulsar_docker_pid=$!

          cd docker
          ./build-venice-docker-images.sh venicedb latest-dev "venice-controller venice-server venice-router venice-client"
          cd ..

          wait $pulsar_docker_pid || { echo "Pulsar Docker build failed"; exit 1; }

      - name: Run the test
        shell: bash
        run: |
          ./gradlew :tests:venice-pulsar-test:pulsarIntegrationTest -i

      - name: Package Build Artifacts
        if: success() || failure()
        shell: bash
        run: |
          mkdir ${{ github.job }}-artifacts
          find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list
          rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts
          tar -zcvf ${{ github.job }}-artifacts.tar.gz ${{ github.job }}-artifacts
      - name: Upload Build Artifacts
        if: success() || failure()
        uses: actions/upload-artifact@v6
        with:
          name: ${{ github.job }}
          path: ${{ github.job }}-artifacts.tar.gz
          retention-days: 30

  # This workflow gets skipped if all upstream jobs succeeded, but it runs (and fails) if any upstream failed.
  # This is what we want since GitHub required actions do not block merging when skipped, but can block it when failed.
  CompatibilityTestsFailureAlert:
    strategy:
      fail-fast: false
    runs-on: ubuntu-latest
    permissions:
      contents: read
    needs: [AvroCompatibilityTests, DuckVinciIntegrationTests, PulsarVeniceIntegrationTests]
    timeout-minutes: 30
    if: ${{ cancelled() || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure') }}
    steps:
      - name: NoGood
        shell: bash
        run: |
          echo "Some workflows have failed!"
          exit 1