From 44183203d5880e9dfe848fdea6d7c56a61079a3a Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 25 Jan 2023 06:15:32 -0800 Subject: [PATCH 1/4] Add extended regex -E flag to sed call --- dask_planner/update-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dask_planner/update-dependencies.sh b/dask_planner/update-dependencies.sh index a94c53dad..cc1f2177f 100644 --- a/dask_planner/update-dependencies.sh +++ b/dask_planner/update-dependencies.sh @@ -2,7 +2,7 @@ UPDATE_ALL_CARGO_DEPS="${UPDATE_ALL_CARGO_DEPS:-true}" # Update datafusion dependencies in the dask-planner to the latest revision from the default branch -sed -i -r 's/^datafusion-([a-z]+).*/datafusion-\1 = { git = "https:\/\/github.com\/apache\/arrow-datafusion\/" }/g' Cargo.toml +sed -i -r -E 's/^datafusion-([a-z]+).*/datafusion-\1 = { git = "https:\/\/github.com\/apache\/arrow-datafusion\/" }/g' Cargo.toml if [ "$UPDATE_ALL_CARGO_DEPS" = true ] ; then cargo update From 1c9812c67bd2e13777113127076f9488c395ff2f Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 25 Jan 2023 13:00:40 -0800 Subject: [PATCH 2/4] Update test to accept test-df-upstream trigger to test with upstream df as well --- .github/workflows/test.yml | 51 ++++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0dd086e9c..5d805ed59 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,24 +17,38 @@ defaults: shell: bash -l {0} jobs: - detect-ci-trigger: - name: Check for upstream trigger phrase + detect-upstream-trigger: + name: Check for upstream trigger phrase to test with upstream dask runs-on: ubuntu-latest if: github.repository == 'dask-contrib/dask-sql' outputs: - triggered: ${{ steps.detect-trigger.outputs.trigger-found }} + triggered: ${{ steps.detect-upstream-trigger.outputs.trigger-found }} steps: - uses: actions/checkout@v3 with: fetch-depth: 2 - uses: xarray-contrib/ci-trigger@v1.2 - id: detect-trigger + id: detect-upstream-trigger with: keyword: "[test-upstream]" + detect-datafusion-upstream-trigger: + name: Check for upstream trigger phrase to test with upstream datafusion + runs-on: ubuntu-latest + if: github.repository == 'dask-contrib/dask-sql' + outputs: + triggered: ${{ steps.detect-datafusion-upstream-trigger.outputs.trigger-found }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + - uses: xarray-contrib/ci-trigger@v1.2 + id: detect-datafusion-upstream-trigger + with: + keyword: "[test-df-upstream]" test: name: "Build & Test (${{ matrix.os }}, python: ${{ matrix.python }})" - needs: [detect-ci-trigger] + needs: [detect-upstream-trigger, detect-datafusion-upstream-trigger] runs-on: ${{ matrix.os }} env: CONDA_FILE: continuous_integration/environment-${{ matrix.python }}-dev.yaml @@ -54,6 +68,11 @@ jobs: channel-priority: strict activate-environment: dask-sql environment-file: ${{ env.CONDA_FILE }} + - name: Optionally update upstream datafusion & cargo dependencies + if: needs.detect-datafusion-upstream-trigger.outputs.triggered == 'true' + run: | + cd dask_planner + bash update-dependencies.sh - name: Build the Rust DataFusion bindings run: | python setup.py build install @@ -64,7 +83,7 @@ jobs: docker pull bde2020/hive:2.3.2-postgresql-metastore docker pull bde2020/hive-metastore-postgresql:2.3.0 - name: Optionally install upstream dev Dask - if: needs.detect-ci-trigger.outputs.triggered == 'true' + if: needs.detect-upstream-trigger.outputs.triggered == 'true' run: | mamba install --no-channel-priority dask/label/dev::dask - name: Test with pytest @@ -82,7 +101,7 @@ jobs: cluster: name: "Test in a dask cluster" - needs: [detect-ci-trigger] + needs: [detect-upstream-trigger, detect-datafusion-upstream-trigger] runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -95,6 +114,11 @@ jobs: channel-priority: strict activate-environment: dask-sql environment-file: continuous_integration/environment-3.9-dev.yaml + - name: Optionally update upstream datafusion & cargo dependencies + if: needs.detect-datafusion-upstream-trigger.outputs.triggered == 'true' + run: | + cd dask_planner + bash update-dependencies.sh - name: Build the Rust DataFusion bindings run: | python setup.py build install @@ -106,12 +130,12 @@ jobs: pip list mamba list - name: Optionally install upstream dev Dask - if: needs.detect-ci-trigger.outputs.triggered == 'true' + if: needs.detect-upstream-trigger.outputs.triggered == 'true' run: | mamba install --no-channel-priority dask/label/dev::dask - name: run a dask cluster env: - UPSTREAM: ${{ needs.detect-ci-trigger.outputs.triggered }} + UPSTREAM: ${{ needs.detect-upstream-trigger.outputs.triggered }} run: | if [[ $UPSTREAM == "true" ]]; then docker-compose -f .github/cluster-upstream.yml up -d @@ -130,7 +154,7 @@ jobs: import: name: "Test importing with bare requirements" - needs: [detect-ci-trigger] + needs: [detect-upstream-trigger, detect-datafusion-upstream-trigger] runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -141,6 +165,11 @@ jobs: use-mamba: true python-version: "3.8" channel-priority: strict + - name: Optionally update upstream datafusion & cargo dependencies + if: needs.detect-datafusion-upstream-trigger.outputs.triggered == 'true' + run: | + cd dask_planner + bash update-dependencies.sh - name: Install dependencies and nothing else run: | mamba install setuptools-rust @@ -150,7 +179,7 @@ jobs: pip list mamba list - name: Optionally install upstream dev Dask - if: needs.detect-ci-trigger.outputs.triggered == 'true' + if: needs.detect-upstream-trigger.outputs.triggered == 'true' run: | python -m pip install --no-deps git+https://github.com/dask/dask python -m pip install --no-deps git+https://github.com/dask/distributed From 470fc916fef9ca6abe74f60545bef6a86a160bc1 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 25 Jan 2023 13:08:24 -0800 Subject: [PATCH 3/4] [test-df-upstream] From 7bd4eac9baeab7133e55f4be7a460573d7d5fcfc Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 25 Jan 2023 13:52:32 -0800 Subject: [PATCH 4/4] rerun tests