Skip to content

Commit eda01e6

Browse files
samansminksamansmink
committed
fix ci
1 parent a5ebaf4 commit eda01e6

File tree

8 files changed

+177
-169
lines changed

8 files changed

+177
-169
lines changed

‎.github/workflows/Linux.yml‎

Lines changed: 0 additions & 84 deletions
This file was deleted.

‎.github/workflows/MacOS.yml‎

Lines changed: 0 additions & 62 deletions
This file was deleted.

‎.github/workflows/MainDistributionPipeline.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#
2+
# This workflow calls the main distribution pipeline from DuckDB to build, test and (optionally) release the extension
3+
#
4+
name: Main Extension Distribution Pipeline
5+
on:
6+
push:
7+
pull_request:
8+
workflow_dispatch:
9+
10+
concurrency:
11+
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
12+
cancel-in-progress: true
13+
14+
jobs:
15+
duckdb-stable-build:
16+
name: Build extension binaries
17+
uses: duckdb/duckdb/.github/workflows/_extension_distribution.yml@5b391512a8fc7800a0c22d98714e6d8f25c002cf
18+
with:
19+
extension_name: arrow
20+
exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64;windows_amd64_rtools'
21+
duckdb_version: v0.9.2
22+
23+
duckdb-stable-deploy:
24+
name: Deploy extension binaries
25+
needs: duckdb-stable-build
26+
uses: ./.github/workflows/_extension_deploy.yml
27+
secrets: inherit
28+
with:
29+
extension_name: arrow
30+
exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64;windows_amd64_rtools'
31+
duckdb_version: v0.9.2
32+
deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}

‎.github/workflows/NodeJS.yml‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,21 +18,28 @@ jobs:
1818
- uses: actions/checkout@v2
1919
with:
2020
fetch-depth: 0
21+
submodules: 'true'
2122

2223
- uses: actions/setup-python@v2
2324
with:
2425
python-version: '3.9'
2526

26-
- name: Update DuckDB submodule
27+
- uses: actions/setup-node@v4
28+
with:
29+
node-version: '20'
30+
31+
- name: Install required node packages
2732
run: |
28-
git config --global --add safe.directory '*'
29-
make pull
33+
sudo npm install -g duckdb apache-arrow mocha
34+
sudo npm install duckdb apache-arrow mocha
35+
npm -v
36+
node -v
37+
3038
31-
- name: Install Node
39+
- name: Build duckdb
3240
run: |
33-
cd duckdb
34-
source scripts/install_node.sh 16
41+
make
3542
36-
- name: Build & Test
43+
- name: Run JS tests
3744
run: |
38-
make test_debug_js
45+
make test_release_js

‎.github/workflows/_extension_deploy.yml‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
#
2+
# Reusable workflow that deploys the artifacts produced by github.com/duckdb/duckdb/.github/workflows/_extension_distribution.yml
3+
#
4+
# note: this workflow needs to be located in the extension repository, as it requires secrets to be passed to the
5+
# deploy script. However, it should generally not be necessary to modify this workflow in your extension repository, as
6+
# this workflow can be configured to use a custom deploy script.
7+
8+
9+
name: Extension Deployment
10+
on:
11+
workflow_call:
12+
inputs:
13+
# The name of the extension
14+
extension_name:
15+
required: true
16+
type: string
17+
# DuckDB version to build against
18+
duckdb_version:
19+
required: true
20+
type: string
21+
# ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64'
22+
exclude_archs:
23+
required: false
24+
type: string
25+
default: ""
26+
# Whether to upload this deployment as the latest. This may overwrite a previous deployment.
27+
deploy_latest:
28+
required: false
29+
type: boolean
30+
default: false
31+
# Whether to upload this deployment under a versioned path. These will not be deleted automatically
32+
deploy_versioned:
33+
required: false
34+
type: boolean
35+
default: false
36+
# Postfix added to artifact names. Can be used to guarantee unique names when this workflow is called multiple times
37+
artifact_postfix:
38+
required: false
39+
type: string
40+
default: ""
41+
# Override the default deploy script with a custom script
42+
deploy_script:
43+
required: false
44+
type: string
45+
default: "./scripts/extension-upload.sh"
46+
# Override the default matrix parse script with a custom script
47+
matrix_parse_script:
48+
required: false
49+
type: string
50+
default: "./duckdb/scripts/modify_distribution_matrix.py"
51+
52+
jobs:
53+
generate_matrix:
54+
name: Generate matrix
55+
runs-on: ubuntu-latest
56+
outputs:
57+
deploy_matrix: ${{ steps.parse-matrices.outputs.deploy_matrix }}
58+
steps:
59+
- uses: actions/checkout@v3
60+
with:
61+
fetch-depth: 0
62+
submodules: 'true'
63+
64+
- name: Checkout DuckDB to version
65+
run: |
66+
cd duckdb
67+
git checkout ${{ inputs.duckdb_version }}
68+
69+
- id: parse-matrices
70+
run: |
71+
python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty
72+
deploy_matrix="`cat deploy_matrix.json`"
73+
echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT
74+
echo `cat $GITHUB_OUTPUT`
75+
76+
deploy:
77+
name: Deploy
78+
runs-on: ubuntu-latest
79+
needs: generate_matrix
80+
if: ${{ needs.generate_matrix.outputs.deploy_matrix != '{}' && needs.generate_matrix.outputs.deploy_matrix != '' }}
81+
strategy:
82+
matrix: ${{fromJson(needs.generate_matrix.outputs.deploy_matrix)}}
83+
84+
steps:
85+
- uses: actions/checkout@v3
86+
with:
87+
fetch-depth: 0
88+
submodules: 'true'
89+
90+
- name: Checkout DuckDB to version
91+
run: |
92+
cd duckdb
93+
git checkout ${{ inputs.duckdb_version }}
94+
95+
- uses: actions/download-artifact@v2
96+
with:
97+
name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}
98+
path: |
99+
/tmp/extension
100+
101+
- name: Deploy
102+
shell: bash
103+
env:
104+
AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }}
105+
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }}
106+
AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }}
107+
BUCKET_NAME: ${{ secrets.S3_BUCKET }}
108+
DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }}
109+
run: |
110+
pwd
111+
python3 -m pip install pip awscli
112+
git config --global --add safe.directory '*'
113+
cd duckdb
114+
git fetch --tags
115+
export DUCKDB_VERSION=`git tag --points-at HEAD`
116+
export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`}
117+
cd ..
118+
git fetch --tags
119+
export EXT_VERSION=`git tag --points-at HEAD`
120+
export EXT_VERSION=${EXT_VERSION:=`git log -1 --format=%h`}
121+
${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.deploy_latest || 'true' && 'false'}} ${{inputs.deploy_versioned || 'true' && 'false'}}

‎Makefile‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -56,12 +56,6 @@ release:
5656
cmake $(GENERATOR) $(FORCE_COLOR) $(EXTENSION_FLAGS) ${CLIENT_FLAGS} ${CMAKE_VARS} -DEXTENSION_STATIC_BUILD=1 -DCMAKE_BUILD_TYPE=Release ${BUILD_FLAGS} -S ./duckdb/ -B build/release && \
5757
cmake --build build/release --config Release
5858

59-
# Client build
60-
debug_js: CLIENT_FLAGS=-DBUILD_NODE=1
61-
debug_js: debug
62-
release_js: CLIENT_FLAGS=-DBUILD_NODE=1
63-
release_js: release
64-
6559
# Main tests
6660
test: test_release
6761

@@ -74,11 +68,11 @@ test_debug: debug
7468
# Client tests
7569
DEBUG_EXT_PATH='$(PROJ_DIR)build/debug/extension/arrow/arrow.duckdb_extension'
7670
RELEASE_EXT_PATH='$(PROJ_DIR)build/release/extension/arrow/arrow.duckdb_extension'
77-
test_js: test_debug_js
78-
test_debug_js: debug_js
79-
cd duckdb/tools/nodejs && ARROW_EXTENSION_BINARY_PATH=$(DEBUG_EXT_PATH) npm run test-path -- "../../../test/nodejs/**/*.js"
80-
test_release_js: release_js
81-
cd duckdb/tools/nodejs && ARROW_EXTENSION_BINARY_PATH=$(RELEASE_EXT_PATH) npm run test-path -- "../../../test/nodejs/**/*.js"
71+
test_js:
72+
test_debug_js:
73+
ARROW_EXTENSION_BINARY_PATH=$(DEBUG_EXT_PATH) mocha -R spec --timeout 480000 -n expose-gc --exclude 'test/*.ts' -- "test/nodejs/**/*.js"
74+
test_release_js:
75+
ARROW_EXTENSION_BINARY_PATH=$(RELEASE_EXT_PATH) mocha -R spec --timeout 480000 -n expose-gc --exclude 'test/*.ts' -- "test/nodejs/**/*.js"
8276

8377
format:
8478
find src/ -iname *.hpp -o -iname *.cpp | xargs clang-format --sort-includes=0 -style=file -i

‎duckdb‎

Submodule duckdb updated 413 files

‎test/nodejs/arrow_test.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
var duckdb = require('../../duckdb/tools/nodejs');
2-
var arrow = require('../../duckdb/tools/nodejs/node_modules/apache-arrow')
1+
var arrow = require('apache-arrow')
2+
var duckdb = require('duckdb');
33
var assert = require('assert');
44

5-
const parquet_file_path = "../../../data/parquet-testing/lineitem_sf0_01.parquet";
5+
const parquet_file_path = "data/parquet-testing/lineitem_sf0_01.parquet";
66

77
// Wrapper for tests, materializes whole stream
88
const arrow_ipc_stream = async (conn, sql) => {

0 commit comments

Comments
 (0)