Skip to content

Commit 50b7870

Browse files
MytherinMytherin
authored
Merge pull request #95 from carlopi/absorb_patches
Absorb more patches
2 parents d355dd0 + 0a9240c commit 50b7870

File tree

7 files changed

+398
-17
lines changed

7 files changed

+398
-17
lines changed

‎.github/workflows/MainDistributionPipeline.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ concurrency:
1414
jobs:
1515
duckdb-stable-build:
1616
name: Build extension binaries
17-
uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main
17+
uses: ./.github/workflows/_extension_distribution.yml
1818
with:
1919
duckdb_version: main
2020
extension_name: sqlite_scanner

‎.github/workflows/_extension_distribution.yml

Lines changed: 381 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,381 @@
1+
# Reusable workflow for building DuckDB extensions using a standardized environment
2+
#
3+
# The workflow:
4+
# - builds the extension using the CI workflow from the corresponding DuckDB version
5+
# - uploads the extensions as gh actions artifacts in the following format:
6+
# <ext_name>-<duckdb_version>-extension-<arch><optional_postfix>
7+
#
8+
# note: extensions are simply uploaded to GitHub actions, deploying the extensions is done a separate step. More info on
9+
# this can be found in https://github.com/duckdb/extension-template
10+
11+
name: Extension distribution
12+
on:
13+
workflow_call:
14+
inputs:
15+
# The name with which the extension will be built
16+
extension_name:
17+
required: true
18+
type: string
19+
# DuckDB version to build against, should in most cases be identical to
20+
duckdb_version:
21+
required: true
22+
type: string
23+
# ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64'
24+
exclude_archs:
25+
required: false
26+
type: string
27+
default: ""
28+
# Postfix added to artifact names. Can be used to guarantee unique names when this workflow is called multiple times
29+
artifact_postfix:
30+
required: false
31+
type: string
32+
default: ""
33+
# Override the default vcpkg commit used by this version of DuckDB
34+
vcpkg_commit:
35+
required: false
36+
type: string
37+
default: "a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6"
38+
# Override the default script producing the matrices. Allows specifying custom matrices.
39+
matrix_parse_script:
40+
required: false
41+
type: string
42+
default: "./duckdb/scripts/modify_distribution_matrix.py"
43+
# Enable building the DuckDB Shell
44+
build_duckdb_shell:
45+
required: false
46+
type: boolean
47+
default: true
48+
49+
jobs:
50+
generate_matrix:
51+
name: Generate matrix
52+
runs-on: ubuntu-latest
53+
outputs:
54+
linux_matrix: ${{ steps.set-matrix-linux.outputs.linux_matrix }}
55+
windows_matrix: ${{ steps.set-matrix-windows.outputs.windows_matrix }}
56+
osx_matrix: ${{ steps.set-matrix-osx.outputs.osx_matrix }}
57+
wasm_matrix: ${{ steps.set-matrix-wasm.outputs.wasm_matrix }}
58+
steps:
59+
- uses: actions/checkout@v3
60+
with:
61+
fetch-depth: 0
62+
submodules: 'true'
63+
64+
- name: Checkout DuckDB to version
65+
run: |
66+
cd duckdb
67+
git checkout ${{ inputs.duckdb_version }}
68+
69+
- id: parse-matrices
70+
run: |
71+
python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --select_os linux --output linux_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty
72+
python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --select_os osx --output osx_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty
73+
python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --select_os windows --output windows_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty
74+
python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --select_os wasm --output wasm_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty
75+
76+
- id: set-matrix-linux
77+
run: |
78+
linux_matrix="`cat linux_matrix.json`"
79+
echo linux_matrix=$linux_matrix >> $GITHUB_OUTPUT
80+
echo `cat $GITHUB_OUTPUT`
81+
82+
- id: set-matrix-osx
83+
run: |
84+
osx_matrix="`cat osx_matrix.json`"
85+
echo osx_matrix=$osx_matrix >> $GITHUB_OUTPUT
86+
echo `cat $GITHUB_OUTPUT`
87+
88+
- id: set-matrix-windows
89+
run: |
90+
windows_matrix="`cat windows_matrix.json`"
91+
echo windows_matrix=$windows_matrix >> $GITHUB_OUTPUT
92+
echo `cat $GITHUB_OUTPUT`
93+
94+
- id: set-matrix-wasm
95+
run: |
96+
wasm_matrix="`cat wasm_matrix.json`"
97+
echo wasm_matrix=$wasm_matrix >> $GITHUB_OUTPUT
98+
echo `cat $GITHUB_OUTPUT`
99+
100+
linux:
101+
name: Linux
102+
runs-on: ubuntu-latest
103+
container: ${{ matrix.container }}
104+
needs: generate_matrix
105+
if: ${{ needs.generate_matrix.outputs.linux_matrix != '{}' && needs.generate_matrix.outputs.linux_matrix != '' }}
106+
strategy:
107+
matrix: ${{fromJson(needs.generate_matrix.outputs.linux_matrix)}}
108+
env:
109+
VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }}
110+
VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
111+
GEN: Ninja
112+
BUILD_SHELL: ${{ inputs.build_duckdb_shell && '1' || '0' }}
113+
DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }}
114+
115+
steps:
116+
- name: Install required ubuntu packages
117+
if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }}
118+
run: |
119+
apt-get update -y -qq
120+
apt-get install -y -qq software-properties-common
121+
add-apt-repository ppa:git-core/ppa
122+
apt-get update -y -qq
123+
apt-get install -y -qq ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client
124+
125+
- name: Install Git 2.18.5
126+
if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }}
127+
run: |
128+
wget https://github.com/git/git/archive/refs/tags/v2.18.5.tar.gz
129+
tar xvf v2.18.5.tar.gz
130+
cd git-2.18.5
131+
make
132+
make prefix=/usr install
133+
git --version
134+
135+
- uses: actions/checkout@v3
136+
with:
137+
fetch-depth: 0
138+
submodules: 'true'
139+
140+
- name: Checkout DuckDB to version
141+
run: |
142+
cd duckdb
143+
git checkout ${{ inputs.duckdb_version }}
144+
145+
- name: Setup ManyLinux2014
146+
if: ${{ matrix.duckdb_arch == 'linux_amd64_gcc4' }}
147+
run: |
148+
./duckdb/scripts/setup_manylinux2014.sh general aws-cli ccache ssh python_alias openssl
149+
150+
- name: Setup Ccache
151+
uses: hendrikmuhs/[email protected] # Note: pinned due to GLIBC incompatibility in later releases
152+
continue-on-error: true
153+
with:
154+
key: ${{ github.job }}-${{ matrix.duckdb_arch }}
155+
156+
- name: Setup Ubuntu
157+
if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }}
158+
uses: ./duckdb/.github/actions/ubuntu_18_setup
159+
with:
160+
aarch64_cross_compile: ${{ matrix.duckdb_arch == 'linux_arm64' && 1 }}
161+
162+
- name: Setup vcpkg
163+
uses: lukka/[email protected]
164+
with:
165+
vcpkgGitCommitId: ${{ inputs.vcpkg_commit }}
166+
167+
- name: Build extension
168+
env:
169+
GEN: ninja
170+
CC: ${{ matrix.duckdb_arch == 'linux_arm64' && 'aarch64-linux-gnu-gcc' || '' }}
171+
CXX: ${{ matrix.duckdb_arch == 'linux_arm64' && 'aarch64-linux-gnu-g++' || '' }}
172+
DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }}
173+
run: |
174+
make release
175+
176+
- name: Test extension
177+
if: ${{ matrix.duckdb_arch != 'linux_arm64'}}
178+
run: |
179+
make test
180+
181+
- uses: actions/upload-artifact@v3
182+
with:
183+
if-no-files-found: error
184+
name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}
185+
path: |
186+
build/release/extension/${{ inputs.extension_name }}/${{ inputs.extension_name }}.duckdb_extension
187+
188+
macos:
189+
name: MacOS
190+
runs-on: macos-latest
191+
needs: generate_matrix
192+
if: ${{ needs.generate_matrix.outputs.osx_matrix != '{}' && needs.generate_matrix.outputs.osx_matrix != '' }}
193+
strategy:
194+
matrix: ${{fromJson(needs.generate_matrix.outputs.osx_matrix)}}
195+
env:
196+
VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
197+
VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }}
198+
OSX_BUILD_ARCH: ${{ matrix.osx_build_arch }}
199+
GEN: Ninja
200+
BUILD_SHELL: ${{ inputs.build_duckdb_shell && '1' || '0' }}
201+
DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }}
202+
203+
steps:
204+
- uses: actions/checkout@v3
205+
with:
206+
fetch-depth: 0
207+
submodules: 'true'
208+
209+
- name: Install Ninja
210+
run: |
211+
brew install ninja
212+
213+
- name: Setup Ccache
214+
uses: hendrikmuhs/ccache-action@main
215+
continue-on-error: true
216+
with:
217+
key: ${{ github.job }}-${{ matrix.duckdb_arch }}
218+
219+
- uses: actions/setup-python@v5
220+
with:
221+
python-version: '3.11'
222+
223+
- name: Checkout DuckDB to version
224+
run: |
225+
cd duckdb
226+
git checkout ${{ inputs.duckdb_version }}
227+
228+
- name: Setup vcpkg
229+
uses: lukka/[email protected]
230+
with:
231+
vcpkgGitCommitId: ${{ inputs.vcpkg_commit }}
232+
233+
- name: Build extension
234+
shell: bash
235+
env:
236+
DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }}
237+
run: |
238+
make release
239+
240+
- name: Test Extension
241+
if: ${{ matrix.osx_build_arch == 'x86_64'}}
242+
shell: bash
243+
run: |
244+
make test
245+
246+
- uses: actions/upload-artifact@v3
247+
with:
248+
if-no-files-found: error
249+
name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}
250+
path: |
251+
build/release/extension/${{ inputs.extension_name }}/${{ inputs.extension_name }}.duckdb_extension
252+
253+
windows:
254+
name: Windows
255+
runs-on: windows-latest
256+
needs: generate_matrix
257+
if: ${{ needs.generate_matrix.outputs.windows_matrix != '{}' && needs.generate_matrix.outputs.windows_matrix != '' }}
258+
strategy:
259+
matrix: ${{fromJson(needs.generate_matrix.outputs.windows_matrix)}}
260+
env:
261+
GEN: Ninja
262+
VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
263+
VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }}
264+
BUILD_SHELL: ${{ inputs.build_duckdb_shell && '1' || '0' }}
265+
DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }}
266+
CC: ${{ matrix.duckdb_arch == 'windows_amd64_rtools' && 'gcc' || '' }}
267+
CXX: ${{ matrix.duckdb_arch == 'windows_amd64_rtools' && 'g++' || '' }}
268+
269+
steps:
270+
- name: Keep \n line endings
271+
shell: bash
272+
run: |
273+
git config --global core.autocrlf false
274+
git config --global core.eol lf
275+
276+
- uses: actions/checkout@v3
277+
with:
278+
fetch-depth: 0
279+
submodules: 'true'
280+
281+
- uses: actions/setup-python@v5
282+
with:
283+
python-version: '3.11'
284+
285+
- uses: r-lib/actions/setup-r@v2
286+
if: matrix.duckdb_arch == 'windows_amd64_rtools'
287+
with:
288+
r-version: 'devel'
289+
update-rtools: true
290+
rtools-version: '42' # linker bug in 43
291+
292+
- name: Checkout DuckDB to version
293+
run: |
294+
cd duckdb
295+
git checkout ${{ inputs.duckdb_version }}
296+
297+
- name: Setup Ccache
298+
uses: hendrikmuhs/ccache-action@main
299+
continue-on-error: true
300+
with:
301+
key: ${{ github.job }}-${{ matrix.duckdb_arch }}
302+
303+
- name: Setup vcpkg
304+
uses: lukka/[email protected]
305+
with:
306+
vcpkgGitCommitId: ${{ inputs.vcpkg_commit }}
307+
308+
- name: Fix for MSVC issue
309+
shell: bash
310+
env:
311+
OVERLAY_TRIPLET_SRC: ${{ github.workspace }}/vcpkg/triplets/community/x64-windows-static-md.cmake
312+
OVERLAY_TRIPLET_DST: ${{ github.workspace }}/overlay_triplets/x64-windows-static-md.cmake
313+
run: |
314+
mkdir overlay_triplets
315+
cp $OVERLAY_TRIPLET_SRC $OVERLAY_TRIPLET_DST
316+
echo "set(VCPKG_PLATFORM_TOOLSET_VERSION "14.39")" >> $OVERLAY_TRIPLET_DST
317+
318+
- name: Build & test extension
319+
env:
320+
VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/overlay_triplets"
321+
DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }}
322+
DUCKDB_PLATFORM_RTOOLS: ${{ matrix.duckdb_arch == 'windows_amd64_rtools' && 1 || 0 }}
323+
run: |
324+
make test_release
325+
326+
- uses: actions/upload-artifact@v3
327+
with:
328+
if-no-files-found: error
329+
name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}
330+
path: |
331+
build/release/extension/${{ inputs.extension_name }}/${{ inputs.extension_name }}.duckdb_extension
332+
333+
wasm:
334+
name: DuckDB-Wasm
335+
runs-on: ubuntu-latest
336+
needs: generate_matrix
337+
if: ${{ needs.generate_matrix.outputs.wasm_matrix != '{}' && needs.generate_matrix.outputs.wasm_matrix != '' }}
338+
strategy:
339+
matrix: ${{fromJson(needs.generate_matrix.outputs.wasm_matrix)}}
340+
env:
341+
VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }}
342+
VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
343+
GEN: Ninja
344+
DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }}
345+
346+
steps:
347+
- uses: actions/checkout@v3
348+
with:
349+
fetch-depth: 0
350+
submodules: 'true'
351+
352+
- name: Checkout DuckDB to version
353+
run: |
354+
cd duckdb
355+
git checkout ${{ inputs.duckdb_version }}
356+
357+
- uses: mymindstorm/setup-emsdk@v13
358+
with:
359+
version: 'latest'
360+
361+
- name: Setup vcpkg
362+
uses: lukka/[email protected]
363+
with:
364+
vcpkgGitCommitId: ${{ inputs.vcpkg_commit }}
365+
366+
- name: Setup Ccache
367+
uses: hendrikmuhs/ccache-action@main
368+
continue-on-error: true
369+
with:
370+
key: ${{ github.job }}-${{ matrix.duckdb_arch }}
371+
372+
- name: Build Wasm module
373+
run: |
374+
make ${{ matrix.duckdb_arch }}
375+
376+
- uses: actions/upload-artifact@v3
377+
with:
378+
if-no-files-found: error
379+
name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}
380+
path: |
381+
build/${{ matrix.duckdb_arch }}/extension/${{ inputs.extension_name }}/${{ inputs.extension_name }}.duckdb_extension.wasm

0 commit comments

Comments
 (0)