Skip to content

Commit 56d24d4

Browse files
samansminksamansmink
authored
Merge pull request #16 from samansmink/expand-ci-to-all-archs
Update DuckDB
2 parents 098eee3 + 0728598 commit 56d24d4

File tree

6 files changed

+167
-63
lines changed

6 files changed

+167
-63
lines changed

‎.github/workflows/Linux.yml

+84
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
name: Linux
2+
on: [push, pull_request,repository_dispatch]
3+
concurrency:
4+
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
5+
cancel-in-progress: true
6+
defaults:
7+
run:
8+
shell: bash
9+
10+
jobs:
11+
linux:
12+
name: Linux Release
13+
runs-on: ubuntu-latest
14+
container: ${{ matrix.container }}
15+
strategy:
16+
matrix:
17+
# Add commits/tags to build against other DuckDB versions
18+
duckdb_version: [ '<submodule_version>' ]
19+
arch: ['linux_amd64', 'linux_arm64', 'linux_amd64_gcc4']
20+
include:
21+
- arch: 'linux_amd64_gcc4'
22+
container: 'quay.io/pypa/manylinux2014_x86_64'
23+
- arch: 'linux_amd64'
24+
container: 'ubuntu:18.04'
25+
- arch: 'linux_arm64'
26+
container: 'ubuntu:18.04'
27+
env:
28+
GEN: Ninja
29+
30+
steps:
31+
- name: Install required ubuntu packages
32+
if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }}
33+
run: |
34+
apt-get update -y -qq
35+
apt-get install -y -qq software-properties-common
36+
add-apt-repository ppa:git-core/ppa
37+
apt-get update -y -qq
38+
apt-get install -y -qq ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client
39+
40+
- name: Install Git 2.18.5
41+
if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }}
42+
run: |
43+
wget https://github.com/git/git/archive/refs/tags/v2.18.5.tar.gz
44+
tar xvf v2.18.5.tar.gz
45+
cd git-2.18.5
46+
make
47+
make prefix=/usr install
48+
git --version
49+
50+
- uses: actions/checkout@v3
51+
with:
52+
fetch-depth: 0
53+
submodules: 'true'
54+
55+
- name: Checkout DuckDB to version
56+
if: ${{ matrix.duckdb_version != '<submodule_version>'}}
57+
run: |
58+
cd duckdb
59+
git checkout ${{ matrix.duckdb_version }}
60+
61+
- name: Setup ManyLinux2014
62+
if: ${{ matrix.arch == 'linux_amd64_gcc4' }}
63+
run: |
64+
./duckdb/scripts/setup_manylinux2014.sh general aws-cli ccache ssh python_alias
65+
66+
- name: Setup Ubuntu
67+
if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }}
68+
uses: ./duckdb/.github/actions/ubuntu_18_setup
69+
with:
70+
aarch64_cross_compile: ${{ matrix.arch == 'linux_arm64' && 1 }}
71+
72+
- name: Build extension
73+
env:
74+
GEN: ninja
75+
STATIC_LIBCPP: 1
76+
CC: ${{ matrix.arch == 'linux_arm64' && 'aarch64-linux-gnu-gcc' || '' }}
77+
CXX: ${{ matrix.arch == 'linux_arm64' && 'aarch64-linux-gnu-g++' || '' }}
78+
run: |
79+
make release
80+
81+
- name: Test extension
82+
if: ${{ matrix.arch != 'linux_arm64'}}
83+
run: |
84+
make test

‎.github/workflows/MacOS.yml

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
name: MacOS
2+
on: [push, pull_request,repository_dispatch]
3+
concurrency:
4+
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
5+
cancel-in-progress: true
6+
defaults:
7+
run:
8+
shell: bash
9+
10+
jobs:
11+
macos:
12+
name: MacOS Release (${{ matrix.osx_build_arch }})
13+
runs-on: macos-latest
14+
strategy:
15+
matrix:
16+
# Add commits/tags to build against other DuckDB versions
17+
duckdb_version: [ '<submodule_version>' ]
18+
osx_build_arch: [ 'x86_64', 'arm64' ]
19+
include:
20+
- osx_build_arch: 'x86_64'
21+
duckdb_arch: 'osx_amd64'
22+
- osx_build_arch: 'arm64'
23+
duckdb_arch: 'osx_arm64'
24+
25+
env:
26+
OSX_BUILD_ARCH: ${{ matrix.osx_build_arch }}
27+
GEN: Ninja
28+
29+
steps:
30+
- uses: actions/checkout@v3
31+
with:
32+
fetch-depth: 0
33+
submodules: 'true'
34+
35+
- name: Install Ninja
36+
run: brew install ninja
37+
38+
- name: Setup Ccache
39+
uses: hendrikmuhs/ccache-action@main
40+
with:
41+
key: ${{ github.job }}-${{ matrix.duckdb_version }}
42+
43+
- uses: actions/setup-python@v2
44+
with:
45+
python-version: '3.7'
46+
47+
- name: Checkout DuckDB to version
48+
if: ${{ matrix.duckdb_version != '<submodule_version>'}}
49+
run: |
50+
cd duckdb
51+
git checkout ${{ matrix.duckdb_version }}
52+
53+
- name: Build extension
54+
shell: bash
55+
run: |
56+
make release
57+
58+
- name: Test Extension
59+
if: ${{ matrix.osx_build_arch == 'x86_64'}}
60+
shell: bash
61+
run: |
62+
make test

‎.github/workflows/SQL.yml

-31
This file was deleted.

‎CMakeLists.txt

+4-4
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,10 @@ set(EXTENSION_SOURCES
1313
src/arrow_scan_ipc.cpp
1414
src/arrow_to_ipc.cpp)
1515

16-
if(OSX_BUILD_UNIVERSAL EQUAL 1)
17-
set(OSX_UNIVERSAL_FLAG -DCMAKE_OSX_ARCHITECTURES=x86_64$<SEMICOLON>arm64)
16+
if(NOT "${OSX_BUILD_ARCH}" STREQUAL "")
17+
set(OSX_ARCH_FLAG -DCMAKE_OSX_ARCHITECTURES=${OSX_BUILD_ARCH})
1818
else()
19-
set(OSX_UNIVERSAL_FLAG "")
19+
set(OSX_ARCH_FLAG "")
2020
endif()
2121

2222
# Building Arrow
@@ -29,7 +29,7 @@ ExternalProject_Add(
2929
INSTALL_DIR "${CMAKE_BINARY_DIR}/third_party/arrow/install"
3030
BUILD_BYPRODUCTS <INSTALL_DIR>/lib/libarrow.a
3131
CONFIGURE_COMMAND
32-
${CMAKE_COMMAND} -G${CMAKE_GENERATOR} ${OSX_UNIVERSAL_FLAG}
32+
${CMAKE_COMMAND} -G${CMAKE_GENERATOR} ${OSX_ARCH_FLAG}
3333
-DCMAKE_BUILD_TYPE=Release
3434
-DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/third_party/arrow/install
3535
-DCMAKE_INSTALL_LIBDIR=lib -DARROW_BUILD_STATIC=ON -DARROW_BUILD_SHARED=OFF

‎Makefile

+16-27
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,17 @@ all: release
55
MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
66
PROJ_DIR := $(dir $(MKFILE_PATH))
77

8+
CMAKE_VARS ?=
89
OSX_BUILD_ARCH_FLAG=
910
ifneq (${OSX_BUILD_ARCH}, "")
1011
OSX_BUILD_ARCH_FLAG=-DOSX_BUILD_ARCH=${OSX_BUILD_ARCH}
1112
endif
1213
ifeq (${STATIC_LIBCPP}, 1)
1314
STATIC_LIBCPP=-DSTATIC_LIBCPP=TRUE
1415
endif
16+
ifeq (${DISABLE_EXTENSION_LOAD}, 1)
17+
CMAKE_VARS:=${CMAKE_VARS} -DDISABLE_EXTENSION_LOAD=1
18+
endif
1519

1620
ifeq ($(GEN),ninja)
1721
GENERATOR=-G "Ninja"
@@ -23,7 +27,12 @@ BUILD_FLAGS=-DEXTENSION_STATIC_BUILD=1 -DBUILD_EXTENSIONS="" ${OSX_BUILD_ARCH_FL
2327
CLIENT_FLAGS :=
2428

2529
# These flags will make DuckDB build the extension
26-
EXTENSION_FLAGS=-DDUCKDB_EXTENSION_NAMES="arrow" -DDUCKDB_EXTENSION_ARROW_PATH="$(PROJ_DIR)" -DDUCKDB_EXTENSION_ARROW_SHOULD_LINK=1 -DDUCKDB_EXTENSION_ARROW_INCLUDE_PATH="$(PROJ_DIR)src/include"
30+
EXTENSION_FLAGS=\
31+
-DDUCKDB_EXTENSION_NAMES="arrow" \
32+
-DDUCKDB_EXTENSION_ARROW_PATH="$(PROJ_DIR)" \
33+
-DDUCKDB_EXTENSION_ARROW_LOAD_TESTS=1 \
34+
-DDUCKDB_EXTENSION_ARROW_TEST_PATH=$(PROJ_DIR)test \
35+
-DDUCKDB_EXTENSION_ARROW_INCLUDE_PATH="$(PROJ_DIR)src/include"
2736

2837
pull:
2938
git submodule init
@@ -37,41 +46,28 @@ clean:
3746
# Main build
3847
debug:
3948
mkdir -p build/debug && \
40-
cmake $(GENERATOR) $(FORCE_COLOR) $(EXTENSION_FLAGS) ${CLIENT_FLAGS} -DEXTENSION_STATIC_BUILD=1 -DCMAKE_BUILD_TYPE=Debug ${BUILD_FLAGS} -S ./duckdb/ -B build/debug && \
49+
cmake $(GENERATOR) $(FORCE_COLOR) $(EXTENSION_FLAGS) -DDUCKDB_EXTENSION_ARROW_SHOULD_LINK=0 ${CLIENT_FLAGS} ${CMAKE_VARS} -DEXTENSION_STATIC_BUILD=1 -DCMAKE_BUILD_TYPE=Debug ${BUILD_FLAGS} -S ./duckdb/ -B build/debug && \
4150
cmake --build build/debug --config Debug
4251

4352
release:
4453
mkdir -p build/release && \
45-
cmake $(GENERATOR) $(FORCE_COLOR) $(EXTENSION_FLAGS) ${CLIENT_FLAGS} -DEXTENSION_STATIC_BUILD=1 -DCMAKE_BUILD_TYPE=Release ${BUILD_FLAGS} -S ./duckdb/ -B build/release && \
54+
cmake $(GENERATOR) $(FORCE_COLOR) $(EXTENSION_FLAGS) -DDUCKDB_EXTENSION_ARROW_SHOULD_LINK=0 ${CLIENT_FLAGS} ${CMAKE_VARS} -DEXTENSION_STATIC_BUILD=1 -DCMAKE_BUILD_TYPE=Release ${BUILD_FLAGS} -S ./duckdb/ -B build/release && \
4655
cmake --build build/release --config Release
4756

4857
# Client build
49-
debug_js: CLIENT_FLAGS=-DBUILD_NODE=1 -DBUILD_EXTENSIONS=json
58+
debug_js: CLIENT_FLAGS=-DBUILD_NODE=1 -DBUILD_EXTENSIONS=json -DDUCKDB_EXTENSION_ARROW_SHOULD_LINK=1
5059
debug_js: debug
51-
52-
debug_r: CLIENT_FLAGS=-DBUILD_R=1
53-
debug_r: debug
54-
55-
debug_python: CLIENT_FLAGS=-DBUILD_PYTHON=1 -DBUILD_EXTENSIONS=json;fts;tpch;visualizer;tpcds
56-
debug_python: debug
57-
58-
release_js: CLIENT_FLAGS=-DBUILD_NODE=1 -DBUILD_EXTENSIONS=json
60+
release_js: CLIENT_FLAGS=-DBUILD_NODE=1 -DBUILD_EXTENSIONS=json -DDUCKDB_EXTENSION_ARROW_SHOULD_LINK=1
5961
release_js: release
6062

61-
release_r: CLIENT_FLAGS=-DBUILD_R=1
62-
release_r: release
63-
64-
release_python: CLIENT_FLAGS=-DBUILD_PYTHON=1 -DBUILD_EXTENSIONS=json;fts;tpch;visualizer;tpcds
65-
release_python: release
66-
6763
# Main tests
6864
test: test_release
6965

7066
test_release: release
71-
./build/release/test/unittest --test-dir . "[sql]"
67+
./build/release/test/unittest "$(PROJ_DIR)test/*"
7268

7369
test_debug: debug
74-
./build/debug/test/unittest --test-dir . "[sql]"
70+
./build/release/test/unittest "$(PROJ_DIR)test/*"
7571

7672
# Client tests
7773
test_js: test_debug_js
@@ -81,13 +77,6 @@ test_debug_js: debug_js
8177
test_release_js: release_js
8278
cd duckdb/tools/nodejs && npm run test-path -- "../../../test/nodejs/**/*.js"
8379

84-
test_python: test_debug_python
85-
test_debug_python: debug_python
86-
cd test/python && python3 -m pytest
87-
88-
test_release_python: release_python
89-
cd test/python && python3 -m pytest
90-
9180
format:
9281
find src/ -iname *.hpp -o -iname *.cpp | xargs clang-format --sort-includes=0 -style=file -i
9382
cmake-format -i CMakeLists.txt

‎duckdb

Submodule duckdb updated 874 files

0 commit comments

Comments
 (0)