Skip to content

Commit c5d8c38

Browse files
teo-tsirpanisteo-tsirpanis
committed
Merge branch 'dev' into teo/fragment-metadata-c.41
2 parents 2f29592 + 6dff6fa commit c5d8c38

File tree

384 files changed

+18313
-16932
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

384 files changed

+18313
-16932
lines changed

‎.github/workflows/append-release-cmake.yml

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
name: Append Release CMake
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
ref:
7+
description: 'Ref to be used as release'
8+
default: 'latest'
9+
required: true
10+
type: string
11+
workflow_call:
12+
inputs:
13+
ref:
14+
description: 'Ref to be used as release'
15+
default: 'latest'
16+
required: true
17+
type: string
18+
19+
jobs:
20+
generate_cmake_files:
21+
runs-on: ubuntu-latest
22+
steps:
23+
- name: Checkout TileDB
24+
uses: actions/checkout@v3
25+
26+
- name: Make release and output directories
27+
run: |
28+
mkdir release output
29+
30+
- name: Github release data
31+
id: release_data
32+
uses: KevinRohn/[email protected]
33+
with:
34+
# repository: 'TileDB-Inc/TileDB'
35+
version: ${{ inputs.ref }}
36+
asset-file: '*.zip,*.tar.gz'
37+
asset-output: './release/'
38+
39+
- name: Render template
40+
run: |
41+
PATTERN="tiledb-([^-]+)-([^-]+)(-noavx2)?-(.+)\.(tar\.gz|zip)$"
42+
RELLIST="output/releases.csv"
43+
MODULE="output/DownloadPrebuiltTileDB.cmake"
44+
cp cmake/inputs/DownloadPrebuiltTileDB.cmake $MODULE
45+
echo "platform,url,sha256" > $RELLIST
46+
47+
for FILE in $(ls release)
48+
do
49+
if [[ $FILE =~ $PATTERN ]]
50+
then
51+
OS=${BASH_REMATCH[1]^^}
52+
ARCH=${BASH_REMATCH[2]^^}
53+
NOAVX2=${BASH_REMATCH[3]^^}
54+
PLATFORM=${OS}-${ARCH}${NOAVX2}
55+
56+
URL="${{ github.server_url }}/${{ github.repository }}/releases/download/${{ inputs.ref }}/$FILE"
57+
HASH=$(cat release/$FILE.sha256 | cut -d \t -f 1)
58+
59+
echo "${PLATFORM},${URL},${HASH}" >> $RELLIST
60+
fi
61+
done
62+
63+
SOURCE_FILE_NAME=$(ls release/tiledb-source-*.tar.gz)
64+
URL_TILEDB_SOURCE="${{ github.server_url }}/${{ github.repository }}/releases/download/${{ inputs.ref }}/$(basename $SOURCE_FILE_NAME)"
65+
HASH_TILEDB_SOURCE=$(cat $SOURCE_FILE_NAME.sha256 | cut -d \t -f 1)
66+
67+
echo "source,${URL_TILEDB_SOURCE},${HASH_TILEDB_SOURCE}" >> $RELLIST
68+
69+
HASH=$(sha256sum $RELLIST | cut -d " " -f 1)
70+
echo $HASH > $RELLIST.sha256
71+
72+
cat $RELLIST
73+
74+
- name: Upload template to release
75+
uses: svenstaro/upload-release-action@v2
76+
with:
77+
file: output/*
78+
tag: ${{ steps.release_data.outputs.tag_name }}
79+
overwrite: true
80+
file_glob: true

‎.github/workflows/build-docs.yml

+9
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ jobs:
2828
runs-on: ubuntu-22.04
2929
timeout-minutes: 90
3030
name: Build Docs
31+
env:
32+
VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite'
3133
steps:
3234
- uses: actions/checkout@v3
3335
- name: 'Print env'
@@ -41,6 +43,13 @@ jobs:
4143
printenv
4244
shell: bash
4345

46+
- name: Set environment variables for vcpkg binary caching
47+
uses: actions/github-script@v6
48+
with:
49+
script: |
50+
core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
51+
core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
52+
4453
- name: Set up Python
4554
uses: actions/setup-python@v4
4655

‎.github/workflows/build-ubuntu20.04-backwards-compatibility.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ jobs:
5959
- ubuntu-20.04
6060
# Note: v2_1_0 arrays were never created so its currently skipped
6161
# Note: This matrix is used to set the value of TILEDB_COMPATIBILITY_VERSION
62-
tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1"]
62+
tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1", "v2_20_1", "v2_21_1"]
6363
timeout-minutes: 30
6464
name: ${{ matrix.tiledb_version }}
6565
steps:

‎.github/workflows/build-windows.yml

+31-11
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,22 @@ jobs:
6060
TILEDB_ARROW_TESTS: ${{ matrix.TILEDB_ARROW_TESTS }}
6161
TILEDB_WEBP: ${{ matrix.TILEDB_WEBP }}
6262
TILEDB_CMAKE_BUILD_TYPE: 'Release'
63+
# On windows-2019 we are using the Visual Studio generator, which is multi-config and places the build artifacts in a subdirectory
64+
CONFIG_PATH_FIXUP: ${{ matrix.os == 'windows-2019' && 'Release' || '' }}
6365
VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite'
6466
steps:
67+
# By default Visual Studio chooses the earliest installed toolset version
68+
# for the main build and vcpkg chooses the latest. Force it to use the
69+
# latest (14.39 currently).
70+
- name: Setup MSVC toolset (VS 2022)
71+
uses: TheMrMilchmann/setup-msvc-dev@v3
72+
if: matrix.os == 'windows-2022'
73+
with:
74+
arch: x64
75+
toolset: 14.39
76+
- name: Install Ninja (VS 2022)
77+
uses: seanmiddleditch/gha-setup-ninja@v4
78+
if: matrix.os == 'windows-2022'
6579
- name: 'tiledb env prep'
6680
run: |
6781
$env:BUILD_BUILDDIRECTORY = $env:GITHUB_WORKSPACE.replace("TileDB\TileDB","tdbbd") # 't'ile'db' 'b'uild 'd'ir
@@ -141,7 +155,7 @@ jobs:
141155
# allow double-checking path
142156
cmd /c "echo $PATH"
143157
144-
$bootstrapOptions = $env:TILEDB_BASE_BOOTSTRAP_OPTIONS
158+
$bootstrapOptions = $env:TILEDB_BASE_BOOTSTRAP_OPTIONS + " -CMakeGenerator ${{ matrix.os == 'windows-2022' && 'Ninja' || '`"Visual Studio 16 2019`"' }}"
145159
if ($env:TILEDB_S3 -eq "ON") {
146160
$bootstrapOptions = "-EnableS3 " + $bootstrapOptions
147161
}
@@ -239,9 +253,7 @@ jobs:
239253
}
240254
241255
# CMake exits with non-0 status if there are any warnings during the build, so
242-
cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target tiledb_unit --config $CMakeBuildType -- /verbosity:minimal
243-
cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target tiledb_regression --config $CMakeBuildType -- /verbosity:minimal
244-
cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target all_link_complete --config $CMakeBuildType -- /verbosity:minimal
256+
cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target tiledb_unit unit_vfs tiledb_regression all_link_complete --config $CMakeBuildType
245257
246258
if ($env:TILEDB_AZURE -eq "ON") {
247259
if($env.TILEDB_USE_CUSTOM_NODE_JS) {
@@ -272,15 +284,23 @@ jobs:
272284
273285
# Actually run tests
274286
275-
$cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\$CMakeBuildType\tiledb_unit.exe -d=yes"
287+
$cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\$env:CONFIG_PATH_FIXUP\tiledb_unit.exe -d=yes"
276288
Write-Host "cmds: '$cmds'"
277289
Invoke-Expression $cmds
278290
if ($LastExitCode -ne 0) {
279291
Write-Host "Tests failed. tiledb_unit exit status: " $LastExitCocde
280292
$host.SetShouldExit($LastExitCode)
281293
}
294+
295+
$cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\tiledb\sm\filesystem\test\$env:CONFIG_PATH_FIXUP\unit_vfs -d=yes"
296+
Write-Host "cmds: '$cmds'"
297+
Invoke-Expression $cmds
298+
if ($LastExitCode -ne 0) {
299+
Write-Host "Tests failed. tiledb_vfs exit status: " $LastExitCocde
300+
$host.SetShouldExit($LastExitCode)
301+
}
282302
283-
$cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\ci\$CMakeBuildType\test_assert.exe -d=yes"
303+
$cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\ci\$env:CONFIG_PATH_FIXUP\test_assert.exe -d=yes"
284304
Invoke-Expression $cmds
285305
if ($LastExitCode -ne 0) {
286306
Write-Host "Tests failed. test_assert exit status: " $LastExitCocde
@@ -302,7 +322,7 @@ jobs:
302322
303323
$TestAppDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api")
304324
$TestAppDataDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api\test_app_data")
305-
Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api\$CMakeBuildType") -Filter *.exe |
325+
Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api\$env:CONFIG_PATH_FIXUP\") -Filter *.exe |
306326
Foreach-Object {
307327
try {
308328
Set-Location -path $TestAppDir
@@ -335,7 +355,7 @@ jobs:
335355
336356
$TestAppDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api")
337357
$TestAppDataDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api\test_app_data")
338-
Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api\$CMakeBuildType") -Filter *.exe |
358+
Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api\$env:CONFIG_PATH_FIXUP\") -Filter *.exe |
339359
Foreach-Object {
340360
try {
341361
Set-Location -path $TestAppDir
@@ -380,12 +400,12 @@ jobs:
380400
cd build
381401
382402
# Build zip artifact
383-
cmake -A X64 -DCMAKE_PREFIX_PATH="$env:BUILD_BUILDDIRECTORY\dist;$env:BUILD_BUILDDIRECTORY\vcpkg_installed\x64-windows" ..
403+
cmake ${{ matrix.os != 'windows-2019' && '-G Ninja' || '' }} -DCMAKE_BUILD_TYPE="$CMakeBuildType" -DCMAKE_PREFIX_PATH="$env:BUILD_BUILDDIRECTORY\dist;$env:BUILD_BUILDDIRECTORY\vcpkg_installed\x64-windows" ..
384404
385405
cmake --build . --config $CMakeBuildType -v
386406
387-
#.\$CMakeBuildType\ExampleExe.exe
388-
$cmd = ".\$CMakeBuildType\ExampleExe.exe"
407+
#.\$env:CONFIG_PATH_FIXUP\ExampleExe.exe
408+
$cmd = ".\$env:CONFIG_PATH_FIXUP\ExampleExe.exe"
389409
Write-Host "cmd: '$cmd'"
390410
Invoke-Expression $cmd
391411

‎.github/workflows/ci-linux_mac.yml

+1
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,7 @@ jobs:
191191
./tiledb/test/regression/tiledb_regression -d yes
192192
./tiledb/test/ci/test_assert -d yes
193193
./tiledb/test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
194+
./tiledb/tiledb/sm/filesystem/test/unit_vfs -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
194195
195196
###################################################
196197
# Stop helper processes, if applicable

‎.github/workflows/ci-rest.yml

+18-8
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,43 @@
11
name: REST CI
22

33
on:
4-
- workflow_call
5-
- workflow_dispatch
4+
workflow_call:
5+
workflow_dispatch:
6+
inputs:
7+
ref:
8+
type: string
9+
required: false
10+
11+
push:
12+
branches:
13+
- dev
14+
- release-*
15+
- refs/tags/*
616

717
jobs:
818
rest-ci:
919
runs-on: ubuntu-latest
1020

1121
steps:
1222
# For easy access to lookup dispatched CI job.
13-
- name: Print URL for TileDB-REST-CI actions
14-
run: echo https://github.com/TileDB-Inc/TileDB-REST-CI/actions
23+
- name: Print URL for REST CI actions
24+
run: echo https://github.com/TileDB-Inc/TileDB-Internal/actions
1525

1626
# If this workflow fails on the remote repository, this CI job will also fail.
17-
- name: Workflow dispatch to TileDB-REST-CI
27+
- name: Workflow dispatch to REST CI
1828
id: trigger-step
1929
uses: aurelien-baudet/workflow-dispatch@v2
2030
env:
2131
TILEDB_REST_CI_PAT: ${{ secrets.TILEDB_REST_CI_PAT }}
2232
# Skip if no PAT is set (e.g. for PRs from forks).
2333
if: env.TILEDB_REST_CI_PAT != null
2434
with:
25-
repo: TileDB-Inc/TileDB-REST-CI
26-
# Trigger workflow on TileDB-REST-CI at this ref.
35+
repo: TileDB-Inc/TileDB-Internal
36+
# Trigger workflow on TileDB-Internal at this ref.
2737
ref: "main"
2838
workflow: full-ci.yml
2939
token: ${{ secrets.TILEDB_REST_CI_PAT }}
3040
# Pass TileDB core ref to test against REST.
3141
# github.head_ref will only be set for PRs, so fallback to github.ref_name if triggered via push event.
32-
inputs: '{ "tiledb_ref": "${{ github.head_ref || github.ref_name }}"}'
42+
inputs: '{ "tiledb_ref": "${{ github.event.inputs.ref || github.head_ref || github.ref_name }}"}'
3343
wait-for-completion-timeout: 90m

‎.github/workflows/full-ci.yml

+5-19
Original file line numberDiff line numberDiff line change
@@ -54,46 +54,37 @@ jobs:
5454
bootstrap_args: '--enable-serialization --enable-release-symbols'
5555

5656
ci4:
57-
uses: ./.github/workflows/ci-linux_mac.yml
58-
with:
59-
ci_backend: HDFS
60-
matrix_image: ubuntu-22.04
61-
matrix_compiler_cc: 'gcc-13'
62-
matrix_compiler_cxx: 'g++-13'
63-
bootstrap_args: '--enable-hdfs --enable-static-tiledb --disable-werror'
64-
65-
ci5:
6657
uses: ./.github/workflows/ci-linux_mac.yml
6758
with:
6859
ci_backend: S3
6960
matrix_image: macos-12
7061
timeout: 120
7162
bootstrap_args: '--enable=s3,serialization,tools --enable-release-symbols'
7263

73-
ci6:
64+
ci5:
7465
uses: ./.github/workflows/ci-linux_mac.yml
7566
with:
7667
ci_backend: GCS
7768
matrix_image: macos-12
7869
timeout: 120
7970
bootstrap_args: '--enable-gcs --enable-release-symbols'
8071

81-
ci7:
72+
ci6:
8273
uses: ./.github/workflows/ci-linux_mac.yml
8374
with:
8475
ci_option: Experimental
8576
matrix_image: ubuntu-22.04
8677
bootstrap_args: '--enable=experimental-features,serialization --enable-release-symbols'
8778

88-
ci8:
79+
ci7:
8980
uses: ./.github/workflows/ci-linux_mac.yml
9081
with:
9182
ci_backend: S3
9283
matrix_image: ubuntu-22.04
9384
timeout: 120
9485
bootstrap_args: '--enable-s3 --enable-release-symbols'
9586

96-
ci9:
87+
ci8:
9788
uses: ./.github/workflows/ci-linux_mac.yml
9889
with:
9990
ci_option: ASAN
@@ -104,7 +95,7 @@ jobs:
10495
bootstrap_args: '--enable-serialization'
10596
asan: true
10697

107-
ci10:
98+
ci9:
10899
uses: ./.github/workflows/ci-linux_mac.yml
109100
with:
110101
ci_backend: AZURE
@@ -138,10 +129,6 @@ jobs:
138129
ci_docker:
139130
uses: ./.github/workflows/build-dockerfile.yml
140131

141-
ci_rest:
142-
uses: ./.github/workflows/ci-rest.yml
143-
secrets: inherit
144-
145132
# dummy job for branch protection check
146133
full_ci_passed:
147134
needs: [
@@ -155,7 +142,6 @@ jobs:
155142
ci7,
156143
ci8,
157144
ci9,
158-
ci10,
159145
ci_manylinux,
160146
ci_msvc,
161147
backward_compatibility,

‎.github/workflows/nightly-test.yml

+15-3
Original file line numberDiff line numberDiff line change
@@ -47,13 +47,13 @@ jobs:
4747
EXPERIMENTAL: ${{ matrix.experimental || 'OFF' }}
4848
working-directory: ${{ matrix.working_directory || github.workspace }}
4949
run: |
50-
cmake -B build ${{ matrix.config != 'Debug' && '-DTILEDB_VCPKG=OFF' }} -DTILEDB_WERROR=ON -DTILEDB_SERIALIZATION=ON -DTILEDB_EXPERIMENTAL_FEATURES=$EXPERIMENTAL -DCMAKE_BUILD_TYPE=${{ matrix.config || 'Release' }} -DTILEDB_SANITIZER=$SANITIZER_ARG -DTILEDB_VCPKG_BASE_TRIPLET=${{ matrix.base_triplet }}
50+
cmake -B build -DTILEDB_WERROR=ON -DTILEDB_SERIALIZATION=ON -DTILEDB_EXPERIMENTAL_FEATURES=$EXPERIMENTAL -DCMAKE_BUILD_TYPE=${{ matrix.config || 'Release' }} -DTILEDB_SANITIZER=$SANITIZER_ARG -DTILEDB_VCPKG_BASE_TRIPLET=${{ matrix.base_triplet }}
5151
5252
- name: Configure TileDB CMake (Windows)
5353
if: contains(matrix.os, 'windows')
5454
working-directory: ${{ matrix.working_directory || github.workspace }}
5555
run: |
56-
cmake -B build -S $env:GITHUB_WORKSPACE ${{ matrix.config != 'Debug' && '-DTILEDB_VCPKG=OFF' }} -DTILEDB_WERROR=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=${{ matrix.config || 'Release' }}
56+
cmake -B build -S $env:GITHUB_WORKSPACE -DTILEDB_WERROR=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=${{ matrix.config || 'Release' }}
5757
5858
- name: Build TileDB
5959
working-directory: ${{ matrix.working_directory || github.workspace }}
@@ -65,11 +65,23 @@ jobs:
6565
run: |
6666
cmake --build build --target check --config ${{ matrix.config || 'Release' }}
6767
68+
test_hdfs:
69+
uses: ./.github/workflows/ci-linux_mac.yml
70+
with:
71+
ci_backend: HDFS
72+
matrix_image: ubuntu-22.04
73+
matrix_compiler_cc: 'gcc-13'
74+
matrix_compiler_cxx: 'g++-13'
75+
timeout: 300
76+
bootstrap_args: '--enable-hdfs --enable-static-tiledb --disable-werror'
77+
6878
create_issue_on_fail:
6979
permissions:
7080
issues: write
7181
runs-on: ubuntu-latest
72-
needs: test
82+
needs:
83+
- test
84+
- test_hdfs
7385
if: failure() || cancelled()
7486
steps:
7587
- name: Checkout TileDB `dev`

0 commit comments

Comments
 (0)