diff --git a/.gitignore b/.gitignore index 797098608..89fb9fe2f 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ __pycache__ # meson subprojects - wrap files need to be kept to let meson download # dependencies as needed, but dependencies themselves should not be versioned subprojects/* +!subprojects/packagefiles !subprojects/*.wrap compile_commands.json diff --git a/ci/scripts/build-with-meson.sh b/ci/scripts/build-with-meson.sh index 41592121a..cd76c6db8 100755 --- a/ci/scripts/build-with-meson.sh +++ b/ci/scripts/build-with-meson.sh @@ -68,12 +68,12 @@ function main() { show_header "Run test suite" meson configure -Dtests=true -Db_coverage=true meson compile - meson test --wrap valgrind + meson test --wrap='valgrind --track-origins=yes --leak-check=full' --print-errorlogs show_header "Run benchmarks" meson configure -Dbenchmarks=true meson compile - meson test --benchmark + meson test --benchmark --print-errorlogs show_header "Generate coverage reports" ninja coverage diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 26a8b0f61..6372a56ac 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -15,6 +15,3 @@ dist/flatcc.c src/nanoarrow/nanoarrow_ipc_flatcc_generated.h thirdparty/* python/src/nanoarrow/dlpack_abi.h -subprojects/google-benchmark.wrap -subprojects/gtest.wrap -subprojects/nlohmann_json.wrap diff --git a/meson.build b/meson.build index c56bf4bb1..bd918d011 100644 --- a/meson.build +++ b/meson.build @@ -34,3 +34,14 @@ subdir('src/nanoarrow') if get_option('benchmarks') subdir('dev/benchmarks') endif + + +if get_option('apps') + if get_option('ipc') + executable( + 'dump_stream', + 'src/apps/dump_stream.c', + dependencies: [nanoarrow_dep, nanoarrow_ipc_dep] + ) + endif +endif diff --git a/meson.options b/meson.options index 73cbccd02..b91509820 100644 --- a/meson.options +++ b/meson.options @@ -17,6 +17,8 @@ option('tests', type: 'boolean', description: 'Build tests', value: false) option('benchmarks', type: 'boolean', description: 'Build benchmarks', value: false) +option('apps', type: 'boolean', description: 'Build utility applications', value: false) +option('ipc', type: 'boolean', description: 'Build IPC libraries', value: false) option('integration_tests', type: 'boolean', description: 'Build cross-implementation Arrow integration tests', value: false) diff --git a/src/apps/dump_stream.c b/src/apps/dump_stream.c index f26e566fe..ba1baec4d 100644 --- a/src/apps/dump_stream.c +++ b/src/apps/dump_stream.c @@ -23,7 +23,7 @@ void dump_schema_to_stdout(struct ArrowSchema* schema, int level, char* buf, int buf_size) { - int n_chars = ArrowSchemaToString(schema, buf, buf_size, 0); + ArrowSchemaToString(schema, buf, buf_size, 0); for (int i = 0; i < level; i++) { fprintf(stdout, " "); diff --git a/src/nanoarrow/meson.build b/src/nanoarrow/meson.build index f6f257161..ccf59c990 100644 --- a/src/nanoarrow/meson.build +++ b/src/nanoarrow/meson.build @@ -64,6 +64,26 @@ incdir = include_directories('..') nanoarrow_dep = declare_dependency(include_directories: [curdir, incdir], link_with: nanoarrow_lib) +if get_option('ipc') + cmake = import('cmake') + cmake_opts = cmake.subproject_options() + cmake_opts.add_cmake_defines({'CMAKE_POSITION_INDEPENDENT_CODE': true}) + flatcc_subproj = cmake.subproject('flatcc', options: cmake_opts) + flatcc_dep = flatcc_subproj.dependency('flatccrt') + + nanoarrow_ipc_lib = build_target( + 'nanoarrow_ipc', + 'nanoarrow_ipc_decoder.c', + 'nanoarrow_ipc_reader.c', + dependencies: [nanoarrow_dep, flatcc_dep], + install: true, + target_type: libtype, + ) + nanoarrow_ipc_dep = declare_dependency(include_directories: [incdir], + link_with: nanoarrow_ipc_lib, + dependencies: [nanoarrow_dep, flatcc_dep]) +endif + if get_option('tests') or get_option('integration_tests') nlohmann_json_dep = dependency('nlohmann_json') @@ -147,4 +167,36 @@ if get_option('tests') include_directories: incdir) test('c_data_integration test', c_data_integration_test) + if get_option('ipc') + zlib_dep = dependency('zlib') + ipc_test_files = { + 'nanoarrow-ipc-decoder': { + 'deps': [nanoarrow_ipc_dep, arrow_dep, gtest_dep], + }, + 'nanoarrow-ipc-reader': { + 'deps': [nanoarrow_ipc_dep, arrow_dep, gtest_dep], + }, + 'nanoarrow-ipc-files': { + 'deps': [ + nanoarrow_ipc_dep, + zlib_dep, + arrow_dep, + gtest_dep, + nlohmann_json_dep + ], + }, + 'nanoarrow-ipc-hpp': { + 'deps': [nanoarrow_ipc_dep, gtest_dep], + }, + } + + foreach name, config : ipc_test_files + exc = executable( + name + '-test', + name.replace('-', '_') + '_test.cc', + dependencies: config['deps'] + ) + test(name, exc) + endforeach + endif endif diff --git a/src/nanoarrow/nanoarrow.hpp b/src/nanoarrow/nanoarrow.hpp index 0de2371b4..5f8aabbac 100644 --- a/src/nanoarrow/nanoarrow.hpp +++ b/src/nanoarrow/nanoarrow.hpp @@ -864,8 +864,8 @@ class ViewArrayStream { }; internal::InputRange range_; - ArrowError* error_; ArrowErrorCode* code_; + ArrowError* error_; ArrowError internal_error_ = {}; ArrowErrorCode internal_code_; bool code_was_accessed_ = false; diff --git a/src/nanoarrow/nanoarrow_ipc_decoder_test.cc b/src/nanoarrow/nanoarrow_ipc_decoder_test.cc index db098dc76..4141b9b08 100644 --- a/src/nanoarrow/nanoarrow_ipc_decoder_test.cc +++ b/src/nanoarrow/nanoarrow_ipc_decoder_test.cc @@ -54,7 +54,6 @@ struct ArrowIpcDecoderPrivate { static enum ArrowIpcEndianness ArrowIpcSystemEndianness(void) { uint32_t check = 1; char first_byte; - enum ArrowIpcEndianness system_endianness; memcpy(&first_byte, &check, sizeof(char)); if (first_byte) { return NANOARROW_IPC_ENDIANNESS_LITTLE; @@ -501,8 +500,6 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeSimpleRecordBatchFromShared) { data.size_bytes = sizeof(kSimpleRecordBatch); ArrowIpcDecoderInit(&decoder); - auto decoder_private = - reinterpret_cast(decoder.private_data); ASSERT_EQ(ArrowIpcDecoderSetSchema(&decoder, &schema, nullptr), NANOARROW_OK); EXPECT_EQ(ArrowIpcDecoderDecodeHeader(&decoder, data, &error), NANOARROW_OK); @@ -602,8 +599,10 @@ TEST(NanoarrowIpcTest, NanoarrowIpcSharedBufferThreadSafeDecode) { std::thread threads[10]; for (int i = 0; i < 10; i++) { threads[i] = std::thread([&arrays, i, &one_two_three_le] { - memcmp(arrays[i].children[0]->buffers[1], one_two_three_le, - sizeof(one_two_three_le)); + auto result = memcmp(arrays[i].children[0]->buffers[1], one_two_three_le, + sizeof(one_two_three_le)); + // discard result to silence -Wunused-value + NANOARROW_UNUSED(result); ArrowArrayRelease(arrays + i); }); } @@ -791,7 +790,7 @@ TEST_P(ArrowTypeIdParameterizedTestFixture, NanoarrowIpcDecodeSwapEndian) { // Make a data buffer long enough for 10 Decimal256s with a pattern // where an endian swap isn't silently the same value (e.g., 0s) uint8_t data_buffer[32 * 10]; - for (int64_t i = 0; i < sizeof(data_buffer); i++) { + for (size_t i = 0; i < sizeof(data_buffer); i++) { data_buffer[i] = i % 256; } diff --git a/src/nanoarrow/nanoarrow_testing_test.cc b/src/nanoarrow/nanoarrow_testing_test.cc index 884210a67..ebd1853a5 100644 --- a/src/nanoarrow/nanoarrow_testing_test.cc +++ b/src/nanoarrow/nanoarrow_testing_test.cc @@ -470,7 +470,7 @@ TEST(NanoarrowTestingTest, NanoarrowTestingTestFieldMetadata) { NANOARROW_RETURN_NOT_OK(ArrowSchemaSetMetadata(schema, "\0\0\0\0")); return NANOARROW_OK; }, - [](ArrowArray* array) { return NANOARROW_OK; }, &WriteFieldJSON, + [](ArrowArray*) { return NANOARROW_OK; }, &WriteFieldJSON, R"({"name": null, "nullable": true, "type": {"name": "null"}, "children": []})", [](TestingJSONWriter& writer) { writer.set_include_metadata(false); }); } diff --git a/subprojects/flatcc.wrap b/subprojects/flatcc.wrap new file mode 100644 index 000000000..311425ae2 --- /dev/null +++ b/subprojects/flatcc.wrap @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[wrap-file] +directory = flatcc-0.6.1 +source_url = https://github.com/dvidelabs/flatcc/archive/refs/tags/v0.6.1.tar.gz +source_filename = flatcc-0.6.1.tar.gz +source_hash = 2533c2f1061498499f15acc7e0937dcf35bc68e685d237325124ae0d6c600c2b +patch_directory = flatcc diff --git a/subprojects/google-benchmark.wrap b/subprojects/google-benchmark.wrap index 91ff9528d..3b6704639 100644 --- a/subprojects/google-benchmark.wrap +++ b/subprojects/google-benchmark.wrap @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [wrap-file] directory = benchmark-1.8.4 source_url = https://github.com/google/benchmark/archive/refs/tags/v1.8.4.tar.gz diff --git a/subprojects/gtest.wrap b/subprojects/gtest.wrap index adb8a9a6d..ede25fc0b 100644 --- a/subprojects/gtest.wrap +++ b/subprojects/gtest.wrap @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [wrap-file] directory = googletest-1.14.0 source_url = https://github.com/google/googletest/archive/refs/tags/v1.14.0.tar.gz diff --git a/subprojects/nlohmann_json.wrap b/subprojects/nlohmann_json.wrap index bf5d7000b..28d4cbf53 100644 --- a/subprojects/nlohmann_json.wrap +++ b/subprojects/nlohmann_json.wrap @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [wrap-file] directory = nlohmann_json-3.11.2 lead_directory_missing = true diff --git a/subprojects/packagefiles/flatcc/meson.build b/subprojects/packagefiles/flatcc/meson.build new file mode 100644 index 000000000..9e12be21f --- /dev/null +++ b/subprojects/packagefiles/flatcc/meson.build @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +project('flatcc', + 'c', + version : '0.6.1', + license : 'Apache-2.0', +) + +incdir = include_directories('include') + +flatcc_lib = library( + 'flatcc', + sources: [ + 'src/runtime/builder.c', + 'src/runtime/emitter.c', + 'src/runtime/verifier.c', + 'src/runtime/refmap.c', + ], + include_directories: [incdir], +) + +flatcc_dep = declare_dependency( + include_directories: [incdir], + link_with: [flatcc_lib], +) diff --git a/subprojects/zlib.wrap b/subprojects/zlib.wrap new file mode 100644 index 000000000..f96fcef03 --- /dev/null +++ b/subprojects/zlib.wrap @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[wrap-file] +directory = zlib-1.3.1 +source_url = http://zlib.net/fossils/zlib-1.3.1.tar.gz +source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/zlib_1.3.1-1/zlib-1.3.1.tar.gz +source_filename = zlib-1.3.1.tar.gz +source_hash = 9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23 +patch_filename = zlib_1.3.1-1_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/zlib_1.3.1-1/get_patch +patch_hash = e79b98eb24a75392009cec6f99ca5cdca9881ff20bfa174e8b8926d5c7a47095 +wrapdb_version = 1.3.1-1 + +[provide] +zlib = zlib_dep