feat: Meson build system for nanoarrow-ipc extension (#483)

WillAyd · web-flow · commit 41e9e7183866 · 2024-06-11T17:23:58.000-03:00
diff --git a/.gitignore b/.gitignore
@@ -31,6 +31,7 @@ __pycache__
 # meson subprojects - wrap files need to be kept to let meson download
 # dependencies as needed, but dependencies themselves should not be versioned
 subprojects/*
+!subprojects/packagefiles
 !subprojects/*.wrap
 
 compile_commands.json
diff --git a/ci/scripts/build-with-meson.sh b/ci/scripts/build-with-meson.sh
@@ -68,12 +68,12 @@ function main() {
     show_header "Run test suite"
     meson configure -Dtests=true -Db_coverage=true
     meson compile
-    meson test --wrap valgrind
+    meson test --wrap='valgrind --track-origins=yes --leak-check=full' --print-errorlogs
 
     show_header "Run benchmarks"
     meson configure -Dbenchmarks=true
     meson compile
-    meson test --benchmark
+    meson test --benchmark --print-errorlogs
 
     show_header "Generate coverage reports"
     ninja coverage
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
@@ -15,6 +15,3 @@ dist/flatcc.c
 src/nanoarrow/nanoarrow_ipc_flatcc_generated.h
 thirdparty/*
 python/src/nanoarrow/dlpack_abi.h
-subprojects/google-benchmark.wrap
-subprojects/gtest.wrap
-subprojects/nlohmann_json.wrap
diff --git a/meson.build b/meson.build
@@ -34,3 +34,14 @@ subdir('src/nanoarrow')
 if get_option('benchmarks')
   subdir('dev/benchmarks')
 endif
+
+
+if get_option('apps')
+  if get_option('ipc')
+  executable(
+      'dump_stream',
+      'src/apps/dump_stream.c',
+      dependencies: [nanoarrow_dep, nanoarrow_ipc_dep]
+  )
+  endif
+endif
diff --git a/meson.options b/meson.options
@@ -17,6 +17,8 @@
 
 option('tests', type: 'boolean', description: 'Build tests', value: false)
 option('benchmarks', type: 'boolean', description: 'Build benchmarks', value: false)
+option('apps', type: 'boolean', description: 'Build utility applications', value: false)
+option('ipc', type: 'boolean', description: 'Build IPC libraries', value: false)
 option('integration_tests', type: 'boolean',
        description: 'Build cross-implementation Arrow integration tests',
        value: false)
diff --git a/src/apps/dump_stream.c b/src/apps/dump_stream.c
@@ -23,7 +23,7 @@
 
 void dump_schema_to_stdout(struct ArrowSchema* schema, int level, char* buf,
                            int buf_size) {
-  int n_chars = ArrowSchemaToString(schema, buf, buf_size, 0);
+  ArrowSchemaToString(schema, buf, buf_size, 0);
 
   for (int i = 0; i < level; i++) {
     fprintf(stdout, "  ");
diff --git a/src/nanoarrow/meson.build b/src/nanoarrow/meson.build
@@ -64,6 +64,26 @@ incdir = include_directories('..')
 nanoarrow_dep = declare_dependency(include_directories: [curdir, incdir],
                                    link_with: nanoarrow_lib)
 
+if get_option('ipc')
+    cmake = import('cmake')
+    cmake_opts = cmake.subproject_options()
+    cmake_opts.add_cmake_defines({'CMAKE_POSITION_INDEPENDENT_CODE': true})
+    flatcc_subproj = cmake.subproject('flatcc', options: cmake_opts)
+    flatcc_dep = flatcc_subproj.dependency('flatccrt')
+
+    nanoarrow_ipc_lib = build_target(
+        'nanoarrow_ipc',
+        'nanoarrow_ipc_decoder.c',
+        'nanoarrow_ipc_reader.c',
+        dependencies: [nanoarrow_dep, flatcc_dep],
+        install: true,
+        target_type: libtype,
+    )
+    nanoarrow_ipc_dep = declare_dependency(include_directories: [incdir],
+                                           link_with: nanoarrow_ipc_lib,
+                                           dependencies: [nanoarrow_dep, flatcc_dep])
+endif
+
 if get_option('tests') or get_option('integration_tests')
   nlohmann_json_dep = dependency('nlohmann_json')
 
@@ -147,4 +167,36 @@ if get_option('tests')
                                        include_directories: incdir)
   test('c_data_integration test', c_data_integration_test)
 
+  if get_option('ipc')
+      zlib_dep = dependency('zlib')
+      ipc_test_files = {
+          'nanoarrow-ipc-decoder': {
+              'deps': [nanoarrow_ipc_dep, arrow_dep, gtest_dep],
+          },
+          'nanoarrow-ipc-reader': {
+              'deps': [nanoarrow_ipc_dep, arrow_dep, gtest_dep],
+          },
+          'nanoarrow-ipc-files': {
+              'deps': [
+                  nanoarrow_ipc_dep,
+                  zlib_dep,
+                  arrow_dep,
+                  gtest_dep,
+                  nlohmann_json_dep
+              ],
+          },
+          'nanoarrow-ipc-hpp': {
+              'deps': [nanoarrow_ipc_dep, gtest_dep],
+          },
+      }
+
+      foreach name, config : ipc_test_files
+          exc = executable(
+              name + '-test',
+              name.replace('-', '_') + '_test.cc',
+              dependencies: config['deps']
+          )
+          test(name, exc)
+      endforeach
+  endif
 endif
diff --git a/src/nanoarrow/nanoarrow.hpp b/src/nanoarrow/nanoarrow.hpp
@@ -864,8 +864,8 @@ class ViewArrayStream {
   };
 
   internal::InputRange<Next> range_;
-  ArrowError* error_;
   ArrowErrorCode* code_;
+  ArrowError* error_;
   ArrowError internal_error_ = {};
   ArrowErrorCode internal_code_;
   bool code_was_accessed_ = false;
diff --git a/src/nanoarrow/nanoarrow_ipc_decoder_test.cc b/src/nanoarrow/nanoarrow_ipc_decoder_test.cc
@@ -54,7 +54,6 @@ struct ArrowIpcDecoderPrivate {
 static enum ArrowIpcEndianness ArrowIpcSystemEndianness(void) {
   uint32_t check = 1;
   char first_byte;
-  enum ArrowIpcEndianness system_endianness;
   memcpy(&first_byte, &check, sizeof(char));
   if (first_byte) {
     return NANOARROW_IPC_ENDIANNESS_LITTLE;
@@ -501,8 +500,6 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeSimpleRecordBatchFromShared) {
   data.size_bytes = sizeof(kSimpleRecordBatch);
 
   ArrowIpcDecoderInit(&decoder);
-  auto decoder_private =
-      reinterpret_cast<struct ArrowIpcDecoderPrivate*>(decoder.private_data);
 
   ASSERT_EQ(ArrowIpcDecoderSetSchema(&decoder, &schema, nullptr), NANOARROW_OK);
   EXPECT_EQ(ArrowIpcDecoderDecodeHeader(&decoder, data, &error), NANOARROW_OK);
@@ -602,8 +599,10 @@ TEST(NanoarrowIpcTest, NanoarrowIpcSharedBufferThreadSafeDecode) {
   std::thread threads[10];
   for (int i = 0; i < 10; i++) {
     threads[i] = std::thread([&arrays, i, &one_two_three_le] {
-      memcmp(arrays[i].children[0]->buffers[1], one_two_three_le,
-             sizeof(one_two_three_le));
+      auto result = memcmp(arrays[i].children[0]->buffers[1], one_two_three_le,
+                           sizeof(one_two_three_le));
+      // discard result to silence -Wunused-value
+      NANOARROW_UNUSED(result);
       ArrowArrayRelease(arrays + i);
     });
   }
@@ -791,7 +790,7 @@ TEST_P(ArrowTypeIdParameterizedTestFixture, NanoarrowIpcDecodeSwapEndian) {
   // Make a data buffer long enough for 10 Decimal256s with a pattern
   // where an endian swap isn't silently the same value (e.g., 0s)
   uint8_t data_buffer[32 * 10];
-  for (int64_t i = 0; i < sizeof(data_buffer); i++) {
+  for (size_t i = 0; i < sizeof(data_buffer); i++) {
     data_buffer[i] = i % 256;
   }
 
diff --git a/src/nanoarrow/nanoarrow_testing_test.cc b/src/nanoarrow/nanoarrow_testing_test.cc
@@ -470,7 +470,7 @@ TEST(NanoarrowTestingTest, NanoarrowTestingTestFieldMetadata) {
         NANOARROW_RETURN_NOT_OK(ArrowSchemaSetMetadata(schema, "\0\0\0\0"));
         return NANOARROW_OK;
       },
-      [](ArrowArray* array) { return NANOARROW_OK; }, &WriteFieldJSON,
+      [](ArrowArray*) { return NANOARROW_OK; }, &WriteFieldJSON,
       R"({"name": null, "nullable": true, "type": {"name": "null"}, "children": []})",
       [](TestingJSONWriter& writer) { writer.set_include_metadata(false); });
 }
diff --git a/subprojects/flatcc.wrap b/subprojects/flatcc.wrap
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[wrap-file]
+directory = flatcc-0.6.1
+source_url = https://github.com/dvidelabs/flatcc/archive/refs/tags/v0.6.1.tar.gz
+source_filename = flatcc-0.6.1.tar.gz
+source_hash = 2533c2f1061498499f15acc7e0937dcf35bc68e685d237325124ae0d6c600c2b
+patch_directory = flatcc
diff --git a/subprojects/google-benchmark.wrap b/subprojects/google-benchmark.wrap
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 [wrap-file]
 directory = benchmark-1.8.4
 source_url = https://github.com/google/benchmark/archive/refs/tags/v1.8.4.tar.gz
diff --git a/subprojects/gtest.wrap b/subprojects/gtest.wrap
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 [wrap-file]
 directory = googletest-1.14.0
 source_url = https://github.com/google/googletest/archive/refs/tags/v1.14.0.tar.gz
diff --git a/subprojects/nlohmann_json.wrap b/subprojects/nlohmann_json.wrap
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 [wrap-file]
 directory = nlohmann_json-3.11.2
 lead_directory_missing = true
diff --git a/subprojects/packagefiles/flatcc/meson.build b/subprojects/packagefiles/flatcc/meson.build
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+project('flatcc',
+  'c',
+  version : '0.6.1',
+  license : 'Apache-2.0',
+)
+
+incdir = include_directories('include')
+
+flatcc_lib = library(
+    'flatcc',
+    sources: [
+        'src/runtime/builder.c',
+        'src/runtime/emitter.c',
+        'src/runtime/verifier.c',
+        'src/runtime/refmap.c',
+    ],
+    include_directories: [incdir],
+)
+
+flatcc_dep = declare_dependency(
+    include_directories: [incdir],
+    link_with: [flatcc_lib],
+)
diff --git a/subprojects/zlib.wrap b/subprojects/zlib.wrap
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[wrap-file]
+directory = zlib-1.3.1
+source_url = http://zlib.net/fossils/zlib-1.3.1.tar.gz
+source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/zlib_1.3.1-1/zlib-1.3.1.tar.gz
+source_filename = zlib-1.3.1.tar.gz
+source_hash = 9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23
+patch_filename = zlib_1.3.1-1_patch.zip
+patch_url = https://wrapdb.mesonbuild.com/v2/zlib_1.3.1-1/get_patch
+patch_hash = e79b98eb24a75392009cec6f99ca5cdca9881ff20bfa174e8b8926d5c7a47095
+wrapdb_version = 1.3.1-1
+
+[provide]
+zlib = zlib_dep

Original file line number	Diff line number	Diff line change
`@@ -470,7 +470,7 @@ TEST(NanoarrowTestingTest, NanoarrowTestingTestFieldMetadata) {`
`470`	`470`	`NANOARROW_RETURN_NOT_OK(ArrowSchemaSetMetadata(schema, "\0\0\0\0"));`
`471`	`471`	`return NANOARROW_OK;`
`472`	`472`	`},`
`473`		`- [](ArrowArray* array) { return NANOARROW_OK; }, &WriteFieldJSON,`
	`473`	`+ [](ArrowArray*) { return NANOARROW_OK; }, &WriteFieldJSON,`
`474`	`474`	`R"({"name": null, "nullable": true, "type": {"name": "null"}, "children": []})",`
`475`	`475`	`[](TestingJSONWriter& writer) { writer.set_include_metadata(false); });`
`476`	`476`	`}`