refactor: Consolidate IPC extension into main project (#511)

Author: WillAyd

.gitattributes

@@ -16,5 +16,5 @@
 # under the License.
 
 dist/* linguist-generated
-extensions/nanoarrow_ipc/thirdparty/* linguist-vendored
+thirdparty/* linguist-vendored
 python/src/nanoarrow/*.pxi linguist-generated

.github/workflows/build-and-test-ipc.yaml

@@ -28,7 +28,6 @@ on:
       - 'CMakeLists.txt'
       - '.github/workflows/build-and-test-ipc.yaml'
       - 'src/nanoarrow/**'
-      - 'extensions/nanoarrow_ipc/**'
 
 permissions:
   contents: read
@@ -44,13 +43,13 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - {label: default-build, cmake_args: "-DNANOARROW_IPC_BUILD_APPS=ON"}
+          - {label: default-build, cmake_args: "-DNANOARROW_BUILD_APPS=ON"}
           - {label: default-noatomics, cmake_args: "-DCMAKE_C_FLAGS='-DNANOARROW_IPC_USE_STDATOMIC=0'"}
           - {label: namespaced-build, cmake_args: "-DNANOARROW_NAMESPACE=SomeUserNamespace"}
-          - {label: bundled-build, cmake_args: "-DNANOARROW_IPC_BUNDLE=ON"}
+          - {label: bundled-build, cmake_args: "-DNANOARROW_BUNDLE=ON"}
 
     env:
-      SUBDIR: 'extensions/nanoarrow_ipc'
+      SUBDIR: '${{ github.workspace }}'
       NANOARROW_ARROW_TESTING_DIR: '${{ github.workspace }}/arrow-testing'
 
     steps:
@@ -91,8 +90,9 @@ jobs:
           mkdir build
           cd build
 
-          cmake .. -DCMAKE_BUILD_TYPE=Debug -DNANOARROW_IPC_BUILD_TESTS=ON \
-            -DCMAKE_PREFIX_PATH="${ARROW_PATH}" ${{ matrix.config.cmake_args }}
+          cmake .. -DCMAKE_BUILD_TYPE=Debug -DNANOARROW_BUILD_TESTS=ON \
+            -DNANOARROW_IPC=ON -DCMAKE_PREFIX_PATH="${ARROW_PATH}" \
+            ${{ matrix.config.cmake_args }}
 
           cmake --build .
 
@@ -147,4 +147,4 @@ jobs:
         uses: actions/upload-artifact@main
         with:
           name: nanoarrow-ipc-memcheck
-          path: extensions/nanoarrow_ipc/build/Testing/Temporary/MemoryChecker.*.log
+          path: build/Testing/Temporary/MemoryChecker.*.log

.github/workflows/bundle.yaml

@@ -44,11 +44,11 @@ jobs:
 
       - name: Bundle nanoarrow_ipc
         run: |
-          cd extensions/nanoarrow_ipc
+          rm -rf build
           mkdir build && cd build
-          cmake .. -DNANOARROW_IPC_BUNDLE=ON
+          cmake .. -DNANOARROW_BUNDLE=ON -DNANOARROW_IPC=ON
           cmake --build .
-          cmake --install . --prefix=../../../nanoarrow-latest
+          cmake --install . --prefix=../nanoarrow-latest
 
       - name: Bundle nanoarrow_device
         run: |

.github/workflows/coverage.yaml

@@ -31,7 +31,6 @@ on:
       - 'docker-compose.yml'
       - 'CMakeLists.txt'
       - 'src/nanoarrow/**'
-      - 'extensions/nanoarrow_ipc/**'
       - 'r/**'
 
 jobs:

.github/workflows/docs.yaml

@@ -29,7 +29,6 @@ on:
       - 'ci/scripts/coverage.sh'
       - 'ci/docker/ubuntu.dockerfile'
       - 'docker-compose.yml'
-      - 'extensions/nanoarrow_ipc/src/**'
       - 'src/**'
       - 'r/**'
       - 'docs/source/**'

.github/workflows/examples.yaml

@@ -90,7 +90,7 @@ jobs:
           cmake --build .
           cmake --install . --prefix=../src/nanoarrow
           mkdir ../build_ipc && cd ../build_ipc
-          cmake ../../../extensions/nanoarrow_ipc -DNANOARROW_IPC_BUNDLE=ON
+          cmake ../../.. -DNANOARROW_BUNDLE=ON -DNANOARROW_IPC=ON
           cmake --build .
           cmake --install . --prefix=../src/nanoarrow
 

.pre-commit-config.yaml

@@ -35,7 +35,7 @@ repos:
       - id: clang-format
         args: [-i]
         types_or: [c, c++]
-        exclude: "(^extensions/nanoarrow_ipc/thirdparty/.*$)|(nanoarrow_ipc_flatcc_generated.h)"
+        exclude: "(^thirdparty/.*$)|(nanoarrow_ipc_flatcc_generated.h)"
   - repo: https://github.com/cheshirekow/cmake-format-precommit
     rev: v0.6.13
     hooks:
@@ -65,7 +65,7 @@ repos:
         additional_dependencies: [tomli]
         exclude: |
             (?x)
-            ^extensions/nanoarrow_ipc/thirdparty
+            ^thirdparty
             |nanoarrow_ipc_flatcc_generated.h
             |nanoarrow_device_metal.cc
 

CMakeLists.txt

@@ -34,9 +34,15 @@ set(NANOARROW_VERSION_PATCH "${nanoarrow_VERSION_PATCH}")
 
 # Feature options
 option(NANOARROW_IPC "Build IPC extension" OFF)
+option(NANOARROW_FLATCC_ROOT_DIR "Root directory for flatcc include and lib directories"
+       OFF)
+option(NANOARROW_FLATCC_INCLUDE_DIR "Include directory for flatcc includes" OFF)
+option(NANOARROW_FLATCC_LIB_DIR "Library directory that contains libflatccrt.a" OFF)
+
 option(NANOARROW_DEVICE "Build device extension" OFF)
 
 # Development options
+option(NANOARROW_BUILD_APPS "Build utility applications" OFF)
 option(NANOARROW_BUILD_TESTS "Build tests" OFF)
 option(NANOARROW_BUILD_BENCHMARKS "Build benchmarks" OFF)
 option(NANOARROW_BUILD_INTEGRATION_TESTS
@@ -47,6 +53,10 @@ option(NANOARROW_NAMESPACE "A prefix for exported symbols" OFF)
 option(NANOARROW_ARROW_STATIC
        "Use a statically-linked Arrow C++ build when linking tests" OFF)
 
+if(NANOARROW_IPC)
+  add_library(ipc_coverage_config INTERFACE)
+endif()
+
 if(NANOARROW_NAMESPACE)
   set(NANOARROW_NAMESPACE_DEFINE "#define NANOARROW_NAMESPACE ${NANOARROW_NAMESPACE}")
 else()
@@ -63,6 +73,46 @@ endif()
 
 configure_file(src/nanoarrow/nanoarrow_config.h.in generated/nanoarrow_config.h)
 
+if(NANOARROW_IPC AND (NANOARROW_BUILD_TESTS OR NOT NANOARROW_BUNDLE))
+  # Add the flatcc (runtime) dependency
+  set(FLATCC_RTONLY
+      ON
+      CACHE INTERNAL "")
+  set(FLATCC_REFLECTION
+      OFF
+      CACHE INTERNAL "")
+
+  # A flatcc installation is unlikely, so default to building the vendored one
+  if(NOT NANOARROW_FLATCC_INCLUDE_DIR AND NOT NANOARROW_FLATCC_ROOT_DIR)
+    add_library(flatccrt STATIC
+                thirdparty/flatcc/src/runtime/builder.c
+                thirdparty/flatcc/src/runtime/emitter.c
+                thirdparty/flatcc/src/runtime/verifier.c
+                thirdparty/flatcc/src/runtime/refmap.c)
+
+    set(NANOARROW_FLATCC_INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/thirdparty/flatcc/include)
+    target_include_directories(flatccrt
+                               PUBLIC $<BUILD_INTERFACE:${NANOARROW_FLATCC_INCLUDE_DIR}>
+                                      $<INSTALL_INTERFACE:include>)
+  elseif(NOT NANOARROW_FLATCC_ROOT_DIR)
+    add_library(flatccrt STATIC IMPORTED)
+    set_target_properties(flatccrt
+                          PROPERTIES IMPORTED_LOCATION
+                                     "${NANOARROW_FLATCC_LIB_DIR}/libflatccrt.a"
+                                     INTERFACE_INCLUDE_DIRECTORIES
+                                     "${NANOARROW_FLATCC_INCLUDE_DIR}")
+
+  elseif(NOT NANOARROW_FLATCC_INCLUDE_DIR)
+    set(NANOARROW_FLATCC_INCLUDE_DIR ${NANOARROW_FLATCC_ROOT_DIR}/include)
+    add_library(flatccrt STATIC IMPORTED)
+    set_target_properties(flatccrt
+                          PROPERTIES IMPORTED_LOCATION
+                                     "${NANOARROW_FLATCC_ROOT_DIR}/lib/libflatccrt.a"
+                                     INTERFACE_INCLUDE_DIRECTORIES
+                                     "${NANOARROW_FLATCC_INCLUDE_DIR}")
+  endif()
+endif()
+
 if(NANOARROW_BUNDLE)
   # Combine all headers into amalgamation/nanoarrow.h in the build directory
   file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/amalgamation)
@@ -133,6 +183,60 @@ if(NANOARROW_BUNDLE)
                 ${NANOARROW_GTEST_UTIL_HPP_TEMP}
                 ${NANOARROW_TESTING_HPP_TEMP}
           DESTINATION ".")
+
+  if(NANOARROW_IPC)
+    # nanoarrow_ipc.h is already standalone
+    set(NANOARROW_IPC_H_TEMP ${CMAKE_BINARY_DIR}/amalgamation/nanoarrow/nanoarrow_ipc.h)
+    file(READ src/nanoarrow/nanoarrow_ipc.h SRC_FILE_CONTENTS)
+    file(WRITE ${NANOARROW_IPC_H_TEMP} "${SRC_FILE_CONTENTS}")
+
+    set(NANOARROW_IPC_C_TEMP ${CMAKE_BINARY_DIR}/amalgamation/nanoarrow/nanoarrow_ipc.c)
+    file(READ src/nanoarrow/nanoarrow_ipc_flatcc_generated.h SRC_FILE_CONTENTS)
+    file(WRITE ${NANOARROW_IPC_C_TEMP} "${SRC_FILE_CONTENTS}")
+    file(READ src/nanoarrow/nanoarrow_ipc_decoder.c SRC_FILE_CONTENTS)
+    file(APPEND ${NANOARROW_IPC_C_TEMP} "${SRC_FILE_CONTENTS}")
+    file(READ src/nanoarrow/nanoarrow_ipc_reader.c SRC_FILE_CONTENTS)
+    file(APPEND ${NANOARROW_IPC_C_TEMP} "${SRC_FILE_CONTENTS}")
+
+    # remove the include for the generated files in the bundled version
+    file(READ ${NANOARROW_IPC_C_TEMP} SRC_FILE_CONTENTS)
+    string(REGEX REPLACE "#include \"nanoarrow_ipc_flatcc_generated.h\"" ""
+                         SRC_FILE_CONTENTS "${SRC_FILE_CONTENTS}")
+    file(WRITE ${NANOARROW_IPC_C_TEMP} "${SRC_FILE_CONTENTS}")
+
+    # combine the flatcc sources
+    set(FLATCC_C_TEMP ${CMAKE_BINARY_DIR}/amalgamation/nanoarrow/flatcc.c)
+    file(READ thirdparty/flatcc/src/runtime/builder.c SRC_FILE_CONTENTS)
+    file(WRITE ${FLATCC_C_TEMP} "${SRC_FILE_CONTENTS}")
+    file(READ thirdparty/flatcc/src/runtime/emitter.c SRC_FILE_CONTENTS)
+    file(APPEND ${FLATCC_C_TEMP} "${SRC_FILE_CONTENTS}")
+    file(READ thirdparty/flatcc/src/runtime/verifier.c SRC_FILE_CONTENTS)
+    file(APPEND ${FLATCC_C_TEMP} "${SRC_FILE_CONTENTS}")
+    file(READ thirdparty/flatcc/src/runtime/refmap.c SRC_FILE_CONTENTS)
+    file(APPEND ${FLATCC_C_TEMP} "${SRC_FILE_CONTENTS}")
+
+    # Add a library that the tests can link against (but don't install it)
+    if(NANOARROW_BUILD_TESTS)
+      # Bundle flatcc into the nanoarrow_ipc library instead of
+      # link to the flatccrt static lib we declared above.
+      add_library(nanoarrow_ipc ${CMAKE_BINARY_DIR}/amalgamation/nanoarrow/nanoarrow_ipc.c
+                                ${CMAKE_BINARY_DIR}/amalgamation/nanoarrow/flatcc.c)
+
+      target_include_directories(nanoarrow_ipc
+                                 PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src>
+                                        $<BUILD_INTERFACE:${nanoarrow_SOURCE_DIR}/src/nanoarrow>
+                                        $<BUILD_INTERFACE:${nanoarrow_BINARY_DIR}/generated>
+                                        $<BUILD_INTERFACE:${NANOARROW_FLATCC_INCLUDE_DIR}>
+      )
+    endif()
+
+    # Install the amalgamated header and sources
+    install(FILES ${NANOARROW_IPC_H_TEMP} ${NANOARROW_IPC_C_TEMP} ${FLATCC_C_TEMP}
+            DESTINATION ".")
+
+    # Also install the flatcc headers
+    install(DIRECTORY thirdparty/flatcc/include/flatcc DESTINATION ".")
+  endif()
 else()
   add_library(nanoarrow src/nanoarrow/array.c src/nanoarrow/schema.c
                         src/nanoarrow/array_stream.c src/nanoarrow/utils.c)
@@ -215,12 +319,31 @@ else()
               NAMESPACE nanoarrow::)
     endif()
   endforeach()
+
+  if(NANOARROW_IPC)
+    add_library(nanoarrow_ipc src/nanoarrow/nanoarrow_ipc_decoder.c
+                              src/nanoarrow/nanoarrow_ipc_reader.c)
+    target_link_libraries(nanoarrow_ipc PRIVATE flatccrt)
+
+    target_include_directories(nanoarrow_ipc
+                               PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src>
+                                      $<BUILD_INTERFACE:${nanoarrow_SOURCE_DIR}/src/nanoarrow>
+                                      $<BUILD_INTERFACE:${nanoarrow_BINARY_DIR}/generated>
+                                      $<BUILD_INTERFACE:${NANOARROW_IPC_FLATCC_INCLUDE_DIR}>
+                                      $<INSTALL_INTERFACE:include>)
+
+    install(TARGETS nanoarrow_ipc DESTINATION lib)
+    install(FILES src/nanoarrow/nanoarrow_ipc.h
+                  src/nanoarrow/nanoarrow_ipc_flatcc_generated.h
+            DESTINATION include/nanoarrow)
+  endif()
 endif()
 
 # Always build integration test if building tests
 if(NANOARROW_BUILD_TESTS OR NANOARROW_BUILD_INTEGRATION_TESTS)
   add_subdirectory("thirdparty/nlohmann_json")
 
+  set_target_properties(nanoarrow PROPERTIES POSITION_INDEPENDENT_CODE ON)
   add_library(nanoarrow_c_data_integration SHARED
               src/nanoarrow/integration/c_data_integration.cc)
   target_include_directories(nanoarrow_c_data_integration
@@ -310,10 +433,73 @@ if(NANOARROW_BUILD_TESTS)
   gtest_discover_tests(nanoarrow_hpp_test DISCOVERY_TIMEOUT 10)
   gtest_discover_tests(nanoarrow_testing_test DISCOVERY_TIMEOUT 10)
   gtest_discover_tests(c_data_integration_test DISCOVERY_TIMEOUT 10)
+
+  if(NANOARROW_IPC)
+
+    # zlib to decode gzipped integration testing JSON files
+    # We don't use Arrow C++ for this because building Arrow C++ with zlib
+    # is not trivial on Windows.
+    find_package(ZLIB)
+    if(NOT ZLIB_FOUND)
+      # Wrapper around FetchContent that better isolates the zlib CMakeLists.txt
+      message(STATUS "Using FetchContent to build a static zlib")
+      add_subdirectory(thirdparty/zlib)
+    endif()
+
+    enable_testing()
+
+    add_executable(nanoarrow_ipc_decoder_test src/nanoarrow/nanoarrow_ipc_decoder_test.cc)
+    add_executable(nanoarrow_ipc_reader_test src/nanoarrow/nanoarrow_ipc_reader_test.cc)
+    add_executable(nanoarrow_ipc_files_test src/nanoarrow/nanoarrow_ipc_files_test.cc)
+    add_executable(nanoarrow_ipc_hpp_test src/nanoarrow/nanoarrow_ipc_hpp_test.cc)
+
+    if(NANOARROW_CODE_COVERAGE)
+      target_compile_options(ipc_coverage_config INTERFACE -O0 -g --coverage)
+      target_link_options(ipc_coverage_config INTERFACE --coverage)
+      target_link_libraries(nanoarrow_ipc PRIVATE ipc_coverage_config)
+    endif()
+    target_link_libraries(nanoarrow_ipc_decoder_test
+                          nanoarrow_ipc
+                          nanoarrow
+                          flatccrt
+                          ${NANOARROW_ARROW_TARGET}
+                          gtest_main
+                          ipc_coverage_config)
+    target_link_libraries(nanoarrow_ipc_reader_test
+                          nanoarrow_ipc
+                          nanoarrow
+                          flatccrt
+                          gtest_main
+                          ipc_coverage_config)
+    target_link_libraries(nanoarrow_ipc_files_test
+                          nanoarrow_ipc
+                          nanoarrow
+                          flatccrt
+                          ${NANOARROW_ARROW_TARGET}
+                          nlohmann_json
+                          ZLIB::ZLIB
+                          gtest_main
+                          ipc_coverage_config)
+    target_link_libraries(nanoarrow_ipc_hpp_test
+                          nanoarrow_ipc
+                          nanoarrow
+                          ${NANOARROW_ARROW_TARGET}
+                          gtest_main
+                          ipc_coverage_config)
+
+    include(GoogleTest)
+    gtest_discover_tests(nanoarrow_ipc_decoder_test)
+    gtest_discover_tests(nanoarrow_ipc_reader_test)
+    gtest_discover_tests(nanoarrow_ipc_files_test)
+    gtest_discover_tests(nanoarrow_ipc_hpp_test)
+  endif()
 endif()
 
-if(NANOARROW_BUILD_BENCHMARKS OR NANOARROW_IPC)
-  add_subdirectory(extensions/nanoarrow_ipc)
+if(NANOARROW_BUILD_APPS)
+  if(NANOARROW_IPC)
+    add_executable(dump_stream src/apps/dump_stream.c)
+    target_link_libraries(dump_stream nanoarrow_ipc nanoarrow)
+  endif()
 endif()
 
 if(NANOARROW_DEVICE)

LICENSE.txt

@@ -203,7 +203,6 @@
 
 --------------------------------------------------------------------------------
 
-extensions/nanoarrow_ipc/thirdparty/flatcc, dist/flatcc.c,
-  and dist/flatcc: Apache 2.0
+thirdparty/flatcc, dist/flatcc.c, and dist/flatcc: Apache 2.0
 
 Copyright 2015-2023 Mikkel F. Jørgensen, dvide.com

ci/scripts/build-docs.sh

@@ -72,11 +72,6 @@ main() {
    doxygen
    popd
 
-   show_header "Run Doxygen for IPC extension"
-   pushd extensions/nanoarrow_ipc/src/apidoc
-   doxygen
-   popd
-
    show_header "Run Doxygen for device extension"
    pushd extensions/nanoarrow_device/src/apidoc
    doxygen

ci/scripts/coverage.sh

@@ -67,25 +67,13 @@ function main() {
     fi
     mkdir "${SANDBOX_DIR}"
 
-    # Bulid + run tests with gcov for core library
+    # Bulid + run tests with gcov
     show_header "Build + test nanoarrow"
     mkdir "${SANDBOX_DIR}/nanoarrow"
     pushd "${SANDBOX_DIR}/nanoarrow"
 
     cmake "${TARGET_NANOARROW_DIR}" \
-        -DNANOARROW_BUILD_TESTS=ON -DNANOARROW_CODE_COVERAGE=ON
-    cmake --build .
-    CTEST_OUTPUT_ON_FAILURE=1 ctest .
-
-    popd
-
-    # Build + run tests with gcov for IPC extension
-    show_header "Build + test nanoarrow_ipc"
-    mkdir "${SANDBOX_DIR}/nanoarrow_ipc"
-    pushd "${SANDBOX_DIR}/nanoarrow_ipc"
-
-    cmake "${TARGET_NANOARROW_DIR}/extensions/nanoarrow_ipc" \
-        -DNANOARROW_IPC_BUILD_TESTS=ON -DNANOARROW_IPC_CODE_COVERAGE=ON
+        -DNANOARROW_IPC=ON -DNANOARROW_BUILD_TESTS=ON -DNANOARROW_CODE_COVERAGE=ON
     cmake --build .
     CTEST_OUTPUT_ON_FAILURE=1 ctest .