refactor: Consolidate device extension into main project (#517)

.github/workflows/build-and-test-device.yaml

@@ -28,7 +28,6 @@ on:
       - 'CMakeLists.txt'
       - '.github/workflows/build-and-test-device.yaml'
       - 'src/nanoarrow/**'
-      - 'extensions/nanoarrow_device/**'
 
 permissions:
   contents: read
@@ -46,28 +45,52 @@ jobs:
         config:
           - {label: default-build}
           - {label: namespaced-build, cmake_args: "-DNANOARROW_NAMESPACE=SomeUserNamespace"}
-          - {label: bundled-build, cmake_args: "-DNANOARROW_DEVICE_BUNDLE=ON"}
+          - {label: bundled-build, cmake_args: "-DNANOARROW_BUNDLE=ON"}
 
     env:
-      SUBDIR: 'extensions/nanoarrow_device'
+      SUBDIR: '${{ github.workspace }}'
       NANOARROW_ARROW_TESTING_DIR: '${{ github.workspace }}/arrow-testing'
 
     steps:
       - uses: actions/checkout@v4
 
+      - name: Checkout arrow-testing
+        uses: actions/checkout@v4
+        with:
+          repository: apache/arrow-testing
+          path: arrow-testing
+
       - name: Install memcheck dependencies
         if: matrix.config.label == 'default-build'
         run: |
           sudo apt-get update && sudo apt-get install -y valgrind
 
+      - name: Cache Arrow C++ Build
+        id: cache-arrow-build
+        uses: actions/cache@v4
+        with:
+          path: arrow
+          # Bump the number at the end of this line to force a new Arrow C++ build
+          key: arrow-${{ runner.os }}-${{ runner.arch }}-1
+
+      - name: Build Arrow C++
+        if: steps.cache-arrow-build.outputs.cache-hit != 'true'
+        shell: bash
+        run: |
+          ci/scripts/build-arrow-cpp-minimal.sh 15.0.2 arrow
+
       - name: Build
         run: |
+          ARROW_PATH="$(pwd)/arrow"
           cd $SUBDIR
           export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:`pwd`/dist/lib
           sudo ldconfig
           mkdir build
           cd build
-          cmake .. -DCMAKE_BUILD_TYPE=Debug -DNANOARROW_DEVICE_BUILD_TESTS=ON ${{ matrix.config.cmake_args }}
+          cmake .. -DCMAKE_BUILD_TYPE=Debug -DNANOARROW_DEVICE=ON \
+            -DNANOARROW_BUILD_TESTS=ON -DCMAKE_PREFIX_PATH="${ARROW_PATH}" \
+            ${{ matrix.config.cmake_args }}
+
           cmake --build .
 
       - name: Check for non-namespaced symbols in namespaced build
@@ -112,4 +135,4 @@ jobs:
         uses: actions/upload-artifact@main
         with:
           name: nanoarrow-device-memcheck
-          path: extensions/nanoarrow_device/build/Testing/Temporary/MemoryChecker.*.log
+          path: build/Testing/Temporary/MemoryChecker.*.log

.github/workflows/bundle.yaml

@@ -52,11 +52,11 @@ jobs:
 
       - name: Bundle nanoarrow_device
         run: |
-          cd extensions/nanoarrow_device
+          rm -rf build
           mkdir build && cd build
-          cmake .. -DNANOARROW_DEVICE_BUNDLE=ON
+          cmake .. -DNANOARROW_BUNDLE=ON -DNANOARROW_DEVICE=ON
           cmake --build .
-          cmake --install . --prefix=../../../nanoarrow-latest
+          cmake --install . --prefix=../nanoarrow-latest
 
       - name: Compress bundle
         run: |

CMakeLists.txt

@@ -53,10 +53,24 @@ option(NANOARROW_NAMESPACE "A prefix for exported symbols" OFF)
 option(NANOARROW_ARROW_STATIC
        "Use a statically-linked Arrow C++ build when linking tests" OFF)
 
+option(NANOARROW_DEVICE_WITH_METAL "Build Apple metal libraries" OFF)
+option(NANOARROW_DEVICE_WITH_CUDA "Build CUDA libraries" OFF)
+
+set(NANOARROW_DEVICE
+    NANOARROW_DEVICE
+    OR
+    NANOARROW_DEVICE_WITH_MDETAL
+    OR
+    NANOARROW_DEVICE_WITH_CUDA)
+
 if(NANOARROW_IPC)
   add_library(ipc_coverage_config INTERFACE)
 endif()
 
+if(NANOARROW_DEVICE)
+  add_library(device_coverage_config INTERFACE)
+endif()
+
 if(NANOARROW_NAMESPACE)
   set(NANOARROW_NAMESPACE_DEFINE "#define NANOARROW_NAMESPACE ${NANOARROW_NAMESPACE}")
 else()
@@ -237,6 +251,39 @@ if(NANOARROW_BUNDLE)
     # Also install the flatcc headers
     install(DIRECTORY thirdparty/flatcc/include/flatcc DESTINATION ".")
   endif()
+
+  if(NANOARROW_DEVICE)
+    # The CMake build step is creating nanoarrow_device.c and nanoarrow_device.h;
+    # the CMake install step is copying them to a specific location
+    file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/amalgamation)
+    file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/amalgamation/nanoarrow)
+
+    # nanoarrow_device.h is currently standalone
+    set(NANOARROW_DEVICE_H_TEMP
+        ${CMAKE_BINARY_DIR}/amalgamation/nanoarrow/nanoarrow_device.h)
+    file(READ src/nanoarrow/nanoarrow_device.h SRC_FILE_CONTENTS)
+    file(WRITE ${NANOARROW_DEVICE_H_TEMP} "${SRC_FILE_CONTENTS}")
+
+    # nanoarrow_device.c is currently standalone
+    set(NANOARROW_DEVICE_C_TEMP
+        ${CMAKE_BINARY_DIR}/amalgamation/nanoarrow/nanoarrow_device.c)
+    file(READ src/nanoarrow/nanoarrow_device.c SRC_FILE_CONTENTS)
+    file(WRITE ${NANOARROW_DEVICE_C_TEMP} "${SRC_FILE_CONTENTS}")
+
+    # Add a library that the tests can link against (but don't install it)
+    if(NANOARROW_BUILD_TESTS)
+      add_library(nanoarrow_device ${NANOARROW_DEVICE_C_TEMP})
+
+      target_include_directories(nanoarrow_device
+                                 PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src>
+                                        $<BUILD_INTERFACE:${nanoarrow_SOURCE_DIR}/src/nanoarrow>
+                                        $<BUILD_INTERFACE:${nanoarrow_BINARY_DIR}/generated>
+      )
+    endif()
+
+    # Install the amalgamated header and sources
+    install(FILES ${NANOARROW_DEVICE_H_TEMP} ${NANOARROW_DEVICE_C_TEMP} DESTINATION ".")
+  endif()
 else()
   add_library(nanoarrow src/nanoarrow/array.c src/nanoarrow/schema.c
                         src/nanoarrow/array_stream.c src/nanoarrow/utils.c)
@@ -329,14 +376,78 @@ else()
                                PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src>
                                       $<BUILD_INTERFACE:${nanoarrow_SOURCE_DIR}/src/nanoarrow>
                                       $<BUILD_INTERFACE:${nanoarrow_BINARY_DIR}/generated>
-                                      $<BUILD_INTERFACE:${NANOARROW_IPC_FLATCC_INCLUDE_DIR}>
+                                      $<BUILD_INTERFACE:${NANOARROW_IPC_FLATCC_INCLUDE_DIR}
                                       $<INSTALL_INTERFACE:include>)
 
     install(TARGETS nanoarrow_ipc DESTINATION lib)
     install(FILES src/nanoarrow/nanoarrow_ipc.h
                   src/nanoarrow/nanoarrow_ipc_flatcc_generated.h
             DESTINATION include/nanoarrow)
   endif()
+
+  if(NANOARROW_DEVICE)
+    if(NANOARROW_DEVICE_WITH_METAL)
+      if(NOT EXISTS "${CMAKE_BINARY_DIR}/metal-cpp")
+        message(STATUS "Fetching metal-cpp")
+        file(DOWNLOAD
+             "https://developer.apple.com/metal/cpp/files/metal-cpp_macOS12_iOS15.zip"
+             "${CMAKE_BINARY_DIR}/metal-cpp.zip")
+        file(ARCHIVE_EXTRACT
+             INPUT
+             ${CMAKE_BINARY_DIR}/metal-cpp.zip
+             DESTINATION
+             ${CMAKE_BINARY_DIR})
+      endif()
+
+      if(NOT DEFINED CMAKE_CXX_STANDARD)
+        set(CMAKE_CXX_STANDARD 17)
+      endif()
+      set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+      find_library(METAL_LIBRARY Metal REQUIRED)
+      message(STATUS "Metal framework found at '${METAL_LIBRARY}'")
+
+      find_library(FOUNDATION_LIBRARY Foundation REQUIRED)
+      message(STATUS "Foundation framework found at '${FOUNDATION_LIBRARY}'")
+
+      find_library(QUARTZ_CORE_LIBRARY QuartzCore REQUIRED)
+      message(STATUS "CoreFoundation framework found at '${QUARTZ_CORE_LIBRARY}'")
+
+      set(NANOARROW_DEVICE_SOURCES_METAL src/nanoarrow/nanoarrow_device_metal.cc)
+      set(NANOARROW_DEVICE_INCLUDE_METAL ${CMAKE_BINARY_DIR}/metal-cpp)
+      set(NANOARROW_DEVICE_LIBS_METAL ${METAL_LIBRARY} ${FOUNDATION_LIBRARY}
+                                      ${QUARTZ_CORE_LIBRARY})
+      set(NANOARROW_DEVICE_DEFS_METAL "NANOARROW_DEVICE_WITH_METAL")
+    endif()
+
+    if(NANOARROW_DEVICE_WITH_CUDA)
+      find_package(CUDAToolkit REQUIRED)
+      set(NANOARROW_DEVICE_SOURCES_CUDA src/nanoarrow/nanoarrow_device_cuda.c)
+      set(NANOARROW_DEVICE_LIBS_CUDA CUDA::cuda_driver)
+      set(NANOARROW_DEVICE_DEFS_CUDA "NANOARROW_DEVICE_WITH_CUDA")
+    endif()
+
+    add_library(nanoarrow_device
+                src/nanoarrow/nanoarrow_device.c ${NANOARROW_DEVICE_SOURCES_METAL}
+                ${NANOARROW_DEVICE_SOURCES_CUDA})
+
+    target_include_directories(nanoarrow_device
+                               PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/nanoarrow>
+                                      $<BUILD_INTERFACE:${nanoarrow_SOURCE_DIR}/src/nanoarrow>
+                                      $<BUILD_INTERFACE:${nanoarrow_BINARY_DIR}/generated>
+                                      $<BUILD_INTERFACE:${NANOARROW_DEVICE_INCLUDE_METAL}>
+                                      $<INSTALL_INTERFACE:include>)
+
+    target_compile_definitions(nanoarrow_device PRIVATE ${NANOARROW_DEVICE_DEFS_METAL}
+                                                        ${NANOARROW_DEVICE_DEFS_CUDA})
+    target_link_libraries(nanoarrow_device PUBLIC ${NANOARROW_DEVICE_LIBS_METAL}
+                                                  ${NANOARROW_DEVICE_LIBS_CUDA})
+    target_compile_definitions(nanoarrow_device
+                               PUBLIC "$<$<CONFIG:Debug>:NANOARROW_DEBUG>")
+
+    install(TARGETS nanoarrow_device DESTINATION lib)
+    install(FILES src/nanoarrow/nanoarrow_device.h DESTINATION include/nanoarrow)
+  endif()
 endif()
 
 # Always build integration test if building tests
@@ -493,6 +604,55 @@ if(NANOARROW_BUILD_TESTS)
     gtest_discover_tests(nanoarrow_ipc_files_test)
     gtest_discover_tests(nanoarrow_ipc_hpp_test)
   endif()
+
+  if(NANOARROW_DEVICE)
+    enable_testing()
+    add_executable(nanoarrow_device_test src/nanoarrow/nanoarrow_device_test.cc)
+    add_executable(nanoarrow_device_hpp_test src/nanoarrow/nanoarrow_device_hpp_test.cc)
+
+    if(NANOARROW_DEVICE_CODE_COVERAGE)
+      target_compile_options(device_coverage_config INTERFACE -O0 -g --coverage)
+      target_link_options(device_coverage_config INTERFACE --coverage)
+      target_link_libraries(nanoarrow_device PRIVATE device_coverage_config)
+    endif()
+
+    target_link_libraries(nanoarrow_device_test
+                          nanoarrow_device
+                          nanoarrow
+                          gtest_main
+                          device_coverage_config)
+    target_link_libraries(nanoarrow_device_hpp_test
+                          nanoarrow_device
+                          nanoarrow
+                          gtest_main
+                          device_coverage_config)
+
+    include(GoogleTest)
+    gtest_discover_tests(nanoarrow_device_test)
+    gtest_discover_tests(nanoarrow_device_hpp_test)
+
+    if(NANOARROW_DEVICE_WITH_METAL)
+      add_executable(nanoarrow_device_metal_test
+                     src/nanoarrow/nanoarrow_device_metal_test.cc)
+      target_link_libraries(nanoarrow_device_metal_test
+                            nanoarrow_device
+                            nanoarrow
+                            gtest_main
+                            device_coverage_config)
+      gtest_discover_tests(nanoarrow_device_metal_test)
+    endif()
+
+    if(NANOARROW_DEVICE_WITH_CUDA)
+      add_executable(nanoarrow_device_cuda_test
+                     src/nanoarrow/nanoarrow_device_cuda_test.cc)
+      target_link_libraries(nanoarrow_device_cuda_test
+                            nanoarrow_device
+                            nanoarrow
+                            gtest_main
+                            device_coverage_config)
+      gtest_discover_tests(nanoarrow_device_cuda_test)
+    endif()
+  endif()
 endif()
 
 if(NANOARROW_BUILD_APPS)
@@ -502,10 +662,6 @@ if(NANOARROW_BUILD_APPS)
   endif()
 endif()
 
-if(NANOARROW_DEVICE)
-  add_subdirectory(extensions/nanoarrow_device)
-endif()
-
 if(NANOARROW_BUILD_BENCHMARKS)
   add_subdirectory(dev/benchmarks)
 endif()

ci/scripts/build-docs.sh

@@ -72,11 +72,6 @@ main() {
    doxygen
    popd
 
-   show_header "Run Doxygen for device extension"
-   pushd extensions/nanoarrow_device/src/apidoc
-   doxygen
-   popd
-
    show_header "Build nanoarrow Python"
 
    pushd python

ci/scripts/coverage.sh

@@ -73,21 +73,8 @@ function main() {
     pushd "${SANDBOX_DIR}/nanoarrow"
 
     cmake "${TARGET_NANOARROW_DIR}" \
-        -DNANOARROW_IPC=ON -DNANOARROW_BUILD_TESTS=ON -DNANOARROW_CODE_COVERAGE=ON
-    cmake --build .
-    CTEST_OUTPUT_ON_FAILURE=1 ctest .
-
-    popd
-
-    pushd "${SANDBOX_DIR}"
-
-    # Build + run tests with gcov for device extension
-    show_header "Build + test nanoarrow_device"
-    mkdir "${SANDBOX_DIR}/nanoarrow_device"
-    pushd "${SANDBOX_DIR}/nanoarrow_device"
-
-    cmake "${TARGET_NANOARROW_DIR}/extensions/nanoarrow_device" \
-        -DNANOARROW_DEVICE_BUILD_TESTS=ON -DNANOARROW_DEVICE_CODE_COVERAGE=ON
+          -DNANOARROW_DEVICE=ON -DNANOARROW_IPC=ON \
+          -DNANOARROW_BUILD_TESTS=ON -DNANOARROW_CODE_COVERAGE=ON
     cmake --build .
     CTEST_OUTPUT_ON_FAILURE=1 ctest .
 

docs/source/conf.py

@@ -66,8 +66,6 @@ def get_version():
 # Breathe configuration
 breathe_projects = {
     "nanoarrow_c": "../../src/apidoc/xml",
-    "nanoarrow_ipc": "../../src/apidoc/xml",
-    "nanoarrow_device": "../../extensions/nanoarrow_device/src/apidoc/xml",
 }
 breathe_default_project = "nanoarrow_c"
 

docs/source/reference/device.rst

@@ -22,20 +22,20 @@ C API
 ------------------------
 
 .. doxygengroup:: nanoarrow_device
-   :project: nanoarrow_device
+   :project: nanoarrow_c
    :members:
 
 C++ Helpers
 ------------------------
 
 .. doxygengroup:: nanoarrow_device_hpp-unique
-   :project: nanoarrow_device
+   :project: nanoarrow_c
    :members:
 
 Arrow C Device Interface
 ------------------------
 
 .. doxygengroup:: nanoarrow_device-arrow-cdata
-   :project: nanoarrow_device
+   :project: nanoarrow_c
    :members:
    :undoc-members:

docs/source/reference/ipc.rst

@@ -22,12 +22,12 @@ C API
 ------------------------
 
 .. doxygengroup:: nanoarrow_ipc
-   :project: nanoarrow_ipc
+   :project: nanoarrow_c
    :members:
 
 C++ Helpers
 ------------------------
 
 .. doxygengroup:: nanoarrow_ipc_hpp-unique
-   :project: nanoarrow_ipc
+   :project: nanoarrow_c
    :members: