Skip to content

Commit e92c364

Browse files
WillAydWillAyd
authored
refactor: Consolidate device extension into main project (#517)
1 parent 8911a01 commit e92c364

27 files changed

+200
-847
lines changed

‎.github/workflows/build-and-test-device.yaml‎

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ on:
2828
- 'CMakeLists.txt'
2929
- '.github/workflows/build-and-test-device.yaml'
3030
- 'src/nanoarrow/**'
31-
- 'extensions/nanoarrow_device/**'
3231

3332
permissions:
3433
contents: read
@@ -46,28 +45,52 @@ jobs:
4645
config:
4746
- {label: default-build}
4847
- {label: namespaced-build, cmake_args: "-DNANOARROW_NAMESPACE=SomeUserNamespace"}
49-
- {label: bundled-build, cmake_args: "-DNANOARROW_DEVICE_BUNDLE=ON"}
48+
- {label: bundled-build, cmake_args: "-DNANOARROW_BUNDLE=ON"}
5049

5150
env:
52-
SUBDIR: 'extensions/nanoarrow_device'
51+
SUBDIR: '${{ github.workspace }}'
5352
NANOARROW_ARROW_TESTING_DIR: '${{ github.workspace }}/arrow-testing'
5453

5554
steps:
5655
- uses: actions/checkout@v4
5756

57+
- name: Checkout arrow-testing
58+
uses: actions/checkout@v4
59+
with:
60+
repository: apache/arrow-testing
61+
path: arrow-testing
62+
5863
- name: Install memcheck dependencies
5964
if: matrix.config.label == 'default-build'
6065
run: |
6166
sudo apt-get update && sudo apt-get install -y valgrind
6267
68+
- name: Cache Arrow C++ Build
69+
id: cache-arrow-build
70+
uses: actions/cache@v4
71+
with:
72+
path: arrow
73+
# Bump the number at the end of this line to force a new Arrow C++ build
74+
key: arrow-${{ runner.os }}-${{ runner.arch }}-1
75+
76+
- name: Build Arrow C++
77+
if: steps.cache-arrow-build.outputs.cache-hit != 'true'
78+
shell: bash
79+
run: |
80+
ci/scripts/build-arrow-cpp-minimal.sh 15.0.2 arrow
81+
6382
- name: Build
6483
run: |
84+
ARROW_PATH="$(pwd)/arrow"
6585
cd $SUBDIR
6686
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:`pwd`/dist/lib
6787
sudo ldconfig
6888
mkdir build
6989
cd build
70-
cmake .. -DCMAKE_BUILD_TYPE=Debug -DNANOARROW_DEVICE_BUILD_TESTS=ON ${{ matrix.config.cmake_args }}
90+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DNANOARROW_DEVICE=ON \
91+
-DNANOARROW_BUILD_TESTS=ON -DCMAKE_PREFIX_PATH="${ARROW_PATH}" \
92+
${{ matrix.config.cmake_args }}
93+
7194
cmake --build .
7295
7396
- name: Check for non-namespaced symbols in namespaced build
@@ -112,4 +135,4 @@ jobs:
112135
uses: actions/upload-artifact@main
113136
with:
114137
name: nanoarrow-device-memcheck
115-
path: extensions/nanoarrow_device/build/Testing/Temporary/MemoryChecker.*.log
138+
path: build/Testing/Temporary/MemoryChecker.*.log

‎.github/workflows/bundle.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,11 +52,11 @@ jobs:
5252
5353
- name: Bundle nanoarrow_device
5454
run: |
55-
cd extensions/nanoarrow_device
55+
rm -rf build
5656
mkdir build && cd build
57-
cmake .. -DNANOARROW_DEVICE_BUNDLE=ON
57+
cmake .. -DNANOARROW_BUNDLE=ON -DNANOARROW_DEVICE=ON
5858
cmake --build .
59-
cmake --install . --prefix=../../../nanoarrow-latest
59+
cmake --install . --prefix=../nanoarrow-latest
6060
6161
- name: Compress bundle
6262
run: |

‎CMakeLists.txt‎

Lines changed: 161 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,24 @@ option(NANOARROW_NAMESPACE "A prefix for exported symbols" OFF)
5353
option(NANOARROW_ARROW_STATIC
5454
"Use a statically-linked Arrow C++ build when linking tests" OFF)
5555

56+
option(NANOARROW_DEVICE_WITH_METAL "Build Apple metal libraries" OFF)
57+
option(NANOARROW_DEVICE_WITH_CUDA "Build CUDA libraries" OFF)
58+
59+
set(NANOARROW_DEVICE
60+
NANOARROW_DEVICE
61+
OR
62+
NANOARROW_DEVICE_WITH_MDETAL
63+
OR
64+
NANOARROW_DEVICE_WITH_CUDA)
65+
5666
if(NANOARROW_IPC)
5767
add_library(ipc_coverage_config INTERFACE)
5868
endif()
5969

70+
if(NANOARROW_DEVICE)
71+
add_library(device_coverage_config INTERFACE)
72+
endif()
73+
6074
if(NANOARROW_NAMESPACE)
6175
set(NANOARROW_NAMESPACE_DEFINE "#define NANOARROW_NAMESPACE ${NANOARROW_NAMESPACE}")
6276
else()
@@ -237,6 +251,39 @@ if(NANOARROW_BUNDLE)
237251
# Also install the flatcc headers
238252
install(DIRECTORY thirdparty/flatcc/include/flatcc DESTINATION ".")
239253
endif()
254+
255+
if(NANOARROW_DEVICE)
256+
# The CMake build step is creating nanoarrow_device.c and nanoarrow_device.h;
257+
# the CMake install step is copying them to a specific location
258+
file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/amalgamation)
259+
file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/amalgamation/nanoarrow)
260+
261+
# nanoarrow_device.h is currently standalone
262+
set(NANOARROW_DEVICE_H_TEMP
263+
${CMAKE_BINARY_DIR}/amalgamation/nanoarrow/nanoarrow_device.h)
264+
file(READ src/nanoarrow/nanoarrow_device.h SRC_FILE_CONTENTS)
265+
file(WRITE ${NANOARROW_DEVICE_H_TEMP} "${SRC_FILE_CONTENTS}")
266+
267+
# nanoarrow_device.c is currently standalone
268+
set(NANOARROW_DEVICE_C_TEMP
269+
${CMAKE_BINARY_DIR}/amalgamation/nanoarrow/nanoarrow_device.c)
270+
file(READ src/nanoarrow/nanoarrow_device.c SRC_FILE_CONTENTS)
271+
file(WRITE ${NANOARROW_DEVICE_C_TEMP} "${SRC_FILE_CONTENTS}")
272+
273+
# Add a library that the tests can link against (but don't install it)
274+
if(NANOARROW_BUILD_TESTS)
275+
add_library(nanoarrow_device ${NANOARROW_DEVICE_C_TEMP})
276+
277+
target_include_directories(nanoarrow_device
278+
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src>
279+
$<BUILD_INTERFACE:${nanoarrow_SOURCE_DIR}/src/nanoarrow>
280+
$<BUILD_INTERFACE:${nanoarrow_BINARY_DIR}/generated>
281+
)
282+
endif()
283+
284+
# Install the amalgamated header and sources
285+
install(FILES ${NANOARROW_DEVICE_H_TEMP} ${NANOARROW_DEVICE_C_TEMP} DESTINATION ".")
286+
endif()
240287
else()
241288
add_library(nanoarrow src/nanoarrow/array.c src/nanoarrow/schema.c
242289
src/nanoarrow/array_stream.c src/nanoarrow/utils.c)
@@ -329,14 +376,78 @@ else()
329376
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src>
330377
$<BUILD_INTERFACE:${nanoarrow_SOURCE_DIR}/src/nanoarrow>
331378
$<BUILD_INTERFACE:${nanoarrow_BINARY_DIR}/generated>
332-
$<BUILD_INTERFACE:${NANOARROW_IPC_FLATCC_INCLUDE_DIR}>
379+
$<BUILD_INTERFACE:${NANOARROW_IPC_FLATCC_INCLUDE_DIR}
333380
$<INSTALL_INTERFACE:include>)
334381

335382
install(TARGETS nanoarrow_ipc DESTINATION lib)
336383
install(FILES src/nanoarrow/nanoarrow_ipc.h
337384
src/nanoarrow/nanoarrow_ipc_flatcc_generated.h
338385
DESTINATION include/nanoarrow)
339386
endif()
387+
388+
if(NANOARROW_DEVICE)
389+
if(NANOARROW_DEVICE_WITH_METAL)
390+
if(NOT EXISTS "${CMAKE_BINARY_DIR}/metal-cpp")
391+
message(STATUS "Fetching metal-cpp")
392+
file(DOWNLOAD
393+
"https://developer.apple.com/metal/cpp/files/metal-cpp_macOS12_iOS15.zip"
394+
"${CMAKE_BINARY_DIR}/metal-cpp.zip")
395+
file(ARCHIVE_EXTRACT
396+
INPUT
397+
${CMAKE_BINARY_DIR}/metal-cpp.zip
398+
DESTINATION
399+
${CMAKE_BINARY_DIR})
400+
endif()
401+
402+
if(NOT DEFINED CMAKE_CXX_STANDARD)
403+
set(CMAKE_CXX_STANDARD 17)
404+
endif()
405+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
406+
407+
find_library(METAL_LIBRARY Metal REQUIRED)
408+
message(STATUS "Metal framework found at '${METAL_LIBRARY}'")
409+
410+
find_library(FOUNDATION_LIBRARY Foundation REQUIRED)
411+
message(STATUS "Foundation framework found at '${FOUNDATION_LIBRARY}'")
412+
413+
find_library(QUARTZ_CORE_LIBRARY QuartzCore REQUIRED)
414+
message(STATUS "CoreFoundation framework found at '${QUARTZ_CORE_LIBRARY}'")
415+
416+
set(NANOARROW_DEVICE_SOURCES_METAL src/nanoarrow/nanoarrow_device_metal.cc)
417+
set(NANOARROW_DEVICE_INCLUDE_METAL ${CMAKE_BINARY_DIR}/metal-cpp)
418+
set(NANOARROW_DEVICE_LIBS_METAL ${METAL_LIBRARY} ${FOUNDATION_LIBRARY}
419+
${QUARTZ_CORE_LIBRARY})
420+
set(NANOARROW_DEVICE_DEFS_METAL "NANOARROW_DEVICE_WITH_METAL")
421+
endif()
422+
423+
if(NANOARROW_DEVICE_WITH_CUDA)
424+
find_package(CUDAToolkit REQUIRED)
425+
set(NANOARROW_DEVICE_SOURCES_CUDA src/nanoarrow/nanoarrow_device_cuda.c)
426+
set(NANOARROW_DEVICE_LIBS_CUDA CUDA::cuda_driver)
427+
set(NANOARROW_DEVICE_DEFS_CUDA "NANOARROW_DEVICE_WITH_CUDA")
428+
endif()
429+
430+
add_library(nanoarrow_device
431+
src/nanoarrow/nanoarrow_device.c ${NANOARROW_DEVICE_SOURCES_METAL}
432+
${NANOARROW_DEVICE_SOURCES_CUDA})
433+
434+
target_include_directories(nanoarrow_device
435+
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/nanoarrow>
436+
$<BUILD_INTERFACE:${nanoarrow_SOURCE_DIR}/src/nanoarrow>
437+
$<BUILD_INTERFACE:${nanoarrow_BINARY_DIR}/generated>
438+
$<BUILD_INTERFACE:${NANOARROW_DEVICE_INCLUDE_METAL}>
439+
$<INSTALL_INTERFACE:include>)
440+
441+
target_compile_definitions(nanoarrow_device PRIVATE ${NANOARROW_DEVICE_DEFS_METAL}
442+
${NANOARROW_DEVICE_DEFS_CUDA})
443+
target_link_libraries(nanoarrow_device PUBLIC ${NANOARROW_DEVICE_LIBS_METAL}
444+
${NANOARROW_DEVICE_LIBS_CUDA})
445+
target_compile_definitions(nanoarrow_device
446+
PUBLIC "$<$<CONFIG:Debug>:NANOARROW_DEBUG>")
447+
448+
install(TARGETS nanoarrow_device DESTINATION lib)
449+
install(FILES src/nanoarrow/nanoarrow_device.h DESTINATION include/nanoarrow)
450+
endif()
340451
endif()
341452

342453
# Always build integration test if building tests
@@ -493,6 +604,55 @@ if(NANOARROW_BUILD_TESTS)
493604
gtest_discover_tests(nanoarrow_ipc_files_test)
494605
gtest_discover_tests(nanoarrow_ipc_hpp_test)
495606
endif()
607+
608+
if(NANOARROW_DEVICE)
609+
enable_testing()
610+
add_executable(nanoarrow_device_test src/nanoarrow/nanoarrow_device_test.cc)
611+
add_executable(nanoarrow_device_hpp_test src/nanoarrow/nanoarrow_device_hpp_test.cc)
612+
613+
if(NANOARROW_DEVICE_CODE_COVERAGE)
614+
target_compile_options(device_coverage_config INTERFACE -O0 -g --coverage)
615+
target_link_options(device_coverage_config INTERFACE --coverage)
616+
target_link_libraries(nanoarrow_device PRIVATE device_coverage_config)
617+
endif()
618+
619+
target_link_libraries(nanoarrow_device_test
620+
nanoarrow_device
621+
nanoarrow
622+
gtest_main
623+
device_coverage_config)
624+
target_link_libraries(nanoarrow_device_hpp_test
625+
nanoarrow_device
626+
nanoarrow
627+
gtest_main
628+
device_coverage_config)
629+
630+
include(GoogleTest)
631+
gtest_discover_tests(nanoarrow_device_test)
632+
gtest_discover_tests(nanoarrow_device_hpp_test)
633+
634+
if(NANOARROW_DEVICE_WITH_METAL)
635+
add_executable(nanoarrow_device_metal_test
636+
src/nanoarrow/nanoarrow_device_metal_test.cc)
637+
target_link_libraries(nanoarrow_device_metal_test
638+
nanoarrow_device
639+
nanoarrow
640+
gtest_main
641+
device_coverage_config)
642+
gtest_discover_tests(nanoarrow_device_metal_test)
643+
endif()
644+
645+
if(NANOARROW_DEVICE_WITH_CUDA)
646+
add_executable(nanoarrow_device_cuda_test
647+
src/nanoarrow/nanoarrow_device_cuda_test.cc)
648+
target_link_libraries(nanoarrow_device_cuda_test
649+
nanoarrow_device
650+
nanoarrow
651+
gtest_main
652+
device_coverage_config)
653+
gtest_discover_tests(nanoarrow_device_cuda_test)
654+
endif()
655+
endif()
496656
endif()
497657

498658
if(NANOARROW_BUILD_APPS)
@@ -502,10 +662,6 @@ if(NANOARROW_BUILD_APPS)
502662
endif()
503663
endif()
504664

505-
if(NANOARROW_DEVICE)
506-
add_subdirectory(extensions/nanoarrow_device)
507-
endif()
508-
509665
if(NANOARROW_BUILD_BENCHMARKS)
510666
add_subdirectory(dev/benchmarks)
511667
endif()

‎ci/scripts/build-docs.sh‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -72,11 +72,6 @@ main() {
7272
doxygen
7373
popd
7474

75-
show_header "Run Doxygen for device extension"
76-
pushd extensions/nanoarrow_device/src/apidoc
77-
doxygen
78-
popd
79-
8075
show_header "Build nanoarrow Python"
8176

8277
pushd python

‎ci/scripts/coverage.sh‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -73,21 +73,8 @@ function main() {
7373
pushd "${SANDBOX_DIR}/nanoarrow"
7474

7575
cmake "${TARGET_NANOARROW_DIR}" \
76-
-DNANOARROW_IPC=ON -DNANOARROW_BUILD_TESTS=ON -DNANOARROW_CODE_COVERAGE=ON
77-
cmake --build .
78-
CTEST_OUTPUT_ON_FAILURE=1 ctest .
79-
80-
popd
81-
82-
pushd "${SANDBOX_DIR}"
83-
84-
# Build + run tests with gcov for device extension
85-
show_header "Build + test nanoarrow_device"
86-
mkdir "${SANDBOX_DIR}/nanoarrow_device"
87-
pushd "${SANDBOX_DIR}/nanoarrow_device"
88-
89-
cmake "${TARGET_NANOARROW_DIR}/extensions/nanoarrow_device" \
90-
-DNANOARROW_DEVICE_BUILD_TESTS=ON -DNANOARROW_DEVICE_CODE_COVERAGE=ON
76+
-DNANOARROW_DEVICE=ON -DNANOARROW_IPC=ON \
77+
-DNANOARROW_BUILD_TESTS=ON -DNANOARROW_CODE_COVERAGE=ON
9178
cmake --build .
9279
CTEST_OUTPUT_ON_FAILURE=1 ctest .
9380

‎docs/source/conf.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,6 @@ def get_version():
6666
# Breathe configuration
6767
breathe_projects = {
6868
"nanoarrow_c": "../../src/apidoc/xml",
69-
"nanoarrow_ipc": "../../src/apidoc/xml",
70-
"nanoarrow_device": "../../extensions/nanoarrow_device/src/apidoc/xml",
7169
}
7270
breathe_default_project = "nanoarrow_c"
7371

‎docs/source/reference/device.rst‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,20 +22,20 @@ C API
2222
------------------------
2323

2424
.. doxygengroup:: nanoarrow_device
25-
:project: nanoarrow_device
25+
:project: nanoarrow_c
2626
:members:
2727

2828
C++ Helpers
2929
------------------------
3030

3131
.. doxygengroup:: nanoarrow_device_hpp-unique
32-
:project: nanoarrow_device
32+
:project: nanoarrow_c
3333
:members:
3434

3535
Arrow C Device Interface
3636
------------------------
3737

3838
.. doxygengroup:: nanoarrow_device-arrow-cdata
39-
:project: nanoarrow_device
39+
:project: nanoarrow_c
4040
:members:
4141
:undoc-members:

‎docs/source/reference/ipc.rst‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,12 @@ C API
2222
------------------------
2323

2424
.. doxygengroup:: nanoarrow_ipc
25-
:project: nanoarrow_ipc
25+
:project: nanoarrow_c
2626
:members:
2727

2828
C++ Helpers
2929
------------------------
3030

3131
.. doxygengroup:: nanoarrow_ipc_hpp-unique
32-
:project: nanoarrow_ipc
32+
:project: nanoarrow_c
3333
:members:

‎extensions/nanoarrow_device/.gitignore‎

Lines changed: 0 additions & 18 deletions
This file was deleted.

0 commit comments

Comments
 (0)