Bootstrap into subprojects directory

Author: WillAyd

.github/workflows/build-and-test-device.yaml

@@ -28,7 +28,8 @@ on:
       - 'CMakeLists.txt'
       - '.github/workflows/build-and-test-device.yaml'
       - 'src/nanoarrow/**'
-      - 'python/setup.py'
+      - 'python/meson.build'
+      - 'python/src/nanoarrow/meson.build'
 
 permissions:
   contents: read
@@ -156,4 +157,4 @@ jobs:
         if: matrix.config.label == 'with-cuda'
         run: |
           cd python
-          python3 -m pytest -vv
+          python3 -m pytest -vv tests

.github/workflows/python-wheels.yaml

@@ -24,9 +24,11 @@ on:
       - main
     paths:
       - '.github/workflows/python-wheels.yaml'
-      - 'python/setup.py'
+      - 'python/meson.build'
+      - 'python/src/nanoarrow/meson.build'
       - 'python/pyproject.toml'
       - 'python/bootstrap.py'
+      - 'python/generate_dist.py'
       - 'python/MANIFEST.in'
   push:
     branches:

.gitignore

@@ -33,5 +33,6 @@ __pycache__
 subprojects/*
 !subprojects/packagefiles
 !subprojects/*.wrap
+python/subprojects/*
 
 compile_commands.json

dev/release/rat_exclude_files.txt

@@ -15,3 +15,4 @@ dist/flatcc.c
 src/nanoarrow/ipc/flatcc_generated.h
 thirdparty/*
 python/src/nanoarrow/dlpack_abi.h
+python/subprojects/arrow-nanoarrow

dev/release/source_build.sh

@@ -43,7 +43,7 @@ main() {
     # Resolve all hard and symbolic links
     rm -rf "${base_name}.tmp/"
     mv "${base_name}/" "${base_name}.tmp/"
-    cp -R -L "${base_name}.tmp" "${base_name}"
+    cp -R -d "${base_name}.tmp" "${base_name}"
     rm -rf "${base_name}.tmp/"
 
     # Create new tarball

docker-compose.yml

@@ -43,7 +43,7 @@ services:
       # Don't mix the "dev tools" and "source" checkouts
       - ./ci/scripts/build-docs.sh:/build-docs.sh
       - ${NANOARROW_DOCKER_SOURCE_DIR}:/nanoarrow
-    command: "/bin/bash /build-docs.sh /nanoarrow"
+    command: "/build-docs.sh /nanoarrow"
 
   integration:
     # Don't cache here (building the image takes about the same amount of time

meson.build

@@ -103,7 +103,7 @@ if get_option('ipc')
     )
     nanoarrow_ipc_dep = declare_dependency(include_directories: [incdir],
                                            link_with: nanoarrow_ipc_lib,
-                                           dependencies: [nanoarrow_dep, flatcc_dep])
+                                           dependencies: [nanoarrow_dep])
 endif
 
 needs_device = get_option('device') or get_option('metal') or get_option('cuda')
@@ -137,6 +137,10 @@ if needs_device
       install: true,
       cpp_args: device_defines,
   )
+
+  nanoarrow_device_dep = declare_dependency(include_directories: [incdir],
+                                            link_with: nanoarrow_device_lib,
+                                            dependencies: device_deps)
 endif
 
 needs_testing = get_option('testing') or get_option('tests')

python/bootstrap.py

@@ -223,10 +223,6 @@ def _write_defs(self, output):
         output.write(b"\n")
 
 
-# Runs cmake -DNANOARROW_BUNDLE=ON if cmake exists or copies nanoarrow.c/h
-# from ../dist if it does not. Running cmake is safer because it will sync
-# any changes from nanoarrow C library sources in the checkout but is not
-# strictly necessary for things like installing from GitHub.
 def copy_or_generate_nanoarrow_c(target_dir: pathlib.Path):
     vendored_files = [
         "nanoarrow.h",
@@ -239,21 +235,12 @@ def copy_or_generate_nanoarrow_c(target_dir: pathlib.Path):
     dst = {name: target_dir / name for name in vendored_files}
 
     this_dir = pathlib.Path(__file__).parent.resolve()
-    source_dir = this_dir.parent
-    is_cmake_dir = (source_dir / "CMakeLists.txt").exists()
-    is_in_nanoarrow_repo = (
-        is_cmake_dir and (source_dir / "src" / "nanoarrow" / "nanoarrow.h").exists()
-    )
-
-    if not is_in_nanoarrow_repo:
-        raise ValueError(
-            "Attempt to build source distribution outside the nanoarrow repo"
-        )
+    arrow_proj_dir = this_dir / "subprojects" / "arrow-nanoarrow"
 
     subprocess.run(
         [
             sys.executable,
-            source_dir / "ci" / "scripts" / "bundle.py",
+            arrow_proj_dir / "ci" / "scripts" / "bundle.py",
             "--symbol-namespace",
             "PythonPkg",
             "--header-namespace",

python/generate_dist.py

@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import pathlib
+import shutil
+
+
+def main():
+    src_dir = pathlib.Path(os.environ["MESON_SOURCE_ROOT"]).parent.resolve()
+    dist_dir = pathlib.Path(os.environ["MESON_DIST_ROOT"]).resolve()
+    subproj_dir = dist_dir / "subprojects" / "arrow-nanoarrow"
+
+    if subproj_dir.is_symlink():
+        subproj_dir.unlink()
+
+    subproj_dir.mkdir(parents=True)
+    shutil.copy(src_dir / "meson.build", subproj_dir / "meson.build")
+    shutil.copy(src_dir / "meson.options", subproj_dir / "meson.options")
+
+    # Copy over any subproject dependency / wrap files
+    subproj_subproj_dir = subproj_dir / "subprojects"
+    subproj_subproj_dir.mkdir()
+    for f in (src_dir / "subprojects").glob("*.wrap"):
+        shutil.copy(f, subproj_subproj_dir / f.name)
+    shutil.copytree(
+        src_dir / "subprojects" / "packagefiles", subproj_subproj_dir / "packagefiles"
+    )
+
+    target_src_dir = subproj_dir / "src"
+    shutil.copytree(src_dir / "src", target_src_dir)
+
+    # CMake isn't actually required for building, but the bundle.py script reads from
+    # its configuration
+    shutil.copy(src_dir / "CMakeLists.txt", subproj_dir / "CMakeLists.txt")
+
+    subproj_ci_scripts_dir = subproj_dir / "ci" / "scripts"
+    subproj_ci_scripts_dir.mkdir(parents=True)
+    shutil.copy(
+        src_dir / "ci" / "scripts" / "bundle.py", subproj_ci_scripts_dir / "bundle.py"
+    )
+
+
+if __name__ == "__main__":
+    main()

python/meson.build

@@ -25,10 +25,14 @@ project(
         'warning_level=2',
         'c_std=c99',
         'default_library=static',
+        # We need to set these options at the project default_option level
+        # due to https://github.com/mesonbuild/meson/issues/6728
+        'arrow-nanoarrow:ipc=true',
+        'arrow-nanoarrow:device=true',
+        'arrow-nanoarrow:namespace=PythonPkg',
     ],
 )
 
-py = import('python').find_installation(pure: false)
-
-subdir('vendor')
 subdir('src/nanoarrow')
+
+meson.add_dist_script('python', meson.current_source_dir() / 'generate_dist.py')

python/pyproject.toml

@@ -40,7 +40,10 @@ Changelog = "https://github.com/apache/arrow-nanoarrow/blob/main/CHANGELOG.md"
 requires = [
     "meson>=1.3.0",
     "meson-python",
-    "cmake>=3.14",
     "Cython"
 ]
 build-backend = "mesonpy"
+
+
+[tool.meson-python.args]
+install = ['--skip-subprojects']

python/src/nanoarrow/_ipc_lib.pyx

@@ -38,7 +38,7 @@ from nanoarrow._array cimport CArrayView
 from nanoarrow._utils cimport Error
 
 
-cdef extern from "nanoarrow_ipc.h" nogil:
+cdef extern from "nanoarrow/nanoarrow_ipc.h" nogil:
     struct ArrowIpcInputStream:
         ArrowErrorCode (*read)(ArrowIpcInputStream* stream, uint8_t* buf,
                                int64_t buf_size_bytes, int64_t* size_read_out,

python/src/nanoarrow/meson.build

@@ -15,6 +15,40 @@
 # specific language governing permissions and limitations
 # under the License.
 
+nanoarrow_proj = subproject('arrow-nanoarrow')
+nanoarrow_dep = nanoarrow_proj.get_variable('nanoarrow_dep')
+nanoarrow_ipc_dep = nanoarrow_proj.get_variable('nanoarrow_ipc_dep')
+nanoarrow_device_dep = nanoarrow_proj.get_variable('nanoarrow_device_dep')
+
+py = import('python').find_installation(pure: false)
+
+vendored_files = custom_target(
+    'generate-pyx',
+    output: [
+        'flatcc.c',
+        'nanoarrow.c',
+        'nanoarrow_c.pxd',
+        'nanoarrow_device.c',
+        'nanoarrow_device_c.pxd',
+        'nanoarrow_device.h',
+        'nanoarrow_device.hpp',
+        'nanoarrow.h',
+        'nanoarrow.hpp',
+        'nanoarrow_ipc.c',
+        'nanoarrow_ipc.h',
+        'nanoarrow_ipc.hpp',
+    ],
+    command: [
+        py,
+        meson.current_source_dir() + '/../../bootstrap.py',
+        '--output-dir', meson.current_build_dir()
+    ],
+)
+
+nanoarrow_pyx_dep = declare_dependency(
+    sources: vendored_files[2],
+)
+
 cyfiles = [
     '_array.pyx',
     '_array_stream.pyx',
@@ -30,19 +64,19 @@ cython_args = [
     '--include-dir',
     meson.current_source_dir(),
     '--include-dir',
-    meson.project_build_root() / 'vendor',
+    meson.current_build_dir(),
 ]
 if get_option('buildtype') == 'debug'
     cython_args += ['--gdb']
 endif
 
 fs = import('fs')
 foreach cyf : cyfiles
-  cyfile_deps = [nanoarrow_pyx_dep]
+  cyfile_deps = [nanoarrow_pyx_dep, nanoarrow_dep]
 
   stem = fs.stem(cyf)
   if stem in ['_array', '_device']
-    cyfile_deps += [nanoarrow_device_pyx_dep]
+    cyfile_deps += [nanoarrow_device_dep]
   elif stem == '_ipc_lib'
     cyfile_deps += [nanoarrow_ipc_dep]
   endif

python/subprojects/arrow-nanoarrow

@@ -0,0 +1 @@
+../..