[libshortfin] Python build ergonomics. (#177)

* Move bindings/python -> python
* Rework setup.py so that it can be the main driver for Python based
dev:
* Add env vars SHORTFIN_IREE_SOURCE_DIR, SHORTFIN_ENABLE_ASAN,
SHORTFIN_DEV_MODE, SHORTFIN_RUN_CTESTS
  * Update README with python based dev flow commands
  * Change build dir to build/cmake (from build/b)
* Removes a redundant build of IREE in the main CI (was not doing
anything in bundled mode).

With this, I can change my dev process to be entirely based on `pip
install` with a fully editable build and the ability to manipulate cmake
post build. This better aligns with how users will consume it.

Author: stellaraccident

.github/workflows/ci_linux_x64-libshortfin.yml

@@ -57,25 +57,6 @@ jobs:
         git submodule update --init --depth 1 -- third_party/googletest
         git submodule update --init --depth 1 -- third_party/hip-build-deps/
 
-    - name: Build IREE runtime
-      run: |
-        mkdir ${{ env.IREE_REPO_DIR }}/build
-        cd ${{ env.IREE_REPO_DIR }}/build
-        cmake -GNinja \
-          -DCMAKE_C_COMPILER=clang-18 \
-          -DCMAKE_CXX_COMPILER=clang++-18 \
-          -DIREE_ENABLE_LLD=ON \
-          -DIREE_ERROR_ON_MISSING_SUBMODULES=OFF \
-          -DIREE_HAL_DRIVER_DEFAULTS=OFF \
-          -DIREE_HAL_DRIVER_LOCAL_SYNC=ON \
-          -DIREE_HAL_DRIVER_LOCAL_TASK=ON \
-          -DIREE_HAL_DRIVER_HIP=ON \
-          -DIREE_BUILD_COMPILER=OFF \
-          -DIREE_BUILD_SAMPLES=OFF \
-          -DIREE_BUILD_TESTS=OFF \
-          ..
-        cmake --build . --target all
-
     - name: Setup Python
       uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
       with:
@@ -97,7 +78,7 @@ jobs:
           -DCMAKE_CXX_COMPILER=clang++-18 \
           -DCMAKE_LINKER_TYPE=LLD \
           -DSHORTFIN_BUNDLE_DEPS=ON \
-          -DCMAKE_PREFIX_PATH=${{ env.IREE_REPO_DIR }}/build/lib/cmake/IREE \
+          -DSHORTFIN_IREE_SOURCE_DIR=${{ env.IREE_REPO_DIR }} \
           -DSHORTFIN_BUILD_PYTHON_BINDINGS=ON \
           ..
         cmake --build . --target all
@@ -120,7 +101,7 @@ jobs:
           -DCMAKE_C_COMPILER=clang-18 \
           -DCMAKE_CXX_COMPILER=clang++-18 \
           -DCMAKE_LINKER_TYPE=LLD \
-          -DCMAKE_PREFIX_PATH=${{ env.IREE_REPO_DIR }}/build/lib/cmake/IREE \
+          -DSHORTFIN_IREE_SOURCE_DIR=${{ env.IREE_REPO_DIR }} \
           -DSHORTFIN_BUILD_PYTHON_BINDINGS=ON \
           -DSHORTFIN_HAVE_AMDGPU=OFF \
           -DSHORTFIN_BUILD_STATIC=ON \

.github/workflows/ci_linux_x64_asan-libshortfin.yml

@@ -147,27 +147,13 @@ jobs:
     - name: Build libshortfin
       run: |
         eval "$(pyenv init -)"
-        mkdir ${{ env.LIBSHORTFIN_DIR }}/build
-        cd ${{ env.LIBSHORTFIN_DIR }}/build
-        cmake -GNinja \
-          -DCMAKE_BUILD_TYPE=Debug \
-          -DCMAKE_C_COMPILER=clang-18 \
-          -DCMAKE_CXX_COMPILER=clang++-18 \
-          -DCMAKE_LINKER_TYPE=LLD \
-          -DSHORTFIN_BUNDLE_DEPS=ON \
-          -DSHORTFIN_IREE_SOURCE_DIR=${{ env.IREE_SOURCE_DIR }} \
-          -DSHORTFIN_BUILD_PYTHON_BINDINGS=ON \
-          -DSHORTFIN_ENABLE_ASAN=ON \
-          ..
-        cmake --build . --target all
+        cd ${{ env.LIBSHORTFIN_DIR }}
+        SHORTFIN_IREE_SOURCE_DIR="${{ env.IREE_SOURCE_DIR }}" \
+        SHORTFIN_ENABLE_ASAN=ON \
+        SHORTFIN_DEV_MODE=ON \
+        SHORTFIN_RUN_CTESTS=ON \
         pip install -v -e .
 
-    - name: Run ctest
-      if: ${{ !cancelled() }}
-      run: |
-        cd ${{ env.LIBSHORTFIN_DIR }}/build
-        ctest --timeout 30 --output-on-failure
-
     - name: Run pytest
       if: ${{ !cancelled() }}
       env:

libshortfin/CMakeLists.txt

@@ -4,7 +4,7 @@
 # https://llvm.org/LICENSE.txt for license information. SPDX-License-Identifier:
 # Apache-2.0 WITH LLVM-exception
 
-cmake_minimum_required(VERSION 3.28)
+cmake_minimum_required(VERSION 3.29)
 
 if(CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR)
   message(
@@ -138,7 +138,7 @@ if (NOT SHORTFIN_IREE_SOURCE_DIR AND SHORTFIN_BUNDLE_DEPS)
   )
   FetchContent_GetProperties(iree)
   if(NOT iree_POPULATED)
-    FetchContent_Populate(iree)
+    FetchContent_MakeAvailable(iree)
   endif()
   set(SHORTFIN_IREE_SOURCE_DIR ${iree_SOURCE_DIR})
 endif()
@@ -184,7 +184,7 @@ add_subdirectory(src)
 
 if(SHORTFIN_BUILD_PYTHON_BINDINGS)
   find_package(Python 3.8 COMPONENTS Interpreter Development.Module REQUIRED)
-  add_subdirectory(bindings/python)
+  add_subdirectory(python)
   set(SHORTFIN_PYTHON_CPP_PREBUILT "TRUE") # See setup.py.
   configure_file(setup.py setup.py @ONLY)
   configure_file(pyproject.toml pyproject.toml COPYONLY)

libshortfin/README.md

@@ -1,43 +1,19 @@
 # libshortfin - SHARK C++ inference library
 
-## Dev Builds
-
-Library dependencies:
-
-* [spdlog](https://github.com/gabime/spdlog)
-* [xtensor](https://github.com/xtensor-stack/xtensor)
-* [iree runtime](https://github.com/iree-org/iree)
-
-On recent Ubuntu, the primary dependencies can be satisfied via:
-
-```
-apt install libspdlog-dev libxtensor-dev
-```
-
-CMake must be told how to find the IREE runtime, either from a distribution
-tarball, or local build/install dir. For a local build directory, pass:
-
-```
-# Assumes that the iree-build directory is adjacent to this repo.
--DCMAKE_PREFIX_PATH=$(pwd)/../../iree-build/lib/cmake/IREE
-```
-
-One liner recommended CMake command (note that CMAKE_LINKER_TYPE requires
-cmake>=3.29):
+## Native Dev Builds
 
 ```
 cmake -GNinja -S. -Bbuild \
     -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
-    -DCMAKE_LINKER_TYPE=LLD \
-    -DCMAKE_PREFIX_PATH=$(pwd)/../../iree-build/lib/cmake/IREE
+    -DCMAKE_LINKER_TYPE=LLD
 ```
 
-## Building Python Bindings
+## Python Dev Builds
 
 If using a Python based development flow, there are two options:
 
-1. `pip install -v .` to build and install the library (TODO: Not yet implemented).
-2. Build with cmake and `-DSHORTFIN_BUILD_PYTHON_BINDINGS=ON` and then
+1. `pip install -e` based.
+2. Build with cmake as above and `-DSHORTFIN_BUILD_PYTHON_BINDINGS=ON` and then
    from the `build/` directory, run `pip install -v -e .` to create an
    editable install that will update as you build the C++ project.
 
@@ -46,6 +22,34 @@ recommended. Your python install should track any source file changes or
 builds without further interaction. Re-installing will be necessary if package
 structure changes significantly.
 
+For pure Python based dev, everything can be done from pip:
+
+```
+# Install build system pre-reqs (since we are building in dev mode, this
+# is not done for us). See source of truth in pyproject.toml:
+pip install setuptools wheel
+
+# Optionally install cmake and ninja if you don't have them or need a newer
+# version. If doing heavy development in Python, it is strongly recommended
+# to install these natively on your system as it will make it easier to
+# switch Python interpreters and build options (and the launcher in debug/asan
+# builds of Python is much slower). Note CMakeLists.txt for minimum CMake
+# version, which is usually quite recent.
+pip install cmake ninja
+
+# Optional env vars:
+#   SHORTFIN_IREE_SOURCE_DIR=$(pwd)/../../iree
+# Note that the `--no-build-isolation` flag is useful in development setups
+# because it does not create an intermediate venv that will keep later
+# invocations of cmake/ninja from working at the command line. If just doing
+# a one-shot build, it can be ommitted.
+SHORTFIN_DEV_MODE=ON pip install --no-build-isolation -v -e .
+
+Once built the first time, `cmake`, `ninja`, and `ctest` commands can be run
+directly from `build/cmake` and changes will apply directly to the next
+process launch.
+```
+
 ## Running Tests
 
 The project uses a combination of ctest for native C++ tests and pytest. Much