multipy: automatically select ABI based off of torch (#226)

Summary:
This automatically selects the ABI version based off the torch variable `torch._C._GLIBCXX_USE_CXX11_ABI`. This keeps the override settings by setting `-D_GLIBCXX_USE_CXX11_ABI=<0/1>` but defaults to loading from python.

The setup.py override now doesn't do anything since it was only used by PyTorch core CI

Pull Request resolved: #226

Test Plan:
CI

```
pip install -e .
```

Reviewed By: PaliC

Differential Revision: D40654548

Pulled By: d4l3k

fbshipit-source-id: 5f4e75174721a7fb0d48c45e36812437a8fdeb86

Author: d4l3k

.github/workflows/install_test.yaml

@@ -102,6 +102,7 @@ jobs:
       - name: Run pip install with conda for 3.8+
         run: |
           set -eux
-          if [[ ${{ matrix.python-minor-version }} -gt 7 ]]; then \
-          pip install -e .
+          if [[ ${{ matrix.python-minor-version }} -gt 7 ]]; then
+            source /opt/conda/bin/activate
+            pip install -e .
           fi

README.md

@@ -143,12 +143,13 @@ pip install -e . --install-option="--cmakeoff"
 
 cd multipy/runtime
 
-# build runtime
-mkdir build
-cd build
-# use cmake -DABI_EQUALS_1=ON .. instead if you want ABI=1
-cmake ..
-cmake --build . --config Release
+# configure runtime to build/
+cmake -S . -B build
+# if you need to override the ABI setting you can pass
+cmake -S . -B build -D_GLIBCXX_USE_CXX11_ABI=<0/1>
+
+# compile the files in build/
+cmake --build build --config Release -j
 ```
 
 ### Running unit tests for `multipy::runtime`
@@ -271,9 +272,6 @@ set(MULTIPY_PATH ".." CACHE PATH "The repo where multipy is built or the PYTHONP
 # include the multipy utils to help link against
 include(${MULTIPY_PATH}/multipy/runtime/utils.cmake)
 
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
-set(TORCH_CXX_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0")
-
 # add headers from multipy
 include_directories(${MULTIPY_PATH})
 

examples/CMakeLists.txt

@@ -6,9 +6,6 @@ set(MULTIPY_PATH ".." CACHE PATH "The repo where multipy is built or the PYTHONP
 # include the multipy utils to help link against
 include(${MULTIPY_PATH}/multipy/runtime/utils.cmake)
 
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
-set(TORCH_CXX_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0")
-
 # add headers from multipy
 include_directories(${MULTIPY_PATH})
 

multipy/runtime/CMakeLists.txt

@@ -9,38 +9,26 @@ project(MultipyRuntime)
 
 # set ABI by default to 0
 
-option(ABI_EQUALS_1 "Set ABI value to 1, by default it is set to 0. Pytorch by default builds with ABI set to 1." OFF)
 option(BUILD_CUDA_TESTS "Set to ON in order to build cuda tests. By default we do not" OFF)
 option(GDB_ON "Sets to debug mode (for gdb), defaults to OFF" OFF)
 
 OPTION(LEGACY_PYTHON_PRE_3_8 "Whether to use Python 3.7 codepaths." OFF)
 
-if(ABI_EQUALS_1)
-  set(ABI_VALUE 1)
-else()
-  set(ABI_VALUE 0)
-endif()
-
 if(GDB_ON)
   set(CMAKE_BUILD_TYPE Debug)
 endif()
 
 message(STATUS "CMAKE_BUILD_TYPE - ${CMAKE_BUILD_TYPE}" )
 
-add_definitions(-D_GLIBCXX_USE_CXX11_ABI=${ABI_VALUE})
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=${ABI_VALUE} -fabi-version=11 -fno-lto")
-
-SET(INTERPRETER_DIR "${DEPLOY_DIR}/interpreter" )
-SET(INTERPRETER_DIR "${DEPLOY_DIR}/interpreter" PARENT_SCOPE)
-
 set(DEPLOY_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 get_filename_component(MULTIPY_DIR ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY)
 
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fabi-version=11 -fno-lto")
+include(${DEPLOY_DIR}/utils.cmake)
+
 add_subdirectory(interpreter)
 add_subdirectory(third-party/fmt)
 
-include(${DEPLOY_DIR}/utils.cmake)
-
 
 # we do not want to have torch_deployinterpreter linked against libstdc++ or libc because
 # when loading it with RTLD_DEEPBIND it will resolve std::cout/stdout to the copy in libc++/libc instead of the

multipy/runtime/interpreter/CMakeLists.txt

@@ -9,8 +9,6 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 SET(INTERPRETER_DIR "${DEPLOY_DIR}/interpreter" )
 SET(INTERPRETER_DIR "${DEPLOY_DIR}/interpreter" PARENT_SCOPE)
 
-include(${DEPLOY_DIR}/utils.cmake)
-
 SET(MULTIPY_UTILS "${CMAKE_CURRENT_SOURCE_DIR}/../../utils")
 
 OPTION(LEGACY_PYTHON_PRE_3_8 "Whether to use Python 3.7 codepaths." OFF)

multipy/runtime/utils.cmake

@@ -7,6 +7,22 @@
 # prefer the active Python version instead of the latest -- only works on cmake
 # 3.15+
 # https://cmake.org/cmake/help/latest/module/FindPython3.html#hints
+if (NOT DEFINED _GLIBCXX_USE_CXX11_ABI)
+  # infer the ABI setting from the installed version of PyTorch
+  execute_process(
+    COMMAND python -c "import torch; print(1 if torch._C._GLIBCXX_USE_CXX11_ABI else 0)"
+    OUTPUT_VARIABLE _GLIBCXX_USE_CXX11_ABI
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    RESULT_VARIABLE ret
+  )
+  if(ret EQUAL "1")
+    message(FATAL_ERROR "Failed to detect ABI version")
+  endif()
+endif()
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=${_GLIBCXX_USE_CXX11_ABI}")
+add_definitions(-D_GLIBCXX_USE_CXX11_ABI=${_GLIBCXX_USE_CXX11_ABI})
+message(STATUS "_GLIBCXX_USE_CXX11_ABI - ${_GLIBCXX_USE_CXX11_ABI}")
+
 set(Python3_FIND_STRATEGY LOCATION)
 
 find_package (Python3 COMPONENTS Interpreter Development)

setup.py

@@ -37,20 +37,19 @@ class MultipyRuntimeDevelop(MultipyRuntimeCmake, develop):
     def initialize_options(self):
         develop.initialize_options(self)
         self.cmakeoff = None
+
+        # TODO(tristanr): remove once unused
         self.abicxx = None
 
     def finalize_options(self):
         develop.finalize_options(self)
         if self.cmakeoff is not None:
             self.distribution.get_command_obj("build_ext").cmake_off = True
-        if self.abicxx is not None:
-            self.distribution.get_command_obj("build_ext").abicxx = True
 
 
 class MultipyRuntimeBuild(MultipyRuntimeCmake, build_ext):
     user_options = build_ext.user_options + MultipyRuntimeCmake.user_options
     cmake_off = False
-    abicxx = False
 
     def run(self):
         if self.cmake_off:
@@ -71,14 +70,11 @@ def run(self):
         if not os.path.exists(build_dir_abs):
             os.makedirs(build_dir_abs)
         legacy_python_cmake_flag = "OFF" if sys.version_info.minor > 7 else "ON"
-        cxx_abi_flag = "ON" if self.abicxx else "OFF"
 
         print(f"-- Running multipy runtime makefile in dir {build_dir_abs}")
         try:
             subprocess.run(
-                [
-                    f"cmake -DLEGACY_PYTHON_PRE_3_8={legacy_python_cmake_flag} -DABI_EQUALS_1={cxx_abi_flag} .."
-                ],
+                [f"cmake -DLEGACY_PYTHON_PRE_3_8={legacy_python_cmake_flag} .."],
                 cwd=build_dir_abs,
                 shell=True,
                 check=True,