change setup.py to optionally build binaries in source distribution (#245)

* adds initial skbuild configuration load from local lib directory loads local binaries if they are available

* remove git from toml dependencies

* additional cmake args in setup.py

* add description format for pypi

* directory corrections for skbuild

* load local nvrtc-builtins when possible

* make mkl libs static for distribution

* Update README to include information about wheels

* remove extra import platform

Co-authored-by: pradeep <[email protected]>

Author: syurkevi

CMakeLists.txt

@@ -0,0 +1,18 @@
+cmake_minimum_required(VERSION 3.11.0)
+project(arrayfire-python)
+find_package(PythonExtensions REQUIRED)
+include(FetchContent)
+
+set(CMAKE_MODULE_PATH_OLD ${CMAKE_MODULE_PATH})
+set(CMAKE_MODULE_PATH "")
+set(NO_SONAME)
+
+FetchContent_Declare(
+    arrayfire
+    GIT_REPOSITORY https://github.com/arrayfire/arrayfire.git
+    GIT_TAG v3.8
+)
+FetchContent_MakeAvailable(arrayfire)
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH_OLD})
+
+set(ignoreWarning "${SKBUILD}")

README.md

@@ -25,30 +25,29 @@ def calc_pi_device(samples):
 
 Choosing a particular backend can be done using `af.set_backend(name)`  where name is either "_cuda_", "_opencl_", or "_cpu_". The default device is chosen in the same order of preference.
 
-## Requirements
-
-Currently, this project is tested only on Linux and OSX. You also need to have the ArrayFire C/C++ library installed on your machine. You can get it from the following sources.
+## Getting started
+ArrayFire can be installed from a variety of sources. [Pre-built wheels](https://repo.arrayfire.com/python/wheels/3.8.0/) are available for a number of systems and toolkits. Wheels for some systems are available on PyPI. Unsupported systems will require separate installation of the ArrayFire C/C++ libraries and only the python wrapper will be installed in that case. 
+You can get the ArrayFire C/C++ library from the following sources:
 
 - [Download and install binaries](https://arrayfire.com/download)
 - [Build and install from source](https://github.com/arrayfire/arrayfire)
 
-Please check the following links for dependencies.
-
-- [Linux dependencies](http://www.arrayfire.com/docs/using_on_linux.htm)
-- [OSX dependencies](http://www.arrayfire.com/docs/using_on_osx.htm)
-
-## Getting started
-
-**Install the last stable version:**
 
+**Install the last stable version:**  
 ```
 pip install arrayfire
 ```
 
-**Install the development version:**
+**Install a pre-built wheel for a specific CUDA toolkit version:**
+```
+pip install arrayfire==3.8.0+cu112 -f https://repo.arrayfire.com/python/wheels/3.8.0/
+# Replace the +cu112 local version with the desired toolkit
+```
+
+**Install the development source distribution:**
 
 ```
-pip install git+git://github.com/arrayfire/arrayfire-python.git@devel
+pip install git+git://github.com/arrayfire/arrayfire-python.git@master
 ```
 
 **Installing offline:**
@@ -57,10 +56,11 @@ pip install git+git://github.com/arrayfire/arrayfire-python.git@devel
 cd path/to/arrayfire-python
 python setup.py install
 ```
+Rather than installing and building ArrayFire elsewhere in the system, you can also build directly through python by first setting the `AF_BUILD_LOCAL_LIBS=1` environment variable. Additional setup will be required to build ArrayFire, including satisfying dependencies and further CMake configuration. Details on how to pass additional arguments to the build systems can be found in the [scikit-build documentation.](https://scikit-build.readthedocs.io/en/latest/)
 
 **Post Installation:**
 
-Please follow [these instructions](https://github.com/arrayfire/arrayfire-python/wiki) to ensure the arrayfire-python can find the arrayfire libraries.
+If you are not using one of the pre-built wheels, you may need to ensure arrayfire-python can find the installed arrayfire libraries. Please follow [these instructions](https://github.com/arrayfire/arrayfire-python/wiki) to ensure that arrayfire-python can find the arrayfire libraries.
 
 To run arrayfire tests, you can run the following command from command line.
 

arrayfire/library.py

@@ -15,6 +15,7 @@
 import ctypes as ct
 import traceback
 import os
+import sys
 
 c_float_t     = ct.c_float
 c_double_t    = ct.c_double
@@ -492,8 +493,6 @@ class VARIANCE(_Enum):
 _VER_MAJOR_PLACEHOLDER = "__VER_MAJOR__"
 
 def _setup():
-    import platform
-
     platform_name = platform.system()
 
     try:
@@ -527,7 +526,7 @@ def _setup():
             ct.windll.kernel32.SetErrorMode(0x0001 | 0x0002)
 
         if AF_SEARCH_PATH is None:
-            AF_SEARCH_PATH="C:/Program Files/ArrayFire/v" + AF_VER_MAJOR +"/"
+            AF_SEARCH_PATH = "C:/Program Files/ArrayFire/v" + AF_VER_MAJOR +"/"
 
         if CUDA_PATH is not None:
             CUDA_FOUND = os.path.isdir(CUDA_PATH + '/bin') and os.path.isdir(CUDA_PATH + '/nvvm/bin/')
@@ -554,7 +553,12 @@ def _setup():
         post = '.so.' + _VER_MAJOR_PLACEHOLDER
 
         if AF_SEARCH_PATH is None:
-            AF_SEARCH_PATH='/opt/arrayfire-' + AF_VER_MAJOR + '/'
+            if os.path.exists('/opt/arrayfire-' + AF_VER_MAJOR + '/'):
+                AF_SEARCH_PATH = '/opt/arrayfire-' + AF_VER_MAJOR + '/'
+            elif os.path.exists('/opt/arrayfire/'):
+                AF_SEARCH_PATH = '/opt/arrayfire/'
+            else:
+                AF_SEARCH_PATH = '/usr/local/'
 
         if CUDA_PATH is None:
             CUDA_PATH='/usr/local/cuda/'
@@ -566,21 +570,46 @@ def _setup():
     else:
         raise OSError(platform_name + ' not supported')
 
-    if AF_PATH is None:
-        os.environ['AF_PATH'] = AF_SEARCH_PATH
-
-    return pre, post, AF_SEARCH_PATH, CUDA_FOUND
+    return pre, post, AF_PATH, AF_SEARCH_PATH, CUDA_FOUND
 
 class _clibrary(object):
 
+    def __find_nvrtc_builtins_libname(self, search_path):
+        filelist = os.listdir(search_path)
+        for f in filelist:
+            if 'nvrtc-builtins' in f:
+                return f
+        return None
+
     def __libname(self, name, head='af', ver_major=AF_VER_MAJOR):
         post = self.__post.replace(_VER_MAJOR_PLACEHOLDER, ver_major)
         libname = self.__pre + head + name + post
-        if os.path.isdir(self.AF_PATH + '/lib64'):
-            libname_full = self.AF_PATH + '/lib64/' + libname
+
+        if self.AF_PATH:
+            if os.path.isdir(self.AF_PATH + '/lib64'):
+                path_search = self.AF_PATH + '/lib64/'
+            else:
+                path_search = self.AF_PATH + '/lib/'
+        else:
+            if os.path.isdir(self.AF_SEARCH_PATH + '/lib64'):
+                path_search = self.AF_SEARCH_PATH + '/lib64/'
+            else:
+                path_search = self.AF_SEARCH_PATH + '/lib/'
+
+        if platform.architecture()[0][:2] == '64':
+            path_site  = sys.prefix + '/lib64/'
+        else:
+            path_site  = sys.prefix + '/lib/'
+
+        path_local = self.AF_PYMODULE_PATH
+        libpaths = [('', libname),
+                    (path_site, libname),
+                    (path_local,libname)]
+        if self.AF_PATH: #prefer specified AF_PATH if exists
+            libpaths.append((path_search, libname))
         else:
-            libname_full = self.AF_PATH + '/lib/' + libname
-        return (libname, libname_full)
+            libpaths.insert(2, (path_search, libname))
+        return libpaths
 
     def set_unsafe(self, name):
         lib = self.__clibs[name]
@@ -592,13 +621,19 @@ def __init__(self):
 
         more_info_str = "Please look at https://github.com/arrayfire/arrayfire-python/wiki for more information."
 
-        pre, post, AF_PATH, CUDA_FOUND = _setup()
+        pre, post, AF_PATH, AF_SEARCH_PATH, CUDA_FOUND = _setup()
 
         self.__pre = pre
         self.__post = post
         self.AF_PATH = AF_PATH
+        self.AF_SEARCH_PATH = AF_SEARCH_PATH
         self.CUDA_FOUND = CUDA_FOUND
 
+        # prefer locally packaged arrayfire libraries if they exist
+        af_module = __import__(__name__)
+        self.AF_PYMODULE_PATH = af_module.__path__[0] + '/' if af_module.__path__ else None
+
+
         self.__name = None
 
         self.__clibs = {'cuda'    : None,
@@ -618,7 +653,7 @@ def __init__(self):
                                    'opencl'  : 4}
 
         # Try to pre-load forge library if it exists
-        libnames = self.__libname('forge', head='', ver_major=FORGE_VER_MAJOR)
+        libnames = reversed(self.__libname('forge', head='', ver_major=FORGE_VER_MAJOR))
 
         try:
             VERBOSE_LOADS = os.environ['AF_VERBOSE_LOADS'] == '1'
@@ -628,14 +663,15 @@ def __init__(self):
 
         for libname in libnames:
             try:
-                ct.cdll.LoadLibrary(libname)
+                full_libname = libname[0] + libname[1]
+                ct.cdll.LoadLibrary(full_libname)
                 if VERBOSE_LOADS:
-                    print('Loaded ' + libname)
+                    print('Loaded ' + full_libname)
                 break
             except OSError:
                 if VERBOSE_LOADS:
                     traceback.print_exc()
-                    print('Unable to load ' + libname)
+                    print('Unable to load ' + full_libname)
                 pass
 
         c_dim4 = c_dim_t*4
@@ -644,24 +680,39 @@ def __init__(self):
 
         # Iterate in reverse order of preference
         for name in ('cpu', 'opencl', 'cuda', ''):
-            libnames = self.__libname(name)
+            libnames = reversed(self.__libname(name))
             for libname in libnames:
                 try:
-                    ct.cdll.LoadLibrary(libname)
+                    full_libname = libname[0] + libname[1]
+
+                    ct.cdll.LoadLibrary(full_libname)
                     __name = 'unified' if name == '' else name
-                    clib = ct.CDLL(libname)
+                    clib = ct.CDLL(full_libname)
                     self.__clibs[__name] = clib
                     err = clib.af_randu(c_pointer(out), 4, c_pointer(dims), Dtype.f32.value)
                     if (err == ERR.NONE.value):
                         self.__name = __name
                         clib.af_release_array(out)
                         if VERBOSE_LOADS:
-                            print('Loaded ' + libname)
+                            print('Loaded ' + full_libname)
+
+                        # load nvrtc-builtins library if using cuda
+                        if name == 'cuda':
+                            nvrtc_name = self.__find_nvrtc_builtins_libname(libname[0])
+                            if nvrtc_name:
+                                ct.cdll.LoadLibrary(libname[0] + nvrtc_name)
+
+                                if VERBOSE_LOADS:
+                                    print('Loaded ' + libname[0] + nvrtc_name)
+                            else:
+                                if VERBOSE_LOADS:
+                                    print('Could not find local nvrtc-builtins libarary')
+
                         break;
                 except OSError:
                     if VERBOSE_LOADS:
                         traceback.print_exc()
-                        print('Unable to load ' + libname)
+                        print('Unable to load ' + full_libname)
                     pass
 
         if (self.__name is None):
@@ -689,6 +740,7 @@ def parse(self, res):
                 lst.append(key)
         return tuple(lst)
 
+
 backend = _clibrary()
 
 def set_backend(name, unsafe=False):

pyproject.toml

@@ -0,0 +1,2 @@
+[build-system]
+requires = ["setuptools", "wheel", "scikit-build", "cmake", "ninja"]

setup.cfg

@@ -1,9 +1,10 @@
 [metadata]
 name = arrayfire
-version = 3.7.20200213
+version = 3.8.0
 description = Python bindings for ArrayFire
 licence = BSD
 long_description = file: README.md
+long_description_content_type = text/markdown
 maintainer = ArrayFire
 maintainer_email = [email protected]
 url = http://arrayfire.com
@@ -15,6 +16,8 @@ classifiers =
 
 [options]
 packages = find:
+install_requires=
+    scikit-build
 
 [options.packages.find]
 include = arrayfire

setup.py

@@ -9,8 +9,98 @@
 # http://arrayfire.com/licenses/BSD-3-Clause
 ########################################################
 
-# TODO: Look for af libraries during setup
+import os
+import re
 
-from setuptools import setup
+# package can be distributed with arrayfire binaries or
+# just with python wrapper files, the AF_BUILD_LOCAL
+# environment var determines whether to build the arrayfire
+# binaries locally rather than searching in a system install
+
+AF_BUILD_LOCAL_LIBS = os.environ.get('AF_BUILD_LOCAL_LIBS')
+print(f'AF_BUILD_LOCAL_LIBS={AF_BUILD_LOCAL_LIBS}')
+if AF_BUILD_LOCAL_LIBS:
+    print('Proceeding to build ArrayFire libraries')
+else:
+    print('Skipping binaries installation, only python files will be installed')
+
+AF_BUILD_CPU = os.environ.get('AF_BUILD_CPU')
+AF_BUILD_CPU = 1 if AF_BUILD_CPU is None else int(AF_BUILD_CPU)
+AF_BUILD_CPU_CMAKE_STR = '-DAF_BUILD_CPU:BOOL=ON' if (AF_BUILD_CPU == 1) else '-DAF_BUILD_CPU:BOOL=OFF'
+
+AF_BUILD_CUDA = os.environ.get('AF_BUILD_CUDA')
+AF_BUILD_CUDA = 1 if AF_BUILD_CUDA is None else int(AF_BUILD_CUDA)
+AF_BUILD_CUDA_CMAKE_STR = '-DAF_BUILD_CUDA:BOOL=ON' if (AF_BUILD_CUDA == 1) else '-DAF_BUILD_CUDA:BOOL=OFF'
+
+AF_BUILD_OPENCL = os.environ.get('AF_BUILD_OPENCL')
+AF_BUILD_OPENCL = 1 if AF_BUILD_OPENCL is None else int(AF_BUILD_OPENCL)
+AF_BUILD_OPENCL_CMAKE_STR = '-DAF_BUILD_OPENCL:BOOL=ON' if (AF_BUILD_OPENCL == 1) else '-DAF_BUILD_OPENCL:BOOL=OFF'
+
+AF_BUILD_UNIFIED = os.environ.get('AF_BUILD_UNIFIED')
+AF_BUILD_UNIFIED = 1 if AF_BUILD_UNIFIED is None else int(AF_BUILD_UNIFIED)
+AF_BUILD_UNIFIED_CMAKE_STR = '-DAF_BUILD_UNIFIED:BOOL=ON' if (AF_BUILD_UNIFIED == 1) else '-DAF_BUILD_UNIFIED:BOOL=OFF'
+
+if AF_BUILD_LOCAL_LIBS:
+    # invoke cmake and build arrayfire libraries to install locally in package
+    from skbuild import setup
+
+    def filter_af_files(cmake_manifest):
+        cmake_manifest = list(filter(lambda name: not (name.endswith('.h') 
+            or name.endswith('.cpp')
+            or name.endswith('.hpp')
+            or name.endswith('.cmake')
+            or name.endswith('jpg')
+            or name.endswith('png')
+            or name.endswith('libaf.so') #avoids duplicates due to symlinks
+            or re.match('.*libaf\.so\.3\..*', name) is not None
+            or name.endswith('libafcpu.so')
+            or re.match('.*libafcpu\.so\.3\..*', name) is not None
+            or name.endswith('libafcuda.so')
+            or re.match('.*libafcuda\.so\.3\..*', name) is not None
+            or name.endswith('libafopencl.so')
+            or re.match('.*libafopencl\.so\.3\..*', name) is not None
+            or name.endswith('libforge.so')
+            or re.match('.*libforge\.so\.1\..*', name) is not None
+            or 'examples' in name), cmake_manifest))
+        return cmake_manifest
+
+    print('Building CMAKE with following configurable variables: ')
+    print(AF_BUILD_CPU_CMAKE_STR)
+    print(AF_BUILD_CUDA_CMAKE_STR)
+    print(AF_BUILD_OPENCL_CMAKE_STR)
+    print(AF_BUILD_UNIFIED_CMAKE_STR)
+
+
+    setup(
+        packages=['arrayfire'],
+        cmake_install_dir='',
+        cmake_process_manifest_hook=filter_af_files,
+        include_package_data=False,
+        cmake_args=[AF_BUILD_CPU_CMAKE_STR,
+                    AF_BUILD_CUDA_CMAKE_STR,
+                    AF_BUILD_OPENCL_CMAKE_STR,
+                    AF_BUILD_UNIFIED_CMAKE_STR,
+                    # todo: pass additional args from environ
+                    '-DCMAKE_BUILD_TYPE:STRING="RelWithDebInfo"',
+                    '-DFG_USE_STATIC_CPPFLAGS:BOOL=OFF',
+                    '-DFG_WITH_FREEIMAGE:BOOL=OFF',
+                    '-DCUDA_architecture_build_targets:STRING=All', 
+                    '-DAF_BUILD_DOCS:BOOL=OFF',
+                    '-DAF_BUILD_EXAMPLES:BOOL=OFF',
+                    '-DAF_INSTALL_STANDALONE:BOOL=ON',
+                    '-DAF_WITH_IMAGEIO:BOOL=ON',
+                    '-DAF_WITH_LOGGING:BOOL=ON',
+                    '-DBUILD_TESTING:BOOL=OFF',
+                    '-DAF_BUILD_FORGE:BOOL=ON',
+                    '-DAF_INSTALL_LIB_DIR:STRING=arrayfire',
+                    '-DAF_INSTALL_BIN_DIR:STRING=arrayfire',
+                    '-DFG_INSTALL_LIB_DIR:STRING=arrayfire',
+                    '-DAF_WITH_STATIC_MKL=ON',
+                    ]
+    )
+
+else:
+    # ignores local arrayfire libraries, will search system instead
+    from setuptools import setup
+    setup()
 
-setup()