MNT Include all pxd files in the package (scikit-learn#15626)

* include all pxd files in package

* _libsvm _liblinear to pxi + add a test

* simplify stuff + use pathlib

* cln

* more cln

Author: jeremiedbb

maint_tools/check_pxd_in_installation.py

@@ -0,0 +1,59 @@
+"""Utility for testing presence and usability of .pxd files in the installation
+
+Usage:
+------
+python check_pxd_in_installation.py path/to/install_dir/of/scikit-learn
+"""
+
+import os
+import sys
+import pathlib
+import tempfile
+import textwrap
+import subprocess
+
+
+sklearn_dir = pathlib.Path(sys.argv[1])
+pxd_files = list(sklearn_dir.glob("**/*.pxd"))
+
+print("> Found pxd files:")
+for pxd_file in pxd_files:
+    print(' -', pxd_file)
+
+print("\n> Trying to compile a cython extension cimporting all corresponding "
+      "modules\n")
+with tempfile.TemporaryDirectory() as tmpdir:
+    tmpdir = pathlib.Path(tmpdir)
+    # A cython test file which cimports all modules corresponding to found
+    # pxd files.
+    # e.g. sklearn/tree/_utils.pxd becomes `cimport sklearn.tree._utils`
+    with open(tmpdir / 'tst.pyx', 'w') as f:
+        for pxd_file in pxd_files:
+            to_import = str(pxd_file.relative_to(sklearn_dir))
+            to_import = to_import.replace(os.path.sep, '.')
+            to_import = to_import.replace('.pxd', '')
+            f.write('cimport sklearn.' + to_import + '\n')
+
+    # A basic setup file to build the test file.
+    # We set the language to c++ and we use numpy.get_include() because
+    # some modules require it.
+    with open(tmpdir / 'setup_tst.py', 'w') as f:
+        f.write(textwrap.dedent(
+            """
+            from distutils.core import setup
+            from distutils.extension import Extension
+            from Cython.Build import cythonize
+            import numpy
+
+            extensions = [Extension("tst",
+                                    sources=["tst.pyx"],
+                                    language="c++",
+                                    include_dirs=[numpy.get_include()])]
+
+            setup(ext_modules=cythonize(extensions))
+            """))
+
+    subprocess.run(["python", "setup_tst.py", "build_ext", "-i"],
+                   check=True, cwd=tmpdir)
+
+    print("\n> Compilation succeeded !")

setup.py

@@ -259,6 +259,7 @@ def setup_package():
                         'scipy>={}'.format(SCIPY_MIN_VERSION),
                         'joblib>={}'.format(JOBLIB_MIN_VERSION)
                     ],
+                    package_data={'': ['*.pxd']},
                     **extra_setuptools_args)
 
     if len(sys.argv) == 1 or (

sklearn/svm/_liblinear.pxd renamed to sklearn/svm/_liblinear.pxi

@@ -1,6 +1,3 @@
-cimport numpy as np
-
-
 cdef extern from "_cython_blas_helpers.h":
     ctypedef double (*dot_func)(int, double*, int, double*, int)
     ctypedef void (*axpy_func)(int, double, double*, int, double*, int)
@@ -12,6 +9,7 @@ cdef extern from "_cython_blas_helpers.h":
         scal_func scal
         nrm2_func nrm2
 
+
 cdef extern from "linear.h":
     cdef struct feature_node
     cdef struct problem
@@ -28,6 +26,7 @@ cdef extern from "linear.h":
     void free_and_destroy_model (model **)
     void destroy_param (parameter *)
 
+
 cdef extern from "liblinear_helper.c":
     void copy_w(void *, model *, int)
     parameter *set_parameter(int, double, double, int, char *, char *, int, int, double)

sklearn/svm/_liblinear.pyx

@@ -9,6 +9,8 @@ cimport numpy as np
 
 from ..utils._cython_blas cimport _dot, _axpy, _scal, _nrm2
 
+include "_liblinear.pxi"
+
 np.import_array()
 
 

sklearn/svm/_libsvm.pxd renamed to sklearn/svm/_libsvm.pxi

@@ -1,5 +1,3 @@
-cimport numpy as np
-
 ################################################################################
 # Includes
 

sklearn/svm/_libsvm.pyx

@@ -35,6 +35,8 @@ import  numpy as np
 cimport numpy as np
 from libc.stdlib cimport free
 
+include "_libsvm.pxi"
+
 cdef extern from *:
     ctypedef struct svm_parameter:
         pass

sklearn/tree/setup.py

@@ -31,10 +31,6 @@ def configuration(parent_package="", top_path=None):
                          extra_compile_args=["-O3"])
 
     config.add_subpackage("tests")
-    config.add_data_files("_criterion.pxd")
-    config.add_data_files("_splitter.pxd")
-    config.add_data_files("_tree.pxd")
-    config.add_data_files("_utils.pxd")
 
     return config
 

sklearn/utils/_weight_vector.pxd

@@ -1,12 +1,5 @@
 """Efficient (dense) parameter vector implementation for linear models. """
 
-cimport numpy as np
-
-
-cdef extern from "math.h":
-    cdef extern double sqrt(double x)
-
-
 cdef class WeightVector(object):
     cdef double *w_data_ptr
     cdef double *aw_data_ptr