MNT CI dropping python-3.5 (scikit-learn#15106)

* dropping python-3.5

* fix install guide

* fix setup.py

* fix circleci

* advanced_installation

* index.html

* readme

* pyparsing.py

* remove clean_warning_registry

* 1.13.1 and 1.19.1

* don't use 16.04

* 18.04 libatlas-dev -> libatlas-base-dev

* min pillow version for 3.6 is 4.2.1

* echo commands

* fix conflict, and 32 bit

* fix conflict for circle

* fix scikit-image version dep

* mark tests as xfail on 32bit py3.6

* move to 1.13.3 min numpy version, and simplify old code

* remove the rest of _object_dtype_isnan usages

* Revert "remove the rest of _object_dtype_isnan usages"

This reverts commit c6e867e.

* fix issues raised by jeremy

* minor fix

* minor fixes mentioned by Olivier

Author: adrinjalali

.circleci/config.yml

@@ -9,17 +9,17 @@ jobs:
       - MKL_NUM_THREADS: 2
       - MINICONDA_PATH: ~/miniconda
       - CONDA_ENV_NAME: testenv
-      - PYTHON_VERSION: 3.5
-      - NUMPY_VERSION: 1.11.0
-      - SCIPY_VERSION: 0.17.0
-      - MATPLOTLIB_VERSION: 1.5.1
+      - PYTHON_VERSION: 3.6
+      - NUMPY_VERSION: 1.13.3
+      - SCIPY_VERSION: 0.19.1
+      - MATPLOTLIB_VERSION: 2.1.1
       # on conda, this is the latest for python 3.5
       # The following places need to be in sync with regard to Cython version:
       # - .circleci config file
       # - sklearn/_build_utils/__init__.py
       # - advanced installation guide
       - CYTHON_VERSION: 0.28.5
-      - SCIKIT_IMAGE_VERSION: 0.12.3
+      - SCIKIT_IMAGE_VERSION: 0.13
     steps:
       - checkout
       - run: ./build_tools/circle/checkout_merge_commit.sh

README.rst

@@ -47,18 +47,18 @@ Dependencies
 
 scikit-learn requires:
 
-- Python (>= 3.5)
-- NumPy (>= 1.11.0)
-- SciPy (>= 0.17.0)
+- Python (>= 3.6)
+- NumPy (>= 1.13.3)
+- SciPy (>= 0.19.1)
 - joblib (>= 0.11)
 
 **Scikit-learn 0.20 was the last version to support Python 2.7 and Python 3.4.**
-scikit-learn 0.21 and later require Python 3.5 or newer.
+scikit-learn 0.23 and later require Python 3.6 or newer.
 
 Scikit-learn plotting capabilities (i.e., functions start with ``plot_``
-and classes end with "Display") require Matplotlib (>= 1.5.1). For running the
-examples Matplotlib >= 1.5.1 is required. A few examples require
-scikit-image >= 0.12.3, a few examples require pandas >= 0.18.0.
+and classes end with "Display") require Matplotlib (>= 2.1.1). For running the
+examples Matplotlib >= 2.1.1 is required. A few examples require
+scikit-image >= 0.13, a few examples require pandas >= 0.18.0.
 
 User installation
 ~~~~~~~~~~~~~~~~~

azure-pipelines.yml

@@ -3,7 +3,7 @@ jobs:
 - job: linting
   displayName: Linting
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
   steps:
     - bash: echo "##vso[task.prependpath]$CONDA/bin"
       displayName: Add conda to PATH
@@ -27,7 +27,7 @@ jobs:
 - template: build_tools/azure/posix.yml
   parameters:
     name: Linux_Runs
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
     matrix:
       pylatest_conda_mkl:
         DISTRIB: 'conda'
@@ -44,31 +44,31 @@ jobs:
 - template: build_tools/azure/posix.yml
   parameters:
     name: Linux
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
     dependsOn: [linting]
     matrix:
       # Linux environment to test that scikit-learn can be built against
-      # versions of numpy, scipy with ATLAS that comes with Ubuntu Xenial 16.04
-      # i.e. numpy 1.11 and scipy 0.17
-      py35_ubuntu_atlas:
+      # versions of numpy, scipy with ATLAS that comes with Ubuntu Bionic 18.04
+      # i.e. numpy 1.13.3 and scipy 0.19
+      py36_ubuntu_atlas:
         DISTRIB: 'ubuntu'
-        PYTHON_VERSION: '3.5'
+        PYTHON_VERSION: '3.6'
         JOBLIB_VERSION: '0.11'
-      # Linux + Python 3.5 build with OpenBLAS and without SITE_JOBLIB
-      py35_conda_openblas:
+      # Linux + Python 3.6 build with OpenBLAS and without SITE_JOBLIB
+      py36_conda_openblas:
         DISTRIB: 'conda'
-        PYTHON_VERSION: '3.5'
+        PYTHON_VERSION: '3.6'
         BLAS: 'openblas'
-        NUMPY_VERSION: '1.11.0'
-        SCIPY_VERSION: '0.17.0'
+        NUMPY_VERSION: '1.13.3'
+        SCIPY_VERSION: '0.19.1'
         PANDAS_VERSION: '*'
         CYTHON_VERSION: '*'
         # temporary pin pytest due to unknown failure with pytest 5.3
         PYTEST_VERSION: '5.2'
-        PILLOW_VERSION: '4.0.0'
-        MATPLOTLIB_VERSION: '1.5.1'
-        # later version of joblib are not packaged in conda for Python 3.5
-        JOBLIB_VERSION: '0.12.3'
+        PILLOW_VERSION: '4.2.1'
+        MATPLOTLIB_VERSION: '2.1.1'
+        # latest version of joblib available in conda for Python 3.6
+        JOBLIB_VERSION: '0.13.2'
         COVERAGE: 'true'
       # Linux environment to test the latest available dependencies and MKL.
       # It runs tests requiring lightgbm, pandas and PyAMG.
@@ -84,13 +84,13 @@ jobs:
 - template: build_tools/azure/posix-32.yml
   parameters:
     name: Linux32
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
     dependsOn: [linting]
     matrix:
-      py35_ubuntu_atlas_32bit:
+      py36_ubuntu_atlas_32bit:
         DISTRIB: 'ubuntu-32'
-        PYTHON_VERSION: '3.5'
-        JOBLIB_VERSION: '0.11'
+        PYTHON_VERSION: '3.6'
+        JOBLIB_VERSION: '0.13'
 
 - template: build_tools/azure/posix.yml
   parameters:
@@ -135,6 +135,6 @@ jobs:
         PYTHON_ARCH: '64'
         PYTEST_VERSION: '*'
         COVERAGE: 'true'
-      py35_pip_openblas_32bit:
-        PYTHON_VERSION: '3.5'
+      py36_pip_openblas_32bit:
+        PYTHON_VERSION: '3.6'
         PYTHON_ARCH: '32'

build_tools/azure/install.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 set -e
+set -x
 
 UNAMESTR=`uname`
 
@@ -73,13 +74,13 @@ if [[ "$DISTRIB" == "conda" ]]; then
 elif [[ "$DISTRIB" == "ubuntu" ]]; then
     sudo add-apt-repository --remove ppa:ubuntu-toolchain-r/test
     sudo apt-get update
-    sudo apt-get install python3-scipy python3-matplotlib libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv
+    sudo apt-get install python3-scipy python3-matplotlib libatlas3-base libatlas-base-dev python3-virtualenv
     python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV
     source $VIRTUALENV/bin/activate
     python -m pip install pytest==$PYTEST_VERSION pytest-cov cython joblib==$JOBLIB_VERSION
 elif [[ "$DISTRIB" == "ubuntu-32" ]]; then
     apt-get update
-    apt-get install -y python3-dev python3-scipy python3-matplotlib libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv
+    apt-get install -y python3-dev python3-scipy python3-matplotlib libatlas3-base libatlas-base-dev python3-virtualenv
     python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV
     source $VIRTUALENV/bin/activate
     python -m pip install pytest==$PYTEST_VERSION pytest-cov cython joblib==$JOBLIB_VERSION

build_tools/azure/posix-32.yml

@@ -40,7 +40,7 @@ jobs:
         -e OMP_NUM_THREADS=$OMP_NUM_THREADS
         -e OPENBLAS_NUM_THREADS=$OPENBLAS_NUM_THREADS
         -e SKLEARN_SKIP_NETWORK_TESTS=$SKLEARN_SKIP_NETWORK_TESTS
-        i386/ubuntu:16.04
+        i386/ubuntu:18.04
         sleep 1000000
       displayName: 'Start container'
     - script: >

build_tools/circle/build_test_pypy.sh

@@ -3,7 +3,7 @@ set -x
 set -e
 
 apt-get -yq update
-apt-get -yq install libatlas-dev libatlas-base-dev liblapack-dev gfortran ccache libopenblas-dev
+apt-get -yq install libatlas-base-dev liblapack-dev gfortran ccache libopenblas-dev
 
 pip install virtualenv
 

doc/developers/advanced_installation.rst

@@ -83,9 +83,9 @@ Runtime dependencies
 Scikit-learn requires the following dependencies both at build time and at
 runtime:
 
-- Python (>= 3.5),
-- NumPy (>= 1.11),
-- SciPy (>= 0.17),
+- Python (>= 3.6),
+- NumPy (>= 1.13.3),
+- SciPy (>= 0.19),
 - Joblib (>= 0.11).
 
 Those dependencies are **automatically installed by pip** if they were missing

doc/install.rst

@@ -132,14 +132,16 @@ If you must install scikit-learn and its dependencies with pip, you can install
 it as ``scikit-learn[alldeps]``.
 
 Scikit-learn plotting capabilities (i.e., functions start with "plot\_"
-and classes end with "Display") require Matplotlib (>= 1.5.1). For running the
-examples Matplotlib >= 1.5.1 is required. A few examples require
-scikit-image >= 0.12.3, a few examples require pandas >= 0.18.0.
+and classes end with "Display") require Matplotlib (>= 2.1.1). For running the
+examples Matplotlib >= 2.1.1 is required. A few examples require
+scikit-image >= 0.13, a few examples require pandas >= 0.18.0.
 
 .. warning::
 
     Scikit-learn 0.20 was the last version to support Python 2.7 and Python 3.4.
-    Scikit-learn now requires Python 3.5 or newer.
+    Scikit-learn 0.21 supported Python 3.5-3.7.
+    Scikit-learn 0.22 supported Python 3.5-3.8.
+    Scikit-learn now requires Python 3.6 or newer.
 
 
 .. note::

doc/templates/index.html

@@ -156,6 +156,8 @@ <h4 class="sk-landing-call-header">News</h4>
         <li><strong>On-going development:</strong>
         <a href="https://scikit-learn.org/dev/whats_new.html"><strong>What's new</strong> (Changelog)</a>
         </li>
+        <li><strong>Scikit-learn from 0.23 requires Python 3.6 or greater.</strong>
+        </li>
         <li><strong>January 2020.</strong> scikit-learn 0.22.1 is available for download (<a href="whats_new/v0.22.html#version-0-22-1">Changelog</a>).
         <li><strong>December 2019.</strong> scikit-learn 0.22 is available for download (<a href="whats_new/v0.22.html#version-0-22-0">Changelog</a>).
         </li>

doc/tutorial/machine_learning_map/pyparsing.py

@@ -1020,20 +1020,14 @@ def _trim_arity(func, maxargs=2):
     limit = [0]
     foundArity = [False]
 
-    # traceback return data structure changed in Py3.5 - normalize back to plain tuples
-    if system_version[:2] >= (3,5):
-        def extract_stack(limit=0):
-            # special handling for Python 3.5.0 - extra deep call stack by 1
-            offset = -3 if system_version == (3,5,0) else -2
-            frame_summary = traceback.extract_stack(limit=-offset+limit-1)[offset]
-            return [(frame_summary.filename, frame_summary.lineno)]
-        def extract_tb(tb, limit=0):
-            frames = traceback.extract_tb(tb, limit=limit)
-            frame_summary = frames[-1]
-            return [(frame_summary.filename, frame_summary.lineno)]
-    else:
-        extract_stack = traceback.extract_stack
-        extract_tb = traceback.extract_tb
+    def extract_stack(limit=0):
+        offset = -2
+        frame_summary = traceback.extract_stack(limit=-offset+limit-1)[offset]
+        return [(frame_summary.filename, frame_summary.lineno)]
+    def extract_tb(tb, limit=0):
+        frames = traceback.extract_tb(tb, limit=limit)
+        frame_summary = frames[-1]
+        return [(frame_summary.filename, frame_summary.lineno)]
 
     # synthesize what would be returned by traceback.extract_stack at the call to 
     # user's parse action 'func', so that we don't incur call penalty at parse time

setup.py

@@ -15,7 +15,7 @@
 try:
     import builtins
 except ImportError:
-    # Python 2 compat: just to be able to declare that Python >=3.5 is needed.
+    # Python 2 compat: just to be able to declare that Python >=3.6 is needed.
     import __builtin__ as builtins
 
 # This is a bit (!) hackish: we are setting a global variable so that the
@@ -52,8 +52,8 @@
     SCIPY_MIN_VERSION = '1.1.0'
     NUMPY_MIN_VERSION = '1.14.0'
 else:
-    SCIPY_MIN_VERSION = '0.17.0'
-    NUMPY_MIN_VERSION = '1.11.0'
+    SCIPY_MIN_VERSION = '0.19.1'
+    NUMPY_MIN_VERSION = '1.13.3'
 
 JOBLIB_MIN_VERSION = '0.11'
 
@@ -139,7 +139,7 @@ def build_extensions(self):
 
 except ImportError:
     # Numpy should not be a dependency just to be able to introspect
-    # that python 3.5 is required.
+    # that python 3.6 is required.
     pass
 
 
@@ -244,7 +244,6 @@ def setup_package():
                                  'Operating System :: Unix',
                                  'Operating System :: MacOS',
                                  'Programming Language :: Python :: 3',
-                                 'Programming Language :: Python :: 3.5',
                                  'Programming Language :: Python :: 3.6',
                                  'Programming Language :: Python :: 3.7',
                                  'Programming Language :: Python :: 3.8',
@@ -254,7 +253,7 @@ def setup_package():
                                   'Implementation :: PyPy')
                                  ],
                     cmdclass=cmdclass,
-                    python_requires=">=3.5",
+                    python_requires=">=3.6",
                     install_requires=[
                         'numpy>={}'.format(NUMPY_MIN_VERSION),
                         'scipy>={}'.format(SCIPY_MIN_VERSION),
@@ -282,9 +281,9 @@ def setup_package():
 
         metadata['version'] = VERSION
     else:
-        if sys.version_info < (3, 5):
+        if sys.version_info < (3, 6):
             raise RuntimeError(
-                "Scikit-learn requires Python 3.5 or later. The current"
+                "Scikit-learn requires Python 3.6 or later. The current"
                 " Python version is %s installed in %s."
                 % (platform.python_version(), sys.executable))
 

sklearn/feature_extraction/tests/test_text.py

@@ -32,7 +32,6 @@
 from sklearn.utils import IS_PYPY
 from sklearn.utils._testing import (assert_almost_equal,
                                    assert_warns_message, assert_raise_message,
-                                   clean_warning_registry,
                                    SkipTest, assert_no_warnings,
                                    fails_if_pypy, assert_allclose_dense_sparse,
                                    skip_if_32bit)
@@ -388,7 +387,6 @@ def test_tfidf_no_smoothing():
          [1, 0, 0]]
     tr = TfidfTransformer(smooth_idf=False, norm='l2')
 
-    clean_warning_registry()
     with warnings.catch_warnings(record=True) as w:
         1. / np.array([0.])
         numpy_provides_div0_warning = len(w) == 1

sklearn/gaussian_process/tests/test_gpr.py

@@ -3,6 +3,7 @@
 # Author: Jan Hendrik Metzen <[email protected]>
 # License: BSD 3 clause
 
+import sys
 import numpy as np
 
 from scipy.optimize import approx_fprime
@@ -46,6 +47,9 @@ def f(x):
 
 @pytest.mark.parametrize('kernel', kernels)
 def test_gpr_interpolation(kernel):
+    if sys.maxsize <= 2 ** 32 and sys.version_info[:2] == (3, 6):
+        pytest.xfail("This test may fail on 32bit Py3.6")
+
     # Test the interpolating property for different kernels.
     gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
     y_pred, y_cov = gpr.predict(X, return_cov=True)
@@ -70,6 +74,9 @@ def test_gpr_interpolation_structured():
 
 @pytest.mark.parametrize('kernel', non_fixed_kernels)
 def test_lml_improving(kernel):
+    if sys.maxsize <= 2 ** 32 and sys.version_info[:2] == (3, 6):
+        pytest.xfail("This test may fail on 32bit Py3.6")
+
     # Test that hyperparameter-tuning improves log-marginal likelihood.
     gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
     assert (gpr.log_marginal_likelihood(gpr.kernel_.theta) >
@@ -175,6 +182,9 @@ def test_no_optimizer():
 
 @pytest.mark.parametrize('kernel', kernels)
 def test_predict_cov_vs_std(kernel):
+    if sys.maxsize <= 2 ** 32 and sys.version_info[:2] == (3, 6):
+        pytest.xfail("This test may fail on 32bit Py3.6")
+
     # Test that predicted std.-dev. is consistent with cov's diagonal.
     gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
     y_mean, y_cov = gpr.predict(X2, return_cov=True)

sklearn/utils/_mask.py

@@ -5,7 +5,7 @@
 
 
 def _get_mask(X, value_to_mask):
-    """Compute the boolean mask X == missing_values."""
+    """Compute the boolean mask X == value_to_mask."""
     if is_scalar_nan(value_to_mask):
         if X.dtype.kind == "f":
             return np.isnan(X)
@@ -16,6 +16,4 @@ def _get_mask(X, value_to_mask):
             # np.isnan does not work on object dtypes.
             return _object_dtype_isnan(X)
     else:
-        # X == value_to_mask with object dtypes does not always perform
-        # element-wise for old versions of numpy
-        return np.equal(X, value_to_mask)
+        return X == value_to_mask