Merge remote-tracking branch 'upstream/main' into bisect

Author: simonjayhawkins

.github/workflows/32-bit-linux.yml

@@ -23,9 +23,15 @@ jobs:
 
       - name: Run 32-bit manylinux2014 Docker Build / Tests
         run: |
+          # Without this (line 34), versioneer will not be able to determine the pandas version.
+          # This is because of a security update to git that blocks it from reading the config folder if
+          # it is not owned by the current user. We hit this since the "mounted" folder is not hit by the
+          # Docker container.
+          # xref https://github.com/pypa/manylinux/issues/1309
           docker pull quay.io/pypa/manylinux2014_i686
           docker run --platform linux/386 -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \
           /bin/bash -xc "cd pandas && \
+          git config --global --add safe.directory /pandas && \
           /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
           . ~/virtualenvs/pandas-dev/bin/activate && \
           python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \

.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
     -   id: absolufy-imports
         files: ^pandas/
 -   repo: https://github.com/jendrikseipp/vulture
-    rev: 'v2.3'
+    rev: 'v2.4'
     hooks:
       - id: vulture
         entry: python scripts/run_vulture.py
@@ -60,7 +60,7 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.32.0
+    rev: v2.32.1
     hooks:
     -   id: pyupgrade
         args: [--py38-plus]
@@ -75,7 +75,7 @@ repos:
         types: [text]  # overwrite types: [rst]
         types_or: [python, rst]
 -   repo: https://github.com/sphinx-contrib/sphinx-lint
-    rev: v0.4.1
+    rev: v0.6
     hooks:
     - id: sphinx-lint
 -   repo: https://github.com/asottile/yesqa

asv_bench/benchmarks/reshape.py

@@ -268,7 +268,9 @@ def setup(self, bins):
         self.datetime_series = pd.Series(
             np.random.randint(N, size=N), dtype="datetime64[ns]"
         )
-        self.interval_bins = pd.IntervalIndex.from_breaks(np.linspace(0, N, bins))
+        self.interval_bins = pd.IntervalIndex.from_breaks(
+            np.linspace(0, N, bins), "right"
+        )
 
     def time_cut_int(self, bins):
         pd.cut(self.int_series, bins)

ci/deps/actions-310.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.10
 
   # test dependencies
-  - cython=0.29.24
+  - cython=0.29.30
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-38-downstream_compat.yaml

@@ -6,7 +6,7 @@ dependencies:
   - python=3.8
 
   # test dependencies
-  - cython=0.29.24
+  - cython=0.29.30
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-38-minimum_versions.yaml

@@ -7,7 +7,7 @@ dependencies:
   - python=3.8.0
 
   # test dependencies
-  - cython=0.29.24
+  - cython=0.29.30
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-38.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.8
 
   # test dependencies
-  - cython=0.29.24
+  - cython=0.29.30
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-39.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.9
 
   # test dependencies
-  - cython=0.29.24
+  - cython=0.29.30
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-pypy-38.yaml

@@ -8,7 +8,7 @@ dependencies:
   - python=3.8[build=*_pypy] # TODO: use this once pypy3.8 is available
 
   # tools
-  - cython>=0.29.24
+  - cython>=0.29.30
   - pytest>=6.0
   - pytest-cov
   - pytest-asyncio

ci/deps/circle-38-arm64.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.8
 
   # test dependencies
-  - cython=0.29.24
+  - cython=0.29.30
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

doc/redirects.csv

@@ -741,11 +741,11 @@ generated/pandas.Index.values,../reference/api/pandas.Index.values
 generated/pandas.Index.view,../reference/api/pandas.Index.view
 generated/pandas.Index.where,../reference/api/pandas.Index.where
 generated/pandas.infer_freq,../reference/api/pandas.infer_freq
-generated/pandas.Interval.closed,../reference/api/pandas.Interval.closed
+generated/pandas.Interval.inclusive,../reference/api/pandas.Interval.inclusive
 generated/pandas.Interval.closed_left,../reference/api/pandas.Interval.closed_left
 generated/pandas.Interval.closed_right,../reference/api/pandas.Interval.closed_right
 generated/pandas.Interval,../reference/api/pandas.Interval
-generated/pandas.IntervalIndex.closed,../reference/api/pandas.IntervalIndex.closed
+generated/pandas.IntervalIndex.inclusive,../reference/api/pandas.IntervalIndex.inclusive
 generated/pandas.IntervalIndex.contains,../reference/api/pandas.IntervalIndex.contains
 generated/pandas.IntervalIndex.from_arrays,../reference/api/pandas.IntervalIndex.from_arrays
 generated/pandas.IntervalIndex.from_breaks,../reference/api/pandas.IntervalIndex.from_breaks

doc/source/reference/arrays.rst

@@ -303,7 +303,7 @@ Properties
 .. autosummary::
    :toctree: api/
 
-   Interval.closed
+   Interval.inclusive
    Interval.closed_left
    Interval.closed_right
    Interval.is_empty
@@ -340,7 +340,7 @@ A collection of intervals may be stored in an :class:`arrays.IntervalArray`.
 
       arrays.IntervalArray.left
       arrays.IntervalArray.right
-      arrays.IntervalArray.closed
+      arrays.IntervalArray.inclusive
       arrays.IntervalArray.mid
       arrays.IntervalArray.length
       arrays.IntervalArray.is_empty

doc/source/reference/indexing.rst

@@ -242,7 +242,7 @@ IntervalIndex components
    IntervalIndex.left
    IntervalIndex.right
    IntervalIndex.mid
-   IntervalIndex.closed
+   IntervalIndex.inclusive
    IntervalIndex.length
    IntervalIndex.values
    IntervalIndex.is_empty

doc/source/reference/testing.rst

@@ -41,6 +41,7 @@ Exceptions and warnings
    errors.ParserError
    errors.ParserWarning
    errors.PerformanceWarning
+   errors.SettingWithCopyError
    errors.SpecificationError
    errors.UnsortedIndexError
    errors.UnsupportedFunctionCall

doc/source/user_guide/advanced.rst

@@ -1020,7 +1020,7 @@ Trying to select an ``Interval`` that is not exactly contained in the ``Interval
 
    In [7]: df.loc[pd.Interval(0.5, 2.5)]
    ---------------------------------------------------------------------------
-   KeyError: Interval(0.5, 2.5, closed='right')
+   KeyError: Interval(0.5, 2.5, inclusive='right')
 
 Selecting all ``Intervals`` that overlap a given ``Interval`` can be performed using the
 :meth:`~IntervalIndex.overlaps` method to create a boolean indexer.

doc/source/user_guide/indexing.rst

@@ -1885,7 +1885,7 @@ chained indexing expression, you can set the :ref:`option <options>`
 ``mode.chained_assignment`` to one of these values:
 
 * ``'warn'``, the default, means a ``SettingWithCopyWarning`` is printed.
-* ``'raise'`` means pandas will raise a ``SettingWithCopyException``
+* ``'raise'`` means pandas will raise a ``SettingWithCopyError``
   you have to deal with.
 * ``None`` will suppress the warnings entirely.
 
@@ -1953,7 +1953,7 @@ Last, the subsequent example will **not** work at all, and so should be avoided:
    >>> dfd.loc[0]['a'] = 1111
    Traceback (most recent call last)
         ...
-   SettingWithCopyException:
+   SettingWithCopyError:
         A value is trying to be set on a copy of a slice from a DataFrame.
         Try using .loc[row_index,col_indexer] = value instead
 

doc/source/whatsnew/v0.20.0.rst

@@ -448,7 +448,7 @@ Selecting via a specific interval:
 
 .. ipython:: python
 
-   df.loc[pd.Interval(1.5, 3.0)]
+   df.loc[pd.Interval(1.5, 3.0, "right")]
 
 Selecting via a scalar value that is contained *in* the intervals.
 

doc/source/whatsnew/v0.25.0.rst

@@ -584,18 +584,18 @@ this would previously return ``True`` for any ``Interval`` overlapping an ``Inte
 
 .. code-block:: python
 
-   In [4]: pd.Interval(1, 2, closed='neither') in ii
+   In [4]: pd.Interval(1, 2, inclusive='neither') in ii
    Out[4]: True
 
-   In [5]: pd.Interval(-10, 10, closed='both') in ii
+   In [5]: pd.Interval(-10, 10, inclusive='both') in ii
    Out[5]: True
 
 *New behavior*:
 
 .. ipython:: python
 
-   pd.Interval(1, 2, closed='neither') in ii
-   pd.Interval(-10, 10, closed='both') in ii
+   pd.Interval(1, 2, inclusive='neither') in ii
+   pd.Interval(-10, 10, inclusive='both') in ii
 
 The :meth:`~IntervalIndex.get_loc` method now only returns locations for exact matches to ``Interval`` queries, as opposed to the previous behavior of
 returning locations for overlapping matches.  A ``KeyError`` will be raised if an exact match is not found.
@@ -619,7 +619,7 @@ returning locations for overlapping matches.  A ``KeyError`` will be raised if a
 
    In [7]: ii.get_loc(pd.Interval(2, 6))
    ---------------------------------------------------------------------------
-   KeyError: Interval(2, 6, closed='right')
+   KeyError: Interval(2, 6, inclusive='right')
 
 Likewise, :meth:`~IntervalIndex.get_indexer` and :meth:`~IntervalIndex.get_indexer_non_unique` will also only return locations for exact matches
 to ``Interval`` queries, with ``-1`` denoting that an exact match was not found.
@@ -680,11 +680,11 @@ Similarly, a ``KeyError`` will be raised for non-exact matches instead of return
 
    In [6]: s[pd.Interval(2, 3)]
    ---------------------------------------------------------------------------
-   KeyError: Interval(2, 3, closed='right')
+   KeyError: Interval(2, 3, inclusive='right')
 
    In [7]: s.loc[pd.Interval(2, 3)]
    ---------------------------------------------------------------------------
-   KeyError: Interval(2, 3, closed='right')
+   KeyError: Interval(2, 3, inclusive='right')
 
 The :meth:`~IntervalIndex.overlaps` method can be used to create a boolean indexer that replicates the
 previous behavior of returning overlapping matches.

doc/source/whatsnew/v1.4.3.rst

@@ -19,16 +19,18 @@ Fixed regressions
 - Fixed regression in :func:`read_fwf` raising ``ValueError`` when ``widths`` was specified with ``usecols`` (:issue:`46580`)
 - Fixed regression in :meth:`.Groupby.transform` and :meth:`.Groupby.agg` failing with ``engine="numba"`` when the index was a :class:`MultiIndex` (:issue:`46867`)
 - Fixed regression is :meth:`.Styler.to_latex` and :meth:`.Styler.to_html` where ``buf`` failed in combination with ``encoding`` (:issue:`47053`)
+- Fixed regression in :func:`read_csv` with ``index_col=False`` identifying first row as index names when ``header=None`` (:issue:`46955`)
 - Fixed regression in :meth:`.DataFrameGroupBy.agg` when used with list-likes or dict-likes and ``axis=1`` that would give incorrect results; now raises ``NotImplementedError`` (:issue:`46995`)
 - Fixed regression in :meth:`DataFrame.resample` and :meth:`DataFrame.rolling` when used with list-likes or dict-likes and ``axis=1`` that would raise an unintuitive error message; now raises ``NotImplementedError`` (:issue:`46904`)
+- Fixed regression in :meth:`DataFrame.shift` when ``axis`` is ``columns`` and ``fill_value`` is absent, ``freq`` is ignored (:issue:`47039`)
 
 .. ---------------------------------------------------------------------------
 
 .. _whatsnew_143.bug_fixes:
 
 Bug fixes
 ~~~~~~~~~
--
+- Most I/O methods do no longer suppress ``OSError`` and ``ValueError`` when closing file handles (:issue:`47136`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -37,7 +39,7 @@ Bug fixes
 
 Other
 ~~~~~
--
+- The minimum version of Cython needed to compile pandas is now ``0.29.30`` (:issue:`41935`)
 -
 
 .. ---------------------------------------------------------------------------

doc/source/whatsnew/v1.5.0.rst

@@ -151,7 +151,7 @@ Other enhancements
 - A :class:`errors.PerformanceWarning` is now thrown when using ``string[pyarrow]`` dtype with methods that don't dispatch to ``pyarrow.compute`` methods (:issue:`42613`)
 - Added ``numeric_only`` argument to :meth:`Resampler.sum`, :meth:`Resampler.prod`, :meth:`Resampler.min`, :meth:`Resampler.max`, :meth:`Resampler.first`, and :meth:`Resampler.last` (:issue:`46442`)
 - ``times`` argument in :class:`.ExponentialMovingWindow` now accepts ``np.timedelta64`` (:issue:`47003`)
-- :class:`DataError` and :class:`SpecificationError` are now exposed in ``pandas.errors`` (:issue:`27656`)
+- :class:`DataError`, :class:`SpecificationError`, and :class:`SettingWithCopyError` are now exposed in ``pandas.errors`` (:issue:`27656`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_150.notable_bug_fixes:
@@ -362,7 +362,7 @@ If installed, we now require:
 +-----------------+-----------------+----------+---------+
 | Package         | Minimum Version | Required | Changed |
 +=================+=================+==========+=========+
-| mypy (dev)      | 0.950           |          |    X    |
+| mypy (dev)      | 0.960           |          |    X    |
 +-----------------+-----------------+----------+---------+
 | beautifulsoup4  | 4.9.3           |          |    X    |
 +-----------------+-----------------+----------+---------+
@@ -669,7 +669,12 @@ Other Deprecations
 - Deprecated the methods :meth:`DataFrame.mad`, :meth:`Series.mad`, and the corresponding groupby methods (:issue:`11787`)
 - Deprecated positional arguments to :meth:`Index.join` except for ``other``, use keyword-only arguments instead of positional arguments (:issue:`46518`)
 - Deprecated indexing on a timezone-naive :class:`DatetimeIndex` using a string representing a timezone-aware datetime (:issue:`46903`, :issue:`36148`)
--
+- Deprecated the ``closed`` argument in :class:`Interval` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
+- Deprecated the ``closed`` argument in :class:`IntervalIndex` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
+- Deprecated the ``closed`` argument in :class:`IntervalDtype` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
+- Deprecated the ``closed`` argument in :class:`IntervalArray` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
+- Deprecated the ``closed`` argument in :class:`intervaltree` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
+- Deprecated the ``closed`` argument in :class:`ArrowInterval` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_150.performance:
@@ -780,7 +785,7 @@ Indexing
 - Bug in setting large integer values into :class:`Series` with ``float32`` or ``float16`` dtype incorrectly altering these values instead of coercing to ``float64`` dtype (:issue:`45844`)
 - Bug in :meth:`Series.asof` and :meth:`DataFrame.asof` incorrectly casting bool-dtype results to ``float64`` dtype (:issue:`16063`)
 - Bug in :meth:`NDFrame.xs`, :meth:`DataFrame.iterrows`, :meth:`DataFrame.loc` and :meth:`DataFrame.iloc` not always propagating metadata (:issue:`28283`)
--
+- Bug in :meth:`DataFrame.sum` min_count changes dtype if input contains NaNs (:issue:`46947`)
 
 Missing
 ^^^^^^^
@@ -809,10 +814,12 @@ I/O
 - Bug in :func:`read_parquet` when ``engine="pyarrow"`` which caused partial write to disk when column of unsupported datatype was passed (:issue:`44914`)
 - Bug in :func:`DataFrame.to_excel` and :class:`ExcelWriter` would raise when writing an empty DataFrame to a ``.ods`` file (:issue:`45793`)
 - Bug in :func:`read_html` where elements surrounding ``<br>`` were joined without a space between them (:issue:`29528`)
+- Bug in :func:`read_csv` when data is longer than header leading to issues with callables in ``usecols`` expecting strings (:issue:`46997`)
 - Bug in Parquet roundtrip for Interval dtype with ``datetime64[ns]`` subtype (:issue:`45881`)
 - Bug in :func:`read_excel` when reading a ``.ods`` file with newlines between xml elements (:issue:`45598`)
 - Bug in :func:`read_parquet` when ``engine="fastparquet"`` where the file was not closed on error (:issue:`46555`)
 - :meth:`to_html` now excludes the ``border`` attribute from ``<table>`` elements when ``border`` keyword is set to ``False``.
+- Bug in :func:`read_sas` returned ``None`` rather than an empty DataFrame for SAS7BDAT files with zero rows (:issue:`18198`)
 -
 
 Period
@@ -855,7 +862,7 @@ Groupby/resample/rolling
 - Bug in :meth:`Grouper.__repr__` where ``dropna`` was not included. Now it is (:issue:`46754`)
 - Bug in :meth:`DataFrame.rolling` gives ValueError when center=True, axis=1 and win_type is specified (:issue:`46135`)
 - Bug in :meth:`.DataFrameGroupBy.describe` and :meth:`.SeriesGroupBy.describe` produces inconsistent results for empty datasets (:issue:`41575`)
--
+- Bug in :meth:`DataFrame.resample` reduction methods when used with ``on`` would attempt to aggregate the provided column (:issue:`47079`)
 
 Reshaping
 ^^^^^^^^^

environment.yml

@@ -15,7 +15,7 @@ dependencies:
   # The compiler packages are meta-packages and install the correct compiler (activation) packages on the respective platforms.
   - c-compiler
   - cxx-compiler
-  - cython>=0.29.24
+  - cython>=0.29.30
 
   # code checks
   - black=22.3.0
@@ -24,7 +24,7 @@ dependencies:
   - flake8-bugbear=21.3.2  # used by flake8, find likely bugs
   - flake8-comprehensions=3.7.0  # used by flake8, linting of unnecessary comprehensions
   - isort>=5.2.1  # check that imports are in the right order
-  - mypy=0.950
+  - mypy=0.960
   - pre-commit>=2.15.0
   - pycodestyle  # used by flake8
   - pyupgrade