Merge remote-tracking branch 'origin/master' into fix-25557

* origin/master:
  DOC: clean bug fix section in whatsnew (pandas-dev#25792)
  DOC: Fixed PeriodArray api ref (pandas-dev#25526)
  Move locale code out of tm, into _config (pandas-dev#25757)
  Unpin pycodestyle (pandas-dev#25789)
  Add test for rdivmod on EA array (GH23287) (pandas-dev#24047)
  ENH: Support datetime.timezone objects (pandas-dev#25065)
  Cython language level 3 (pandas-dev#24538)
  API: concat on sparse values (pandas-dev#25719)
  TST: assert_produces_warning works with filterwarnings (pandas-dev#25721)
  make core.config self-contained (pandas-dev#25613)
  CLN: replace %s syntax with .format in pandas.io.parsers (pandas-dev#24721)
  TST: Check pytables<3.5.1 when skipping (pandas-dev#25773)
  DOC: Fix typo in docstring of DataFrame.memory_usage  (pandas-dev#25770)
  Replace dicts with OrderedDicts in groupby aggregation functions (pandas-dev#25693)
  TST: Fixturize tests/frame/test_missing.py (pandas-dev#25640)
  DOC: Improve the docsting of Series.iteritems (pandas-dev#24879)
  DOC: Fix function name. (pandas-dev#25751)
  Implementing iso_week_year support for to_datetime (pandas-dev#25541)
  DOC: clarify corr behaviour when using a callable (pandas-dev#25732)
  remove unnecessary check_output (pandas-dev#25755)

# Conflicts:
#	doc/source/whatsnew/v0.25.0.rst

Author: sighingnow

doc/source/reference/arrays.rst

@@ -259,7 +259,7 @@ Every period in a ``PeriodArray`` must have the same ``freq``.
 .. autosummary::
    :toctree: api/
 
-   arrays.DatetimeArray
+   arrays.PeriodArray
    PeriodDtype
 
 .. _api.arrays.interval:

doc/source/user_guide/timeseries.rst

@@ -2149,12 +2149,9 @@ Time Zone Handling
 ------------------
 
 pandas provides rich support for working with timestamps in different time
-zones using the ``pytz`` and ``dateutil`` libraries.
+zones using the ``pytz`` and ``dateutil`` libraries or class:`datetime.timezone`
+objects from the standard library.
 
-.. note::
-
-    pandas does not yet support ``datetime.timezone`` objects from the standard
-    library.
 
 Working with Time Zones
 ~~~~~~~~~~~~~~~~~~~~~~~
@@ -2197,6 +2194,15 @@ To return ``dateutil`` time zone objects, append ``dateutil/`` before the string
                            tz=dateutil.tz.tzutc())
    rng_utc.tz
 
+.. versionadded:: 0.25.0
+
+.. ipython:: python
+
+   # datetime.timezone
+   rng_utc = pd.date_range('3/6/2012 00:00', periods=3, freq='D',
+                           tz=datetime.timezone.utc)
+   rng_utc.tz
+
 Note that the ``UTC`` time zone is a special case in ``dateutil`` and should be constructed explicitly
 as an instance of ``dateutil.tz.tzutc``. You can also construct other time
 zones objects explicitly first.

doc/source/whatsnew/v0.25.0.rst

@@ -19,6 +19,7 @@ including other versions of pandas.
 Other Enhancements
 ^^^^^^^^^^^^^^^^^^
 
+- Added support for ISO week year format ('%G-%V-%u') when parsing datetimes using :meth: `to_datetime` (:issue:`16607`)
 - Indexing of ``DataFrame`` and ``Series`` now accepts zerodim ``np.ndarray`` (:issue:`24919`)
 - :meth:`Timestamp.replace` now supports the ``fold`` argument to disambiguate DST transition times (:issue:`25017`)
 - :meth:`DataFrame.at_time` and :meth:`Series.at_time` now support :meth:`datetime.time` objects with timezones (:issue:`24043`)
@@ -27,6 +28,7 @@ Other Enhancements
 - :meth:`DatetimeIndex.union` now supports the ``sort`` argument. The behaviour of the sort parameter matches that of :meth:`Index.union` (:issue:`24994`)
 - :meth:`DataFrame.rename` now supports the ``errors`` argument to raise errors when attempting to rename nonexistent keys (:issue:`13473`)
 - :class:`RangeIndex` has gained :attr:`~RangeIndex.start`, :attr:`~RangeIndex.stop`, and :attr:`~RangeIndex.step` attributes (:issue:`25710`)
+- :class:`datetime.timezone` objects are now supported as arguments to timezone methods and constructors (:issue:`25065`)
 
 .. _whatsnew_0250.api_breaking:
 
@@ -65,6 +67,42 @@ is respected in indexing. (:issue:`24076`, :issue:`16785`)
     df = pd.DataFrame([0], index=pd.DatetimeIndex(['2019-01-01'], tz='US/Pacific'))
     df['2019-01-01 12:00:00+04:00':'2019-01-01 13:00:00+04:00']
 
+Concatenating Sparse Values
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When passed DataFrames whose values are sparse, :func:`concat` will now return a
+Series or DataFrame with sparse values, rather than a ``SparseDataFrame`` (:issue:`25702`).
+
+.. ipython:: python
+
+   df = pd.DataFrame({"A": pd.SparseArray([0, 1])})
+
+*Previous Behavior:*
+
+.. code-block:: ipython
+
+   In [2]: type(pd.concat([df, df]))
+   pandas.core.sparse.frame.SparseDataFrame
+
+*New Behavior:*
+
+.. ipython:: python
+
+   type(pd.concat([df, df]))
+
+
+This now matches the existing behavior of :class:`concat` on ``Series`` with sparse values.
+:func:`concat` will continue to return a ``SparseDataFrame`` when all the values
+are instances of ``SparseDataFrame``.
+
+This change also affects routines using :func:`concat` internally, like :func:`get_dummies`,
+which now returns a :class:`DataFrame` in all cases (previously a ``SparseDataFrame`` was
+returned if all the columns were dummy encoded, and a :class:`DataFrame` otherwise).
+
+Providing any ``SparseSeries`` or ``SparseDataFrame`` to :func:`concat` will
+cause a ``SparseSeries`` or ``SparseDataFrame`` to be returned, as before.
+
+
 .. _whatsnew_0250.api_breaking.deps:
 
 Increased minimum versions for dependencies
@@ -137,10 +175,7 @@ Performance Improvements
 
 Bug Fixes
 ~~~~~~~~~
-- Bug in :func:`to_datetime` which would raise an (incorrect) ``ValueError`` when called with a date far into the future and the ``format`` argument specified instead of raising ``OutOfBoundsDatetime`` (:issue:`23830`)
-- Bug in an error message in :meth:`DataFrame.plot`. Improved the error message if non-numerics are passed to :meth:`DataFrame.plot` (:issue:`25481`)
-- Bug in error messages in :meth:`DataFrame.corr` and :meth:`Series.corr`. Added the possibility of using a callable. (:issue:`25729`)
-- Bug in :meth:`Series.divmod` and :meth:`Series.rdivmod` which would raise an (incorrect) ``ValueError`` rather than return a pair of :class:`Series` object as result (:issue:`25557`)
+
 
 Categorical
 ^^^^^^^^^^^
@@ -152,6 +187,7 @@ Categorical
 Datetimelike
 ^^^^^^^^^^^^
 
+- Bug in :func:`to_datetime` which would raise an (incorrect) ``ValueError`` when called with a date far into the future and the ``format`` argument specified instead of raising ``OutOfBoundsDatetime`` (:issue:`23830`)
 -
 -
 -
@@ -175,6 +211,8 @@ Numeric
 
 - Bug in :meth:`to_numeric` in which large negative numbers were being improperly handled (:issue:`24910`)
 - Bug in :meth:`to_numeric` in which numbers were being coerced to float, even though ``errors`` was not ``coerce`` (:issue:`24910`)
+- Bug in error messages in :meth:`DataFrame.corr` and :meth:`Series.corr`. Added the possibility of using a callable. (:issue:`25729`)
+- Bug in :meth:`Series.divmod` and :meth:`Series.rdivmod` which would raise an (incorrect) ``ValueError`` rather than return a pair of :class:`Series` objects as result (:issue:`25557`)
 -
 -
 -
@@ -242,6 +280,7 @@ Plotting
 ^^^^^^^^
 
 - Fixed bug where :class:`api.extensions.ExtensionArray` could not be used in matplotlib plotting (:issue:`25587`)
+- Bug in an error message in :meth:`DataFrame.plot`. Improved the error message if non-numerics are passed to :meth:`DataFrame.plot` (:issue:`25481`)
 -
 -
 -
@@ -253,6 +292,7 @@ Groupby/Resample/Rolling
 - Bug in :meth:`pandas.core.groupby.DataFrameGroupBy.nunique` in which the names of column levels were lost (:issue:`23222`)
 - Bug in :func:`pandas.core.groupby.GroupBy.agg` when applying a aggregation function to timezone aware data (:issue:`23683`)
 - Bug in :func:`pandas.core.groupby.GroupBy.first` and :func:`pandas.core.groupby.GroupBy.last` where timezone information would be dropped (:issue:`21603`)
+- Ensured that ordering of outputs in ``groupby`` aggregation functions is consistent across all versions of Python (:issue:`25692`)
 
 
 Reshaping

environment.yml

@@ -19,7 +19,7 @@ dependencies:
   - hypothesis>=3.82
   - isort
   - moto
-  - pycodestyle=2.4
+  - pycodestyle
   - pytest>=4.0.2
   - pytest-mock
   - sphinx

pandas/_config/__init__.py

@@ -0,0 +1,4 @@
+"""
+pandas._config is considered explicitly upstream of everything else in pandas,
+should have no intra-pandas dependencies.
+"""

pandas/_config/localization.py

@@ -0,0 +1,93 @@
+"""
+Helpers for configuring locale settings.
+
+Name `localization` is chosen to avoid overlap with builtin `locale` module.
+"""
+from contextlib import contextmanager
+import locale
+
+
+@contextmanager
+def set_locale(new_locale, lc_var=locale.LC_ALL):
+    """
+    Context manager for temporarily setting a locale.
+
+    Parameters
+    ----------
+    new_locale : str or tuple
+        A string of the form <language_country>.<encoding>. For example to set
+        the current locale to US English with a UTF8 encoding, you would pass
+        "en_US.UTF-8".
+    lc_var : int, default `locale.LC_ALL`
+        The category of the locale being set.
+
+    Notes
+    -----
+    This is useful when you want to run a particular block of code under a
+    particular locale, without globally setting the locale. This probably isn't
+    thread-safe.
+    """
+    current_locale = locale.getlocale()
+
+    try:
+        locale.setlocale(lc_var, new_locale)
+        normalized_locale = locale.getlocale()
+        if all(x is not None for x in normalized_locale):
+            yield '.'.join(normalized_locale)
+        else:
+            yield new_locale
+    finally:
+        locale.setlocale(lc_var, current_locale)
+
+
+def can_set_locale(lc, lc_var=locale.LC_ALL):
+    """
+    Check to see if we can set a locale, and subsequently get the locale,
+    without raising an Exception.
+
+    Parameters
+    ----------
+    lc : str
+        The locale to attempt to set.
+    lc_var : int, default `locale.LC_ALL`
+        The category of the locale being set.
+
+    Returns
+    -------
+    is_valid : bool
+        Whether the passed locale can be set
+    """
+
+    try:
+        with set_locale(lc, lc_var=lc_var):
+            pass
+    except (ValueError, locale.Error):
+        # horrible name for a Exception subclass
+        return False
+    else:
+        return True
+
+
+def _valid_locales(locales, normalize):
+    """
+    Return a list of normalized locales that do not throw an ``Exception``
+    when set.
+
+    Parameters
+    ----------
+    locales : str
+        A string where each locale is separated by a newline.
+    normalize : bool
+        Whether to call ``locale.normalize`` on each locale.
+
+    Returns
+    -------
+    valid_locales : list
+        A list of valid locales.
+    """
+    if normalize:
+        normalizer = lambda x: locale.normalize(x.strip())
+    else:
+        normalizer = lambda x: x.strip()
+
+    return list(filter(can_set_locale, map(normalizer, locales)))

pandas/_libs/__init__.py

@@ -2,4 +2,4 @@
 # flake8: noqa
 
 from .tslibs import (
-    iNaT, NaT, Timestamp, Timedelta, OutOfBoundsDatetime, Period)
+    iNaT, NaT, NaTType, Timestamp, Timedelta, OutOfBoundsDatetime, Period)

pandas/_libs/groupby.pyx

@@ -57,10 +57,10 @@ cdef inline float64_t median_linear(float64_t* a, int n) nogil:
         n -= na_count
 
     if n % 2:
-        result = kth_smallest_c( a, n / 2, n)
+        result = kth_smallest_c( a, n // 2, n)
     else:
-        result = (kth_smallest_c(a, n / 2, n) +
-                  kth_smallest_c(a, n / 2 - 1, n)) / 2
+        result = (kth_smallest_c(a, n // 2, n) +
+                  kth_smallest_c(a, n // 2 - 1, n)) / 2
 
     if na_count:
         free(a)

pandas/_libs/parsers.pyx

@@ -948,7 +948,7 @@ cdef class TextReader:
             status = tokenize_nrows(self.parser, nrows)
 
         if self.parser.warn_msg != NULL:
-            print >> sys.stderr, self.parser.warn_msg
+            print(self.parser.warn_msg, file=sys.stderr)
             free(self.parser.warn_msg)
             self.parser.warn_msg = NULL
 
@@ -976,7 +976,7 @@ cdef class TextReader:
                 status = tokenize_all_rows(self.parser)
 
             if self.parser.warn_msg != NULL:
-                print >> sys.stderr, self.parser.warn_msg
+                print(self.parser.warn_msg, file=sys.stderr)
                 free(self.parser.warn_msg)
                 self.parser.warn_msg = NULL
 

pandas/_libs/tslib.pyx

@@ -275,7 +275,7 @@ def format_array_from_datetime(ndarray[int64_t] values, object tz=None,
                                                    dts.sec)
 
             if show_ns:
-                ns = dts.ps / 1000
+                ns = dts.ps // 1000
                 res += '.%.9d' % (ns + 1000 * dts.us)
             elif show_us:
                 res += '.%.6d' % dts.us

pandas/_libs/tslibs/__init__.py

@@ -2,7 +2,7 @@
 # flake8: noqa
 
 from .conversion import normalize_date, localize_pydatetime, tz_convert_single
-from .nattype import NaT, iNaT, is_null_datetimelike
+from .nattype import NaT, NaTType, iNaT, is_null_datetimelike
 from .np_datetime import OutOfBoundsDatetime
 from .period import Period, IncompatibleFrequency
 from .timestamps import Timestamp

pandas/_libs/tslibs/ccalendar.pyx

@@ -9,6 +9,8 @@ import cython
 from numpy cimport int64_t, int32_t
 
 from locale import LC_TIME
+
+from pandas._config.localization import set_locale
 from pandas._libs.tslibs.strptime import LocaleTime
 
 # ----------------------------------------------------------------------
@@ -159,7 +161,7 @@ cpdef int32_t get_week_of_year(int year, int month, int day) nogil:
     # estimate
     woy = (doy - 1) - dow + 3
     if woy >= 0:
-        woy = woy / 7 + 1
+        woy = woy // 7 + 1
 
     # verify
     if woy < 0:
@@ -206,7 +208,7 @@ cpdef int32_t get_day_of_year(int year, int month, int day) nogil:
     return day_of_year
 
 
-cpdef get_locale_names(object name_type, object locale=None):
+def get_locale_names(name_type: object, locale: object=None):
     """Returns an array of localized day or month names
 
     Parameters
@@ -218,9 +220,6 @@ cpdef get_locale_names(object name_type, object locale=None):
     Returns
     -------
     list of locale names
-
     """
-    from pandas.util.testing import set_locale
-
     with set_locale(locale, LC_TIME):
         return getattr(LocaleTime(), name_type)

pandas/_libs/tslibs/conversion.pyx

@@ -462,8 +462,8 @@ cdef _TSObject convert_str_to_tsobject(object ts, object tz, object unit,
                     dt = datetime(obj.dts.year, obj.dts.month, obj.dts.day,
                                   obj.dts.hour, obj.dts.min, obj.dts.sec,
                                   obj.dts.us, obj.tzinfo)
-                    obj = convert_datetime_to_tsobject(dt, tz,
-                                                       nanos=obj.dts.ps / 1000)
+                    obj = convert_datetime_to_tsobject(
+                        dt, tz, nanos=obj.dts.ps // 1000)
                     return obj
 
             else:

pandas/_libs/tslibs/fields.pyx

@@ -478,7 +478,7 @@ def get_date_field(int64_t[:] dtindex, object field):
                     continue
 
                 dt64_to_dtstruct(dtindex[i], &dts)
-                out[i] = dts.ps / 1000
+                out[i] = dts.ps // 1000
         return out
     elif field == 'doy':
         with nogil:
@@ -522,7 +522,7 @@ def get_date_field(int64_t[:] dtindex, object field):
 
                 dt64_to_dtstruct(dtindex[i], &dts)
                 out[i] = dts.month
-                out[i] = ((out[i] - 1) / 3) + 1
+                out[i] = ((out[i] - 1) // 3) + 1
         return out
 
     elif field == 'dim':

pandas/_libs/tslibs/nattype.pyx

@@ -353,7 +353,7 @@ class NaTType(_NaT):
 
         .. versionadded:: 0.23.0
         """)
-    day_name = _make_nan_func('day_name', # noqa:E128
+    day_name = _make_nan_func('day_name',  # noqa:E128
         """
         Return the day name of the Timestamp with specified locale.
 

pandas/_libs/tslibs/offsets.pyx

@@ -587,7 +587,7 @@ def shift_day(other: datetime, days: int) -> datetime:
 
 cdef inline int year_add_months(npy_datetimestruct dts, int months) nogil:
     """new year number after shifting npy_datetimestruct number of months"""
-    return dts.year + (dts.month + months - 1) / 12
+    return dts.year + (dts.month + months - 1) // 12
 
 
 cdef inline int month_add_months(npy_datetimestruct dts, int months) nogil: