Merge remote-tracking branch 'upstream/master' into str_infer

Author: h-vetinari

asv_bench/benchmarks/reshape.py

@@ -49,21 +49,33 @@ def time_unstack(self):
 
 class Unstack(object):
 
-    def setup(self):
+    params = ['int', 'category']
+
+    def setup(self, dtype):
         m = 100
         n = 1000
 
         levels = np.arange(m)
         index = MultiIndex.from_product([levels] * 2)
         columns = np.arange(n)
-        values = np.arange(m * m * n).reshape(m * m, n)
+        if dtype == 'int':
+            values = np.arange(m * m * n).reshape(m * m, n)
+        else:
+            # the category branch is ~20x slower than int. So we
+            # cut down the size a bit. Now it's only ~3x slower.
+            n = 50
+            columns = columns[:n]
+            indices = np.random.randint(0, 52, size=(m * m, n))
+            values = np.take(list(string.ascii_letters), indices)
+            values = [pd.Categorical(v) for v in values.T]
+
         self.df = DataFrame(values, index, columns)
         self.df2 = self.df.iloc[:-1]
 
-    def time_full_product(self):
+    def time_full_product(self, dtype):
         self.df.unstack()
 
-    def time_without_last_row(self):
+    def time_without_last_row(self, dtype):
         self.df2.unstack()
 
 

ci/azure/linux.yml

@@ -54,3 +54,28 @@ jobs:
       inputs:
         testResultsFiles: 'test-data-*.xml'
         testRunTitle: 'Linux'
+    - powershell: |
+        $junitXml = "test-data-single.xml"
+        $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"'
+        if ($matches[1] -eq 0)
+        {
+          Write-Host "No test failures in test-data-single"
+        }
+        else
+        {
+          # note that this will produce $LASTEXITCODE=1
+          Write-Error "$($matches[1]) tests failed"
+        }
+
+        $junitXmlMulti = "test-data-multiple.xml"
+        $(Get-Content $junitXmlMulti | Out-String) -match 'failures="(.*?)"'
+        if ($matches[1] -eq 0)
+        {
+          Write-Host "No test failures in test-data-multi"
+        }
+        else
+        {
+          # note that this will produce $LASTEXITCODE=1
+          Write-Error "$($matches[1]) tests failed"
+        }
+      displayName: Check for test failures

ci/azure/macos.yml

@@ -41,3 +41,28 @@ jobs:
       inputs:
         testResultsFiles: 'test-data-*.xml'
         testRunTitle: 'MacOS-35'
+    - powershell: |
+        $junitXml = "test-data-single.xml"
+        $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"'
+        if ($matches[1] -eq 0)
+        {
+          Write-Host "No test failures in test-data-single"
+        }
+        else
+        {
+          # note that this will produce $LASTEXITCODE=1
+          Write-Error "$($matches[1]) tests failed"
+        }
+
+        $junitXmlMulti = "test-data-multiple.xml"
+        $(Get-Content $junitXmlMulti | Out-String) -match 'failures="(.*?)"'
+        if ($matches[1] -eq 0)
+        {
+          Write-Host "No test failures in test-data-multi"
+        }
+        else
+        {
+          # note that this will produce $LASTEXITCODE=1
+          Write-Error "$($matches[1]) tests failed"
+        }
+      displayName: Check for test failures

ci/azure/windows-py27.yml

@@ -43,3 +43,16 @@ jobs:
       inputs:
         testResultsFiles: 'test-data.xml'
         testRunTitle: 'Windows 27'
+    - powershell: |
+        $junitXml = "test-data.xml"
+        $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"'
+        if ($matches[1] -eq 0)
+        {
+          Write-Host "No test failures in test-data"
+        }
+        else
+        {
+          # note that this will produce $LASTEXITCODE=1
+          Write-Error "$($matches[1]) tests failed"
+        }
+      displayName: Check for test failures

ci/azure/windows.yml

@@ -34,3 +34,16 @@ jobs:
       inputs:
         testResultsFiles: 'test-data.xml'
         testRunTitle: 'Windows 36'
+    - powershell: |
+        $junitXml = "test-data.xml"
+        $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"'
+        if ($matches[1] -eq 0)
+        {
+          Write-Host "No test failures in test-data"
+        }
+        else
+        {
+          # note that this will produce $LASTEXITCODE=1
+          Write-Error "$($matches[1]) tests failed"
+        }
+      displayName: Check for test failures

ci/deps/azure-macos-35.yaml

@@ -12,7 +12,7 @@ dependencies:
   - nomkl
   - numexpr
   - numpy=1.12.0
-  - openpyxl
+  - openpyxl=2.5.5
   - pytables
   - python=3.5*
   - pytz

doc/source/whatsnew/v0.24.0.txt

@@ -855,7 +855,7 @@ update the ``ExtensionDtype._metadata`` tuple to match the signature of your
 - Updated the ``.type`` attribute for ``PeriodDtype``, ``DatetimeTZDtype``, and ``IntervalDtype`` to be instances of the dtype (``Period``, ``Timestamp``, and ``Interval`` respectively) (:issue:`22938`)
 - :func:`ExtensionArray.isna` is allowed to return an ``ExtensionArray`` (:issue:`22325`).
 - Support for reduction operations such as ``sum``, ``mean`` via opt-in base class method override (:issue:`22762`)
-- :meth:`Series.unstack` no longer converts extension arrays to object-dtype ndarrays. The output ``DataFrame`` will now have the same dtype as the input. This changes behavior for Categorical and Sparse data (:issue:`23077`).
+- :meth:`Series.unstack` and :meth:`DataFrame.unstack` no longer convert extension arrays to object-dtype ndarrays. Each column in the output ``DataFrame`` will now have the same dtype as the input (:issue:`23077`).
 - Bug when grouping :meth:`Dataframe.groupby()` and aggregating on ``ExtensionArray`` it was not returning the actual ``ExtensionArray`` dtype (:issue:`23227`).
 
 .. _whatsnew_0240.api.incompatibilities:
@@ -1092,6 +1092,7 @@ Categorical
 - Bug when indexing with a boolean-valued ``Categorical``. Now a boolean-valued ``Categorical`` is treated as a boolean mask (:issue:`22665`)
 - Constructing a :class:`CategoricalIndex` with empty values and boolean categories was raising a ``ValueError`` after a change to dtype coercion (:issue:`22702`).
 - Bug in :meth:`Categorical.take` with a user-provided ``fill_value`` not encoding the ``fill_value``, which could result in a ``ValueError``, incorrect results, or a segmentation fault (:issue:`23296`).
+- In meth:`Series.unstack`, specifying a ``fill_value`` not present in the categories now raises a ``TypeError`` rather than ignoring the ``fill_value`` (:issue:`23284`)
 - Bug when resampling :meth:`Dataframe.resample()` and aggregating on categorical data, the categorical dtype was getting lost. (:issue:`23227`)
 
 Datetimelike
@@ -1123,6 +1124,7 @@ Datetimelike
 - Bug in :func:`date_range` with frequency of ``Day`` or higher where dates sufficiently far in the future could wrap around to the past instead of raising ``OutOfBoundsDatetime`` (:issue:`14187`)
 - Bug in :class:`PeriodIndex` with attribute ``freq.n`` greater than 1 where adding a :class:`DateOffset` object would return incorrect results (:issue:`23215`)
 - Bug in :class:`Series` that interpreted string indices as lists of characters when setting datetimelike values (:issue:`23451`)
+- Bug in :class:`Timestamp` constructor which would drop the frequency of an input :class:`Timestamp` (:issue:`22311`)
 
 Timedelta
 ^^^^^^^^^
@@ -1207,6 +1209,7 @@ Indexing
 ^^^^^^^^
 
 - The traceback from a ``KeyError`` when asking ``.loc`` for a single missing label is now shorter and more clear (:issue:`21557`)
+- :class:`PeriodIndex` now emits a ``KeyError`` when a malformed string is looked up, which is consistent with the behavior of :class:`DateTimeIndex` (:issue:`22803`)
 - When ``.ix`` is asked for a missing integer label in a :class:`MultiIndex` with a first level of integer type, it now raises a ``KeyError``, consistently with the case of a flat :class:`Int64Index`, rather than falling back to positional indexing (:issue:`21593`)
 - Bug in :meth:`DatetimeIndex.reindex` when reindexing a tz-naive and tz-aware :class:`DatetimeIndex` (:issue:`8306`)
 - Bug in :meth:`Series.reindex` when reindexing an empty series with a ``datetime64[ns, tz]`` dtype (:issue:`20869`)
@@ -1223,7 +1226,7 @@ Indexing
 - Bug in `MultiIndex.set_levels` when levels value is not subscriptable (:issue:`23273`)
 - Bug where setting a timedelta column by ``Index`` causes it to be casted to double, and therefore lose precision (:issue:`23511`)
 - Bug in :func:`Index.union` and :func:`Index.intersection` where name of the ``Index`` of the result was not computed correctly for certain cases (:issue:`9943`, :issue:`9862`)
-
+- Bug in :class:`Index` slicing with boolean :class:`Index` may raise ``TypeError`` (:issue:`22533`)
 
 Missing
 ^^^^^^^

pandas/_libs/tslibs/timestamps.pyx

@@ -16,7 +16,8 @@ from cpython.datetime cimport (datetime,
 PyDateTime_IMPORT
 
 from util cimport (is_datetime64_object, is_timedelta64_object,
-                   is_integer_object, is_string_object, is_array)
+                   is_integer_object, is_string_object, is_array,
+                   is_offset_object)
 
 cimport ccalendar
 from conversion import tz_localize_to_utc, normalize_i8_timestamps
@@ -734,6 +735,9 @@ class Timestamp(_Timestamp):
 
         if is_string_object(freq):
             freq = to_offset(freq)
+        elif not is_offset_object(freq):
+            # GH 22311: Try to extract the frequency of a given Timestamp input
+            freq = getattr(ts_input, 'freq', None)
 
         return create_timestamp_from_ts(ts.value, ts.dts, ts.tzinfo, freq)
 

pandas/compat/numpy/function.py

@@ -19,11 +19,13 @@
 """
 
 from numpy import ndarray
-from pandas.util._validators import (validate_args, validate_kwargs,
-                                     validate_args_and_kwargs)
-from pandas.errors import UnsupportedFunctionCall
-from pandas.core.dtypes.common import is_integer, is_bool
+
 from pandas.compat import OrderedDict
+from pandas.errors import UnsupportedFunctionCall
+from pandas.util._validators import (
+    validate_args, validate_args_and_kwargs, validate_kwargs)
+
+from pandas.core.dtypes.common import is_bool, is_integer
 
 
 class CompatValidator(object):

pandas/compat/pickle_compat.py

@@ -2,12 +2,14 @@
 Support pre-0.12 series pickle compatibility.
 """
 
-import sys
-import pandas  # noqa
 import copy
 import pickle as pkl
-from pandas import compat, Index
-from pandas.compat import u, string_types  # noqa
+import sys
+
+from pandas.compat import string_types, u  # noqa
+
+import pandas  # noqa
+from pandas import Index, compat
 
 
 def load_reduce(self):

pandas/core/arrays/datetimelike.py

@@ -161,7 +161,7 @@ def __getitem__(self, key):
             return self._box_func(val)
 
         if com.is_bool_indexer(key):
-            key = np.asarray(key)
+            key = np.asarray(key, dtype=bool)
             if key.all():
                 key = slice(0, None, None)
             else: