Skip to content

Backport PR #33629 on branch 1.0.x (BUG: Fix Categorical use_inf_as_n… #34001

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/source/whatsnew/v1.0.4.rst
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ including other versions of pandas.

Fixed regressions
~~~~~~~~~~~~~~~~~
- Bug where :meth:`Series.isna` and :meth:`DataFrame.isna` would raise for categorical dtype when ``pandas.options.mode.use_inf_as_na`` was set to ``True`` (:issue:`33594`)
- Bug in :meth:`GroupBy.first` and :meth:`GroupBy.last` where None is not preserved in object dtype (:issue:`32800`)
-

Expand Down
10 changes: 9 additions & 1 deletion pandas/core/dtypes/missing.py
Original file line number Diff line number Diff line change
Expand Up @@ -269,10 +269,18 @@ def _isna_ndarraylike(obj):


def _isna_ndarraylike_old(obj):
is_extension = is_extension_array_dtype(obj)

values = getattr(obj, "values", obj)
dtype = values.dtype

if is_string_dtype(dtype):
if is_extension:
if isinstance(obj, (ABCIndexClass, ABCSeries)):
values = obj._values
else:
values = obj
result = values.isna() | (values == -np.inf) | (values == np.inf)
elif is_string_dtype(dtype):
# Working around NumPy ticket 1542
shape = values.shape

Expand Down
53 changes: 52 additions & 1 deletion pandas/tests/arrays/categorical/test_missing.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@

from pandas.core.dtypes.dtypes import CategoricalDtype

from pandas import Categorical, Index, Series, isna
import pandas as pd
from pandas import Categorical, DataFrame, Index, Series, isna
import pandas._testing as tm


Expand Down Expand Up @@ -82,3 +83,53 @@ def test_fillna_iterable_category(self, named):
expected = Categorical([Point(0, 0), Point(0, 1), Point(0, 0)])

tm.assert_categorical_equal(result, expected)

@pytest.mark.parametrize(
"values, expected",
[
([1, 2, 3], np.array([False, False, False])),
([1, 2, np.nan], np.array([False, False, True])),
([1, 2, np.inf], np.array([False, False, True])),
([1, 2, pd.NA], np.array([False, False, True])),
],
)
def test_use_inf_as_na(self, values, expected):
# https://github.com/pandas-dev/pandas/issues/33594
with pd.option_context("mode.use_inf_as_na", True):
cat = Categorical(values)
result = cat.isna()
tm.assert_numpy_array_equal(result, expected)

result = Series(cat).isna()
expected = Series(expected)
tm.assert_series_equal(result, expected)

result = DataFrame(cat).isna()
expected = DataFrame(expected)
tm.assert_frame_equal(result, expected)

@pytest.mark.parametrize(
"values, expected",
[
([1, 2, 3], np.array([False, False, False])),
([1, 2, np.nan], np.array([False, False, True])),
([1, 2, np.inf], np.array([False, False, True])),
([1, 2, pd.NA], np.array([False, False, True])),
],
)
def test_use_inf_as_na_outside_context(self, values, expected):
# https://github.com/pandas-dev/pandas/issues/33594
# Using isna directly for Categorical will fail in general here
cat = Categorical(values)

with pd.option_context("mode.use_inf_as_na", True):
result = pd.isna(cat)
tm.assert_numpy_array_equal(result, expected)

result = pd.isna(Series(cat))
expected = Series(expected)
tm.assert_series_equal(result, expected)

result = pd.isna(DataFrame(cat))
expected = DataFrame(expected)
tm.assert_frame_equal(result, expected)