Skip to content

TST: make tests involving use_numexpr more robust #44601

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 17 commits into from
Nov 26, 2021
Merged

TST: make tests involving use_numexpr more robust #44601

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
0601176
TEMP disable some workflows
jorisvandenbossche Nov 24, 2021
0f04786
TEMP add some prints to test_eval.py
jorisvandenbossche Nov 24, 2021
ee2bd87
TEMP also disable circleci/azure
jorisvandenbossche Nov 24, 2021
eba06c8
print output
jorisvandenbossche Nov 24, 2021
a8d1560
use default pytest capture
jorisvandenbossche Nov 24, 2021
2fc7509
print output for passed tests as well
jorisvandenbossche Nov 24, 2021
07ed0da
try different way to get verbose output
jorisvandenbossche Nov 24, 2021
8dd6b5c
run with BlockManager + run directly with pytest command
jorisvandenbossche Nov 24, 2021
2adc091
not parallel
jorisvandenbossche Nov 24, 2021
7beabe3
test ArrayManager again
jorisvandenbossche Nov 25, 2021
eba07ff
add autouse fixture to print
jorisvandenbossche Nov 25, 2021
ccda93b
reset use_numexpr after dask test
jorisvandenbossche Nov 25, 2021
9cf1e93
add print statement to dask test
jorisvandenbossche Nov 25, 2021
3b69f92
more robust resetting of option
jorisvandenbossche Nov 25, 2021
bf00198
undo all temporary changes
jorisvandenbossche Nov 25, 2021
6db8378
remove remaining print statements
jorisvandenbossche Nov 25, 2021
1f98ca9
use set_option instead of expr.set_use_numexpr in tests to make them …
jorisvandenbossche Nov 25, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions pandas/_testing/contexts.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@

import numpy as np

from pandas import set_option

from pandas.io.common import get_handle


Expand Down Expand Up @@ -202,11 +204,11 @@ def use_numexpr(use, min_elements=None):

olduse = expr.USE_NUMEXPR
oldmin = expr._MIN_ELEMENTS
expr.set_use_numexpr(use)
set_option("compute.use_numexpr", use)
expr._MIN_ELEMENTS = min_elements
yield
expr._MIN_ELEMENTS = oldmin
expr.set_use_numexpr(olduse)
set_option("compute.use_numexpr", olduse)


class RNGContext:
Expand Down
20 changes: 14 additions & 6 deletions pandas/tests/test_downstream.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

import pandas.util._test_decorators as td

import pandas as pd
from pandas import DataFrame
import pandas._testing as tm

Expand All @@ -32,14 +33,21 @@ def df():
@pytest.mark.filterwarnings("ignore:.*64Index is deprecated:FutureWarning")
def test_dask(df):

toolz = import_module("toolz") # noqa:F841
dask = import_module("dask") # noqa:F841
# dask sets "compute.use_numexpr" to False, so catch the current value
# and ensure to reset it afterwards to avoid impacting other tests
olduse = pd.get_option("compute.use_numexpr")

import dask.dataframe as dd
try:
toolz = import_module("toolz") # noqa:F841
dask = import_module("dask") # noqa:F841

import dask.dataframe as dd

ddf = dd.from_pandas(df, npartitions=3)
assert ddf.A is not None
assert ddf.compute() is not None
ddf = dd.from_pandas(df, npartitions=3)
assert ddf.A is not None
assert ddf.compute() is not None
finally:
pd.set_option("compute.use_numexpr", olduse)


def test_xarray(df):
Expand Down
29 changes: 15 additions & 14 deletions pandas/tests/test_expressions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import numpy as np
import pytest

from pandas import set_option
import pandas._testing as tm
from pandas.core.api import (
DataFrame,
Expand Down Expand Up @@ -65,9 +66,9 @@ def call_op(df, other, flex: bool, opname: str):
else:
op = getattr(operator, opname)

expr.set_use_numexpr(False)
set_option("compute.use_numexpr", False)
expected = op(df, other)
expr.set_use_numexpr(True)
set_option("compute.use_numexpr", True)

expr.get_test_result()

Expand Down Expand Up @@ -107,9 +108,9 @@ def run_binary(self, df, other, flex: bool):
def run_frame(self, df, other, flex: bool):
self.run_arithmetic(df, other, flex)

expr.set_use_numexpr(False)
set_option("compute.use_numexpr", False)
binary_comp = other + 1
expr.set_use_numexpr(True)
set_option("compute.use_numexpr", True)
self.run_binary(df, binary_comp, flex)

for i in range(len(df.columns)):
Expand Down Expand Up @@ -179,9 +180,9 @@ def testit():
result = expr._can_use_numexpr(op, op_str, right, right, "evaluate")
assert not result

expr.set_use_numexpr(False)
set_option("compute.use_numexpr", False)
testit()
expr.set_use_numexpr(True)
set_option("compute.use_numexpr", True)
expr.set_numexpr_threads(1)
testit()
expr.set_numexpr_threads()
Expand Down Expand Up @@ -215,9 +216,9 @@ def testit():
result = expr._can_use_numexpr(op, op_str, right, f22, "evaluate")
assert not result

expr.set_use_numexpr(False)
set_option("compute.use_numexpr", False)
testit()
expr.set_use_numexpr(True)
set_option("compute.use_numexpr", True)
expr.set_numexpr_threads(1)
testit()
expr.set_numexpr_threads()
Expand All @@ -233,9 +234,9 @@ def testit():
expected = np.where(c, df.values, df.values + 1)
tm.assert_numpy_array_equal(result, expected)

expr.set_use_numexpr(False)
set_option("compute.use_numexpr", False)
testit()
expr.set_use_numexpr(True)
set_option("compute.use_numexpr", True)
expr.set_numexpr_threads(1)
testit()
expr.set_numexpr_threads()
Expand Down Expand Up @@ -360,9 +361,9 @@ def test_frame_series_axis(self, axis, arith):

op_func = getattr(df, arith)

expr.set_use_numexpr(False)
set_option("compute.use_numexpr", False)
expected = op_func(other, axis=axis)
expr.set_use_numexpr(True)
set_option("compute.use_numexpr", True)

result = op_func(other, axis=axis)
tm.assert_frame_equal(expected, result)
Expand All @@ -387,9 +388,9 @@ def test_python_semantics_with_numexpr_installed(self, op, box, scalar):
result = method(scalar)

# compare result with numpy
expr.set_use_numexpr(False)
set_option("compute.use_numexpr", False)
expected = method(scalar)
expr.set_use_numexpr(True)
set_option("compute.use_numexpr", True)
tm.assert_equal(result, expected)

# compare result element-wise with Python
Expand Down