From 0601176b7b6e86c9789c23301c805bfbad9ba1d8 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 24 Nov 2021 08:54:51 +0100 Subject: [PATCH 01/17] TEMP disable some workflows --- .github/workflows/ci.yml | 2 ++ .github/workflows/database.yml | 1 + .github/workflows/posix.yml | 1 + .github/workflows/python-dev.yml | 1 + .github/workflows/sdist.yml | 1 + 5 files changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23e452f682b60..553a3f0ab8091 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,7 @@ env: jobs: checks: name: Checks + if: ${{ false }} runs-on: ubuntu-latest defaults: run: @@ -99,6 +100,7 @@ jobs: web_and_docs: name: Web and docs + if: ${{ false }} runs-on: ubuntu-latest concurrency: diff --git a/.github/workflows/database.yml b/.github/workflows/database.yml index 6da47c86026ed..bad8450252dd0 100644 --- a/.github/workflows/database.yml +++ b/.github/workflows/database.yml @@ -20,6 +20,7 @@ env: jobs: Linux_py38_IO: + if: ${{ false }} runs-on: ubuntu-latest defaults: run: diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index f37f31686ef69..e4d1de64ff856 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -18,6 +18,7 @@ env: jobs: pytest: + if: ${{ false }} runs-on: ubuntu-latest defaults: run: diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index 824e7de3bde41..a91c9b2f1d1d9 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -21,6 +21,7 @@ env: jobs: build: + if: ${{ false }} runs-on: ${{ matrix.os }} strategy: fail-fast: false diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 92a9f2a5fb97c..faa66b0d3f9a4 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -14,6 +14,7 @@ on: jobs: build: + if: ${{ false }} runs-on: ubuntu-latest timeout-minutes: 60 defaults: From 0f04786e0382955f1aa1547c3d649337763c6c1b Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 24 Nov 2021 08:55:31 +0100 Subject: [PATCH 02/17] TEMP add some prints to test_eval.py --- pandas/tests/computation/test_eval.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 5c614dac2bcb9..1743065345e4c 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -35,6 +35,7 @@ PandasExprVisitor, PythonExprVisitor, ) +import pandas.core.computation.expressions as expressions_module from pandas.core.computation.expressions import ( NUMEXPR_INSTALLED, USE_NUMEXPR, @@ -47,6 +48,8 @@ _unary_math_ops, ) +print("test_eval.py: ", expressions_module.USE_NUMEXPR, USE_NUMEXPR) + @pytest.fixture( params=( @@ -1128,6 +1131,12 @@ class TestOperationsNumExprPandas: @classmethod def setup_class(cls): + print( + "TestOperationsNumExprPandas: ", + expressions_module.USE_NUMEXPR, + td.USE_NUMEXPR, + USE_NUMEXPR, + ) cls.arith_ops = [ op for op in expr.ARITH_OPS_SYMS + expr.CMP_OPS_SYMS @@ -1340,6 +1349,8 @@ def f(): tm.assert_frame_equal(df, expected) def test_column_in(self): + print("test_column_in: ", expressions_module.USE_NUMEXPR, USE_NUMEXPR) + # GH 11235 df = DataFrame({"a": [11], "b": [-32]}) result = df.eval("a in [11, -32]") @@ -2001,6 +2012,8 @@ def test_bool_ops_fails_on_scalars(lhs, cmp, rhs, engine, parser): ], ) def test_equals_various(other): + print("test_equals_various: ", expressions_module.USE_NUMEXPR, USE_NUMEXPR) + df = DataFrame({"A": ["a", "b", "c"]}) result = df.eval(f"A == {other}") expected = Series([False, False, False], name="A") From ee2bd874866a7ce3e5ac68108cd860470c6c3e52 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 24 Nov 2021 11:13:13 +0100 Subject: [PATCH 03/17] TEMP also disable circleci/azure --- .circleci/config.yml | 8 ++--- .github/workflows/pre-commit.yml | 1 + azure-pipelines.yml | 57 -------------------------------- 3 files changed, 5 insertions(+), 61 deletions(-) delete mode 100644 azure-pipelines.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index dc357101e79fd..2ea38c52ede7b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ jobs: - run: ci/setup_env.sh - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh -workflows: - test: - jobs: - - test-arm +# workflows: +# test: +# jobs: +# - test-arm diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 93e30e4d551af..ac160322cafce 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -9,6 +9,7 @@ on: jobs: pre-commit: + if: ${{ false }} runs-on: ubuntu-latest concurrency: group: ${{ github.ref }}-pre-commit diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 6c685d09ab55a..0000000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,57 +0,0 @@ -# Adapted from https://github.com/numba/numba/blob/master/azure-pipelines.yml -trigger: - branches: - include: - - master - - 1.3.x - paths: - exclude: - - 'doc/*' - -pr: - autoCancel: true - branches: - include: - - master - - 1.3.x - -variables: - PYTEST_WORKERS: auto - PYTEST_TARGET: pandas - -jobs: -# Mac and Linux use the same template -- template: ci/azure/posix.yml - parameters: - name: macOS - vmImage: macOS-10.15 - -- template: ci/azure/windows.yml - parameters: - name: Windows - vmImage: windows-2019 - -- job: py38_32bit - pool: - vmImage: ubuntu-18.04 - - steps: - - script: | - docker pull quay.io/pypa/manylinux2014_i686 - docker run -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \ - /bin/bash -xc "cd pandas && \ - /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \ - . ~/virtualenvs/pandas-dev/bin/activate && \ - python -m pip install --no-deps -U pip wheel setuptools && \ - pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines && \ - python setup.py build_ext -q -j2 && \ - python -m pip install --no-build-isolation -e . && \ - pytest -m 'not slow and not network and not clipboard' pandas --junitxml=test-data.xml" - displayName: 'Run 32-bit manylinux2014 Docker Build / Tests' - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - failTaskOnFailedTests: true - testRunTitle: 'Publish test results for Python 3.8-32 bit full Linux' From eba06c860088f4e0b71c92f592ce3069ff672647 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 24 Nov 2021 11:14:32 +0100 Subject: [PATCH 04/17] print output --- ci/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 9fea696b6ea81..1406658247682 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -19,7 +19,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi -PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="${XVFB}pytest -s -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ $(uname) != "Linux" && $(uname) != "Darwin" ]]; then PYTEST_CMD="$PYTEST_CMD --ignore=pandas/tests/plotting/" From a8d1560af3f97f1631e77691338fef8ed9f76c3a Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 24 Nov 2021 12:17:58 +0100 Subject: [PATCH 05/17] use default pytest capture --- ci/run_tests.sh | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 1406658247682..9fea696b6ea81 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -19,7 +19,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi -PYTEST_CMD="${XVFB}pytest -s -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ $(uname) != "Linux" && $(uname) != "Darwin" ]]; then PYTEST_CMD="$PYTEST_CMD --ignore=pandas/tests/plotting/" diff --git a/pyproject.toml b/pyproject.toml index 0c3e078d8761a..fcfe2ac1d1c10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ exclude = ''' [tool.pytest.ini_options] # sync minversion with pyproject.toml & install.rst minversion = "6.0" -addopts = "--strict-data-files --strict-markers --capture=no --durations=30 --junitxml=test-data.xml" +addopts = "--strict-data-files --strict-markers --durations=30 --junitxml=test-data.xml" xfail_strict = true testpaths = "pandas" doctest_optionflags = [ From 2fc7509ba76b1922ab60e6f7a6413d206b828bb5 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 24 Nov 2021 13:12:02 +0100 Subject: [PATCH 06/17] print output for passed tests as well --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fcfe2ac1d1c10..e7c7328bbe15d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ exclude = ''' [tool.pytest.ini_options] # sync minversion with pyproject.toml & install.rst minversion = "6.0" -addopts = "--strict-data-files --strict-markers --durations=30 --junitxml=test-data.xml" +addopts = "-rfEsP --strict-data-files --strict-markers --durations=30 --junitxml=test-data.xml" xfail_strict = true testpaths = "pandas" doctest_optionflags = [ From 07ed0da5a6662bde67529a006e759509d8b036e8 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 24 Nov 2021 15:47:57 +0100 Subject: [PATCH 07/17] try different way to get verbose output --- ci/run_tests.sh | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 9fea696b6ea81..69cb76882fff6 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -19,7 +19,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi -PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="${XVFB}pytest -s -v -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ $(uname) != "Linux" && $(uname) != "Darwin" ]]; then PYTEST_CMD="$PYTEST_CMD --ignore=pandas/tests/plotting/" diff --git a/pyproject.toml b/pyproject.toml index e7c7328bbe15d..fcfe2ac1d1c10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ exclude = ''' [tool.pytest.ini_options] # sync minversion with pyproject.toml & install.rst minversion = "6.0" -addopts = "-rfEsP --strict-data-files --strict-markers --durations=30 --junitxml=test-data.xml" +addopts = "--strict-data-files --strict-markers --durations=30 --junitxml=test-data.xml" xfail_strict = true testpaths = "pandas" doctest_optionflags = [ From 8dd6b5c6433af92122b23433b391da8008a1eca6 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 24 Nov 2021 17:03:45 +0100 Subject: [PATCH 08/17] run with BlockManager + run directly with pytest command --- .github/workflows/ci.yml | 4 ++-- ci/run_tests.sh | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 553a3f0ab8091..8d3457a8dbc93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,13 +180,13 @@ jobs: - name: Run tests env: - PANDAS_DATA_MANAGER: array + PANDAS_DATA_MANAGER: block PATTERN: ${{ matrix.pattern }} PYTEST_WORKERS: "auto" PYTEST_TARGET: pandas run: | source activate pandas-dev - ci/run_tests.sh + pytest -s -v -m "not slow and not network and not clipboard" -n auto --dist=loadfile pandas - name: Print skipped tests run: python ci/print_skipped.py diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 69cb76882fff6..fe383918928d4 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -28,10 +28,10 @@ fi echo $PYTEST_CMD sh -c "$PYTEST_CMD" -if [[ "$PANDAS_DATA_MANAGER" != "array" ]]; then - # The ArrayManager tests should have already been run by PYTEST_CMD if PANDAS_DATA_MANAGER was already set to array - PYTEST_AM_CMD="PANDAS_DATA_MANAGER=array pytest -m \"$PATTERN and arraymanager\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE pandas" +# if [[ "$PANDAS_DATA_MANAGER" != "array" ]]; then +# # The ArrayManager tests should have already been run by PYTEST_CMD if PANDAS_DATA_MANAGER was already set to array +# PYTEST_AM_CMD="PANDAS_DATA_MANAGER=array pytest -m \"$PATTERN and arraymanager\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE pandas" - echo $PYTEST_AM_CMD - sh -c "$PYTEST_AM_CMD" -fi +# echo $PYTEST_AM_CMD +# sh -c "$PYTEST_AM_CMD" +# fi From 2adc091864dfd6ba3d7b206a3196bd7e425c9981 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 24 Nov 2021 22:07:56 +0100 Subject: [PATCH 09/17] not parallel --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d3457a8dbc93..bb216b81b3331 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -186,7 +186,7 @@ jobs: PYTEST_TARGET: pandas run: | source activate pandas-dev - pytest -s -v -m "not slow and not network and not clipboard" -n auto --dist=loadfile pandas + pytest -s -v -m "not slow and not network and not clipboard" pandas - name: Print skipped tests run: python ci/print_skipped.py From 7beabe356233d4d6fb05ef4b44d4ce11b0e6071d Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 25 Nov 2021 07:47:41 +0100 Subject: [PATCH 10/17] test ArrayManager again --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb216b81b3331..f5097de2d9b23 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,7 +180,7 @@ jobs: - name: Run tests env: - PANDAS_DATA_MANAGER: block + PANDAS_DATA_MANAGER: array PATTERN: ${{ matrix.pattern }} PYTEST_WORKERS: "auto" PYTEST_TARGET: pandas From eba07fffb0bd78f253f3933fae9b83ac389d06f1 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 25 Nov 2021 08:29:10 +0100 Subject: [PATCH 11/17] add autouse fixture to print --- pandas/tests/computation/test_eval.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 1743065345e4c..00d186ce8b5d1 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -51,6 +51,13 @@ print("test_eval.py: ", expressions_module.USE_NUMEXPR, USE_NUMEXPR) +@pytest.fixture(autouse=True) +def print_debug(request): + print("DEBUG-START: ", expressions_module.USE_NUMEXPR, td.USE_NUMEXPR, USE_NUMEXPR) + yield + print("DEBUG-END: ", expressions_module.USE_NUMEXPR, td.USE_NUMEXPR, USE_NUMEXPR) + + @pytest.fixture( params=( pytest.param( @@ -2012,7 +2019,7 @@ def test_bool_ops_fails_on_scalars(lhs, cmp, rhs, engine, parser): ], ) def test_equals_various(other): - print("test_equals_various: ", expressions_module.USE_NUMEXPR, USE_NUMEXPR) + print("DEBUG: ", expressions_module.USE_NUMEXPR, td.USE_NUMEXPR, USE_NUMEXPR) df = DataFrame({"A": ["a", "b", "c"]}) result = df.eval(f"A == {other}") From ccda93b974a1fa4c7e3fa267432210570f54df54 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 25 Nov 2021 09:50:55 +0100 Subject: [PATCH 12/17] reset use_numexpr after dask test --- pandas/tests/test_downstream.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/pandas/tests/test_downstream.py b/pandas/tests/test_downstream.py index 3ea95b8a56eea..4dafc3eca5696 100644 --- a/pandas/tests/test_downstream.py +++ b/pandas/tests/test_downstream.py @@ -32,14 +32,23 @@ def df(): @pytest.mark.filterwarnings("ignore:.*64Index is deprecated:FutureWarning") def test_dask(df): - toolz = import_module("toolz") # noqa:F841 - dask = import_module("dask") # noqa:F841 + # dask sets "compute.use_numexpr" to False, so catch the current value + # and ensure to reset it afterwards to avoid impacting other tests + from pandas.core.computation import expressions as expr - import dask.dataframe as dd + olduse = expr.USE_NUMEXPR - ddf = dd.from_pandas(df, npartitions=3) - assert ddf.A is not None - assert ddf.compute() is not None + try: + toolz = import_module("toolz") # noqa:F841 + dask = import_module("dask") # noqa:F841 + + import dask.dataframe as dd + + ddf = dd.from_pandas(df, npartitions=3) + assert ddf.A is not None + assert ddf.compute() is not None + finally: + expr.set_use_numexpr(olduse) def test_xarray(df): From 9cf1e933fb0cb3ef24ce11ffc9a52a806da9fa3c Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 25 Nov 2021 10:54:16 +0100 Subject: [PATCH 13/17] add print statement to dask test --- pandas/tests/test_downstream.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandas/tests/test_downstream.py b/pandas/tests/test_downstream.py index 4dafc3eca5696..c4fa6a1d03f22 100644 --- a/pandas/tests/test_downstream.py +++ b/pandas/tests/test_downstream.py @@ -37,6 +37,7 @@ def test_dask(df): from pandas.core.computation import expressions as expr olduse = expr.USE_NUMEXPR + print("test_dask START: ", olduse) try: toolz = import_module("toolz") # noqa:F841 @@ -50,6 +51,8 @@ def test_dask(df): finally: expr.set_use_numexpr(olduse) + print("test_dask END: ", expr.USE_NUMEXPR) + def test_xarray(df): From 3b69f92e445b0579cbdae2a16ad2314875f32b23 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 25 Nov 2021 13:30:26 +0100 Subject: [PATCH 14/17] more robust resetting of option --- pandas/tests/test_downstream.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/tests/test_downstream.py b/pandas/tests/test_downstream.py index c4fa6a1d03f22..5b688a1a902c0 100644 --- a/pandas/tests/test_downstream.py +++ b/pandas/tests/test_downstream.py @@ -10,6 +10,7 @@ import pandas.util._test_decorators as td +import pandas as pd from pandas import DataFrame import pandas._testing as tm @@ -36,7 +37,7 @@ def test_dask(df): # and ensure to reset it afterwards to avoid impacting other tests from pandas.core.computation import expressions as expr - olduse = expr.USE_NUMEXPR + olduse = pd.get_option("compute.use_numexpr") print("test_dask START: ", olduse) try: @@ -49,7 +50,7 @@ def test_dask(df): assert ddf.A is not None assert ddf.compute() is not None finally: - expr.set_use_numexpr(olduse) + pd.set_option("compute.use_numexpr", olduse) print("test_dask END: ", expr.USE_NUMEXPR) From bf001989b8d86fd1d55437b6f0c3fe3e13dd8dc4 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 25 Nov 2021 20:06:35 +0100 Subject: [PATCH 15/17] undo all temporary changes --- .circleci/config.yml | 8 ++-- .github/workflows/ci.yml | 4 +- .github/workflows/database.yml | 1 - .github/workflows/posix.yml | 1 - .github/workflows/pre-commit.yml | 1 - .github/workflows/python-dev.yml | 1 - .github/workflows/sdist.yml | 1 - azure-pipelines.yml | 57 +++++++++++++++++++++++++++ ci/run_tests.sh | 14 +++---- pandas/tests/computation/test_eval.py | 20 ---------- pyproject.toml | 2 +- 11 files changed, 70 insertions(+), 40 deletions(-) create mode 100644 azure-pipelines.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 2ea38c52ede7b..dc357101e79fd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ jobs: - run: ci/setup_env.sh - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh -# workflows: -# test: -# jobs: -# - test-arm +workflows: + test: + jobs: + - test-arm diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5097de2d9b23..23e452f682b60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,6 @@ env: jobs: checks: name: Checks - if: ${{ false }} runs-on: ubuntu-latest defaults: run: @@ -100,7 +99,6 @@ jobs: web_and_docs: name: Web and docs - if: ${{ false }} runs-on: ubuntu-latest concurrency: @@ -186,7 +184,7 @@ jobs: PYTEST_TARGET: pandas run: | source activate pandas-dev - pytest -s -v -m "not slow and not network and not clipboard" pandas + ci/run_tests.sh - name: Print skipped tests run: python ci/print_skipped.py diff --git a/.github/workflows/database.yml b/.github/workflows/database.yml index bad8450252dd0..6da47c86026ed 100644 --- a/.github/workflows/database.yml +++ b/.github/workflows/database.yml @@ -20,7 +20,6 @@ env: jobs: Linux_py38_IO: - if: ${{ false }} runs-on: ubuntu-latest defaults: run: diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index e4d1de64ff856..f37f31686ef69 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -18,7 +18,6 @@ env: jobs: pytest: - if: ${{ false }} runs-on: ubuntu-latest defaults: run: diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index ac160322cafce..93e30e4d551af 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -9,7 +9,6 @@ on: jobs: pre-commit: - if: ${{ false }} runs-on: ubuntu-latest concurrency: group: ${{ github.ref }}-pre-commit diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index a91c9b2f1d1d9..824e7de3bde41 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -21,7 +21,6 @@ env: jobs: build: - if: ${{ false }} runs-on: ${{ matrix.os }} strategy: fail-fast: false diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index faa66b0d3f9a4..92a9f2a5fb97c 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -14,7 +14,6 @@ on: jobs: build: - if: ${{ false }} runs-on: ubuntu-latest timeout-minutes: 60 defaults: diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000000000..6c685d09ab55a --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,57 @@ +# Adapted from https://github.com/numba/numba/blob/master/azure-pipelines.yml +trigger: + branches: + include: + - master + - 1.3.x + paths: + exclude: + - 'doc/*' + +pr: + autoCancel: true + branches: + include: + - master + - 1.3.x + +variables: + PYTEST_WORKERS: auto + PYTEST_TARGET: pandas + +jobs: +# Mac and Linux use the same template +- template: ci/azure/posix.yml + parameters: + name: macOS + vmImage: macOS-10.15 + +- template: ci/azure/windows.yml + parameters: + name: Windows + vmImage: windows-2019 + +- job: py38_32bit + pool: + vmImage: ubuntu-18.04 + + steps: + - script: | + docker pull quay.io/pypa/manylinux2014_i686 + docker run -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \ + /bin/bash -xc "cd pandas && \ + /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \ + . ~/virtualenvs/pandas-dev/bin/activate && \ + python -m pip install --no-deps -U pip wheel setuptools && \ + pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines && \ + python setup.py build_ext -q -j2 && \ + python -m pip install --no-build-isolation -e . && \ + pytest -m 'not slow and not network and not clipboard' pandas --junitxml=test-data.xml" + displayName: 'Run 32-bit manylinux2014 Docker Build / Tests' + + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testResultsFiles: '**/test-*.xml' + failTaskOnFailedTests: true + testRunTitle: 'Publish test results for Python 3.8-32 bit full Linux' diff --git a/ci/run_tests.sh b/ci/run_tests.sh index fe383918928d4..9fea696b6ea81 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -19,7 +19,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi -PYTEST_CMD="${XVFB}pytest -s -v -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ $(uname) != "Linux" && $(uname) != "Darwin" ]]; then PYTEST_CMD="$PYTEST_CMD --ignore=pandas/tests/plotting/" @@ -28,10 +28,10 @@ fi echo $PYTEST_CMD sh -c "$PYTEST_CMD" -# if [[ "$PANDAS_DATA_MANAGER" != "array" ]]; then -# # The ArrayManager tests should have already been run by PYTEST_CMD if PANDAS_DATA_MANAGER was already set to array -# PYTEST_AM_CMD="PANDAS_DATA_MANAGER=array pytest -m \"$PATTERN and arraymanager\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE pandas" +if [[ "$PANDAS_DATA_MANAGER" != "array" ]]; then + # The ArrayManager tests should have already been run by PYTEST_CMD if PANDAS_DATA_MANAGER was already set to array + PYTEST_AM_CMD="PANDAS_DATA_MANAGER=array pytest -m \"$PATTERN and arraymanager\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE pandas" -# echo $PYTEST_AM_CMD -# sh -c "$PYTEST_AM_CMD" -# fi + echo $PYTEST_AM_CMD + sh -c "$PYTEST_AM_CMD" +fi diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 00d186ce8b5d1..5c614dac2bcb9 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -35,7 +35,6 @@ PandasExprVisitor, PythonExprVisitor, ) -import pandas.core.computation.expressions as expressions_module from pandas.core.computation.expressions import ( NUMEXPR_INSTALLED, USE_NUMEXPR, @@ -48,15 +47,6 @@ _unary_math_ops, ) -print("test_eval.py: ", expressions_module.USE_NUMEXPR, USE_NUMEXPR) - - -@pytest.fixture(autouse=True) -def print_debug(request): - print("DEBUG-START: ", expressions_module.USE_NUMEXPR, td.USE_NUMEXPR, USE_NUMEXPR) - yield - print("DEBUG-END: ", expressions_module.USE_NUMEXPR, td.USE_NUMEXPR, USE_NUMEXPR) - @pytest.fixture( params=( @@ -1138,12 +1128,6 @@ class TestOperationsNumExprPandas: @classmethod def setup_class(cls): - print( - "TestOperationsNumExprPandas: ", - expressions_module.USE_NUMEXPR, - td.USE_NUMEXPR, - USE_NUMEXPR, - ) cls.arith_ops = [ op for op in expr.ARITH_OPS_SYMS + expr.CMP_OPS_SYMS @@ -1356,8 +1340,6 @@ def f(): tm.assert_frame_equal(df, expected) def test_column_in(self): - print("test_column_in: ", expressions_module.USE_NUMEXPR, USE_NUMEXPR) - # GH 11235 df = DataFrame({"a": [11], "b": [-32]}) result = df.eval("a in [11, -32]") @@ -2019,8 +2001,6 @@ def test_bool_ops_fails_on_scalars(lhs, cmp, rhs, engine, parser): ], ) def test_equals_various(other): - print("DEBUG: ", expressions_module.USE_NUMEXPR, td.USE_NUMEXPR, USE_NUMEXPR) - df = DataFrame({"A": ["a", "b", "c"]}) result = df.eval(f"A == {other}") expected = Series([False, False, False], name="A") diff --git a/pyproject.toml b/pyproject.toml index fcfe2ac1d1c10..0c3e078d8761a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ exclude = ''' [tool.pytest.ini_options] # sync minversion with pyproject.toml & install.rst minversion = "6.0" -addopts = "--strict-data-files --strict-markers --durations=30 --junitxml=test-data.xml" +addopts = "--strict-data-files --strict-markers --capture=no --durations=30 --junitxml=test-data.xml" xfail_strict = true testpaths = "pandas" doctest_optionflags = [ From 6db8378724c0e0f8247ae4011d58b00d3766094c Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 25 Nov 2021 20:07:42 +0100 Subject: [PATCH 16/17] remove remaining print statements --- pandas/tests/test_downstream.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pandas/tests/test_downstream.py b/pandas/tests/test_downstream.py index 5b688a1a902c0..1972fbbe0f414 100644 --- a/pandas/tests/test_downstream.py +++ b/pandas/tests/test_downstream.py @@ -35,10 +35,7 @@ def test_dask(df): # dask sets "compute.use_numexpr" to False, so catch the current value # and ensure to reset it afterwards to avoid impacting other tests - from pandas.core.computation import expressions as expr - olduse = pd.get_option("compute.use_numexpr") - print("test_dask START: ", olduse) try: toolz = import_module("toolz") # noqa:F841 @@ -52,8 +49,6 @@ def test_dask(df): finally: pd.set_option("compute.use_numexpr", olduse) - print("test_dask END: ", expr.USE_NUMEXPR) - def test_xarray(df): From 1f98ca9a92bf00ea9d57f248b9bdd8c3e5cf93dd Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 25 Nov 2021 20:12:58 +0100 Subject: [PATCH 17/17] use set_option instead of expr.set_use_numexpr in tests to make them more robust --- pandas/_testing/contexts.py | 6 ++++-- pandas/tests/test_expressions.py | 29 +++++++++++++++-------------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/pandas/_testing/contexts.py b/pandas/_testing/contexts.py index e20d2d58e499f..b92772761e0a7 100644 --- a/pandas/_testing/contexts.py +++ b/pandas/_testing/contexts.py @@ -14,6 +14,8 @@ import numpy as np +from pandas import set_option + from pandas.io.common import get_handle @@ -202,11 +204,11 @@ def use_numexpr(use, min_elements=None): olduse = expr.USE_NUMEXPR oldmin = expr._MIN_ELEMENTS - expr.set_use_numexpr(use) + set_option("compute.use_numexpr", use) expr._MIN_ELEMENTS = min_elements yield expr._MIN_ELEMENTS = oldmin - expr.set_use_numexpr(olduse) + set_option("compute.use_numexpr", olduse) class RNGContext: diff --git a/pandas/tests/test_expressions.py b/pandas/tests/test_expressions.py index c6666a8c91e8d..115b8d716dd3f 100644 --- a/pandas/tests/test_expressions.py +++ b/pandas/tests/test_expressions.py @@ -5,6 +5,7 @@ import numpy as np import pytest +from pandas import set_option import pandas._testing as tm from pandas.core.api import ( DataFrame, @@ -65,9 +66,9 @@ def call_op(df, other, flex: bool, opname: str): else: op = getattr(operator, opname) - expr.set_use_numexpr(False) + set_option("compute.use_numexpr", False) expected = op(df, other) - expr.set_use_numexpr(True) + set_option("compute.use_numexpr", True) expr.get_test_result() @@ -107,9 +108,9 @@ def run_binary(self, df, other, flex: bool): def run_frame(self, df, other, flex: bool): self.run_arithmetic(df, other, flex) - expr.set_use_numexpr(False) + set_option("compute.use_numexpr", False) binary_comp = other + 1 - expr.set_use_numexpr(True) + set_option("compute.use_numexpr", True) self.run_binary(df, binary_comp, flex) for i in range(len(df.columns)): @@ -179,9 +180,9 @@ def testit(): result = expr._can_use_numexpr(op, op_str, right, right, "evaluate") assert not result - expr.set_use_numexpr(False) + set_option("compute.use_numexpr", False) testit() - expr.set_use_numexpr(True) + set_option("compute.use_numexpr", True) expr.set_numexpr_threads(1) testit() expr.set_numexpr_threads() @@ -215,9 +216,9 @@ def testit(): result = expr._can_use_numexpr(op, op_str, right, f22, "evaluate") assert not result - expr.set_use_numexpr(False) + set_option("compute.use_numexpr", False) testit() - expr.set_use_numexpr(True) + set_option("compute.use_numexpr", True) expr.set_numexpr_threads(1) testit() expr.set_numexpr_threads() @@ -233,9 +234,9 @@ def testit(): expected = np.where(c, df.values, df.values + 1) tm.assert_numpy_array_equal(result, expected) - expr.set_use_numexpr(False) + set_option("compute.use_numexpr", False) testit() - expr.set_use_numexpr(True) + set_option("compute.use_numexpr", True) expr.set_numexpr_threads(1) testit() expr.set_numexpr_threads() @@ -360,9 +361,9 @@ def test_frame_series_axis(self, axis, arith): op_func = getattr(df, arith) - expr.set_use_numexpr(False) + set_option("compute.use_numexpr", False) expected = op_func(other, axis=axis) - expr.set_use_numexpr(True) + set_option("compute.use_numexpr", True) result = op_func(other, axis=axis) tm.assert_frame_equal(expected, result) @@ -387,9 +388,9 @@ def test_python_semantics_with_numexpr_installed(self, op, box, scalar): result = method(scalar) # compare result with numpy - expr.set_use_numexpr(False) + set_option("compute.use_numexpr", False) expected = method(scalar) - expr.set_use_numexpr(True) + set_option("compute.use_numexpr", True) tm.assert_equal(result, expected) # compare result element-wise with Python