From bbd5097aa899f43492e5cb189daa0c729db9157d Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 18 Mar 2021 08:45:21 +0100 Subject: [PATCH 1/5] [ArrayManager] TST: enable remaining top-level tests --- .github/workflows/ci.yml | 1 + pandas/tests/test_downstream.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b40670d743e18..ccc9ee59efcef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -165,6 +165,7 @@ jobs: pytest pandas/tests/series/ pytest pandas/tests/indexing/ + pytest pandas/tests/test_* pytest pandas/tests/api/ pytest pandas/tests/apply/ pytest pandas/tests/arrays/ diff --git a/pandas/tests/test_downstream.py b/pandas/tests/test_downstream.py index 911f1c7ebe31c..ea95f90d3a2cb 100644 --- a/pandas/tests/test_downstream.py +++ b/pandas/tests/test_downstream.py @@ -29,6 +29,9 @@ def df(): return DataFrame({"A": [1, 2, 3]}) +# TODO(ArrayManager) dask is still accessing the blocks +# https://github.com/dask/dask/pull/7318 +@td.skip_array_manager_not_yet_implemented def test_dask(df): toolz = import_module("toolz") # noqa From f78309e947aa1d6193ffc584adeafae9950f0968 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 18 Mar 2021 08:51:26 +0100 Subject: [PATCH 2/5] add internals --- .github/workflows/ci.yml | 1 + pandas/tests/internals/test_internals.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ccc9ee59efcef..014c666a17084 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,6 +176,7 @@ jobs: pytest pandas/tests/extension/ pytest pandas/tests/generic/ pytest pandas/tests/indexes/ + pytest pandas/tests/internals/ pytest pandas/tests/io/test_* -m "not slow and not clipboard" pytest pandas/tests/io/excel/ -m "not slow and not clipboard" pytest pandas/tests/io/formats/ -m "not slow and not clipboard" diff --git a/pandas/tests/internals/test_internals.py b/pandas/tests/internals/test_internals.py index ba85ff1a044d6..c63d5271f1fae 100644 --- a/pandas/tests/internals/test_internals.py +++ b/pandas/tests/internals/test_internals.py @@ -9,6 +9,7 @@ import pytest from pandas._libs.internals import BlockPlacement +import pandas.util._test_decorators as td from pandas.core.dtypes.common import is_scalar @@ -38,6 +39,10 @@ ) from pandas.core.internals.blocks import new_block +# this file contains BlockManager specific tests +# TODO(ArrayManager) factor out interleave_dtype tests +pytestmark = td.skip_array_manager_invalid_test + @pytest.fixture(params=[new_block, make_block]) def block_maker(request): From e4dcc0580c81ba0c306fb75a1f63969a9df23452 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 18 Mar 2021 09:09:27 +0100 Subject: [PATCH 3/5] [ArrayManager] TST: run all tests with separate not slow / slow build --- .github/workflows/ci.yml | 44 +++++++--------------------------------- 1 file changed, 7 insertions(+), 37 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 014c666a17084..e8955f0cc297f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,6 +141,9 @@ jobs: data_manager: name: Test experimental data manager runs-on: ubuntu-latest + strategy: + matrix: + pattern: ["not slow and not network and not clipboard", "slow"] steps: - name: Checkout @@ -152,43 +155,10 @@ jobs: - name: Run tests env: PANDAS_DATA_MANAGER: array + PATTERN: ${{ matrix.pattern }} run: | source activate pandas-dev + ci/run_tests.sh - pytest pandas/tests/frame/ - pytest pandas/tests/reductions/ - pytest pandas/tests/generic/test_generic.py - pytest pandas/tests/arithmetic/ - pytest pandas/tests/groupby/ - pytest pandas/tests/resample/ - pytest pandas/tests/reshape/merge - pytest pandas/tests/series/ - pytest pandas/tests/indexing/ - - pytest pandas/tests/test_* - pytest pandas/tests/api/ - pytest pandas/tests/apply/ - pytest pandas/tests/arrays/ - pytest pandas/tests/base/ - pytest pandas/tests/computation/ - pytest pandas/tests/config/ - pytest pandas/tests/dtypes/ - pytest pandas/tests/extension/ - pytest pandas/tests/generic/ - pytest pandas/tests/indexes/ - pytest pandas/tests/internals/ - pytest pandas/tests/io/test_* -m "not slow and not clipboard" - pytest pandas/tests/io/excel/ -m "not slow and not clipboard" - pytest pandas/tests/io/formats/ -m "not slow and not clipboard" - pytest pandas/tests/io/parser/ -m "not slow and not clipboard" - pytest pandas/tests/io/sas/ -m "not slow and not clipboard" - pytest pandas/tests/io/xml/ -m "not slow and not clipboard" - pytest pandas/tests/libs/ - pytest pandas/tests/plotting/ - pytest pandas/tests/scalar/ - pytest pandas/tests/strings/ - pytest pandas/tests/tools/ - pytest pandas/tests/tseries/ - pytest pandas/tests/tslibs/ - pytest pandas/tests/util/ - pytest pandas/tests/window/ + - name: Print skipped tests + run: python ci/print_skipped.py From b470fb6e52cf7ed516e79a8e32bd5ed184783c70 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 18 Mar 2021 09:33:20 +0100 Subject: [PATCH 4/5] specify n workers --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8955f0cc297f..d6744f578560c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -156,6 +156,7 @@ jobs: env: PANDAS_DATA_MANAGER: array PATTERN: ${{ matrix.pattern }} + PYTEST_WORKERS: "auto" run: | source activate pandas-dev ci/run_tests.sh From 367bb80a484a9525c98b296388f19b39609425b0 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 18 Mar 2021 11:19:14 +0100 Subject: [PATCH 5/5] skip few remaining concat tests --- pandas/tests/reshape/concat/__init__.py | 4 ++++ pandas/tests/reshape/test_crosstab.py | 3 +++ pandas/tests/reshape/test_pivot.py | 3 +++ pandas/tests/reshape/test_pivot_multilevel.py | 3 +++ 4 files changed, 13 insertions(+) diff --git a/pandas/tests/reshape/concat/__init__.py b/pandas/tests/reshape/concat/__init__.py index e69de29bb2d1d..777923be02398 100644 --- a/pandas/tests/reshape/concat/__init__.py +++ b/pandas/tests/reshape/concat/__init__.py @@ -0,0 +1,4 @@ +import pandas.util._test_decorators as td + +# TODO(ArrayManager) concat axis=0 +pytestmark = td.skip_array_manager_not_yet_implemented diff --git a/pandas/tests/reshape/test_crosstab.py b/pandas/tests/reshape/test_crosstab.py index e467dbb7d49b6..5cc65feee869b 100644 --- a/pandas/tests/reshape/test_crosstab.py +++ b/pandas/tests/reshape/test_crosstab.py @@ -1,6 +1,8 @@ import numpy as np import pytest +import pandas.util._test_decorators as td + from pandas.core.dtypes.common import is_categorical_dtype from pandas import ( @@ -438,6 +440,7 @@ def test_crosstab_normalize_arrays(self): ) tm.assert_frame_equal(test_case, norm_sum) + @td.skip_array_manager_not_yet_implemented # TODO(ArrayManager) concat axis=0 def test_crosstab_with_empties(self): # Check handling of empties df = DataFrame( diff --git a/pandas/tests/reshape/test_pivot.py b/pandas/tests/reshape/test_pivot.py index c50886ba43019..8d8a83c233444 100644 --- a/pandas/tests/reshape/test_pivot.py +++ b/pandas/tests/reshape/test_pivot.py @@ -8,6 +8,8 @@ import numpy as np import pytest +import pandas.util._test_decorators as td + import pandas as pd from pandas import ( Categorical, @@ -1197,6 +1199,7 @@ def test_pivot_table_with_margins_set_margin_name(self, margin_name): margins_name=margin_name, ) + @td.skip_array_manager_not_yet_implemented # TODO(ArrayManager) concat axis=0 def test_pivot_timegrouper(self): df = DataFrame( { diff --git a/pandas/tests/reshape/test_pivot_multilevel.py b/pandas/tests/reshape/test_pivot_multilevel.py index df2ae0d52c660..ab41a94d1ff25 100644 --- a/pandas/tests/reshape/test_pivot_multilevel.py +++ b/pandas/tests/reshape/test_pivot_multilevel.py @@ -1,6 +1,8 @@ import numpy as np import pytest +import pandas.util._test_decorators as td + import pandas as pd from pandas import ( Index, @@ -196,6 +198,7 @@ def test_pivot_list_like_columns( tm.assert_frame_equal(result, expected) +@td.skip_array_manager_not_yet_implemented # TODO(ArrayManager) concat axis=0 def test_pivot_multiindexed_rows_and_cols(): # GH 36360