Merge pull request #244 from honno/ci-updates

CI updates

Author: honno

Diff for: .github/workflows/numpy.yml renamed to .github/workflows/test.yml

@@ -1,4 +1,4 @@
-name: NumPy Array API
+name: Test Array API Strict
 
 on: [push, pull_request]
 
@@ -22,10 +22,12 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install numpy==1.26.2
+        python -m pip install array-api-strict
         python -m pip install -r requirements.txt
     - name: Run the test suite
       env:
-        ARRAY_API_TESTS_MODULE: numpy.array_api
+        ARRAY_API_TESTS_MODULE: array_api_strict
       run: |
-        pytest -v -rxXfE --ci --skips-file numpy-skips.txt
+        pytest -v -rxXfE --skips-file array-api-strict-skips.txt array_api_tests/
+        # We also have internal tests that isn't really necessary for adopters
+        pytest -v -rxXfE meta_tests/

Diff for: README.md

@@ -138,9 +138,9 @@ issues](https://github.com/data-apis/array-api-tests/issues/) to us.
 
 ## Running on CI
 
-See our existing [GitHub Actions workflow for
-Numpy](https://github.com/data-apis/array-api-tests/blob/master/.github/workflows/numpy.yml)
-for an example of using the test suite on CI.
+See our existing [GitHub Actions workflow for `array-api-strict`](https://github.com/data-apis/array-api-tests/blob/master/.github/workflows/test.yml)
+for an example of using the test suite on CI. Note [`array-api-strict`](https://github.com/data-apis/array-api-strict)
+is an implementation of the array API that uses NumPy under the hood.
 
 ### Releases
 
@@ -161,12 +161,6 @@ You can specify the API version to use when testing via the
 array module's `__array_api_version__` value, and if that attribute doesn't
 exist then we fallback to `"2021.12"`.
 
-#### CI flag
-
-Use the `--ci` flag to run only the primary and special cases tests. You can
-ignore the other test cases as they are redundant for the purposes of checking
-compliance.
-
 #### Data-dependent shapes
 
 Use the `--disable-data-dependent-shapes` flag to skip testing functions which have

Diff for: array-api-strict-skips.txt

@@ -0,0 +1,8 @@
+# Known special case issue in NumPy. Not worth working around here
+# https://github.com/numpy/numpy/issues/21213
+array_api_tests/test_special_cases.py::test_iop[__ipow__(x1_i is -infinity and x2_i > 0 and not (x2_i.is_integer() and x2_i % 2 == 1)) -> +infinity]
+array_api_tests/test_special_cases.py::test_iop[__ipow__(x1_i is -0 and x2_i > 0 and not (x2_i.is_integer() and x2_i % 2 == 1)) -> +0]
+
+# The test suite is incorrectly checking sums that have loss of significance
+# (https://github.com/data-apis/array-api-tests/issues/168)
+array_api_tests/test_statistical_functions.py::test_sum

Diff for: array_api_tests/pytest_helpers.py

@@ -485,7 +485,7 @@ def assert_array_elements(
         >>> assert xp.all(out == x)
 
     """
-    # __tracebackhide__ = True
+    __tracebackhide__ = True
     dh.result_type(out.dtype, expected.dtype)  # sanity check
     assert_shape(func_name, out_shape=out.shape, expected=expected.shape, kw=kw)  # sanity check
     f_func = f"[{func_name}({fmt_kw(kw)})]"

Diff for: array_api_tests/test_array_object.py

@@ -16,8 +16,6 @@
 from . import xp as _xp
 from .typing import DataType, Index, Param, Scalar, ScalarType, Shape
 
-pytestmark = pytest.mark.ci
-
 
 def scalar_objects(
     dtype: DataType, shape: Shape
@@ -107,6 +105,7 @@ def test_getitem(shape, dtype, data):
         ph.assert_array_elements("__getitem__", out=out, expected=expected)
 
 
+@pytest.mark.unvectorized
 @given(
     shape=hh.shapes(),
     dtypes=hh.oneway_promotable_dtypes(dh.all_dtypes),
@@ -154,6 +153,7 @@ def test_setitem(shape, dtypes, data):
         )
 
 
+@pytest.mark.unvectorized
 @pytest.mark.data_dependent_shapes
 @given(hh.shapes(), st.data())
 def test_getitem_masking(shape, data):
@@ -199,6 +199,7 @@ def test_getitem_masking(shape, data):
                 )
 
 
+@pytest.mark.unvectorized
 @given(hh.shapes(), st.data())
 def test_setitem_masking(shape, data):
     x = data.draw(hh.arrays(xps.scalar_dtypes(), shape=shape), label="x")

Diff for: array_api_tests/test_constants.py

@@ -7,8 +7,6 @@
 from . import xp
 from .typing import Array
 
-pytestmark = pytest.mark.ci
-
 
 def assert_scalar_float(name: str, c: Any):
     assert isinstance(c, SupportsFloat), f"{name}={c!r} does not look like a float"

Diff for: array_api_tests/test_creation_functions.py

@@ -3,7 +3,6 @@
 from itertools import count
 from typing import Iterator, NamedTuple, Union
 
-import pytest
 from hypothesis import assume, given, note
 from hypothesis import strategies as st
 
@@ -15,8 +14,6 @@
 from . import xps
 from .typing import DataType, Scalar
 
-pytestmark = pytest.mark.ci
-
 
 class frange(NamedTuple):
     start: float

Diff for: array_api_tests/test_data_type_functions.py

@@ -14,8 +14,6 @@
 from . import xp as _xp
 from .typing import DataType
 
-pytestmark = pytest.mark.ci
-
 
 # TODO: test with complex dtypes
 def non_complex_dtypes():

Diff for: array_api_tests/test_fft.py

@@ -17,7 +17,6 @@
 from . import xp
 
 pytestmark = [
-    pytest.mark.ci,
     pytest.mark.xp_extension("fft"),
     pytest.mark.min_version("2022.12"),
 ]

Diff for: array_api_tests/test_has_names.py

@@ -9,8 +9,6 @@
 from .stubs import (array_attributes, array_methods, category_to_funcs,
                     extension_to_funcs, EXTENSIONS)
 
-pytestmark = pytest.mark.ci
-
 has_name_params = []
 for ext, stubs in extension_to_funcs.items():
     for stub in stubs:

Diff for: array_api_tests/test_indexing_functions.py

@@ -9,9 +9,8 @@
 from . import shape_helpers as sh
 from . import xps
 
-pytestmark = pytest.mark.ci
-
 
+@pytest.mark.unvectorized
 @pytest.mark.min_version("2022.12")
 @given(
     x=hh.arrays(xps.scalar_dtypes(), hh.shapes(min_dims=1, min_side=1)),

Diff for: array_api_tests/test_linalg.py

@@ -43,8 +43,6 @@
 from . import _array_module as xp
 from ._array_module import linalg
 
-pytestmark = pytest.mark.ci
-
 def assert_equal(x, y, msg_extra=None):
     extra = '' if not msg_extra else f' ({msg_extra})'
     if x.dtype in dh.all_float_dtypes:

Diff for: array_api_tests/test_manipulation_functions.py

@@ -14,8 +14,6 @@
 from . import xps
 from .typing import Array, Shape
 
-pytestmark = pytest.mark.ci
-
 MAX_SIDE = hh.MAX_ARRAY_SIZE // 64
 MAX_DIMS = min(hh.MAX_ARRAY_SIZE // MAX_SIDE, 32)  # NumPy only supports up to 32 dims
 
@@ -121,6 +119,7 @@ def test_concat(dtypes, base_shape, data):
                     )
 
 
+@pytest.mark.unvectorized
 @given(
     x=hh.arrays(dtype=xps.scalar_dtypes(), shape=shared_shapes()),
     axis=shared_shapes().flatmap(
@@ -149,6 +148,7 @@ def test_expand_dims(x, axis):
     )
 
 
+@pytest.mark.unvectorized
 @given(
     x=hh.arrays(
         dtype=xps.scalar_dtypes(), shape=hh.shapes(min_side=1).filter(lambda s: 1 in s)
@@ -186,6 +186,7 @@ def test_squeeze(x, data):
     assert_array_ndindex("squeeze", x, x_indices=sh.ndindex(x.shape), out=out, out_indices=sh.ndindex(out.shape))
 
 
+@pytest.mark.unvectorized
 @given(
     x=hh.arrays(dtype=xps.scalar_dtypes(), shape=hh.shapes()),
     data=st.data(),
@@ -210,6 +211,7 @@ def test_flip(x, data):
                              out_indices=reverse_indices, kw=kw)
 
 
+@pytest.mark.unvectorized
 @given(
     x=hh.arrays(dtype=xps.scalar_dtypes(), shape=shared_shapes(min_dims=1)),
     axes=shared_shapes(min_dims=1).flatmap(
@@ -250,6 +252,7 @@ def reshape_shapes(draw, shape):
     return tuple(rshape)
 
 
+@pytest.mark.unvectorized
 @pytest.mark.skip("flaky")  # TODO: fix!
 @given(
     x=hh.arrays(dtype=xps.scalar_dtypes(), shape=hh.shapes(max_side=MAX_SIDE)),
@@ -282,6 +285,7 @@ def roll_ndindex(shape: Shape, shifts: Tuple[int], axes: Tuple[int]) -> Iterator
         yield tuple((i + sh) % si for i, sh, si in zip(idx, all_shifts, shape))
 
 
+@pytest.mark.unvectorized
 @given(hh.arrays(dtype=xps.scalar_dtypes(), shape=shared_shapes()), st.data())
 def test_roll(x, data):
     shift_strat = st.integers(-hh.MAX_ARRAY_SIZE, hh.MAX_ARRAY_SIZE)
@@ -321,6 +325,7 @@ def test_roll(x, data):
         assert_array_ndindex("roll", x, x_indices=sh.ndindex(x.shape), out=out, out_indices=shifted_indices, kw=kw)
 
 
+@pytest.mark.unvectorized
 @given(
     shape=shared_shapes(min_dims=1),
     dtypes=hh.mutually_promotable_dtypes(None),

Diff for: array_api_tests/test_operators_and_elementwise_functions.py

@@ -21,7 +21,8 @@
 from . import xps
 from .typing import Array, DataType, Param, Scalar, ScalarType, Shape
 
-pytestmark = pytest.mark.ci
+
+pytestmark = pytest.mark.unvectorized
 
 
 def all_integer_dtypes() -> st.SearchStrategy[DataType]:
@@ -457,7 +458,7 @@ class UnaryParamContext(NamedTuple):
 
     @property
     def id(self) -> str:
-        return f"{self.func_name}"
+        return self.func_name
 
     def __repr__(self):
         return f"UnaryParamContext(<{self.id}>)"

Diff for: array_api_tests/test_searching_functions.py

@@ -11,7 +11,8 @@
 from . import shape_helpers as sh
 from . import xps
 
-pytestmark = pytest.mark.ci
+
+pytestmark = pytest.mark.unvectorized
 
 
 @given(

Diff for: array_api_tests/test_set_functions.py

@@ -13,7 +13,7 @@
 from . import shape_helpers as sh
 from . import xps
 
-pytestmark = [pytest.mark.ci, pytest.mark.data_dependent_shapes]
+pytestmark = [pytest.mark.data_dependent_shapes, pytest.mark.unvectorized]
 
 
 @given(hh.arrays(dtype=xps.scalar_dtypes(), shape=hh.shapes(min_side=1)))

Diff for: array_api_tests/test_signatures.py

@@ -33,8 +33,6 @@ def squeeze(x, /, axis):
 from . import xp
 from .stubs import array_methods, category_to_funcs, extension_to_funcs, name_to_func
 
-pytestmark = pytest.mark.ci
-
 ParameterKind = Literal[
     Parameter.POSITIONAL_ONLY,
     Parameter.VAR_POSITIONAL,

Diff for: array_api_tests/test_sorting_functions.py

@@ -14,8 +14,6 @@
 from . import xps
 from .typing import Scalar, Shape
 
-pytestmark = pytest.mark.ci
-
 
 def assert_scalar_in_set(
     func_name: str,
@@ -32,6 +30,7 @@ def assert_scalar_in_set(
 
 
 # TODO: Test with signed zeros and NaNs (and ignore them somehow)
+@pytest.mark.unvectorized
 @given(
     x=hh.arrays(
         dtype=xps.real_dtypes(),
@@ -91,6 +90,7 @@ def test_argsort(x, data):
                     )
 
 
+@pytest.mark.unvectorized
 # TODO: Test with signed zeros and NaNs (and ignore them somehow)
 @given(
     x=hh.arrays(

Diff for: array_api_tests/test_special_cases.py

@@ -35,8 +35,6 @@
 from . import xp, xps
 from .stubs import category_to_funcs
 
-pytestmark = pytest.mark.ci
-
 UnaryCheck = Callable[[float], bool]
 BinaryCheck = Callable[[float, float], bool]
 
@@ -1212,6 +1210,7 @@ def parse_binary_case_block(case_block: str) -> List[BinaryCase]:
 assert len(iop_params) != 0
 
 
+@pytest.mark.unvectorized
 @pytest.mark.parametrize("func_name, func, case", unary_params)
 @given(
     x=hh.arrays(dtype=xps.floating_dtypes(), shape=hh.shapes(min_side=1)),
@@ -1250,6 +1249,7 @@ def test_unary(func_name, func, case, x, data):
 )
 
 
+@pytest.mark.unvectorized
 @pytest.mark.parametrize("func_name, func, case", binary_params)
 @given(x1=x1_strat, x2=x2_strat, data=st.data())
 def test_binary(func_name, func, case, x1, x2, data):
@@ -1294,6 +1294,7 @@ def test_binary(func_name, func, case, x1, x2, data):
     assume(good_example)
 
 
+@pytest.mark.unvectorized
 @pytest.mark.parametrize("iop_name, iop, case", iop_params)
 @given(
     oneway_dtypes=hh.oneway_promotable_dtypes(dh.real_float_dtypes),

Diff for: array_api_tests/test_statistical_functions.py

@@ -15,8 +15,6 @@
 from ._array_module import _UndefinedStub
 from .typing import DataType
 
-pytestmark = pytest.mark.ci
-
 
 def kwarg_dtypes(dtype: DataType) -> st.SearchStrategy[Optional[DataType]]:
     dtypes = [d2 for d1, d2 in dh.promotion_table if d1 == dtype]
@@ -25,6 +23,7 @@ def kwarg_dtypes(dtype: DataType) -> st.SearchStrategy[Optional[DataType]]:
     return st.none() | st.sampled_from(dtypes)
 
 
+@pytest.mark.unvectorized
 @given(
     x=hh.arrays(
         dtype=xps.real_dtypes(),
@@ -77,6 +76,7 @@ def test_mean(x, data):
     # Values testing mean is too finicky
 
 
+@pytest.mark.unvectorized
 @given(
     x=hh.arrays(
         dtype=xps.real_dtypes(),
@@ -107,6 +107,7 @@ def test_min(x, data):
         ph.assert_scalar_equals("min", type_=scalar_type, idx=out_idx, out=min_, expected=expected)
 
 
+@pytest.mark.unvectorized
 @given(
     x=hh.arrays(
         dtype=xps.numeric_dtypes(),
@@ -195,6 +196,7 @@ def test_std(x, data):
     # We can't easily test the result(s) as standard deviation methods vary a lot
 
 
+@pytest.mark.unvectorized
 @pytest.mark.skip("flaky")  # TODO: fix!
 @given(
     x=hh.arrays(
@@ -247,6 +249,7 @@ def test_sum(x, data):
         ph.assert_scalar_equals("sum", type_=scalar_type, idx=out_idx, out=sum_, expected=expected)
 
 
+@pytest.mark.unvectorized
 @pytest.mark.skip(reason="flaky")  # TODO: fix!
 @given(
     x=hh.arrays(