Merge pull request #92 from honno/ci-skips

Skip rather than xfail on CI

Author: honno

.github/workflows/numpy.yml

@@ -25,8 +25,8 @@ jobs:
       env:
         ARRAY_API_TESTS_MODULE: numpy.array_api
       run: |
-        # Mark some known issues as XFAIL
-        cat << EOF >> xfails.txt
+        # Skip test cases with known issues
+        cat << EOF >> skips.txt
 
         # copy not implemented
         array_api_tests/test_creation_functions.py::test_asarray_arrays

array_api_tests/test_linalg.py

@@ -229,6 +229,7 @@ def true_diag(x_stack):
 
     _test_stacks(linalg.diagonal, x, **kw, res=res, dims=1, true_val=true_diag)
 
+@pytest.mark.skip(reason="Inputs need to be restricted")  # TODO
 @pytest.mark.xp_extension('linalg')
 @given(x=symmetric_matrices(finite=True))
 def test_eigh(x):

array_api_tests/test_statistical_functions.py

@@ -69,15 +69,7 @@ def test_mean(x, data):
     ph.assert_keepdimable_shape(
         "mean", out.shape, x.shape, _axes, kw.get("keepdims", False), **kw
     )
-    for indices, out_idx in zip(sh.axes_ndindex(x.shape, _axes), sh.ndindex(out.shape)):
-        mean = float(out[out_idx])
-        assume(not math.isinf(mean))  # mean may become inf due to internal overflows
-        elements = []
-        for idx in indices:
-            s = float(x[idx])
-            elements.append(s)
-        expected = sum(elements) / len(elements)
-        ph.assert_scalar_equals("mean", float, out_idx, mean, expected)
+    # Values testing mean is too finicky
 
 
 @given(

conftest.py

@@ -71,14 +71,14 @@ def xp_has_ext(ext: str) -> bool:
         return False
 
 
-xfail_ids = []
-xfails_path = Path(__file__).parent / "xfails.txt"
-if xfails_path.exists():
-    with open(xfails_path) as f:
+skip_ids = []
+skips_path = Path(__file__).parent / "skips.txt"
+if skips_path.exists():
+    with open(skips_path) as f:
         for line in f:
             if line.startswith("array_api_tests"):
                 id_ = line.strip("\n")
-                xfail_ids.append(id_)
+                skip_ids.append(id_)
 
 
 def pytest_collection_modifyitems(config, items):
@@ -96,10 +96,10 @@ def pytest_collection_modifyitems(config, items):
                 )
             elif not xp_has_ext(ext):
                 item.add_marker(mark.skip(reason=f"{ext} not found in array module"))
-        # xfail if specified in xfails.txt
-        for id_ in xfail_ids:
+        # skip if specified in skips.txt
+        for id_ in skip_ids:
             if item.nodeid.startswith(id_):
-                item.add_marker(mark.xfail(reason="xfails.txt"))
+                item.add_marker(mark.skip(reason="skips.txt"))
                 break
         # skip if test not appropiate for CI
         if ci: