Merge pull request #373 from ev-br/skip_xfails

ENH: add ARRAY_API_TESTS_XFAIL_MARK to turn xfails into skips

Author: ev-br

README.md

@@ -305,6 +305,26 @@ ARRAY_API_TESTS_SKIP_DTYPES=uint16,uint32,uint64 pytest array_api_tests/
 Note that skipping certain essential dtypes such as `bool` and the default
 floating-point dtype is not supported.
 
+#### Turning xfails into skips
+
+Keeping a large number of ``xfails`` can have drastic effects on the run time. This is due
+to the way `hypothesis` works: when it detects a failure, it does a large amount
+of work to simplify the failing example.
+If the run time of the test suite becomes a problem, you can use the
+``ARRAY_API_TESTS_XFAIL_MARK`` environment variable: setting it to ``skip`` skips the
+entries from the ``xfail.txt`` file instead of xfailing them. Anecdotally, we saw
+speed-ups by a factor of 4-5---which allowed us to use 4-5 larger values of
+``--max-examples`` within the same time budget.
+
+#### Limiting the array sizes
+
+The test suite generates random arrays as inputs to functions it tests. "unvectorized"
+tests iterate over elements of arrays, which might be slow. If the run time becomes
+a problem, you can limit the maximum number of elements in generated arrays by
+setting the environment variable ``ARRAY_API_TESTS_MAX_ARRAY_SIZE`` to the
+desired value. By default, it is set to 1024.
+
+
 ## Contributing
 
 ### Remain in-scope

array_api_tests/hypothesis_helpers.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import os
 import re
 from contextlib import contextmanager
 from functools import wraps
@@ -232,7 +233,7 @@ def all_floating_dtypes() -> SearchStrategy[DataType]:
     lambda i: getattr(xp, i))
 
 # Limit the total size of an array shape
-MAX_ARRAY_SIZE = 10000
+MAX_ARRAY_SIZE = int(os.environ.get("ARRAY_API_TESTS_MAX_ARRAY_SIZE", 1024))
 # Size to use for 2-dim arrays
 SQRT_MAX_ARRAY_SIZE = int(math.sqrt(MAX_ARRAY_SIZE))
 

array_api_tests/test_creation_functions.py

@@ -499,7 +499,7 @@ def test_meshgrid(dtype, data):
     shapes = data.draw(
         st.integers(1, 5).flatmap(
             lambda n: hh.mutually_broadcastable_shapes(
-                n, min_dims=1, max_dims=1, max_side=5
+                n, min_dims=1, max_dims=1, max_side=4
             )
         ),
         label="shapes",
@@ -509,7 +509,7 @@ def test_meshgrid(dtype, data):
         x = data.draw(hh.arrays(dtype=dtype, shape=shape), label=f"x{i}")
         arrays.append(x)
     # sanity check
-    assert math.prod(math.prod(x.shape) for x in arrays) <= hh.MAX_ARRAY_SIZE
+    # assert math.prod(math.prod(x.shape) for x in arrays) <= hh.MAX_ARRAY_SIZE
     out = xp.meshgrid(*arrays)
     for i, x in enumerate(out):
         ph.assert_dtype("meshgrid", in_dtype=dtype, out_dtype=x.dtype, repr_name=f"out[{i}].dtype")

conftest.py

@@ -144,6 +144,20 @@ def check_id_match(id_, pattern):
     return False
 
 
+def get_xfail_mark():
+    """Skip or xfail tests from the xfails-file.txt."""
+    m = os.environ.get("ARRAY_API_TESTS_XFAIL_MARK", "xfail")
+    if m == "xfail":
+        return mark.xfail
+    elif m == "skip":
+        return mark.skip
+    else:
+        raise ValueError(
+            f'ARRAY_API_TESTS_XFAIL_MARK value should be one of "skip" or "xfail" '
+            f'got {m} instead.'
+        )
+
+
 def pytest_collection_modifyitems(config, items):
     # 1. Prepare for iterating over items
     # -----------------------------------
@@ -187,6 +201,8 @@ def pytest_collection_modifyitems(config, items):
     # 2. Iterate through items and apply markers accordingly
     # ------------------------------------------------------
 
+    xfail_mark = get_xfail_mark()
+
     for item in items:
         markers = list(item.iter_markers())
         # skip if specified in skips file
@@ -198,7 +214,7 @@ def pytest_collection_modifyitems(config, items):
         # xfail if specified in xfails file
         for id_ in xfail_ids:
             if check_id_match(item.nodeid, id_):
-                item.add_marker(mark.xfail(reason=f"--xfails-file ({xfails_file})"))
+                item.add_marker(xfail_mark(reason=f"--xfails-file ({xfails_file})"))
                 xfail_id_matched[id_] = True
                 break
         # skip if disabled or non-existent extension