ENH: add ARRAY_API_TESTS_XFAIL_MARK to turn xfails into skips

Author: ev-br

README.md

@@ -305,6 +305,18 @@ ARRAY_API_TESTS_SKIP_DTYPES=uint16,uint32,uint64 pytest array_api_tests/
 Note that skipping certain essential dtypes such as `bool` and the default
 floating-point dtype is not supported.
 
+#### Turning xfails into skips
+
+Keeping a large number of ``xfails`` can have drastic effects on the run time. This is due
+to the way `hypothesis` works: when it detects a failure, it does a large amount
+of work to simplify the failing example.
+If the run time of the test suite becomes a problem, you can use the
+``ARRAY_API_TESTS_XFAIL_MARK`` environment variable: setting it to ``skip`` skips the
+entries from the ``xfail.txt`` file instead of xfailing them. Anecdotally, we saw
+speed-ups by a factor of 4-5---which allowed us to use 4-5 larger values of
+``--max-examples`` within the same time budget.
+
+
 ## Contributing
 
 ### Remain in-scope

conftest.py

@@ -144,6 +144,20 @@ def check_id_match(id_, pattern):
     return False
 
 
+def get_xfail_mark():
+    """Skip or xfail tests from the xfails-file.txt."""
+    m = os.environ.get("ARRAY_API_TESTS_XFAIL_MARK", "xfail")
+    if m == "xfail":
+        return mark.xfail
+    elif m == "skip":
+        return mark.skip
+    else:
+        raise ValueError(
+            f'ARRAY_API_TESTS_XFAIL_MARK value should be one of "skip" or "xfail" '
+            f'got {m} instead.'
+        )
+
+
 def pytest_collection_modifyitems(config, items):
     # 1. Prepare for iterating over items
     # -----------------------------------
@@ -187,6 +201,8 @@ def pytest_collection_modifyitems(config, items):
     # 2. Iterate through items and apply markers accordingly
     # ------------------------------------------------------
 
+    xfail_mark = get_xfail_mark()
+
     for item in items:
         markers = list(item.iter_markers())
         # skip if specified in skips file
@@ -198,7 +214,7 @@ def pytest_collection_modifyitems(config, items):
         # xfail if specified in xfails file
         for id_ in xfail_ids:
             if check_id_match(item.nodeid, id_):
-                item.add_marker(mark.xfail(reason=f"--xfails-file ({xfails_file})"))
+                item.add_marker(xfail_mark(reason=f"--xfails-file ({xfails_file})"))
                 xfail_id_matched[id_] = True
                 break
         # skip if disabled or non-existent extension