From a4c64a5df513d0a45adb092cbfdbea577f9b183c Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Mon, 30 Jun 2025 21:05:28 +0100 Subject: [PATCH 1/6] add support for pytest-xdist --- pytest_mpl/plugin.py | 89 ++++++++++++++++++++++++++++------ tests/subtests/helpers.py | 14 +++++- tests/subtests/test_subtest.py | 47 +++++++++++++++++- tox.ini | 3 +- 4 files changed, 134 insertions(+), 19 deletions(-) diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 49029d6b..969d745a 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -31,6 +31,7 @@ import io import os import json +import uuid import shutil import hashlib import logging @@ -216,6 +217,12 @@ def pytest_addoption(parser): parser.addini(option, help=msg) +class XdistPlugin: + def pytest_configure_node(self, node): + node.workerinput["pytest_mpl_uid"] = node.config.pytest_mpl_uid + node.workerinput["pytest_mpl_results_dir"] = node.config.pytest_mpl_results_dir + + def pytest_configure(config): config.addinivalue_line( @@ -288,12 +295,20 @@ def get_cli_or_ini(name, default=None): if not _hash_library_from_cli: hash_library = os.path.abspath(hash_library) + if not hasattr(config, "workerinput"): + uid = uuid.uuid4().hex + results_dir_path = results_dir or tempfile.mkdtemp() + config.pytest_mpl_uid = uid + config.pytest_mpl_results_dir = results_dir_path + + if config.pluginmanager.hasplugin("xdist"): + config.pluginmanager.register(XdistPlugin(), name="pytest_mpl_xdist_plugin") + plugin = ImageComparison( config, baseline_dir=baseline_dir, baseline_relative_dir=baseline_relative_dir, generate_dir=generate_dir, - results_dir=results_dir, hash_library=hash_library, generate_hash_library=generate_hash_lib, generate_summary=generate_summary, @@ -356,7 +371,6 @@ def __init__( baseline_dir=None, baseline_relative_dir=None, generate_dir=None, - results_dir=None, hash_library=None, generate_hash_library=None, generate_summary=None, @@ -372,7 +386,7 @@ def __init__( self.baseline_dir = baseline_dir self.baseline_relative_dir = path_is_not_none(baseline_relative_dir) self.generate_dir = path_is_not_none(generate_dir) - self.results_dir = path_is_not_none(results_dir) + self.results_dir = None self.hash_library = path_is_not_none(hash_library) self._hash_library_from_cli = _hash_library_from_cli # for backwards compatibility self.generate_hash_library = path_is_not_none(generate_hash_library) @@ -394,11 +408,6 @@ def __init__( self.deterministic = deterministic self.default_backend = default_backend - # Generate the containing dir for all test results - if not self.results_dir: - self.results_dir = Path(tempfile.mkdtemp(dir=self.results_dir)) - self.results_dir.mkdir(parents=True, exist_ok=True) - # Decide what to call the downloadable results hash library if self.hash_library is not None: self.results_hash_library_name = self.hash_library.name @@ -411,6 +420,14 @@ def __init__( self._test_stats = None self.return_value = {} + def pytest_sessionstart(self, session): + config = session.config + if hasattr(config, "workerinput"): + config.pytest_mpl_uid = config.workerinput["pytest_mpl_uid"] + config.pytest_mpl_results_dir = config.workerinput["pytest_mpl_results_dir"] + self.results_dir = Path(config.pytest_mpl_results_dir) + self.results_dir.mkdir(parents=True, exist_ok=True) + def get_logger(self): # configure a separate logger for this pluggin which is independent # of the options that are configured for pytest or for the code that @@ -932,27 +949,65 @@ def pytest_runtest_call(self, item): # noqa result._result = None result._excinfo = (type(e), e, e.__traceback__) + def generate_hash_library_json(self): + if hasattr(self.config, "workerinput"): + uid = self.config.pytest_mpl_uid + worker_id = os.environ.get("PYTEST_XDIST_WORKER") + json_file = self.results_dir / f"generated-hashes-xdist-{uid}-{worker_id}.json" + else: + json_file = Path(self.config.rootdir) / self.generate_hash_library + json_file.parent.mkdir(parents=True, exist_ok=True) + with open(json_file, 'w') as f: + json.dump(self._generated_hash_library, f, indent=2) + return json_file + def generate_summary_json(self): - json_file = self.results_dir / 'results.json' + filename = "results.json" + if hasattr(self.config, "workerinput"): + uid = self.config.pytest_mpl_uid + worker_id = os.environ.get("PYTEST_XDIST_WORKER") + filename = f"results-xdist-{uid}-{worker_id}.json" + json_file = self.results_dir / filename with open(json_file, 'w') as f: json.dump(self._test_results, f, indent=2) return json_file - def pytest_unconfigure(self, config): + def pytest_sessionfinish(self, session): """ Save out the hash library at the end of the run. """ + config = session.config + try: + import xdist + is_xdist_controller = xdist.is_xdist_controller(session) + is_xdist_worker = xdist.is_xdist_worker(session) + except ImportError: + is_xdist_controller = False + is_xdist_worker = False + except Exception as e: + if "xdist" not in session.config.option: + is_xdist_controller = False + is_xdist_worker = False + else: + raise e + + if is_xdist_controller: # Merge results from workers + uid = config.pytest_mpl_uid + for worker_hashes in self.results_dir.glob(f"generated-hashes-xdist-{uid}-*.json"): + with worker_hashes.open() as f: + self._generated_hash_library.update(json.load(f)) + for worker_results in self.results_dir.glob(f"results-xdist-{uid}-*.json"): + with worker_results.open() as f: + self._test_results.update(json.load(f)) + result_hash_library = self.results_dir / (self.results_hash_library_name or "temp.json") if self.generate_hash_library is not None: - hash_library_path = Path(config.rootdir) / self.generate_hash_library - hash_library_path.parent.mkdir(parents=True, exist_ok=True) - with open(hash_library_path, "w") as fp: - json.dump(self._generated_hash_library, fp, indent=2) - if self.results_always: # Make accessible in results directory + hash_library_path = self.generate_hash_library_json() + if self.results_always and not is_xdist_worker: # Make accessible in results directory # Use same name as generated result_hash_library = self.results_dir / hash_library_path.name shutil.copy(hash_library_path, result_hash_library) - elif self.results_always and self.results_hash_library_name: + elif self.results_always and self.results_hash_library_name and not is_xdist_worker: result_hashes = {k: v['result_hash'] for k, v in self._test_results.items() if v['result_hash']} if len(result_hashes) > 0: # At least one hash comparison test @@ -964,6 +1019,8 @@ def pytest_unconfigure(self, config): if 'json' in self.generate_summary: summary = self.generate_summary_json() print(f"A JSON report can be found at: {summary}") + if is_xdist_worker: + return if result_hash_library.exists(): # link to it in the HTML kwargs["hash_library"] = result_hash_library.name if 'html' in self.generate_summary: diff --git a/tests/subtests/helpers.py b/tests/subtests/helpers.py index 87a2f5b6..b48e2e03 100644 --- a/tests/subtests/helpers.py +++ b/tests/subtests/helpers.py @@ -1,4 +1,3 @@ -import os import re import json from pathlib import Path @@ -8,6 +7,8 @@ __all__ = ['diff_summary', 'assert_existence', 'patch_summary', 'apply_regex', 'remove_specific_hashes', 'transform_hashes', 'transform_images'] +MIN_EXPECTED_ITEMS = 20 # Rough minimum number of items in a summary to be valid + class MatchError(Exception): pass @@ -39,15 +40,26 @@ def diff_summary(baseline, result, baseline_hash_library=None, result_hash_libra # Load "correct" baseline hashes with open(baseline_hash_library, 'r') as f: baseline_hash_library = json.load(f) + if len(baseline_hash_library.keys()) < MIN_EXPECTED_ITEMS: + raise ValueError(f"baseline_hash_library only has {len(baseline_hash_library.keys())} items") else: baseline_hash_library = {} if result_hash_library and result_hash_library.exists(): # Load "correct" result hashes with open(result_hash_library, 'r') as f: result_hash_library = json.load(f) + if len(result_hash_library.keys()) < MIN_EXPECTED_ITEMS: + raise ValueError(f"result_hash_library only has {len(result_hash_library.keys())} items") else: result_hash_library = {} + b = baseline.get("a", baseline) + if len(b.keys()) < MIN_EXPECTED_ITEMS: + raise ValueError(f"baseline only has {len(b.keys())} items {b}") + r = result.get("a", result) + if len(r.keys()) < MIN_EXPECTED_ITEMS: + raise ValueError(f"result only has {len(r.keys())} items {r}") + # Get test names baseline_tests = set(baseline.keys()) result_tests = set(result.keys()) diff --git a/tests/subtests/test_subtest.py b/tests/subtests/test_subtest.py index 73f7c52e..23dd4e14 100644 --- a/tests/subtests/test_subtest.py +++ b/tests/subtests/test_subtest.py @@ -48,7 +48,7 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=True, - has_result_hashes=False, generating_hashes=False, testing_hashes=False, + has_result_hashes=False, generating_hashes=False, testing_hashes=False, n_xdist_workers=None, update_baseline=UPDATE_BASELINE, update_summary=UPDATE_SUMMARY): """ Run pytest (within pytest) and check JSON summary report. @@ -72,6 +72,9 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=Tru both of `--mpl-hash-library` and `hash_library=` were not. testing_hashes : bool, optional, default=False Whether the subtest is comparing hashes and therefore needs baseline hashes generated. + n_xdist_workers : str or int, optional, default=None + Number of xdist workers to use, or "auto" to use all available cores. + None will disable xdist. If pytest-xdist is not installed, this will be ignored. """ if update_baseline and update_summary: raise ValueError("Cannot enable both `update_baseline` and `update_summary`.") @@ -109,6 +112,15 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=Tru shutil.copy(expected_result_hash_library, baseline_hash_library) transform_hashes(baseline_hash_library) + try: + import xdist + if n_xdist_workers is None: + pytest_args += ["-p", "no:xdist"] + else: + pytest_args += ["-n", str(n_xdist_workers)] + except ImportError: + pass + # Run the test and record exit status status = subprocess.call(pytest_args + mpl_args + args) @@ -206,6 +218,21 @@ def test_html(tmp_path): assert (tmp_path / 'results' / 'styles.css').exists() +@pytest.mark.parametrize("num_workers", [None, 0, 1, 2]) +def test_html_xdist(request, tmp_path, num_workers): + if not request.config.pluginmanager.hasplugin("xdist"): + pytest.skip("Skipping: pytest-xdist is not installed") + run_subtest('test_results_always', tmp_path, + [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS], summaries=['html'], + has_result_hashes=True, n_xdist_workers=num_workers) + assert (tmp_path / 'results' / 'fig_comparison.html').exists() + assert (tmp_path / 'results' / 'extra.js').exists() + assert (tmp_path / 'results' / 'styles.css').exists() + if num_workers is not None: + assert len(list((tmp_path / 'results').glob('generated-hashes-xdist-*-*.json'))) == 0 + assert len(list((tmp_path / 'results').glob('results-xdist-*-*.json'))) == num_workers + + def test_html_hashes_only(tmp_path): run_subtest('test_html_hashes_only', tmp_path, [HASH_LIBRARY_FLAG, *HASH_COMPARISON_MODE], @@ -260,6 +287,24 @@ def test_html_generate(tmp_path): assert (tmp_path / 'results' / 'fig_comparison.html').exists() +@pytest.mark.parametrize("num_workers", [None, 0, 1, 2]) +def test_html_generate_xdist(request, tmp_path, num_workers): + # generating hashes and images; no testing + if not request.config.pluginmanager.hasplugin("xdist"): + pytest.skip("Skipping: pytest-xdist is not installed") + run_subtest('test_html_generate', tmp_path, + [rf'--mpl-generate-path={tmp_path}', + rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], + summaries=['html'], xfail=False, has_result_hashes="test_hashes.json", + generating_hashes=True, n_xdist_workers=num_workers) + assert (tmp_path / 'results' / 'fig_comparison.html').exists() + assert (tmp_path / 'results' / 'extra.js').exists() + assert (tmp_path / 'results' / 'styles.css').exists() + if num_workers is not None: + assert len(list((tmp_path / 'results').glob('generated-hashes-xdist-*-*.json'))) == num_workers + assert len(list((tmp_path / 'results').glob('results-xdist-*-*.json'))) == num_workers + + def test_html_generate_images_only(tmp_path): # generating images; no testing run_subtest('test_html_generate_images_only', tmp_path, diff --git a/tox.ini b/tox.ini index cff73251..eb6ccfe5 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,7 @@ setenv = changedir = .tmp/{envname} description = run tests deps = + pytest-xdist mpl20: matplotlib==2.0.* mpl21: matplotlib==2.1.* mpl22: matplotlib==2.2.* @@ -58,7 +59,7 @@ commands = # Make sure the tests pass with and without --mpl # Use -m so pytest skips "subtests" which always apply --mpl pytest '{toxinidir}' -m "mpl_image_compare" {posargs} - coverage run --source=pytest_mpl -m pytest '{toxinidir}' --mpl + coverage run --source=pytest_mpl -m pytest '{toxinidir}' -n auto --mpl coverage xml -o '{toxinidir}{/}coverage.xml' [testenv:codestyle] From b66e2a991de3991097b4516b53975008e895b3e1 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Wed, 30 Jul 2025 00:38:40 +0100 Subject: [PATCH 2/6] run pytester subprocess when matplotlib built with pybind11 v3 --- tests/conftest.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 9c3572d9..39829226 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,3 +7,12 @@ @pytest.fixture def pytester(testdir): return testdir + + +def pytest_configure(config): + # Matplotlib versions build with pybind11 3.0.0 or later + # encounter import issues unless run in subprocess mode. + # See: https://github.com/matplotlib/pytest-mpl/issues/248 + import matplotlib + if Version(matplotlib.__version__) > Version("3.10.3"): + config.option.runpytest = "subprocess" From 79dbd43d9394f1d63bb0e515c8dcc5a616e064f4 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Thu, 16 Oct 2025 01:13:31 +0100 Subject: [PATCH 3/6] Revert "run pytester subprocess when matplotlib built with pybind11 v3" This reverts commit b66e2a991de3991097b4516b53975008e895b3e1. --- tests/conftest.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 39829226..9c3572d9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,12 +7,3 @@ @pytest.fixture def pytester(testdir): return testdir - - -def pytest_configure(config): - # Matplotlib versions build with pybind11 3.0.0 or later - # encounter import issues unless run in subprocess mode. - # See: https://github.com/matplotlib/pytest-mpl/issues/248 - import matplotlib - if Version(matplotlib.__version__) > Version("3.10.3"): - config.option.runpytest = "subprocess" From 337005c4438c67701ee1824cd6a79a20f110b9cf Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Fri, 17 Oct 2025 19:24:12 +0100 Subject: [PATCH 4/6] Verify HTML outputs are not empty --- tests/subtests/test_subtest.py | 50 +++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/tests/subtests/test_subtest.py b/tests/subtests/test_subtest.py index 23dd4e14..d2a568d4 100644 --- a/tests/subtests/test_subtest.py +++ b/tests/subtests/test_subtest.py @@ -213,7 +213,10 @@ def test_html(tmp_path): run_subtest('test_results_always', tmp_path, [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS], summaries=['html'], has_result_hashes=True) - assert (tmp_path / 'results' / 'fig_comparison.html').exists() + html_path = tmp_path / 'results' / 'fig_comparison.html' + assert html_path.exists() + assert html_path.stat().st_size > 200_000 + assert "Baseline image differs" in html_path.read_text() assert (tmp_path / 'results' / 'extra.js').exists() assert (tmp_path / 'results' / 'styles.css').exists() @@ -225,7 +228,10 @@ def test_html_xdist(request, tmp_path, num_workers): run_subtest('test_results_always', tmp_path, [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS], summaries=['html'], has_result_hashes=True, n_xdist_workers=num_workers) - assert (tmp_path / 'results' / 'fig_comparison.html').exists() + html_path = tmp_path / 'results' / 'fig_comparison.html' + assert html_path.exists() + assert html_path.stat().st_size > 200_000 + assert "Baseline image differs" in html_path.read_text() assert (tmp_path / 'results' / 'extra.js').exists() assert (tmp_path / 'results' / 'styles.css').exists() if num_workers is not None: @@ -237,14 +243,20 @@ def test_html_hashes_only(tmp_path): run_subtest('test_html_hashes_only', tmp_path, [HASH_LIBRARY_FLAG, *HASH_COMPARISON_MODE], summaries=['html'], has_result_hashes=True) - assert (tmp_path / 'results' / 'fig_comparison.html').exists() + html_path = tmp_path / 'results' / 'fig_comparison.html' + assert html_path.exists() + assert html_path.stat().st_size > 100_000 + assert "Baseline hash differs" in html_path.read_text() assert (tmp_path / 'results' / 'extra.js').exists() assert (tmp_path / 'results' / 'styles.css').exists() def test_html_images_only(tmp_path): run_subtest('test_html_images_only', tmp_path, [*IMAGE_COMPARISON_MODE], summaries=['html']) - assert (tmp_path / 'results' / 'fig_comparison.html').exists() + html_path = tmp_path / 'results' / 'fig_comparison.html' + assert html_path.exists() + assert html_path.stat().st_size > 200_000 + assert "Baseline image differs" in html_path.read_text() assert (tmp_path / 'results' / 'extra.js').exists() assert (tmp_path / 'results' / 'styles.css').exists() @@ -253,7 +265,10 @@ def test_basic_html(tmp_path): run_subtest('test_results_always', tmp_path, [HASH_LIBRARY_FLAG, *BASELINE_IMAGES_FLAG_REL], summaries=['basic-html'], has_result_hashes=True) - assert (tmp_path / 'results' / 'fig_comparison_basic.html').exists() + html_path = tmp_path / 'results' / 'fig_comparison_basic.html' + assert html_path.exists() + assert html_path.stat().st_size > 20_000 + assert "hash comparison, although" in html_path.read_text() def test_generate(tmp_path): @@ -284,7 +299,10 @@ def test_html_generate(tmp_path): rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], summaries=['html'], xfail=False, has_result_hashes="test_hashes.json", generating_hashes=True) - assert (tmp_path / 'results' / 'fig_comparison.html').exists() + html_path = tmp_path / 'results' / 'fig_comparison.html' + assert html_path.exists() + assert html_path.stat().st_size > 100_000 + assert "Baseline image was generated" in html_path.read_text() @pytest.mark.parametrize("num_workers", [None, 0, 1, 2]) @@ -297,7 +315,10 @@ def test_html_generate_xdist(request, tmp_path, num_workers): rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], summaries=['html'], xfail=False, has_result_hashes="test_hashes.json", generating_hashes=True, n_xdist_workers=num_workers) - assert (tmp_path / 'results' / 'fig_comparison.html').exists() + html_path = tmp_path / 'results' / 'fig_comparison.html' + assert html_path.exists() + assert html_path.stat().st_size > 100_000 + assert "Baseline image was generated" in html_path.read_text() assert (tmp_path / 'results' / 'extra.js').exists() assert (tmp_path / 'results' / 'styles.css').exists() if num_workers is not None: @@ -310,7 +331,10 @@ def test_html_generate_images_only(tmp_path): run_subtest('test_html_generate_images_only', tmp_path, [rf'--mpl-generate-path={tmp_path}', *IMAGE_COMPARISON_MODE], summaries=['html'], xfail=False) - assert (tmp_path / 'results' / 'fig_comparison.html').exists() + html_path = tmp_path / 'results' / 'fig_comparison.html' + assert html_path.exists() + assert html_path.stat().st_size > 100_000 + assert "Baseline image was generated" in html_path.read_text() def test_html_generate_hashes_only(tmp_path): @@ -318,7 +342,10 @@ def test_html_generate_hashes_only(tmp_path): run_subtest('test_html_generate_hashes_only', tmp_path, [rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], summaries=['html'], has_result_hashes="test_hashes.json", generating_hashes=True) - assert (tmp_path / 'results' / 'fig_comparison.html').exists() + html_path = tmp_path / 'results' / 'fig_comparison.html' + assert html_path.exists() + assert html_path.stat().st_size > 200_000 + assert "Baseline hash was generated" in html_path.read_text() def test_html_run_generate_hashes_only(tmp_path): @@ -327,7 +354,10 @@ def test_html_run_generate_hashes_only(tmp_path): [rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}', HASH_LIBRARY_FLAG, *HASH_COMPARISON_MODE], summaries=['html'], has_result_hashes="test_hashes.json") - assert (tmp_path / 'results' / 'fig_comparison.html').exists() + html_path = tmp_path / 'results' / 'fig_comparison.html' + assert html_path.exists() + assert html_path.stat().st_size > 100_000 + assert "Baseline hash differs" in html_path.read_text() # Run a hybrid mode test last so if generating hash libraries, it includes all the hashes. From 8659069eb9d8238df90bd74eff6e381ad2aae193 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Sat, 18 Oct 2025 21:15:16 +0100 Subject: [PATCH 5/6] Simplify xdist detection --- pytest_mpl/plugin.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 969d745a..00c1cc0a 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -977,19 +977,12 @@ def pytest_sessionfinish(self, session): Save out the hash library at the end of the run. """ config = session.config - try: - import xdist - is_xdist_controller = xdist.is_xdist_controller(session) - is_xdist_worker = xdist.is_xdist_worker(session) - except ImportError: - is_xdist_controller = False - is_xdist_worker = False - except Exception as e: - if "xdist" not in session.config.option: - is_xdist_controller = False - is_xdist_worker = False - else: - raise e + is_xdist_worker = hasattr(config, "workerinput") + is_xdist_controller = ( + config.pluginmanager.hasplugin("xdist") + and not is_xdist_worker + and getattr(config.option, "dist", "") != "no" + ) if is_xdist_controller: # Merge results from workers uid = config.pytest_mpl_uid From d9ded2d3f96a313536e1a32f05a25ab51d514e60 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Sat, 18 Oct 2025 21:52:28 +0100 Subject: [PATCH 6/6] Always generate JSON summaries for xdist workers --- pytest_mpl/plugin.py | 5 +++-- tests/subtests/test_subtest.py | 36 ++++++++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 00c1cc0a..4367dd89 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -1008,12 +1008,13 @@ def pytest_sessionfinish(self, session): json.dump(result_hashes, fp, indent=2) if self.generate_summary: + if is_xdist_worker: + self.generate_summary_json() + return kwargs = {} if 'json' in self.generate_summary: summary = self.generate_summary_json() print(f"A JSON report can be found at: {summary}") - if is_xdist_worker: - return if result_hash_library.exists(): # link to it in the HTML kwargs["hash_library"] = result_hash_library.name if 'html' in self.generate_summary: diff --git a/tests/subtests/test_subtest.py b/tests/subtests/test_subtest.py index d2a568d4..a8deb286 100644 --- a/tests/subtests/test_subtest.py +++ b/tests/subtests/test_subtest.py @@ -47,6 +47,17 @@ ] +def xdist_args(n_workers): + try: + import xdist + if n_workers is None: + return ["-p", "no:xdist"] + else: + return ["-n", str(n_workers)] + except ImportError: + return [] + + def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=True, has_result_hashes=False, generating_hashes=False, testing_hashes=False, n_xdist_workers=None, update_baseline=UPDATE_BASELINE, update_summary=UPDATE_SUMMARY): @@ -112,14 +123,7 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=Tru shutil.copy(expected_result_hash_library, baseline_hash_library) transform_hashes(baseline_hash_library) - try: - import xdist - if n_xdist_workers is None: - pytest_args += ["-p", "no:xdist"] - else: - pytest_args += ["-n", str(n_xdist_workers)] - except ImportError: - pass + pytest_args.extend(xdist_args(n_xdist_workers)) # Run the test and record exit status status = subprocess.call(pytest_args + mpl_args + args) @@ -363,3 +367,19 @@ def test_html_run_generate_hashes_only(tmp_path): # Run a hybrid mode test last so if generating hash libraries, it includes all the hashes. def test_hybrid(tmp_path): run_subtest('test_hybrid', tmp_path, [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS], testing_hashes=True) + + +@pytest.mark.parametrize("num_workers", [None, 0, 1, 2]) +def test_html_no_json(tmp_path, num_workers): + # Previous tests require JSON summary to be generated to function correctly. + # This test ensures HTML summary generation works without JSON summary. + results_path = tmp_path / 'results' + results_path.mkdir() + mpl_args = ['--mpl', rf'--mpl-results-path={results_path.as_posix()}', + '--mpl-generate-summary=html', *xdist_args(num_workers)] + subprocess.call([sys.executable, '-m', 'pytest', str(TEST_FILE), *mpl_args]) + assert not (tmp_path / 'results' / 'results.json').exists() + html_path = tmp_path / 'results' / 'fig_comparison.html' + assert html_path.exists() + assert html_path.stat().st_size > 200_000 + assert "Baseline image differs" in html_path.read_text()