From e567ea71d293dc227cdcb7dc7da706b96596081c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 7 Nov 2026 19:26:21 +0000 Subject: [PATCH 1/2] Hold module-level skips and missing modules in the Spec test gate A test_spec_*.py module skipped while it is collected (module-level skip or importorskip) is now a failed collection, and a run over the whole tests folder fails when no test was collected from a module of EXPECTED_MODULES (a deleted or emptied Spec module). Fixes #203 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Scma6W3LC11s3qjXHZuGDm --- CHANGELOG.md | 5 +++ docs/development.md | 5 ++- tests/conftest.py | 4 +- tests/spec_gate.py | 78 ++++++++++++++++++++++++++++++++++++- tests/spec_gate_plugin.py | 13 ++++++- tests/test_spec_gate.py | 81 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 180 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a56a3404..0bbb0f3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ are in [docs/plan.md](docs/plan.md) ยง12. Upgrading from v0.3: see the ## Unreleased +- **The Spec test gate holds module-level skips and missing modules (#203).** A `test_spec_*.py` + module skipped while it is collected (`pytest.skip(..., allow_module_level=True)`, a module-level + `importorskip`) is now a failed collection, and a run over the whole tests folder fails when no + test was collected from a module of `EXPECTED_MODULES` in `tests/spec_gate.py` (deleted or + emptied module). Regression tests in `tests/test_spec_gate.py`. - **Re-pin the vendored Spec to 7dbae0c (MorphoBricks-Spec PR 9).** New in the pin: the byte rules and golden files of the derived `experiment_design.csv` (section 11; `fixtures/expected/` for `v13`, `v33`, `v34`, not yet derived by any code: Stage 1 PR 10), the byte form of one diff --git a/docs/development.md b/docs/development.md index 81c0ea12..6f318265 100644 --- a/docs/development.md +++ b/docs/development.md @@ -113,7 +113,10 @@ commit SHA in its `PIN.toml`, never a branch. of a fixture cannot make a reader pass. Never edit the vendored files; updating the pin is a reviewed change (`tests/spec_fixtures/README.md`). - **Skip is failure:** every test module named `tests/test_spec_*.py` is held by - `tests/spec_gate.py`: a skipped or xfailed test there fails the run. + `tests/spec_gate.py`: a skipped or xfailed test there fails the run, and so does a module + skipped while it is collected (module-level `skip` or `importorskip`). A run over the whole + tests folder also fails when no test was collected from a module of `EXPECTED_MODULES` in + that file (a deleted or emptied Spec module); a new Spec module is added to that list. - **No second copy of the rules:** `tests/test_spec_no_fallback.py` scans the reader's modules (`description*.py`) for a second table of codes or names and for lenient, older or default-version paths. diff --git a/tests/conftest.py b/tests/conftest.py index 82b7800a..5f9358f8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,9 +4,9 @@ # Lets tests import the shared simulation helpers. sys.path.insert(0, str(Path(__file__).parent)) -from spec_gate import SkipIsFailure # noqa: E402 +from spec_gate import EXPECTED_MODULES, SkipIsFailure # noqa: E402 def pytest_configure(config): # A skipped Spec conformance test is a failure (conformance plan section 1). - config.pluginmanager.register(SkipIsFailure(), "spec-skip-is-failure") + config.pluginmanager.register(SkipIsFailure(EXPECTED_MODULES), "spec-skip-is-failure") diff --git a/tests/spec_gate.py b/tests/spec_gate.py index c83a2549..a98e7972 100644 --- a/tests/spec_gate.py +++ b/tests/spec_gate.py @@ -4,18 +4,61 @@ test is skipped (``pytest.skip``, ``skipif``, ``importorskip``, a skipping fixture) or expected to fail (``xfail``), the report is turned into a failure, so a missing package, file or weight cannot make the release gate pass silently. + +Two cases never reach a test item, so two more checks hold them (issue #203): + +- a ``test_spec_*.py`` module skipped while it is collected (``pytest.skip(..., + allow_module_level=True)``, a module-level ``importorskip``) is reported as a failed + collection (:meth:`SkipIsFailure.pytest_collectreport`); +- a deleted or emptied module: a run over the whole tests folder must have collected at + least one item from every module of :data:`EXPECTED_MODULES`, else it is refused + (:meth:`SkipIsFailure.pytest_collection_finish`). Adding a Spec module means listing it + there; removing one is a reviewed change to this list. """ from __future__ import annotations +from collections import Counter +from collections.abc import Collection +from pathlib import Path + import pytest MARKER = "spec_conformance" MODULE_PREFIX = "test_spec_" +EXPECTED_MODULES = ( + "test_spec_catalogue.py", + "test_spec_discovery.py", + "test_spec_encoding.py", + "test_spec_gate.py", + "test_spec_no_fallback.py", + "test_spec_pin.py", + "test_spec_publication.py", + "test_spec_reader.py", + "test_spec_reader_phases_1_3.py", + "test_spec_reconcile.py", + "test_spec_refval_crosscheck.py", + "test_spec_rules_table.py", + "test_spec_transitions.py", + "test_spec_vendored_missing.py", +) class SkipIsFailure: - """A pytest plugin: marks the Spec tests and fails the ones that did not run.""" + """A pytest plugin: marks the Spec tests and fails the ones that did not run. + + Parameters + ---------- + expected + Names of the Spec modules a run over ``root`` must collect items from. + root + The folder holding them (default: this file's folder, the tests folder). + """ + + def __init__(self, expected: Collection[str] = (), root: Path | None = None) -> None: + self.expected = tuple(expected) + self.root = (root or Path(__file__).resolve().parent).resolve() + self.collected: Counter[str] = Counter() def pytest_configure(self, config: pytest.Config) -> None: """Register the marker.""" @@ -27,6 +70,39 @@ def pytest_collection_modifyitems(self, items: list[pytest.Item]) -> None: if item.path.name.startswith(MODULE_PREFIX): item.add_marker(MARKER) + def pytest_itemcollected(self, item: pytest.Item) -> None: + """Count the items collected from each ``test_spec_*.py`` module (before any -k).""" + if item.path.name.startswith(MODULE_PREFIX): + self.collected[item.path.name] += 1 + + @pytest.hookimpl(tryfirst=True) + def pytest_collectreport(self, report: pytest.CollectReport) -> None: + """Fail a ``test_spec_*.py`` module that was skipped while it was collected.""" + name = Path(report.nodeid).name + if not report.skipped or "::" in report.nodeid or not name.startswith(MODULE_PREFIX): + return + reason = report.longrepr[2] if isinstance(report.longrepr, tuple) else str(report.longrepr) + report.outcome = "failed" + report.longrepr = f"a Spec conformance module must be collected, it was skipped: {reason}" + + def pytest_collection_finish(self, session: pytest.Session) -> None: + """Refuse a whole-folder run that collected no item from an expected Spec module.""" + config = session.config + folders = [Path(arg) for arg in config.args] + # A run of chosen files, tests or an unrelated folder checks only what it selects. + if not self.expected or not all( + f.is_dir() and (f.resolve() == self.root or f.resolve() in self.root.parents) + for f in folders + ): + return + missing = [name for name in self.expected if not self.collected[name]] + if missing: + raise pytest.UsageError( + "no test was collected from the Spec conformance module(s) " + + ", ".join(missing) + + ": deleted, emptied or skipped while collected (tests/spec_gate.py)" + ) + @pytest.hookimpl(hookwrapper=True) def pytest_runtest_makereport(self, item: pytest.Item, call: pytest.CallInfo): """Turn a skipped or xfailed report of a marked test into a failure.""" diff --git a/tests/spec_gate_plugin.py b/tests/spec_gate_plugin.py index 017f4bcf..b4a53443 100644 --- a/tests/spec_gate_plugin.py +++ b/tests/spec_gate_plugin.py @@ -1,9 +1,18 @@ -"""Registers ``SkipIsFailure`` in a pytest run started with ``-p spec_gate_plugin``.""" +"""Registers ``SkipIsFailure`` in a pytest run started with ``-p spec_gate_plugin``. + +``SPEC_GATE_EXPECTED`` (comma-separated module names) sets the modules a run over the current +folder must collect (default: none), for the gate's own tests. +""" from __future__ import annotations +import os +from pathlib import Path + from spec_gate import SkipIsFailure def pytest_configure(config): - config.pluginmanager.register(SkipIsFailure(), "spec-skip-is-failure") + expected = [name for name in os.environ.get("SPEC_GATE_EXPECTED", "").split(",") if name] + plugin = SkipIsFailure(expected, Path.cwd()) + config.pluginmanager.register(plugin, "spec-skip-is-failure") diff --git a/tests/test_spec_gate.py b/tests/test_spec_gate.py index d6331a66..5747a117 100644 --- a/tests/test_spec_gate.py +++ b/tests/test_spec_gate.py @@ -60,3 +60,84 @@ def test_other_tests_may_still_skip(tmp_path): assert "FAILED" not in out and "ERROR" not in out assert "1 passed" in out and "3 skipped" in out and "1 xfailed" in out assert code == 0 + + +def _gated(folder: Path, expected: str = "") -> tuple[int, str]: + result = subprocess.run( + [sys.executable, "-m", "pytest", "-p", "no:cacheprovider", "-p", "spec_gate_plugin", "-q"], + cwd=folder, + capture_output=True, + text=True, + env={**os.environ, "PYTHONPATH": str(TESTS), "SPEC_GATE_EXPECTED": expected}, + check=False, + ) + return result.returncode, result.stdout + result.stderr + + +PASSING = "def test_passes():\n pass\n" + + +def test_a_module_level_skip_in_a_spec_module_fails_the_run(tmp_path): + (tmp_path / "test_spec_demo.py").write_text( + 'import pytest\npytest.skip("no weights", allow_module_level=True)\n' + PASSING, + encoding="utf-8", + ) + code, out = _gated(tmp_path) + assert code != 0 + assert "a Spec conformance module must be collected, it was skipped: Skipped: no weights" in out + + +def test_a_module_level_importorskip_in_a_spec_module_fails_the_run(tmp_path): + (tmp_path / "test_spec_demo.py").write_text( + 'import pytest\npytest.importorskip("a_module_that_is_not_installed")\n' + PASSING, + encoding="utf-8", + ) + code, out = _gated(tmp_path) + assert code != 0 and "a Spec conformance module must be collected" in out + + +def test_a_module_level_skip_in_another_module_may_still_skip(tmp_path): + (tmp_path / "test_other.py").write_text( + 'import pytest\npytest.importorskip("a_module_that_is_not_installed")\n' + PASSING, + encoding="utf-8", + ) + (tmp_path / "test_spec_demo.py").write_text(PASSING, encoding="utf-8") + code, out = _gated(tmp_path) + assert code == 0 and "1 passed" in out and "1 skipped" in out + + +def test_a_missing_spec_module_fails_a_whole_folder_run(tmp_path): + (tmp_path / "test_spec_here.py").write_text(PASSING, encoding="utf-8") + code, out = _gated(tmp_path, "test_spec_here.py,test_spec_gone.py") + assert code != 0 + assert "test_spec_gone.py" in out and "test_spec_here.py:" not in out + (tmp_path / "test_spec_gone.py").write_text(PASSING, encoding="utf-8") + assert _gated(tmp_path, "test_spec_here.py,test_spec_gone.py")[0] == 0 + + +def test_a_spec_module_without_a_collected_test_fails_a_whole_folder_run(tmp_path): + (tmp_path / "test_spec_empty.py").write_text("X = 1\n", encoding="utf-8") + code, out = _gated(tmp_path, "test_spec_empty.py") + assert code != 0 and "test_spec_empty.py" in out + + +def test_a_run_of_one_chosen_file_does_not_need_the_other_spec_modules(tmp_path): + (tmp_path / "test_spec_here.py").write_text(PASSING, encoding="utf-8") + env = {**os.environ, "PYTHONPATH": str(TESTS), "SPEC_GATE_EXPECTED": "test_spec_gone.py"} + command = [sys.executable, "-m", "pytest", "-p", "no:cacheprovider", "-p", "spec_gate_plugin"] + result = subprocess.run( + [*command, "-q", "test_spec_here.py"], + cwd=tmp_path, + capture_output=True, + text=True, + env=env, + check=False, + ) + assert result.returncode == 0 and "1 passed" in result.stdout + + +def test_every_spec_module_of_the_repository_is_listed(): + from spec_gate import EXPECTED_MODULES, MODULE_PREFIX + + found = sorted(p.name for p in TESTS.glob(f"{MODULE_PREFIX}*.py")) + assert found == sorted(EXPECTED_MODULES) From fce7fc8d36221ce2fcc2ead287cd8fe401463f54 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 7 Nov 2026 20:38:28 +0000 Subject: [PATCH 2/2] List test_spec_derived_design.py in the Spec gate #196 added the module on main; the gate refuses an unlisted Spec module, as designed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Scma6W3LC11s3qjXHZuGDm --- tests/spec_gate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/spec_gate.py b/tests/spec_gate.py index a98e7972..1a85b2b9 100644 --- a/tests/spec_gate.py +++ b/tests/spec_gate.py @@ -28,6 +28,7 @@ MODULE_PREFIX = "test_spec_" EXPECTED_MODULES = ( "test_spec_catalogue.py", + "test_spec_derived_design.py", "test_spec_discovery.py", "test_spec_encoding.py", "test_spec_gate.py",