Hold module-level skips and missing modules in the Spec test gate

CHANGELOG.md

@@ -8,6 +8,11 @@ are in [docs/plan.md](docs/plan.md) §12. Upgrading from v0.3: see the
 
 ## Unreleased
 
+- **The Spec test gate holds module-level skips and missing modules (#203).** A `test_spec_*.py`
+  module skipped while it is collected (`pytest.skip(..., allow_module_level=True)`, a module-level
+  `importorskip`) is now a failed collection, and a run over the whole tests folder fails when no
+  test was collected from a module of `EXPECTED_MODULES` in `tests/spec_gate.py` (deleted or
+  emptied module). Regression tests in `tests/test_spec_gate.py`.
 - **A mask changed after the last check is no longer published (#198).** Every run, also one
   with `verify_content` off, fingerprints each mask again in the commit's `before_commit`
   callback, right before the manifest is replaced; a changed mask stops the run with

docs/development.md

@@ -113,7 +113,10 @@ commit SHA in its `PIN.toml`, never a branch.
   of a fixture cannot make a reader pass. Never edit the vendored files; updating the pin
   is a reviewed change (`tests/spec_fixtures/README.md`).
 - **Skip is failure:** every test module named `tests/test_spec_*.py` is held by
-  `tests/spec_gate.py`: a skipped or xfailed test there fails the run.
+  `tests/spec_gate.py`: a skipped or xfailed test there fails the run, and so does a module
+  skipped while it is collected (module-level `skip` or `importorskip`). A run over the whole
+  tests folder also fails when no test was collected from a module of `EXPECTED_MODULES` in
+  that file (a deleted or emptied Spec module); a new Spec module is added to that list.
 - **No second copy of the rules:** `tests/test_spec_no_fallback.py` scans the reader's
   modules (`description*.py`) for a second table of codes or names and for lenient, older or
   default-version paths.

tests/conftest.py

@@ -4,9 +4,9 @@
 # Lets tests import the shared simulation helpers.
 sys.path.insert(0, str(Path(__file__).parent))
 
-from spec_gate import SkipIsFailure  # noqa: E402
+from spec_gate import EXPECTED_MODULES, SkipIsFailure  # noqa: E402
 
 
 def pytest_configure(config):
     # A skipped Spec conformance test is a failure (conformance plan section 1).
-    config.pluginmanager.register(SkipIsFailure(), "spec-skip-is-failure")
+    config.pluginmanager.register(SkipIsFailure(EXPECTED_MODULES), "spec-skip-is-failure")

tests/spec_gate.py

@@ -4,18 +4,62 @@
 test is skipped (``pytest.skip``, ``skipif``, ``importorskip``, a skipping fixture) or
 expected to fail (``xfail``), the report is turned into a failure, so a missing package,
 file or weight cannot make the release gate pass silently.
+
+Two cases never reach a test item, so two more checks hold them (issue #203):
+
+- a ``test_spec_*.py`` module skipped while it is collected (``pytest.skip(...,
+  allow_module_level=True)``, a module-level ``importorskip``) is reported as a failed
+  collection (:meth:`SkipIsFailure.pytest_collectreport`);
+- a deleted or emptied module: a run over the whole tests folder must have collected at
+  least one item from every module of :data:`EXPECTED_MODULES`, else it is refused
+  (:meth:`SkipIsFailure.pytest_collection_finish`). Adding a Spec module means listing it
+  there; removing one is a reviewed change to this list.
 """
 
 from __future__ import annotations
 
+from collections import Counter
+from collections.abc import Collection
+from pathlib import Path
+
 import pytest
 
 MARKER = "spec_conformance"
 MODULE_PREFIX = "test_spec_"
+EXPECTED_MODULES = (
+    "test_spec_catalogue.py",
+    "test_spec_derived_design.py",
+    "test_spec_discovery.py",
+    "test_spec_encoding.py",
+    "test_spec_gate.py",
+    "test_spec_no_fallback.py",
+    "test_spec_pin.py",
+    "test_spec_publication.py",
+    "test_spec_reader.py",
+    "test_spec_reader_phases_1_3.py",
+    "test_spec_reconcile.py",
+    "test_spec_refval_crosscheck.py",
+    "test_spec_rules_table.py",
+    "test_spec_transitions.py",
+    "test_spec_vendored_missing.py",
+)
 
 
 class SkipIsFailure:
-    """A pytest plugin: marks the Spec tests and fails the ones that did not run."""
+    """A pytest plugin: marks the Spec tests and fails the ones that did not run.
+
+    Parameters
+    ----------
+    expected
+        Names of the Spec modules a run over ``root`` must collect items from.
+    root
+        The folder holding them (default: this file's folder, the tests folder).
+    """
+
+    def __init__(self, expected: Collection[str] = (), root: Path | None = None) -> None:
+        self.expected = tuple(expected)
+        self.root = (root or Path(__file__).resolve().parent).resolve()
+        self.collected: Counter[str] = Counter()
 
     def pytest_configure(self, config: pytest.Config) -> None:
         """Register the marker."""
@@ -27,6 +71,39 @@ def pytest_collection_modifyitems(self, items: list[pytest.Item]) -> None:
             if item.path.name.startswith(MODULE_PREFIX):
                 item.add_marker(MARKER)
 
+    def pytest_itemcollected(self, item: pytest.Item) -> None:
+        """Count the items collected from each ``test_spec_*.py`` module (before any -k)."""
+        if item.path.name.startswith(MODULE_PREFIX):
+            self.collected[item.path.name] += 1
+
+    @pytest.hookimpl(tryfirst=True)
+    def pytest_collectreport(self, report: pytest.CollectReport) -> None:
+        """Fail a ``test_spec_*.py`` module that was skipped while it was collected."""
+        name = Path(report.nodeid).name
+        if not report.skipped or "::" in report.nodeid or not name.startswith(MODULE_PREFIX):
+            return
+        reason = report.longrepr[2] if isinstance(report.longrepr, tuple) else str(report.longrepr)
+        report.outcome = "failed"
+        report.longrepr = f"a Spec conformance module must be collected, it was skipped: {reason}"
+
+    def pytest_collection_finish(self, session: pytest.Session) -> None:
+        """Refuse a whole-folder run that collected no item from an expected Spec module."""
+        config = session.config
+        folders = [Path(arg) for arg in config.args]
+        # A run of chosen files, tests or an unrelated folder checks only what it selects.
+        if not self.expected or not all(
+            f.is_dir() and (f.resolve() == self.root or f.resolve() in self.root.parents)
+            for f in folders
+        ):
+            return
+        missing = [name for name in self.expected if not self.collected[name]]
+        if missing:
+            raise pytest.UsageError(
+                "no test was collected from the Spec conformance module(s) "
+                + ", ".join(missing)
+                + ": deleted, emptied or skipped while collected (tests/spec_gate.py)"
+            )
+
     @pytest.hookimpl(hookwrapper=True)
     def pytest_runtest_makereport(self, item: pytest.Item, call: pytest.CallInfo):
         """Turn a skipped or xfailed report of a marked test into a failure."""

tests/spec_gate_plugin.py

@@ -1,9 +1,18 @@
-"""Registers ``SkipIsFailure`` in a pytest run started with ``-p spec_gate_plugin``."""
+"""Registers ``SkipIsFailure`` in a pytest run started with ``-p spec_gate_plugin``.
+
+``SPEC_GATE_EXPECTED`` (comma-separated module names) sets the modules a run over the current
+folder must collect (default: none), for the gate's own tests.
+"""
 
 from __future__ import annotations
 
+import os
+from pathlib import Path
+
 from spec_gate import SkipIsFailure
 
 
 def pytest_configure(config):
-    config.pluginmanager.register(SkipIsFailure(), "spec-skip-is-failure")
+    expected = [name for name in os.environ.get("SPEC_GATE_EXPECTED", "").split(",") if name]
+    plugin = SkipIsFailure(expected, Path.cwd())
+    config.pluginmanager.register(plugin, "spec-skip-is-failure")

tests/test_spec_gate.py

@@ -60,3 +60,84 @@ def test_other_tests_may_still_skip(tmp_path):
     assert "FAILED" not in out and "ERROR" not in out
     assert "1 passed" in out and "3 skipped" in out and "1 xfailed" in out
     assert code == 0
+
+
+def _gated(folder: Path, expected: str = "") -> tuple[int, str]:
+    result = subprocess.run(
+        [sys.executable, "-m", "pytest", "-p", "no:cacheprovider", "-p", "spec_gate_plugin", "-q"],
+        cwd=folder,
+        capture_output=True,
+        text=True,
+        env={**os.environ, "PYTHONPATH": str(TESTS), "SPEC_GATE_EXPECTED": expected},
+        check=False,
+    )
+    return result.returncode, result.stdout + result.stderr
+
+
+PASSING = "def test_passes():\n    pass\n"
+
+
+def test_a_module_level_skip_in_a_spec_module_fails_the_run(tmp_path):
+    (tmp_path / "test_spec_demo.py").write_text(
+        'import pytest\npytest.skip("no weights", allow_module_level=True)\n' + PASSING,
+        encoding="utf-8",
+    )
+    code, out = _gated(tmp_path)
+    assert code != 0
+    assert "a Spec conformance module must be collected, it was skipped: Skipped: no weights" in out
+
+
+def test_a_module_level_importorskip_in_a_spec_module_fails_the_run(tmp_path):
+    (tmp_path / "test_spec_demo.py").write_text(
+        'import pytest\npytest.importorskip("a_module_that_is_not_installed")\n' + PASSING,
+        encoding="utf-8",
+    )
+    code, out = _gated(tmp_path)
+    assert code != 0 and "a Spec conformance module must be collected" in out
+
+
+def test_a_module_level_skip_in_another_module_may_still_skip(tmp_path):
+    (tmp_path / "test_other.py").write_text(
+        'import pytest\npytest.importorskip("a_module_that_is_not_installed")\n' + PASSING,
+        encoding="utf-8",
+    )
+    (tmp_path / "test_spec_demo.py").write_text(PASSING, encoding="utf-8")
+    code, out = _gated(tmp_path)
+    assert code == 0 and "1 passed" in out and "1 skipped" in out
+
+
+def test_a_missing_spec_module_fails_a_whole_folder_run(tmp_path):
+    (tmp_path / "test_spec_here.py").write_text(PASSING, encoding="utf-8")
+    code, out = _gated(tmp_path, "test_spec_here.py,test_spec_gone.py")
+    assert code != 0
+    assert "test_spec_gone.py" in out and "test_spec_here.py:" not in out
+    (tmp_path / "test_spec_gone.py").write_text(PASSING, encoding="utf-8")
+    assert _gated(tmp_path, "test_spec_here.py,test_spec_gone.py")[0] == 0
+
+
+def test_a_spec_module_without_a_collected_test_fails_a_whole_folder_run(tmp_path):
+    (tmp_path / "test_spec_empty.py").write_text("X = 1\n", encoding="utf-8")
+    code, out = _gated(tmp_path, "test_spec_empty.py")
+    assert code != 0 and "test_spec_empty.py" in out
+
+
+def test_a_run_of_one_chosen_file_does_not_need_the_other_spec_modules(tmp_path):
+    (tmp_path / "test_spec_here.py").write_text(PASSING, encoding="utf-8")
+    env = {**os.environ, "PYTHONPATH": str(TESTS), "SPEC_GATE_EXPECTED": "test_spec_gone.py"}
+    command = [sys.executable, "-m", "pytest", "-p", "no:cacheprovider", "-p", "spec_gate_plugin"]
+    result = subprocess.run(
+        [*command, "-q", "test_spec_here.py"],
+        cwd=tmp_path,
+        capture_output=True,
+        text=True,
+        env=env,
+        check=False,
+    )
+    assert result.returncode == 0 and "1 passed" in result.stdout
+
+
+def test_every_spec_module_of_the_repository_is_listed():
+    from spec_gate import EXPECTED_MODULES, MODULE_PREFIX
+
+    found = sorted(p.name for p in TESTS.glob(f"{MODULE_PREFIX}*.py"))
+    assert found == sorted(EXPECTED_MODULES)