diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e806e29..29aba5e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,7 @@ jobs: python scripts/mine_report.py --selftest python scripts/oracle_compare.py --selftest python scripts/metamorphic.py --selftest + python scripts/metamorphic_facts.py --selftest tests: name: tests (py${{ matrix.python-version }}) diff --git a/docs/notes/metamorphic.md b/docs/notes/metamorphic.md index d7b0ed28..c1cfb591 100644 --- a/docs/notes/metamorphic.md +++ b/docs/notes/metamorphic.md @@ -49,33 +49,43 @@ text below were caught by codex on the first cut — see PR #45.) control flow / borrow blocks and any pair that shares a variable, so it never emits an unsound swap.) -### The result so far +### The result so far — two targets -The whole `.own` corpus (gallery + examples + corpus, 28 programs) is **invariant -under both transforms** — the expected baseline for a core built on symbol -identity + dataflow (it *should* be name/order-agnostic). That is a real, if -modest, robustness result, and the framework now ratchets it. The harness is not -vacuous: a **teeth test** asserts the (code, line) key actually distinguishes a -leak from a clean run, and that the transforms genuinely fire. +**Core (`.own`, `metamorphic.py`).** The whole `.own` corpus (gallery + examples + +corpus, 28 programs) is **invariant under both transforms** — the expected baseline +for a core built on symbol identity + dataflow (it *should* be name/order-agnostic). + +**Bridge (OwnIR facts, `metamorphic_facts.py`).** The same idea one level down: the +JSON the extractor emits is a *set of records* (components, their resources, DI +services, contracts), so **reversing** any record list or **consistently renaming** +a component/service identifier cannot change which leaks exist. All **18** committed +fact fixtures (plus a captive-DI set) are invariant under `check_facts` — including +`DI001` (a singleton capturing a scoped service), which holds under both service +reordering *and* a consistent rename of the dependency graph. Higher-signal than the +core, since the bridge carries the incidental complexity (DI graph, finding dedup, +source-lifetime tiering) — and still dotnet-free. + +Both are real, if modest, robustness results, and the framework now ratchets them. +Neither harness is vacuous: a **teeth test** asserts the code key actually +distinguishes a leak from a clean run, and that the transforms genuinely fire. ## Run it ```sh -python scripts/metamorphic.py examples corpus # sweep, report any non-invariance -python scripts/metamorphic.py --selftest # corpus invariance + teeth test (CI) +python scripts/metamorphic.py examples corpus # core: sweep .own +python scripts/metamorphic.py --selftest # core: invariance + teeth (CI) +python scripts/metamorphic_facts.py tests/fixtures/ownir # bridge: sweep *.facts.json +python scripts/metamorphic_facts.py --selftest # bridge: invariance + teeth (CI) ``` -`--selftest` runs on every push (CI `script selftests` job), beside the miner and -oracle selftests. +Both `--selftest`s run on every push (CI `script selftests` job), beside the miner +and oracle selftests. ## Follow-ups (where the bug-finding power grows) -- **More sound transforms** — dead-branch wrapping for statements that bind no - later-used name; a redundant borrow/`use`; statement reorder into nested bodies. -- **OwnIR-fact target** — mutate the JSON facts (reorder a component's resource - records, rename component/event/handler symbols) and re-run `check_facts`. The - bridge has more incidental complexity than the core, so this is higher-signal — - and still dotnet-free. +- **More sound transforms** — core: dead-branch wrapping for statements that bind + no later-used name, a redundant borrow/`use`, reorder into nested bodies; bridge: + splitting independent components, or inserting a no-op released record. - **C# source target** — mutate `.cs` (rename locals, reorder members) → re-run the extractor → check. This tests the *extractor*, where the syntactic-FP bugs actually lived — but it needs the Roslyn frontend, so it is CI-only. diff --git a/scripts/metamorphic_facts.py b/scripts/metamorphic_facts.py new file mode 100644 index 00000000..db8a0337 --- /dev/null +++ b/scripts/metamorphic_facts.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +""" +Metamorphic testing for the Own.NET OwnIR bridge (`check_facts`) — analyzer QA. + +The sibling of `scripts/metamorphic.py`, one level down: instead of mutating `.own` +source it mutates the **OwnIR facts** (the JSON the C# extractor emits) and asserts +the bridge's diagnostics are invariant. The facts are *sets of records* — components, +their resources, DI services, contracts — so reordering them, or consistently +renaming an identifier, cannot change *which* leaks exist. If `check_facts`'s +verdict moves, the bridge is order/name-sensitive where it must not be. Higher +signal than the core harness: the bridge carries the incidental complexity (the DI +captive-dependency graph, finding dedup, source-lifetime tiering). + +Sound transforms (v1), each meaning-preserving: + - **reverse**: reverse a list of records — the top-level component/service/function + lists, a component's resource list, a service's deps. Independent records commute. + - **rename**: consistently rename a component/service identifier at the fact + graph's *identifier* sites only (a `name`/`source_type` field, a `deps` entry) — + alpha-equivalence over the fact graph. Semantic literals (a `lifetime`, a + `source` kind, an `event`/`handler`) are never touched, so a name equal to such + a literal stays sound. + +Compared on the **multiset of diagnostic codes** (not lines — same reasoning as the +core harness: a record's line is intrinsic, but order/name must not move a *code*). + +dotnet-free: drives the same bridge entry the CLI uses — `check_facts(load(path))`. + +Usage: + metamorphic_facts.py ... # sweep *.facts.json; report non-invariance + metamorphic_facts.py --selftest +""" + +from __future__ import annotations + +import copy +import json +import os +import sys +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING, Any + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from ownlang.ownir import OwnIRError, check_facts, load + +if TYPE_CHECKING: + from collections.abc import Iterator + +_LIST_KEYS = ("components", "services", "functions") +# Fact-graph sites that hold a component/service *identifier* (vs a semantic literal +# like a `lifetime` or a `source` kind). A rename only ever touches these. +_NAME_KEYS = ("name", "source_type") # scalar identifier fields +_DEP_KEY = "deps" # a list of identifier references + + +def code_key(facts: dict[str, Any]) -> tuple[str, ...]: + """The sorted multiset of diagnostic codes `check_facts` produces — the property + a meaning-preserving fact rewrite must not change.""" + return tuple(sorted(d.code for d in check_facts(facts))) + + +def _strings(node: Any) -> set[str]: + """Every string value anywhere in the facts (to pick a guaranteed-fresh rename).""" + if isinstance(node, str): + return {node} + if isinstance(node, dict): + return set().union(set(), *(_strings(v) for v in node.values())) + if isinstance(node, list): + return set().union(set(), *(_strings(v) for v in node)) + return set() + + +def _rename_id(node: Any, old: str, new: str) -> Any: + """A deep copy of `node` with the identifier `old` renamed to `new` only at the + fact graph's *identifier* sites — a component/service `name`, a `source_type` + reference, or an entry of a `deps` list. A semantic literal under any other key + (a `lifetime`, a `source` kind, an `event`/`handler`/`file`) is left untouched, + so the rename stays meaning-preserving even if a name equals such a literal.""" + if isinstance(node, dict): + out: dict[str, Any] = {} + for k, v in node.items(): + if k in _NAME_KEYS and v == old: + out[k] = new + elif k == _DEP_KEY and isinstance(v, list): + out[k] = [new if x == old else x for x in v] + else: + out[k] = _rename_id(v, old, new) + return out + if isinstance(node, list): + return [_rename_id(x, old, new) for x in node] + return node + + +def reverse_variants(facts: dict[str, Any]) -> Iterator[tuple[str, dict[str, Any]]]: + """Reverse each list of records: the top-level component/service/function lists, + each component's resource list, and each service's deps. Records are a set, so + their order is not meaning. Each variant reverses its *own* (deep-copied) list, + never aliasing the source graph.""" + for key in _LIST_KEYS: + seq = facts.get(key) + if isinstance(seq, list) and len(seq) > 1: + v = copy.deepcopy(facts) + v[key] = list(reversed(v[key])) + yield (f"reverse {key}", v) + comps = facts.get("components") + if isinstance(comps, list): + for i, c in enumerate(comps): + subs = c.get("subscriptions") if isinstance(c, dict) else None + if isinstance(subs, list) and len(subs) > 1: + v = copy.deepcopy(facts) + v["components"][i]["subscriptions"] = list( + reversed(v["components"][i]["subscriptions"])) + yield (f"reverse components[{i}].subscriptions", v) + svcs = facts.get("services") + if isinstance(svcs, list): + for i, s in enumerate(svcs): + deps = s.get("deps") if isinstance(s, dict) else None + if isinstance(deps, list) and len(deps) > 1: + v = copy.deepcopy(facts) + v["services"][i]["deps"] = list(reversed(v["services"][i]["deps"])) + yield (f"reverse services[{i}].deps", v) + + +def _identifiers(facts: dict[str, Any]) -> list[str]: + """Component + service names — the identifiers safe to consistently rename.""" + out: list[str] = [] + for key in ("components", "services"): + seq = facts.get(key) + if isinstance(seq, list): + out += [r["name"] for r in seq + if isinstance(r, dict) and isinstance(r.get("name"), str)] + return out + + +def rename_variants(facts: dict[str, Any]) -> Iterator[tuple[str, dict[str, Any]]]: + """One variant per component/service name, consistently renamed to a fresh name + at the fact graph's identifier sites. The verdict must not depend on the name.""" + used = _strings(facts) + for name in _identifiers(facts): + fresh = f"{name}_mr" + while fresh in used: + fresh += "x" + yield (f"rename {name}->{fresh}", _rename_id(facts, name, fresh)) + + +_TRANSFORMS = (reverse_variants, rename_variants) + + +def violations(facts: dict[str, Any]) -> list[str]: + """Every metamorphic violation for one *validated* fact set (the caller loads it + through `ownir.load`): a meaning-preserving variant whose code multiset differs + from the original. Empty == invariant.""" + base = code_key(copy.deepcopy(facts)) + out: list[str] = [] + for transform in _TRANSFORMS: + for label, variant in transform(facts): + try: + got = code_key(variant) + except Exception as e: # a crash on a valid variant is itself a finding + out.append(f"{label}: variant raised {type(e).__name__}: {e}") + continue + if got != base: + out.append(f"{label}: base={list(base)} variant={list(got)}") + return out + + +def sweep(paths: list[str]) -> int: + """Run the harness over every *.facts.json under the given files/dirs. Returns a + process exit status: 0 only if at least one file loaded and every loaded fact set + is invariant; 1 on any violation, any load error, or an empty input set (so a + sweep that evaluated nothing is not a false green).""" + files: list[Path] = [] + for p in paths: + pp = Path(p) + files.extend(sorted(pp.rglob("*.facts.json")) if pp.is_dir() else [pp]) + bad = load_errors = loaded = 0 + for f in files: + try: + facts = load(str(f)) # the real pipeline's entry — validates the schema + except (OSError, OwnIRError) as e: + print(f"{f}: cannot load ({e})") + load_errors += 1 + continue + loaded += 1 + vs = violations(facts) + if vs: + bad += 1 + print(f"\n{f.name}: {len(vs)} violation(s):") + for v in vs: + print(f" - {v}") + print(f"\nmetamorphic-facts: {loaded - bad}/{loaded} loaded fact set(s) invariant " + f"under {len(_TRANSFORMS)} transform class(es).") + if not loaded: + print("metamorphic-facts: no loadable *.facts.json inputs") + return 1 if (bad or load_errors or not loaded) else 0 + + +def _selftest() -> int: + fails: list[str] = [] + repo = Path(__file__).resolve().parent.parent + + # 1) Robustness: every committed fact fixture must load *through the validator* + # and be invariant. A fixture that no longer loads is a loud failure. + fix = repo / "tests" / "fixtures" / "ownir" + files = sorted(fix.rglob("*.facts.json")) if fix.exists() else [] + bad: list[str] = [] + for f in files: + try: + facts = load(str(f)) + except OwnIRError as e: + bad.append(f"{f.name}: does not load ({e})") + continue + vs = violations(facts) + if vs: + bad.append(f"{f.name}: {vs[0]}") + if not files: + fails.append("no .facts.json fixtures found to sweep") + elif bad: + fails.append(f"fixtures not all loadable+invariant: {len(bad)} file(s), e.g. {bad[0]}") + + # 2) Teeth: the code key must distinguish a leak from a clean run. + leak = {"module": "M", "components": [{"name": "Vm", "file": "Vm.cs", + "subscriptions": [{"event": "b.X", "handler": "h", "line": 5, "released": False}]}]} + clean = {"module": "M", "components": [{"name": "Vm", "file": "Vm.cs", + "subscriptions": [{"event": "b.X", "handler": "h", "line": 5, "released": True}]}]} + if code_key(leak) == code_key(clean): + fails.append("teeth: code key does not distinguish a leak from a clean run") + if "OWN001" not in code_key(leak): + fails.append("teeth: expected OWN001 on the leak fixture") + + # 3) The transforms fire on a fact set that admits them, and a two-component + + # multi-service (captive-DI) set is invariant under reorder and rename. + multi = {"module": "M", + "components": [ + {"name": "A", "file": "A.cs", "subscriptions": [ + {"event": "b.X", "handler": "hx", "line": 5, "released": False}, + {"event": "b.Y", "handler": "hy", "line": 6, "released": False}]}, + {"name": "B", "file": "B.cs", "subscriptions": [ + {"event": "b.Z", "handler": "hz", "line": 7, "released": True}]}], + "services": [ + {"name": "Sender", "lifetime": "singleton", "file": "S.cs", "line": 1, + "deps": ["Db"]}, + {"name": "Db", "lifetime": "scoped", "file": "S.cs", "line": 2, "deps": []}]} + if sum(1 for _ in reverse_variants(multi)) < 2: + fails.append("expected >=2 reverse variants") + if sum(1 for _ in rename_variants(multi)) < 2: + fails.append("expected >=2 rename variants") + if violations(multi): + fails.append(f"multi-component/service set should be invariant: {violations(multi)}") + + # 4) rename stays sound when an identifier collides with a semantic literal: a + # service *named* "scoped" must not have its `lifetime: "scoped"` rewritten. + collide = {"module": "M", "components": [], + "services": [{"name": "scoped", "lifetime": "scoped", "file": "S.cs", + "line": 1, "deps": []}]} + collide_ok = False + for _lbl, mod in rename_variants(collide): + svcs = mod.get("services") + if isinstance(svcs, list) and svcs and isinstance(svcs[0], dict): + collide_ok = svcs[0].get("lifetime") == "scoped" + if not collide_ok: + fails.append("rename clobbered a semantic literal (lifetime) that equals a name") + + # 5) The harness drives the real pipeline (load -> check_facts), so a + # schema-invalid fact set (a future ownir_version) is rejected at the gate, + # not silently swept as "invariant" (codex #46). + fd, p = tempfile.mkstemp(suffix=".facts.json") + try: + with os.fdopen(fd, "w", encoding="utf-8") as fh: + json.dump({"ownir_version": 999, "module": "M", "components": []}, fh) + try: + load(p) + fails.append("load() should reject a future ownir_version") + except OwnIRError: + pass + finally: + os.unlink(p) + + for msg in fails: + print(f"METAMORPHIC-FACTS SELFTEST FAIL: {msg}") + total = 8 + print(f"metamorphic-facts selftest: {total - len(fails)}/{total} checks passed " + f"(swept {len(files)} fixture(s))") + return 1 if fails else 0 + + +def main(argv: list[str]) -> int: + if argv == ["--selftest"]: + return _selftest() + if not argv or any(a.startswith("-") for a in argv): + print(__doc__) + return 2 + return sweep(argv) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:]))