From b2deb07905737e75fb7a6d81621003d04430880a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 20 Jun 2026 11:12:50 +0000 Subject: [PATCH 1/4] =?UTF-8?q?feat(eval):=20corpus=20benchmark=20?= =?UTF-8?q?=E2=80=94=20recall=20+=20specificity=20on=20real=20C#=20(P-012?= =?UTF-8?q?=20slice=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The labeled corpus was already half a benchmark (before.cs/after.cs/expected/ case.own per case), but the only thing scoring it — tests/test_corpus.py — checks the .own *reduction* and even conceded "not that the tool scanned real C#." The P-001 extractor exists now, so close the gap. scripts/benchmark.py runs the ACTUAL before.cs/after.cs through the extractor + core (own-check.sh --format sarif) and measures what the .own check cannot: - recall — the bug is CAUGHT in real before.cs (>= 1 verdict); - specificity — the real after.cs (the fix) is SILENT (0 verdicts, no false alarm). The metric is code-agnostic (OWN001 vs OWN014 both count as caught), so it survives a sound reclassification an exact-code match would spuriously fail; a verdict is an error/warning SARIF result, the advisory OWN050 note excluded. Aggregate: one defensible line "N caught / N · K fixes clean / N · F false positives". Validated two ways (the harness pattern): --selftest pins the SARIF-parse + scoring logic with no SDK (wired into the lint job's selftests), and a new dotnet-backed corpus-benchmark CI job runs the real benchmark — materializing the WindowsDesktop ref pack (OWN_EXTRA_REF_DIRS) so framework events resolve, same as the oracle/mine jobs. The gate: every before.cs caught and every after.cs silent. This is the measurement spine — a reproducible recall/specificity number over real C#, pinned against regression, and the verifiable reward for any future learning loop (built before any proposer/LLM layer, never trusting a source). docs: P-012 status, the stale test_corpus note, and docs/notes/corpus-benchmark.md. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Rg8kSk1YT14x7A1vo5zgED --- .github/workflows/ci.yml | 48 ++++- docs/notes/corpus-benchmark.md | 67 ++++++ docs/proposals/P-012-bug-corpus-mining.md | 9 +- scripts/benchmark.py | 247 ++++++++++++++++++++++ tests/test_corpus.py | 8 +- 5 files changed, 374 insertions(+), 5 deletions(-) create mode 100644 docs/notes/corpus-benchmark.md create mode 100644 scripts/benchmark.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6221e0a3..3e2e11ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,12 +34,13 @@ jobs: # analyzer tester) carry embedded fixtures / sweep the .own corpus; run their # selftests here so the parsers/aggregators and the robustness invariants stay # honest on every push, not only on workflow_dispatch. - - name: script selftests (miner + oracle + metamorphic) + - name: script selftests (miner + oracle + metamorphic + benchmark) run: | python scripts/mine_report.py --selftest python scripts/oracle_compare.py --selftest python scripts/metamorphic.py --selftest python scripts/metamorphic_facts.py --selftest + python scripts/benchmark.py --selftest tests: name: tests (py${{ matrix.python-version }}) @@ -579,3 +580,48 @@ jobs: sarif_file: ${{ steps.own.outputs.sarif-file }} category: own-net-samples + # P-012 slice 1: score the checker against the labeled corpus on REAL C# — not + # just the .own reduction tests/test_corpus.py checks. Per case: the bug must be + # CAUGHT in before.cs (recall) and the fix must be SILENT in after.cs + # (specificity / no false alarm). A defensible, regression-pinned number — and + # the RLVR reward scaffold: a deterministic verifier over labeled real-C# data. + corpus-benchmark: + name: corpus benchmark (real C# recall + specificity) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: "8.0.x" + # Some corpus cases subscribe to framework events (WPF Window, Microsoft.Win32 + # SystemEvents); the type-aware extractor needs those refs to bind a `+=` to an + # event (else an OWN050 note, not a leak). Materialize the WindowsDesktop ref + # pack and export OWN_EXTRA_REF_DIRS — same mechanism as the oracle/mine jobs. + # Harmless for the self-contained cases (deduped against the runtime TPA). + - name: Materialize framework reference assemblies + continue-on-error: true + run: | + tmp=$(mktemp -d) + printf '%s\n' \ + '' \ + ' ' \ + ' net8.0-windows' \ + ' true' \ + ' true' \ + ' true' \ + ' ' \ + '' > "$tmp/ref.csproj" + dotnet restore "$tmp/ref.csproj" >/dev/null 2>&1 || echo "ref restore failed (continuing)" + d=$(find "$HOME/.nuget/packages/microsoft.windowsdesktop.app.ref" -type d -name 'net8.0' 2>/dev/null | sort | tail -1 || true) + if [ -n "$d" ]; then + echo "OWN_EXTRA_REF_DIRS=$d" >> "$GITHUB_ENV" + echo "framework refs: $d ($(find "$d" -name '*.dll' | wc -l) dlls)" + else + echo "framework refs not found — own-check resolves runtime types only" + fi + - name: Score the corpus on real C# + run: python scripts/benchmark.py + diff --git a/docs/notes/corpus-benchmark.md b/docs/notes/corpus-benchmark.md new file mode 100644 index 00000000..38b9ae5c --- /dev/null +++ b/docs/notes/corpus-benchmark.md @@ -0,0 +1,67 @@ +# Corpus benchmark — recall + specificity on real C# (P-012 slice 1) + +The labeled corpus (`corpus///`) was already half a benchmark: every +case carries `before.cs` (buggy), `after.cs` (fixed), `expected-diagnostics.txt` +and a `case.own` reduction. But the only thing scoring it — `tests/test_corpus.py` +— checks the **`.own` reduction**, and its own note conceded *"not that the tool +scanned real C#."* That note is now stale: the P-001 extractor exists. + +`scripts/benchmark.py` closes the gap. It runs the **actual C#** through the +extractor + core (`own-check.sh --format sarif`) and measures the two things the +`.own` check cannot: + +- **recall** — the bug is *caught* in the real `before.cs` (≥ 1 verdict); +- **specificity** — the real `after.cs` (the fix) is *silent* (0 verdicts — no + false alarm on correct code). + +The aggregate is one defensible line: + +``` +benchmark: 9/9 bugs caught in real C# · 9/9 fixes clean · 0 false positive(s) on fixes +``` + +## Why catch/clean, not exact-code match + +The metric is deliberately **code-agnostic**: a leak reported as `OWN001` (token +leak) vs `OWN014` (region escape) both count as "caught". `test_corpus.py` pins the +exact code on the `.own` reduction; the real-C# benchmark answers the blunter, more +honest product question — *did we catch the real bug, and did we stay silent on the +real fix?* — which survives a sound reclassification of the leak that an exact-code +assertion would spuriously fail. (`expected-diagnostics.txt` is still reported as a +secondary `expected_hit` signal, just not part of the gate.) + +A **verdict** is any SARIF result at error/warning level. The advisory `note` level +(`OWN050` "resolution skipped") is coverage honesty, not a verdict, so it is +neither a catch nor a false positive — a `before.cs` whose framework type didn't +resolve reads as a *miss*, not a fake catch. + +## Validated two ways (the harness pattern) + +- **`--selftest` (no SDK)** — the SARIF-parsing and scoring/aggregation logic is + pinned on embedded fixtures (verdict levels counted, `note` excluded, malformed + input safe, the catch/clean/FP arithmetic), wired into the lint job alongside the + miner/oracle/metamorphic selftests. Keeps the harness honest on every push. +- **`corpus-benchmark` CI job (dotnet)** — runs the real benchmark. Some cases + subscribe to framework events (WPF `Window`, `Microsoft.Win32.SystemEvents`), so + it materializes the WindowsDesktop ref pack and exports `OWN_EXTRA_REF_DIRS` (the + same mechanism as the oracle/mine jobs) — else a `+=` to an unresolved event is an + `OWN050` note, not a leak. The gate: **every** `before.cs` caught and **every** + `after.cs` silent; a recall or specificity regression turns the job red. + +## Why it matters + +This is the **measurement spine**. Until now "does Own.NET work?" was answered by +the `.own` logic check and anecdotal oracle overlaps; now there is a reproducible +recall/specificity number over real C#, pinned against regression. It is also the +**verifiable reward** for any future learning loop (RLVR): a deterministic verifier +over labeled real-C# data is exactly the clean reward signal — built *before* any +proposer/LLM layer, never trusting a source. + +## Next + +- Grow the corpus (P-012 stage 1 mining) — every new mined `before`/`after` pair is + a new benchmark row for free; the number gets more defensible as N grows. +- Stage 2 prevalence scan (hits per 1k LOC across 50–100 repos) feeding the + `docs/ROADMAP.md` priority matrix — replacing the proxy estimates with counts. +- Per-code recall and a precision breakdown once the corpus is large enough for the + rates to mean something. diff --git a/docs/proposals/P-012-bug-corpus-mining.md b/docs/proposals/P-012-bug-corpus-mining.md index 31c73c47..ec8754a8 100644 --- a/docs/proposals/P-012-bug-corpus-mining.md +++ b/docs/proposals/P-012-bug-corpus-mining.md @@ -1,6 +1,13 @@ # P-012 — Real-world bug corpus & mining pipeline -- **Status:** draft +- **Status:** in progress — the **corpus benchmark** (slice 1) is built: + `scripts/benchmark.py` scores the checker against the labeled corpus on the + *real* `before.cs`/`after.cs` (not just the `.own` reduction), measuring recall + (the bug is caught) and specificity (the fix is silent), gated in the + `corpus-benchmark` CI job. This is the measurement spine — the defensible number, + and the verifiable reward for any future learning loop. Still ahead: GitHub + mining at scale (stage 1) and the 50–100-repo prevalence scan (stage 2). See + [docs/notes/corpus-benchmark.md](../notes/corpus-benchmark.md). - **Depends on:** P-001 (C# → OwnIR extractor — the scanner that does stage 2); the existing `corpus/` layout (`before.cs`, `after.cs`, `expected-diagnostics.txt`, `notes.md`/`source.md`). diff --git a/scripts/benchmark.py b/scripts/benchmark.py new file mode 100644 index 00000000..6350ab95 --- /dev/null +++ b/scripts/benchmark.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +""" +Corpus benchmark — score the checker against the labeled real-world corpus. + +Each ``corpus///`` holds a real bug: ``before.cs`` (buggy), +``after.cs`` (fixed), ``expected-diagnostics.txt`` (the codes), ``case.own`` (a +reduction) and ``notes.md``. ``tests/test_corpus.py`` checks the ``.own`` +*reduction*; this harness runs the **actual C#** through the extractor + core +(``own-check.sh``) and measures the two things the ``.own`` check cannot: + + - **recall** — the bug is *caught* in the real ``before.cs`` (>= 1 verdict); + - **specificity** — the real ``after.cs`` (the fix) is *silent* (0 verdicts, i.e. + no false alarm on correct code). + +The aggregate is one defensible line: *"N cases - caught C/N in real C# - clean +K/N fixes - F false positives"*. That is the RLVR reward scaffold: a deterministic +verifier over labeled real-C# data. + +A *verdict* is any SARIF result at error/warning level. The advisory note level +(``OWN050`` "resolution skipped") is coverage honesty, not a verdict, so it counts +as neither a catch nor a false positive. The catch/clean metric is deliberately +code-agnostic: a leak reported as OWN001 vs OWN014 both count as "caught", so the +benchmark survives a classifier reclassification that ``test_corpus.py``'s +exact-code match would not. + +Needs a .NET SDK (``own-check.sh`` runs the extractor); some WPF cases also need +``OWN_EXTRA_REF_DIRS`` to resolve framework events. ``--selftest`` validates the +scoring + SARIF-parsing logic with no SDK (embedded fixtures), so the lint job +keeps the harness honest on every push. + +Usage: + python scripts/benchmark.py [--root REPO] [--corpus DIR ...] # run the benchmark + python scripts/benchmark.py --selftest # logic check (no SDK) +""" +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from dataclasses import dataclass + +# A verdict is a real finding; these SARIF levels are not (note = the advisory +# OWN050 "resolution skipped"; none = suppressed) — neither a catch nor an FP. +_NONVERDICT_LEVELS = frozenset({"note", "none"}) + + +def sarif_codes(sarif_text: str) -> set[str]: + """The set of *verdict* rule codes in a SARIF log: results at error/warning + level. A note-level result (OWN050 resolution-skipped) is advisory coverage + honesty, not a verdict, so it is excluded. Malformed input yields no codes + (the caller treats that as "no verdict", surfaced as a missed catch).""" + try: + doc = json.loads(sarif_text) + except (json.JSONDecodeError, ValueError): + return set() + if not isinstance(doc, dict): + return set() + codes: set[str] = set() + runs = doc.get("runs") + for run in runs if isinstance(runs, list) else []: + if not isinstance(run, dict): + continue + results = run.get("results") + for res in results if isinstance(results, list) else []: + if not isinstance(res, dict): + continue + # SARIF's default level is "warning" — a result without one is a verdict. + level = res.get("level", "warning") + rid = res.get("ruleId") + if isinstance(rid, str) and level not in _NONVERDICT_LEVELS: + codes.add(rid) + return codes + + +@dataclass +class CaseScore: + """One corpus case scored on real C#: the expected codes, and the verdict + codes found on ``before.cs`` (buggy) and ``after.cs`` (fixed).""" + + name: str + expected: set[str] + before: set[str] + after: set[str] + + @property + def caught(self) -> bool: + """The bug is caught in the real ``before.cs``: at least one verdict.""" + return bool(self.before) + + @property + def clean(self) -> bool: + """The real fix (``after.cs``) is silent: no verdict (no false alarm).""" + return not self.after + + @property + def expected_hit(self) -> bool: + """Secondary signal: the specific expected code(s) appear on ``before`` + (a stronger match than "some verdict"). Not part of the gate, so a sound + reclassification of the leak code does not fail the benchmark.""" + return bool(self.expected) and self.expected <= self.before + + +def summarize(scores: list[CaseScore]) -> tuple[int, int, int, int]: + """``(caught, clean, total, false_positives)`` over the scored cases.""" + caught = sum(1 for s in scores if s.caught) + clean = sum(1 for s in scores if s.clean) + fps = sum(len(s.after) for s in scores) + return caught, clean, len(scores), fps + + +# ---- the SDK-backed half (the real run) -------------------------------------- + +def _own_check(root: str, cs_file: str) -> str: + """Run ``own-check.sh --format sarif`` over one .cs file; return its SARIF + stdout (build chatter goes to stderr, so stdout is a clean SARIF log).""" + script = os.path.join(root, "scripts", "own-check.sh") + proc = subprocess.run( + [script, "--root", root, "--format", "sarif", "--", cs_file], + capture_output=True, text=True, + ) + return proc.stdout + + +def discover(corpus_dirs: list[str]) -> list[str]: + """Case directories carrying before.cs/after.cs/expected-diagnostics.txt.""" + cases: list[str] = [] + for base in corpus_dirs: + if not os.path.isdir(base): + continue + for name in sorted(os.listdir(base)): + d = os.path.join(base, name) + if (os.path.isdir(d) + and os.path.exists(os.path.join(d, "before.cs")) + and os.path.exists(os.path.join(d, "after.cs")) + and os.path.exists(os.path.join(d, "expected-diagnostics.txt"))): + cases.append(d) + return cases + + +def score_corpus(root: str, corpus_dirs: list[str]) -> list[CaseScore]: + """Run the extractor + core over every case's before.cs and after.cs.""" + scores: list[CaseScore] = [] + for d in discover(corpus_dirs): + with open(os.path.join(d, "expected-diagnostics.txt"), encoding="utf-8") as f: + expected = {w for w in f.read().split() if w} + before = sarif_codes(_own_check(root, os.path.join(d, "before.cs"))) + after = sarif_codes(_own_check(root, os.path.join(d, "after.cs"))) + scores.append(CaseScore(os.path.basename(d), expected, before, after)) + return scores + + +def run(root: str, corpus_dirs: list[str]) -> int: + """Score the corpus on real C# and print the scorecard. The gate: every real + bug caught and every real fix silent (recall + specificity at 100%).""" + scores = score_corpus(root, corpus_dirs) + if not scores: + print("BENCHMARK FAIL: no corpus cases found") + return 1 + width = max(len(s.name) for s in scores) + print("corpus benchmark (real C# through the extractor + core):") + for s in scores: + catch = "caught" if s.caught else "MISSED" + clean = "clean" if s.clean else f"FP:{','.join(sorted(s.after))}" + note = ("" if s.expected_hit + else f" (expected {sorted(s.expected)}, got {sorted(s.before)})") + print(f" {s.name:<{width}} before[{catch}: {','.join(sorted(s.before)) or '-'}]" + f" after[{clean}]{note}") + caught, clean, total, fps = summarize(scores) + print(f"benchmark: {caught}/{total} bugs caught in real C# · " + f"{clean}/{total} fixes clean · {fps} false positive(s) on fixes") + if caught != total or clean != total: + print("BENCHMARK FAIL: recall or specificity below the corpus baseline " + "(every before.cs must be caught and every after.cs must be silent)") + return 1 + return 0 + + +# ---- selftest (no SDK) ------------------------------------------------------- + +def _selftest() -> int: + fails: list[str] = [] + + # 1) sarif_codes: verdict levels counted (deduped), note level excluded, junk safe. + sarif = json.dumps({"runs": [{"results": [ + {"ruleId": "OWN001", "level": "error"}, + {"ruleId": "OWN001", "level": "warning"}, # dedupes with the above + {"ruleId": "DI001", "level": "error"}, + {"ruleId": "OWN050", "level": "note"}, # advisory -> excluded + {"ruleId": "OWN999", "level": "none"}, # suppressed -> excluded + ]}]}) + got = sarif_codes(sarif) + if got != {"OWN001", "DI001"}: + fails.append(f"sarif_codes: expected {{OWN001,DI001}}, got {sorted(got)}") + for bad in ("not json", "{}", "[]", json.dumps({"runs": [{"results": []}]})): + if sarif_codes(bad) != set(): + fails.append(f"sarif_codes: {bad!r} must yield no codes") + # a result with no level defaults to a verdict (SARIF's default is "warning"). + if sarif_codes(json.dumps({"runs": [{"results": [{"ruleId": "OWN001"}]}]})) != {"OWN001"}: + fails.append("sarif_codes: a level-less result must count as a verdict") + + # 2) scoring + aggregation. + cases = [ + CaseScore("hit_clean", {"OWN001"}, {"OWN001"}, set()), # caught + clean + hit + CaseScore("drift_clean", {"OWN001"}, {"OWN014"}, set()), # caught + clean, drifted + CaseScore("missed", {"OWN003"}, set(), set()), # not caught + CaseScore("leaky_fix", {"OWN001"}, {"OWN001"}, {"OWN001"}), # caught, fix has an FP + ] + checks = [ + (cases[0].caught and cases[0].clean and cases[0].expected_hit, "hit_clean misjudged"), + (cases[1].caught and cases[1].clean and not cases[1].expected_hit, + "drift case must be caught+clean but not an expected_hit"), + (not cases[2].caught and cases[2].clean, "missed case must be not-caught but clean"), + (cases[3].caught and not cases[3].clean, "leaky_fix must be caught but not clean"), + ] + for ok, msg in checks: + if not ok: + fails.append(f"scoring: {msg}") + if summarize(cases) != (3, 3, 4, 1): + fails.append(f"summarize: expected (3,3,4,1), got {summarize(cases)}") + + for f in fails: + print(f"SELFTEST FAIL: {f}") + print(f"benchmark selftest: {'OK' if not fails else 'FAIL'} " + f"— sarif-parse + scoring/aggregation ({len(checks) + 2} checks)") + return 1 if fails else 0 + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser(description="Corpus benchmark for the Own.NET checker.") + ap.add_argument("--selftest", action="store_true", + help="validate the harness logic with no .NET SDK") + ap.add_argument("--root", default=None, help="repo root (default: this script's repo)") + ap.add_argument("--corpus", action="append", default=None, metavar="DIR", + help="corpus base dir(s) (default: corpus/real-world + corpus/wpf)") + args = ap.parse_args(argv) + if args.selftest: + return _selftest() + root = args.root or os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + corpus_dirs = args.corpus or [os.path.join(root, "corpus", "real-world"), + os.path.join(root, "corpus", "wpf")] + return run(root, corpus_dirs) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/tests/test_corpus.py b/tests/test_corpus.py index 9e836324..a12c401a 100644 --- a/tests/test_corpus.py +++ b/tests/test_corpus.py @@ -11,9 +11,11 @@ expected-diagnostics.txt next to it, so the corpus stays honest: if the checker ever stops catching one of these real patterns, the suite goes red. -NOTE: case.own is a hand reduction of the C# pattern, not C# the checker -ingested -- OwnLang has no C# front-end. The corpus shows the ownership *logic* -maps onto real bugs, not that the tool scanned real C#. +NOTE: case.own is a hand reduction of the C# pattern; this test checks the +ownership *logic* maps onto the real bug. The actual before.cs/after.cs are now +also scanned end-to-end (extractor + core) by scripts/benchmark.py — the +real-C# recall/specificity benchmark — which runs in the dotnet-backed +`corpus-benchmark` CI job (this Python-only test needs no SDK). Run: python tests/test_corpus.py python tests/run_tests.py (runs it as part of the suite) From a1e6f6ebde865bed9a8a0c236a7dc6d89d37c2d7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 20 Jun 2026 11:23:03 +0000 Subject: [PATCH 2/4] =?UTF-8?q?fix(eval):=20honest=20benchmark=20gate=20?= =?UTF-8?q?=E2=80=94=20precision=20absolute,=20recall=20pinned=20at=20floo?= =?UTF-8?q?r?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first real run measured 3/9 bugs caught in real C# · 9/9 fixes clean · 0 false positives. The .own reductions all fire (tests/test_corpus.py); the 6 missed are cases the C# *frontend* does not yet extract (pool double-return/use-after-return, ownership-handoff, a few dispose/escape shapes) — the extractor's recall debt, now measured. So gate honestly rather than hard-asserting a 9/9 the tool cannot yet deliver: precision is absolute (every after.cs silent, zero false positives — a regression there is crying wolf on correct code), recall is pinned at a floor (--min-recall, set to 3 in CI) that ratchets up as extraction coverage grows. The number is *reported* and forbidden from regressing — exactly what a measurement spine does. benchmark.py: extract gate() (pure, selftested), run() takes min_recall, --min-recall CLI; selftest now covers the gate (11 checks). docs/notes/corpus-benchmark.md and P-012 record the honest first number + the itemized recall backlog. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Rg8kSk1YT14x7A1vo5zgED --- .github/workflows/ci.yml | 7 ++- docs/notes/corpus-benchmark.md | 25 ++++++++-- docs/proposals/P-012-bug-corpus-mining.md | 6 ++- scripts/benchmark.py | 57 ++++++++++++++++++----- 4 files changed, 78 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e2e11ba..4022f3a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -623,5 +623,10 @@ jobs: echo "framework refs not found — own-check resolves runtime types only" fi - name: Score the corpus on real C# - run: python scripts/benchmark.py + # Precision is gated absolutely (every fix silent, zero false positives); + # recall is pinned at the measured floor (the frontend catches the + # subscription/region class; pool/dispose/handoff shapes are the itemized + # extraction backlog) and ratchets up as the extractor grows. Raise the + # floor whenever recall improves — a drop below it is a regression. + run: python scripts/benchmark.py --min-recall 3 diff --git a/docs/notes/corpus-benchmark.md b/docs/notes/corpus-benchmark.md index 38b9ae5c..49a23c3a 100644 --- a/docs/notes/corpus-benchmark.md +++ b/docs/notes/corpus-benchmark.md @@ -14,12 +14,24 @@ extractor + core (`own-check.sh --format sarif`) and measures the two things the - **specificity** — the real `after.cs` (the fix) is *silent* (0 verdicts — no false alarm on correct code). -The aggregate is one defensible line: +The aggregate is one defensible line. The **first measurement** (9 cases): ``` -benchmark: 9/9 bugs caught in real C# · 9/9 fixes clean · 0 false positive(s) on fixes +benchmark: 3/9 bugs caught in real C# · 9/9 fixes clean · 0 false positive(s) on fixes ``` +That is the honest day-one number, and it is *sharp*: **specificity is perfect** +(every real fix is silent, zero false positives — the checker does not cry wolf on +correct code), and **recall is 3/9** — the three caught are exactly the +subscription/region class the extractor is strongest at (`zombie-viewmodel` → +OWN001, two static-event escapes → OWN014). The six missed +(`arraypool-double-return`, `arraypool-use-after-return`, `ownership-handoff-consume`, +`screentogif-loaded-subscription`, `handler-use-after-dispose`, +`viewmodel-escapes-to-app`) are cases the `.own` reductions *all* catch but the C# +**frontend** does not yet extract — pool double-return/use-after-return, +ownership-handoff, and a few dispose/escape shapes. The benchmark just quantified +the frontend's recall debt and turned it into an itemized to-do list. + ## Why catch/clean, not exact-code match The metric is deliberately **code-agnostic**: a leak reported as `OWN001` (token @@ -45,8 +57,13 @@ resolve reads as a *miss*, not a fake catch. subscribe to framework events (WPF `Window`, `Microsoft.Win32.SystemEvents`), so it materializes the WindowsDesktop ref pack and exports `OWN_EXTRA_REF_DIRS` (the same mechanism as the oracle/mine jobs) — else a `+=` to an unresolved event is an - `OWN050` note, not a leak. The gate: **every** `before.cs` caught and **every** - `after.cs` silent; a recall or specificity regression turns the job red. + `OWN050` note, not a leak. The gate is **asymmetric and honest**: precision is + absolute (**every** `after.cs` silent, **zero** false positives — a regression + there means crying wolf on correct code), while recall is pinned at a **floor** + (`--min-recall`, currently 3) that ratchets up as the frontend's extraction + coverage grows. We do *not* hard-assert 9/9 the tool cannot yet deliver — the + benchmark *reports* the recall number and forbids it regressing, which is exactly + what a measurement spine should do. ## Why it matters diff --git a/docs/proposals/P-012-bug-corpus-mining.md b/docs/proposals/P-012-bug-corpus-mining.md index ec8754a8..97293d99 100644 --- a/docs/proposals/P-012-bug-corpus-mining.md +++ b/docs/proposals/P-012-bug-corpus-mining.md @@ -5,7 +5,11 @@ *real* `before.cs`/`after.cs` (not just the `.own` reduction), measuring recall (the bug is caught) and specificity (the fix is silent), gated in the `corpus-benchmark` CI job. This is the measurement spine — the defensible number, - and the verifiable reward for any future learning loop. Still ahead: GitHub + and the verifiable reward for any future learning loop. **First measurement: 3/9 + caught · 9/9 fixes clean · 0 false positives** — perfect precision, and the C# + *frontend's* recall debt is now a tracked number (the `.own` reductions all fire; + the 6 missed are pool/dispose/handoff shapes the extractor does not yet lower — + the itemized extraction backlog). Still ahead: raising recall case-by-case, GitHub mining at scale (stage 1) and the 50–100-repo prevalence scan (stage 2). See [docs/notes/corpus-benchmark.md](../notes/corpus-benchmark.md). - **Depends on:** P-001 (C# → OwnIR extractor — the scanner that does stage 2); diff --git a/scripts/benchmark.py b/scripts/benchmark.py index 6350ab95..6f981316 100644 --- a/scripts/benchmark.py +++ b/scripts/benchmark.py @@ -151,9 +151,28 @@ def score_corpus(root: str, corpus_dirs: list[str]) -> list[CaseScore]: return scores -def run(root: str, corpus_dirs: list[str]) -> int: - """Score the corpus on real C# and print the scorecard. The gate: every real - bug caught and every real fix silent (recall + specificity at 100%).""" +def gate(caught: int, clean: int, total: int, fps: int, min_recall: int) -> list[str]: + """The regression gate, as a list of problem strings (empty == pass). + + Precision is non-negotiable — *every* fix must be silent and there must be no + false positive on a fix; a regression there means the checker started crying + wolf on correct code. Recall on real C# is a *tracked* number that ratchets up + as the frontend's extraction coverage grows (the missed cases are the + extractor's to-do list, not a failure of the core logic — test_corpus.py + already shows the .own reductions all fire); the gate only forbids it dropping + below the pinned floor.""" + problems: list[str] = [] + if clean != total: + problems.append(f"specificity regressed: only {clean}/{total} fixes silent") + if fps != 0: + problems.append(f"precision regressed: {fps} false positive(s) on fixes") + if caught < min_recall: + problems.append(f"recall regressed: {caught}/{total} caught, floor is {min_recall}") + return problems + + +def run(root: str, corpus_dirs: list[str], min_recall: int = 0) -> int: + """Score the corpus on real C#, print the scorecard, and apply the gate.""" scores = score_corpus(root, corpus_dirs) if not scores: print("BENCHMARK FAIL: no corpus cases found") @@ -169,12 +188,12 @@ def run(root: str, corpus_dirs: list[str]) -> int: f" after[{clean}]{note}") caught, clean, total, fps = summarize(scores) print(f"benchmark: {caught}/{total} bugs caught in real C# · " - f"{clean}/{total} fixes clean · {fps} false positive(s) on fixes") - if caught != total or clean != total: - print("BENCHMARK FAIL: recall or specificity below the corpus baseline " - "(every before.cs must be caught and every after.cs must be silent)") - return 1 - return 0 + f"{clean}/{total} fixes clean · {fps} false positive(s) on fixes " + f"(recall floor {min_recall})") + problems = gate(caught, clean, total, fps, min_recall) + for p in problems: + print(f"BENCHMARK FAIL: {p}") + return 1 if problems else 0 # ---- selftest (no SDK) ------------------------------------------------------- @@ -220,10 +239,23 @@ def _selftest() -> int: if summarize(cases) != (3, 3, 4, 1): fails.append(f"summarize: expected (3,3,4,1), got {summarize(cases)}") + # 3) gate: precision absolute (a dirty fix or any FP fails regardless of recall), + # recall gated only against the floor. + gate_checks = [ + (gate(3, 9, 9, 0, min_recall=3) == [], "measured baseline (floor 3) must pass"), + (gate(2, 9, 9, 0, min_recall=3) != [], "recall below floor must fail"), + (gate(9, 8, 9, 0, min_recall=3) != [], "a non-silent fix must fail even at full recall"), + (gate(9, 9, 9, 1, min_recall=3) != [], "a false positive on a fix must fail"), + (gate(3, 9, 9, 0, min_recall=0) == [], "floor 0 with clean fixes must pass"), + ] + for ok, msg in gate_checks: + if not ok: + fails.append(f"gate: {msg}") + for f in fails: print(f"SELFTEST FAIL: {f}") print(f"benchmark selftest: {'OK' if not fails else 'FAIL'} " - f"— sarif-parse + scoring/aggregation ({len(checks) + 2} checks)") + f"— sarif-parse + scoring + gate ({len(checks) + len(gate_checks) + 2} checks)") return 1 if fails else 0 @@ -234,13 +266,16 @@ def main(argv: list[str]) -> int: ap.add_argument("--root", default=None, help="repo root (default: this script's repo)") ap.add_argument("--corpus", action="append", default=None, metavar="DIR", help="corpus base dir(s) (default: corpus/real-world + corpus/wpf)") + ap.add_argument("--min-recall", type=int, default=0, metavar="N", + help="fail if fewer than N before.cs cases are caught (the pinned " + "recall floor; specificity + zero-FP are always required)") args = ap.parse_args(argv) if args.selftest: return _selftest() root = args.root or os.path.dirname(os.path.dirname(os.path.abspath(__file__))) corpus_dirs = args.corpus or [os.path.join(root, "corpus", "real-world"), os.path.join(root, "corpus", "wpf")] - return run(root, corpus_dirs) + return run(root, corpus_dirs, args.min_recall) if __name__ == "__main__": From c02f63f1d786f9dd9da90d4bec6c68b23761dd47 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 20 Jun 2026 11:44:21 +0000 Subject: [PATCH 3/4] fix(eval): benchmark fails fast on own-check errors; reject negative floor (review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex + CodeRabbit: the benchmark must not pass on output that never ran. - own-check exits 0 for clean AND findings (no --fail-on-finding), so any non-zero return means the file was not analyzed (extractor crash, no SDK, drifted facts). _scan now routes stdout/returncode through _verdicts_or_raise, which raises BenchmarkError on a non-zero exit (surfacing stderr) instead of letting sarif_codes read empty output as "no verdict" — which would score a failed after.cs as 'clean' and hide a sub-floor before.cs miss. run() catches it and fails loudly. Added a subprocess timeout (300s) so a hung extractor cannot stall CI (CodeRabbit). - --min-recall now uses a non-negative-int argparse type; a negative floor would trivially pass the recall gate and silently weaken the contract (CodeRabbit). - docs/notes/corpus-benchmark.md: language tag on the output fence (MD040). selftest grows to 15 checks (the fail-fast guard + the arg validation). The honest 3/9 baseline is unchanged — those scans all exit 0. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Rg8kSk1YT14x7A1vo5zgED --- docs/notes/corpus-benchmark.md | 2 +- scripts/benchmark.py | 86 +++++++++++++++++++++++++++++----- 2 files changed, 74 insertions(+), 14 deletions(-) diff --git a/docs/notes/corpus-benchmark.md b/docs/notes/corpus-benchmark.md index 49a23c3a..7363fbae 100644 --- a/docs/notes/corpus-benchmark.md +++ b/docs/notes/corpus-benchmark.md @@ -16,7 +16,7 @@ extractor + core (`own-check.sh --format sarif`) and measures the two things the The aggregate is one defensible line. The **first measurement** (9 cases): -``` +```text benchmark: 3/9 bugs caught in real C# · 9/9 fixes clean · 0 false positive(s) on fixes ``` diff --git a/scripts/benchmark.py b/scripts/benchmark.py index 6f981316..f7d8d71b 100644 --- a/scripts/benchmark.py +++ b/scripts/benchmark.py @@ -112,15 +112,41 @@ def summarize(scores: list[CaseScore]) -> tuple[int, int, int, int]: # ---- the SDK-backed half (the real run) -------------------------------------- -def _own_check(root: str, cs_file: str) -> str: - """Run ``own-check.sh --format sarif`` over one .cs file; return its SARIF - stdout (build chatter goes to stderr, so stdout is a clean SARIF log).""" +class BenchmarkError(RuntimeError): + """own-check could not analyze a file — a hard failure (extractor crash, no + .NET SDK, drifted/bad facts: a non-zero return). The benchmark must fail + loudly rather than score an unanalyzed file as 'clean' and pass on output that + never actually ran.""" + + +def _verdicts_or_raise(stdout: str, returncode: int, cs_file: str, + stderr: str = "") -> set[str]: + """Verdict codes from a *completed* own-check run. own-check exits 0 for clean + AND for findings (it is run without --fail-on-finding), so any non-zero return + means the file was not analyzed — raise rather than treat empty/partial output + as 'no verdict' (Codex: a silent analysis failure must not read as a clean fix + or a hidden sub-floor miss).""" + if returncode != 0: + detail = stderr.strip() + raise BenchmarkError( + f"own-check failed (rc={returncode}) on {cs_file}" + + (f"\n{detail}" if detail else "")) + return sarif_codes(stdout) + + +def _scan(root: str, cs_file: str, timeout: int = 300) -> set[str]: + """Run own-check.sh over one .cs file and return its verdict codes (build + chatter goes to stderr, so stdout is a clean SARIF log). Raises BenchmarkError + on a hard own-check failure or a timeout (a hung extractor must not stall CI).""" script = os.path.join(root, "scripts", "own-check.sh") - proc = subprocess.run( - [script, "--root", root, "--format", "sarif", "--", cs_file], - capture_output=True, text=True, - ) - return proc.stdout + try: + proc = subprocess.run( + [script, "--root", root, "--format", "sarif", "--", cs_file], + capture_output=True, text=True, timeout=timeout, + ) + except subprocess.TimeoutExpired as e: + raise BenchmarkError(f"own-check timed out ({e.timeout}s) on {cs_file}") from e + return _verdicts_or_raise(proc.stdout, proc.returncode, cs_file, proc.stderr) def discover(corpus_dirs: list[str]) -> list[str]: @@ -145,8 +171,8 @@ def score_corpus(root: str, corpus_dirs: list[str]) -> list[CaseScore]: for d in discover(corpus_dirs): with open(os.path.join(d, "expected-diagnostics.txt"), encoding="utf-8") as f: expected = {w for w in f.read().split() if w} - before = sarif_codes(_own_check(root, os.path.join(d, "before.cs"))) - after = sarif_codes(_own_check(root, os.path.join(d, "after.cs"))) + before = _scan(root, os.path.join(d, "before.cs")) + after = _scan(root, os.path.join(d, "after.cs")) scores.append(CaseScore(os.path.basename(d), expected, before, after)) return scores @@ -173,7 +199,13 @@ def gate(caught: int, clean: int, total: int, fps: int, min_recall: int) -> list def run(root: str, corpus_dirs: list[str], min_recall: int = 0) -> int: """Score the corpus on real C#, print the scorecard, and apply the gate.""" - scores = score_corpus(root, corpus_dirs) + try: + scores = score_corpus(root, corpus_dirs) + except BenchmarkError as e: + # A hard own-check failure (extractor crash / no SDK / timeout) must fail + # the benchmark loudly — never pass on output that never actually ran. + print(f"BENCHMARK FAIL: {e}") + return 1 if not scores: print("BENCHMARK FAIL: no corpus cases found") return 1 @@ -252,13 +284,41 @@ def _selftest() -> int: if not ok: fails.append(f"gate: {msg}") + # 4) fail-fast guards: a hard own-check failure must raise, not score as clean; + # a negative recall floor must be rejected at the CLI. + ok_sarif = json.dumps({"runs": [{"results": [{"ruleId": "OWN001", "level": "error"}]}]}) + if _verdicts_or_raise(ok_sarif, 0, "f.cs") != {"OWN001"}: + fails.append("verdicts_or_raise: rc==0 must return the parsed codes") + try: + _verdicts_or_raise("", 2, "f.cs", "drifted facts") + fails.append("verdicts_or_raise: a non-zero own-check must raise BenchmarkError") + except BenchmarkError: + pass + if _non_negative_int("3") != 3: + fails.append("_non_negative_int: must accept a non-negative value") + try: + _non_negative_int("-1") + fails.append("_non_negative_int: must reject a negative value") + except argparse.ArgumentTypeError: + pass + for f in fails: print(f"SELFTEST FAIL: {f}") print(f"benchmark selftest: {'OK' if not fails else 'FAIL'} " - f"— sarif-parse + scoring + gate ({len(checks) + len(gate_checks) + 2} checks)") + f"— sarif-parse + scoring + gate + guards " + f"({len(checks) + len(gate_checks) + 6} checks)") return 1 if fails else 0 +def _non_negative_int(value: str) -> int: + """An argparse int type that rejects negatives — a negative recall floor would + trivially pass the gate and quietly weaken the regression contract.""" + n = int(value) + if n < 0: + raise argparse.ArgumentTypeError("--min-recall must be a non-negative integer") + return n + + def main(argv: list[str]) -> int: ap = argparse.ArgumentParser(description="Corpus benchmark for the Own.NET checker.") ap.add_argument("--selftest", action="store_true", @@ -266,7 +326,7 @@ def main(argv: list[str]) -> int: ap.add_argument("--root", default=None, help="repo root (default: this script's repo)") ap.add_argument("--corpus", action="append", default=None, metavar="DIR", help="corpus base dir(s) (default: corpus/real-world + corpus/wpf)") - ap.add_argument("--min-recall", type=int, default=0, metavar="N", + ap.add_argument("--min-recall", type=_non_negative_int, default=0, metavar="N", help="fail if fewer than N before.cs cases are caught (the pinned " "recall floor; specificity + zero-FP are always required)") args = ap.parse_args(argv) From 7a6dfa9e6e3508000d084fdce59379039747ecee Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 20 Jun 2026 11:51:06 +0000 Subject: [PATCH 4/4] =?UTF-8?q?fix(eval):=20benchmark=20hardens=20fail-lou?= =?UTF-8?q?d=20=E2=80=94=20launch=20failures=20+=20malformed=20SARIF=20(re?= =?UTF-8?q?view)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeRabbit, two more fail-loud holes in the benchmark's own-check path: - A process-launch failure (own-check.sh missing / not executable) raises OSError, which bypassed run()'s BenchmarkError handler. _scan now catches OSError and raises BenchmarkError, like the timeout path. - _verdicts_or_raise trusted rc==0 and handed stdout to the *permissive* sarif_codes parser, which returns an empty set for malformed/empty output — so an unanalyzed file with a 0 exit could score as a clean 'no verdict'. Now validate stdout is a well-formed SARIF log (a runs list) before parsing; a valid log with zero results stays the legitimate clean case. selftest grows to 17 checks (valid/empty/invalid SARIF, missing-script launch failure, the existing rc/gate/arg guards). The 3/9 baseline is unchanged — real scans exit 0 with well-formed SARIF. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Rg8kSk1YT14x7A1vo5zgED --- scripts/benchmark.py | 45 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/scripts/benchmark.py b/scripts/benchmark.py index f7d8d71b..94263c67 100644 --- a/scripts/benchmark.py +++ b/scripts/benchmark.py @@ -131,6 +131,17 @@ def _verdicts_or_raise(stdout: str, returncode: int, cs_file: str, raise BenchmarkError( f"own-check failed (rc={returncode}) on {cs_file}" + (f"\n{detail}" if detail else "")) + # rc==0 means clean OR findings — either way own-check emits a well-formed SARIF + # log (build_sarif always writes runs:[{...}], possibly with empty results). If + # stdout is not parseable SARIF, the file was not really analyzed; fail loudly + # rather than let the permissive parser score garbage as a clean 'no verdict' + # (CodeRabbit). A valid log with zero results is the legitimate clean case. + try: + doc = json.loads(stdout) + except (json.JSONDecodeError, ValueError) as e: + raise BenchmarkError(f"own-check emitted invalid SARIF on {cs_file}") from e + if not isinstance(doc, dict) or not isinstance(doc.get("runs"), list): + raise BenchmarkError(f"own-check emitted malformed SARIF (no runs) on {cs_file}") return sarif_codes(stdout) @@ -146,6 +157,10 @@ def _scan(root: str, cs_file: str, timeout: int = 300) -> set[str]: ) except subprocess.TimeoutExpired as e: raise BenchmarkError(f"own-check timed out ({e.timeout}s) on {cs_file}") from e + except OSError as e: + # own-check.sh missing / not executable / no interpreter — a launch failure + # must fail loud like any other, not escape run()'s BenchmarkError handler. + raise BenchmarkError(f"own-check could not start on {cs_file}: {e}") from e return _verdicts_or_raise(proc.stdout, proc.returncode, cs_file, proc.stderr) @@ -284,14 +299,29 @@ def _selftest() -> int: if not ok: fails.append(f"gate: {msg}") - # 4) fail-fast guards: a hard own-check failure must raise, not score as clean; - # a negative recall floor must be rejected at the CLI. + # 4) fail-fast guards: a hard own-check failure (bad rc, malformed SARIF, or a + # launch failure) must raise — never score garbage as clean; a valid empty + # log is the legitimate clean case; a negative recall floor is rejected. ok_sarif = json.dumps({"runs": [{"results": [{"ruleId": "OWN001", "level": "error"}]}]}) + empty_sarif = json.dumps({"runs": [{"results": []}]}) if _verdicts_or_raise(ok_sarif, 0, "f.cs") != {"OWN001"}: - fails.append("verdicts_or_raise: rc==0 must return the parsed codes") - try: - _verdicts_or_raise("", 2, "f.cs", "drifted facts") - fails.append("verdicts_or_raise: a non-zero own-check must raise BenchmarkError") + fails.append("verdicts_or_raise: rc==0 valid SARIF must return parsed codes") + if _verdicts_or_raise(empty_sarif, 0, "f.cs") != set(): + fails.append("verdicts_or_raise: a valid empty SARIF is a clean 'no verdict'") + raise_cases = [ + ("non-zero exit", ("", 2, "f.cs", "drift")), + ("invalid SARIF json", ("not json", 0, "f.cs")), + ("SARIF without runs", ("{}", 0, "f.cs")), + ] + for label, vargs in raise_cases: + try: + _verdicts_or_raise(*vargs) + fails.append(f"verdicts_or_raise: {label} must raise BenchmarkError") + except BenchmarkError: + pass + try: # a missing own-check.sh (launch failure) must also fail loud + _scan(os.path.join(os.sep, "no", "such", "ownnet-root"), "x.cs", timeout=5) + fails.append("_scan: a missing own-check.sh must raise BenchmarkError") except BenchmarkError: pass if _non_negative_int("3") != 3: @@ -302,11 +332,12 @@ def _selftest() -> int: except argparse.ArgumentTypeError: pass + guard_count = 2 + len(raise_cases) + 1 + 2 for f in fails: print(f"SELFTEST FAIL: {f}") print(f"benchmark selftest: {'OK' if not fails else 'FAIL'} " f"— sarif-parse + scoring + gate + guards " - f"({len(checks) + len(gate_checks) + 6} checks)") + f"({len(checks) + len(gate_checks) + guard_count} checks)") return 1 if fails else 0