From 79ecc6c69be450ec30ee6ba763e8f42fddd7e3b9 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 21:04:43 +0000 Subject: [PATCH 1/2] feat: cross-tool oracle comparison + related-work positioning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validate the leak check against mature C# analysers (Infer#, CodeQL) on the same code, and state honestly where Own.NET sits among them. - scripts/oracle_compare.py: diff Own.NET vs Infer#/CodeQL leak findings into agree / own-only / oracle-only buckets (plus Own.NET-only defect classes and oracle out-of-scope context). Reuses the miner's own-check parser; one SARIF parser for both oracles; basename+line matching (--line-tol); --json output; --selftest (9/9). - .github/workflows/oracle.yml: workflow_dispatch running own-check + CodeQL (build-mode none) + Infer# (built binaries) over a target, diffing to the run summary + an artifact. Each oracle step is continue-on-error -- the oracles need the target to build, our extractor does not. - docs/notes/oracle.md: methodology, the three buckets, how to read it, gaps. - README: "Related work / позиционирование" -- Infer#/CodeQL/IDisposableAnalyzers/ CA2000 are the recall bar and the oracle; our difference is borrow + region + double-dispose in one model. Closest same-idea-different-language is C++/Rust lifetime work (Polonius). - ci.yml: run the miner + oracle selftests on every push, not just on dispatch. Verified locally: ruff clean (whole tree), oracle selftest 9/9, miner 7/7, run_tests.py exit 0 (analysis 125/125, fuzz 3000, gallery/corpus/wpf/loops/spec/ ownir all green). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Rg8kSk1YT14x7A1vo5zgED --- .github/workflows/ci.yml | 7 + .github/workflows/oracle.yml | 153 ++++++++++++ README.md | 50 ++++ docs/notes/oracle.md | 101 ++++++++ scripts/oracle_compare.py | 445 +++++++++++++++++++++++++++++++++++ 5 files changed, 756 insertions(+) create mode 100644 .github/workflows/oracle.yml create mode 100644 docs/notes/oracle.md create mode 100644 scripts/oracle_compare.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a88e00d..fa78cec6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,13 @@ jobs: run: ruff check . - name: mypy --strict (ownlang) run: mypy + # The evaluation scripts (corpus miner, cross-tool oracle diff) carry + # embedded fixtures; run their selftests here so the parsers/aggregators + # stay honest on every push, not only on workflow_dispatch. + - name: script selftests (miner + oracle) + run: | + python scripts/mine_report.py --selftest + python scripts/oracle_compare.py --selftest tests: name: tests (py${{ matrix.python-version }}) diff --git a/.github/workflows/oracle.yml b/.github/workflows/oracle.yml new file mode 100644 index 00000000..ade7e7d6 --- /dev/null +++ b/.github/workflows/oracle.yml @@ -0,0 +1,153 @@ +name: oracle (cross-tool) + +# On-demand cross-tool validation: run Own.NET's leak check, Infer#, and CodeQL +# over the SAME public C# repo and diff their leak-class findings into an +# agreement report (scripts/oracle_compare.py). Evaluation tooling — the mature +# detectors are both a recall bar and an oracle. See docs/notes/oracle.md. +# +# Trigger from the Actions tab ("Run workflow") or the API. Inputs reach the +# shell via env (never interpolated into a `run:` script) to avoid injection. +# +# Honest notes: +# * Own.NET needs no build (error-tolerant SemanticModel). The two oracles do: +# CodeQL builds a database (build-mode: none, from source); Infer# analyses +# compiled .dll+.pdb, so the target must `dotnet build`. Each oracle step is +# continue-on-error, so a build failure still yields a partial report. +# * The diff core (oracle_compare.py) is unit-tested (--selftest, run first); +# this orchestration is validated on dispatch, like mine.yml. + +on: + workflow_dispatch: + inputs: + repo: + description: "Target: owner/repo (e.g. DapperLib/Dapper) or a git URL" + required: true + ref: + description: "Branch / tag / sha to analyse (optional, default: repo HEAD)" + required: false + default: "" + paths: + description: "Subdir to scan with own-check (optional, default: whole repo)" + required: false + default: "" + build: + description: "Project/solution under the target to `dotnet build` for Infer# (optional, default: repo root)" + required: false + default: "" + +permissions: + contents: read + security-events: write # required by the CodeQL action internals (upload is off) + +jobs: + oracle: + name: oracle ${{ inputs.repo }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: "8.0.x" + + # Fast fail: the diff logic is unit-tested before we clone/build anything. + - name: Comparator selftest + run: python scripts/oracle_compare.py --selftest + + - name: Clone the target + env: + REPO: ${{ inputs.repo }} + REF: ${{ inputs.ref }} + run: | + case "$REPO" in + http://*|https://*|git@*) url="$REPO" ;; + *) url="/${REPO}.git" ;; + esac + args=(--depth 1 --quiet) + [[ -n "$REF" ]] && args+=(--branch "$REF") + git clone "${args[@]}" "$url" target + echo "COMMIT=$(git -C target rev-parse HEAD)" >> "$GITHUB_ENV" + + # Own.NET — no build needed; scans .cs directly. + - name: Own.NET own-check + env: + PATHS: ${{ inputs.paths }} + run: | + scan="target"; [[ -n "$PATHS" ]] && scan="target/$PATHS" + set +e + scripts/own-check.sh --format human -- "$scan" > own.txt 2> own-extract.log + echo "own-check rc=$? ; $(wc -l < own.txt) finding line(s)" + + # CodeQL — database from source (no build), default queries; we filter to + # the dispose/leak family in the comparator. + - name: CodeQL init + uses: github/codeql-action/init@v3 + continue-on-error: true + with: + languages: csharp + build-mode: none + source-root: target + - name: CodeQL analyze + uses: github/codeql-action/analyze@v3 + continue-on-error: true + with: + category: ownnet-oracle + output: codeql-out + upload: false + + # Infer# — needs compiled binaries; build the target into one output dir. + - name: Build the target (for Infer#) + env: + BUILD: ${{ inputs.build }} + continue-on-error: true + run: | + tgt="target"; [[ -n "$BUILD" ]] && tgt="target/$BUILD" + if dotnet build "$tgt" -c Release -o _bin -v quiet; then + echo "BUILD_OK=1" >> "$GITHUB_ENV" + else + echo "target build failed — Infer# will be skipped, report stays partial" + fi + - name: Run Infer# + if: env.BUILD_OK == '1' + uses: microsoft/infersharpaction@v1.5 + continue-on-error: true + with: + binary-path: _bin + + - name: Diff Own.NET vs the oracles + if: always() + env: + REPO: ${{ inputs.repo }} + run: | + args=(--own own.txt --target "$REPO" --commit "${COMMIT:-}" + --strip "$GITHUB_WORKSPACE/target" --strip target + --json report.json) + [[ -f infer-out/report.sarif ]] && args+=(--infersharp infer-out/report.sarif) + cq=$(find codeql-out -name '*.sarif' -type f 2>/dev/null | head -1 || true) + [[ -n "$cq" ]] && args+=(--codeql "$cq") + python scripts/oracle_compare.py "${args[@]}" > report.md + cat report.md + + - name: Publish the report to the run summary + if: always() + run: | + if [[ -s report.md ]]; then + cat report.md >> "$GITHUB_STEP_SUMMARY" + else + echo "no report produced (see the Diff step log)" >> "$GITHUB_STEP_SUMMARY" + fi + - name: Upload the report and raw outputs + if: always() + uses: actions/upload-artifact@v4 + with: + name: oracle-report + path: | + report.md + report.json + own.txt + own-extract.log + infer-out/report.sarif + codeql-out/*.sarif + if-no-files-found: warn diff --git a/README.md b/README.md index d29255b8..e85faa44 100644 --- a/README.md +++ b/README.md @@ -696,6 +696,56 @@ OWN010 в новой схеме занят «maybe-move».) --- +## Related work / позиционирование + +Честно: **мы не первый и не лучший детектор утечек ресурсов для C#.** Ниша плотно +занята зрелыми инструментами, и притворяться иначе — ровно та декорация, против +которой вся затея: + +| Инструмент | Что ловит | Как | +|---|---|---| +| **Infer#** (Microsoft, на базе Facebook Infer) | resource leak, null-deref, thread-safety, taint | интерпроцедурно, separation logic; по скомпилированным `.dll`+`.pdb` | +| **CodeQL** `cs/local-not-disposed` | локальный `IDisposable` без `Dispose` | dataflow по собранной CodeQL-базе | +| **IDisposableAnalyzers** (`IDISP0xx`) | dispose / ownership-transfer | Roslyn-аналайзер (синтаксис + символы), в IDE | +| **CA2000 / CA2213** (.NET SDK analyzers) | dispose до выхода из scope; недиспоженные поля | flow-sensitive, но transfer-распознавание — список типов | +| **SonarC#, PVS-Studio (V3178), ReSharper `[MustDisposeResource]`** | dispose-утечки | паттерны / аннотации | + +Все они сильнее нас в **leak-recall** на больших базах: интерпроцедурные, +обстрелянные, без нашего «honest skip». Это **планка**, и мы это признаём. + +**Чем мы отличаемся — модель, а не охват.** Перечисленные инструменты по сути +отвечают на один вопрос: *«этот `IDisposable` освобождён?»*. Own.NET моделирует +**владение целиком** в духе Rust — и из этого выпадают классы дефектов, которых у +leak-only инструментов нет в их основном запросе: + +- **double-dispose (OWN003)** и **use-after-dispose (OWN002)** — отдельные коды, + не «leak». В leak-запросах Infer#/CodeQL их попросту нет. +- **loans + permissions (OWN006–013)** — алиасинг и эксклюзивность borrow'ов + (mutable-while-shared и пр.). Ни один C#-инструмент не делает этого для + `IDisposable`; C#-ный `ref safety` / `scoped` / `Span` — это escape-safety для + ref/span-значений, не ownership ресурсов, и пересечения почти нет. +- **region/lifetime escape (OWN014)** — промоушн объекта в более долгоживущий + регион (zombie-ViewModel). Это lifetime-анализ, а не dispose-чек. + +Ближайший «та же идея, другой язык» — не в C#, а в C++/Rust: **C++ Lifetime +profile** (Sutter / MSVC, opt-in), экспериментальная **lifetime-safety в Clang** +(2025, вдохновлённая Polonius) и сам **Polonius** — Datalog-формулировка +borrow-чека Rust ([rust-lang/polonius](https://github.com/rust-lang/polonius)). +Их факты (`loan_issued_at` / `cfg_edge` / `loan_killed_at` / `subset`) — ровно тот +словарь, в котором написан наш OwnIR (`acquire`/`use`/`release`/`return` + +back-edge); мы воспроизводим **region-based** модель, просто на другом движке +(питоновский worklist-fixpoint вместо Datalog). Попытки «Rust-подобного C#» +(вроде RLC#) — заброшены. + +**А ещё зрелые детекторы для нас — оракул.** Раз они сильны в leak-detection, их +можно гонять на том же коде и сверять находки: пересечение = high-confidence, +*only-oracle* = наш recall-gap (что пропустили), *only-own* = кандидат в FP +**или** уникальный улов (тот самый double-dispose). Это валидационный харнес +поверх mining — `scripts/oracle_compare.py` + workflow **oracle (cross-tool)**, +подробности в [`docs/notes/oracle.md`](docs/notes/oracle.md). + +--- + ## Структура ``` diff --git a/docs/notes/oracle.md b/docs/notes/oracle.md new file mode 100644 index 00000000..e892c3da --- /dev/null +++ b/docs/notes/oracle.md @@ -0,0 +1,101 @@ +# Oracle comparison — validating the leak check against Infer# and CodeQL + +We are **not** the first resource-leak detector for C#. Infer# (Microsoft, on +Facebook Infer) and CodeQL (`cs/local-not-disposed`) are mature, interprocedural, +battle-tested. That is precisely what makes them useful here: run all three over +the **same** repo and diff the leak-class findings. Cross-tool agreement is a +strong correctness signal; disagreement points straight at our precision or +recall gaps. This is evaluation tooling — a companion to corpus mining +([`mining.md`](mining.md)), with an external reference instead of just our own +verdict. + +## The three buckets + +Restricted to the comparable class — *resource leak / not disposed* (OWN001 vs +Infer#'s `DOTNET_RESOURCE_LEAK` vs CodeQL's `cs/local-not-disposed` & friends): + +| bucket | meaning | what to do | +|---|---|---| +| **agree** | a `(file, line)` flagged by Own.NET **and** an oracle | high confidence — nothing, this is the win | +| **own-only** | flagged by us, by no oracle | triage: a candidate **false positive** to harden, *or* a real catch the oracle's leak query can't express | +| **oracle-only** | flagged by an oracle, not by us | our **recall gap** — reduce to a minimal `.cs`, then model it or record it as a known limitation | + +Two classes sit **outside** the three-way diff and are reported separately: + +- **Own.NET-only defect classes** — `OWN002` (use-after-dispose) and `OWN003` + (double-dispose). The oracle *leak* queries have no equivalent, so counting + them as "own-only leaks" would be misleading. They are a feature, not noise. +- **Oracle findings outside our scope** — Infer#'s `NULL_DEREFERENCE`, + thread-safety, taint, etc. Listed as context (counts by rule), not a gap. + +## Why this is a fair-but-honest comparison + +- **Own.NET needs no build.** The Roslyn extractor reads a best-effort + `SemanticModel` without `dotnet restore`/build (unresolved externals become an + honest `OWN050`, not a guess). **Both oracles need the target to build**: + CodeQL constructs a database (here via `build-mode: none`, from source), Infer# + analyses compiled `.dll`+`.pdb`. So the oracle run can fail where ours doesn't + — that asymmetry is the point, and each oracle step is `continue-on-error` so a + build failure still yields a partial report. +- **Path/line matching is deliberately loose.** Tools disagree on the exact line + (allocation site vs declaration) and on path prefixes. The comparator matches + on **basename + a line window** (`--line-tol`, default 3). Robust to prefixes; + same-named files in different dirs can theoretically collide (rare — the line + disambiguates). The file-level overlap is the most robust signal. + +## Run it + +In CI (no local Infer#/CodeQL/Docker needed) — Actions tab → **oracle +(cross-tool)** → *Run workflow*. The report lands in the run summary and as an +artifact (`report.md`, `report.json`, plus each tool's raw output): + +```text +inputs: repo = DapperLib/Dapper ref = (optional) + paths = (optional own-check subdir) build = (optional proj/sln for Infer#) +``` + +The diff core runs anywhere on already-produced outputs (this is what `--selftest` +exercises, and it gates CI): + +```sh +python scripts/oracle_compare.py \ + --own own.txt \ + --infersharp infer-out/report.sarif \ + --codeql codeql-out/csharp.sarif \ + --strip "$PWD/target" \ + --target DapperLib/Dapper --commit "$SHA" --json report.json +``` + +`--own` is `own-check`'s human output (same format the miner reads). The two +oracle inputs are SARIF — Infer# and CodeQL both emit it, so one parser handles +both. Extra SARIF oracles can be added with `--sarif tool=path`. + +## What "agree" buys us, concretely + +The first mine of Dapper found **zero** real leaks (a well-disciplined library). +A clean run is a precision signal — but on its own it can't tell "we correctly +found nothing" from "we silently skipped everything". The oracle closes that: + +- if the oracles also find ~nothing → genuine agreement, the codebase is clean; +- if the oracles find leaks we missed → **oracle-only**, a concrete recall target + (likely interprocedural, a field, or a `for`/`do`/`try` shape we honestly skip); +- if we flag something they don't → **own-only**, either a precision bug to fix or + a defect class (double-dispose) they don't model. + +Pair this with the extractor's planned `--stats` coverage (methods analysed vs +skipped) and the picture is complete: how much we looked at, and how our verdicts +line up with two independent engines. + +## Honest gaps (v1) + +- **No tool versions pinned in the report yet.** `microsoft/infersharpaction@v1.5` + and `github/codeql-action@v3` float on tags; the report header names the tools + but not exact analyser versions. A later pass can stamp them. +- **CodeQL runs the default suite, filtered in the comparator** (rather than a + single-query pack). Simpler and robust to suite/version drift; the filter keys + on the dispose/leak rule family. +- **One target, by hand.** Same discipline as mining: a deliberate spot-check, + not a crawler. Be a good citizen (shallow, read-only). +- **Agreement is necessary, not sufficient.** Two tools can share a blind spot. + The oracle raises confidence; it does not prove soundness (that is the Boogie/ + Dafny backend's job, still roadmap). diff --git a/scripts/oracle_compare.py b/scripts/oracle_compare.py new file mode 100644 index 00000000..2e96e6ea --- /dev/null +++ b/scripts/oracle_compare.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python3 +""" +Oracle comparison — cross-check Own.NET's leak findings against mature C# +analysers (Infer#, CodeQL) on the same codebase. See docs/notes/oracle.md. + +We are *not* the first resource-leak detector: Infer# and CodeQL are strong at +it. That is exactly what makes them an **oracle** — run all three on one repo and +diff the leak-class findings: + + - agree a (file, line) flagged by Own.NET AND an oracle -> high confidence + - own-only flagged by us, by no oracle -> candidate false positive, OR a + defect class the oracle's leak query can't express (double-dispose) + - oracle-only flagged by an oracle, not by us -> our recall gap (what we missed) + +Own.NET findings come from `own-check` (human format; same parser as the miner). +Infer# and CodeQL both emit SARIF, so a single parser reads both oracles. + +dotnet-free: the three tools run upstream (the oracle workflow / CI); this only +reads their outputs and diffs them, so it runs anywhere. `--selftest` exercises +the diff on embedded fixtures. + +Usage: + oracle_compare.py --own own.txt --infersharp infer-out/report.sarif \\ + --codeql codeql.sarif --strip "$PWD/target" \\ + --target owner/repo --commit SHA [--json out.json] + oracle_compare.py --selftest +""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections import Counter +from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + + +def _load_titles() -> dict[str, str]: + """OWN-code -> human title, for labelling Own.NET-only defect classes. Imported + from the core when this runs inside the checkout; an empty map is a fine + fallback when it is not importable.""" + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + try: + from ownlang.diagnostics import TITLES + except Exception: + return {} + return dict(TITLES) + + +TITLES = _load_titles() + +# Which rule of each tool is the comparable "resource leak / not disposed" class. +# Only these are diffed three ways; everything else is reported as context. +OWN_LEAK = {"OWN001"} # owned resource not released on a path +OWN_USE_AFTER = {"OWN002", "OWN009"} # use after release (definite / maybe) +OWN_DOUBLE = {"OWN003"} # double release +INFER_LEAK = {"DOTNET_RESOURCE_LEAK", "RESOURCE_LEAK", "MEMORY_LEAK"} +# CodeQL ids vary by suite/version; match the dispose/leak family by id too. +CODEQL_LEAK = { + "cs/local-not-disposed", + "cs/missing-dispose", + "cs/dispose-not-called-on-throw", + "cs/late-dispose", +} + + +@dataclass +class Finding: + tool: str # "own" | "infersharp" | "codeql" | ... + path: str # normalised, repo-relative where possible + line: int + rule: str # OWN001 / DOTNET_RESOURCE_LEAK / cs/local-not-disposed / ... + message: str + cls: str # "leak" | "use-after" | "double" | "other" + fkey: str = field(init=False, default="") + + def __post_init__(self) -> None: + # File identity for cross-tool matching is the basename, lower-cased: + # robust to the path-prefix differences between tools (one reports + # `src/Dapper/X.cs`, another `Dapper/X.cs`, a third an absolute path). + self.fkey = self.path.lower().rsplit("/", 1)[-1] + + +def _own_class(code: str) -> str: + if code in OWN_LEAK: + return "leak" + if code in OWN_USE_AFTER: + return "use-after" + if code in OWN_DOUBLE: + return "double" + return "other" + + +def _oracle_class(tool: str, rule: str) -> str: + r = rule or "" + if tool == "infersharp": + return "leak" if r in INFER_LEAK else "other" + # codeql (and any other SARIF oracle): the dispose/leak family by id. + rl = r.lower() + if r in CODEQL_LEAK or "not-disposed" in rl or "dispose" in rl: + return "leak" + return "other" + + +def norm_path(raw: str, strips: list[str]) -> str: + """Normalise a finding path to a repo-relative-ish form: forward slashes, no + `file://` scheme, and the longest matching `--strip` prefix removed. Matching + ultimately keys on the basename, so this is mostly for readable output.""" + p = raw.replace("\\", "/") + for scheme in ("file://", "file:"): + if p.startswith(scheme): + p = p[len(scheme):] + prefixes = sorted((s.replace("\\", "/").rstrip("/") for s in strips), + key=len, reverse=True) + for pre in prefixes: + if pre and p.startswith(pre): + p = p[len(pre):] + break + if p.startswith("./"): + p = p[2:] + return p.lstrip("/") + + +def build_own(text: str, strips: list[str]) -> list[Finding]: + """Parse own-check human output into Findings (reusing the miner's parser).""" + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from mine_report import parse # local import: scripts/ is on the path now + raw, _ = parse(text) + return [ + Finding("own", norm_path(f["file"], strips), int(f["line"]), + f["code"], f["message"], _own_class(f["code"])) + for f in raw + ] + + +def _first_location(res: dict[str, Any]) -> tuple[str, int] | None: + """The primary (uri, startLine) of a SARIF result, or None if it has none.""" + for loc in res.get("locations", []): + phys = loc.get("physicalLocation") or {} + uri = (phys.get("artifactLocation") or {}).get("uri") + if not uri: + continue + line = (phys.get("region") or {}).get("startLine") + return uri, line if isinstance(line, int) else 0 + return None + + +def parse_sarif(text: str, tool: str, strips: list[str]) -> list[Finding]: + """Parse a SARIF log (Infer# or CodeQL) into Findings.""" + data = json.loads(text) + out: list[Finding] = [] + for run in data.get("runs", []): + for res in run.get("results", []): + rule = res.get("ruleId") or (res.get("rule") or {}).get("id") or "" + msg = ((res.get("message") or {}).get("text") or "").strip() + loc = _first_location(res) + if loc is None: + continue + uri, line = loc + out.append(Finding(tool, norm_path(uri, strips), line, rule, msg, + _oracle_class(tool, rule))) + return out + + +def compare(own: list[Finding], oracles: list[Finding], tol: int) -> dict[str, Any]: + """Bucket the leak-class findings three ways (agree / own-only / oracle-only), + plus file overlap, Own.NET-only defect classes, and out-of-scope oracle hits.""" + own_leak = [f for f in own if f.cls == "leak"] + ora_leak = [g for g in oracles if g.cls == "leak"] + + def near(a: Finding, b: Finding) -> bool: + return a.fkey == b.fkey and abs(a.line - b.line) <= tol + + agree: list[tuple[Finding, list[Finding]]] = [] + own_only: list[tuple[Finding, list[Finding]]] = [] + for f in own_leak: + hits = [g for g in ora_leak if near(f, g)] + (agree if hits else own_only).append((f, hits)) + oracle_only = [g for g in ora_leak if not any(near(f, g) for f in own_leak)] + + own_files = {f.fkey for f in own_leak} + ora_files = {g.fkey for g in ora_leak} + return { + "own_leak": own_leak, + "oracle_leak": ora_leak, + "agree": agree, + "own_only": own_only, + "oracle_only": oracle_only, + "own_unique": [f for f in own if f.cls in ("use-after", "double")], + "oracle_other": [g for g in oracles if g.cls == "other"], + "files_both": own_files & ora_files, + "files_own_only": own_files - ora_files, + "files_oracle_only": ora_files - own_files, + } + + +def _fmt_files(s: set[str], cap: int = 12) -> str: + items = sorted(s) + shown = ", ".join(f"`{x}`" for x in items[:cap]) + if len(items) > cap: + shown += f", … (+{len(items) - cap})" + return shown or "—" + + +def render_md(result: dict[str, Any], target: str, commit: str, + oracles: list[str], tol: int, max_list: int = 50) -> str: + """The human-facing comparison report.""" + own_leak = result["own_leak"] + ora_leak = result["oracle_leak"] + agree = result["agree"] + own_only = result["own_only"] + oracle_only = result["oracle_only"] + own_unique = result["own_unique"] + oracle_other = result["oracle_other"] + by_tool = Counter(g.tool for g in ora_leak) + + out: list[str] = [ + f"# Oracle comparison — `{target or '?'}`", + "", + f"- commit: `{commit or '?'}`", + f"- generated: {datetime.now(UTC):%Y-%m-%d %H:%M UTC}", + f"- tools: Own.NET + {', '.join(oracles) or '(no oracle SARIF supplied)'}", + f"- file match: basename + line within ±{tol}", + "", + "Leak / not-disposed class only — the question all three tools can answer. " + "Own.NET's use-after-release and double-release are listed separately: the " + "oracle leak queries have no equivalent.", + "", + "## Leak-class totals", + "", + "| tool | leak findings |", + "|---|---:|", + f"| Own.NET | {len(own_leak)} |", + ] + out += [f"| {t} | {by_tool.get(t, 0)} |" for t in oracles] + + out += ["", f"## Agree — {len(agree)} (Own.NET and an oracle; high confidence)", ""] + out += ["_(none)_"] if not agree else [ + f"- `{f.path}:{f.line}` **[{f.rule}]** — also: " + f"{', '.join(sorted({h.tool for h in hits}))}" + for f, hits in agree[:max_list] + ] + + out += ["", f"## Own.NET only — {len(own_only)} " + "(candidate FP, or a catch the oracle misses)", ""] + out += ["_(none)_"] if not own_only else [ + f"- `{f.path}:{f.line}` **[{f.rule}]** {f.message}" for f, _ in own_only[:max_list] + ] + + out += ["", f"## Oracle only — {len(oracle_only)} (our recall gap, or an oracle FP)", ""] + out += ["_(none)_"] if not oracle_only else [ + f"- `{g.path}:{g.line}` **[{g.tool}:{g.rule}]** {g.message}" + for g in oracle_only[:max_list] + ] + + fb, fo, fx = (result["files_both"], result["files_own_only"], + result["files_oracle_only"]) + out += [ + "", "## File overlap (leak class)", "", + f"- both: {len(fb)} — {_fmt_files(fb)}", + f"- Own.NET only: {len(fo)} — {_fmt_files(fo)}", + f"- oracle only: {len(fx)} — {_fmt_files(fx)}", + ] + + out += ["", f"## Own.NET-only defect classes — {len(own_unique)} " + "(no oracle leak-query equivalent)", ""] + if not own_unique: + out += ["_(none)_"] + else: + out += [f"- **{rule}** x{n} — {TITLES.get(rule, '')}" + for rule, n in Counter(f.rule for f in own_unique).most_common()] + + out += ["", f"## Oracle findings outside our leak scope — {len(oracle_other)} (context)", ""] + if not oracle_other: + out += ["_(none)_"] + else: + out += [f"- {k} x{n}" for k, n in + Counter(f"{g.tool}:{g.rule}" for g in oracle_other).most_common(20)] + + out += [ + "", "## How to read this", "", + "- **Agree** is the high-confidence set: two independent models flag the same spot.", + "- **Own.NET only** is the triage queue — each is a candidate false positive to " + "harden, *or* a real catch the oracle's leak query can't express (double-dispose, " + "use-after-dispose, a non-allowlisted owning type).", + "- **Oracle only** is our recall gap: reduce one to a minimal `.cs`, decide if it " + "is in scope (interprocedural? a field? a loop/`try` shape we skip?), then model " + "it or record it as a known limitation.", + "- File match is by basename + a line window, so cross-tool path prefixes do not " + "matter; same-named files in different directories can theoretically collide " + "(rare — the line number disambiguates in practice).", + "", + ] + return "\n".join(out) + + +def _fd(f: Finding) -> dict[str, Any]: + return {"tool": f.tool, "path": f.path, "line": f.line, + "rule": f.rule, "cls": f.cls, "message": f.message} + + +def to_json(result: dict[str, Any], target: str, commit: str) -> dict[str, Any]: + return { + "target": target, + "commit": commit, + "totals": { + "own_leak": len(result["own_leak"]), + "oracle_leak": len(result["oracle_leak"]), + "agree": len(result["agree"]), + "own_only": len(result["own_only"]), + "oracle_only": len(result["oracle_only"]), + "own_unique": len(result["own_unique"]), + "oracle_other": len(result["oracle_other"]), + }, + "agree": [{"finding": _fd(f), "oracles": [_fd(h) for h in hits]} + for f, hits in result["agree"]], + "own_only": [_fd(f) for f, _ in result["own_only"]], + "oracle_only": [_fd(g) for g in result["oracle_only"]], + "own_unique": [_fd(f) for f in result["own_unique"]], + "oracle_other": [_fd(g) for g in result["oracle_other"]], + "files": { + "both": sorted(result["files_both"]), + "own_only": sorted(result["files_own_only"]), + "oracle_only": sorted(result["files_oracle_only"]), + }, + } + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser( + description="Diff Own.NET leak findings against Infer#/CodeQL (oracle).") + ap.add_argument("--own", help="own-check human output file") + ap.add_argument("--infersharp", help="Infer# SARIF (e.g. infer-out/report.sarif)") + ap.add_argument("--codeql", help="CodeQL SARIF") + ap.add_argument("--sarif", action="append", default=[], metavar="TOOL=PATH", + help="extra SARIF oracle as tool=path (repeatable)") + ap.add_argument("--strip", action="append", default=[], metavar="PREFIX", + help="path prefix to strip from finding paths (repeatable)") + ap.add_argument("--target", default="", help="owner/repo (report header)") + ap.add_argument("--commit", default="", help="commit SHA (report header)") + ap.add_argument("--line-tol", type=int, default=3, + help="line window for a cross-tool match (default 3)") + ap.add_argument("--json", dest="json_out", default="", + help="also write the structured comparison as JSON") + ap.add_argument("--selftest", action="store_true", + help="run built-in diff checks and exit") + args = ap.parse_args(argv) + + if args.selftest: + return _selftest() + + own = build_own(Path(args.own).read_text(encoding="utf-8"), args.strip) if args.own else [] + sarif_inputs: list[tuple[str, str]] = [] + if args.infersharp: + sarif_inputs.append(("infersharp", args.infersharp)) + if args.codeql: + sarif_inputs.append(("codeql", args.codeql)) + for spec in args.sarif: + tool, _, path = spec.partition("=") + if not path: + ap.error(f"--sarif expects tool=path, got {spec!r}") + sarif_inputs.append((tool, path)) + + oracles: list[Finding] = [] + present: list[str] = [] + for tool, path in sarif_inputs: + present.append(tool) + oracles += parse_sarif(Path(path).read_text(encoding="utf-8"), tool, args.strip) + + result = compare(own, oracles, args.line_tol) + if args.json_out: + Path(args.json_out).write_text( + json.dumps(to_json(result, args.target, args.commit), indent=2), + encoding="utf-8") + print(render_md(result, args.target, args.commit, present, args.line_tol)) + return 0 + + +def _selftest() -> int: + own_txt = ( + "src/A.cs:12: error: [OWN001] IDisposable local 'a' is never disposed " + "(leak) [resource: disposable]\n" + "src/B.cs:5: error: [OWN001] IDisposable local 'b' is never disposed " + "(leak) [resource: disposable]\n" + "src/C.cs:9: error: [OWN003] 'c' is disposed twice [resource: disposable]\n" + "src/D.cs:3: warning: [OWN050] cannot verify 'X.Y' — unresolved " + "[resource: unresolved reference]\n" + ) + infer = json.dumps({"runs": [{"results": [ + {"ruleId": "DOTNET_RESOURCE_LEAK", "message": {"text": "resource leak"}, + "locations": [{"physicalLocation": { + "artifactLocation": {"uri": "src/A.cs"}, "region": {"startLine": 12}}}]}, + {"ruleId": "NULL_DEREFERENCE", "message": {"text": "npe"}, + "locations": [{"physicalLocation": { + "artifactLocation": {"uri": "src/D.cs"}, "region": {"startLine": 3}}}]}, + ]}]}) + codeql = json.dumps({"runs": [{"results": [ + {"ruleId": "cs/local-not-disposed", "message": {"text": "not disposed"}, + "locations": [{"physicalLocation": { + "artifactLocation": {"uri": "A.cs"}, "region": {"startLine": 13}}}]}, + {"ruleId": "cs/local-not-disposed", "message": {"text": "not disposed"}, + "locations": [{"physicalLocation": { + "artifactLocation": {"uri": "E.cs"}, "region": {"startLine": 7}}}]}, + ]}]}) + + own = build_own(own_txt, []) + oracles = parse_sarif(infer, "infersharp", []) + parse_sarif(codeql, "codeql", []) + r = compare(own, oracles, tol=3) + + fails: list[str] = [] + if len(r["own_leak"]) != 2: + fails.append(f"own_leak: expected 2, got {len(r['own_leak'])}") + if len(r["agree"]) != 1: + fails.append(f"agree: expected 1, got {len(r['agree'])}") + elif {h.tool for h in r["agree"][0][1]} != {"infersharp", "codeql"}: + fails.append(f"agree oracles wrong: {[h.tool for h in r['agree'][0][1]]}") + if [f.fkey for f, _ in r["own_only"]] != ["b.cs"]: + fails.append(f"own_only wrong: {[f.fkey for f, _ in r['own_only']]}") + if [g.fkey for g in r["oracle_only"]] != ["e.cs"]: + fails.append(f"oracle_only wrong: {[g.fkey for g in r['oracle_only']]}") + if [f.rule for f in r["own_unique"]] != ["OWN003"]: + fails.append(f"own_unique wrong: {[f.rule for f in r['own_unique']]}") + if [g.rule for g in r["oracle_other"]] != ["NULL_DEREFERENCE"]: + fails.append(f"oracle_other wrong: {[g.rule for g in r['oracle_other']]}") + if r["files_both"] != {"a.cs"}: + fails.append(f"files_both wrong: {r['files_both']}") + md = render_md(r, "o/r", "abc123", ["infersharp", "codeql"], 3) + if "Oracle comparison" not in md or "## Agree" not in md: + fails.append("markdown render missing sections") + js = to_json(r, "o/r", "abc123") + if js["totals"]["agree"] != 1 or js["totals"]["oracle_only"] != 1: + fails.append(f"json totals wrong: {js['totals']}") + + total = 9 + for f in fails: + print(f"ORACLE SELFTEST FAIL: {f}") + print(f"oracle_compare selftest: {total - len(fails)}/{total} checks passed") + return 1 if fails else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 5a533fab431f071553100d92f9bff77aaa5053c5 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 21:13:58 +0000 Subject: [PATCH 2/2] fix: address CodeRabbit review on the oracle harness (#21) - oracle.yml: clone then fetch + checkout the ref, so a raw commit SHA works as the `ref` input documents (was `git clone --branch`, which only takes a branch/tag and would fail a SHA dispatch). - oracle_compare.py: surface own-check parser drift instead of dropping it. build_own now returns the unparsed-line count; main warns on stderr and the report header flags it. Chose a non-fatal note over a hard raise: this is eval tooling run over third-party repos, so one stray stdout line shouldn't sink the whole comparison (the miner surfaces its unparsed count the same way). Selftest is now 12/12 and covers the drift path. Verified: ruff clean (whole tree), oracle selftest 12/12, run_tests.py exit 0. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Rg8kSk1YT14x7A1vo5zgED --- .github/workflows/oracle.yml | 9 ++++--- scripts/oracle_compare.py | 47 ++++++++++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/.github/workflows/oracle.yml b/.github/workflows/oracle.yml index ade7e7d6..3a6a81dd 100644 --- a/.github/workflows/oracle.yml +++ b/.github/workflows/oracle.yml @@ -65,9 +65,12 @@ jobs: http://*|https://*|git@*) url="$REPO" ;; *) url="/${REPO}.git" ;; esac - args=(--depth 1 --quiet) - [[ -n "$REF" ]] && args+=(--branch "$REF") - git clone "${args[@]}" "$url" target + git clone --depth 1 --quiet "$url" target + if [[ -n "$REF" ]]; then + # --branch only accepts a branch/tag; fetch+checkout also takes a raw SHA. + git -C target fetch --depth 1 --quiet origin "$REF" + git -C target checkout --quiet --detach FETCH_HEAD + fi echo "COMMIT=$(git -C target rev-parse HEAD)" >> "$GITHUB_ENV" # Own.NET — no build needed; scans .cs directly. diff --git a/scripts/oracle_compare.py b/scripts/oracle_compare.py index 2e96e6ea..29d2993f 100644 --- a/scripts/oracle_compare.py +++ b/scripts/oracle_compare.py @@ -124,16 +124,19 @@ def norm_path(raw: str, strips: list[str]) -> str: return p.lstrip("/") -def build_own(text: str, strips: list[str]) -> list[Finding]: - """Parse own-check human output into Findings (reusing the miner's parser).""" +def build_own(text: str, strips: list[str]) -> tuple[list[Finding], int]: + """Parse own-check human output into Findings (reusing the miner's parser). + Also returns the unparsed-line count, so drift in the own-check format is + surfaced rather than silently dropped (which would inflate `oracle-only`).""" sys.path.insert(0, str(Path(__file__).resolve().parent)) from mine_report import parse # local import: scripts/ is on the path now - raw, _ = parse(text) - return [ + raw, unparsed = parse(text) + findings = [ Finding("own", norm_path(f["file"], strips), int(f["line"]), f["code"], f["message"], _own_class(f["code"])) for f in raw ] + return findings, unparsed def _first_location(res: dict[str, Any]) -> tuple[str, int] | None: @@ -206,7 +209,8 @@ def _fmt_files(s: set[str], cap: int = 12) -> str: def render_md(result: dict[str, Any], target: str, commit: str, - oracles: list[str], tol: int, max_list: int = 50) -> str: + oracles: list[str], tol: int, own_unparsed: int = 0, + max_list: int = 50) -> str: """The human-facing comparison report.""" own_leak = result["own_leak"] ora_leak = result["oracle_leak"] @@ -224,6 +228,11 @@ def render_md(result: dict[str, Any], target: str, commit: str, f"- generated: {datetime.now(UTC):%Y-%m-%d %H:%M UTC}", f"- tools: Own.NET + {', '.join(oracles) or '(no oracle SARIF supplied)'}", f"- file match: basename + line within ±{tol}", + ] + if own_unparsed: + out.append(f"- **warning:** {own_unparsed} own-check line(s) did not parse " + "(format drift?) — Own.NET findings may be incomplete") + out += [ "", "Leak / not-disposed class only — the question all three tools can answer. " "Own.NET's use-after-release and double-release are listed separately: the " @@ -352,7 +361,15 @@ def main(argv: list[str] | None = None) -> int: if args.selftest: return _selftest() - own = build_own(Path(args.own).read_text(encoding="utf-8"), args.strip) if args.own else [] + own: list[Finding] = [] + own_unparsed = 0 + if args.own: + own, own_unparsed = build_own( + Path(args.own).read_text(encoding="utf-8"), args.strip) + if own_unparsed: + print(f"warning: {own_unparsed} own-check line(s) did not parse " + "(format drift?); Own.NET findings may be incomplete", + file=sys.stderr) sarif_inputs: list[tuple[str, str]] = [] if args.infersharp: sarif_inputs.append(("infersharp", args.infersharp)) @@ -375,7 +392,8 @@ def main(argv: list[str] | None = None) -> int: Path(args.json_out).write_text( json.dumps(to_json(result, args.target, args.commit), indent=2), encoding="utf-8") - print(render_md(result, args.target, args.commit, present, args.line_tol)) + print(render_md(result, args.target, args.commit, present, args.line_tol, + own_unparsed)) return 0 @@ -406,7 +424,7 @@ def _selftest() -> int: "artifactLocation": {"uri": "E.cs"}, "region": {"startLine": 7}}}]}, ]}]}) - own = build_own(own_txt, []) + own, own_unparsed = build_own(own_txt, []) oracles = parse_sarif(infer, "infersharp", []) + parse_sarif(codeql, "codeql", []) r = compare(own, oracles, tol=3) @@ -433,8 +451,17 @@ def _selftest() -> int: js = to_json(r, "o/r", "abc123") if js["totals"]["agree"] != 1 or js["totals"]["oracle_only"] != 1: fails.append(f"json totals wrong: {js['totals']}") - - total = 9 + # parser-drift surfacing: a clean input drops nothing; an unrecognised line + # is counted and rendered as a header warning, not silently swallowed. + if own_unparsed != 0: + fails.append(f"clean own input should have 0 unparsed, got {own_unparsed}") + _, drift = build_own("a line that is not a finding\n", []) + if drift != 1: + fails.append(f"parser drift not surfaced: expected 1 unparsed, got {drift}") + if "warning:" not in render_md(r, "o/r", "abc123", ["infersharp"], 3, drift): + fails.append("unparsed warning not rendered in header") + + total = 12 for f in fails: print(f"ORACLE SELFTEST FAIL: {f}") print(f"oracle_compare selftest: {total - len(fails)}/{total} checks passed")