From 4c35043383a94308a6f9f36579215d01ddf04e44 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 17 Jun 2026 05:49:27 +0000 Subject: [PATCH 1/2] fix(oracle): count Infer# PULSE_RESOURCE_LEAK + add product-only scoping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first real three-way run (DapperLib/Dapper) exposed two issues: 1. Infer# reported 0 leaks in the comparison, but it actually found 3 — modern Infer# (Pulse engine, v1.2+) emits `PULSE_RESOURCE_LEAK`, not the older `RESOURCE_LEAK`, so the comparator's INFER_LEAK set missed them and they were mis-filed under "out of scope" context. Now match the Infer leak family by substring (`*RESOURCE_LEAK*` / `*MEMORY_LEAK*`), robust to version drift. 2. Scope asymmetry: Infer# builds only the product project (the #24 heuristic), while own-check and CodeQL scan the whole source tree — so their leak counts were inflated by test/benchmark code the others never saw (agree was depressed accordingly). Add `--exclude-tests` (comparator) + an `include_tests` workflow input (default off => product-only). The exclusion is applied uniformly to all tools in the comparator rather than per-tool, because CodeQL's `paths-ignore` is unreliable for compiled C# with `build-mode: none`. Report header notes the scope. docs/notes/oracle.md updated (scope note + the input). selftest now 15/15 (adds PULSE classification + the exclude-tests predicate); ruff clean; run_tests.py exit 0. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Rg8kSk1YT14x7A1vo5zgED --- .github/workflows/oracle.yml | 8 +++++ docs/notes/oracle.md | 11 ++++++- scripts/oracle_compare.py | 59 ++++++++++++++++++++++++++++++++---- 3 files changed, 71 insertions(+), 7 deletions(-) diff --git a/.github/workflows/oracle.yml b/.github/workflows/oracle.yml index 520f8fee..6a512dbc 100644 --- a/.github/workflows/oracle.yml +++ b/.github/workflows/oracle.yml @@ -34,6 +34,10 @@ on: description: "Project/solution under the target to `dotnet build` for Infer# (optional, default: repo root)" required: false default: "" + include_tests: + description: "Also analyse test/benchmark/sample code (optional, default: product code only)" + required: false + default: "false" permissions: contents: read @@ -158,10 +162,14 @@ jobs: if: always() env: REPO: ${{ inputs.repo }} + INCLUDE_TESTS: ${{ inputs.include_tests }} run: | args=(--own own.txt --target "$REPO" --commit "${COMMIT:-}" --strip "$GITHUB_WORKSPACE/target" --strip target --json report.json) + # Compare on product code by default: tests/benchmarks pollute the diff + # (and Infer# only built the product project). include_tests keeps them. + [[ "${INCLUDE_TESTS:-false}" == "true" ]] || args+=(--exclude-tests) [[ -f infer-out/report.sarif ]] && args+=(--infersharp infer-out/report.sarif) cq=$(find codeql-out -name '*.sarif' -type f 2>/dev/null | head -1 || true) [[ -n "$cq" ]] && args+=(--codeql "$cq") diff --git a/docs/notes/oracle.md b/docs/notes/oracle.md index 277950b4..0b03b4c9 100644 --- a/docs/notes/oracle.md +++ b/docs/notes/oracle.md @@ -12,7 +12,7 @@ verdict. ## The three buckets Restricted to the comparable class — *resource leak / not disposed* (OWN001 vs -Infer#'s `DOTNET_RESOURCE_LEAK` vs CodeQL's `cs/local-not-disposed` & friends): +Infer#'s `PULSE_RESOURCE_LEAK` vs CodeQL's `cs/local-not-disposed` & friends): | bucket | meaning | what to do | |---|---|---| @@ -43,6 +43,14 @@ Two classes sit **outside** the three-way diff and are reported separately: to a single root `*.sln`/`*.slnx`, then a single solution anywhere, then the dir. The `build` input overrides. The shallow clone is deepened first, since version tools like Nerdbank.GitVersioning need history.) +- **All three are compared on the *product* code by default.** Infer# only builds + the product project (above), so own-check / CodeQL — which scan the whole source + tree — would otherwise count test/benchmark leaks the others never saw. The + comparator drops findings under `test` / `benchmark` / `sample` / `example` + paths (`--exclude-tests`, the workflow default); set `include_tests` to compare + across everything. Doing it in the comparator keeps one uniform rule for all + tools (CodeQL's `paths-ignore` is unreliable for compiled C# with + `build-mode: none`). - **Path/line matching is deliberately loose.** Tools disagree on the exact line (allocation site vs declaration) and on path prefixes. The comparator matches on **basename + a line window** (`--line-tol`, default 3). Robust to prefixes; @@ -58,6 +66,7 @@ artifact (`report.md`, `report.json`, plus each tool's raw output): ```text inputs: repo = DapperLib/Dapper ref = (optional) paths = (optional own-check subdir) build = (optional proj/sln for Infer#) + include_tests = false (default: compare product code only; true keeps tests/benchmarks) ``` The diff core runs anywhere on already-produced outputs (this is what `--selftest` diff --git a/scripts/oracle_compare.py b/scripts/oracle_compare.py index 29d2993f..323dce43 100644 --- a/scripts/oracle_compare.py +++ b/scripts/oracle_compare.py @@ -57,7 +57,8 @@ def _load_titles() -> dict[str, str]: OWN_LEAK = {"OWN001"} # owned resource not released on a path OWN_USE_AFTER = {"OWN002", "OWN009"} # use after release (definite / maybe) OWN_DOUBLE = {"OWN003"} # double release -INFER_LEAK = {"DOTNET_RESOURCE_LEAK", "RESOURCE_LEAK", "MEMORY_LEAK"} +INFER_LEAK = {"PULSE_RESOURCE_LEAK", "DOTNET_RESOURCE_LEAK", "RESOURCE_LEAK", + "MEMORY_LEAK", "PULSE_MEMORY_LEAK"} # canonical ids; matched by substring # CodeQL ids vary by suite/version; match the dispose/leak family by id too. CODEQL_LEAK = { "cs/local-not-disposed", @@ -97,7 +98,10 @@ def _own_class(code: str) -> str: def _oracle_class(tool: str, rule: str) -> str: r = rule or "" if tool == "infersharp": - return "leak" if r in INFER_LEAK else "other" + # Infer's Pulse engine renamed the rule (RESOURCE_LEAK -> PULSE_RESOURCE_LEAK); + # match the family by substring so version drift doesn't silence it. + ru = r.upper() + return "leak" if "RESOURCE_LEAK" in ru or "MEMORY_LEAK" in ru else "other" # codeql (and any other SARIF oracle): the dispose/leak family by id. rl = r.lower() if r in CODEQL_LEAK or "not-disposed" in rl or "dispose" in rl: @@ -210,7 +214,7 @@ def _fmt_files(s: set[str], cap: int = 12) -> str: def render_md(result: dict[str, Any], target: str, commit: str, oracles: list[str], tol: int, own_unparsed: int = 0, - max_list: int = 50) -> str: + excluded_tests: int = 0, max_list: int = 50) -> str: """The human-facing comparison report.""" own_leak = result["own_leak"] ora_leak = result["oracle_leak"] @@ -232,6 +236,9 @@ def render_md(result: dict[str, Any], target: str, commit: str, if own_unparsed: out.append(f"- **warning:** {own_unparsed} own-check line(s) did not parse " "(format drift?) — Own.NET findings may be incomplete") + if excluded_tests: + out.append(f"- scope: **product code only** — {excluded_tests} finding(s) under " + "test/benchmark/sample/example paths excluded (set `include_tests` to keep)") out += [ "", "Leak / not-disposed class only — the question all three tools can answer. " @@ -338,6 +345,17 @@ def to_json(result: dict[str, Any], target: str, commit: str) -> dict[str, Any]: } +def _is_test_path(path: str) -> bool: + """True if a finding path lives under a test/benchmark/sample/example tree — + non-product code. `--exclude-tests` drops these so the three tools are diffed + on the product code only (Infer# builds just the product project, so without + this own-check/CodeQL would also count test/benchmark leaks the others can't).""" + for seg in path.lower().split("/"): + if seg in ("test", "tests") or seg.startswith(("benchmark", "sample", "example")): + return True + return False + + def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser( description="Diff Own.NET leak findings against Infer#/CodeQL (oracle).") @@ -352,6 +370,9 @@ def main(argv: list[str] | None = None) -> int: ap.add_argument("--commit", default="", help="commit SHA (report header)") ap.add_argument("--line-tol", type=int, default=3, help="line window for a cross-tool match (default 3)") + ap.add_argument("--exclude-tests", action="store_true", + help="drop findings under test/benchmark/sample/example paths, " + "comparing the product code only across all tools") ap.add_argument("--json", dest="json_out", default="", help="also write the structured comparison as JSON") ap.add_argument("--selftest", action="store_true", @@ -387,13 +408,23 @@ def main(argv: list[str] | None = None) -> int: present.append(tool) oracles += parse_sarif(Path(path).read_text(encoding="utf-8"), tool, args.strip) + excluded = 0 + if args.exclude_tests: + before = len(own) + len(oracles) + own = [f for f in own if not _is_test_path(f.path)] + oracles = [g for g in oracles if not _is_test_path(g.path)] + excluded = before - len(own) - len(oracles) + if excluded: + print(f"--exclude-tests: dropped {excluded} finding(s) under " + "test/benchmark/sample/example paths", file=sys.stderr) + result = compare(own, oracles, args.line_tol) if args.json_out: Path(args.json_out).write_text( json.dumps(to_json(result, args.target, args.commit), indent=2), encoding="utf-8") print(render_md(result, args.target, args.commit, present, args.line_tol, - own_unparsed)) + own_unparsed, excluded)) return 0 @@ -460,8 +491,24 @@ def _selftest() -> int: fails.append(f"parser drift not surfaced: expected 1 unparsed, got {drift}") if "warning:" not in render_md(r, "o/r", "abc123", ["infersharp"], 3, drift): fails.append("unparsed warning not rendered in header") - - total = 12 + # Infer#'s Pulse engine emits PULSE_RESOURCE_LEAK (not RESOURCE_LEAK) — it must + # still classify as a leak, not out-of-scope context. + pulse_sarif = json.dumps({"runs": [{"results": [ + {"ruleId": "PULSE_RESOURCE_LEAK", "message": {"text": "leak"}, + "locations": [{"physicalLocation": { + "artifactLocation": {"uri": "Dapper/SqlMapper.cs"}, + "region": {"startLine": 10}}}]}]}]}) + pulse = parse_sarif(pulse_sarif, "infersharp", []) + if [g.cls for g in pulse] != ["leak"]: + fails.append(f"PULSE_RESOURCE_LEAK not classed as leak: {[g.cls for g in pulse]}") + # --exclude-tests predicate: matches non-product trees, not product code. + if not all(_is_test_path(p) for p in + ("tests/Foo/Bar.cs", "benchmarks/X/Y.cs", "src/Test/Z.cs")): + fails.append("_is_test_path should match test/benchmark trees") + if any(_is_test_path(p) for p in ("Dapper/SqlMapper.cs", "src/Lib/A.cs")): + fails.append("_is_test_path should not match product paths") + + total = 15 for f in fails: print(f"ORACLE SELFTEST FAIL: {f}") print(f"oracle_compare selftest: {total - len(fails)}/{total} checks passed") From ae11bff477b6d4783ce777bbcfb77c991f4eca3d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 17 Jun 2026 05:56:25 +0000 Subject: [PATCH 2/2] fix(oracle): harden include_tests input + scope note (CodeRabbit #26) Both Minor, both valid: - oracle.yml: `include_tests` was a free-form string matched against exact "true", so `True`/`TRUE` (e.g. via the API) would silently still exclude tests. Declare it `type: boolean` (dispatch UI checkbox; normalised to "true"/"false") and lower-case the shell compare (`${INCLUDE_TESTS,,}`). - oracle_compare.py: the scope note was gated on `excluded_tests > 0`, so a product-only run that excluded nothing read like a full-scope run. Gate the line on the mode (`--exclude-tests` enabled), not the count; it now always states the scope (even "0 excluded"). selftest 16/16 (adds the scope-note-at-zero check); ruff clean; oracle.yml valid YAML. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Rg8kSk1YT14x7A1vo5zgED --- .github/workflows/oracle.yml | 7 ++++--- scripts/oracle_compare.py | 13 +++++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/oracle.yml b/.github/workflows/oracle.yml index 6a512dbc..2d393f79 100644 --- a/.github/workflows/oracle.yml +++ b/.github/workflows/oracle.yml @@ -35,9 +35,10 @@ on: required: false default: "" include_tests: - description: "Also analyse test/benchmark/sample code (optional, default: product code only)" + description: "Also analyse test/benchmark/sample code (default: product code only)" required: false - default: "false" + type: boolean + default: false permissions: contents: read @@ -169,7 +170,7 @@ jobs: --json report.json) # Compare on product code by default: tests/benchmarks pollute the diff # (and Infer# only built the product project). include_tests keeps them. - [[ "${INCLUDE_TESTS:-false}" == "true" ]] || args+=(--exclude-tests) + [[ "${INCLUDE_TESTS,,}" == "true" ]] || args+=(--exclude-tests) [[ -f infer-out/report.sarif ]] && args+=(--infersharp infer-out/report.sarif) cq=$(find codeql-out -name '*.sarif' -type f 2>/dev/null | head -1 || true) [[ -n "$cq" ]] && args+=(--codeql "$cq") diff --git a/scripts/oracle_compare.py b/scripts/oracle_compare.py index 323dce43..d66ef5eb 100644 --- a/scripts/oracle_compare.py +++ b/scripts/oracle_compare.py @@ -214,7 +214,8 @@ def _fmt_files(s: set[str], cap: int = 12) -> str: def render_md(result: dict[str, Any], target: str, commit: str, oracles: list[str], tol: int, own_unparsed: int = 0, - excluded_tests: int = 0, max_list: int = 50) -> str: + excluded_tests: int = 0, exclude_tests_mode: bool = False, + max_list: int = 50) -> str: """The human-facing comparison report.""" own_leak = result["own_leak"] ora_leak = result["oracle_leak"] @@ -236,7 +237,7 @@ def render_md(result: dict[str, Any], target: str, commit: str, if own_unparsed: out.append(f"- **warning:** {own_unparsed} own-check line(s) did not parse " "(format drift?) — Own.NET findings may be incomplete") - if excluded_tests: + if exclude_tests_mode: out.append(f"- scope: **product code only** — {excluded_tests} finding(s) under " "test/benchmark/sample/example paths excluded (set `include_tests` to keep)") out += [ @@ -424,7 +425,7 @@ def main(argv: list[str] | None = None) -> int: json.dumps(to_json(result, args.target, args.commit), indent=2), encoding="utf-8") print(render_md(result, args.target, args.commit, present, args.line_tol, - own_unparsed, excluded)) + own_unparsed, excluded, args.exclude_tests)) return 0 @@ -507,8 +508,12 @@ def _selftest() -> int: fails.append("_is_test_path should match test/benchmark trees") if any(_is_test_path(p) for p in ("Dapper/SqlMapper.cs", "src/Lib/A.cs")): fails.append("_is_test_path should not match product paths") + # the scope note is gated on mode, not count: a product-only run that excluded + # nothing must still say so (else it reads like a full-scope run). + if "product code only" not in render_md(r, "o/r", "abc", ["codeql"], 3, 0, 0, True): + fails.append("scope note must render when exclude-tests mode is on (even at 0)") - total = 15 + total = 16 for f in fails: print(f"ORACLE SELFTEST FAIL: {f}") print(f"oracle_compare selftest: {total - len(fails)}/{total} checks passed")