diff --git a/.github/ISSUE_TEMPLATE/owen_cli_report.yml b/.github/ISSUE_TEMPLATE/owen_cli_report.yml new file mode 100644 index 00000000..7073ee27 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/owen_cli_report.yml @@ -0,0 +1,70 @@ +name: "owen CLI problem (alpha)" +description: "Something went wrong running `owen check` — a crash, a wrong exit code, a confusing message, or a suspected wrong finding." +labels: ["bug", "owen-cli"] +body: + - type: markdown + attributes: + value: | + Thanks for trying the alpha. Two quick pointers before you file: + - **Exit 4** ("no supported input") and **exit 3** (no Python ≥ 3.11) are + documented behavior, not bugs — see `owen --help`. If the message was + unclear or wrong for your case, that IS worth filing. + - **Exit 5** means owen hit an internal error and wrote a diagnostic + report to `~/.owen/diag/last-failure.json` — attaching it makes the fix + much faster. It contains tool/OS/runtime identity, the command line and + the failure cause, and **no source file contents**. + - type: input + id: version + attributes: + label: "owen --version" + placeholder: "0.1.0" + validations: + required: true + - type: input + id: os + attributes: + label: "OS and .NET SDK" + description: "e.g. `Windows 11 / .NET SDK 8.0.4` or `Ubuntu 24.04 / .NET SDK 8.0.4`" + validations: + required: true + - type: input + id: command + attributes: + label: "Exact command" + placeholder: "owen check MyApp.sln --format sarif" + validations: + required: true + - type: input + id: exit-code + attributes: + label: "Exit code" + description: "`echo $?` (bash) / `echo $LASTEXITCODE` (PowerShell) right after the run" + validations: + required: true + - type: textarea + id: output + attributes: + label: "Output (stderr/stdout)" + description: "What owen printed. If you can, re-run with `--debug` and include that instead." + render: shell + validations: + required: true + - type: textarea + id: artifact + attributes: + label: "Diagnostic report / sanitized repro (optional but very helpful)" + description: | + - For exit 5: attach `~/.owen/diag/last-failure.json`. + - For a wrong/missed finding: a minimal `.cs` snippet that shows it, or the + facts file from `owen check --emit-facts facts.json` — **review it + before attaching**; it contains file paths and code structure (names, + lines), though no full source text. Never attach anything you consider + private without sanitizing it first. + validations: + required: false + - type: textarea + id: expected + attributes: + label: "What you expected instead" + validations: + required: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 854a1150..51c60b0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2209,7 +2209,13 @@ jobs: # corpus: DI has no `.own` form, so it is not scanned by the Python `test_corpus` runner). # Remaining backlog: a full-length view STORED into another field, a TWO-plus-hop indirect field # use, and an injected-source region-escape. A drop below the floor is a regression. - run: python scripts/benchmark.py --min-recall 25 + run: python scripts/benchmark.py --min-recall 25 --json "$RUNNER_TEMP/benchmark-scorecard.json" + - name: publish the scorecard artifact (numbers + corpus + revision + methodology, A1) + uses: actions/upload-artifact@v4 + with: + name: benchmark-scorecard + path: ${{ runner.temp }}/benchmark-scorecard.json + retention-days: 90 # Alpha gate A (issue #202): the single delightful command, proven end-to-end # on a clean runner — install -> check -> findings. Packaging only, no @@ -2327,6 +2333,23 @@ jobs: echo "$out" [ "$rc" -ge 2 ] || { echo "FAIL: expected a non-zero exit for an unknown command, got $rc"; exit 1; } echo "$out" | grep -q "^owen: unknown command" || { echo "FAIL: expected an 'owen: unknown command' prefix"; exit 1; } + - name: "owen check is a usage error (exit 2), not a phantom path (A1)" + run: | + set +e + out=$(owen check --verbose . 2>&1) + rc=$? + set -e + echo "$out" + [ "$rc" -eq 2 ] || { echo "FAIL: expected exit 2 for an unknown option, got $rc"; exit 1; } + echo "$out" | grep -q "unknown option '--verbose'" || { echo "FAIL: expected an 'unknown option' message"; exit 1; } + echo "$out" | grep -q "does not exist" && { echo "FAIL: the typo was treated as a path"; exit 1; } + true + - name: "owen --help documents the exit-code contract incl. internal-error 5 (A1)" + run: | + out=$(owen --help) + echo "$out" | grep -q "Exit codes:" || { echo "FAIL: --help must document exit codes"; exit 1; } + echo "$out" | grep -q "5 internal error" || { echo "FAIL: --help must document exit 5"; exit 1; } + echo "$out" | grep -q -- "--debug" || { echo "FAIL: --help must document --debug"; exit 1; } - name: owen check finds the leak (installed execution, outside any checkout) run: | set +e @@ -2344,6 +2367,125 @@ jobs: set -e echo "$out" [ "$rc" -eq 0 ] || { echo "FAIL: expected exit 0 on clean code, got $rc"; exit 1; } + - name: "flagship console repro: bad is OWN001, ok is clean (A2)" + run: | + set +e + out=$(owen check "$GITHUB_WORKSPACE/examples/flagship/console/bad" --fail-on-finding 2>&1) + rc=$? + set -e + echo "$out" + [ "$rc" -eq 1 ] || { echo "FAIL: flagship bad must exit 1 (findings), got $rc"; exit 1; } + echo "$out" | grep -q "OWN001" || { echo "FAIL: flagship bad must be flagged OWN001"; exit 1; } + set +e + out=$(owen check "$GITHUB_WORKSPACE/examples/flagship/console/ok" --fail-on-finding 2>&1) + rc=$? + set -e + echo "$out" + [ "$rc" -eq 0 ] || { echo "FAIL: flagship ok must scan clean (exit 0), got $rc"; exit 1; } + - name: "core internal crash surfaces as owen exit 5, politely — never a clean scan (A1)" + if: runner.os == 'Linux' + run: | + # A crash-injection python: forwards everything to the real python3 + # (so PythonResolver's version probe passes) but crashes the core + # stage exactly like `ownlang.run()` reports an internal error. + # Pre-A1 a core crash exited 1 and, without --fail-on-finding, owen + # mapped it to a CLEAN 0. (Cache sabotage cannot simulate this: the + # content-addressed core cache self-heals — an earlier step pins that.) + cat > "$RUNNER_TEMP/crashing-python" <<'EOF' + #!/bin/sh + case "$*" in + *"-m ownlang"*) + if [ "$OWNLANG_DEBUG" = "1" ]; then + echo "Traceback (most recent call last):" >&2 + echo " synthetic gate-A crash frame" >&2 + else + echo "ownlang: internal error: RuntimeError: synthetic gate-A crash" >&2 + fi + exit 70 ;; + *) exec python3 "$@" ;; + esac + EOF + chmod +x "$RUNNER_TEMP/crashing-python" + set +e + out=$(OWEN_PYTHON="$RUNNER_TEMP/crashing-python" owen check "$RUNNER_TEMP/owen-sample" 2>&1) + rc=$? + set -e + echo "$out" + [ "$rc" -eq 5 ] || { echo "FAIL: expected exit 5 for a core crash, got $rc"; exit 1; } + echo "$out" | grep -q "ownlang: internal error" || { echo "FAIL: expected the core's polite one-liner"; exit 1; } + echo "$out" | grep -q "This is a bug in owen" || { echo "FAIL: expected owen's polite framing"; exit 1; } + echo "$out" | grep -q "Diagnostic report" || { echo "FAIL: the core-crash path must write the diagnostic report (Codex P2)"; exit 1; } + echo "$out" | grep -q "Traceback (most recent call last)" && { echo "FAIL: raw traceback leaked without --debug"; exit 1; } + echo "$out" | grep -qE "^0 findings\.$" && { echo "FAIL: a core crash must never read as a clean scan"; exit 1; } + true + - name: "the same crash with --debug shows the full cause and still exits 5 (A1)" + if: runner.os == 'Linux' + run: | + set +e + out=$(OWEN_PYTHON="$RUNNER_TEMP/crashing-python" owen check --debug "$RUNNER_TEMP/owen-sample" 2>&1) + rc=$? + set -e + echo "$out" + [ "$rc" -eq 5 ] || { echo "FAIL: expected exit 5 in debug mode too, got $rc"; exit 1; } + echo "$out" | grep -q "Traceback (most recent call last)" || { echo "FAIL: --debug must surface the full cause"; exit 1; } + - name: "retention-path witness MVP builds and its usage surface is honest (A3)" + run: | + dotnet build "$GITHUB_WORKSPACE/audit/runtime/RetentionPath" -c Release -v quiet + set +e + out=$(dotnet "$GITHUB_WORKSPACE"/audit/runtime/RetentionPath/bin/Release/net8.0/RetentionPath.dll 2>&1) + rc=$? + set -e + echo "$out" + [ "$rc" -eq 2 ] || { echo "FAIL: bare usage must exit 2 (never clean), got $rc"; exit 1; } + echo "$out" | grep -q "RETAINED (root path shown) | OBSERVED_ONLY" || { echo "FAIL: usage must document the verdict vocabulary"; exit 1; } + set +e + out=$(dotnet "$GITHUB_WORKSPACE"/audit/runtime/RetentionPath/bin/Release/net8.0/RetentionPath.dll roots --pid 999999 --type X 2>&1) + rc=$? + set -e + echo "$out" + [ "$rc" -eq 2 ] || { echo "FAIL: a failed attach must exit 2, never read as clean, got $rc"; exit 1; } + dotnet "$GITHUB_WORKSPACE"/audit/runtime/RetentionPath/bin/Release/net8.0/RetentionPath.dll selftest + - name: "flagship demo orchestrator end-to-end: bad DEMONSTRATED, ok VERIFIED (A3/A4)" + if: runner.os == 'Linux' + run: | + # The ONE reproducible demo entrypoint (scripts/flagship-demo.sh) is + # itself the CI proof: build -> hold the app -> attach the witness + # through its public CLI -> machine-validate JSON against the human + # verdict -> one stable summary line. bad must demonstrate the + # static-event retention; ok must verify no established retention + # (the loop-local stack root correctly reads OBSERVED_ONLY, not + # RETAINED — the verdict consults the classification). + # + # GitHub runners ship Yama ptrace_scope=1, which blocks same-user + # non-ancestor PTRACE_ATTACH — the ClrMD live attach needs classic + # scope. CI-runner-only relaxation; the demo script itself stays + # sudo-free (a real user attaching to their own app under scope 1 + # gets the witness's polite exit-2, not silence). + sudo sysctl -w kernel.yama.ptrace_scope=0 + ./scripts/flagship-demo.sh bad + ./scripts/flagship-demo.sh ok + - name: "a top-level .NET exception exits 5 in BOTH modes — debug changes volume, not semantics (A1 P2)" + if: runner.os == 'Linux' + run: | + # TMPDIR pointing at a directory that does not exist makes + # Path.GetTempFileName() throw before any inner catch — the honest + # top-level owen exception, unreachable via the child stages. + set +e + out=$(TMPDIR=/definitely/does/not/exist owen check "$RUNNER_TEMP/owen-sample" 2>&1) + rc=$? + set -e + echo "$out" + [ "$rc" -eq 5 ] || { echo "FAIL: expected exit 5 for a top-level exception, got $rc"; exit 1; } + echo "$out" | grep -q "owen: internal error" || { echo "FAIL: expected owen's polite framing"; exit 1; } + echo "$out" | grep -q "Unhandled exception" && { echo "FAIL: raw runtime crash banner leaked"; exit 1; } + echo "$out" | grep -q " at " && { echo "FAIL: stack trace leaked without --debug"; exit 1; } + set +e + out=$(TMPDIR=/definitely/does/not/exist owen check --debug "$RUNNER_TEMP/owen-sample" 2>&1) + rc=$? + set -e + echo "$out" + [ "$rc" -eq 5 ] || { echo "FAIL: --debug must keep exit 5 (a rethrow would exit a runtime-chosen code), got $rc"; exit 1; } + echo "$out" | grep -q " at " || { echo "FAIL: --debug must print the full .NET stack"; exit 1; } - name: owen check --format sarif -- Owen-branded SARIF driver name run: | out=$(owen check "$RUNNER_TEMP/owen-sample" --format sarif) diff --git a/audit/runtime/RetentionPath/Heap.cs b/audit/runtime/RetentionPath/Heap.cs new file mode 100644 index 00000000..67f71c12 --- /dev/null +++ b/audit/runtime/RetentionPath/Heap.cs @@ -0,0 +1,529 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Microsoft.Diagnostics.Runtime; + +namespace OwnNet.Audit.Runtime +{ + /// + /// Mark-from-roots over a target's managed heap, and the root -> object paths for a + /// suspect type. + /// + /// WHY THIS IS NOT HeapCounter. answers "how many instances + /// of T are on the heap". That is a different question from "how many are RETAINED", + /// because ClrHeap.EnumerateObjects() walks the heap segments linearly and + /// returns everything allocated — including garbage the GC has not collected yet. A + /// big heap is not evidence of a leak. HeapCounter mitigates this by forcing a GC in + /// the target first (SematixTrace), which works when you can drive the target; this + /// type does not need to, because marking from the roots answers the question: + /// + /// reachable ≈ heap -> genuinely retained; something holds it + /// reachable << heap -> not a leak; the GC simply has not collected yet + /// + /// WHY IT SAMPLES. "Who holds this object" is ill-posed for an object reachable from + /// many roots — there are as many answers as there are paths, and the shortest one is + /// an arbitrary pick, not an explanation. Ask instead: **what holds the typical + /// instance?** So the walk takes a SAMPLE of the retained instances, computes each + /// one's shortest path in a single BFS, and reports the paths as a HISTOGRAM. The + /// retainer that accounts for 129,900 of 130,000 instances is the leak; the three that + /// hang off the stack or a prototype are noise, and reading one of them as "the answer" + /// is how a leak hunt goes wrong. + /// + /// The principled version of this is a dominator tree (which single reference, if cut, + /// frees the object — and how much memory that frees). See the README. + /// + internal sealed class RetentionWalker : IDisposable + { + private readonly DataTarget _target; + private readonly ClrRuntime _runtime; + + /// Attach to a LIVE process (suspends it for the read). No procdump needed. + public static RetentionWalker AttachToProcess(int pid) => + new RetentionWalker(DataTarget.AttachToProcess(pid, suspend: true)); + + /// Read a full dump — the right choice when the target must not be paused. + public static RetentionWalker LoadDump(string path) => + new RetentionWalker(DataTarget.LoadDump(path)); + + private RetentionWalker(DataTarget target) + { + _target = target; + var clr = _target.ClrVersions.FirstOrDefault() + ?? throw new InvalidOperationException( + "the target contains no CLR — is it a managed process / a full (-ma) dump?"); + _runtime = clr.CreateRuntime(); + } + + private ClrHeap Heap => _runtime.Heap; + + /// + /// One mark pass. Returns the retained set (by type) alongside the raw heap totals, + /// so the caller can state the retained SHARE rather than a bare object count. + /// + public HeapCensus Census() + { + long heapObjects = 0, heapBytes = 0; + foreach (var o in Heap.EnumerateObjects()) + { + if (!o.IsValid || o.Type == null) continue; + heapObjects++; + heapBytes += (long)o.Size; + } + + var seen = new HashSet(); + var stack = new Stack(); + foreach (var root in Heap.EnumerateRoots()) + { + var o = root.Object; + if (o.IsValid && seen.Add(o.Address)) stack.Push(o.Address); + } + int rootCount = seen.Count; + + var byType = new Dictionary(); + long liveObjects = 0, liveBytes = 0; + while (stack.Count > 0) + { + var obj = Heap.GetObject(stack.Pop()); + if (!obj.IsValid || obj.Type == null) continue; + + liveObjects++; + long size = (long)obj.Size; + liveBytes += size; + + string name = obj.Type.Name ?? ""; + if (!byType.TryGetValue(name, out var tally)) tally = new TypeTally(); + tally.Count++; + tally.Bytes += size; + byType[name] = tally; + + foreach (var child in obj.EnumerateReferences()) + if (child.IsValid && seen.Add(child.Address)) stack.Push(child.Address); + } + + return new HeapCensus(rootCount, heapObjects, heapBytes, liveObjects, liveBytes, byType); + } + + /// + /// Find what retains the instances of : one breadth-first pass + /// from the whole root set (which gives each node its shortest path for free), then group + /// the resolved paths by shape, ranked by how many instances each shape holds — the answer + /// to "what is holding all of this", as opposed to "here is a path to one of them". + /// + /// INVARIANT (the display/verdict boundary): reachability and the durable/transient census + /// are computed over EVERY instance on the heap. bounds only how + /// many paths are RESOLVED for display; bounds only how many + /// hops are RENDERED. No display limit may alter discovery, classification, aggregation, + /// or the verdict/exit code — a presentation flag that can change the diagnosis is a + /// lottery, not an option. + /// + public RetentionReport FindRetainers(string typeName, int sample, int maxHops) + { + // ---- 1. the targets --------------------------------------------------------- + // Match the TYPE, not the type's spelling. A naive substring match on the type name + // matches `System.Func` when you asked for + // `GTDGoody` — a cached lambda whose *generic argument* happens to mention it — and then + // confidently reports a path to the wrong object. A tool that points at the wrong culprit + // is worse than no tool. + // ALL matching instances are targeted (Codex P1: taking the first + // `sample` in heap-enumeration order could catch only old garbage + // and miss a later durably-held instance — a false OBSERVED_ONLY). + // The verdict is exact over the whole population; only PATH + // RESOLUTION below is sampled (`sample` paths), keeping the + // expensive part bounded. + var targets = new Dictionary(); + long totalOfType = 0; + foreach (var o in Heap.EnumerateObjects()) + { + if (!o.IsValid || o.Type?.Name == null) continue; + if (!IsType(o.Type.Name, typeName)) continue; + totalOfType++; + targets[o.Address] = o.Type.Name; + } + if (targets.Count == 0) + return new RetentionReport(typeName, 0, 0, 0, 0, new List()); + + // ---- 2. one BFS from every root; parent pointers only (no strings) ------------ + // Storing a label per node would cost hundreds of MB on a 4M-object heap. Store the + // parent address, and resolve type/field names later, for the sampled paths only. + var parent = new Dictionary(); // child -> parent (0 = root) + var rootKind = new Dictionary(); + var queue = new Queue(); + + // Seed DURABLE roots (handles/statics) before TRANSIENT ones + // (stack frames, the finalizer queue). Parent-pointer BFS credits an + // object to whichever root reaches it first; a Main local that + // happens to hold the static publisher in a register would + // otherwise claim it as [stack] and mask the real static-event + // retention (observed live on net8, gate A pins it). + var allRoots = Heap.EnumerateRoots().ToList(); + int reachedTargets = 0; + + void Bfs() + { + while (queue.Count > 0 && reachedTargets < targets.Count) + { + ulong addr = queue.Dequeue(); + if (targets.ContainsKey(addr)) reachedTargets++; + + var obj = Heap.GetObject(addr); + if (!obj.IsValid || obj.Type == null) continue; + + foreach (var child in obj.EnumerateReferences()) + { + if (!child.IsValid || parent.ContainsKey(child.Address)) continue; + parent[child.Address] = addr; + queue.Enqueue(child.Address); + } + } + } + + // Two BFS PHASES, not merely two seeding passes: durable roots + // (handles/statics) are seeded and walked TO EXHAUSTION before any + // transient root (stack frame, finalizer queue) enters the graph. + // An object can be a stack-root itself AND reachable from a pinned + // static — a Main local holding the static publisher is exactly + // that — and seeding it as a stack ROOT would mask the durable + // static-event retention behind it (observed live on net8; the + // gate-A end-to-end smoke pins the corrected verdict). Retention + // analysis prefers durable evidence; the stack only explains what + // nothing durable can. + // + // INVARIANT (why the shared `parent` map cannot bury a transient + // path): durable traversal claims reachable targets; transient + // traversal may need shared intermediates, but every intermediate + // the durable phase claimed was WALKED TO EXHAUSTION — so any + // target reachable through a durably-claimed node is already + // durably claimed itself. A target left for phase 2 is, by + // construction, unreachable from every durable root, and its + // transient path cannot pass through a durably-claimed node. (The + // early exit on `reachedTargets` fires only when ALL targets are + // claimed, which preserves the property.) Transient ownership + // never overwrites durable ownership, and no explainable object + // is left unexplained. + foreach (var seedTransient in new[] { false, true }) + { + foreach (var root in allRoots) + { + bool transientRoot = Retainer.IsTransientRootKind(root.RootKind); + if (transientRoot != seedTransient) continue; + var o = root.Object; + if (!o.IsValid || parent.ContainsKey(o.Address)) continue; + parent[o.Address] = 0; + rootKind[o.Address] = root.RootKind; + queue.Enqueue(o.Address); + } + Bfs(); + } + + // ---- 3. root-kind census over EVERY reachable instance ------------------------ + // The census walks each reachable target's parent chain to its true root — + // dictionary hops only, no type/field resolution — so the VERDICT sees the + // root kind of the whole population. Classifying only the resolved-path + // sample would re-admit the Codex P1 bias one level up: 200+ finalizer- + // queue-reachable corpses sitting ahead of one durably-held instance in + // heap order would read as a false OBSERVED_ONLY. + var durableAddrs = new List(); + var transientAddrs = new List(); + foreach (var kv in targets) + { + if (!parent.ContainsKey(kv.Key)) continue; // not reachable — genuinely garbage + ulong cur = kv.Key; + while (parent.TryGetValue(cur, out ulong p) && p != 0) cur = p; + var rk = rootKind.TryGetValue(cur, out var k) ? k : ClrRootKind.None; + // An unknown/None kind lands on the durable side, matching Classify's + // `unsupported-root:*` doctrine (fail-closed toward visibility). + if (Retainer.IsTransientRootKind(rk)) transientAddrs.Add(kv.Key); + else durableAddrs.Add(kv.Key); + } + + // ---- 4. resolve up to `sample` paths for display, durable instances first ---- + // Path resolution (type/field names) is the expensive, bounded part. + // Durable-first ordering guarantees that whenever the census found durable + // retention, at least one durable path is on display: RETAINED never ships + // without its root path. + var groups = new Dictionary(); + long pathsResolved = 0; + foreach (var addr in durableAddrs.Concat(transientAddrs)) + { + if (pathsResolved >= sample) break; + pathsResolved++; + + var hops = Unwind(addr, parent, rootKind, maxHops, out ClrRootKind kind); + // The signature must carry the CLASSIFICATION and the fields, + // not only hop type names (Codex P1): a stack-rooted and a + // durably-rooted instance can share a type sequence, and + // merging them would classify the whole group by whichever + // came first — hiding a durable retainer or inventing one. + string signature = Retainer.Classify(kind, hops) + " | " + + string.Join(" -> ", hops.Select(h => h.ToString())); + + if (!groups.TryGetValue(signature, out var retainer)) + { + retainer = new Retainer(hops, kind); + groups[signature] = retainer; + } + retainer.Instances++; + } + + var ranked = groups.Values.OrderByDescending(r => r.Instances).ToList(); + return new RetentionReport(targets.Values.First(), totalOfType, + durableAddrs.Count + transientAddrs.Count, durableAddrs.Count, pathsResolved, ranked); + } + + /// + /// Walk the parent chain back to a root, naming the field traversed at every hop. The field + /// name is what turns "this object is alive" into "THIS FIELD is holding it" — the sentence a + /// developer can act on — so it is resolved here (by re-reading the parent's references), + /// rather than carried through the BFS at the cost of hundreds of megabytes. + /// + private List Unwind(ulong target, Dictionary parent, + Dictionary rootKind, int maxHops, + out ClrRootKind kind) + { + var chain = new List(); + ulong cur = target; + bool truncated = false; + while (true) + { + chain.Add(cur); + if (!parent.TryGetValue(cur, out ulong p) || p == 0) break; + cur = p; + if (chain.Count > maxHops) { truncated = true; break; } + } + if (truncated) + { + // Keep walking the (acyclic) parent chain WITHOUT recording + // hops: the rendered path stays bounded, but the verdict must + // see the true root — stopping mid-chain yielded + // ClrRootKind.None -> 'unsupported-root:None', which the + // verdict counts as durable, turning a long stack-only path + // into a false RETAINED (Codex P1). + while (parent.TryGetValue(cur, out ulong p2) && p2 != 0) cur = p2; + } + kind = rootKind.TryGetValue(cur, out var k) ? k : ClrRootKind.None; + chain.Reverse(); + + var hops = new List(chain.Count); + for (int i = 0; i < chain.Count; i++) + { + var obj = Heap.GetObject(chain[i]); + string type = obj.Type?.Name ?? "?"; + string? field = null; + if (i > 0) + { + var owner = Heap.GetObject(chain[i - 1]); + if (owner.IsValid && owner.Type != null) + { + foreach (var r in owner.EnumerateReferencesWithFields()) + { + if (r.Object.Address != chain[i]) continue; + field = r.Field?.Name; + break; + } + } + } + hops.Add(new Hop(type, field)); + } + return hops; + } + + /// + /// Does name the type the caller asked for? Compares the SIMPLE + /// name with generic arguments stripped, so `GTDGoody` matches `BrokerDataClasses.GTDGoody` + /// but NOT `System.Func<BrokerDataClasses.GTDGoody, System.Boolean>`. A fully-qualified + /// request (`BrokerDataClasses.GTDGoody`) is matched exactly. + /// + internal static bool IsType(string heapType, string wanted) + { + if (string.Equals(heapType, wanted, StringComparison.Ordinal)) return true; + + int lt = heapType.IndexOf('<'); // Func -> Func + string bare = lt >= 0 ? heapType.Substring(0, lt) : heapType; + if (string.Equals(bare, wanted, StringComparison.Ordinal)) return true; + + int dot = bare.LastIndexOf('.'); // Ns.GTDGoody -> GTDGoody + string simple = dot >= 0 ? bare.Substring(dot + 1) : bare; + return string.Equals(simple, wanted, StringComparison.Ordinal); + } + + /// + /// The dominator tree of the whole live graph, with retained sizes. This is the well-posed + /// version of "who holds it": not a path, but the one reference whose removal frees the object. + /// + + public void Dispose() + { + _runtime.Dispose(); + _target.Dispose(); + } + } + + internal struct TypeTally + { + public long Count; + public long Bytes; + } + + internal sealed class Hop + { + public readonly string Type; + public readonly string? Field; + + public Hop(string type, string? field) + { + Type = type; + Field = field; + } + + public override string ToString() => + Field == null ? Type : Type + " (." + Field + ")"; + } + + /// One distinct retention shape, and how many of the RESOLVED paths land on it + /// (display evidence; the verdict rests on the whole-population census, not on these). + internal sealed class Retainer + { + public readonly IReadOnlyList Path; + public readonly ClrRootKind RootKind; + public long Instances; + + public Retainer(IReadOnlyList path, ClrRootKind rootKind) + { + Path = path; + RootKind = rootKind; + } + + /// + /// Map a ClrMD root kind onto the `runtime.json` kinds (OwnAudit/docs/runtime-contract.md: + /// static-field, static-event, gc-handle, thread-local, timer). + /// + /// Note there is no `StaticVar` root kind: on .NET Framework a class's statics live in a + /// pinned `System.Object[]` handed to the runtime as a **PinnedHandle**, which is why a + /// static-field leak surfaces as `[PinnedHandle] System.Object[] -> …`. A **delegate hop** + /// further down the path is what makes it a static *event* rather than a plain static field — + /// the distinction correlate.py's `high` tier keys on. + /// + /// `Stack` and `FinalizerQueue` are reported as themselves, deliberately: an object rooted + /// only by the stack is merely *live right now*, not retained, and reading it as a leak is how + /// a leak hunt goes wrong. + /// + public string ContractKind() => Classify(RootKind, Path); + + /// + /// The transient/durable split at the ClrMD level — the single source of + /// truth shared by the BFS phase seeding, the whole-population census, and + /// (via Classify's 'stack'/'finalizer' cases) the string-level verdict + /// rule; the selftest pins that the layers agree. Every other kind — + /// including an UNKNOWN one — is durable for verdict purposes (fail-closed + /// toward visibility). + /// + public static bool IsTransientRootKind(ClrRootKind kind) => + kind == ClrRootKind.Stack || kind == ClrRootKind.FinalizerQueue; + + /// + /// The classifier boundary (kept pure over its evidence so the + /// selftest pins it without a heap): a ClrMD root kind plus the path's + /// delegate evidence map onto the `runtime.json` kinds. Every KNOWN + /// kind is named explicitly; an UNKNOWN kind is an honest + /// `unsupported-root:` — visible evidence the mapping must be + /// taught, never silently classified as non-root or as a handle. + /// + public static string Classify(ClrRootKind rootKind, IReadOnlyList path) + { + // The delegate evidence: an event subscription retains through the + // handler chain — EventHandler/MulticastDelegate hop types, or the + // multicast `_invocationList` field. Field evidence is checked as a + // FIELD, hop types as TYPES: an unrelated type merely named + // "...StackFrame..." or a field named "stackCache" is not evidence. + bool viaDelegate = path.Any(h => + h.Type.IndexOf("EventHandler", StringComparison.Ordinal) >= 0 || + h.Type.IndexOf("MulticastDelegate", StringComparison.Ordinal) >= 0 || + (h.Field != null && h.Field.IndexOf("invocationList", StringComparison.OrdinalIgnoreCase) >= 0)); + + switch (rootKind) + { + case ClrRootKind.Stack: + return "stack"; // live in a frame right now — not retention + case ClrRootKind.FinalizerQueue: + return "finalizer"; // awaiting finalization — a stall, not a reference leak + case ClrRootKind.PinnedHandle: + // statics live in a pinned object[] on both runtimes + return viaDelegate ? "static-event" : "static-field"; + case ClrRootKind.StrongHandle: + case ClrRootKind.AsyncPinnedHandle: + case ClrRootKind.RefCountedHandle: + case ClrRootKind.SizedRefHandle: + return viaDelegate ? "static-event" : "gc-handle"; + default: + return $"unsupported-root:{rootKind}"; + } + } + + /// The object one hop above the target — the thing actually holding the reference. + public string Holder => Path.Count >= 2 ? Path[Path.Count - 2].Type : Path[0].Type; + + /// The field on that object, when the reference came from a named field. + public string? Member => Path.Count >= 1 ? Path[Path.Count - 1].Field : null; + + public string Render() + { + var sb = new StringBuilder(); + for (int i = 0; i < Path.Count; i++) + sb.Append(" ").Append(Path[i]).Append(Environment.NewLine); + return sb.ToString(); + } + } + + internal sealed class RetentionReport + { + public readonly string TypeName; + public readonly long TotalOnHeap; + /// Root-reachable instances — exact over the whole population. + public readonly long Retained; + /// Reachable instances whose true root is durable — exact; the + /// verdict's input, deliberately independent of the --sample display budget. + public readonly long DurableRetained; + /// How many paths were resolved for display (bounded by --sample) — + /// the denominator for every rendered share. + public readonly long PathsResolved; + public readonly IReadOnlyList Retainers; + + public RetentionReport(string typeName, long totalOnHeap, long retained, + long durableRetained, long pathsResolved, + IReadOnlyList retainers) + { + TypeName = typeName; + TotalOnHeap = totalOnHeap; + Retained = retained; + DurableRetained = durableRetained; + PathsResolved = pathsResolved; + Retainers = retainers; + } + } + + internal sealed class HeapCensus + { + public readonly int Roots; + public readonly long HeapObjects; + public readonly long HeapBytes; + public readonly long RetainedObjects; + public readonly long RetainedBytes; + public readonly IReadOnlyDictionary ByType; + + public HeapCensus(int roots, long heapObjects, long heapBytes, + long retainedObjects, long retainedBytes, + IReadOnlyDictionary byType) + { + Roots = roots; + HeapObjects = heapObjects; + HeapBytes = heapBytes; + RetainedObjects = retainedObjects; + RetainedBytes = retainedBytes; + ByType = byType; + } + + /// The number that decides whether this is a leak hunt at all. + public double RetainedShare => HeapBytes == 0 ? 0 : 100.0 * RetainedBytes / HeapBytes; + } +} diff --git a/audit/runtime/RetentionPath/Program.cs b/audit/runtime/RetentionPath/Program.cs new file mode 100644 index 00000000..502630e4 --- /dev/null +++ b/audit/runtime/RetentionPath/Program.cs @@ -0,0 +1,416 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.Diagnostics.Runtime; +using Newtonsoft.Json; + +namespace OwnNet.Audit.Runtime +{ + /// + /// Retention paths (Plan.md §4): the half of the runtime arm that HeapCounter leaves + /// undone. HeapCounter counts instances of named types; this answers the two questions + /// that actually decide a leak hunt: + /// + /// 1. is any of it RETAINED, or is the heap just full of uncollected garbage? + /// 2. if it is retained — WHO is holding it? + /// + /// Emits the `runtime.json` contract (OwnAudit/docs/runtime-contract.md) so + /// OwnAudit's runtime/correlate.py consumes the output directly: a `confirmed` finding + /// is a static leak finding whose type also shows up here as retained, and a + /// `runtime-only` finding — retention with nothing static to explain it — is the + /// analyzer's blind spot, i.e. a rule request. + /// + /// Usage: + /// RetentionPath census --pid N | --dump D [--out runtime.json] [--top 25] + /// RetentionPath roots --pid N | --dump D --type TypeName [--sample 200] [--max-hops 40] + /// + /// `census` prints the retained SHARE first, on purpose: if only 5% of the heap is + /// reachable, there is no leak to hunt and the next step is a GC question, not a + /// reference question. + /// + /// `roots` SAMPLES the instances and reports the paths as a ranked histogram, because + /// "who holds this object" is ill-posed for an object reachable from many roots — there + /// are as many answers as there are paths, and the shortest is an arbitrary pick. The + /// question worth asking is "what holds the TYPICAL instance": the retainer that + /// accounts for 129,900 of 130,000 is the leak, and the three hanging off the stack or a + /// prototype are noise. + /// + internal static class Program + { + private static int Main(string[] args) + { + if (args.Length == 0) return Usage(); + string verb = args[0].ToLowerInvariant(); + + // The classifier boundary, pinned without a heap: the live net8 + // static-event shape (gate A also proves it end-to-end), the + // negative neighbours, and the honest-refusal case. + if (verb == "selftest") + return ClassifierSelfTest() ? 0 : 1; + + int pid = ArgInt(args, "--pid", 0); + string? dump = Arg(args, "--dump"); + if (pid == 0 && dump == null) + { + Console.Error.WriteLine("retention-path: need --pid or --dump "); + return 2; + } + + try + { + using var walker = dump != null + ? RetentionWalker.LoadDump(dump) + : RetentionWalker.AttachToProcess(pid); + + switch (verb) + { + case "census": return Census(walker, args); + case "roots": return Roots(walker, args); + default: return Usage(); + } + } + catch (Exception ex) + { + // A failed read must not read as "clean" — exit 2, distinct from + // 0 (analysed, nothing retained) and 1 (analysed, retention found). + Console.Error.WriteLine($"retention-path: {ex.GetType().Name}: {ex.Message}"); + return 2; + } + } + + private static int Census(RetentionWalker walker, string[] args) + { + var c = walker.Census(); + int top = ArgInt(args, "--top", 25); + + Console.WriteLine($"roots : {c.Roots,12:N0} objects"); + Console.WriteLine($"on the heap : {c.HeapObjects,12:N0} objects {Mb(c.HeapBytes),10:N0} MB"); + Console.WriteLine($"REACHABLE from roots : {c.RetainedObjects,12:N0} objects {Mb(c.RetainedBytes),10:N0} MB"); + Console.WriteLine($"uncollected garbage : {c.HeapObjects - c.RetainedObjects,12:N0} objects {Mb(c.HeapBytes - c.RetainedBytes),10:N0} MB"); + Console.WriteLine(); + Console.WriteLine(c.RetainedShare > 50 + ? $">>> {c.RetainedShare:N1}% of the heap is genuinely RETAINED — something holds it; run `roots`" + : $">>> only {c.RetainedShare:N1}% of the heap is retained — the rest is garbage the GC has not collected"); + Console.WriteLine(); + Console.WriteLine($"{"type",-62}{"count",14}{"MB",12}"); + foreach (var kv in c.ByType.OrderByDescending(k => k.Value.Bytes).Take(top)) + Console.WriteLine($"{Short(kv.Key),-62}{kv.Value.Count,14:N0}{Mb(kv.Value.Bytes),12:N1}"); + + string? outPath = Arg(args, "--out"); + if (outPath != null) + { + // The runtime.json contract. `expected` is left at 0 — the collector does not + // know the budget; the scenario/config does, and correlate.py applies it. + var retained = c.ByType + .OrderByDescending(k => k.Value.Bytes) + .Take(top) + .Select(kv => new Dictionary + { + ["type"] = kv.Key, + ["count"] = kv.Value.Count, + ["expected"] = 0, + ["bytes"] = kv.Value.Bytes, + ["roots"] = new object[0], + }) + .ToList(); + + var doc = new Dictionary + { + ["schema"] = "own-runtime/1", + ["collector"] = CollectorIdentity(args), + ["retained"] = retained, + }; + File.WriteAllText(outPath, JsonConvert.SerializeObject(doc, Formatting.Indented)); + Console.WriteLine(); + Console.WriteLine($"runtime.json written to {outPath}"); + } + + return c.RetainedShare > 50 ? 1 : 0; + } + + private static int Roots(RetentionWalker walker, string[] args) + { + string? type = Arg(args, "--type"); + if (type == null) + { + Console.Error.WriteLine("retention-path roots: need --type "); + return 2; + } + // Display budgets only — clamped to at least 1 so a pathological + // `--sample 0` cannot suppress the root path a RETAINED verdict + // must ship with. Neither flag can alter the verdict (the census + // in FindRetainers is exact over the whole population). + int sample = Math.Max(1, ArgInt(args, "--sample", 200)); + int maxHops = Math.Max(1, ArgInt(args, "--max-hops", 40)); + + var report = walker.FindRetainers(type, sample, maxHops); + if (report.TotalOnHeap == 0) + { + Console.WriteLine($"verdict: ABSENT — no instance of {type} is on the heap"); + WriteArtifact(args, "ABSENT", type, 0, new List()); + return 0; + } + if (report.Retained == 0) + { + // Honest mode split (A3): instances exist but NO retention path was + // established — never call this a proven leak. + Console.WriteLine($"verdict: OBSERVED_ONLY — {report.TotalOnHeap:N0} instance(s) of {type} on the " + + "heap, but none of them is reachable from a GC root " + + "(garbage awaiting collection, not an established retention)"); + WriteArtifact(args, "OBSERVED_ONLY", type, report.TotalOnHeap, new List()); + return 0; + } + + // The verdict CONSULTS the classification (the ok-variant of the + // flagship demo pinned this): a path from a transient root (stack + // frame, finalizer queue) proves the object is live RIGHT NOW, + // not that anything retains it — a loop local still in a register + // is not a leak. RETAINED requires at least one durable retainer; + // an unknown root kind counts as durable on purpose (fail-closed + // toward visibility: unknown evidence must surface loudly, never + // quietly demote the verdict). The census behind DurableRetained + // covers EVERY reachable instance, so the verdict cannot change + // with the --sample display budget. + if (report.DurableRetained == 0) + { + Console.WriteLine($"verdict: OBSERVED_ONLY — {report.TypeName}: {report.TotalOnHeap:N0} on the " + + $"heap, {report.Retained:N0} reachable, " + + "but ONLY from transient roots (stack/finalizer) — live right now, not durable " + + "retention"); + foreach (var r in report.Retainers.Take(3)) + { + Console.WriteLine(); + Console.WriteLine($" via [{r.ContractKind()}], {r.Path.Count} hops:"); + Console.Write(r.Render()); + } + WriteArtifact(args, "OBSERVED_ONLY", type, report.TotalOnHeap, report.Retainers); + return 0; + } + + Console.WriteLine($"verdict: RETAINED — {report.TypeName}: {report.TotalOnHeap:N0} on the heap, " + + $"{report.Retained:N0} reachable, {report.DurableRetained:N0} durably retained " + + $"({report.PathsResolved:N0} path(s) resolved for display)"); + Console.WriteLine(); + Console.WriteLine("RETAINERS, ranked — what holds the TYPICAL instance, not merely one of them:"); + + int rank = 0; + foreach (var r in report.Retainers) + { + rank++; + // Shares are shares OF THE RESOLVED PATHS — the display sample — + // never of the full population the verdict was computed over. + double share = 100.0 * r.Instances / report.PathsResolved; + Console.WriteLine(); + Console.WriteLine($"#{rank} {r.Instances:N0}/{report.PathsResolved:N0} resolved ({share:N1}%) " + + $"— via [{r.ContractKind()}], {r.Path.Count} hops"); + Console.Write(r.Render()); + if (rank >= 5) break; // the tail is noise; raise --sample for resolution + } + + Console.WriteLine(); + var dominant = report.Retainers[0]; + double dominantShare = 100.0 * dominant.Instances / report.PathsResolved; + if (dominantShare >= 50 && dominant.ContractKind() != "stack") + { + string member = dominant.Member != null ? "." + dominant.Member : ""; + Console.WriteLine($">>> {dominantShare:N1}% of the resolved paths hang off ONE reference: " + + $"{dominant.Holder}{member} [{dominant.ContractKind()}]"); + } + else + { + Console.WriteLine(">>> no single dominant retainer in this sample — raise --sample, or the type " + + "really is held from many places"); + } + + WriteArtifact(args, "RETAINED", report.TypeName, report.TotalOnHeap, report.Retainers); + + return 1; // retention found + } + + /// The verdict rule, pure over classification kinds (the + /// selftest pins it): RETAINED requires at least one DURABLE + /// retainer. Transient kinds (stack frame, finalizer queue) prove + /// 'live right now', never retention; an `unsupported-root:*` kind + /// counts as durable on purpose — unknown evidence surfaces loudly, + /// never quietly demotes the verdict. + internal static bool IsDurableKind(string kind) => + kind != "stack" && kind != "finalizer"; + + internal static string VerdictOf(IEnumerable retainerKinds) => + retainerKinds.Any(IsDurableKind) ? "RETAINED" : "OBSERVED_ONLY"; + + /// The one `runtime.json` writer — every verdict emits the + /// artifact when `--out` is given, so the ok-side of a demo is as + /// machine-checkable as the leak side. + private static void WriteArtifact( + string[] args, string verdict, string typeName, long count, IReadOnlyList retainers) + { + string? outPath = Arg(args, "--out"); + if (outPath == null) return; + var doc = new Dictionary + { + ["schema"] = "own-runtime/1", + ["verdict"] = verdict, + ["collector"] = CollectorIdentity(args), + ["retained"] = new object[] + { + new Dictionary + { + ["type"] = typeName, + ["count"] = count, + ["expected"] = 0, + ["bytes"] = 0, + ["roots"] = retainers.Take(5).Select(r => new Dictionary + { + ["kind"] = r.ContractKind(), + ["holder"] = r.Holder, + ["member"] = r.Member ?? "", + ["via"] = r.ContractKind() == "static-event" ? "delegate" : "reference", + ["instances"] = r.Instances, + ["path"] = r.Path.Select(h => h.ToString()).ToList(), + }).ToList(), + }, + }, + }; + File.WriteAllText(outPath, JsonConvert.SerializeObject(doc, Formatting.Indented)); + Console.WriteLine(); + Console.WriteLine($"runtime.json written to {outPath}"); + } + + /// Who read the heap and how — so the artifact is auditable + /// (A3): the target (pid or dump path), the collector runtime, the OS. + /// No timestamps: identical heaps must yield identical artifacts. + private static Dictionary CollectorIdentity(string[] args) + { + return new Dictionary + { + ["tool"] = "retention-path", + ["mode"] = Arg(args, "--dump") != null ? "dump" : "attach", + ["target"] = Arg(args, "--dump") ?? Arg(args, "--pid") ?? "?", + ["runtime"] = Environment.Version.ToString(), + ["os"] = Environment.OSVersion.ToString(), + }; + } + + private static double Mb(long bytes) => bytes / 1024.0 / 1024.0; + + private static string Short(string t) => + t.Length <= 60 ? t : t.Substring(0, 28) + "…" + t.Substring(t.Length - 30); + + private static string? Arg(string[] args, string name) + { + int i = Array.IndexOf(args, name); + return i >= 0 && i + 1 < args.Length ? args[i + 1] : null; + } + + private static int ArgInt(string[] args, string name, int fallback) + { + var v = Arg(args, name); + return v != null && int.TryParse(v, out int n) ? n : fallback; + } + + private static bool ClassifierSelfTest() + { + var fails = new List(); + void Check(string name, string got, string want) + { + if (got != want) fails.Add($"{name}: got '{got}', want '{want}'"); + } + + // 1. THE live net8 static-event path, verbatim from the flagship + // bad app (statics live in a pinned object[] on both runtimes). + var staticEvent = new List + { + new("Owen.Flagship.AppSettings", null), + new("System.ComponentModel.PropertyChangedEventHandler", "PropertyChanged"), + new("System.Object[]", "_invocationList"), + new("System.ComponentModel.PropertyChangedEventHandler", null), + new("Owen.Flagship.DocumentView", "_target"), + }; + Check("net8 static event (PinnedHandle)", + Retainer.Classify(ClrRootKind.PinnedHandle, staticEvent), "static-event"); + Check("static event through a strong handle stays an event", + Retainer.Classify(ClrRootKind.StrongHandle, staticEvent), "static-event"); + + // 2. Negative neighbour: 'stack'-flavoured NAMES are not evidence — + // a type/field merely containing the word must classify by its + // root kind, not by string contagion. + var stackishNames = new List + { + new("My.App.StackMachine", "stackCache"), + new("My.App.Node", "next"), + }; + Check("stack-flavoured names stay a plain handle", + Retainer.Classify(ClrRootKind.StrongHandle, stackishNames), "gc-handle"); + + // 3. Doctrine: a genuine Stack root is 'live right now', never + // retention — even when the path has delegate evidence. + Check("stack root stays stack even via a delegate", + Retainer.Classify(ClrRootKind.Stack, staticEvent), "stack"); + Check("finalizer root is a stall, not a reference leak", + Retainer.Classify(ClrRootKind.FinalizerQueue, staticEvent), "finalizer"); + + // 4. A pinned root WITHOUT delegate evidence is a static field. + var plainStatic = new List + { + new("My.App.Config", null), + new("My.App.Cache", "_entries"), + }; + Check("pinned root without a delegate is static-field", + Retainer.Classify(ClrRootKind.PinnedHandle, plainStatic), "static-field"); + + // 5. Honest refusal: an unknown root kind is REPORTED as + // unsupported, never silently classified as non-root/handle. + Check("unknown root kind refuses honestly", + Retainer.Classify((ClrRootKind)999, plainStatic), "unsupported-root:999"); + + // The verdict rule (deterministic here; the live ok-demo only has + // to prove the absence of a false RETAINED — JIT liveness of a + // loop local is not a public contract to test against). + Check("stack-only reachability is OBSERVED_ONLY", + VerdictOf(new[] { "stack" }), "OBSERVED_ONLY"); + Check("finalizer-only reachability is OBSERVED_ONLY", + VerdictOf(new[] { "finalizer", "stack" }), "OBSERVED_ONLY"); + Check("one durable retainer makes it RETAINED", + VerdictOf(new[] { "stack", "static-event" }), "RETAINED"); + Check("unknown evidence surfaces as RETAINED, never demotes", + VerdictOf(new[] { "unsupported-root:999" }), "RETAINED"); + + // 6. The doctrine lives at TWO layers — the ClrMD-level split + // (BFS phase seeding + the whole-population census) and the + // string-level verdict rule — and they must never disagree: + // a kind the census calls transient must classify to a kind + // the verdict calls non-durable, and vice versa. + foreach (var kind in new[] { ClrRootKind.Stack, ClrRootKind.FinalizerQueue, + ClrRootKind.PinnedHandle, ClrRootKind.StrongHandle, + (ClrRootKind)999 }) + { + bool transient = Retainer.IsTransientRootKind(kind); + bool durable = IsDurableKind(Retainer.Classify(kind, plainStatic)); + if (transient == durable) + fails.Add($"census/verdict split disagrees for {kind}: " + + $"IsTransientRootKind={transient}, IsDurableKind(Classify)={durable}"); + } + + foreach (var f in fails) + Console.Error.WriteLine($"FAIL: classifier {f}"); + if (fails.Count == 0) + Console.WriteLine("retention-path classifier selftest OK: 16 checks passed"); + return fails.Count == 0; + } + + private static int Usage() + { + Console.Error.WriteLine("usage:"); + Console.Error.WriteLine(" RetentionPath selftest # classifier fixtures, no target needed"); + Console.Error.WriteLine(" RetentionPath census --pid | --dump [--out runtime.json] [--top 25]"); + Console.Error.WriteLine(" RetentionPath roots --pid | --dump --type [--sample 200] [--max-hops 40] [--out runtime.json]"); + Console.Error.WriteLine(); + Console.Error.WriteLine(" census is there anything retained at all, or is the heap just uncollected garbage?"); + Console.Error.WriteLine(" roots what holds the TYPICAL instance of a type (exact verdict; sampled, ranked paths);"); + Console.Error.WriteLine(" verdicts: RETAINED (root path shown) | OBSERVED_ONLY (no path established) | ABSENT"); + return 2; + } + } +} diff --git a/audit/runtime/RetentionPath/RetentionPath.csproj b/audit/runtime/RetentionPath/RetentionPath.csproj new file mode 100644 index 00000000..0c1a5e38 --- /dev/null +++ b/audit/runtime/RetentionPath/RetentionPath.csproj @@ -0,0 +1,34 @@ + + + + Exe + net8.0 + latest + enable + RetentionPath + OwnNet.Audit.Runtime + + + + + + diff --git a/examples/flagship/README.md b/examples/flagship/README.md new file mode 100644 index 00000000..994dec86 --- /dev/null +++ b/examples/flagship/README.md @@ -0,0 +1,49 @@ +# The flagship leak — "a `-=` exists" is not "a `-=` runs" + +The one-case demo of what Owen catches that ordinary cleanup matching misses. +Distilled from a real, heap-proven production leak (issue #278: a formally +present unsubscribe behind a parameter guard nobody ever passed `false` to — +66% of the process heap retained), reduced to ~80 cross-platform console +lines. + +## The bug (`console/bad/`) + +A static, process-lifetime publisher; a view subscribed in its constructor; +an unsubscribe that *exists* but sits behind `if (!keepAlive)` — and every +close path calls `Cleanup(keepAlive: true)`. The `-=` never runs; every +closed view stays pinned to the publisher forever. + +``` +dotnet run --project examples/flagship/console/bad + → opened and closed 1000 views; 1000 still subscribed — every one of them + is retained by the static publisher. + +owen check examples/flagship/console/bad --fail-on-finding + → OWN001 … subscribed but never provably unsubscribed (exit 1) +``` + +A checker that pairs `+=` with any `-=` in the class calls this clean. Owen +demands the release be *provable*: a parameter-guarded `-=` in a non-teardown +method is not evidence (the corpus pins this predicate family — +`corpus/wpf/subscription-teardown-early-return-guard` and friends). + +## The fix (`console/ok/`) + +Move the release where it provably runs: `Dispose()`, unconditionally, called +on every close path. + +``` +dotnet run --project examples/flagship/console/ok + → opened and closed 1000 views; 0 still subscribed. + +owen check examples/flagship/console/ok --fail-on-finding + → exit 0 +``` + +Same publisher, same subscription, same handler — the only change is that +teardown became a teardown. The static finding disappears *and* the runtime +count goes to zero: the two halves of the same evidence. + +Both variants are smoke-checked in CI (gate A) against the installed +`Owen.Cli` on Linux and Windows: `bad` must exit 1 with OWN001, `ok` must +exit 0. diff --git a/examples/flagship/console/bad/BadDocumentApp.csproj b/examples/flagship/console/bad/BadDocumentApp.csproj new file mode 100644 index 00000000..ee942bde --- /dev/null +++ b/examples/flagship/console/bad/BadDocumentApp.csproj @@ -0,0 +1,8 @@ + + + Exe + net8.0 + enable + Owen.Flagship + + diff --git a/examples/flagship/console/bad/DocumentApp.cs b/examples/flagship/console/bad/DocumentApp.cs new file mode 100644 index 00000000..e8964d95 --- /dev/null +++ b/examples/flagship/console/bad/DocumentApp.cs @@ -0,0 +1,88 @@ +// The flagship console leak (the #278 shape, minus the corporate archaeology). +// +// A static publisher lives for the whole process. Every "document view" +// subscribes in its constructor. An unsubscribe EXISTS — `Cleanup` — but it +// sits behind a parameter guard, and the close path calls `Cleanup(keepAlive: +// true)`, so the `-=` never runs. Every closed view stays reachable through +// the publisher's delegate list: the process accumulates one dead view per +// open/close cycle, forever. +// +// This is exactly the bug class where "a matching -= exists somewhere in the +// class" reads as safe. Owen does not accept existence as evidence: a +// parameter-guarded `-=` in a non-teardown method cannot be proven to run, so +// the subscription is flagged (OWN001). +// +// Run it: dotnet run --project examples/flagship/console/bad +// Analyze it: owen check examples/flagship/console/bad --fail-on-finding +using System; +using System.ComponentModel; + +namespace Owen.Flagship; + +/// Process-lifetime settings hub (the static publisher). +public sealed class AppSettings : INotifyPropertyChanged +{ + public static readonly AppSettings Instance = new(); + + public event PropertyChangedEventHandler? PropertyChanged; + + public int SubscriberCount => PropertyChanged?.GetInvocationList().Length ?? 0; + + public void Touch() => + PropertyChanged?.Invoke(this, new PropertyChangedEventArgs("Theme")); +} + +/// Opened per document; subscribes to the process-lifetime hub. +public sealed class DocumentView +{ + private readonly AppSettings _settings; + + public DocumentView(AppSettings settings) + { + _settings = settings; + _settings.PropertyChanged += OnSettingsChanged; + } + + private void OnSettingsChanged(object? sender, PropertyChangedEventArgs e) + { + // re-render with the new settings + } + + public void Cleanup(bool keepAlive) + { + if (!keepAlive) + { + _settings.PropertyChanged -= OnSettingsChanged; + } + // detach only the cheap extras when the caller asked to keep the core + } + + // The bug: every close path keeps the core subscription alive. + public void Close() => Cleanup(keepAlive: true); +} + +public static class Program +{ + public static void Main() + { + for (var i = 0; i < 1000; i++) + { + var view = new DocumentView(AppSettings.Instance); + view.Close(); + } + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + Console.WriteLine( + $"opened and closed 1000 views; " + + $"{AppSettings.Instance.SubscriberCount} still subscribed — " + + "every one of them is retained by the static publisher."); + if (Environment.GetEnvironmentVariable("OWEN_FLAGSHIP_HOLD") == "1") + { + // Keep the heap alive for a runtime witness (the demo script and + // the CI end-to-end smoke attach retention-path to this process). + Console.WriteLine($"holding (pid {Environment.ProcessId}) — send a line to exit."); + Console.ReadLine(); + } + } +} diff --git a/examples/flagship/console/ok/DocumentApp.cs b/examples/flagship/console/ok/DocumentApp.cs new file mode 100644 index 00000000..04b2164a --- /dev/null +++ b/examples/flagship/console/ok/DocumentApp.cs @@ -0,0 +1,76 @@ +// The fix for the flagship console leak: teardown belongs in a teardown. +// +// The subscription is released in `Dispose()` — unconditionally, on every +// close path. Owen treats a `-=` in a real teardown as a provable release, so +// this variant scans clean; at runtime the publisher's delegate list stays +// empty after the views are closed. +// +// Run it: dotnet run --project examples/flagship/console/ok +// Analyze it: owen check examples/flagship/console/ok --fail-on-finding +using System; +using System.ComponentModel; + +namespace Owen.Flagship; + +/// Process-lifetime settings hub (the static publisher). +public sealed class AppSettings : INotifyPropertyChanged +{ + public static readonly AppSettings Instance = new(); + + public event PropertyChangedEventHandler? PropertyChanged; + + public int SubscriberCount => PropertyChanged?.GetInvocationList().Length ?? 0; + + public void Touch() => + PropertyChanged?.Invoke(this, new PropertyChangedEventArgs("Theme")); +} + +/// Opened per document; subscribes to the process-lifetime hub. +public sealed class DocumentView : IDisposable +{ + private readonly AppSettings _settings; + + public DocumentView(AppSettings settings) + { + _settings = settings; + _settings.PropertyChanged += OnSettingsChanged; + } + + private void OnSettingsChanged(object? sender, PropertyChangedEventArgs e) + { + // re-render with the new settings + } + + public void Dispose() + { + _settings.PropertyChanged -= OnSettingsChanged; + } + + // Every close path releases the subscription. + public void Close() => Dispose(); +} + +public static class Program +{ + public static void Main() + { + for (var i = 0; i < 1000; i++) + { + var view = new DocumentView(AppSettings.Instance); + view.Close(); + } + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + Console.WriteLine( + $"opened and closed 1000 views; " + + $"{AppSettings.Instance.SubscriberCount} still subscribed."); + if (Environment.GetEnvironmentVariable("OWEN_FLAGSHIP_HOLD") == "1") + { + // Keep the heap alive for a runtime witness (the demo script and + // the CI end-to-end smoke attach retention-path to this process). + Console.WriteLine($"holding (pid {Environment.ProcessId}) — send a line to exit."); + Console.ReadLine(); + } + } +} diff --git a/examples/flagship/console/ok/OkDocumentApp.csproj b/examples/flagship/console/ok/OkDocumentApp.csproj new file mode 100644 index 00000000..ee942bde --- /dev/null +++ b/examples/flagship/console/ok/OkDocumentApp.csproj @@ -0,0 +1,8 @@ + + + Exe + net8.0 + enable + Owen.Flagship + + diff --git a/frontend/roslyn/OwnSharp.Cli/CheckCommand.cs b/frontend/roslyn/OwnSharp.Cli/CheckCommand.cs index de6ab957..2b498972 100644 --- a/frontend/roslyn/OwnSharp.Cli/CheckCommand.cs +++ b/frontend/roslyn/OwnSharp.Cli/CheckCommand.cs @@ -93,20 +93,59 @@ public static async Task RunAsync(string[] args) var factsPath = Path.GetTempFileName(); try { - var extractRc = await RunExtractorAsync(paths, factsPath, legacy, stats, bodyThrowEdges) - .ConfigureAwait(false); - if (extractRc != 0) + var (extractRc, extractOutput) = + await RunExtractorAsync(paths, factsPath, legacy, stats, bodyThrowEdges) + .ConfigureAwait(false); + // The extractor's own contract is 0 / 2 (usage) / 4 (nothing to + // analyze after expansion) — those pass through with its output. + // Anything else is a crash (an unhandled exception's runtime + // code): frame it politely and keep the raw trace in the + // diagnostic report (or on stderr in --debug mode) — A1. + if (extractRc is 2 or 4) { + Console.Error.Write(extractOutput); return extractRc; } + if (extractRc != 0) + { + if (CrashReport.Debug) + { + Console.Error.Write(extractOutput); + } + return CrashReport.Child("extractor", extractRc, args, extractOutput); + } + Console.Error.Write(extractOutput); if (emitFacts is not null) { - File.Copy(factsPath, emitFacts, overwrite: true); + try + { + File.Copy(factsPath, emitFacts, overwrite: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException + or ArgumentException or NotSupportedException or DirectoryNotFoundException) + { + Console.Error.WriteLine( + $"owen check: cannot write --emit-facts '{emitFacts}': {ex.Message}"); + return 2; + } } var cacheRoot = CoreVendor.EnsureUnpacked(); var rc = await RunCoreAsync(python, cacheRoot, factsPath, format, severity).ConfigureAwait(false); + // The core self-reports internal errors as exit 70 (EX_SOFTWARE) + // with one polite line (ownlang `run()`): surface them as OUR + // internal error — pre-A1 a core crash exited 1 and, without + // --fail-on-finding, was silently mapped to a CLEAN scan. + if (rc == 70) + { + // The most important internal-error path must honor the whole + // contract, including the diagnostic report (Codex P2) — the + // core's polite one-liner is already on stderr above. + Console.Error.WriteLine( + "owen: the analysis core failed internally — the line above has the short cause."); + return CrashReport.Child("analysis core", rc, args, capturedOutput: null); + } if (failOnFinding) { @@ -151,7 +190,20 @@ private static (string Format, string Severity, bool FailOnFinding, bool Legacy, case "--legacy": legacy = true; break; case "--stats": stats = true; break; case "--body-throw-edges": bodyThrowEdges = true; break; - default: paths.Add(a); break; + case "--debug": CrashReport.DebugFlag = true; break; + default: + // A mistyped flag must be a usage error, not a phantom + // path: pre-A1 `owen check --verbose .` fell through to + // "path '--verbose' does not exist" (exit 4), which reads + // as an input problem instead of the actual typo. + if (a.StartsWith('-')) + { + throw new InvalidOperationException( + $"owen check: unknown option '{a}' (see `owen --help`; " + + "put `--` before paths that begin with '-')"); + } + paths.Add(a); + break; } } @@ -206,10 +258,12 @@ private static bool HasSupportedInput(IReadOnlyList paths, out string re return false; } - /// Stage 1: run the bundled extractor as a child process. All of its - /// own output (build/run chatter, if any) goes to OUR stderr, keeping - /// stdout clean for stage 2 — same as own-check.sh's `1>&2` on this stage. - private static async Task RunExtractorAsync( + /// Stage 1: run the bundled extractor as a child process. Its + /// output (build/run chatter, warnings — and, on a crash, the raw trace) + /// is CAPTURED and returned; the caller decides what reaches OUR stderr + /// (everything for the contract codes, report-only for a crash — A1), + /// keeping stdout clean for stage 2 like own-check.sh's `1>&2`. + private static async Task<(int Rc, string Output)> RunExtractorAsync( IReadOnlyList paths, string factsPath, bool legacy, bool stats, bool bodyThrowEdges) { // "ownsharp-extract.dll" is OwnSharp.Extractor's own real AssemblyName/output @@ -218,10 +272,10 @@ private static async Task RunExtractorAsync( var extractorDll = Path.Combine(AppContext.BaseDirectory, "ownsharp-extract.dll"); if (!File.Exists(extractorDll)) { - Console.Error.WriteLine( + return (2, $"owen: bundled extractor not found at '{extractorDll}' — a corrupt or " + - "incomplete tool install. Try `dotnet tool uninstall --global Owen.Cli` and reinstall."); - return 2; + "incomplete tool install. Try `dotnet tool uninstall --global Owen.Cli` " + + "and reinstall." + Environment.NewLine); } var psi = new ProcessStartInfo(ResolveDotnetMuxer()) @@ -258,9 +312,7 @@ private static async Task RunExtractorAsync( await proc.WaitForExitAsync().ConfigureAwait(false); var stdout = await stdoutTask.ConfigureAwait(false); var stderr = await stderrTask.ConfigureAwait(false); - if (stdout.Length > 0) Console.Error.Write(stdout); - if (stderr.Length > 0) Console.Error.Write(stderr); - return proc.ExitCode; + return (proc.ExitCode, stdout + stderr); } /// The `dotnet` muxer used to `exec` the bundled extractor dll. A @@ -313,6 +365,13 @@ private static async Task RunCoreAsync( // cwd to sys.path[0], but own-check.sh/.ps1 both set PYTHONPATH // explicitly too, and matching that is cheap insurance. psi.EnvironmentVariables["PYTHONPATH"] = cacheRoot; + // Debug passthrough (A1): the core's catch-all (`ownlang.run`) prints + // one polite line and exits 70; with OWNLANG_DEBUG=1 it re-raises the + // full traceback instead — that is what `owen check --debug` asks for. + if (CrashReport.Debug) + { + psi.EnvironmentVariables["OWNLANG_DEBUG"] = "1"; + } using var proc = Process.Start(psi) ?? throw new InvalidOperationException("owen: failed to start the Python core process"); diff --git a/frontend/roslyn/OwnSharp.Cli/CrashReport.cs b/frontend/roslyn/OwnSharp.Cli/CrashReport.cs new file mode 100644 index 00000000..7d513ef1 --- /dev/null +++ b/frontend/roslyn/OwnSharp.Cli/CrashReport.cs @@ -0,0 +1,118 @@ +using System.Text.Json; + +namespace OwnSharp.Cli; + +/// +/// The first-run failure contract (alpha gate A1): an internal error must +/// never surface as a raw stack trace, a runtime-chosen exit code, or — +/// worst — a clean scan. Every internal failure exits +/// (5) with one short, actionable message and a deterministic diagnostic +/// report at ~/.owen/diag/last-failure.json. +/// +/// The report contains tool/OS/runtime identity, the command line, the +/// failure stage and the technical cause. It deliberately contains NO source +/// file contents — facts/source sharing stays an explicit user action +/// (`--emit-facts`). Debug mode (`--debug` or OWEN_DEBUG=1) prints the full +/// .NET trace to stderr — it changes the VOLUME of diagnostics, never the +/// machine semantics: the exit code is 5 in both modes (a re-throw would +/// exit with a runtime-chosen, platform-dependent code and break the +/// published contract). +/// +internal static class CrashReport +{ + /// Exit code for "owen (or a stage it drives) hit a bug" — + /// distinct from usage (2), no Python (3), and no input (4), and never + /// collapsible into findings (1) or clean (0). + public const int ExitCode = 5; + + /// Set by `owen check --debug`; OWEN_DEBUG=1 is the env twin. + public static bool DebugFlag { get; set; } + + public static bool Debug => + DebugFlag || Environment.GetEnvironmentVariable("OWEN_DEBUG") == "1"; + + private const string ReportIssueUrl = + "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/PhysShell/Own.NET/issues/new/choose"; + + /// Handle an uncaught exception at the top level: polite + /// message + report, exit 5. Debug mode additionally prints the full + /// exception (type, message, stack) — the exit code stays 5. + public static int Handle(Exception ex, string[] args) + { + if (Debug) + { + Console.Error.WriteLine(ex); + } + var report = TryWrite(args, stage: "owen", + cause: $"{ex.GetType().FullName}: {ex.Message}", + detail: ex.ToString(), childOutput: null); + Console.Error.WriteLine($"owen: internal error ({ex.GetType().Name}: {ex.Message})"); + Emit(report); + return ExitCode; + } + + /// Frame a child stage's crash (unexpected exit code) without + /// dumping its raw output on the user; the full capture goes into the + /// report instead. In debug mode the caller prints the raw output. + public static int Child(string stage, int rc, string[] args, string? capturedOutput) + { + var report = TryWrite(args, stage, + cause: $"{stage} exited with unexpected code {rc}", + detail: null, childOutput: capturedOutput); + Console.Error.WriteLine( + $"owen: the {stage} stage failed internally (exit {rc})."); + Emit(report); + return ExitCode; + } + + private static void Emit(string? reportPath) + { + Console.Error.WriteLine( + " This is a bug in owen, not in your code. Re-run with --debug " + + "(or OWEN_DEBUG=1) for the full technical cause."); + if (reportPath is not null) + { + Console.Error.WriteLine( + $" Diagnostic report (no source contents collected): {reportPath}"); + } + Console.Error.WriteLine($" Please report it: {ReportIssueUrl}"); + } + + /// One deterministic JSON report, overwritten in place (a single + /// well-known path beats an ever-growing directory). Best-effort: a + /// failure to write the report must never mask the original failure. + private static string? TryWrite( + string[] args, string stage, string cause, string? detail, string? childOutput) + { + try + { + var dir = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + ".owen", "diag"); + Directory.CreateDirectory(dir); + var path = Path.Combine(dir, "last-failure.json"); + var report = new + { + schema = 1, + tool = "owen", + version = ToolVersion.Current, + timestamp_utc = DateTime.UtcNow.ToString("o"), + os = Environment.OSVersion.ToString(), + runtime = Environment.Version.ToString(), + command = "check", + args, + stage, + cause, + detail, + child_output = childOutput, + }; + File.WriteAllText(path, JsonSerializer.Serialize( + report, new JsonSerializerOptions { WriteIndented = true })); + return path; + } + catch (Exception) + { + return null; + } + } +} diff --git a/frontend/roslyn/OwnSharp.Cli/Program.cs b/frontend/roslyn/OwnSharp.Cli/Program.cs index 75b7fa4b..ce1cef0a 100644 --- a/frontend/roslyn/OwnSharp.Cli/Program.cs +++ b/frontend/roslyn/OwnSharp.Cli/Program.cs @@ -30,7 +30,17 @@ return 2; } -return await CheckCommand.RunAsync(args[1..]).ConfigureAwait(false); +// A1 first-run contract: no user ever sees a raw .NET stack trace (or a +// runtime-chosen exit code) from a bug of ours by default. Debug mode +// re-throws inside CrashReport.Handle. +try +{ + return await CheckCommand.RunAsync(args[1..]).ConfigureAwait(false); +} +catch (Exception ex) +{ + return CrashReport.Handle(ex, args); +} // Product framing is deliberately language-neutral (Owen finds lifetime and // resource-contract bugs; the OwnIR/core layer is not C#-specific) while the @@ -56,6 +66,16 @@ owen check [more paths...] [options] --legacy use the flat name-based local-IDisposable detector --stats print flow-locals coverage to stderr --body-throw-edges opt-in: flag body-level (no-try) dispose-not-called-on-throw + --debug full technical causes on failure (raw traces; OWEN_DEBUG=1 works too) + + Exit codes: + 0 clean (or findings, without --fail-on-finding) + 1 findings (with --fail-on-finding) + 2 usage or contract error + 3 no usable Python (>=3.11) found + 4 no supported input found (never a silent clean scan) + 5 internal error — a bug in owen, never silence; a diagnostic report + (no source contents) is written under ~/.owen/diag/ Python: resolved via OWEN_PYTHON (OWN_PYTHON is a deprecated, temporary fallback), else `py -3` (Windows) / `python3` (elsewhere); must be diff --git a/frontend/roslyn/OwnSharp.Cli/README.md b/frontend/roslyn/OwnSharp.Cli/README.md index c8d48c6b..93cbef85 100644 --- a/frontend/roslyn/OwnSharp.Cli/README.md +++ b/frontend/roslyn/OwnSharp.Cli/README.md @@ -75,9 +75,44 @@ as above (bump `--version` if you rebuilt with a new ``). | `--stats` | off | print flow-locals coverage to stderr | | `--body-throw-edges` | off | opt-in: flag body-level (no-`try`) dispose-not-called-on-throw | -Exit codes: `0` clean, `1` findings (only with `--fail-on-finding`), `>=2` a -core hard error (bad facts, a drifted contract), `3` no usable Python found, -`4` no supported input found (nothing matching the included frontend). +Exit codes: `0` clean, `1` findings (only with `--fail-on-finding`), `2` a +usage or contract error (bad flags, bad facts, a drifted contract), `3` no +usable Python found, `4` no supported input found (nothing matching the +included frontend), `5` an **internal error** — a bug in owen or a stage it +drives (extractor/core crash). An internal error is never silence, never a +clean scan, and never a raw stack trace by default: one short message, plus a +deterministic diagnostic report at `~/.owen/diag/last-failure.json` (tool/OS/ +runtime identity, command line, stage, cause — **no source file contents**; +sharing facts stays the explicit `--emit-facts` action). `--debug` (or +`OWEN_DEBUG=1`) prints the full technical cause instead. + +## Known limitations (alpha) + +What is *unsupported by design* — distinct from bugs (which we want reported): + +- **.NET / C# frontend only.** `.cs`, `.csproj`, `.sln`. Anything else is + exit 4 ("no supported input"), never a silent clean scan. +- **A non-compiling project is analyzed anyway** — symbol-tolerantly. Roslyn + compile errors are deliberately ignored (the analysis reads symbols, not + IL); unresolved external references degrade to *advisory* notes + (OWN050/OWN051), never to invented findings. Consequence: a broken build + does not fail `owen check`, and findings that depend on an unresolved type + may be missed — check the project compiles if a finding you expected is + absent. +- **Alpha rule scope**, not a general leak detector: event-subscription + lifetime (the WPF/WinForms `+=`-without-reachable-`-=` family), timers, + local `IDisposable` flows, DI lifetime mismatches, pooled-buffer misuse. +- **Static analysis only.** A finding is a lifetime-contract violation with + the evidence the code shows — not a runtime-proven leak. Runtime retention + proof is separate tooling. +- **Vocabulary is versioned and fails loud.** Facts from a mismatched + extractor/core pair are a hard exit 2 by contract — never a guess. +- **Python ≥ 3.11 required** at run time; never auto-installed. +- Analysis of WPF-shaped code does **not** require Windows; only *running* + WPF apps does. + +Anything outside this list that ends in a crash, a wrong exit code, or a +wrong finding is a bug — please use the "owen CLI problem" issue template. ## Release process diff --git a/ownlang/__main__.py b/ownlang/__main__.py index 318c8291..d32ff3b9 100644 --- a/ownlang/__main__.py +++ b/ownlang/__main__.py @@ -928,5 +928,30 @@ def main(argv: list[str]) -> int: "summaries": cmd_summaries}[cmd](path) +def run(argv: list[str]) -> int: + """`main` behind the first-run contract: an internal crash exits 70 + (EX_SOFTWARE) with ONE actionable line — never a traceback a first-time + user has to parse, and never an exit code a caller could mistake for + findings (1) or clean (0). `OWNLANG_DEBUG=1` re-raises for the full + technical cause (the `owen check --debug` passthrough). Deliberate + contract errors keep their own codes: they raise nothing.""" + try: + return main(argv) + except Exception as exc: # the whole point is the catch-all + import os + if os.environ.get("OWNLANG_DEBUG"): + # Debug shows the full cause but KEEPS the exit-code contract: a + # re-raise would exit 1, which a caller maps as "findings" (and + # `owen check` without --fail-on-finding maps to a clean 0). + import traceback + traceback.print_exc() + return 70 + print(f"ownlang: internal error: {type(exc).__name__}: {exc}\n" + f" This is a bug in the analyzer, not in your code. Re-run with " + f"OWNLANG_DEBUG=1 for the full traceback and please report it.", + file=sys.stderr) + return 70 + + if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) + raise SystemExit(run(sys.argv[1:])) diff --git a/ownlang/ownir.py b/ownlang/ownir.py index 31d74bde..32f50f5c 100644 --- a/ownlang/ownir.py +++ b/ownlang/ownir.py @@ -524,7 +524,11 @@ def build_sarif(findings: list[Finding], severity: str = "error") -> dict[str, A def load(path: str) -> dict[str, Any]: """Load and shape-check an OwnIR facts file (it is external input — a malformed file should fail with a clear error, not a deep traceback).""" - with open(path, encoding="utf-8") as f: + try: + f = open(path, encoding="utf-8") + except OSError as e: + raise OwnIRError(f"cannot read {path}: {e}") from e + with f: try: result: Any = json.load(f) except json.JSONDecodeError as e: diff --git a/scripts/benchmark.py b/scripts/benchmark.py index 9a9d3fc1..9735229b 100644 --- a/scripts/benchmark.py +++ b/scripts/benchmark.py @@ -212,7 +212,75 @@ def gate(caught: int, clean: int, total: int, fps: int, min_recall: int) -> list return problems -def run(root: str, corpus_dirs: list[str], min_recall: int = 0) -> int: +def scorecard(scores: list[CaseScore], min_recall: int, + corpus_counts: list[tuple[str, int]], revision: str) -> dict: + """The publishable scorecard (alpha gate A1): the aggregate numbers WITH the + corpus, the revision and the methodology attached — a metric published + without those is just a nice percentage nobody can audit. Pure function of + its inputs, so `--selftest` pins its shape with no SDK.""" + caught, clean, total, fps = summarize(scores) + return { + "schema": 1, + "benchmark": "own.net corpus (real C# through the extractor + core)", + "revision": revision, + "methodology": { + "recall": "a case is 'caught' when its before.cs yields >=1 SARIF " + "result at error/warning level (note/none are advisory, " + "not verdicts); code-agnostic on purpose", + "specificity": "a fix is 'clean' when its after.cs yields 0 " + "verdict-level results (any verdict on a fix is a " + "false positive)", + "gate": "specificity and zero FPs are unconditional; recall is " + "gated against the pinned floor and ratchets up", + "runner": "scripts/own-check.sh --format sarif per file", + }, + "corpus": [{"dir": d, "cases": n} for d, n in corpus_counts], + "totals": {"cases": total, "caught": caught, "clean": clean, + "false_positives": fps, "recall_floor": min_recall}, + "cases": [{"name": s.name, "expected": sorted(s.expected), + "before": sorted(s.before), "after": sorted(s.after), + "caught": s.caught, "clean": s.clean} for s in scores], + } + + +def _revision(root: str) -> str: + try: + proc = subprocess.run(["git", "rev-parse", "HEAD"], cwd=root, + capture_output=True, text=True, timeout=30, + check=False) + return proc.stdout.strip() or "unknown" + except OSError: + return "unknown" + + +def _publish(card: dict, json_path: str | None) -> None: + """Write the scorecard artifact and, under GitHub Actions, the job summary + — the same numbers the terminal prints, in auditable/linkable form.""" + if json_path: + with open(json_path, "w", encoding="utf-8") as f: + json.dump(card, f, indent=2, sort_keys=True) + f.write("\n") + print(f"benchmark: scorecard written to {json_path}") + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + t = card["totals"] + corpus = " + ".join(f"`{c['dir']}` ({c['cases']})" for c in card["corpus"]) + with open(summary_path, "a", encoding="utf-8") as f: + f.write( + "### Corpus benchmark (real C#)\n\n" + f"| metric | value |\n|---|---|\n" + f"| bugs caught | {t['caught']}/{t['cases']} |\n" + f"| fixes clean | {t['clean']}/{t['cases']} |\n" + f"| false positives on fixes | {t['false_positives']} |\n" + f"| recall floor (gate) | {t['recall_floor']} |\n\n" + f"Corpus: {corpus} · revision `{card['revision'][:12]}`\n\n" + "Methodology: a *catch* is ≥1 SARIF error/warning on `before.cs`; " + "a *clean fix* is 0 on `after.cs`; advisory notes count as " + "neither. Specificity and zero-FP are unconditional gates.\n") + + +def run(root: str, corpus_dirs: list[str], min_recall: int = 0, + json_path: str | None = None) -> int: """Score the corpus on real C#, print the scorecard, and apply the gate.""" try: scores = score_corpus(root, corpus_dirs) @@ -237,6 +305,10 @@ def run(root: str, corpus_dirs: list[str], min_recall: int = 0) -> int: print(f"benchmark: {caught}/{total} bugs caught in real C# · " f"{clean}/{total} fixes clean · {fps} false positive(s) on fixes " f"(recall floor {min_recall})") + corpus_counts = [(os.path.relpath(d, root), len(discover([d]))) + for d in corpus_dirs] + _publish(scorecard(scores, min_recall, corpus_counts, _revision(root)), + json_path) problems = gate(caught, clean, total, fps, min_recall) for p in problems: print(f"BENCHMARK FAIL: {p}") @@ -286,6 +358,22 @@ def _selftest() -> int: if summarize(cases) != (3, 3, 4, 1): fails.append(f"summarize: expected (3,3,4,1), got {summarize(cases)}") + # 2b) scorecard: the publishable artifact must carry the SAME totals as + # summarize(), plus the corpus/revision/methodology context — and be + # deterministic (a metric nobody can audit or reproduce is marketing). + card = scorecard(cases, 3, [("corpus/x", 4)], "deadbeef") + if (card["totals"] != {"cases": 4, "caught": 3, "clean": 3, + "false_positives": 1, "recall_floor": 3} + or card["corpus"] != [{"dir": "corpus/x", "cases": 4}] + or card["revision"] != "deadbeef" + or {"recall", "specificity", "gate", "runner"} - set(card["methodology"]) + or [c["name"] for c in card["cases"]] != [s.name for s in cases]): + fails.append(f"scorecard: shape/totals wrong: {card}") + if (json.dumps(card, sort_keys=True) + != json.dumps(scorecard(cases, 3, [("corpus/x", 4)], "deadbeef"), + sort_keys=True)): + fails.append("scorecard: must be deterministic for identical inputs") + # 3) gate: precision absolute (a dirty fix or any FP fails regardless of recall), # recall gated only against the floor. gate_checks = [ @@ -360,6 +448,9 @@ def main(argv: list[str]) -> int: ap.add_argument("--min-recall", type=_non_negative_int, default=0, metavar="N", help="fail if fewer than N before.cs cases are caught (the pinned " "recall floor; specificity + zero-FP are always required)") + ap.add_argument("--json", default=None, metavar="PATH", + help="also write the publishable scorecard (numbers + corpus + " + "revision + methodology) as JSON") args = ap.parse_args(argv) if args.selftest: return _selftest() @@ -367,7 +458,7 @@ def main(argv: list[str]) -> int: corpus_dirs = args.corpus or [os.path.join(root, "corpus", "real-world"), os.path.join(root, "corpus", "wpf"), os.path.join(root, "corpus", "di")] - return run(root, corpus_dirs, args.min_recall) + return run(root, corpus_dirs, args.min_recall, args.json) if __name__ == "__main__": diff --git a/scripts/flagship-demo.sh b/scripts/flagship-demo.sh new file mode 100755 index 00000000..0d5e35da --- /dev/null +++ b/scripts/flagship-demo.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# The flagship leak-to-retention-path demo, as a DUMB orchestrator (alpha A4). +# +# It prepares nothing clever and decides nothing clever: build the sample and +# the witness, run the bad app held live, attach the witness through its +# PUBLIC CLI, machine-validate the JSON artifact against the human verdict, +# print ONE stable summary line, clean up. All classification/verdict logic +# stays in the witness — a demo script that grows its own opinion about +# retention becomes a second, worse witness. +# +# Usage: scripts/flagship-demo.sh [bad|ok] (default: bad) +# +# Exit: 0 — the variant behaved exactly as documented (bad: RETAINED via +# static-event; ok: no established retention); +# 1 — it did not, or the human and JSON verdicts DISAGREE; +# 2 — orchestration failure (build/launch/timeout). +set -u + +VARIANT="${1:-bad}" +case "$VARIANT" in bad|ok) ;; *) echo "usage: $0 [bad|ok]" >&2; exit 2 ;; esac + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +APP_DIR="$ROOT/examples/flagship/console/$VARIANT" +WITNESS_DIR="$ROOT/audit/runtime/RetentionPath" +WORK="$(mktemp -d)" +APP_PID="" + +cleanup() { + [ -n "$APP_PID" ] && kill "$APP_PID" 2>/dev/null + rm -rf "$WORK" +} +trap cleanup EXIT INT TERM + +fail() { echo "flagship-demo: FAIL: $*" >&2; exit "${2:-2}"; } + +# ---- build (quietly; full logs kept for the failure path) -------------------- +dotnet build "$APP_DIR" -c Release -v quiet > "$WORK/build-app.log" 2>&1 \ + || { cat "$WORK/build-app.log" >&2; fail "sample build failed"; } +dotnet build "$WITNESS_DIR" -c Release -v quiet > "$WORK/build-witness.log" 2>&1 \ + || { cat "$WORK/build-witness.log" >&2; fail "witness build failed"; } + +APP_DLL=$(ls "$APP_DIR"/bin/Release/net8.0/*DocumentApp.dll) || fail "sample dll not found" +WITNESS_DLL="$WITNESS_DIR/bin/Release/net8.0/RetentionPath.dll" + +# ---- launch the app held live (the OWEN_FLAGSHIP_HOLD contract) -------------- +mkfifo "$WORK/stdin" +OWEN_FLAGSHIP_HOLD=1 dotnet "$APP_DLL" > "$WORK/app.log" 2>&1 < "$WORK/stdin" & +APP_PID=$! +exec 9> "$WORK/stdin" # keep ReadLine blocked + +for _ in $(seq 1 30); do + grep -q "holding (pid" "$WORK/app.log" 2>/dev/null && break + kill -0 "$APP_PID" 2>/dev/null || { cat "$WORK/app.log" >&2; fail "app exited before holding"; } + sleep 1 +done +grep -q "holding (pid" "$WORK/app.log" || { cat "$WORK/app.log" >&2; fail "app did not reach the hold point in 30s"; } + +# ---- the witness, through its public CLI ------------------------------------- +timeout 120 dotnet "$WITNESS_DLL" roots --pid "$APP_PID" \ + --type Owen.Flagship.DocumentView --out "$WORK/witness.json" \ + > "$WORK/witness.log" 2>&1 +WITNESS_RC=$? +echo >&9 || true # release the app + +# ---- machine validation: JSON is the artifact, grep is not a parser ---------- +SUMMARY=$(python3 - "$VARIANT" "$WITNESS_RC" "$WORK/witness.json" "$WORK/witness.log" "$WORK/app.log" <<'PY' +import json, re, sys +variant, rc, json_path, log_path, app_log = sys.argv[1], int(sys.argv[2]), sys.argv[3], sys.argv[4], sys.argv[5] +human = open(log_path, encoding="utf-8", errors="replace").read() +app = open(app_log, encoding="utf-8", errors="replace").read() +m = re.search(r"still subscribed", app) +subs = re.search(r"(\d[\d,]*) still subscribed", app) +problems = [] + +if variant == "bad": + if rc != 1: + problems.append(f"witness exit {rc}, want 1 (RETAINED)") + try: + doc = json.load(open(json_path, encoding="utf-8")) + except Exception as e: + problems.append(f"JSON artifact unreadable: {e}") + doc = {} + if doc.get("verdict") != "RETAINED": + problems.append(f"JSON verdict {doc.get('verdict')!r}, want RETAINED") + if "verdict: RETAINED" not in human: + problems.append("human output lacks 'verdict: RETAINED'") + roots = (doc.get("retained") or [{}])[0].get("roots") or [] + kinds = {r.get("kind") for r in roots} + if "static-event" not in kinds: + problems.append(f"no static-event root in JSON (got {sorted(kinds)})") + # stable semantic anchors, not verbatim addresses/paths + path_text = " ".join(" ".join(r.get("path", [])) for r in roots) + for anchor in ("AppSettings", "PropertyChanged", "_invocationList", "DocumentView"): + if anchor not in path_text + " ".join(str(r.get("holder", "")) for r in roots) + human: + problems.append(f"retention path lacks the '{anchor}' anchor") + if ("verdict: RETAINED" in human) != (doc.get("verdict") == "RETAINED"): + problems.append("human and JSON verdicts DISAGREE") + verdict_line = "leak DEMONSTRATED: static-event retention, path root->view established" +else: + # The user-level contract only: exit 0, verdict in {ABSENT, OBSERVED_ONLY}, + # zero durable retainers. Which of the two verdicts shows up depends on JIT + # liveness of a loop local — an internal CLR decision, not a public API; + # the witness selftest pins the verdict semantics deterministically. + if rc != 0: + problems.append(f"witness exit {rc}, want 0 (nothing durably retained)") + try: + doc = json.load(open(json_path, encoding="utf-8")) + except Exception as e: + problems.append(f"JSON artifact unreadable: {e}") + doc = {} + if doc.get("verdict") not in ("ABSENT", "OBSERVED_ONLY"): + problems.append(f"JSON verdict {doc.get('verdict')!r}, want ABSENT or OBSERVED_ONLY") + roots = (doc.get("retained") or [{}])[0].get("roots") or [] + durable = [r.get("kind") for r in roots if r.get("kind") not in ("stack", "finalizer")] + if durable: + problems.append(f"ok variant has durable retainer(s): {durable}") + if "verdict: RETAINED" in human: + problems.append("ok variant must not be RETAINED") + if (doc.get("verdict") in ("ABSENT", "OBSERVED_ONLY")) != ("verdict: RETAINED" not in human): + problems.append("human and JSON verdicts DISAGREE") + verdict_line = f"fix VERIFIED: no established retention ({doc.get('verdict')})" + +if problems: + for p in problems: + print(f"flagship-demo: FAIL: {p}", file=sys.stderr) + sys.exit(1) +subs_n = subs.group(1) if subs else "?" +print(f"flagship-demo[{variant}]: app reported {subs_n} live subscription(s); {verdict_line}") +PY +) || { cat "$WORK/witness.log" >&2; fail "validation failed (witness log above)" 1; } + +echo "$SUMMARY" diff --git a/spec/CLI.md b/spec/CLI.md index b19a4f99..cc3a7adb 100644 --- a/spec/CLI.md +++ b/spec/CLI.md @@ -17,6 +17,10 @@ Notes: - `check`'s non-zero exit on errors is what makes it usable as a CI gate. +- An **internal crash** of any command exits **70** (EX_SOFTWARE) with one + actionable line — never a traceback by default (`OWNLANG_DEBUG=1` re-raises), + and never a code a caller could read as findings (1) or clean (0) + (`ownlang.__main__.run`). The `owen` CLI maps 70 to its own exit 5. - **`own-check --config `** (the shell/Action wrapper, `scripts/own-check.sh`) reads the same file via `config` and forwards the declared weak-subscribe wrapper names to the Roslyn extractor (`--weak-subscribe "SimpleType.Method"`, internal diff --git a/tests/test_cli_contract.py b/tests/test_cli_contract.py new file mode 100644 index 00000000..44e340c5 --- /dev/null +++ b/tests/test_cli_contract.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""The first-run failure contract of `python -m ownlang` (alpha gate A1). + +An internal crash of any command exits 70 (EX_SOFTWARE) with ONE actionable +line — never a traceback a first-time user has to parse, and never an exit +code a caller could mistake for findings (1) or clean (0): pre-A1 a core +crash exited 1, and `owen check` without `--fail-on-finding` mapped that to +a CLEAN scan. `OWNLANG_DEBUG=1` prints the full traceback but KEEPS exit 70 +(a re-raise would exit 1 and reopen the same hole). Deliberate contract +errors keep their own codes (spec/CLI.md). + +Run: python tests/test_cli_contract.py + python tests/run_tests.py (runs it in the suite) +""" + +from __future__ import annotations + +import os +import subprocess +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +_CRASH_DRIVER = """ +import sys; sys.path.insert(0, {root!r}) +from unittest import mock +import ownlang.__main__ as m +with mock.patch.object(m, "main", side_effect=RuntimeError("synthetic crash")): + sys.exit(m.run(["ownir", "x.json"])) +""" + + +def _crash(env_extra: dict[str, str]) -> subprocess.CompletedProcess[str]: + root = os.path.join(os.path.dirname(__file__), "..") + env = {k: v for k, v in os.environ.items() if k != "OWNLANG_DEBUG"} + env.update(env_extra) + return subprocess.run( + [sys.executable, "-c", _CRASH_DRIVER.format(root=root)], + capture_output=True, text=True, env=env, check=False) + + +def run() -> int: + fails: list[str] = [] + checks = 0 + + # 1. Internal crash -> exit 70, one polite line, no traceback. + checks += 1 + r = _crash({}) + if r.returncode != 70: + fails.append(f"internal crash must exit 70 (EX_SOFTWARE), got {r.returncode}") + if "Traceback" in r.stderr: + fails.append("internal crash must not print a raw traceback by default") + if "ownlang: internal error" not in r.stderr: + fails.append(f"expected the polite one-liner, got: {r.stderr!r}") + + # 2. OWNLANG_DEBUG=1 -> full traceback, exit STILL 70 (never 1). + checks += 1 + r = _crash({"OWNLANG_DEBUG": "1"}) + if r.returncode != 70: + fails.append(f"debug crash must still exit 70, got {r.returncode} — " + f"exit 1 reads as findings and owen maps it to clean") + if "Traceback" not in r.stderr: + fails.append("OWNLANG_DEBUG=1 must print the full traceback") + + # 3. A missing facts file is a deliberate contract error (2), politely. + checks += 1 + r = subprocess.run( + [sys.executable, "-m", "ownlang", "ownir", "/no/such/facts.json"], + capture_output=True, text=True, check=False, + cwd=os.path.join(os.path.dirname(__file__), "..")) + if r.returncode != 2 or "Traceback" in r.stderr or "cannot read" not in r.stderr: + fails.append(f"missing facts must be a polite exit 2, got rc {r.returncode}: " + f"{r.stderr!r}") + + # 4. Usage errors keep exit 2 (the catch-all must not swallow them). + checks += 1 + r = subprocess.run( + [sys.executable, "-m", "ownlang", "no-such-command"], + capture_output=True, text=True, check=False, + cwd=os.path.join(os.path.dirname(__file__), "..")) + if r.returncode != 2: + fails.append(f"unknown command must stay exit 2, got {r.returncode}") + + if fails: + for f in fails: + print(f"FAIL: cli contract {f}") + return 1 + print(f"cli first-run contract OK: {checks} checks passed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(run())