From b1cb56585d24c83616063fabbf7f26d629ffef43 Mon Sep 17 00:00:00 2001 From: Dmitrii Fedorov Date: Thu, 1 Oct 2026 13:50:41 -0300 Subject: [PATCH] feat(reproduce): index every capability card against a run of the profile (Refs #989) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit S12 of #988, fourth part: a machine-readable capability -> spec -> artifact -> build/test -> evidence index, with every capability measured complete, explicitly blocked or excluded with a reason, and no figure that counts catalog entries. tools/capabilities.py index --record R verifies the vendored contracts against their lock, imports the S01 checker itself (external/t27/tools/trinity_manifest.py) for its inventory and check of this tree, reads a record of the headless profile (installs with sha256, compiled test roots, the test verdict; a record of another revision is refused), runs the planned acceptance commands, and writes capabilities-.json and a table for the job summary. A card is excluded by its disposition, its profile, or because nothing the profile builds, tests or runs measures it; measured when every install it owns is hashed in the record, every root of the profile's test step it owns passed, and its acceptance is the profile's command, ran and exited 0, or is declined with a reason -- at least one of these being real evidence; blocked otherwise, and a blocked card must be in KNOWN_BLOCKED, as one there must still be blocked. Each row links its canonical spec with the lock's sha256 and what the pinned compiler made of its tests. specs/reproduce/capabilities.t27 holds only the measuring: what runs, what does not and why, the known blocks, and which S01 checker findings are drift. reproduce.yml builds the index in both legs. At this head on aarch64: 51 cards, 21 measured, 3 blocked, 27 excluded with a reason. The blocks: mcp.needle-mcp and mcp.trinity-mcp name steps build.zig comments out (the S01 inventory counts them because its parser does not skip // lines), and registry.commands' acceptance cannot pass because the export writes its own time into generated_at. Importing the vendored checker wrote a __pycache__ into external/t27, which the lock check then refused; bytecode is no longer written there. Индекс всех карточек возможностей по прогону профиля. S12 эпика #988, четвёртая часть: машиночитаемый индекс возможность -> спека -> артефакт -> сборка/тесты -> доказательство, где каждая возможность измерена, явно заблокирована или исключена с причиной, и ни одна цифра не считает записи каталога. tools/capabilities.py index --record R сверяет скопированные контракты с их lock'ом, импортирует сам проверяющий S01 (external/t27/tools/trinity_manifest.py) ради его инвентаря и проверки этого дерева, читает запись headless-профиля (установки с sha256, собранные корни тестов, вердикт тестов; запись другой ревизии отвергается), запускает запланированные acceptance-команды и пишет capabilities-.json и таблицу для сводки job. Карточка исключается по своему disposition, профилю или потому что её ничто из того, что профиль собирает, тестирует или запускает, не меряет; измерена, когда каждая её установка захэширована в записи, каждый её корень тестового шага профиля прошёл, а её acceptance -- команда профиля, запущена и вышла с 0 или отклонена с причиной, -- и хотя бы одно из этого настоящее доказательство; иначе заблокирована, и заблокированная карточка должна быть в KNOWN_BLOCKED, как и любая оттуда должна всё ещё быть заблокированной. Каждая строка связывает каноническую спеку с её sha256 из lock'а и тем, что закреплённый компилятор сделал с её тестами. specs/reproduce/capabilities.t27 содержит только способ измерения: что запускается, что нет и почему, известные блоки и какие находки проверяющего S01 -- дрейф. reproduce.yml строит индекс в обеих ногах. На этом коммите на aarch64: 51 карточка, 21 измерена, 3 заблокированы, 27 исключены с причиной. Блоки: mcp.needle-mcp и mcp.trinity-mcp называют шаги, которые build.zig закомментировал (инвентарь S01 их считает -- его парсер не пропускает строки //), а acceptance registry.commands не может пройти, потому что экспорт пишет своё время в generated_at. Импорт скопированного проверяющего писал __pycache__ в external/t27, и проверка lock'а его отвергала; байткод там больше не пишется. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/reproduce.yml | 15 +- specs/reproduce/capabilities.t27 | 60 +++++ tools/capabilities.py | 418 +++++++++++++++++++++++++++++++ 3 files changed, 492 insertions(+), 1 deletion(-) create mode 100644 specs/reproduce/capabilities.t27 create mode 100644 tools/capabilities.py diff --git a/.github/workflows/reproduce.yml b/.github/workflows/reproduce.yml index ef653fbb1d..94c923954b 100644 --- a/.github/workflows/reproduce.yml +++ b/.github/workflows/reproduce.yml @@ -6,7 +6,8 @@ name: Reproduce the headless profile # empty directory and setup-zig keeps no cache, so nothing built before is reused. # The record of the run -- each command with its exit code, the compile failures by # root, the run steps, every compiled test root and every install with its sha256 -- -# is uploaded as the artifact reproduce-headless-. +# is uploaded as the artifact reproduce-headless-, with the capability index of +# tools/capabilities.py built on it (capabilities-.json, and a table in the job summary). on: pull_request: @@ -18,6 +19,8 @@ on: - "external/**" - "deploy/**/*.zig" - "tools/reproduce.py" + - "tools/capabilities.py" + - "tools/contracts.py" - "specs/reproduce/**" - ".github/workflows/reproduce.yml" push: @@ -61,6 +64,15 @@ jobs: - name: Build and test the headless profile with fresh caches run: python3 tools/reproduce.py headless --out reproduce-headless.json + # Every S01 capability card held to this run: measured (installs hashed, test roots passed, + # acceptance commands run), blocked (each in KNOWN_BLOCKED with its reason) or excluded with + # its reason -- specs/reproduce/capabilities.t27. The table goes to the job summary. + - name: Negative control of the capability index + run: python3 tools/capabilities.py --self-check + + - name: Index the capabilities against this run + run: python3 tools/capabilities.py index --record reproduce-headless.json --out capabilities-${{ matrix.arch }}.json --summary "$GITHUB_STEP_SUMMARY" + - name: Upload the record if: always() uses: actions/upload-artifact@v4 @@ -71,4 +83,5 @@ jobs: path: | reproduce-headless.json reproduce-headless.*.log + capabilities-*.json if-no-files-found: error diff --git a/specs/reproduce/capabilities.t27 b/specs/reproduce/capabilities.t27 new file mode 100644 index 0000000000..60d33ffa87 --- /dev/null +++ b/specs/reproduce/capabilities.t27 @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 +; specs/reproduce/capabilities.t27 -- how this repository measures each capability card of S01 +; S12 of gHashTag/trinity#988 (gHashTag/trinity#989): publish a machine-readable capability -> spec -> +; artifact -> build/test -> evidence index, with every capability measured complete, explicitly blocked +; or intentionally excluded with a reason, and no coverage figure taken from catalog counts. The cards +; are t27's (external/t27, locked byte for byte by specs/reproduce/contracts.t27) and are not restated +; here: this module is only the measuring -- which acceptance commands tools/capabilities.py runs, which +; it does not and why, and which capabilities are known to be blocked, each with what blocks it. +; A headless card with an ACCEPTANCE is in exactly one place: the profile's own commands (BUILD or TEST +; of specs/reproduce/headless.t27, measured by the record of a headless run), RUN, or NOT_RUN. A card in +; none of them, or in two, fails the index; so does a name here that is no card. +; Not a claim that a capability works beyond what its row says was measured. +; ASCII only (L3). phi^2 + 1/phi^2 = 3 | TRINITY + +module reproduce_capabilities; + +pub const KIND : str = "capability-index"; +pub const ID : str = "reproduce/capabilities"; +; Run as the card writes the command, with -Dci=true added to every `zig build` in it (the cards are +; headless, and the headless profile is -Dci=true), stdin closed, at most RUN_TIMEOUT_S seconds each. +; A tracked file the command changes is recorded with the run and restored after it. +pub const RUN : [11]str = [ + "trinity/abi.c-api", + "trinity/agent.phi-loop", + "trinity/cli.tri", + "trinity/convert.b2t", + "trinity/examples.zig", + "trinity/llm.firebird", + "trinity/mcp.needle-mcp", + "trinity/mcp.trinity-mcp", + "trinity/queen.lib", + "trinity/registry.commands", + "trinity/vibee.compiler" +]; +pub const RUN_TIMEOUT_S : u32 = 600; +pub const NOT_RUN : [3]str = ["trinity/agent.tri-api", "trinity/bot.tri-bot", "trinity/node.trinity-node"]; +pub const NOT_RUN_REASONS : [3]str = [ + "zig build tri-api runs the agent against the Anthropic API, which needs a key; the build and install of exe:tri-api are what is measured", + "zig build tri-bot runs the Telegram bot until it is stopped, which needs a token; the build and install of exe:tri-bot are what is measured", + "zig build node runs the inference node until it is stopped; the build and install of exe:trinity-node are what is measured" +]; +; Capabilities known to be blocked, and what blocks each. One that measures complete fails the index +; until its line is removed, and one that is blocked and not here fails it too. +pub const KNOWN_BLOCKED : [3]str = ["trinity/mcp.needle-mcp", "trinity/mcp.trinity-mcp", "trinity/registry.commands"]; +pub const KNOWN_BLOCKED_REASONS : [3]str = [ + "zig build needle-mcp: there is no such step -- build.zig comments it out (around line 1507), and the S01 inventory counts it because tools/trinity_manifest.py does not skip // lines", + "zig build trinity-mcp: there is no such step -- build.zig comments it out (around line 1551), as for needle-mcp", + "zig build export-registry && git diff --exit-code .trinity/registry.json cannot pass: the export writes the time it ran into generated_at" +]; +; Findings of the S01 checker that are drift since the inventory S01 pinned (gHashTag/trinity@976df517) +; and are reported, not failed: the pin itself, and the counts of a tree that kept growing. +pub const DRIFT_CODES : [2]str = ["PIN_MISMATCH", "COUNT_MISMATCH"]; +; Dispositions whose cards are excluded from measuring by what they are, with the card's own reason. +pub const EXCLUDED_DISPOSITIONS : [4]str = ["deprecated", "out-of-scope", "catalog-only", "external"]; +pub const ENABLED : bool = true; + +test the_plan_is_whole { + assert KIND == "capability-index"; + assert RUN_TIMEOUT_S == 600; +} diff --git a/tools/capabilities.py b/tools/capabilities.py new file mode 100644 index 0000000000..7a6fda9950 --- /dev/null +++ b/tools/capabilities.py @@ -0,0 +1,418 @@ +#!/usr/bin/env python3 +"""The capability index: every S01 capability card, held to this tree and to a run of the headless +profile -- measured, blocked, or excluded with its reason. + +WHY THIS EXISTS +--------------- +S12 of gHashTag/trinity#988 (gHashTag/trinity#989) asks for a machine-readable capability -> spec -> +artifact -> build/test -> evidence index, for every capability measured complete, explicitly blocked or +intentionally excluded with a reason, and for no coverage figure that counts catalog entries as code. +The S01 cards say "measured" when a CI run once built the tree; nothing yet held each card to what a +clean run of this tree builds, installs, tests and runs. + +WHAT IT READS +------------- + * the cards and the S01 checker, external/t27 -- verified first against specs/reproduce/contracts.t27 + (tools/contracts.py check); an index built on unverified cards is not built; + * the S01 checker's own inventory and check of this tree (external/t27/tools/trinity_manifest.py, + imported): target ownership, paths and dialects are judged by the canonical code, not a copy; + * the record of a headless run (python3 tools/reproduce.py headless --out R): the installs with their + sha256, the compiled test roots, the test verdict; + * specs/reproduce/capabilities.t27: which acceptance commands to run, which not and why, and the + capabilities known to be blocked. + +WHAT IT DECIDES, PER CARD +------------------------- + excluded its disposition excludes it (deprecated, out-of-scope, catalog-only, external), or its + profile is not headless, or it names nothing to measure -- with the reason; + measured every exe and lib it owns that the profile installs is in the record with its sha256, every + test root it owns compiled in a test step with no failure, and its acceptance is the + profile's own command, or ran and exited 0, or is declined with a reason; + blocked anything of that failed -- with what failed. +A blocked capability must be in KNOWN_BLOCKED, and one there must still be blocked. The S01 checker's +findings fail the index, except DRIFT_CODES (the pin and the counts), which are reported. + +Usage (from the repository root, on a clean tree): + python3 tools/capabilities.py index --record R.json [--out capabilities.json] [--summary FILE] + python3 tools/capabilities.py --self-check +Exit codes: 0 every capability is what the index says and nothing unexpected is blocked; 1 a violation +(each printed); 2 could not run. Standard library only. +""" +from __future__ import annotations + +import argparse +import fnmatch +import importlib.util +import json +import os +import pathlib +import re +import subprocess +import sys +import time + +ROOT = pathlib.Path(__file__).resolve().parent.parent +PLAN = "specs/reproduce/capabilities.t27" +PROFILE_SPEC = "specs/reproduce/headless.t27" +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) +import contracts # noqa: E402 (the lock, its reader and its check) + +CARDS = "specs/trinity/capabilities" + + +def load_checker(root: pathlib.Path, vendor_dir: str): + """The S01 checker as t27 holds it -- the vendored file, imported, never a copy of its logic.""" + path = root / vendor_dir / "tools/trinity_manifest.py" + spec = importlib.util.spec_from_file_location("trinity_manifest", path) + module = importlib.util.module_from_spec(spec) + # No __pycache__ beside it: a .pyc in external/t27 is a file the lock does not hold, and the + # next check of the vendored copy would rightly refuse it. + saved, sys.dont_write_bytecode = sys.dont_write_bytecode, True + try: + spec.loader.exec_module(module) + finally: + sys.dont_write_bytecode = saved + return module + + +def with_ci(command: str) -> str: + """Each `zig build` in the command gains -Dci=true unless it has it: the cards are headless.""" + return re.sub(r"\bzig build\b(?![^&;|]*-Dci=true)([^&;|]*)", lambda m: f"zig build{m.group(1).rstrip()} -Dci=true ", command).strip() + + +# =========================================================================================== +# The decision, pure: cards, ownership, the record, the plan and the runs in; rows and violations out +# =========================================================================================== +def classify(cards: dict, owned: dict, installed: set, record: dict, plan: dict, runs: dict, profile_cmds: set, test_roots: set, head: str) -> tuple: + rows, v = [], [] + + def bad(code, text): + v.append(f"{code}: {text}") + known = set(cards) + for name in ("RUN", "NOT_RUN", "KNOWN_BLOCKED"): + for cid in sorted(set(plan[name]) - known): + bad("PLAN_UNKNOWN", f"{PLAN} {name} names {cid}, which is no card") + for cid in sorted(set(plan["RUN"]) & set(plan["NOT_RUN"])): + bad("PLAN_CONFLICT", f"{cid} is both run and not run") + if record.get("violations"): + bad("RECORD", f"the headless record it reads has violations, so the profile itself did not hold: {record['violations'][:3]}") + if record.get("head") != head: + bad("RECORD", f"the headless record is of {record.get('head')}, the tree indexed is {head}: its installs and tests are another revision's") + t = record["test"] + roots_compiled = t.get("compiled_roots") + test_clean = t.get("rc") == 0 and not t.get("failed_tests") and not t["summary"].get("tests_failed") + declined = dict(zip(plan["NOT_RUN"], plan["NOT_RUN_REASONS"])) + for cid in sorted(cards): + c = cards[cid] + row = {"id": cid, "title": c.get("TITLE", ""), "disposition": c.get("DISPOSITION"), "profile": c.get("PROFILE"), + "work_package": c.get("WORK_PACKAGE"), "card_evidence": c.get("EVIDENCE"), "spec": c.get("CANONICAL_SPEC", ""), + "targets": [], "acceptance": None} + acceptance = c.get("ACCEPTANCE", "") + if c.get("DISPOSITION") in plan["EXCLUDED_DISPOSITIONS"]: + row.update(status="excluded", reason=f"disposition {c['DISPOSITION']}: {c.get('EVIDENCE_SOURCE') or c.get('NOTE', '')}"[:400]) + rows.append(row) + continue + if c.get("PROFILE") != "headless": + row.update(status="excluded", reason=f"profile {c.get('PROFILE')}: S12 reproduces the approved headless profile only") + rows.append(row) + continue + if not owned.get(cid) and not acceptance: + row.update(status="excluded", reason="the card owns no build target and names no acceptance command: nothing to measure") + rows.append(row) + continue + problems = [] + for key in owned.get(cid, []): + kind, name = key.split(":", 1) + entry = {"target": key} + if kind in ("exe", "lib") and key in installed: + path = f"bin/{name}" if kind == "exe" else f"lib/lib{name}.a" + digest = record.get("installs", {}).get(path) + entry.update(artifact=path, sha256=digest) + if not digest: + problems.append(f"{key}: the profile installs {path} by default and the record has no such file") + elif kind in ("exe", "lib"): + entry.update(note="not installed by the profile; built by a named step") + elif kind == "test" and name in test_roots: + compiled = roots_compiled is None or name in roots_compiled + entry.update(compiled=compiled if roots_compiled is not None else "not read (reused caches)", passed=test_clean) + if not compiled: + problems.append(f"{key}: a root of the profile's test step that the step did not compile") + if not test_clean: + problems.append(f"{key}: the test step had failures ({t['summary'].get('line')})") + elif kind == "test": + entry.update(note="a test root of another step, outside the profile's test step") + else: + entry.update(note="a named step; measured only through an acceptance command") + row["targets"].append(entry) + if acceptance: + if acceptance.replace(" --summary all", "") in profile_cmds: + row["acceptance"] = {"command": acceptance, "measured_by": "the headless record", "rc": t.get("rc") if "test" in acceptance else record.get("build", {}).get("rc")} + if row["acceptance"]["rc"] != 0: + problems.append(f"acceptance {acceptance!r} is the profile's own command, and it exited {row['acceptance']['rc']}") + elif cid in plan["RUN"]: + r = runs.get(cid) + row["acceptance"] = r + if r is None: + problems.append("planned to run, and no run is recorded") + elif r["rc"] != 0: + problems.append(f"acceptance {r['command']!r} exited {r['rc']}: {r.get('tail', '')[-300:]}") + elif cid in declined: + row["acceptance"] = {"command": acceptance, "not_run": declined[cid]} + else: + bad("UNPLANNED", f"{cid}: its acceptance {acceptance!r} is neither the profile's, nor run, nor declined with a reason") + # Measured means something was: an install hashed, a root of the profile's test step passed, + # or an acceptance that exited 0. A card that owns only named steps, or roots of other steps, + # and declines or lacks a command has nothing measured, and says so. + evidence = [e for e in row["targets"] if e.get("sha256") or e.get("passed") is True] + acc = row["acceptance"] or {} + if acc.get("rc") == 0: + evidence.append(acc) + if problems: + row.update(status="blocked", problems=problems) + elif evidence: + row.update(status="measured", problems=[]) + else: + row.update(status="excluded", problems=[], reason="nothing the profile builds, tests or runs measures it: it owns " + + (", ".join(e["target"] for e in row["targets"]) or "no target") + (f", and its acceptance is not run: {acc['not_run']}" if acc.get("not_run") else ", and names no acceptance")) + rows.append(row) + blocked = {r["id"] for r in rows if r["status"] == "blocked"} + ledger = dict(zip(plan["KNOWN_BLOCKED"], plan["KNOWN_BLOCKED_REASONS"])) + for cid in sorted(blocked - set(ledger)): + bad("NEW_BLOCKED", f"{cid} is blocked and {PLAN} does not know it: {[r for r in rows if r['id'] == cid][0]['problems'][:2]}") + for cid in sorted(set(ledger) - blocked): + if cid in known: + bad("BLOCKED_NOW_MEASURED", f"{cid} is in KNOWN_BLOCKED and measured complete; remove its line") + for r in rows: + if r["id"] in ledger and r["status"] == "blocked": + r["reason"] = ledger[r["id"]] + return rows, v + + +def checker_verdict(findings: list, drift_codes: list) -> tuple: + """The S01 checker's findings on this tree: drift since its pinned inventory is reported, anything + else -- an unowned target, an absent path, a dialect, a duplicate -- fails the index.""" + v = [] + + def bad(code, text): + v.append(f"{code}: {text}") + drift = [f"{c}: {m}" for c, m in findings if c in drift_codes] + for c, m in findings: + if c not in drift_codes: + bad("CHECKER", f"{c}: {m}") + return drift, v + + +# =========================================================================================== +# The measuring: the lock, the checker on this tree, the cards, the runs +# =========================================================================================== +def run_acceptance(command: str, timeout: int) -> dict: + t0 = time.time() + try: + p = subprocess.run(["bash", "-c", command], cwd=str(ROOT), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, timeout=timeout) + rc, out = p.returncode, p.stdout + except subprocess.TimeoutExpired as e: + rc, out = None, (e.stdout.decode(errors="replace") if isinstance(e.stdout, bytes) else (e.stdout or "")) + changed = [x[3:] for x in subprocess.run(["git", "-C", str(ROOT), "status", "--porcelain", "--untracked-files=no"], + capture_output=True, text=True).stdout.split("\n") if x] + if changed: + subprocess.run(["git", "-C", str(ROOT), "checkout", "--"] + changed, capture_output=True) + return {"command": command, "rc": rc, "seconds": round(time.time() - t0, 1), "changed_tracked_files": changed, "tail": out[-1500:]} + + +def measure(record_path: pathlib.Path) -> dict: + lock = contracts.read_lock(ROOT) + findings = contracts.check(ROOT) + if findings: + raise RuntimeError(f"the vendored contracts are not what the lock says, so no index is built on them: {findings[:3]}") + plan = contracts.constants((ROOT / PLAN).read_text(encoding="utf-8"), PLAN) + profile = contracts.constants((ROOT / PROFILE_SPEC).read_text(encoding="utf-8"), PROFILE_SPEC) + record = json.loads(record_path.read_text(encoding="utf-8")) + tm = load_checker(ROOT, lock["VENDOR_DIR"]) + inv = tm.inventory(ROOT) # before any command can touch the tree: it refuses a dirty one + checker_findings, _ = tm.check(inv, ROOT / lock["VENDOR_DIR"] / "specs/trinity", ROOT / lock["VENDOR_DIR"]) + cards = {} + for path in sorted((ROOT / lock["VENDOR_DIR"] / CARDS).glob("*.t27")): + consts = {k: d["value"] for k, d in tm.parse_spec(path)["consts"].items()} + cards[consts["ID"]] = consts + targets = sorted({a["key"] for kind in ("executables", "libraries", "tests", "steps") for a in inv["build"][kind]}) + owned = {cid: sorted({t for p in c.get("TARGETS", []) for t in targets if fnmatch.fnmatchcase(t, p)}) for cid, c in cards.items()} + installed = set(inv["build"]["installed_by_default"]) + runs = {} + for cid in plan["RUN"]: + if cid in cards and cards[cid].get("ACCEPTANCE"): + runs[cid] = run_acceptance(with_ci(cards[cid]["ACCEPTANCE"]), plan["RUN_TIMEOUT_S"]) + rows, violations = classify(cards, owned, installed, record, plan, runs, {c.replace(" --summary all", "") for c in (profile["BUILD"], profile["TEST"])}, + set(profile["TEST_ROOTS"]), inv["sha"]) + # The spec link of each row, from the lock: where the contract lives, its bytes, and what the + # pinned compiler made of it (tools/contracts.py check --t27c, run by contracts.yml). + locked = dict(zip(lock["FILES"], lock["SHA256"])) + test_blocked = dict(zip(lock["TEST_BLOCKED"], lock["TEST_BLOCKED_ERRORS"])) + dropped = dict(zip(lock["PARSE_DISCARDS"], lock["PARSE_DISCARD_TOKENS"])) + for r in rows: + ref = r["spec"] + if not ref: + r["spec"] = {"ref": "", "note": "the card names no canonical spec"} + elif re.match(r"^[\w.-]+:", ref): + repo, path = ref.split(":", 1) + r["spec"] = {"ref": ref, "in": "this repository" if repo == "trinity" else repo, + "exists": (ROOT / path).is_file() if repo == "trinity" else "not checked here"} + else: + r["spec"] = {"ref": ref, "in": f"gHashTag/t27@{lock['T27_REVISION']}", "sha256": locked.get(ref), + "contract_tests": f"do not build: {test_blocked[ref]}" if ref in test_blocked else + "declares no test" if ref in lock["TEST_NONE"] else "run with the pinned compiler (contracts.yml)", + **({"parser_drops_tokens": dropped[ref]} if ref in dropped else {})} + drift, checker_violations = checker_verdict(checker_findings, plan["DRIFT_CODES"]) + violations += checker_violations + counts = {s: sum(1 for r in rows if r["status"] == s) for s in ("measured", "blocked", "excluded")} + return {"tool": "tools/capabilities.py", "head": inv["sha"], "host": record.get("host"), + "t27": {"revision": lock["T27_REVISION"], "lock": contracts.LOCK}, + "record": {"path": str(record_path), "head": record.get("head"), "fresh_caches": record.get("fresh_caches"), "zig": record.get("zig")}, + "checker": {"drift": drift, "findings": [f"{c}: {m}" for c, m in checker_findings if c not in plan["DRIFT_CODES"]]}, + "counts": dict(counts, total=len(rows)), "capabilities": rows, "violations": violations} + + +def summary_markdown(doc: dict) -> str: + c = doc["counts"] + out = [f"## Capability index at {doc['head'][:12]} ({doc['host']})", "", + f"{c['total']} capability cards of gHashTag/t27@{doc['t27']['revision'][:12]}: **{c['measured']} measured, {c['blocked']} blocked, " + f"{c['excluded']} excluded with a reason.** No figure here counts catalog entries.", "", + "| capability | status | evidence or reason |", "|---|---|---|"] + for r in doc["capabilities"]: + if r["status"] == "measured": + arts = [t["artifact"] for t in r["targets"] if t.get("artifact")] + tests = [t["target"][5:] for t in r["targets"] if t["target"].startswith("test:")] + acc = r["acceptance"] + how = (f"{len(arts)} install(s) hashed" if arts else "") + (f"; {len(tests)} test root(s) passed" if tests else "") + if acc and acc.get("rc") == 0 and "measured_by" not in acc: + how += f"; `{acc['command']}` exited 0" + elif acc and acc.get("rc") == 0: + how += f"; `{acc['command']}` is the profile's own command, exit 0 in the record" + elif acc and acc.get("not_run"): + how += "; acceptance not run (reason in the record)" + text = how.strip("; ") or "measured" + else: + text = r.get("reason") or "; ".join(r.get("problems", [])) + out.append(f"| `{r['id']}` | {r['status']} | {text.replace('|', '/')[:220]} |") + if doc["violations"]: + out += ["", "### Violations", ""] + [f"- {x}" for x in doc["violations"]] + return "\n".join(out) + "\n" + + +# =========================================================================================== +# --self-check: the decision on toy cards, each defect planted once +# =========================================================================================== +def self_check() -> int: + ok = True + seen = set() + + def expect(cond, what): + nonlocal ok + print(f" {'ok ' if cond else 'BAD'} {what}") + ok = ok and bool(cond) + card = lambda cid, **kw: {"ID": cid, "TITLE": cid, "DISPOSITION": "executable", "PROFILE": "headless", "EVIDENCE": "measured", + "ACCEPTANCE": "", "TARGETS": [], **kw} + cards = {"t/lib": card("t/lib", ACCEPTANCE="zig build -Dci=true"), "t/cli": card("t/cli", ACCEPTANCE="zig build cli && ./zig-out/bin/cli --version"), + "t/web": card("t/web", PROFILE="web"), "t/old": card("t/old", DISPOSITION="deprecated", EVIDENCE_SOURCE="kept for history"), + "t/bot": card("t/bot", ACCEPTANCE="zig build bot"), "t/mcp": card("t/mcp", ACCEPTANCE="zig build mcp"), "t/corpus": card("t/corpus"), + "t/steps": card("t/steps")} + owned = {"t/lib": ["lib:core", "test:src/core.zig", "test:src/other_step.zig"], "t/cli": ["exe:cli"], "t/bot": ["exe:bot"], "t/mcp": ["exe:mcp"], + "t/steps": ["step:release", "test:src/other_step.zig"]} + installed = {"lib:core", "exe:cli", "exe:bot", "exe:mcp"} + record = {"host": "Linux x86_64", "head": "f" * 40, "violations": [], "build": {"rc": 0}, + "installs": {"lib/libcore.a": "a" * 64, "bin/cli": "b" * 64, "bin/bot": "c" * 64, "bin/mcp": "d" * 64}, + "test": {"rc": 0, "summary": {"line": "Build Summary: 2/2 steps succeeded; 9/9 tests passed", "tests_failed": 0}, "failed_tests": [], + "compiled_roots": ["src/core.zig"]}} + plan = {"RUN": ["t/cli", "t/mcp"], "NOT_RUN": ["t/bot"], "NOT_RUN_REASONS": ["needs a token"], "KNOWN_BLOCKED": ["t/mcp"], + "KNOWN_BLOCKED_REASONS": ["no such step"], "EXCLUDED_DISPOSITIONS": ["deprecated", "out-of-scope", "catalog-only", "external"], + "DRIFT_CODES": ["PIN_MISMATCH"]} + runs = {"t/cli": {"command": "zig build cli -Dci=true && ./zig-out/bin/cli --version", "rc": 0}, + "t/mcp": {"command": "zig build mcp -Dci=true", "rc": 1, "tail": "no step named 'mcp'"}} + prof = {"zig build -Dci=true", "zig build test -Dci=true"} + + def run(**changes): + args = {"cards": cards, "owned": owned, "installed": installed, "record": record, "plan": plan, "runs": runs, "profile_cmds": prof, + "test_roots": {"src/core.zig"}, "head": "f" * 40} + args.update(changes) + rows, v = classify(**args) + got = {x.split(":")[0] for x in v} + seen.update(got) + return {r["id"]: r["status"] for r in rows}, got + status, got = run() + expect(not got and status == {"t/lib": "measured", "t/cli": "measured", "t/web": "excluded", "t/old": "excluded", "t/bot": "measured", + "t/mcp": "blocked", "t/corpus": "excluded", "t/steps": "excluded"}, + f"a tree that is what the plan says: measured, excluded with reasons, the known block blocked, nothing else ({status}, {got})") + _, got = run(record=dict(record, installs={k: v for k, v in record["installs"].items() if k != "bin/cli"})) + expect("NEW_BLOCKED" in got, "planted: an install the profile makes is missing from the record") + _, got = run(record=dict(record, test=dict(record["test"], rc=1, failed_tests=[{"test": "core.test.x"}]))) + expect("NEW_BLOCKED" in got and "RECORD" not in got, "planted: a failed test under an owned root") + _, got = run(record=dict(record, test=dict(record["test"], compiled_roots=["src/elsewhere.zig"]))) + expect("NEW_BLOCKED" in got, "planted: an owned root of the test step that the step did not compile") + _, got = run(runs=dict(runs, **{"t/cli": dict(runs["t/cli"], rc=2, tail="error")})) + expect("NEW_BLOCKED" in got, "planted: an acceptance command that fails") + _, got = run(runs=dict(runs, **{"t/mcp": dict(runs["t/mcp"], rc=0)})) + expect("BLOCKED_NOW_MEASURED" in got, "planted: a known block that measures complete (the ledger would outlive it)") + _, got = run(plan=dict(plan, RUN=["t/mcp"])) + expect("UNPLANNED" in got, "planted: a headless acceptance nobody planned") + _, got = run(plan=dict(plan, NOT_RUN=["t/bot", "t/cli"], NOT_RUN_REASONS=["needs a token", "x"])) + expect("PLAN_CONFLICT" in got, "planted: a card both run and not run") + _, got = run(plan=dict(plan, KNOWN_BLOCKED=["t/mcp", "t/gone"], KNOWN_BLOCKED_REASONS=["no such step", "x"])) + expect("PLAN_UNKNOWN" in got, "planted: a plan line for a card that does not exist") + _, got = run(record=dict(record, violations=["TEST_FAILED: ..."])) + expect("RECORD" in got, "planted: a headless record whose profile did not hold") + _, got = run(head="e" * 40) + expect("RECORD" in got, "planted: a record of another revision than the tree indexed") + _, got = run(runs={k: v for k, v in runs.items() if k != "t/cli"}) + expect("NEW_BLOCKED" in got, "planted: a planned run that did not happen") + expect(with_ci("zig build tri && ./zig-out/bin/tri --version") == "zig build tri -Dci=true && ./zig-out/bin/tri --version" + and with_ci("zig build examples && zig build e2e") == "zig build examples -Dci=true && zig build e2e -Dci=true" + and with_ci("zig build -Dci=true") == "zig build -Dci=true", f"-Dci=true is added to each zig build, once ({with_ci('zig build examples && zig build e2e')!r})") + drift, cv = checker_verdict([("PIN_MISMATCH", "pinned elsewhere"), ("UNOWNED_TARGET", "exe:new has no card")], plan["DRIFT_CODES"]) + seen.update(x.split(":")[0] for x in cv) + expect(drift == ["PIN_MISMATCH: pinned elsewhere"] and [x.split(":")[0] for x in cv] == ["CHECKER"], + "planted: a checker finding that is not drift fails, the pin is only reported") + src = pathlib.Path(__file__).read_text(encoding="utf-8").split("def run_acceptance(", 1)[0] + rules = set(re.findall(r'bad\("([A-Z_]+)"', src)) + expect(rules <= seen, f"every rule of the decision fired at least once ({len(rules & seen)}/{len(rules)}; never: {sorted(rules - seen)})") + print("capabilities --self-check:", "ok" if ok else "FAILED") + return 0 if ok else 1 + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("command", nargs="?", choices=["index"]) + ap.add_argument("--record", help="the JSON record of python3 tools/reproduce.py headless") + ap.add_argument("--out", default="capabilities.json") + ap.add_argument("--summary", help="also write a Markdown table here (for $GITHUB_STEP_SUMMARY)") + ap.add_argument("--self-check", action="store_true") + a = ap.parse_args() + if a.self_check: + return self_check() + if a.command != "index" or not a.record: + ap.print_help() + return 2 + try: + doc = measure(pathlib.Path(a.record)) + except (OSError, ValueError, RuntimeError, SystemExit) as e: + print(f"capabilities: could not run: {e}", file=sys.stderr) + return 2 + pathlib.Path(a.out).write_text(json.dumps(doc, indent=1, sort_keys=True) + "\n", encoding="utf-8") + md = summary_markdown(doc) + if a.summary: + with open(a.summary, "a", encoding="utf-8") as fh: + fh.write(md) + c = doc["counts"] + print(f"capabilities: {c['total']} cards at {doc['head'][:12]} ({doc['host']}): {c['measured']} measured, {c['blocked']} blocked, {c['excluded']} excluded") + for r in doc["capabilities"]: + if r["status"] != "measured": + print(f" {r['status']:8} {r['id']}: {(r.get('reason') or '; '.join(r.get('problems', [])))[:160]}") + for x in doc["checker"]["drift"]: + print(" drift since the S01 inventory:", x[:160]) + for x in doc["violations"]: + print("VIOLATION:", x) + print("capabilities:", "every capability is what the index says" if not doc["violations"] else f"{len(doc['violations'])} violation(s)") + return 1 if doc["violations"] else 0 + + +if __name__ == "__main__": + sys.exit(main())