diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 382aae17d3..522a66d8cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,21 +32,20 @@ jobs: # Build with -Dci=true to skip GUI targets requiring raylib zig build -Dci=true - # P0.4: Memory leak detection via Zig's General Purpose Allocator (GPA). - # Any memory leaks will cause the test to fail with a "leaked" error message. - # This is a HARD GATE - the build will fail if any tests leak memory. - # The leak checking is enabled by using std.testing.allocator in tests. - # Additional runtime leak checking can be enabled with --zig-lib-dir. - - name: Test (with leak detection) - run: | - # Run tests with Zig's built-in leak detection - # Tests using std.testing.allocator will automatically detect leaks - zig build test -Dci=true 2>&1 | tee test-output.txt - # Check for any memory leak indicators - if grep -i "leak\|memory.*leak\|allocation.*leak" test-output.txt; then - echo "::error::Memory leaks detected in tests" - exit 1 - fi + # P0.4: Memory leak detection via Zig's General Purpose Allocator (GPA) is kept: a + # leak fails this step as before. + # + # The test step is judged, not piped. It used to run `zig build test -Dci=true 2>&1 | tee` + # under bash without pipefail, so its exit code was tee's: on main five test roots do not + # compile -- src/trinity.zig, the library's own test root, among them -- and the step was + # green on every push while its summary said "124/135 steps succeeded; 5 failed" (#616; + # #987 is the tee fix alone). tools/reproduce.py reads the exit code from the process and + # holds the result to specs/reproduce/headless.t27: exactly the roots that file names as + # blocked may fail, a sixth fails this step, and so does a blocked root that compiles again + # (take it off the list); a failed test, a leak, fewer tests or fewer run steps fail it too. + # The record of the run is test-output.json. + - name: Test (judged against specs/reproduce/headless.t27) + run: python3 tools/reproduce.py test --reuse-cache --out test-output.json - name: Format check run: | diff --git a/.github/workflows/reproduce.yml b/.github/workflows/reproduce.yml new file mode 100644 index 0000000000..62732a8537 --- /dev/null +++ b/.github/workflows/reproduce.yml @@ -0,0 +1,59 @@ +name: Reproduce the headless profile + +# S12 of #988 (#989): the declared headless profile, built and tested from a clean +# clone on a runner with no cache, and judged against specs/reproduce/headless.t27. +# tools/reproduce.py points ZIG_GLOBAL_CACHE_DIR and ZIG_LOCAL_CACHE_DIR at a new +# empty directory and setup-zig keeps no cache, so nothing built before is reused. +# The record of the run -- each command with its exit code, the compile failures by +# root, the run steps, every compiled test root and every install with its sha256 -- +# is uploaded as the artifact reproduce-headless. + +on: + pull_request: + branches: [main] + paths: + - "build.zig" + - "build.zig.zon" + - "src/**" + - "external/**" + - "deploy/**/*.zig" + - "tools/reproduce.py" + - "specs/reproduce/**" + - ".github/workflows/reproduce.yml" + push: + branches: [main] + schedule: + - cron: "23 4 * * 1" + workflow_dispatch: + +jobs: + headless: + name: Headless profile from a clean clone + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + + - name: Setup Zig, no cache + uses: mlugg/setup-zig@v2 + with: + version: 0.15.2 + use-cache: false + + # A gate nobody has seen fail is not a gate: the judge is run on planted records + # first -- a swallowed exit code, a new compile failure, a blocked root that compiles, + # a failed test, a leak, a shrunken graph, an empty run step, a missing install -- + # and each must be reported. + - name: Negative control + run: python3 tools/reproduce.py --self-check + + - name: Build and test the headless profile with fresh caches + run: python3 tools/reproduce.py headless --out reproduce-headless.json + + - name: Upload the record + if: always() + uses: actions/upload-artifact@v4 + with: + name: reproduce-headless + path: reproduce-headless.json + if-no-files-found: error diff --git a/specs/reproduce/headless.t27 b/specs/reproduce/headless.t27 new file mode 100644 index 0000000000..5b2ccc7cee --- /dev/null +++ b/specs/reproduce/headless.t27 @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: Apache-2.0 +; specs/reproduce/headless.t27 -- the headless profile of gHashTag/trinity, reproduced from a clean clone +; S12 of gHashTag/trinity#988 (gHashTag/trinity#989): a set of specifications is not a recreated project until +; a clean checkout builds and tests the declared profile, and a failure anywhere fails the gate. This module is +; that profile, measured; tools/reproduce.py runs it and judges every run against it (the command is +; `python3 tools/reproduce.py headless`, the CI job is .github/workflows/reproduce.yml). +; MEASURED at gHashTag/trinity main afc9d38435ad01c48af6c33c311da422c38f046e (2026-09-30) with Zig 0.15.2, twice: +; Linux aarch64 in a container with empty caches, and the Build & Test job of CI run 36875247882 (Linux x86_64). +; Both print the same test summary, "124/135 steps succeeded; 5 failed; 2510/2510 tests passed", and until +; this module CI reported that summary as a success: its test step was `zig build test -Dci=true 2>&1 | tee` +; under bash without pipefail, so the step's exit code was tee's (gHashTag/trinity#616, the tee fix alone is +; gHashTag/trinity#987). Five test roots do not compile, the library's own src/trinity.zig among them, and +; their tests have not run since; they are BLOCKED below, each with the error and the capability card it +; holds back, and the gate fails the day one of them compiles until it is taken off the list. +; Not a claim about any profile but headless; not a claim that the 2510 tests that ran test enough. +; ASCII only (L3). phi^2 + 1/phi^2 = 3 | TRINITY + +module reproduce_headless; + +pub const KIND : str = "reproduce-profile"; +pub const ID : str = "reproduce/headless"; +pub const PROFILE : str = "headless"; +pub const MEASURED_AT : str = "afc9d38435ad01c48af6c33c311da422c38f046e"; +pub const ZIG : str = "0.15.2"; +; The documented clean-clone commands. tools/reproduce.py replaces the leading zig with the binary it is given +; and, unless asked to reuse caches, points ZIG_GLOBAL_CACHE_DIR and ZIG_LOCAL_CACHE_DIR at a new empty +; directory, so nothing a developer built before is reused. +pub const BUILD : str = "zig build -Dci=true --summary all"; +pub const TEST : str = "zig build test -Dci=true --summary all"; +pub const CLEAN_CLONE : str = "git clone https://github.com/gHashTag/trinity && cd trinity && python3 tools/reproduce.py headless"; +; The four packages build.zig.zon pins, exactly: zig verifies a package against its hash, so a changed pin is +; a different dependency that builds just as quietly. Changing one is a decision made here, not only there. +pub const DEPENDENCIES : [4]str = ["emsdk", "raylib", "zig_hdc", "zig_golden_float"]; +pub const DEPENDENCY_URLS : [4]str = ["git+https://github.com/emscripten-core/emsdk?ref=4.0.9#3bcf1dcd01f040f370e10fe673a092d9ed79ebb5", "git+https://github.com/raylib-zig/raylib-zig#cd71c85d571027ac8033357f83b124ee051825b3", "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/gHashTag/zig-hdc/archive/b73b2fa29874f4e30297f167a6ea8a3c2ded9d32.tar.gz", "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/gHashTag/zig-golden-float/archive/e7ce32885de2a8c50b7b6a3030d0592202145dd1.tar.gz"]; +pub const DEPENDENCY_HASHES : [4]str = ["N-V-__8AAJl1DwBezhYo_VE6f53mPVm00R-Fk28NPW7P14EQ", "raylib_zig-5.6.0-dev-KE8REENOBQC-m5nK7M2b5aKSIubJPbPLUYcRhT7aT3RN", "zig_hdc-0.1.0-6NS2Eg99FQCG62j7TmpGwVTnq07LhNb1FUHrwUacgWdO", "golden_float-2.1.0-h7LKhUUNCwAtKVHQ56wjridCZVwXY7_oxT2hXD9xCDdF"]; +; The build: every step succeeds, and these 46 files are installed under zig-out (45 executables and the +; static library) -- the INSTALLED_BY_DEFAULT of the S01 manifest (specs/trinity/project.t27 in gHashTag/t27). +pub const BUILD_STEPS_MIN : u32 = 94; +pub const INSTALLS : [46]str = [ + "bin/agent-entrypoint", "bin/arena", "bin/b2t", "bin/background-agent-api", + "bin/beal", "bin/bench-compress", "bin/bench-core", "bin/claude-ui", + "bin/example-memory", "bin/example-sequence", "bin/example-vm", "bin/firebird", + "bin/fluent", "bin/forge", "bin/hslm-train", "bin/mu-agent", + "bin/needle-mcp", "bin/pas-demo", "bin/phi-loop", "bin/photon-terminal", + "bin/pipeline-guard", "bin/quantum", "bin/railway-create-service", "bin/railway-redeploy", + "bin/railway-set-builder", "bin/railway-set-dockerfile", "bin/railway-set-startcmd", "bin/railway-trigger-redeploy", + "bin/railway-update-service", "bin/ralph-agent", "bin/ralph-hook", "bin/scholar-agent", + "bin/sota-report", "bin/swe-entrypoint", "bin/tri", "bin/tri-api", + "bin/tri-bot", "bin/trinity-cli", "bin/trinity-hybrid", "bin/trinity-mcp", + "bin/trinity-node", "bin/trinity-query", "bin/trinity-search", "bin/vibee", + "bin/vibee-self-improve", "lib/libtrinity.a" +]; +; The test graph: the floors it may not drop below. A graph that shrank is a partial graph, and a run step +; that ran no test is not a passing one: one does today, the step of external/zig-hslm/src/root.zig, the only +; compiled test root without a test declaration. +pub const TEST_STEPS_MIN : u32 = 135; +pub const TESTS_PASSED_MIN : u32 = 2510; +pub const RUN_STEPS_WITH_TESTS_MIN : u32 = 61; +pub const EMPTY_RUN_STEPS_MAX : u32 = 1; +; Every root the test step compiles, read from `--verbose` with empty caches. The five BLOCKED ones are here too: +; the step still tries them, and a root that leaves this list is a test that left the graph. +pub const TEST_ROOTS : [67]str = [ + "external/zig-hslm/src/root.zig", "src/agent_mu/production_hardening_test.zig", + "src/agent_mu/production_test.zig", "src/agent_mu/swarm_collaboration.zig", + "src/api/unified_server.zig", "src/arena/main.zig", + "src/beal.zig", "src/c_api.zig", + "src/depin/network.zig", "src/e2e_test.zig", + "src/firebird/b2t_integration.zig", "src/firebird/depin.zig", + "src/firebird/extension_wasm.zig", "src/firebird/wasm_parser.zig", + "src/igla/trace.zig", "src/queen_api.zig", + "src/trinity.zig", "src/trinity_node/auto_repair.zig", + "src/trinity_node/bandwidth_aggregator.zig", "src/trinity_node/connection_pool.zig", + "src/trinity_node/cross_shard_tx.zig", "src/trinity_node/crypto.zig", + "src/trinity_node/dynamic_erasure.zig", "src/trinity_node/erasure_repair.zig", + "src/trinity_node/file_encoder.zig", "src/trinity_node/galois.zig", + "src/trinity_node/graceful_shutdown.zig", "src/trinity_node/incentive_slashing.zig", + "src/trinity_node/integration_test.zig", "src/trinity_node/manifest_dht.zig", + "src/trinity_node/metrics_http.zig", "src/trinity_node/network_stats.zig", + "src/trinity_node/node_reputation.zig", "src/trinity_node/parallel_saga.zig", + "src/trinity_node/peer_latency.zig", "src/trinity_node/prometheus_http.zig", + "src/trinity_node/prometheus_metrics.zig", "src/trinity_node/proof_of_storage.zig", + "src/trinity_node/protocol.zig", "src/trinity_node/reed_solomon.zig", + "src/trinity_node/region_router.zig", "src/trinity_node/region_topology.zig", + "src/trinity_node/remote_storage.zig", "src/trinity_node/repair_rate_limiter.zig", + "src/trinity_node/reputation_consensus.zig", "src/trinity_node/rs_repair.zig", + "src/trinity_node/saga_coordinator.zig", "src/trinity_node/semantic_index.zig", + "src/trinity_node/shard_manager.zig", "src/trinity_node/shard_rebalancer.zig", + "src/trinity_node/shard_scrubber.zig", "src/trinity_node/slashing_escrow.zig", + "src/trinity_node/stake_delegation.zig", "src/trinity_node/storage.zig", + "src/trinity_node/storage_discovery.zig", "src/trinity_node/token_staking.zig", + "src/trinity_node/transaction_wal.zig", "src/trinity_node/vsa_shard_encoder.zig", + "src/trinity_node/vsa_shard_locks.zig", "src/trinity_node/wal_disk.zig", + "src/vibeec/codegen_tests.zig", "src/vibeec/kg_pipeline.zig", + "src/vibeec/kg_sync.zig", "src/vibeec/prefix_cache_completion.zig", + "src/vibeec/sym_005_demo.zig", "src/vibeec/triples_parser.zig", + "src/vm.zig" +]; +; The roots that do not compile at MEASURED_AT, what the compiler says, and the capability card of the S01 +; manifest each one holds back. Exactly these may fail; one more fails the gate, and one fewer fails it too. +pub const BLOCKED_ROOTS : [5]str = ["src/c_api.zig", "src/e2e_test.zig", "src/queen_api.zig", "src/trinity.zig", "src/vm.zig"]; +pub const BLOCKED_REASONS : [5]str = [ + "src/c_api.zig:270:1: expected statement, found 'a document comment' -- the file does not parse; S05 of the epic recorded the same at the S01 pin", + "src/packed_trit.zig:1:1: unable to load 'packed_trit.zig': FileNotFound -- src/e2e_test.zig:21 imports a file removed in 42490a222 (#517)", + "lib/std/c.zig and lib/std/heap.zig:128: the C allocator without libc (4 errors) -- the queen_api test module does not set link_libc, as the main test module does", + "src/hybrid.zig:4 and src/vsa.zig:7: no module named 'zig-hdc-vsa' (the test module provides it as hdc_vsa); src/science.zig:53, 143, 161, 179, 204: expected optional type, found '[59049]i8' (7 errors)", + "src/vm.zig:1150:15: expected optional type, found '[59049]i8'; src/vsa_jit.zig:117:26: member function expected 1 argument(s), found 2" +]; +pub const BLOCKED_CAPABILITIES : [5]str = ["trinity/abi.c-api", "trinity/examples.zig", "trinity/queen.lib", "trinity/lib.trinity", "trinity/lib.trinity"]; +pub const EVIDENCE : str = "tools/reproduce.py writes zig-out/reproduce/.json: toolchain, head, cache freshness, each command with its exit code and summary, the compile failures by root, the run steps, the compiled roots, every install with its sha256, and the violations"; +pub const ENABLED : bool = true; + +test the_profile_is_headless_and_pinned { + assert PROFILE == "headless"; + assert ZIG == "0.15.2"; + assert BUILD_STEPS_MIN == 94; +} + +test the_floors_hold_together { + assert TESTS_PASSED_MIN == 2510; + assert RUN_STEPS_WITH_TESTS_MIN + EMPTY_RUN_STEPS_MAX + 5 == 67; + assert TEST_STEPS_MIN > RUN_STEPS_WITH_TESTS_MIN; +} + +invariant blocked_roots_are_few { + assert EMPTY_RUN_STEPS_MAX <= 1; +} diff --git a/tools/reproduce.py b/tools/reproduce.py new file mode 100644 index 0000000000..01a9e5c4a7 --- /dev/null +++ b/tools/reproduce.py @@ -0,0 +1,416 @@ +#!/usr/bin/env python3 +"""Reproduce the headless profile of gHashTag/trinity from a clean clone, and refuse to call a broken +graph green. + +WHY THIS EXISTS +--------------- +S12 of gHashTag/trinity#988 (gHashTag/trinity#989): a set of specifications is not a recreated project +until a clean checkout builds and tests the declared profile, and a failure anywhere fails the gate. +Until this tool, CI ran `zig build test -Dci=true 2>&1 | tee test-output.txt` under bash without +pipefail: the step's exit code was tee's, so five test binaries that do not compile -- among them the +library's own test root, src/trinity.zig -- were reported green on every push (gHashTag/trinity#616; +the tee fix alone is gHashTag/trinity#987). The summary line said it plainly, "124/135 steps +succeeded; 5 failed", and nothing read it. + +WHAT IT CHECKS +-------------- +specs/reproduce/headless.t27 is the declared profile: the toolchain, the two commands, the 46 artifacts +the build installs, the test roots the test step compiles, the floors the graph may not drop below, and +the test roots known not to compile, each with its reason and the capability card it blocks. The tool +runs the commands and judges what they did: + + * the exit code of each command is read from the process, never through a pipe; + * a compile failure must be a BLOCKED root, and a BLOCKED root must still fail (a fixed one is + reported, so the ledger cannot outlive the defect); + * a failed test, a leak, or a failure the output does not name fails; + * a missing install fails; every install is hashed into the evidence; + * the number of run steps, of tests passed, and (with fresh caches) the set of compiled test roots + may not drop below the spec -- a graph that shrank is a partial graph; + * a run step that ran no test is counted, and their number may not grow (an empty step is not a + passing one); + * the four packages build.zig.zon pins are the spec's, url and hash. + +Usage (from the repository root of a clean clone): + python3 tools/reproduce.py headless [--out FILE] [--reuse-cache] [--zig ZIG] + python3 tools/reproduce.py test [--out FILE] [--reuse-cache] [--zig ZIG] (the test step only) + python3 tools/reproduce.py --self-check +headless runs both commands with fresh, empty zig caches (ZIG_GLOBAL_CACHE_DIR and ZIG_LOCAL_CACHE_DIR +in a new temporary directory) unless --reuse-cache is given, which the evidence then records. +Exit codes: 0 the profile is what the spec says; 1 it is not (every violation is printed); 2 could not run. +Standard library only. +""" +from __future__ import annotations + +import argparse +import datetime +import hashlib +import json +import os +import pathlib +import platform +import re +import shutil +import subprocess +import sys +import tempfile +import time + +ROOT = pathlib.Path(__file__).resolve().parent.parent +SPEC = ROOT / "specs/reproduce/headless.t27" +CONST_RE = re.compile(r"^pub const (\w+) : ([^=]+?) = (.*);\s*$") + + +# =========================================================================================== +# The spec, read the way the other consumers of .t27 constants read it: one `pub const` per +# statement, string arrays as JSON arrays. A constant this reader cannot read stops the tool. +# =========================================================================================== +def load_spec(path: pathlib.Path) -> dict: + text = path.read_text(encoding="utf-8") + if re.search(r"[^\x00-\x7f]", text): + raise ValueError(f"{path}: non-ASCII byte (L3)") + out, lines, n = {}, text.split("\n"), 0 + while n < len(lines): + line = lines[n] + n += 1 + if line.startswith("pub const") and not line.rstrip().endswith(";"): + while n < len(lines): + line += " " + lines[n].strip() + n += 1 + if lines[n - 1].rstrip().endswith(";"): + break + m = CONST_RE.match(line) + if m: + out[m.group(1)] = value_of(m.group(3).strip(), m.group(2).strip()) + elif line.startswith("pub const"): + raise ValueError(f"{path}:{n}: cannot read constant line") + return out + + +def value_of(raw: str, typ: str): + if typ == "bool": + return raw == "true" + if re.fullmatch(r"[ui]\d+", typ): + return int(raw) + if typ == "str": + return json.loads(raw) + m = re.fullmatch(r"\[(\d+)\](\w+)", typ) + if m: + items = json.loads(raw) if m.group(2) == "str" else [int(x) for x in raw.strip("[]").split(",") if x.strip()] + if len(items) != int(m.group(1)): + raise ValueError(f"annotated {typ}, holds {len(items)}") + return items + raise ValueError(f"unknown type {typ}") + + +# =========================================================================================== +# Reading what zig printed +# =========================================================================================== +def parse_summary(text: str) -> dict: + """'Build Summary: 124/135 steps succeeded; 5 failed; 2510/2510 tests passed' and its variants.""" + line = next((x for x in text.split("\n") if x.startswith("Build Summary:")), None) + if line is None: + return {"found": False} + num = lambda pat: (lambda m: int(m.group(1)) if m else 0)(re.search(pat, line)) + steps = re.search(r"(\d+)/(\d+) steps succeeded", line) + tests = re.search(r"(\d+)/(\d+) tests passed", line) + return {"found": True, "line": line.strip(), + "steps_succeeded": int(steps.group(1)) if steps else 0, "steps_total": int(steps.group(2)) if steps else 0, + "steps_failed": num(r"steps succeeded; (\d+) failed"), + "tests_passed": int(tests.group(1)) if tests else 0, "tests_total": int(tests.group(2)) if tests else 0, + "tests_skipped": num(r"(\d+) skipped"), "tests_failed": num(r"tests passed(?:; \d+ skipped)?; (\d+) failed"), + "tests_leaked": num(r"(\d+) leaked")} + + +def parse_run_steps(text: str) -> dict: + """The run steps of the summary tree: how many, how many ran a test, how many ran none.""" + counted = [int(m.group(1)) for m in re.finditer(r"run test (\d+) passed", text)] + empty = len(re.findall(r"run test success\b", text)) + failed = len(re.findall(r"run test (?:\d+/\d+ passed, )?\d+ failed", text)) + return {"with_tests": len(counted), "empty": empty, "with_failures": failed, "tests_in_run_steps": sum(counted)} + + +def rel(path: str, workspace: str) -> str: + for prefix in (workspace.rstrip("/") + "/", "/work/"): + if path.startswith(prefix): + return path[len(prefix):] + return path + + +def parse_compile_failures(text: str, workspace: str) -> list: + """Every 'the following command failed with N compilation errors:' and the root it compiled.""" + out = [] + for m in re.finditer(r"error: the following command failed with (\d+) compilation errors?:\n(\S.*)", text): + root = re.search(r"-Mroot=(\S+)", m.group(2)) + out.append({"root": rel(root.group(1), workspace) if root else None, "errors": int(m.group(1))}) + return out + + +def parse_compiled_roots(text: str, workspace: str) -> list: + """With --verbose and an empty cache every compile command is printed; the test ones name their root.""" + roots = set() + for line in text.split("\n"): + if re.search(r"\bzig\S* test ", line): + m = re.search(r"-Mroot=(\S+)", line) + if m: + roots.add(rel(m.group(1), workspace)) + return sorted(roots) + + +def read_pins(zon: str) -> dict: + """build.zig.zon's dependencies: name -> [url, hash].""" + return {n: [u, h] for n, u, h in re.findall(r'\.(\w+) = \.\{\s*\.url = "([^"]+)",\s*\.hash = "([^"]+)"', zon)} + + +def first_errors(text: str, n=3) -> list: + return [x.strip()[:200] for x in text.split("\n") if ": error:" in x][:n] + + +# =========================================================================================== +# Running it +# =========================================================================================== +def run(cmd: list, env: dict, timeout: int) -> dict: + t0 = time.time() + try: + p = subprocess.run(cmd, cwd=str(ROOT), env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=timeout) + rc, out = p.returncode, p.stdout + except subprocess.TimeoutExpired as e: + rc, out = None, (e.stdout or b"").decode() if isinstance(e.stdout, bytes) else (e.stdout or "") + return {"command": " ".join(cmd), "rc": rc, "seconds": round(time.time() - t0, 1), "output": out} + + +def git(*args) -> str: + try: + p = subprocess.run(["git", "-C", str(ROOT)] + list(args), capture_output=True, text=True) + except OSError: + return "" + return p.stdout.strip() + + +def sha256_file(p: pathlib.Path) -> str: + h = hashlib.sha256() + with open(p, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def measure(sp: dict, zig: str, reuse_cache: bool, build: bool) -> dict: + env = dict(os.environ) + scratch = None + if not reuse_cache: + scratch = tempfile.mkdtemp(prefix="reproduce-") + env["ZIG_GLOBAL_CACHE_DIR"] = os.path.join(scratch, "global") + env["ZIG_LOCAL_CACHE_DIR"] = os.path.join(scratch, "local") + version = subprocess.run([zig, "version"], capture_output=True, text=True).stdout.strip() + doc = {"tool": "tools/reproduce.py", "spec": str(SPEC.relative_to(ROOT)), "spec_sha256": sha256_file(SPEC), + "at": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "host": f"{platform.system()} {platform.machine()}", "zig": version, "head": git("rev-parse", "HEAD") or "unknown", + "tracked_changes": [x for x in git("status", "--porcelain").split("\n") if x and not x.startswith("??")][:20], + "fresh_caches": not reuse_cache, "workspace": str(ROOT)} + doc["dependencies"] = read_pins((ROOT / "build.zig.zon").read_text(encoding="utf-8")) + if build: + b = run([zig] + sp["BUILD"].split()[1:], env, 7200) + doc["build"] = {"command": b["command"], "rc": b["rc"], "seconds": b["seconds"], "summary": parse_summary(b["output"]), + "compile_failures": parse_compile_failures(b["output"], str(ROOT)), "errors": first_errors(b["output"])} + installs = {} + for rel_path in sp["INSTALLS"]: + p = ROOT / "zig-out" / rel_path + installs[rel_path] = sha256_file(p) if p.is_file() else None + doc["installs"] = installs + test_cmd = [zig] + sp["TEST"].split()[1:] + ([] if reuse_cache else ["--verbose"]) + t = run(test_cmd, env, 7200) + doc["test"] = {"command": t["command"], "rc": t["rc"], "seconds": t["seconds"], "summary": parse_summary(t["output"]), + "run_steps": parse_run_steps(t["output"]), "compile_failures": parse_compile_failures(t["output"], str(ROOT)), + "compiled_roots": parse_compiled_roots(t["output"], str(ROOT)) if not reuse_cache else None, + "leak_lines": [x.strip()[:200] for x in t["output"].split("\n") if re.search(r"\bleak", x, re.I)][:5], + "errors": first_errors(t["output"])} + if scratch: + shutil.rmtree(scratch, ignore_errors=True) + return doc + + +# =========================================================================================== +# Judging it: the whole contract, in one place +# =========================================================================================== +def judge(sp: dict, doc: dict) -> list: + v = [] + + def bad(code, text): + v.append(f"{code}: {text}") + if doc.get("head") in (None, "", "unknown"): + bad("NO_REVISION", "git could not name the checkout's revision, so the record cannot say what it measured and the tree could not be checked for edits") + if not doc["zig"].startswith(sp["ZIG"]): + bad("TOOLCHAIN", f"zig {doc['zig']}, the profile is measured with {sp['ZIG']}") + if doc.get("tracked_changes"): + bad("DIRTY", f"tracked files differ from HEAD: {doc['tracked_changes'][:3]}") + want = {n: [u, h] for n, u, h in zip(sp["DEPENDENCIES"], sp["DEPENDENCY_URLS"], sp["DEPENDENCY_HASHES"])} + if doc.get("dependencies") != want: + changed = sorted(n for n in set(want) | set(doc.get("dependencies") or {}) if want.get(n) != (doc.get("dependencies") or {}).get(n)) + bad("DEPENDENCY_DRIFT", f"build.zig.zon pins differ from the spec for {changed}") + if "build" in doc: + b = doc["build"] + if b["rc"] != 0: + bad("BUILD_FAILED", f"`{b['command']}` exited {b['rc']}: {b['errors'][:2]}") + if not b["summary"].get("found"): + bad("NO_SUMMARY", "the build printed no Build Summary") + elif b["summary"]["steps_failed"] or b["compile_failures"]: + bad("BUILD_STEP_FAILED", f"{b['summary']['line']} {[f['root'] for f in b['compile_failures']]}") + if b["rc"] == 0 and (b["summary"].get("steps_failed") or b["compile_failures"]): + bad("EXIT_SWALLOWED", "the build exited 0 while its output names failed steps") + missing = sorted(k for k, h in doc["installs"].items() if h is None) + if missing: + bad("MISSING_OUTPUT", f"{len(missing)} declared install(s) absent from zig-out: {missing[:5]}") + t = doc["test"] + s, rs = t["summary"], t["run_steps"] + failed_roots = sorted({f["root"] for f in t["compile_failures"] if f["root"]}) + blocked = sorted(sp["BLOCKED_ROOTS"]) + if not s.get("found"): + bad("NO_SUMMARY", f"the test step printed no Build Summary (exit {t['rc']})") + unknown = sorted(set(failed_roots) - set(blocked)) + fixed = sorted(set(blocked) - set(failed_roots)) + if unknown: + bad("NEW_COMPILE_FAILURE", f"test roots that do not compile and that the spec does not name: {unknown}") + if fixed: + bad("BLOCKED_NOW_COMPILES", f"named as blocked but they compiled: {fixed}; remove them from BLOCKED_ROOTS") + if any(f["root"] is None for f in t["compile_failures"]): + bad("UNNAMED_FAILURE", "a compile failure whose root the output does not name") + if s.get("found"): + if s["tests_failed"] or rs["with_failures"]: + bad("TEST_FAILED", f"{s['line']}") + if s["tests_leaked"] or t["leak_lines"]: + bad("LEAK", f"{s['tests_leaked']} leaked; {t['leak_lines'][:2]}") + if s["steps_failed"] != len(t["compile_failures"]) + rs["with_failures"]: + bad("UNEXPLAINED_STEP_FAILURE", f"{s['steps_failed']} failed step(s), {len(t['compile_failures'])} compile failure(s) and {rs['with_failures']} failed run step(s) named") + if s["tests_passed"] < sp["TESTS_PASSED_MIN"]: + bad("PARTIAL_GRAPH", f"{s['tests_passed']} tests passed, the floor is {sp['TESTS_PASSED_MIN']}") + if s["steps_total"] < sp["TEST_STEPS_MIN"]: + bad("PARTIAL_GRAPH", f"{s['steps_total']} steps in the test graph, the floor is {sp['TEST_STEPS_MIN']}") + if rs["with_tests"] < sp["RUN_STEPS_WITH_TESTS_MIN"]: + bad("PARTIAL_GRAPH", f"{rs['with_tests']} run steps ran a test, the floor is {sp['RUN_STEPS_WITH_TESTS_MIN']}") + if rs["empty"] > sp["EMPTY_RUN_STEPS_MAX"]: + bad("EMPTY_STEP", f"{rs['empty']} run step(s) ran no test, at most {sp['EMPTY_RUN_STEPS_MAX']} are known") + if t["compiled_roots"] is not None: + gone = sorted(set(sp["TEST_ROOTS"]) - set(t["compiled_roots"])) + if gone: + bad("PARTIAL_GRAPH", f"declared test roots the test step no longer compiles: {gone}") + # The exit code, read from the process, must agree with what the output says happened. + if t["rc"] == 0 and (failed_roots or (s.get("found") and (s["steps_failed"] or s["tests_failed"]))): + bad("EXIT_SWALLOWED", "the test step exited 0 while its output names failures") + if t["rc"] not in (0, None) and not failed_roots and not (s.get("found") and (s["tests_failed"] or s["tests_leaked"] or s["steps_failed"])): + bad("UNEXPLAINED_EXIT", f"the test step exited {t['rc']} and its output names no failure") + if t["rc"] is None: + bad("TIMEOUT", "the test step did not finish") + return v + + +# =========================================================================================== +# --self-check: every rule above has been seen to fire, and a conforming record stays silent +# =========================================================================================== +def self_check() -> int: + ok = True + + def expect(cond, what): + nonlocal ok + print(f" {'ok ' if cond else 'BAD'} {what}") + ok = ok and bool(cond) + sp = load_spec(SPEC) + blocked = sp["BLOCKED_ROOTS"] + fail_block = "".join(f"error: the following command failed with 1 compilation errors:\n/zig/zig test -ODebug -Mroot=/work/{r} --name test\n" for r in blocked) + steps_total = sp["TEST_STEPS_MIN"] + good_out = (fail_block + f"Build Summary: {steps_total - len(blocked)}/{steps_total} steps succeeded; {len(blocked)} failed; " + f"{sp['TESTS_PASSED_MIN']}/{sp['TESTS_PASSED_MIN']} tests passed\n" + + "".join("+- run test 7 passed 1ms MaxRSS:1M\n" for _ in range(sp["RUN_STEPS_WITH_TESTS_MIN"])) + + "".join("+- run test success 1ms MaxRSS:1M\n" for _ in range(sp["EMPTY_RUN_STEPS_MAX"])) + + "".join(f"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/zig/zig test -ODebug -Mroot=/work/{r} --listen=-\n" for r in sp["TEST_ROOTS"])) + good_build = f"Build Summary: {sp['BUILD_STEPS_MIN']}/{sp['BUILD_STEPS_MIN']} steps succeeded\n" + + good_deps = {n: [u, h] for n, u, h in zip(sp["DEPENDENCIES"], sp["DEPENDENCY_URLS"], sp["DEPENDENCY_HASHES"])} + + def record(test_out=good_out, test_rc=1, build_out=good_build, build_rc=0, installs=None, zig=sp["ZIG"], deps=None): + return {"zig": zig, "head": "0" * 40, "tracked_changes": [], "dependencies": deps if deps is not None else good_deps, + "build": {"command": sp["BUILD"], "rc": build_rc, "summary": parse_summary(build_out), + "compile_failures": parse_compile_failures(build_out, "/work"), "errors": []}, + "installs": installs if installs is not None else {k: "0" * 64 for k in sp["INSTALLS"]}, + "test": {"command": sp["TEST"], "rc": test_rc, "summary": parse_summary(test_out), "run_steps": parse_run_steps(test_out), + "compile_failures": parse_compile_failures(test_out, "/work"), "compiled_roots": parse_compiled_roots(test_out, "/work"), + "leak_lines": [x for x in test_out.split("\n") if re.search(r"\bleak", x, re.I)], "errors": []}} + codes = lambda rec: {x.split(":")[0] for x in judge(sp, rec)} + expect(not judge(sp, record()), f"a record that is what the spec says is silent ({judge(sp, record())[:2]})") + expect("EXIT_SWALLOWED" in codes(record(test_rc=0)), "planted: the test step's failures with exit 0 (the tee of #616)") + extra = "error: the following command failed with 2 compilation errors:\n/zig/zig test -ODebug -Mroot=/work/src/new_broken.zig --name test\n" + out = good_out.replace(f"{len(blocked)} failed;", f"{len(blocked) + 1} failed;") + expect("NEW_COMPILE_FAILURE" in codes(record(test_out=extra + out)), "planted: a test root that stops compiling") + first = f"error: the following command failed with 1 compilation errors:\n/zig/zig test -ODebug -Mroot=/work/{blocked[0]} --name test\n" + out = good_out.replace(first, "", 1).replace(f"{len(blocked)} failed;", f"{len(blocked) - 1} failed;") + expect("BLOCKED_NOW_COMPILES" in codes(record(test_out=out)), "planted: a blocked root that compiles again (the ledger would outlive the defect)") + out = good_out.replace(f"{sp['TESTS_PASSED_MIN']}/{sp['TESTS_PASSED_MIN']} tests passed", f"{sp['TESTS_PASSED_MIN'] - 1}/{sp['TESTS_PASSED_MIN']} tests passed; 1 failed") + expect("TEST_FAILED" in codes(record(test_out=out)), "planted: a failed assertion") + expect("LEAK" in codes(record(test_out=good_out + "error: 'x.test.y' leaked: [gpa] Memory leak detected\n")), "planted: a leak") + out = good_out.replace(f"{sp['TESTS_PASSED_MIN']}/{sp['TESTS_PASSED_MIN']} tests passed", f"{sp['TESTS_PASSED_MIN'] - 400}/{sp['TESTS_PASSED_MIN'] - 400} tests passed") + expect("PARTIAL_GRAPH" in codes(record(test_out=out)), "planted: 400 tests fewer (a partial graph)") + out = good_out.replace(f"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/zig/zig test -ODebug -Mroot=/work/{sp['TEST_ROOTS'][0]} --listen=-\n", "", 1) + expect("PARTIAL_GRAPH" in codes(record(test_out=out)), "planted: a declared test root that is no longer compiled") + expect("EMPTY_STEP" in codes(record(test_out=good_out + "+- run test success 1ms MaxRSS:1M\n")), "planted: one more run step that runs no test") + inst = {k: "0" * 64 for k in sp["INSTALLS"]} + inst[sp["INSTALLS"][0]] = None + expect("MISSING_OUTPUT" in codes(record(installs=inst)), "planted: a declared install that is not there") + expect("BUILD_FAILED" in codes(record(build_rc=1, build_out="Build Summary: 1/2 steps succeeded; 1 failed\n")), "planted: a build that fails") + expect("UNEXPLAINED_EXIT" in codes(record(test_out=good_out.replace(fail_block, "").replace(f"{len(blocked)} failed;", "0 failed;"), test_rc=1)) or + "BLOCKED_NOW_COMPILES" in codes(record(test_out=good_out.replace(fail_block, ""), test_rc=1)), "planted: a non-zero exit with no failure in the output") + expect("TOOLCHAIN" in codes(record(zig="0.16.0")), "planted: another zig") + rec = record() + rec["head"] = "unknown" + expect("NO_REVISION" in codes(rec), "planted: a checkout git cannot read (the dirty-tree check would be skipped)") + rec = record() + rec["tracked_changes"] = [" M build.zig"] + expect("DIRTY" in codes(rec), "planted: a tracked file edited after the commit") + zon = (ROOT / "build.zig.zon").read_text(encoding="utf-8") + expect(read_pins(zon) == good_deps, "the pins build.zig.zon carries are the spec's") + moved = zon.replace(sp["DEPENDENCY_URLS"][2], sp["DEPENDENCY_URLS"][2].replace("b73b2fa2", "0000fa2a"), 1) + expect("DEPENDENCY_DRIFT" in codes(record(deps=read_pins(moved))), "planted: zig_hdc pinned to another revision") + print("reproduce --self-check:", "ok" if ok else "FAILED") + return 0 if ok else 1 + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("command", nargs="?", choices=["headless", "test"]) + ap.add_argument("--out") + ap.add_argument("--reuse-cache", action="store_true") + ap.add_argument("--zig", default="zig") + ap.add_argument("--self-check", action="store_true") + a = ap.parse_args() + if a.self_check: + return self_check() + if a.command is None: + ap.print_help() + return 2 + try: + sp = load_spec(SPEC) + except (OSError, ValueError) as e: + print(f"could not read {SPEC}: {e}", file=sys.stderr) + return 2 + if shutil.which(a.zig) is None and not pathlib.Path(a.zig).exists(): + print(f"no zig at {a.zig}", file=sys.stderr) + return 2 + doc = measure(sp, a.zig, a.reuse_cache, build=(a.command == "headless")) + doc["violations"] = judge(sp, doc) + out = pathlib.Path(a.out) if a.out else ROOT / "zig-out" / "reproduce" / f"{a.command}.json" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(doc, indent=1, sort_keys=True) + "\n", encoding="utf-8") + t = doc["test"] + print(f"reproduce {a.command}: zig {doc['zig']}, {'fresh' if doc['fresh_caches'] else 'reused'} caches, head {doc['head'][:12]}") + if "build" in doc: + print(f" build: exit {doc['build']['rc']}, {doc['build']['summary'].get('line', 'no summary')}, " + f"{sum(1 for h in doc['installs'].values() if h)}/{len(doc['installs'])} installs") + print(f" test: exit {t['rc']}, {t['summary'].get('line', 'no summary')}") + print(f" compile failures {[f['root'] for f in t['compile_failures']]}, run steps {t['run_steps']}") + for x in doc["violations"]: + print("VIOLATION:", x) + print(f" evidence: {out}") + print(" the profile is what specs/reproduce/headless.t27 says" if not doc["violations"] else f" {len(doc['violations'])} violation(s)") + return 1 if doc["violations"] else 0 + + +if __name__ == "__main__": + sys.exit(main())