From c56be35693cf95b044af0ef2dddf2307f6a15c4b Mon Sep 17 00:00:00 2001 From: Austin Gregg-Smith Date: Fri, 14 Aug 2026 20:50:11 +0100 Subject: [PATCH 1/2] timing: fail a mistyped stage name where it is written, scrub both seam stamps Two riders from PR #199's review, both in timing's own domain. `@timing.staged(name)` left the vocabulary check to `stage()`, which runs it only while a recorder is live -- so a typo passed a suite that runs with timing off, which is every suite, and crashed the first launch anyone measured. The check now runs as the decorator is applied: eight of the nine instrumented sites are `@staged`, and import is the one moment it costs nothing per launch and still happens. The lone `with timing.stage(...)` keeps the call-time check, so the off path stays a single `None` check. The suite's env scrub covered the handoff stamp but not the prewarm one. They are one seam, so they are scrubbed together: on its own a prewarm stamp reports nothing today, which is exactly why the asymmetry would be found by the next test that observes an attach shape rather than by this one. --- devlaunch/timing.py | 14 +++++++++++++- test/conftest.py | 11 ++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/devlaunch/timing.py b/devlaunch/timing.py index f248a1da..3735c50a 100644 --- a/devlaunch/timing.py +++ b/devlaunch/timing.py @@ -295,9 +295,14 @@ def stage(name: str): recorder = _recorder if recorder is None: return contextlib.nullcontext() + _check_stage(name) + return _record_stage(recorder, name) + + +def _check_stage(name: str) -> None: + """Refuse a name the vocabulary does not have.""" if name not in STAGES: raise ValueError(f"{name!r} is not a timing stage; expected one of {STAGES}") - return _record_stage(recorder, name) def span(label: str): @@ -316,7 +321,14 @@ def staged(name: str) -> Callable[[_F], _F]: lend the tools in, attach — and marking them where they are defined puts the stage next to the thing it names, rather than at each of the call sites that would all have to agree. + + *name* is checked here, as the decorator is applied, rather than left to + :func:`stage` at call time: the call-time check runs only while recording, + so a typo would pass a suite that runs with timing off — every suite — and + then crash the first launch anyone actually measured. Import is the one + moment the check costs nothing per launch and still happens. """ + _check_stage(name) def decorate(func: _F) -> _F: @functools.wraps(func) diff --git a/test/conftest.py b/test/conftest.py index 78db9037..2515697f 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -170,12 +170,17 @@ def timing_switch_off(monkeypatch): scrubs it rather than going red in their shell. A test that wants the switch on sets it itself. - The handoff stamp goes with it: it is exported by whatever hands off to - dl, and a developer whose shell carries one would otherwise see a handoff - stage in every test that asks for the summary. + Both seam stamps go with it: they are exported by whatever hands off to + dl, and a developer whose shell carries them would otherwise see a handoff + stage, or a prewarm report, in tests that ask for the summary and have no + reason to expect either. The pair is scrubbed together because they are + one seam — scrubbing the handoff stamp alone leaves the trap set for the + next test that observes an attach shape without stamping the environment + itself. """ monkeypatch.delenv(timing.ENV_VAR, raising=False) monkeypatch.delenv(timing.HANDOFF_VAR, raising=False) + monkeypatch.delenv(timing.PREWARM_VAR, raising=False) def pytest_configure(): From d78ca4c3c8d551094d75bcd3c7bcae68c3dfc9f5 Mon Sep 17 00:00:00 2001 From: Austin Gregg-Smith Date: Fri, 14 Aug 2026 20:50:11 +0100 Subject: [PATCH 2/2] bench: --record writes the launch's stages per run, and the median as the point `bench_launch.py --record PATH` asks each timed run for the machine-readable timing document (#199) and writes the whole invocation as one JSON object: the command, the run count, every run's wall time, in-process total and per-stage seconds, and the medians of those. `--shape NAME` labels which launch shape was benched. Without `--record` nothing changes -- no timing mode is forced on the child and stdout is what it was. The median is what a trend publishes and the runs are the evidence under it: the trend baselines a point against the immediately previous point only (#197), so N runs as N points would read ordinary spread as a regression. A stage no run reported is absent rather than zero (#195) -- a warm launch legitimately has no cold-path stages -- and a stage only some runs reported is a median over those runs carrying a count of how many. A run that failed, whose reset failed, or that reported no document at all writes no record: the "no median over a failing command" discipline, extended to the artifact. Nothing else goes in the record. A CI job stamps its own commit, clock and host better than this script can. Documented in the epilog and the README, and the epilog's invocations are now handed to the real parser by the doc guard that already runs its cold reset. --- README.md | 29 +++++ scripts/bench_launch.py | 194 +++++++++++++++++++++++++++++-- test/test_timing.py | 247 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 463 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 99d29229..3e8e6504 100644 --- a/README.md +++ b/README.md @@ -913,6 +913,35 @@ build the image pays more, by an amount this recipe does not measure — but the gap is large: an earlier 3-run median on this same host, reported as its first real launch, was 33.204s. +Every number in the two paragraphs above was copied into this prose by hand. +`--record` is how that stops: it writes the same invocation as one JSON object +a trend job can upload without anyone reading it. + +```bash +python3 scripts/bench_launch.py -n 5 --record warm.json --shape warm \ + -- dl-next owner/repo -- true +``` + +The record holds the command, the run count, each run's wall time and +per-stage seconds, and the medians of those — and nothing else, because a CI +job stamps its own commit, clock and host better than this script can. Four +things about it are load-bearing: + +- **The median is the point, the runs are its evidence.** A trend compares a + point against the immediately previous one, so N runs published as N points + would read ordinary spread as a regression. +- **A stage no run reported is absent, not zero.** A warm launch legitimately + has no cold-path stages; a zero would claim the work happened instantly + rather than not at all. A stage only *some* runs reported is a median over + those runs, carrying a count of how many — a median of two and a median of + five are not the same claim. +- **Recording asks the launch for its stages** (`DEVLAUNCH_TIMING=json`), so a + run that reports no timing document is an error and no record — same + discipline as no median over a failed run. +- **`--shape` labels the trend line** (`warm`, `cold-recreate`). It is the + caller's to say: the same command benches either shape depending on + `--before`, and a wrong label is worse in a trend than a missing one. + ## Worktree Backend For git repositories, devlaunch uses an efficient worktree backend by default: diff --git a/scripts/bench_launch.py b/scripts/bench_launch.py index d7ae93aa..3db7fc82 100755 --- a/scripts/bench_launch.py +++ b/scripts/bench_launch.py @@ -8,12 +8,14 @@ """ import argparse +import json +import os import shlex import statistics import subprocess import sys import time -from typing import List, Optional +from typing import Any, Dict, List, Optional EPILOG = """\ warm: bench_launch.py -n 5 -- dl-next owner/repo -- true @@ -25,6 +27,9 @@ bench_launch.py -n 5 --before 'dl-next owner/repo rm --force' \\ -- dl-next owner/repo -- true +record: bench_launch.py -n 5 --record warm.json --shape warm \\ + -- dl-next owner/repo -- true + A cold median needs the reset per *run*: delete once and bench five times and runs 2..5 are warm, so the median is a warm number under a cold label. The reset is `rm --force`, which also succeeds when there is nothing to remove @@ -42,20 +47,88 @@ machine. The bare-repo cache survives the reset on purpose: "cold" here means no workspace, no worktree, no container -- not a fresh network clone per run. +--record writes one JSON object for the whole invocation: the command, the run +count, every run's wall time and per-stage seconds, and the medians of those. +It asks the launch for its stages itself (DEVLAUNCH_TIMING=json), so a run that +reports no timing document is an error and no record, exactly as a failing run +is no median. The median is the published number, not the runs -- a trend +compares a point against the immediately previous point, so N runs published as +N points would read ordinary spread as a regression. A stage no run reported is +absent rather than zero, and a stage only some runs reported is a median over +those runs with a count saying how many. --shape labels which launch shape was +benched (warm, cold-recreate); nothing is guessed from the runs if it is +omitted. Nothing else goes in: a CI job stamps its own commit and clock better +than this script can. Finding the document means capturing the run's stderr, so +a recorded run shows the launch's own chatter only when the run failed, and the +whole of it then; stdout is untouched either way. + This clock starts outside the process, so it includes interpreter startup and imports; dl's own `dl-timing: total` starts inside main() and excludes them (on `dl --version`: 0.001s total against a 0.061s median here). Quote one -instrument on both sides of a change. +instrument on both sides of a change. The record carries both, as +`wall_seconds` and `total_seconds` -- per run and in the median. """ -def bench(n: int, command: List[str], before: Optional[List[str]] = None) -> int: +# How the launch is asked for its stages, and how its answer is found again in +# a stderr that also carries devpod's chatter. These three are devlaunch's, not +# this script's -- see `devlaunch/timing.py` -- but the script stays stdlib-only +# and importable from a checkout that was never installed, so it names them +# rather than importing them. `test_timing.py` drives this script against a +# stand-in launch built from timing.py's own constants, so a rename there fails +# here rather than quietly recording nothing. +TIMING_VAR = "DEVLAUNCH_TIMING" +TIMING_DOCUMENT = "json" +DOCUMENT_PREFIX = "dl-timing-json:" + + +def recording_env() -> Dict[str, str]: + """This process's environment, with the launch asked for its document. + + Overriding rather than deferring to an exported `DEVLAUNCH_TIMING=1`: the + prose summary is not what a record is made of, and a bench run that + silently recorded nothing because of the developer's shell would be found + much later, in the trend. + """ + env = dict(os.environ) + env[TIMING_VAR] = TIMING_DOCUMENT + return env + + +def stage_document(stderr: str) -> Optional[dict]: + """The one timing document *stderr* carries, or None if it carries no + readable one -- which is the same answer for a launch too old to emit one + and for output that was never a document.""" + found = [ + line[len(DOCUMENT_PREFIX) :] + for line in stderr.splitlines() + if line.startswith(DOCUMENT_PREFIX) + ] + if len(found) != 1: + return None + try: + document = json.loads(found[0]) + except ValueError: + return None + return document if isinstance(document, dict) and "total" in document else None + + +def bench( + n: int, + command: List[str], + before: Optional[List[str]] = None, + record: Optional[str] = None, + shape: Optional[str] = None, +) -> int: """Run *command* *n* times, print each wall time and the median. Returns 0 after n clean runs, otherwise the exit code of the first run -- - or untimed *before* reset -- that failed. + or untimed *before* reset -- that failed. With *record* given, the same + runs are also written there as one JSON record; without it nothing about + the run changes, down to the stdout. """ durations = [] + runs: List[Dict[str, Any]] = [] for i in range(1, n + 1): if before is not None: # nosec B603 - list form, not shell=True; the reset is the user's own argv @@ -69,9 +142,20 @@ def bench(n: int, command: List[str], before: Optional[List[str]] = None) -> int return reset.returncode start = time.perf_counter() # nosec B603 - list form, not shell=True; the command is the user's own argv - result = subprocess.run(command, check=False) + result = subprocess.run( + command, + check=False, + env=recording_env() if record is not None else None, + stderr=subprocess.PIPE if record is not None else None, + text=True, + ) elapsed = time.perf_counter() - start if result.returncode != 0: + if result.stderr: + # Captured for the document, so it has to be handed back: a + # failing run the bench swallowed the error of is worse than + # no bench. + sys.stderr.write(result.stderr) print( f"run {i}/{n} exited {result.returncode}; no median over a failing command", file=sys.stderr, @@ -79,11 +163,81 @@ def bench(n: int, command: List[str], before: Optional[List[str]] = None) -> int return result.returncode print(f"run {i}/{n}: {elapsed:.3f}s") durations.append(elapsed) + if record is not None: + document = stage_document(result.stderr) + if document is None: + sys.stderr.write(result.stderr) + print( + f"run {i}/{n} reported no {DOCUMENT_PREFIX} document; " + "nothing to record its stages from", + file=sys.stderr, + ) + return 1 + runs.append( + { + "wall_seconds": round(elapsed, 6), + "total_seconds": document["total"], + "stages": { + stage["stage"]: stage["seconds"] for stage in document.get("stages", []) + }, + } + ) median = statistics.median(durations) print(f"median of {n}: {median:.3f}s (wall clock, including interpreter startup)") + if record is not None: + write_record(record, command=command, runs=runs, shape=shape) return 0 +def median_of(runs: List[Dict[str, Any]]) -> Dict[str, Any]: + """The one point these runs make: a median per quantity they reported. + + The trend's baseline is the immediately previous point and nothing older + (#197), so what gets published has to be the median already -- N runs + published as N points would compare a run against the run before it and + call ordinary spread a regression. + """ + stages: Dict[str, List[float]] = {} + for run in runs: + for name, seconds in run["stages"].items(): + stages.setdefault(name, []).append(seconds) + return { + "wall_seconds": round(statistics.median([run["wall_seconds"] for run in runs]), 6), + "total_seconds": round(statistics.median([run["total_seconds"] for run in runs]), 6), + "stages": { + name: {"seconds": round(statistics.median(seconds), 6), "runs": len(seconds)} + for name, seconds in stages.items() + }, + } + + +def write_record( + path: str, command: List[str], runs: List[Dict[str, Any]], shape: Optional[str] = None +) -> None: + """Write one bench invocation as one JSON object at *path*. + + Deliberately boring: what was run, how many times, what each run cost and + what the medians of those costs are. No timestamp, no commit, no host -- + a trend job knows all three about itself better than this script can, and + a field it would have to overwrite is a field that can disagree with it. + """ + document = { + "command": list(command), + "n": len(runs), + "runs": runs, + "median": median_of(runs), + } + if shape is not None: + # Absent rather than guessed when the caller did not say: the same + # command benches warm or cold depending on a `--before` this script + # is in no position to interpret, and a wrong label is worse in a + # trend than a missing one. + document["shape"] = shape + with open(path, "w", encoding="utf-8") as handle: + json.dump(document, handle, indent=2, sort_keys=True) + handle.write("\n") + + def positive_int(text: str) -> int: """A run count there is a median of: one or more.""" value = int(text) @@ -92,7 +246,13 @@ def positive_int(text: str) -> int: return value -def main(argv=None) -> int: +def build_parser() -> argparse.ArgumentParser: + """The command line, built where a test can also ask it what it accepts. + + Separate from :func:`main` so the documented invocations in EPILOG can be + parsed without being run: every documented-command defect in this script's + history was a command nobody had ever fed to the thing that reads it. + """ parser = argparse.ArgumentParser( description="Run a command N times and report the median wall time.", epilog=EPILOG, @@ -107,12 +267,30 @@ def main(argv=None) -> int: "use it to re-establish a cold launch's starting state every run" ), ) + parser.add_argument( + "--record", + metavar="PATH", + help="also write this invocation's runs, per-stage seconds and medians there as JSON", + ) + parser.add_argument( + "--shape", + metavar="NAME", + help=( + "the launch shape this invocation benched (e.g. warm, cold-recreate), " + "recorded as the label of the trend line it belongs to" + ), + ) parser.add_argument( "command", nargs=argparse.REMAINDER, metavar="-- command...", help="the command to time, after a `--`", ) + return parser + + +def main(argv=None) -> int: + parser = build_parser() args = parser.parse_args(argv) command = args.command[1:] if args.command[:1] == ["--"] else args.command if not command: @@ -120,7 +298,9 @@ def main(argv=None) -> int: before = shlex.split(args.before) if args.before else None if args.before and not before: parser.error("--before given an empty command") - return bench(args.n, command, before) + if args.shape and args.record is None: + parser.error("--shape has nothing to name without --record") + return bench(args.n, command, before, record=args.record, shape=args.shape) if __name__ == "__main__": diff --git a/test/test_timing.py b/test/test_timing.py index 66770bf7..5e9160f7 100644 --- a/test/test_timing.py +++ b/test/test_timing.py @@ -7,6 +7,7 @@ the CLI entry point on one side, the subprocess module on the other. """ +import importlib.util import io import json import re @@ -695,3 +696,249 @@ def devpod(args, **_kwargs): patch("devlaunch.dl.update_cache_background"), ): assert main(argv) == 0 + + +def a_document(stages, total: float = 0.3) -> str: + """One launch's timing document, exactly as dl's json mode writes the line.""" + return f"{timing.JSON_PREFIX} " + json.dumps( + { + "total": total, + "total_epoch": timing.TOTAL_EPOCH, + "stages": [ + {"stage": name, "seconds": seconds, "outcome": "ok", "spans": []} + for name, seconds in stages + ], + } + ) + + +def a_launch(tmp_path, *runs, exit_code: int = 0) -> list[str]: + """A stand-in launch reporting one of *runs*' documents per invocation. + + It reports a document only when the environment asks for one, the way dl + does, so a bench that never asked gets what the real thing would have + given it: nothing to record. Runs past the last one described repeat it, + and *exit_code* is how a launch that reported its stages and then failed + anyway is asked for. + """ + lines = [a_document(stages, total) for stages, total in runs] + counted = tmp_path / "launches" + counted.mkdir() + source = ( + "import os, sys\n" + f"lines = {lines!r}\n" + f"counted = {str(counted)!r}\n" + "run = len(os.listdir(counted))\n" + "open(os.path.join(counted, str(run)), 'w').close()\n" + f"if os.environ.get({timing.ENV_VAR!r}) == {timing.JSON_VALUE!r}:\n" + " print(lines[min(run, len(lines) - 1)], file=sys.stderr)\n" + f"sys.exit({exit_code})\n" + ) + return [sys.executable, "-c", source] + + +class TestBenchRecordsEachRunsStages: + """#196: the bench writes one machine-readable record per invocation, so a + published number stops being hand-copied out of prose.""" + + def test_a_record_names_the_command_the_run_count_and_every_run(self, tmp_path): + record = tmp_path / "bench.json" + command = a_launch(tmp_path, ((("host-prep", 0.10), ("attach", 0.20)), 0.30)) + result = run_bench("-n", "3", "--record", str(record), "--", *command) + assert result.returncode == 0, result.stderr + written = json.loads(record.read_text()) + assert written["command"] == command + assert written["n"] == 3 + assert len(written["runs"]) == 3 + + def test_each_run_carries_the_stage_seconds_that_run_reported(self, tmp_path): + """The record's point: the stages the launch itself measured, not one + outside wall time a reader has to decompose by hand.""" + record = tmp_path / "bench.json" + command = a_launch(tmp_path, ((("host-prep", 0.10), ("attach", 0.20)), 0.30)) + result = run_bench("-n", "2", "--record", str(record), "--", *command) + assert result.returncode == 0, result.stderr + runs = json.loads(record.read_text())["runs"] + assert [run["stages"] for run in runs] == [{"host-prep": 0.10, "attach": 0.20}] * 2 + assert [run["total_seconds"] for run in runs] == [0.30, 0.30] + + def test_a_run_that_reported_no_document_is_not_a_record(self, tmp_path): + """A wall time with no stages behind it is a silent contract break — + an older dl, or a command that is not a launch at all. Better a loud + failure at the bench than an empty point in the trend.""" + record = tmp_path / "bench.json" + result = run_bench("-n", "2", "--record", str(record), "--", sys.executable, "-c", "pass") + assert result.returncode != 0 + assert not record.exists() + assert "Traceback" not in result.stderr + + def test_the_committed_point_is_the_median_of_the_runs(self, tmp_path): + """The trend compares a point against the immediately previous point + only (#197), so one bench invocation is one point: the median, with + the runs kept underneath it as the evidence it was taken over.""" + record = tmp_path / "bench.json" + command = a_launch( + tmp_path, + ((("devpod-up", 0.50),), 1.00), + ((("devpod-up", 0.10),), 3.00), + ((("devpod-up", 0.30),), 2.00), + ) + result = run_bench("-n", "3", "--record", str(record), "--", *command) + assert result.returncode == 0, result.stderr + median = json.loads(record.read_text())["median"] + assert median["stages"]["devpod-up"] == {"seconds": 0.30, "runs": 3} + assert median["total_seconds"] == 2.00 + + def test_a_stage_no_run_reported_is_absent_rather_than_zero(self, tmp_path): + """A warm launch legitimately has no cold-path stages (#195), and a + zero there would claim the work happened instantly instead of not at + all — a fiction the trend has no way to tell from a reading.""" + record = tmp_path / "bench.json" + command = a_launch(tmp_path, ((("devpod-up", 0.50), ("attach", 1.20)), 1.70)) + result = run_bench("-n", "2", "--record", str(record), "--", *command) + assert result.returncode == 0, result.stderr + written = json.loads(record.read_text()) + assert set(written["median"]["stages"]) == {"devpod-up", "attach"} + assert all(set(run["stages"]) == {"devpod-up", "attach"} for run in written["runs"]) + + def test_a_stage_only_some_runs_reported_is_a_median_over_those_runs(self, tmp_path): + """Counting the missing runs as zero would drag the median toward a + number no run measured, so the median is over the runs that reported + the stage and says how many that was — a median of two and a median + of five are not the same claim.""" + record = tmp_path / "bench.json" + command = a_launch( + tmp_path, + ((("attach", 1.00),), 1.00), + ((("attach", 1.00), ("tools", 4.00)), 5.00), + ((("attach", 1.00), ("tools", 6.00)), 7.00), + ) + result = run_bench("-n", "3", "--record", str(record), "--", *command) + assert result.returncode == 0, result.stderr + stages = json.loads(record.read_text())["median"]["stages"] + assert stages["tools"] == {"seconds": 5.00, "runs": 2} + assert stages["attach"] == {"seconds": 1.00, "runs": 3} + + def test_a_failed_run_writes_no_record_even_having_reported_stages(self, tmp_path): + """The record is the artifact of a completed measurement, so the "no + median over a failing command" discipline covers it: a launch that + reported stages and then failed measured a failure.""" + record = tmp_path / "bench.json" + command = a_launch(tmp_path, ((("attach", 1.00),), 1.00), exit_code=7) + result = run_bench("-n", "2", "--record", str(record), "--", *command) + assert result.returncode == 7 + assert not record.exists() + + def test_a_failed_reset_writes_no_record(self, tmp_path): + record = tmp_path / "bench.json" + command = a_launch(tmp_path, ((("attach", 1.00),), 1.00)) + result = run_bench("-n", "2", "--record", str(record), "--before", FAILING, "--", *command) + assert result.returncode != 0 + assert not record.exists() + + def test_without_the_flag_nothing_about_the_run_changes(self, tmp_path): + """Recording asks the launch for a document it would not otherwise + emit, so an unrecorded bench must not be paying for the instrument — + including on stdout, which #140's callers still read.""" + report = tmp_path / "asked" + command = [ + sys.executable, + "-c", + f"import os; open({str(report)!r}, 'w').write(os.environ.get({timing.ENV_VAR!r}, ''))", + ] + result = run_bench("-n", "2", "--", *command) + assert result.returncode == 0, result.stderr + assert report.read_text() == "" + assert re.search(r"^median of 2: \d+\.\d{3}s", result.stdout, re.M) + + def test_a_record_reads_a_document_the_timing_module_itself_wrote(self, tmp_path): + """Every other test here hands the bench a document built to look like + the launch's. This one has the launch's own emitter write it, so a + field renamed in the document cannot leave the bench parsing a shape + nothing emits any more.""" + record = tmp_path / "bench.json" + emitting = ( + "from devlaunch import timing\n" + "timing.begin()\n" + "with timing.stage('devpod-up'):\n" + " pass\n" + "timing.emit()\n" + ) + result = run_bench("-n", "2", "--record", str(record), "--", sys.executable, "-c", emitting) + assert result.returncode == 0, result.stderr + written = json.loads(record.read_text()) + assert set(written["median"]["stages"]) == {"devpod-up"} + assert written["median"]["stages"]["devpod-up"]["runs"] == 2 + assert written["median"]["total_seconds"] >= 0 + + def test_the_record_names_the_shape_the_caller_benched(self, tmp_path): + """warm and cold-recreate are two trend lines over the same command — + the `--before` reset is what separates them, so the shape is a label + the caller carries rather than something the runs reveal.""" + record = tmp_path / "bench.json" + command = a_launch(tmp_path, ((("attach", 1.00),), 1.00)) + result = run_bench( + "-n", "2", "--record", str(record), "--shape", "cold-recreate", "--", *command + ) + assert result.returncode == 0, result.stderr + assert json.loads(record.read_text())["shape"] == "cold-recreate" + + def test_an_unnamed_shape_is_absent_rather_than_guessed(self, tmp_path): + record = tmp_path / "bench.json" + command = a_launch(tmp_path, ((("attach", 1.00),), 1.00)) + assert run_bench("-n", "2", "--record", str(record), "--", *command).returncode == 0 + assert "shape" not in json.loads(record.read_text()) + + def test_a_shape_with_nothing_to_record_it_in_is_refused(self, tmp_path): + """Otherwise the label is silently dropped, which is how a trend line + ends up unnamed at the far end of a CI job nobody watched.""" + command = a_launch(tmp_path, ((("attach", 1.00),), 1.00)) + result = run_bench("-n", "2", "--shape", "warm", "--", *command) + assert result.returncode == 2 + assert "--record" in result.stderr + assert "Traceback" not in result.stderr + + +class TestAMistypedStageNameFailsWhereItIsWritten: + """The vocabulary is read from outside this repo, so a name that drifted + out of it must fail at the site that wrote it — not on the first launch + somebody happened to measure. A suite that runs with timing off, which is + every suite, is exactly the run a typo used to survive.""" + + def test_decorating_with_a_name_outside_the_vocabulary_is_refused(self): + with pytest.raises(ValueError): + + @timing.staged("host_prep") + def _launch(): + pass + + def test_every_name_in_the_vocabulary_decorates_and_calls_through(self): + for name in timing.STAGES: + assert timing.staged(name)(lambda: "ran")() == "ran" + + +def bench_module(): + """The bench script, imported by path — it is a script, not a package.""" + spec = importlib.util.spec_from_file_location("bench_launch", BENCH) + assert spec is not None and spec.loader is not None, BENCH + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class TestEveryDocumentedInvocationParses: + """Sibling of the cold-reset guard, for the flags rather than the reset. + + Same lesson from the same lineage (#192): every documented-command defect + in this script's history was a command nobody had fed to the thing that + reads it. So each invocation the epilog shows is handed to the real parser + — a flag that was renamed, or documented before it existed, fails here. + """ + + def test_the_epilog_shows_invocations_the_parser_accepts(self): + module = bench_module() + joined = re.sub(r"\\\n\s+", " ", module.EPILOG) + documented = re.findall(r"bench_launch\.py (.+)$", joined, re.M) + assert len(documented) >= 3, documented + for invocation in documented: + module.build_parser().parse_args(shlex.split(invocation))