From 5e08555e76617a288b0b4c6aeb7da618951fe7b5 Mon Sep 17 00:00:00 2001 From: "b10x-bot[bot]" <316511680+b10x-bot[bot]@users.noreply.github.com> Date: Sat, 26 Sep 2026 04:37:49 +0200 Subject: [PATCH 1/2] feat(ess): a catalogue of specification-hardening techniques - New skill ess:hardening: eight techniques for after a conformance suite is green, cheapest first with the design review early, and the rule that a check nobody has seen fail is not evidence. - References: a procedure and a defect to plant per technique, the reference-model pattern (to ship as ess features, beyond10x/ess#114), a design-review brief and spec-diff classification guidance. - ess:conformance offers the catalogue on a green suite, and ess:testing-conformance links it from its mutation section; the checker, README and site list the new skill. Refs: beyond10x/agentplugins#35 --- README.md | 2 +- crates/agentplugins-check/src/main.rs | 33 ++++ plugins/b10x/skills/routing/SKILL.md | 1 + plugins/ess/agents/conformance.md | 3 + plugins/ess/skills/hardening/SKILL.md | 91 +++++++++++ .../hardening/references/design-review.md | 71 ++++++++ .../hardening/references/reference-model.md | 85 ++++++++++ .../skills/hardening/references/spec-diff.md | 57 +++++++ .../skills/hardening/references/techniques.md | 152 ++++++++++++++++++ plugins/ess/skills/init/SKILL.md | 1 + .../ess/skills/testing-conformance/SKILL.md | 5 + website/docs/intro.md | 2 +- website/docs/plugins/ess.md | 1 + website/docs/structure.md | 2 +- 14 files changed, 503 insertions(+), 3 deletions(-) create mode 100644 plugins/ess/skills/hardening/SKILL.md create mode 100644 plugins/ess/skills/hardening/references/design-review.md create mode 100644 plugins/ess/skills/hardening/references/reference-model.md create mode 100644 plugins/ess/skills/hardening/references/spec-diff.md create mode 100644 plugins/ess/skills/hardening/references/techniques.md diff --git a/README.md b/README.md index 03b010f..f99b7b1 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ What each plugin ships: - [`ess`](plugins/ess/) · [docs](website/docs/plugins/ess.md) - - skills: [`init`](plugins/ess/skills/init/SKILL.md) · [`upgrade`](plugins/ess/skills/upgrade/SKILL.md) · [`retrofitting`](plugins/ess/skills/retrofitting/SKILL.md) · [`specifying`](plugins/ess/skills/specifying/SKILL.md) · [`testing-conformance`](plugins/ess/skills/testing-conformance/SKILL.md) + - skills: [`init`](plugins/ess/skills/init/SKILL.md) · [`upgrade`](plugins/ess/skills/upgrade/SKILL.md) · [`hardening`](plugins/ess/skills/hardening/SKILL.md) · [`retrofitting`](plugins/ess/skills/retrofitting/SKILL.md) · [`specifying`](plugins/ess/skills/specifying/SKILL.md) · [`testing-conformance`](plugins/ess/skills/testing-conformance/SKILL.md) - agents: [`author`](plugins/ess/agents/author.md) · [`conformance`](plugins/ess/agents/conformance.md) · [`retrofitter`](plugins/ess/agents/retrofitter.md) - [`aep`](plugins/aep/) · [docs](website/docs/plugins/aep.md) - skills: [`init`](plugins/aep/skills/init/SKILL.md) · [`upgrade`](plugins/aep/skills/upgrade/SKILL.md) · [`implementing`](plugins/aep/skills/implementing/SKILL.md) · [`migrating`](plugins/aep/skills/migrating/SKILL.md) · [`planning`](plugins/aep/skills/planning/SKILL.md) diff --git a/crates/agentplugins-check/src/main.rs b/crates/agentplugins-check/src/main.rs index 9fd9493..8e6d478 100644 --- a/crates/agentplugins-check/src/main.rs +++ b/crates/agentplugins-check/src/main.rs @@ -61,6 +61,11 @@ const PLUGINS: &[(&str, &[&str])] = &[ "skills/specifying/references/syntax.md", "skills/retrofitting/SKILL.md", "skills/testing-conformance/SKILL.md", + "skills/hardening/SKILL.md", + "skills/hardening/references/techniques.md", + "skills/hardening/references/reference-model.md", + "skills/hardening/references/design-review.md", + "skills/hardening/references/spec-diff.md", "agents/author.md", "agents/retrofitter.md", "agents/conformance.md", @@ -1376,6 +1381,34 @@ one product only: `b10x upgrade ess` std::fs::remove_dir_all(&sandbox).expect("the sandbox is removable"); } + /// A green suite is where hardening starts, so the two places an agent stands when it gets + /// there — the conformance agent's charter and the skill's mutation section — hand it the + /// catalogue. Without the link the catalogue is a file nobody finishing a coverage task reads. + #[test] + fn a_green_suite_is_handed_the_hardening_catalogue() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("checker is under repository root"); + let agent = std::fs::read_to_string(root.join("plugins/ess/agents/conformance.md")) + .expect("the conformance agent is readable"); + assert!( + agent.contains("`ess:hardening`"), + "the conformance agent does not offer `ess:hardening`" + ); + let skill = + std::fs::read_to_string(root.join("plugins/ess/skills/testing-conformance/SKILL.md")) + .expect("the testing-conformance skill is readable"); + let mutation = skill + .split_once("## Before anything: does a green run mean anything?") + .and_then(|(_, rest)| rest.split_once("\n## ").map(|(section, _)| section)) + .expect("the mutation section is present"); + assert!( + mutation.contains("`ess:hardening`"), + "the mutation section does not link `ess:hardening`" + ); + } + /// The committed critics carry the pin. This is the half of the check that would catch a fifth /// critic added without one. #[test] diff --git a/plugins/b10x/skills/routing/SKILL.md b/plugins/b10x/skills/routing/SKILL.md index ce1d210..eaff26a 100644 --- a/plugins/b10x/skills/routing/SKILL.md +++ b/plugins/b10x/skills/routing/SKILL.md @@ -30,6 +30,7 @@ Route the request; do not reproduce a specialist plugin's full workflow. | Specify a system or API | `ess:specifying` | | Derive a specification for an existing system | `ess:retrofitting` | | Run or raise a conformance suite | `ess:testing-conformance` | +| Harden a specification once its suite is green | `ess:hardening` | | Create, inspect, finish, or safely clean Git worktrees | `worktree:managing-worktrees` | | Set up providers, inspect Connector readiness, or invoke configured integrations through the CLI | `connectors:integrating` | diff --git a/plugins/ess/agents/conformance.md b/plugins/ess/agents/conformance.md index d0e15ed..cd4bfa4 100644 --- a/plugins/ess/agents/conformance.md +++ b/plugins/ess/agents/conformance.md @@ -16,3 +16,6 @@ Charter: - Never weaken a scenario or a target to turn a failure green. - Report: `passed`/`skipped`/`failed` before and after, the mutation you ran and the scenario it failed, and the exact commands. +- When a coverage task finishes on a green suite, offer the `ess:hardening` catalogue as the next + step, not only mutation: name the techniques it would run first (the design review, the spec + diff in the gate) and what each needs. Offer it; do not start it unasked. diff --git a/plugins/ess/skills/hardening/SKILL.md b/plugins/ess/skills/hardening/SKILL.md new file mode 100644 index 0000000..91f1879 --- /dev/null +++ b/plugins/ess/skills/hardening/SKILL.md @@ -0,0 +1,91 @@ +--- +name: hardening +description: >- + Harden an ESS specification and the implementation held to it once the conformance suite is green — ask the questions a passing suite cannot, with a catalogue of eight techniques from mutation to a design review. Use when the user asks how to harden a specification, whether a green suite can be trusted, what to do after coverage is raised, or asks for mutation testing, random command sequences or a model-based check, replay of a real caller, determinism checks, metamorphic relations, guard analysis, classifying a specification diff as breaking, or reviewing a design document against a specification. Not for raising the number of scenarios that execute, which is the `ess:testing-conformance` skill; not for writing the specification, which is `ess:specifying`. +--- + +# Hardening a specification + +A green conformance suite answers one question: do the scenarios somebody wrote pass. Each +technique here asks a different one, and each runs only after the suite is green — a red suite is +the work, and `ess:testing-conformance` is where it is done. + +## The rule that makes any of this count + +**A check nobody has seen fail is not evidence. Plant a defect first.** Before a technique's green +result is reported, break the implementation (or the specification) in the way the technique claims +to catch, run it, and record the named failure. Then restore and run it green. A technique that +cannot be made to fail on a planted defect has measured nothing, whatever it prints. + +Record both runs: the planted defect, the command, the failure it produced, and the green run after +restoring. + +## The catalogue + +| # | technique | the question | what it catches | needs from the IR | +|---|---|---|---|---| +| 1 | mutation audit | would the suite notice if a declared rule broke? | declared behaviour no scenario pins down: a dropped `from` state, a moved guard boundary, a dropped `sets` entry | lifecycles, guards, `sets`, payloads | +| 2 | random command sequences against a reference model | does the implementation agree with the spec along paths nobody wrote? | engine defects that only show later in a sequence: a view that caps its rows, a refusal that still writes, an identity reused, a state not re-enterable; invariants that hold only through undeclared defaults | everything the [reference model](references/reference-model.md) reads: lifecycles, guards, `sets`, payloads, views, invariants, `wrong_state` | +| 3 | replay of the real caller | does the code that *uses* the implementation send only what the spec accepts? | a caller relying on behaviour the spec does not declare; refusal outcomes the caller never triggers | the reference model, plus actors (who may send which command) | +| 4 | determinism | is the implementation deterministic where the spec assumes it? | clock reads, randomness, iteration order leaking into outcomes | none beyond technique 2's runner | +| 5 | metamorphic relations | do promises that span two runs hold? | a command allowed where the design forbids it ("a pause changes nothing but the pause commands") | the reference model; the relation comes from the design | +| 6 | exhaustive guard analysis | are guards dead, overlapping, gap-leaving, or weak enough to admit an invariant break? | a `when:` that can never match, two that match together, an input no outcome answers | guards, field types, invariants | +| 7 | spec diff in the gate | is a change breaking, and was that acknowledged? | a narrowing released as if it were additive | two compiled revisions | +| 8 | design review by an agent | what does the design say that the spec omits or contradicts? | rules the design states and the spec never declares ("one entry per player", "the count always goes up") | the spec files and the design document; no IR | + +Procedures, with the defect to plant for each: [references/techniques.md](references/techniques.md). + +Formal model checking (a TLA+ or Alloy export) is not in the catalogue. On a spec with 31 commands, +random sequences reached every declared outcome, 59 of 59; reach for a model checker only when +technique 2 reports declared outcomes it never reaches. + +## The order + +Cheapest first, with one exception: **the design review runs early, because it finds the most.** On +one adopter's spec (3 domains, 7 entities, 31 commands, a 121-scenario suite that passed) it +produced 26 findings, the most of any technique. What it finds changes the spec, and +every later technique is then run against the spec that will stay. + +1. **Design review** (8) — no tooling, one agent, the [brief](references/design-review.md). +2. **Spec diff in the gate** (7) — one command and a [classification](references/spec-diff.md); it + protects everything after it. +3. **Mutation audit** (1) — reuses the suite you already have. +4. **Guard analysis** (6) — reads the IR only; no implementation runs. +5. **Reference model and random sequences** (2) — the one piece of code the catalogue needs; build it + once. +6. **Determinism** (4), **metamorphic relations** (5) and **caller replay** (3) — each reuses the + runner from step 5. + +Stop where the cost exceeds what the spec is worth, and say which techniques were not run. + +## What the IR is for + +Techniques 1–6 read the canonical IR, not the YAML: + +```console +ess specify compile --path --format json --out +``` + +The IR is the resolved model: every name qualified, every guard in structured form, every default +applied. A technique that reads the YAML re-implements resolution and disagrees with the compiler +somewhere nobody looked. + +## The reference model + +Techniques 2, 3 and 5 rest on one small interpreter over the IR — about 150 lines — that answers, +for any command in any state, what the spec says happens. `ess` does not ship it yet; +[beyond10x/ess#114](https://github.com/beyond10x/ess/issues/114) proposes shipping the mutation audit +and the sequence runner as `ess` features, and carries a draft for the TypeScript target. Until it +ships, build it from the pattern: [references/reference-model.md](references/reference-model.md). + +## Reporting + +Per technique run: the planted defect and the failure it produced, the green result after restoring, +the command, and the findings — each one a spec gap, an implementation defect or a false positive of +the technique, said which. Name the techniques not run. A finding that belongs to `ess` itself (a +generator or validate gap) is an issue on beyond10x/ess, not a workaround here. + +## Next + +- A finding changes the specification: `ess:specifying`. +- A finding is a missing scenario or a skipped one: `ess:testing-conformance`. diff --git a/plugins/ess/skills/hardening/references/design-review.md b/plugins/ess/skills/hardening/references/design-review.md new file mode 100644 index 0000000..9b5a223 --- /dev/null +++ b/plugins/ess/skills/hardening/references/design-review.md @@ -0,0 +1,71 @@ +# Design review: a brief for an agent + +Technique 8. Dispatch one agent with the brief below, filled in. It found the most of any technique +on the one adopter where all eight were run — 26 findings — so run it early. + +The agent needs read access to three things and write access to none of them. + +--- + +## Brief + +You are reviewing a specification against the design it claims to implement. You do not edit +either. You produce a list of findings. + +**Inputs** + +- the design document: `` +- the mapping, if one exists — the document or table that says which design statement became which + spec declaration: ``, or `none` +- the specification: ``; validate it first with `ess specify validate --path ` + and stop if it refuses + +**Procedure** + +1. Read the design end to end. List every statement that promises behaviour: a rule, a limit, an + ordering, a uniqueness, a "never", an "always", a "only when". +2. For each statement, find where the spec declares it — an entity field or invariant, a lifecycle + transition, a command outcome and its `when:` guard, an actor's grant, a view. +3. Then go the other way: for every spec declaration, find the design statement that justifies it. +4. Classify each mismatch with exactly one of the classifications below. + +**Classifications** + +| classification | meaning | +|---|---| +| `missing` | the design states it; the spec does not declare it | +| `contradicts` | both state it, and they disagree | +| `stale mapping` | the mapping points a design statement at a declaration that no longer says it, or at one that does not exist | +| `spec-only` | the spec declares it; no design statement justifies it | +| `unclear` | the design statement is too vague to decide which of the above applies | + +**Every finding cites both sides.** The design by its line (`design.md:42`, quoted), the spec by +`file:line` (quoted). A `missing` finding cites the spec file and line where the declaration would +belong; a `spec-only` finding cites the nearest design section and says it has nothing. A finding +without both citations is not a finding; drop it. + +**Quote a vague statement and mark it `unclear`; never invent a meaning for it.** "Players should +not be able to cheat" is `unclear` with that sentence quoted. Supplying a plausible rule and then +reporting the spec as missing it is the one failure this review exists to avoid: downstream, the +invented rule reads as the designer's. + +**Output**, one row per finding: + +```text +| # | classification | design (file:line, quoted) | spec (file:line, quoted) | note | +``` + +Then the counts per classification, and the design sections you did not reach, if any. + +--- + +## After the review + +- Each `missing` and `contradicts` finding goes to the design's owner or becomes a spec change + (`ess:specifying`); the review does not decide which. +- `spec-only` findings are candidates for deletion — "a declared refusal nobody executes is a + fiction" (`ess:testing-conformance`). +- `unclear` findings go back to the design's author as questions, with the quote. + +**Plant a defect before trusting a clean review:** on a copy of the spec, delete one declaration the +design plainly states, and run the brief on the copy. It must come back `missing`, citing both sides. diff --git a/plugins/ess/skills/hardening/references/reference-model.md b/plugins/ess/skills/hardening/references/reference-model.md new file mode 100644 index 0000000..42b3655 --- /dev/null +++ b/plugins/ess/skills/hardening/references/reference-model.md @@ -0,0 +1,85 @@ +# The reference model: a pattern + +Techniques 2, 3 and 5 compare an implementation with what the spec says happens. That needs a +**reference model**: a small interpreter over the compiled IR that, given a state and a command, +returns the outcome the spec declares. On one adopter it was about 150 lines of TypeScript. + +**Status.** `ess` does not ship this. [beyond10x/ess#114](https://github.com/beyond10x/ess/issues/114) +proposes shipping the mutation audit and a seeded sequence runner built on this model as `ess` +features, emitted beside the suite for `--target typescript` and `--target go` and driven through the +same `Target` interface; the issue carries a draft (`explore.ts`) for the TypeScript target. Until it +ships, write the model yourself from the pattern below, and check that issue before starting — once +it ships, use what it emits instead. + +## Input + +The canonical JSON IR, and nothing else: + +```console +ess specify compile --path --format json --out +``` + +Read the IR, never the YAML: the IR has qualified names, guards in structured form and defaults +applied. Read field names from your own IR file rather than from this page; the pattern is stable, +the exact keys are the compiler's. + +## State + +```text +instances: map entity -> map identity -> { state, fields } +issued: set of every identity ever created (identity uniqueness) +``` + +Nothing else. A model that tracks anything the IR does not declare is asserting behaviour the spec +does not have. + +## Step: one command + +For `execute(command, input)`: + +1. **Pick the outcome.** Evaluate each outcome's `when:` predicate over the input — comparisons over + `input_field` and `literal` values, combined with and/or/not. Exactly one non-`wrong_state` + outcome must match; zero or two is a spec defect, report it rather than choosing. +2. **Check the lifecycle.** For an outcome that `moves` an instance through a transition, look the + instance up by the outcome's `instance` field. Missing instance, or current state not in the + transition's `from`: answer with the command's `wrong_state` outcome instead, and **change + nothing** — no `sets`, no events. (That a `wrong_state` answer still applied `sets` was one of the + defects this found.) +3. **Apply.** + - `creates`: a new identity (refuse to reuse one in `issued`), the lifecycle's `initial` state, the + fields the outcome sets. + - `moves`: set the state to the transition's `to`. + - `updates` / `sets`: write each field from its value expression (`input.` or a literal). + - `error`: change nothing. +4. **Emit.** For each event in `emits`, build its payload from the outcome's `payload` mapping. +5. **Check invariants** of every instance the step touched, over its fields after the step. + +Return `{ outcome, error, events, payloads }`. + +## Views + +For `query(view)`: take every instance of the view's `source`, keep those its `filter` admits, and +project the view's `fields`. Compare rows **and their order** with the implementation's answer; a +view whose order the spec does not declare compares as a set, and the model says which. + +**Unknown is an answer.** Where a view publishes a field no command ever set on that instance, the +model does not know its value. Report that as its own finding: whatever the implementation shows, +it rests on an undeclared default. That is how three invariants that held only by accident were +found (beyond10x/ess#112). + +## What drives it + +The model is the oracle; a runner drives it and the implementation side by side: + +- a seeded generator picks actor, command and input (valid, at each guard boundary, and invalid) — + the actor from those whose grants include the command, plus one that is not, to exercise refusal; +- after each step, compare the step results, then every view, then identity uniqueness; +- shrink a failing trace by dropping steps while it still fails; +- at the end, list every declared outcome never reached, and fail if the list is not empty. + +## Proving the model is not the implementation's twin + +A model written by reading the implementation agrees with it by construction. Write it from the IR +only, and before trusting it, plant one engine defect in the implementation (a view capped at 5 +rows, a `wrong_state` that still writes, a reused identity, a state that cannot be re-entered). The +runner must catch it. A model that agrees with a planted defect is a copy, not a reference. diff --git a/plugins/ess/skills/hardening/references/spec-diff.md b/plugins/ess/skills/hardening/references/spec-diff.md new file mode 100644 index 0000000..c8ccef8 --- /dev/null +++ b/plugins/ess/skills/hardening/references/spec-diff.md @@ -0,0 +1,57 @@ +# Spec diff in the gate: classifying changes + +Technique 7. The gate compares the specification with the one at the last release tag, and fails +on a breaking change nobody acknowledged. + +## The diff + +`ess verify diff` compares two specification directories: + +```console +ess verify diff --from --to --format json +``` + +`--from` and `--to` are paths, so materialise the tagged revision first, for example +`git archive | tar -x -C `. The `ess-diff` document lists `changes`, each +with a stable `id` (`type//variant-removed/`), a `relation` (`expanded`, `narrowed` +or `changed`) and the change itself. + +`ess` names each change and its relation; **it does not decide whether a change is breaking.** Until +it does, classify with the rule below. + +## The rule + +**Additive** (not breaking): + +- an added item — a type, entity, field, command, outcome, event, view, actor +- an added enum variant +- an added lifecycle transition +- an added grant (an actor `may` one more command) +- an added `accepts` or `publishes` entry +- a wording change — `summary`, `naming.display`, descriptions + +**Breaking:** everything else, and **anything whose `relation` is `narrowed`**, whatever its kind. A +removal, a rename, a changed guard, a changed invariant, a changed type, a changed `wire` name — all +breaking. When a change's kind is not on the additive list, it is breaking; the list grows by +decision, not by argument. + +## Acknowledgement + +A breaking change passes the gate only when acknowledged, and an acknowledgement is a **committed +file keyed to the release tag** — for example `spec-acknowledgements/.yaml` — listing each +acknowledged change by its `id`, with one line saying why it is acceptable. + +The gate: + +1. materialises the spec at the last release tag; +2. runs the diff; +3. classifies each change by the rule; +4. fails on any breaking change whose `id` is not in the acknowledgement file for that tag, naming + the `id`; +5. fails on any acknowledged `id` that no longer appears in the diff — a stale acknowledgement + hides the next change with the same name. + +A new release tag starts an empty acknowledgement file; the previous tag's never carries over. + +**Plant a defect before trusting it:** on a branch, remove one enum variant. The gate must fail and +name the `variant-removed` id; add that id to the acknowledgement file and it must pass. diff --git a/plugins/ess/skills/hardening/references/techniques.md b/plugins/ess/skills/hardening/references/techniques.md new file mode 100644 index 0000000..6955d33 --- /dev/null +++ b/plugins/ess/skills/hardening/references/techniques.md @@ -0,0 +1,152 @@ +# The eight techniques, step by step + +Each procedure ends the same way: plant the named defect, watch the technique fail on it, restore, +run it green. The figures in each section are from one adopter's spec (3 domains, 7 entities, 31 +commands, a JavaScript implementation, a generated suite of 121 scenarios that all passed); they say +what a technique is worth on a real spec, not what yours will find. + +Every technique that reads the model reads the compiled IR: + +```console +ess specify compile --path --format json --out +``` + +## 1. Mutation audit + +**Question:** would the suite notice if a declared rule broke? + +1. From the IR, list one mutant per declared rule. The classes that find gaps: + + | class | example | + |---|---| + | drop a `from` state | `close: from [Open, Held]` → `[Open]` | + | change a transition's `to` | `B → C` becomes `B → B` | + | move a guard boundary | `items >= 0` → `items > 0` | + | drop a `sets` entry | `colour: input.colour` removed | + | invert or weaken a guard | `A && B` → `A \|\| B` | + | wrong error, event or order | `desc` → `asc` | + +2. Apply each mutant — to the implementation's rule table, or to a copy of the specification whose + suite is re-synthesized and run against the unchanged implementation — one at a time. +3. Record, per mutant, the scenarios that failed. A mutant no scenario kills is a declared rule the + suite does not pin down. +4. For each survivor, decide: a missing scenario (author one, then re-run that mutant and see it + killed), a view that does not expose the field (see `ess:testing-conformance`, "does a green run + mean anything?"), or a generator gap (an issue on beyond10x/ess). + +**Planted defect:** the audit is its own plant — but confirm the harness first with one mutant you +know a scenario kills. If it survives, the harness is not applying mutants. + +**Found:** 8 of 35 single-rule mutants passed every scenario; 6 were declared behaviour. After 6 +authored scenarios, 1 survived, and it rested on a value the spec did not declare. + +## 2. Random command sequences against a reference model + +**Question:** does the implementation agree with the spec along paths nobody wrote? + +1. Build the [reference model](reference-model.md) over the IR. +2. Drive a seeded random walk: at each step pick an actor, a command and an input (valid, boundary + and invalid), send it to the implementation through the suite's own `Target` interface + (`executeCommand`, `queryView`), and to the model. +3. After every step compare: the outcome, the error, the events and their payloads, every view's rows + and their order, identity uniqueness, and every invariant. +4. On a disagreement, shrink the trace (drop steps while it still disagrees) and report the shortest. +5. Fail the run when a declared outcome is never reached across all seeds. An unreached outcome is + untested, not passed. +6. Where the model cannot say what a view shows because no command sets the field, report it: the + invariant holds only through an undeclared default. + +**Planted defect:** one engine defect the suite passes — a view returning at most 5 rows, or a +`wrong_state` answer that still applies `sets`. The walk must catch it and shrink it. + +**Found:** 300 walks of 80 steps in about 1.3 s caught 4 of 4 planted engine defects that the +127-scenario suite passed, and surfaced 3 invariants that held only through undeclared defaults. + +## 3. Replay of the real caller + +**Question:** does the code that uses the implementation send only what the spec accepts? + +1. Record the caller's command stream: the application's own traffic, from a log, a test run or a + recorded session. Strip any personal data before it leaves the environment it was recorded in. +2. Replay it through the reference model, from the same initial state. +3. Report every command the model refuses or answers differently from what the caller expected, and + every refusal outcome the caller never triggers (a candidate for a caller-side test, or for + deletion from the spec if it is fiction). +4. Check the actor of each command against the actors' grants. + +**Planted defect:** edit one recorded command so it sends a field the spec does not accept, or runs +from a state the transition does not start from. The replay must name it. + +**Found:** 0 disagreements over 25,980 commands; 25 refusal outcomes the caller never triggers. + +## 4. Determinism + +**Question:** is the implementation deterministic where the spec assumes it? + +1. Run technique 2 twice with the same seed and compare the two command streams and every result, + byte for byte. +2. Scan the implementation's source for clock reads, randomness and unordered iteration in code a + command reaches (`Date.now`, `Math.random`, `time.Now`, `rand.`, iteration over a map or set). + Each hit is either injected (a seed, a clock the target controls) or a finding. + +**Planted defect:** add a `Math.random()` (or the language's equivalent) to one outcome. The two +runs must diverge, and the report must name the first divergent command. + +**Found:** passed; the planted `Math.random()` diverged at command 235. + +## 5. Metamorphic relations + +**Question:** do promises that span two runs hold? + +1. Read the relations from the design, not the spec: "a pause changes nothing but the pause + commands", "cancelling and re-creating equals never creating", "the order two independent + commands arrive in does not matter". +2. For each, generate a base trace with technique 2's walker, derive the paired trace (insert the + pause, swap the independent pair), run both, and compare what the relation says must be equal. + +**Planted defect:** make one command succeed while paused. The pause relation must fail on it. + +**Found:** 1 real defect: an action allowed while paused that the design forbids. + +## 6. Exhaustive guard analysis + +**Question:** are guards dead, overlapping, gap-leaving, or weak enough to admit an invariant break? + +1. For each command, collect its outcomes' `when:` guards from the IR. +2. Build each guard's boundary domain from the field types and every literal the guards compare + against: each literal, one either side, the type's extremes, and for an enum every variant. +3. Evaluate every guard of the command over the product of those domains and report: a guard no + input satisfies (dead), an input two guards both satisfy (overlap), an input no outcome answers + (gap), and an input a guard admits that leaves an invariant false after the outcome's `sets`. +4. Optionally cross-check each finding, and each claimed absence, with `z3`. + +**Planted defect:** in a copy of the spec, where one outcome's guard is `>= 0` and its sibling's +is `< 0`, change the sibling to `<= 0`. The analysis must report the overlap at `0`. If +`ess specify validate` already refuses the planted copy, the compiler covers that class: plant a gap +instead (`< 0` → `< -1`) and record which class the compiler caught. + +**Found:** 11 guards, 374 combinations, 0 findings — a result that counted only because the planted +overlap was reported first. + +## 7. Spec diff in the gate + +**Question:** is a change breaking, and was that acknowledged? + +Follow [spec-diff.md](spec-diff.md). + +**Planted defect:** remove one enum variant on a branch. The gate must fail until the change is +acknowledged. + +**Found:** 20 changes since the release tag, all additive. + +## 8. Design review by an agent + +**Question:** what does the design say that the spec omits or contradicts? + +Dispatch one agent with [design-review.md](design-review.md) as its brief. + +**Planted defect:** delete one declared rule from a copy of the spec that the design states. The +review must report it as `missing`, citing both sides. + +**Found:** 26 findings, the most of any technique — among them "one entry per player" and "the +count always goes up", neither declared at all. diff --git a/plugins/ess/skills/init/SKILL.md b/plugins/ess/skills/init/SKILL.md index 1ab51be..4b0dd34 100644 --- a/plugins/ess/skills/init/SKILL.md +++ b/plugins/ess/skills/init/SKILL.md @@ -56,6 +56,7 @@ validation refuses or a scenario fails, that refusal or failure is the work; rou | write a new specification, add an entity, validate, compile or project one | `ess:specifying` | `ess:author` | | give an existing codebase a specification it never had | `ess:retrofitting` | `ess:retrofitter` | | raise or audit what a conformance suite actually tests | `ess:testing-conformance` | `ess:conformance` | +| harden a specification whose suite is green | `ess:hardening` | — | A skill that is not loaded yet in this session prints with `b10x skill ess:`. diff --git a/plugins/ess/skills/testing-conformance/SKILL.md b/plugins/ess/skills/testing-conformance/SKILL.md index 26f4ff4..4c434b4 100644 --- a/plugins/ess/skills/testing-conformance/SKILL.md +++ b/plugins/ess/skills/testing-conformance/SKILL.md @@ -40,6 +40,10 @@ The mechanism generalises, so learn to spot it by reading rather than by mutatin Do the mutation test once per mapping you rely on, and record in the commit that you did it and what failed. A scenario nobody has ever seen fail is a scenario that has never been tested. +Mutation is one of eight hardening techniques. Once the suite is green, `ess:hardening` carries the +rest — random command sequences against a reference model, caller replay, determinism, metamorphic +relations, guard analysis, a spec diff in the gate and a design review — and the order to run them in. + ## Ranking the skips **Never rank skips by construct.** Counting which ESS constructs appear in skipped scenarios and @@ -273,3 +277,4 @@ run it once something answers it. ## Next - A scenario reveals a gap in the specification: `ess:specifying`. +- The suite is green and the question is what it still misses: `ess:hardening`. diff --git a/website/docs/intro.md b/website/docs/intro.md index 107b03f..01122a0 100644 --- a/website/docs/intro.md +++ b/website/docs/intro.md @@ -15,7 +15,7 @@ directly when the work is already clear. |---|---|---| | [`b10x`](./plugins/b10x.md) | Setup, upgrades and navigation | `init`, `upgrade`, `routing` and `authoring-plugins` skills, the `b10x` binary, drift check | | [`aep`](./plugins/aep.md) | Governed planning and delivery | `planning`, `migrating` and `implementing` skills; decomposer, plan critics, reverse engineer, story scoper, implementor, adversary, security reviewer | -| [`ess`](./plugins/ess.md) | Executable System Specifications | `specifying`, `retrofitting` and `testing-conformance` skills; author, conformance, retrofitter agents | +| [`ess`](./plugins/ess.md) | Executable System Specifications | `specifying`, `retrofitting`, `testing-conformance` and `hardening` skills; author, conformance, retrofitter agents | | [`worktree`](./plugins/worktree.md) | Git workspaces | managed worktrees, leases, recovery proof, and safe cleanup | Every plugin starts with `/:init` and checks itself with `/:upgrade`; `/b10x:init` diff --git a/website/docs/plugins/ess.md b/website/docs/plugins/ess.md index a0391f8..640cdf5 100644 --- a/website/docs/plugins/ess.md +++ b/website/docs/plugins/ess.md @@ -14,6 +14,7 @@ to raise or audit a conformance suite against a real implementation. | `ess:specifying` | write or extend a specification; `references/syntax.md` shows every section in one that validates | `author` | | `ess:retrofitting` | derive a specification for a system that has none | `retrofitter` | | `ess:testing-conformance` | raise or audit what a conformance suite tests | `conformance` | +| `ess:hardening` | after a green suite, the eight techniques that ask what it cannot; `references/` holds each procedure, the reference-model pattern, a design-review brief and spec-diff classification | — | | `ess:upgrade` | check the plugin and CLI, offer the upgrade | — | ```text diff --git a/website/docs/structure.md b/website/docs/structure.md index 395dbf5..40f061e 100644 --- a/website/docs/structure.md +++ b/website/docs/structure.md @@ -25,7 +25,7 @@ tools` checks the skills against the newest CLI releases. A change that breaks a |---|---|---|---| | `b10x` | `init` (guided onboarding), `upgrade` | `routing`, `authoring-plugins` | — | | `aep` | `init`, `upgrade` | `planning`, `migrating`, `implementing` (wave or drive mode) | `planning`: decomposer, four plan critics, plan reviewer, reverse engineer · `implementing`: story scoper, implementor, adversary, security reviewer | -| `ess` | `init`, `upgrade` | `specifying`, `retrofitting`, `testing-conformance` | `specifying`: author · `retrofitting`: retrofitter · `testing-conformance`: conformance | +| `ess` | `init`, `upgrade` | `specifying`, `retrofitting`, `testing-conformance`, `hardening` | `specifying`: author · `retrofitting`: retrofitter · `testing-conformance`: conformance | | `worktree` | `init`, `upgrade` | `managing-worktrees` | — | | `connectors` | `init`, `upgrade` | `integrating` | — | From 7ddf54f2b3e5e555b6f1c09becd2c5ae726feaa9 Mon Sep 17 00:00:00 2001 From: "b10x-bot[bot]" <316511680+b10x-bot[bot]@users.noreply.github.com> Date: Sat, 26 Sep 2026 04:38:03 +0200 Subject: [PATCH 2/2] chore: release 0.14.13 - CHANGELOG section for #35. - Workspace, Cargo.lock and all ten plugin manifests move to 0.14.13. Refs: beyond10x/agentplugins#35 --- CHANGELOG.md | 13 +++++++++++++ Cargo.lock | 4 ++-- Cargo.toml | 2 +- plugins/aep/.claude-plugin/plugin.json | 2 +- plugins/aep/.codex-plugin/plugin.json | 2 +- plugins/b10x/.claude-plugin/plugin.json | 2 +- plugins/b10x/.codex-plugin/plugin.json | 2 +- plugins/connectors/.claude-plugin/plugin.json | 2 +- plugins/connectors/.codex-plugin/plugin.json | 2 +- plugins/ess/.claude-plugin/plugin.json | 2 +- plugins/ess/.codex-plugin/plugin.json | 2 +- plugins/worktree/.claude-plugin/plugin.json | 2 +- plugins/worktree/.codex-plugin/plugin.json | 2 +- 13 files changed, 26 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a15a0f3..911c023 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [0.14.13] — 2026-09-26 + +Hardening a specification after its suite is green. + +- `ess:hardening`: eight specification-hardening techniques for after a conformance suite is green — + mutation, random sequences against a reference model, caller replay, determinism, metamorphic + relations, guard analysis, spec diff in the gate and design review — in cheapest-first order with + the design review early, and the rule that a check nobody has seen fail is not evidence. Its + `references/` carry a procedure per technique, the reference-model pattern (to ship as `ess` + features, beyond10x/ess#114), a design-review brief and spec-diff classification. (#35) +- `ess:conformance` offers the catalogue when a coverage task ends on a green suite, and + `ess:testing-conformance` links it from its mutation section. + ## [0.14.12] — 2026-09-26 Skills that start from what a repository already has, and plans that say what the other host needs. diff --git a/Cargo.lock b/Cargo.lock index 4613267..656897e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,7 +4,7 @@ version = 4 [[package]] name = "agentplugins-check" -version = "0.14.12" +version = "0.14.13" dependencies = [ "clap", "serde", @@ -76,7 +76,7 @@ dependencies = [ [[package]] name = "b10x" -version = "0.14.12" +version = "0.14.13" dependencies = [ "clap", "serde", diff --git a/Cargo.toml b/Cargo.toml index f24fde3..2c41d0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = ["crates/agentplugins-check", "crates/b10x"] [workspace.package] -version = "0.14.12" +version = "0.14.13" edition = "2021" rust-version = "1.85" license = "Apache-2.0" diff --git a/plugins/aep/.claude-plugin/plugin.json b/plugins/aep/.claude-plugin/plugin.json index d8b87b5..f212542 100644 --- a/plugins/aep/.claude-plugin/plugin.json +++ b/plugins/aep/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "name": "aep", "displayName": "AEP", "description": "Plan governed work in the AEP artifact store and deliver it in reviewed waves: decomposition, plan critique, reverse engineering, story scoping, implementation and adversarial review.", - "version": "0.14.12", + "version": "0.14.13", "author": { "name": "Beyond10x" }, diff --git a/plugins/aep/.codex-plugin/plugin.json b/plugins/aep/.codex-plugin/plugin.json index 36c0879..1d11d4c 100644 --- a/plugins/aep/.codex-plugin/plugin.json +++ b/plugins/aep/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "aep", - "version": "0.14.12", + "version": "0.14.13", "description": "Plan governed work in the AEP artifact store and deliver it in reviewed waves.", "author": { "name": "Beyond10x" diff --git a/plugins/b10x/.claude-plugin/plugin.json b/plugins/b10x/.claude-plugin/plugin.json index 6b655a1..04bce58 100644 --- a/plugins/b10x/.claude-plugin/plugin.json +++ b/plugins/b10x/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "name": "b10x", "displayName": "Beyond10x", "description": "Set up, upgrade and check the Beyond10x plugins and binaries, route work to them, and create portable plugins.", - "version": "0.14.12", + "version": "0.14.13", "author": { "name": "Beyond10x" }, diff --git a/plugins/b10x/.codex-plugin/plugin.json b/plugins/b10x/.codex-plugin/plugin.json index 439027a..39320a9 100644 --- a/plugins/b10x/.codex-plugin/plugin.json +++ b/plugins/b10x/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "b10x", - "version": "0.14.12", + "version": "0.14.13", "description": "Set up, upgrade and check the Beyond10x plugins and binaries, route work to them, and create portable plugins.", "author": { "name": "Beyond10x" diff --git a/plugins/connectors/.claude-plugin/plugin.json b/plugins/connectors/.claude-plugin/plugin.json index e7251c4..b426dd1 100644 --- a/plugins/connectors/.claude-plugin/plugin.json +++ b/plugins/connectors/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "connectors", - "version": "0.14.12", + "version": "0.14.13", "description": "Set up, inspect, and invoke governed integrations through the connectors CLI.", "author": { "name": "Beyond10x" }, "license": "Apache-2.0", diff --git a/plugins/connectors/.codex-plugin/plugin.json b/plugins/connectors/.codex-plugin/plugin.json index dc909b0..5d13ea4 100644 --- a/plugins/connectors/.codex-plugin/plugin.json +++ b/plugins/connectors/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "connectors", - "version": "0.14.12", + "version": "0.14.13", "description": "Set up, inspect, and invoke governed integrations through the connectors CLI.", "author": { "name": "Beyond10x" }, "license": "Apache-2.0", diff --git a/plugins/ess/.claude-plugin/plugin.json b/plugins/ess/.claude-plugin/plugin.json index 7db69e4..fc0629f 100644 --- a/plugins/ess/.claude-plugin/plugin.json +++ b/plugins/ess/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "name": "ess", "displayName": "ESS", "description": "Write, retrofit, validate and project Executable System Specifications, and hold implementations to them with conformance suites.", - "version": "0.14.12", + "version": "0.14.13", "author": { "name": "Beyond10x" }, diff --git a/plugins/ess/.codex-plugin/plugin.json b/plugins/ess/.codex-plugin/plugin.json index 232fa9e..063bf10 100644 --- a/plugins/ess/.codex-plugin/plugin.json +++ b/plugins/ess/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "ess", - "version": "0.14.12", + "version": "0.14.13", "description": "Write, retrofit, validate and project Executable System Specifications, and hold implementations to them with conformance suites.", "author": { "name": "Beyond10x" diff --git a/plugins/worktree/.claude-plugin/plugin.json b/plugins/worktree/.claude-plugin/plugin.json index d26ef29..737c00e 100644 --- a/plugins/worktree/.claude-plugin/plugin.json +++ b/plugins/worktree/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "name": "worktree", "displayName": "Worktree", "description": "Create, lease, finish, audit and safely clean isolated Git worktrees through the worktree CLI.", - "version": "0.14.12", + "version": "0.14.13", "author": { "name": "Beyond10x" }, diff --git a/plugins/worktree/.codex-plugin/plugin.json b/plugins/worktree/.codex-plugin/plugin.json index 015ff2e..c74b1ed 100644 --- a/plugins/worktree/.codex-plugin/plugin.json +++ b/plugins/worktree/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "worktree", - "version": "0.14.12", + "version": "0.14.13", "description": "Create, lease, finish, audit and safely clean isolated Git worktrees through the worktree CLI.", "author": { "name": "Beyond10x"