diff --git a/CHANGELOG.md b/CHANGELOG.md index f36e352..7f85054 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## [0.14.10] — 2026-09-25 + +From round 5, the first measured trial round: seven trials on 0.14.9, all isolated. The +full-package trial generated every ESS output and ran `go test` against the synthesized suite: 13 +passed, 0 failed, 0 skipped. + +- `agentplugins-check trial-report` recognises every spelling of `go test` (`go -C test`, + behind environment assignments, after `cd … &&`) and only in command position; it had reported the + full-package run as "not run". +- `b10x setup plan` with no selection keeps every installed product, optional ones included; it + proposed uninstalling an installed `connectors`. +- `ess:specifying`: every `--kind` of `ess generate` (`schema`, `openapi`, `asyncapi`, `docs`, + `site`, `docs-ir`) and `ess generate types`; one `--out` per kind side by side, because nested + output directories are refused; which kinds add their own folder (`site` does not). OpenAPI and + AsyncAPI need a component that says how the service is reached, or they write `0 artifact(s)` + without a refusal (beyond10x/ess#102). +- `ess:testing-conformance`: generate the Go suite inside the implementation's module; return payload + numbers as `float64` (beyond10x/ess#101). +- Trials: `cargo` is allowed (the Rust type library can be compiled); the aep, worktree and upgrade + prompts say nobody will answer questions. + ## [0.14.9] — 2026-09-25 Trials become measurable: their prompts live in the repository, and each run is scored against a diff --git a/Cargo.lock b/Cargo.lock index 208b1c7..0e43bdd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,7 +4,7 @@ version = 4 [[package]] name = "agentplugins-check" -version = "0.14.9" +version = "0.14.10" dependencies = [ "clap", "serde", @@ -76,7 +76,7 @@ dependencies = [ [[package]] name = "b10x" -version = "0.14.9" +version = "0.14.10" dependencies = [ "clap", "serde", diff --git a/Cargo.toml b/Cargo.toml index 33f2baf..2be2824 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = ["crates/agentplugins-check", "crates/b10x"] [workspace.package] -version = "0.14.9" +version = "0.14.10" edition = "2021" rust-version = "1.85" license = "Apache-2.0" diff --git a/Taskfile.yml b/Taskfile.yml index 41b91e8..7df2984 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -91,7 +91,7 @@ tasks: - | (. '{{.T}}/env' && dir="$(cat '{{.T}}/workdir')" && mkdir -p "{{.T}}/work/$dir" && cd "{{.T}}/work/$dir" && timeout 2400 claude -p "$(cat '{{.T}}/prompt.txt')" --permission-mode acceptEdits \ - --allowedTools 'Bash(b10x:*)' 'Bash(ess:*)' 'Bash(aep:*)' 'Bash(worktree:*)' 'Bash(git:*)' 'Bash(go:*)' \ + --allowedTools 'Bash(b10x:*)' 'Bash(ess:*)' 'Bash(aep:*)' 'Bash(worktree:*)' 'Bash(git:*)' 'Bash(go:*)' 'Bash(cargo:*)' \ 'Bash(mkdir:*)' 'Bash(ls:*)' 'Bash(cat:*)' 'Bash(find:*)' 'Bash(curl:*)' 'Bash(tar:*)' \ 'Bash(sha256sum:*)' 'Bash(mv:*)' 'Bash(chmod:*)' Read Write Edit Glob Grep Skill Task Agent \ --strict-mcp-config --output-format stream-json --verbose > '{{.T}}/run.jsonl' 2> '{{.T}}/run.err' diff --git a/crates/agentplugins-check/src/report.rs b/crates/agentplugins-check/src/report.rs index d97d21c..aa04325 100644 --- a/crates/agentplugins-check/src/report.rs +++ b/crates/agentplugins-check/src/report.rs @@ -283,10 +283,44 @@ fn go_counts(output: &str) -> Option { Some(counts) } +/// Whether a shell command runs `go test`, however it is spelled: `go test`, `go -C test`, +/// behind environment assignments or after `cd … &&`. +fn runs_go_test(command: &str) -> bool { + let words: Vec = shlex::split(command) + .unwrap_or_else(|| command.split_whitespace().map(str::to_owned).collect()); + words.iter().enumerate().any(|(index, word)| { + if word != "go" && !word.ends_with("/go") { + return false; + } + // In command position: every word since the last separator is an environment assignment. + let start = words[..index] + .iter() + .rposition(|w| matches!(w.as_str(), "&&" | "||" | ";" | "|" | "then" | "do")) + .map_or(0, |separator| separator + 1); + if !words[start..index] + .iter() + .all(|w| w.contains('=') && !w.starts_with('-')) + { + return false; + } + let mut rest = words[index + 1..].iter(); + while let Some(next) = rest.next() { + match next.as_str() { + "-C" => { + rest.next(); + } + flag if flag.starts_with('-') => {} + verb => return verb == "test", + } + } + false + }) +} + fn go_test(run: &Run) -> Option { run.shells .iter() - .filter(|shell| shell.command.contains("go test")) + .filter(|shell| runs_go_test(&shell.command)) .rev() .find_map(|shell| go_counts(shell.output.as_deref()?)) } @@ -657,6 +691,27 @@ mod tests { std::fs::remove_dir_all(&sandbox).unwrap(); } + #[test] + fn every_spelling_of_go_test_is_recognised() { + for command in [ + "go test ./...", + "cd impl && go test -v ./...", + "go -C /work/impl test -count=1 -v ./... 2>&1 | tail -20", + "ESS_REPORT_OUT=r.json go -C impl test ./...", + "/usr/local/go/bin/go test ./...", + ] { + assert!(runs_go_test(command), "{command}"); + } + for command in [ + "go vet ./...", + "go -C impl build ./...", + "echo go test", + "cargo test", + ] { + assert!(!runs_go_test(command), "{command}"); + } + } + #[test] fn go_test_counts_leaf_tests_of_the_last_run() { let sandbox = scratch("gotest"); diff --git a/crates/b10x/src/plan.rs b/crates/b10x/src/plan.rs index d438e4d..1da847a 100644 --- a/crates/b10x/src/plan.rs +++ b/crates/b10x/src/plan.rs @@ -270,15 +270,9 @@ pub struct Context<'a> { pub fn plan(context: &Context<'_>, inventory: &Inventory) -> Plan { let catalog = context.catalog; let present = present(catalog, inventory); - let selected: BTreeSet = context.selection.clone().unwrap_or_else(|| { - present - .iter() - .filter(|id| { - context.only || catalog.product(id).is_some_and(|product| !product.optional) - }) - .cloned() - .collect() - }); + // With no explicit selection, what is installed stays installed, optional products included: + // leaving one out would uninstall it (connectors, trial 5). + let selected: BTreeSet = context.selection.clone().unwrap_or_else(|| present.clone()); let offers = catalog .products .iter() @@ -1360,6 +1354,32 @@ mod tests { plan(&context, inventory) } + #[test] + fn an_installed_optional_product_stays_when_nothing_is_selected() { + let mut claude = HostState::default(); + claude.marketplaces.push(market()); + claude.plugins.push(installed("b10x", "b10x", "user", None)); + claude + .plugins + .push(installed("connectors", "b10x", "user", None)); + let inventory = Inventory { + claude: Some(claude), + ..Inventory::default() + }; + let plan = run(&inventory, None, &[Host::Claude]); + let commands = commands(&plan); + assert!( + !commands + .iter() + .any(|c| c.contains("uninstall connectors@b10x")), + "{commands:#?}" + ); + assert!(plan + .offers + .iter() + .any(|offer| offer.id == "connectors" && offer.selected)); + } + #[test] fn upgrade_installs_the_replacement_of_a_legacy_plugin_it_removes() { let mut claude = HostState::default(); diff --git a/plugins/aep/.claude-plugin/plugin.json b/plugins/aep/.claude-plugin/plugin.json index 3896e9b..2a0a725 100644 --- a/plugins/aep/.claude-plugin/plugin.json +++ b/plugins/aep/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "name": "aep", "displayName": "AEP", "description": "Plan governed work in the AEP artifact store and deliver it in reviewed waves: decomposition, plan critique, reverse engineering, story scoping, implementation and adversarial review.", - "version": "0.14.9", + "version": "0.14.10", "author": { "name": "Beyond10x" }, diff --git a/plugins/aep/.codex-plugin/plugin.json b/plugins/aep/.codex-plugin/plugin.json index e792fb7..d5ee62d 100644 --- a/plugins/aep/.codex-plugin/plugin.json +++ b/plugins/aep/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "aep", - "version": "0.14.9", + "version": "0.14.10", "description": "Plan governed work in the AEP artifact store and deliver it in reviewed waves.", "author": { "name": "Beyond10x" diff --git a/plugins/b10x/.claude-plugin/plugin.json b/plugins/b10x/.claude-plugin/plugin.json index f775634..8c84369 100644 --- a/plugins/b10x/.claude-plugin/plugin.json +++ b/plugins/b10x/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "name": "b10x", "displayName": "Beyond10x", "description": "Set up, upgrade and check the Beyond10x plugins and binaries, route work to them, and create portable plugins.", - "version": "0.14.9", + "version": "0.14.10", "author": { "name": "Beyond10x" }, diff --git a/plugins/b10x/.codex-plugin/plugin.json b/plugins/b10x/.codex-plugin/plugin.json index 31516e7..f9d3c03 100644 --- a/plugins/b10x/.codex-plugin/plugin.json +++ b/plugins/b10x/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "b10x", - "version": "0.14.9", + "version": "0.14.10", "description": "Set up, upgrade and check the Beyond10x plugins and binaries, route work to them, and create portable plugins.", "author": { "name": "Beyond10x" diff --git a/plugins/connectors/.claude-plugin/plugin.json b/plugins/connectors/.claude-plugin/plugin.json index 8209f32..3d642bc 100644 --- a/plugins/connectors/.claude-plugin/plugin.json +++ b/plugins/connectors/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "connectors", - "version": "0.14.9", + "version": "0.14.10", "description": "Set up, inspect, and invoke governed integrations through the connectors CLI.", "author": { "name": "Beyond10x" }, "license": "Apache-2.0", diff --git a/plugins/connectors/.codex-plugin/plugin.json b/plugins/connectors/.codex-plugin/plugin.json index 2ab5478..42a89d4 100644 --- a/plugins/connectors/.codex-plugin/plugin.json +++ b/plugins/connectors/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "connectors", - "version": "0.14.9", + "version": "0.14.10", "description": "Set up, inspect, and invoke governed integrations through the connectors CLI.", "author": { "name": "Beyond10x" }, "license": "Apache-2.0", diff --git a/plugins/ess/.claude-plugin/plugin.json b/plugins/ess/.claude-plugin/plugin.json index 8bf3d94..303346b 100644 --- a/plugins/ess/.claude-plugin/plugin.json +++ b/plugins/ess/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "name": "ess", "displayName": "ESS", "description": "Write, retrofit, validate and project Executable System Specifications, and hold implementations to them with conformance suites.", - "version": "0.14.9", + "version": "0.14.10", "author": { "name": "Beyond10x" }, diff --git a/plugins/ess/.codex-plugin/plugin.json b/plugins/ess/.codex-plugin/plugin.json index 1bdf152..12fa315 100644 --- a/plugins/ess/.codex-plugin/plugin.json +++ b/plugins/ess/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "ess", - "version": "0.14.9", + "version": "0.14.10", "description": "Write, retrofit, validate and project Executable System Specifications, and hold implementations to them with conformance suites.", "author": { "name": "Beyond10x" diff --git a/plugins/ess/skills/specifying/SKILL.md b/plugins/ess/skills/specifying/SKILL.md index e05979e..6e760c3 100644 --- a/plugins/ess/skills/specifying/SKILL.md +++ b/plugins/ess/skills/specifying/SKILL.md @@ -191,13 +191,18 @@ edit generated output around it. Use the projection command, never a handwritten parallel generator: ```console -ess generate --path --kind schema --out -ess generate project openapi --path --out +ess generate --path --kind --out ``` -Each writes into its own subdirectory of `--out`: `--out generated` gives `generated/schema/…` and -`generated/openapi/…`. Pass the parent, not `generated/schema`, or the files land in -`generated/schema/schema/`. +`--kind` is `schema` (JSON Schema), `openapi`, `asyncapi`, `docs` (Markdown pages), `site` (the +same pages as HTML) or `docs-ir`. Give each kind its own `--out`, side by side (`out/schema`, +`out/openapi`, …): an output directory inside another is refused. `schema`, `openapi`, `asyncapi` +and `docs` add a folder named after the kind below `--out` (`out/schema/schema/…`); `site` writes +into `--out` itself. Type libraries come from `ess generate types --target rust|go|typescript`. + +**OpenAPI and AsyncAPI need a component** that owns the domain and says how it is reached +(`components.yaml`, `reached_by`). Without one, `openapi` writes `0 artifact(s)` and prints no +refusal: that is a missing declaration, not a clean result. The same typed IR must produce the same ordered files and bytes. Compare a regenerated temporary tree with the committed tree before replacing anything. A stale committed file is drift; a file no diff --git a/plugins/ess/skills/testing-conformance/SKILL.md b/plugins/ess/skills/testing-conformance/SKILL.md index 9e0345d..c7a1c2f 100644 --- a/plugins/ess/skills/testing-conformance/SKILL.md +++ b/plugins/ess/skills/testing-conformance/SKILL.md @@ -216,7 +216,11 @@ ess verify conform synthesize --path --target go --out ess verify conform synthesize --path --target typescript --out ``` -The package's `README.md` lists the methods and the wiring test. A method you cannot answer returns +Generate the Go package **inside the implementation's own module** (`--out /conformance`) so +its test can import it; a copy outside the module has no `go.mod` to import from. Numbers in a +payload you return compare equal only as `float64`: an `int64` or a `json.Number` fails although +the type check accepts it (beyond10x/ess#101, until fixed). The package's `README.md` lists the +methods and the wiring test. A method you cannot answer returns `ErrUnsupported` (a skip), never a made-up result. Keep a suite for a domain nobody has implemented; run it once something answers it. diff --git a/plugins/worktree/.claude-plugin/plugin.json b/plugins/worktree/.claude-plugin/plugin.json index fd066d7..384a0cd 100644 --- a/plugins/worktree/.claude-plugin/plugin.json +++ b/plugins/worktree/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "name": "worktree", "displayName": "Worktree", "description": "Create, lease, finish, audit and safely clean isolated Git worktrees through the worktree CLI.", - "version": "0.14.9", + "version": "0.14.10", "author": { "name": "Beyond10x" }, diff --git a/plugins/worktree/.codex-plugin/plugin.json b/plugins/worktree/.codex-plugin/plugin.json index 36d70f0..30b8fc9 100644 --- a/plugins/worktree/.codex-plugin/plugin.json +++ b/plugins/worktree/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "worktree", - "version": "0.14.9", + "version": "0.14.10", "description": "Create, lease, finish, audit and safely clean isolated Git worktrees through the worktree CLI.", "author": { "name": "Beyond10x" diff --git a/trials/aep-backlog/trial.yaml b/trials/aep-backlog/trial.yaml index f139d14..adf5883 100644 --- a/trials/aep-backlog/trial.yaml +++ b/trials/aep-backlog/trial.yaml @@ -4,6 +4,8 @@ prompt: | Our backlog for this little tool is just `TODO.md`. I'd like to plan it properly with AEP from now on: turn what's there into epics and stories, leave out what's already done, and make sure the result validates. Don't delete `TODO.md`. + Nobody will answer questions during this run: where you would ask, decide, write down what you + decided and why, and go on. When you're done, list the skills and agents you used, quote any instruction text that confused you, and paste the last validation output verbatim. diff --git a/trials/baseline.json b/trials/baseline.json index 0967ef4..14cd741 100644 --- a/trials/baseline.json +++ b/trials/baseline.json @@ -1 +1,60 @@ -{} +{ + "aep-backlog": { + "tool_calls": 313 + }, + "ess-full-package": { + "tool_calls": 68, + "validate": "valid", + "synthesis": { + "scenarios": 13, + "refusals": 0 + }, + "unmapped": 0, + "outputs": [ + "asyncapi", + "conformance", + "docs", + "implementation", + "openapi", + "schema", + "site", + "types-go", + "types-rust" + ], + "go_test": { + "passed": 13, + "failed": 0, + "skipped": 0 + } + }, + "ess-new": { + "tool_calls": 12, + "validate": "valid", + "unmapped": 7 + }, + "ess-pipeline": { + "tool_calls": 19, + "validate": "valid", + "synthesis": { + "scenarios": 12, + "refusals": 0 + }, + "outputs": [ + "conformance", + "docs", + "schema" + ] + }, + "ess-retrofit": { + "tool_calls": 29, + "validate": "valid", + "synthesis": { + "scenarios": 16, + "refusals": 2 + }, + "unmapped": 7 + }, + "worktree-onboarding": { + "tool_calls": 19 + } +} diff --git a/trials/upgrade-seeded/trial.yaml b/trials/upgrade-seeded/trial.yaml index a09d481..5776ed8 100644 --- a/trials/upgrade-seeded/trial.yaml +++ b/trials/upgrade-seeded/trial.yaml @@ -2,7 +2,8 @@ name: upgrade-seeded kind: upgrade prompt: | I installed the Beyond10x plugins a while ago and haven't touched them since. Bring everything up - to date, and tell me what changed. + to date, and tell me what changed. Nobody will answer questions during this run: where you would + ask, decide, write down what you decided and why, and go on. When you're done, list the skills and agents you used, quote any instruction text that confused you, and paste the last command output verbatim. diff --git a/trials/worktree-onboarding/trial.yaml b/trials/worktree-onboarding/trial.yaml index c47ade6..25e0930 100644 --- a/trials/worktree-onboarding/trial.yaml +++ b/trials/worktree-onboarding/trial.yaml @@ -4,6 +4,8 @@ prompt: | Set up worktree for this repository so I can work on changes in isolated checkouts. Then use it to fix the typo in `README.md` in an isolated checkout on a branch, commit it there, and tell me where the checkout is. Leave this checkout as it is. + Nobody will answer questions during this run: where you would ask, decide, write down what you + decided and why, and go on. When you're done, list the skills and agents you used, quote any instruction text that confused you, and paste the last command output verbatim.