From 48b0d95a2df5e974571a7e28e1b0ceb0ac121246 Mon Sep 17 00:00:00 2001 From: abrichr Date: Fri, 28 Aug 2026 14:42:34 -0400 Subject: [PATCH] feat: bind published benchmark figures to their upstream artifacts Every claim guard in this organization checks that a page contains an attribution string. None of them compares a published number to the measurement it came from, so a figure can drift from its artifact while every guard stays green. That is how success_count 20 stayed on a public surface for five weeks after the measured artifact said 19. benchmark-claims.json pins four openadapt-flow artifacts at commit aee0941, vendors their bytes under benchmark-claims/upstream/, and binds each figure printed on README.md and profile/README.md to a JSON pointer into one of them. scripts/check_benchmark_claims.py: - re-hashes every vendored snapshot, so the snapshot cannot be edited to bless a wrong figure; - renders each registered figure out of the artifact and requires the exact published text; - cross-checks the derived comparison artifact against the two measured results files; - sweeps both surfaces for figure-shaped tokens and fails on any token no entry claims, so a new figure cannot land unregistered and a stale exemption cannot rot in place; - with --online, re-fetches each pinned path from raw.githubusercontent and compares digests. An unreachable GitHub warns, since an outage is not evidence of drift; a digest mismatch exits non-zero. profile/README.md publishes 20/20 for the OpenEMR compiled arm where the pinned artifact records 19 of 20. This repository does not own that prose, so the mismatch is recorded as a dated drift with a named owner and a 2026-09-25 expiry. The checker prints it on every run, fails by default, and fails unconditionally once the date passes. The offline half runs on every pull request. The online half runs on the daily schedule. Both are standard library only. Co-Authored-By: Claude Opus 5 --- .github/workflows/profile-consistency.yml | 8 + benchmark-claims.json | 272 + .../comparison_artifact/comparison.json | 126 + .../benchmark/effect_e2e/results.json | 6247 +++++++++++++++++ .../benchmark/openemr/results.json | 692 ++ .../benchmark/results.json | 2012 ++++++ scripts/check_benchmark_claims.py | 768 ++ tests/test_benchmark_claims.py | 550 ++ 8 files changed, 10675 insertions(+) create mode 100644 benchmark-claims.json create mode 100644 benchmark-claims/upstream/openadapt-flow/aee094193b232f472f991be6fa9b33c3c4b3f9be/benchmark/comparison_artifact/comparison.json create mode 100644 benchmark-claims/upstream/openadapt-flow/aee094193b232f472f991be6fa9b33c3c4b3f9be/benchmark/effect_e2e/results.json create mode 100644 benchmark-claims/upstream/openadapt-flow/aee094193b232f472f991be6fa9b33c3c4b3f9be/benchmark/openemr/results.json create mode 100644 benchmark-claims/upstream/openadapt-flow/aee094193b232f472f991be6fa9b33c3c4b3f9be/benchmark/results.json create mode 100644 scripts/check_benchmark_claims.py create mode 100644 tests/test_benchmark_claims.py diff --git a/.github/workflows/profile-consistency.yml b/.github/workflows/profile-consistency.yml index a81bfdb..6f6bd20 100644 --- a/.github/workflows/profile-consistency.yml +++ b/.github/workflows/profile-consistency.yml @@ -71,6 +71,14 @@ jobs: --previous-admissions "$RUNNER_TEMP/previous-production-lifecycle-admissions.json" - name: Validate product truth and links run: python3 scripts/check_profile.py + - name: Bind every published benchmark figure to its artifact + # --allow-recorded-drift keeps a dated, reviewed drift record from + # blocking every unrelated change. Each record is still printed, still + # names an owner, and still fails this step once its expiry date passes. + run: python3 scripts/check_benchmark_claims.py --allow-recorded-drift + - name: Re-fetch the pinned benchmark artifacts + if: github.event_name == 'schedule' + run: python3 scripts/check_benchmark_claims.py --online --allow-recorded-drift report-expired-admission: needs: [reject-lifecycle-app, validate-profile] diff --git a/benchmark-claims.json b/benchmark-claims.json new file mode 100644 index 0000000..6080d48 --- /dev/null +++ b/benchmark-claims.json @@ -0,0 +1,272 @@ +{ + "$comment": "Binds every benchmark figure printed on an org front page to the upstream artifact that measured it. Every other claim guard in this org checks that a page contains an attribution string; none of them checks the number. openadapt-web published success_count 20 for the OpenEMR compiled arm for five weeks while the measured artifact said 19, and no guard noticed. scripts/check_benchmark_claims.py re-hashes the vendored snapshots under benchmark-claims/upstream/, renders each registered figure straight out of those bytes, and refuses any figure-shaped token on a registered surface that no entry here claims. So a new figure cannot land on the org profile without a deliberate edit to this file. profile/README.md currently publishes 20/20 for the OpenEMR compiled arm and the pinned artifact says 19 of 20; that one figure is recorded below as an open drift with an expiry date, and the checker fails once the date passes. This repository does not own that prose, so this guard reports the mismatch instead of editing it.", + "schema_version": "openadapt.benchmark-claim-binding/v1", + "upstream": { + "repository": "OpenAdaptAI/openadapt-flow", + "commit": "aee094193b232f472f991be6fa9b33c3c4b3f9be", + "sources": [ + { + "id": "comparison", + "path": "benchmark/comparison_artifact/comparison.json", + "git_blob": "93e8f6ccb050f3b672432229334db9ac2ac3bfcf", + "sha256": "sha256:c5eb58d234ecbd5b03a687745fb6afeb53d0965459b0e42779942441b19dbbd6", + "size_bytes": 4977, + "snapshot": "benchmark-claims/upstream/openadapt-flow/aee094193b232f472f991be6fa9b33c3c4b3f9be/benchmark/comparison_artifact/comparison.json" + }, + { + "id": "openemr", + "path": "benchmark/openemr/results.json", + "git_blob": "368c2857a0e9ffe8fdd82e2ee1903714d48971b0", + "sha256": "sha256:30de6dbb43ad0db4fd8b55737d765d06747b896ffae04522b6957d567dcfd89c", + "size_bytes": 21616, + "snapshot": "benchmark-claims/upstream/openadapt-flow/aee094193b232f472f991be6fa9b33c3c4b3f9be/benchmark/openemr/results.json" + }, + { + "id": "mockmed", + "path": "benchmark/results.json", + "git_blob": "e80539233b1a96a0e96bd6e2704a9065c1a2b299", + "sha256": "sha256:daf6f840f441ad32c1c88716ed41e2c1dec766a2f03762adda00f30f7588d8cc", + "size_bytes": 50086, + "snapshot": "benchmark-claims/upstream/openadapt-flow/aee094193b232f472f991be6fa9b33c3c4b3f9be/benchmark/results.json" + }, + { + "id": "effect-e2e", + "path": "benchmark/effect_e2e/results.json", + "git_blob": "3d01d92a96bbd82588d96a485c4fa6276bb21f69", + "sha256": "sha256:35b577939f63f02580df283e478d6a729fd86bc4a4bb200bdcb0778ed4b773d0", + "size_bytes": 231969, + "snapshot": "benchmark-claims/upstream/openadapt-flow/aee094193b232f472f991be6fa9b33c3c4b3f9be/benchmark/effect_e2e/results.json" + } + ] + }, + "surfaces": [ + "README.md", + "profile/README.md" + ], + "claims": [ + { + "id": "openemr-compiled-success-ratio", + "surface": "profile/README.md", + "context": "compiled replay went **20/20 at 39.2s p50 with zero model calls**", + "text": "20/20", + "renderer": "ratio", + "decimals": null, + "source": "openemr", + "pointers": [ + "/arms/compiled/success_count", + "/arms/compiled/n" + ], + "status": "drift_open", + "drift": { + "upstream_text": "19/20", + "recorded_on": "2026-08-28", + "expires_on": "2026-09-25", + "owner": "the agent that owns org front-page corrections", + "reason": "profile/README.md publishes 20/20 for the OpenEMR compiled arm. The pinned artifact records success_count 19 of n 20, because the saved-row oracle adjudication of 2026-07-28 rejected compiled run 20: the note stayed in the unsaved entry form. The published figure is wrong and must become 19/20. This guard adds the binding and does not edit the prose." + } + }, + { + "id": "openemr-compiled-p50", + "surface": "profile/README.md", + "context": "compiled replay went **20/20 at 39.2s p50 with zero model calls**", + "text": "39.2s", + "renderer": "seconds", + "decimals": 1, + "source": "openemr", + "pointers": [ + "/arms/compiled/wall_s_p50" + ], + "status": "bound", + "drift": null + }, + { + "id": "openemr-agent-success-ratio", + "surface": "profile/README.md", + "context": "the agent went 10/10 at 70.4s p50 at about $0.55 of model charge per run.", + "text": "10/10", + "renderer": "ratio", + "decimals": null, + "source": "openemr", + "pointers": [ + "/arms/agent/success_count", + "/arms/agent/n" + ], + "status": "bound", + "drift": null + }, + { + "id": "openemr-agent-p50", + "surface": "profile/README.md", + "context": "the agent went 10/10 at 70.4s p50 at about $0.55 of model charge per run.", + "text": "70.4s", + "renderer": "seconds", + "decimals": 1, + "source": "openemr", + "pointers": [ + "/arms/agent/wall_s_p50" + ], + "status": "bound", + "drift": null + }, + { + "id": "openemr-agent-cost-per-run", + "surface": "profile/README.md", + "context": "the agent went 10/10 at 70.4s p50 at about $0.55 of model charge per run.", + "text": "$0.55", + "renderer": "usd", + "decimals": 2, + "source": "openemr", + "pointers": [ + "/arms/agent/cost_usd_per_run" + ], + "status": "bound", + "drift": null + }, + { + "id": "mockmed-compiled-success-ratio", + "surface": "profile/README.md", + "context": "run (100/100 compiled, 20/20 agent)", + "text": "100/100", + "renderer": "ratio", + "decimals": null, + "source": "mockmed", + "pointers": [ + "/arms/compiled/success_count", + "/arms/compiled/n" + ], + "status": "bound", + "drift": null + }, + { + "id": "mockmed-agent-success-ratio", + "surface": "profile/README.md", + "context": "run (100/100 compiled, 20/20 agent)", + "text": "20/20", + "renderer": "ratio", + "decimals": null, + "source": "mockmed", + "pointers": [ + "/arms/agent/success_count", + "/arms/agent/n" + ], + "status": "bound", + "drift": null + }, + { + "id": "mockmed-compiled-p50", + "surface": "profile/README.md", + "context": "success rate — 4.9s p50 with zero model calls versus 37.5s p50 for the agent.", + "text": "4.9s", + "renderer": "seconds", + "decimals": 1, + "source": "mockmed", + "pointers": [ + "/arms/compiled/wall_s_p50" + ], + "status": "bound", + "drift": null + }, + { + "id": "mockmed-agent-p50", + "surface": "profile/README.md", + "context": "success rate — 4.9s p50 with zero model calls versus 37.5s p50 for the agent.", + "text": "37.5s", + "renderer": "seconds", + "decimals": 1, + "source": "mockmed", + "pointers": [ + "/arms/agent/wall_s_p50" + ], + "status": "bound", + "drift": null + }, + { + "id": "effect-screen-undetected-rate", + "surface": "profile/README.md", + "context": "oracle silently accepted **75.0%** of the wrong effects that actually occurred (54 of 90 runs).", + "text": "75.0%", + "renderer": "percent", + "decimals": 1, + "source": "effect-e2e", + "pointers": [ + "/metrics/per_arm/screen/undetected_wrong_rate" + ], + "status": "bound", + "drift": null + }, + { + "id": "effect-screen-silent-wrong-count", + "surface": "profile/README.md", + "context": "oracle silently accepted **75.0%** of the wrong effects that actually occurred (54 of 90 runs).", + "text": "54 of 90", + "renderer": "count_of", + "decimals": null, + "source": "effect-e2e", + "pointers": [ + "/metrics/per_arm/screen/silent_wrong_count", + "/metrics/per_arm/screen/n_runs" + ], + "status": "bound", + "drift": null + }, + { + "id": "effect-rest-undetected-rate", + "surface": "profile/README.md", + "context": "cut that to **12.5%** (9 of 90).", + "text": "12.5%", + "renderer": "percent", + "decimals": 1, + "source": "effect-e2e", + "pointers": [ + "/metrics/per_arm/effect_rest/undetected_wrong_rate" + ], + "status": "bound", + "drift": null + }, + { + "id": "effect-rest-silent-wrong-count", + "surface": "profile/README.md", + "context": "cut that to **12.5%** (9 of 90).", + "text": "9 of 90", + "renderer": "count_of", + "decimals": null, + "source": "effect-e2e", + "pointers": [ + "/metrics/per_arm/effect_rest/silent_wrong_count", + "/metrics/per_arm/effect_rest/n_runs" + ], + "status": "bound", + "drift": null + }, + { + "id": "effect-full-silent-wrong-count", + "surface": "profile/README.md", + "context": "A complete read path over every mutable surface reaches 0 of 90,", + "text": "0 of 90", + "renderer": "count_of", + "decimals": null, + "source": "effect-e2e", + "pointers": [ + "/metrics/per_arm/effect_full/silent_wrong_count", + "/metrics/per_arm/effect_full/n_runs" + ], + "status": "bound", + "drift": null + }, + { + "id": "effect-full-undetected-rate", + "surface": "profile/README.md", + "context": "one out-of-band oracle — not the 0%. All nine", + "text": "0%", + "renderer": "percent", + "decimals": 0, + "source": "effect-e2e", + "pointers": [ + "/metrics/per_arm/effect_full/undetected_wrong_rate" + ], + "status": "bound", + "drift": null + } + ], + "non_benchmark_figures": [] +} diff --git a/benchmark-claims/upstream/openadapt-flow/aee094193b232f472f991be6fa9b33c3c4b3f9be/benchmark/comparison_artifact/comparison.json b/benchmark-claims/upstream/openadapt-flow/aee094193b232f472f991be6fa9b33c3c4b3f9be/benchmark/comparison_artifact/comparison.json new file mode 100644 index 0000000..93e8f6c --- /dev/null +++ b/benchmark-claims/upstream/openadapt-flow/aee094193b232f472f991be6fa9b33c3c4b3f9be/benchmark/comparison_artifact/comparison.json @@ -0,0 +1,126 @@ +{ + "generated_by": "benchmark.comparison_artifact.generate", + "model_calls_compiled": 0, + "network_calls": 0, + "provenance": "Every figure is read from the source results.json files; the only prose-sourced figure is the one-time human demonstration cost, quoted from benchmark/openemr/BENCHMARK.md + benchmark/BENCHMARK.md (prose).", + "source_files": { + "openemr": "benchmark/openemr/results.json", + "mockmed": "benchmark/results.json" + }, + "demonstration_cost": { + "text": "about a minute of human demonstration", + "source": "benchmark/openemr/BENCHMARK.md + benchmark/BENCHMARK.md (prose)" + }, + "benchmarks": { + "openemr": { + "label": "OpenEMR public demo \u2014 a real third-party EMR", + "kind": "field", + "reproducible": false, + "source_file": "benchmark/openemr/results.json", + "task": "OpenEMR public demo: log in, search the demo patient, open the chart, scroll the dashboard to the Messages card, open Patient Messages, add a parameterized note, save", + "model": "claude-sonnet-5", + "workflow_steps": 18, + "arms": { + "compiled": { + "n": 20, + "success_count": 19, + "success_rate": 0.95, + "wall_s_p50": 39.170917561999886, + "wall_s_p95": 41.04867044315233, + "cost_usd_per_run": 0.0, + "cost_usd_total": 0.0 + }, + "agent": { + "n": 10, + "success_count": 10, + "success_rate": 1.0, + "wall_s_p50": 70.44607112499943, + "wall_s_p95": 82.59687604810188, + "cost_usd_per_run": 0.552150765, + "cost_usd_total": 5.52150765 + } + }, + "extras": { + "target": "https://demo.openemr.io/openemr/index.php", + "cost_caps_usd": { + "per_run": 1.5, + "total": 8.0 + }, + "pricing_note": "list price; an introductory $2/$10 rate applies through 2026-08-31", + "compiled_self_flag": { + "i": 19, + "success": false, + "first_failure_step": "step_017" + }, + "success_contract": { + "kind": "screen_saved_message_row", + "note_min_contiguous_squashed_chars": 16, + "requires": [ + "Patient Messages Content and Status headers", + "note fragment in the Content column below the headers", + "an aligned New status for the same saved row" + ], + "rejects": "note text visible only in the unsaved entry form" + }, + "oracle_adjudication": { + "corrected_at": "2026-07-28", + "contract": "The final screenshot must show a contiguous note fragment of at least 16 squashed characters inside a saved Patient Messages Content row aligned with its New status. Note text in the unsaved entry form is not saved-row evidence.", + "retained_final_path": "benchmark/openemr/finals/compiled_019.png", + "retained_final_sha256": "8c504ba15bab9cdca8b5987dd1d1ab7b0ba7ae77f67fac5e93ba8481492ae18f", + "old_behavior": "The legacy verifier accepted the requested note anywhere in full-frame OCR and returned success for compiled run 20.", + "new_behavior": "The saved-row verifier rejects compiled run 20 because the note remains in the open textarea and the Save as new message button remains visible; no saved row contains the note.", + "changed_runs": [ + "compiled:19" + ], + "denominators_changed": false, + "latency_aggregates_changed": false + } + } + }, + "mockmed": { + "label": "MockMed \u2014 the bundled demo clinic (CI-reproducible anchor)", + "kind": "reproducible", + "reproducible": true, + "source_file": "benchmark/results.json", + "task": "MockMed triage: login, open first referral, create a Triage encounter, enter the note, save", + "model": "claude-sonnet-5", + "workflow_steps": null, + "arms": { + "compiled": { + "n": 100, + "success_count": 100, + "success_rate": 1.0, + "wall_s_p50": 4.9309526044999075, + "wall_s_p95": 5.055971199399937, + "cost_usd_per_run": 0.0, + "cost_usd_total": 0.0 + }, + "agent": { + "n": 20, + "success_count": 20, + "success_rate": 1.0, + "wall_s_p50": 37.492187666500286, + "wall_s_p95": 43.44962514519993, + "cost_usd_per_run": 0.27155505, + "cost_usd_total": 5.431101 + } + }, + "extras": { + "pricing_note": "list price; an introductory $2/$10 rate applies through 2026-08-31", + "drift_theme": { + "compiled_wall_s": 9.720975041999736, + "compiled_heal_count": 8, + "compiled_success": true, + "agent_wall_s": 87.36445195899978, + "agent_cost_usd": 0.631923, + "agent_actions": 23, + "agent_success": true + } + } + } + }, + "derived": { + "openemr_speedup_p50": 1.8, + "mockmed_speedup_p50": 7.6 + } +} diff --git a/benchmark-claims/upstream/openadapt-flow/aee094193b232f472f991be6fa9b33c3c4b3f9be/benchmark/effect_e2e/results.json b/benchmark-claims/upstream/openadapt-flow/aee094193b232f472f991be6fa9b33c3c4b3f9be/benchmark/effect_e2e/results.json new file mode 100644 index 0000000..3d01d92 --- /dev/null +++ b/benchmark-claims/upstream/openadapt-flow/aee094193b232f472f991be6fa9b33c3c4b3f9be/benchmark/effect_e2e/results.json @@ -0,0 +1,6247 @@ +{ + "instrument": "silent-wrong-effect rate, end-to-end through the replayer", + "generated_at": "2026-07-21T06:40:00.293507+00:00", + "platform": "macOS-15.7.3-arm64-arm-64bit", + "model_calls": 0, + "n_per_scenario": 9, + "arms": [ + "screen", + "effect_rest", + "effect_full" + ], + "scenarios": [ + "ok", + "no_persist", + "partial", + "duplicate", + "wrong_record", + "stale", + "collateral_unaudited", + "optimistic", + "session", + "timeout" + ], + "independence": { + "write_path": "POST /api/encounter via the replayer's ApiActuator (governed actuation) -> on-disk SQLite mutation", + "verifier_readback_path": "GET /api/records (screen arm: GET /api/ui/last-save) -- a different HTTP verb/endpoint/connection than the write; the replayer's effect_verifier CONFIRMS/REFUTES/HALTs", + "ground_truth_path": "direct read-only SQLite file connection over every persisted system-of-record table (discovered dynamically from sqlite_master, not a hardcoded pair; the UI-echo banner excluded), with an independent classifier and its OWN per-table delta audit (NOT the effect kit's audit_table_deltas) -- bypasses the service; the write's success flag never reaches it" + }, + "headline": { + "screen_silent_wrong": 54, + "screen_n_runs": 90, + "effect_rest_silent_wrong": 9, + "effect_rest_n_runs": 90, + "effect_full_silent_wrong": 0, + "effect_full_n_runs": 90, + "slips_through_effect_rest": [ + "collateral_unaudited" + ] + }, + "metrics": { + "per_arm": { + "screen": { + "n_runs": 90, + "n_wrong_effect": 72, + "n_correct_effect": 18, + "silent_wrong_count": 54, + "silent_wrong_action_rate": 0.6, + "undetected_wrong_rate": 0.75, + "caught_count": 18, + "false_abort_count": 9, + "false_abort_rate": 0.5, + "silent_wrong_scenarios": [ + "collateral_unaudited", + "duplicate", + "no_persist", + "partial", + "stale", + "wrong_record" + ], + "per_scenario": { + "ok": { + "n": 9, + "gt_correct": true, + "gt_fault": "correct", + "halted": false, + "silent_wrong_rate": 0.0 + }, + "no_persist": { + "n": 9, + "gt_correct": false, + "gt_fault": "absent", + "halted": false, + "silent_wrong_rate": 1.0 + }, + "partial": { + "n": 9, + "gt_correct": false, + "gt_fault": "partial", + "halted": false, + "silent_wrong_rate": 1.0 + }, + "duplicate": { + "n": 9, + "gt_correct": false, + "gt_fault": "duplicate", + "halted": false, + "silent_wrong_rate": 1.0 + }, + "wrong_record": { + "n": 9, + "gt_correct": false, + "gt_fault": "wrong_record", + "halted": false, + "silent_wrong_rate": 1.0 + }, + "stale": { + "n": 9, + "gt_correct": false, + "gt_fault": "collateral_loss", + "halted": false, + "silent_wrong_rate": 1.0 + }, + "collateral_unaudited": { + "n": 9, + "gt_correct": false, + "gt_fault": "collateral_write", + "halted": false, + "silent_wrong_rate": 1.0 + }, + "optimistic": { + "n": 9, + "gt_correct": false, + "gt_fault": "absent", + "halted": true, + "silent_wrong_rate": 0.0 + }, + "session": { + "n": 9, + "gt_correct": false, + "gt_fault": "absent", + "halted": true, + "silent_wrong_rate": 0.0 + }, + "timeout": { + "n": 9, + "gt_correct": true, + "gt_fault": "correct", + "halted": true, + "silent_wrong_rate": 0.0 + } + } + }, + "effect_rest": { + "n_runs": 90, + "n_wrong_effect": 72, + "n_correct_effect": 18, + "silent_wrong_count": 9, + "silent_wrong_action_rate": 0.1, + "undetected_wrong_rate": 0.125, + "caught_count": 63, + "false_abort_count": 9, + "false_abort_rate": 0.5, + "silent_wrong_scenarios": [ + "collateral_unaudited" + ], + "per_scenario": { + "ok": { + "n": 9, + "gt_correct": true, + "gt_fault": "correct", + "halted": false, + "silent_wrong_rate": 0.0 + }, + "no_persist": { + "n": 9, + "gt_correct": false, + "gt_fault": "absent", + "halted": true, + "silent_wrong_rate": 0.0 + }, + "partial": { + "n": 9, + "gt_correct": false, + "gt_fault": "partial", + "halted": true, + "silent_wrong_rate": 0.0 + }, + "duplicate": { + "n": 9, + "gt_correct": false, + "gt_fault": "duplicate", + "halted": true, + "silent_wrong_rate": 0.0 + }, + "wrong_record": { + "n": 9, + "gt_correct": false, + "gt_fault": "wrong_record", + "halted": true, + "silent_wrong_rate": 0.0 + }, + "stale": { + "n": 9, + "gt_correct": false, + "gt_fault": "collateral_loss", + "halted": true, + "silent_wrong_rate": 0.0 + }, + "collateral_unaudited": { + "n": 9, + "gt_correct": false, + "gt_fault": "collateral_write", + "halted": false, + "silent_wrong_rate": 1.0 + }, + "optimistic": { + "n": 9, + "gt_correct": false, + "gt_fault": "absent", + "halted": true, + "silent_wrong_rate": 0.0 + }, + "session": { + "n": 9, + "gt_correct": false, + "gt_fault": "absent", + "halted": true, + "silent_wrong_rate": 0.0 + }, + "timeout": { + "n": 9, + "gt_correct": true, + "gt_fault": "correct", + "halted": true, + "silent_wrong_rate": 0.0 + } + } + }, + "effect_full": { + "n_runs": 90, + "n_wrong_effect": 72, + "n_correct_effect": 18, + "silent_wrong_count": 0, + "silent_wrong_action_rate": 0.0, + "undetected_wrong_rate": 0.0, + "caught_count": 72, + "false_abort_count": 9, + "false_abort_rate": 0.5, + "silent_wrong_scenarios": [], + "per_scenario": { + "ok": { + "n": 9, + "gt_correct": true, + "gt_fault": "correct", + "halted": false, + "silent_wrong_rate": 0.0 + }, + "no_persist": { + "n": 9, + "gt_correct": false, + "gt_fault": "absent", + "halted": true, + "silent_wrong_rate": 0.0 + }, + "partial": { + "n": 9, + "gt_correct": false, + "gt_fault": "partial", + "halted": true, + "silent_wrong_rate": 0.0 + }, + "duplicate": { + "n": 9, + "gt_correct": false, + "gt_fault": "duplicate", + "halted": true, + "silent_wrong_rate": 0.0 + }, + "wrong_record": { + "n": 9, + "gt_correct": false, + "gt_fault": "wrong_record", + "halted": true, + "silent_wrong_rate": 0.0 + }, + "stale": { + "n": 9, + "gt_correct": false, + "gt_fault": "collateral_loss", + "halted": true, + "silent_wrong_rate": 0.0 + }, + "collateral_unaudited": { + "n": 9, + "gt_correct": false, + "gt_fault": "collateral_write", + "halted": true, + "silent_wrong_rate": 0.0 + }, + "optimistic": { + "n": 9, + "gt_correct": false, + "gt_fault": "absent", + "halted": true, + "silent_wrong_rate": 0.0 + }, + "session": { + "n": 9, + "gt_correct": false, + "gt_fault": "absent", + "halted": true, + "silent_wrong_rate": 0.0 + }, + "timeout": { + "n": 9, + "gt_correct": true, + "gt_fault": "correct", + "halted": true, + "silent_wrong_rate": 0.0 + } + } + } + } + }, + "runs": [ + { + "arm": "screen", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 0, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "screen", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 1, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "screen", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 2, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "screen", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 3, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "screen", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 4, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "screen", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 5, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "screen", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 6, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "screen", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 7, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "screen", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 8, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "screen", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 0, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 1, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 2, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 3, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 4, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 5, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 6, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 7, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 8, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 0, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 1, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 2, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 3, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 4, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 5, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 6, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 7, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 8, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 0, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 1, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 2, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 3, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 4, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 5, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 6, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 7, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 8, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 0, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 1, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 2, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 3, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 4, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 5, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 6, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 7, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 8, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 0, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 1, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 2, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 3, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 4, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 5, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 6, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 7, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 8, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 0, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 1, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 2, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 3, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 4, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 5, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 6, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 7, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 8, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 0, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 1, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 2, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 3, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 4, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 5, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 6, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 7, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 8, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 0, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 1, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 2, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 3, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 4, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 5, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 6, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 7, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 8, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 0, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 1, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 2, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 3, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 4, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 5, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 6, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 7, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "screen", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 8, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 0, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "effect_rest", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 1, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "effect_rest", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 2, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "effect_rest", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 3, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "effect_rest", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 4, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "effect_rest", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 5, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "effect_rest", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 6, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "effect_rest", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 7, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "effect_rest", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 8, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "effect_rest", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 0, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 1, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 2, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 3, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 4, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 5, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 6, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 7, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 8, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 0, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): field_equals refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] field_equals: field 'note' is '', expected 'End-to-end SWER benchmark triage note' (partial save -- the row persisted but this field was dropped or differs) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 1, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): field_equals refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] field_equals: field 'note' is '', expected 'End-to-end SWER benchmark triage note' (partial save -- the row persisted but this field was dropped or differs) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 2, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): field_equals refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] field_equals: field 'note' is '', expected 'End-to-end SWER benchmark triage note' (partial save -- the row persisted but this field was dropped or differs) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 3, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): field_equals refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] field_equals: field 'note' is '', expected 'End-to-end SWER benchmark triage note' (partial save -- the row persisted but this field was dropped or differs) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 4, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): field_equals refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] field_equals: field 'note' is '', expected 'End-to-end SWER benchmark triage note' (partial save -- the row persisted but this field was dropped or differs) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 5, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): field_equals refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] field_equals: field 'note' is '', expected 'End-to-end SWER benchmark triage note' (partial save -- the row persisted but this field was dropped or differs) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 6, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): field_equals refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] field_equals: field 'note' is '', expected 'End-to-end SWER benchmark triage note' (partial save -- the row persisted but this field was dropped or differs) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 7, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): field_equals refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] field_equals: field 'note' is '', expected 'End-to-end SWER benchmark triage note' (partial save -- the row persisted but this field was dropped or differs) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 8, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): field_equals refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] field_equals: field 'note' is '', expected 'End-to-end SWER benchmark triage note' (partial save -- the row persisted but this field was dropped or differs) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 0, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 2 records match the target selector but exactly 1 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 1, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 2 records match the target selector but exactly 1 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 2, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 2 records match the target selector but exactly 1 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 3, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 2 records match the target selector but exactly 1 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 4, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 2 records match the target selector but exactly 1 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 5, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 2 records match the target selector but exactly 1 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 6, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 2 records match the target selector but exactly 1 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 7, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 2 records match the target selector but exactly 1 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 8, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 2 records match the target selector but exactly 1 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 0, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 1, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 2, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 3, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 4, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 5, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 6, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 7, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 8, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 0, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 1 pre-existing record(s) vanished -- collateral loss (stale / concurrent last-write-wins overwrote another actor's row) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 1, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 1 pre-existing record(s) vanished -- collateral loss (stale / concurrent last-write-wins overwrote another actor's row) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 2, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 1 pre-existing record(s) vanished -- collateral loss (stale / concurrent last-write-wins overwrote another actor's row) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 3, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 1 pre-existing record(s) vanished -- collateral loss (stale / concurrent last-write-wins overwrote another actor's row) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 4, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 1 pre-existing record(s) vanished -- collateral loss (stale / concurrent last-write-wins overwrote another actor's row) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 5, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 1 pre-existing record(s) vanished -- collateral loss (stale / concurrent last-write-wins overwrote another actor's row) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 6, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 1 pre-existing record(s) vanished -- collateral loss (stale / concurrent last-write-wins overwrote another actor's row) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 7, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 1 pre-existing record(s) vanished -- collateral loss (stale / concurrent last-write-wins overwrote another actor's row) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 8, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the rest system of record and could not be reconciled (escalated) \u2014 [rest] record_written: 1 pre-existing record(s) vanished -- collateral loss (stale / concurrent last-write-wins overwrote another actor's row) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 0, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 1, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 2, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 3, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 4, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 5, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 6, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 7, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 8, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": true, + "caught": false, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 0, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 1, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 2, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 3, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 4, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 5, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 6, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 7, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 8, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 0, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 1, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 2, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 3, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 4, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 5, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 6, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 7, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 8, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 0, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 1, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 2, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 3, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 4, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 5, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 6, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 7, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "effect_rest", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 8, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 0, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "effect_full", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 1, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "effect_full", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 2, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "effect_full", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 3, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "effect_full", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 4, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "effect_full", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 5, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "effect_full", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 6, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "effect_full", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 7, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "effect_full", + "scenario": "ok", + "blurb": "control: a clean accepted write", + "i": 8, + "reported_success": true, + "halted": false, + "actuation": "api", + "effect_verified": true, + "error": null, + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": false, + "clean_success": true + }, + { + "arm": "effect_full", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 0, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 1, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 2, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 3, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 4, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 5, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 6, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 7, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "no_persist", + "blurb": "2xx banner but nothing persisted (phantom write)", + "i": 8, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 0, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): field_equals refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] field_equals: field 'note' is '', expected 'End-to-end SWER benchmark triage note' (partial save -- the row persisted but this field was dropped or differs) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 1, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): field_equals refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] field_equals: field 'note' is '', expected 'End-to-end SWER benchmark triage note' (partial save -- the row persisted but this field was dropped or differs) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 2, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): field_equals refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] field_equals: field 'note' is '', expected 'End-to-end SWER benchmark triage note' (partial save -- the row persisted but this field was dropped or differs) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 3, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): field_equals refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] field_equals: field 'note' is '', expected 'End-to-end SWER benchmark triage note' (partial save -- the row persisted but this field was dropped or differs) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 4, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): field_equals refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] field_equals: field 'note' is '', expected 'End-to-end SWER benchmark triage note' (partial save -- the row persisted but this field was dropped or differs) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 5, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): field_equals refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] field_equals: field 'note' is '', expected 'End-to-end SWER benchmark triage note' (partial save -- the row persisted but this field was dropped or differs) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 6, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): field_equals refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] field_equals: field 'note' is '', expected 'End-to-end SWER benchmark triage note' (partial save -- the row persisted but this field was dropped or differs) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 7, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): field_equals refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] field_equals: field 'note' is '', expected 'End-to-end SWER benchmark triage note' (partial save -- the row persisted but this field was dropped or differs) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "partial", + "blurb": "row persisted but the note field was dropped", + "i": 8, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): field_equals refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] field_equals: field 'note' is '', expected 'End-to-end SWER benchmark triage note' (partial save -- the row persisted but this field was dropped or differs) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "partial", + "gt_detail": "target encounter persisted with a wrong/dropped note field", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 0, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 2 records match the target selector but exactly 1 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 1, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 2 records match the target selector but exactly 1 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 2, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 2 records match the target selector but exactly 1 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 3, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 2 records match the target selector but exactly 1 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 4, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 2 records match the target selector but exactly 1 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 5, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 2 records match the target selector but exactly 1 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 6, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 2 records match the target selector but exactly 1 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 7, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 2 records match the target selector but exactly 1 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "duplicate", + "blurb": "double-delivered write: the row landed twice", + "i": 8, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 2 records match the target selector but exactly 1 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "duplicate", + "gt_detail": "2 target encounters persisted; exactly one intended", + "table_deltas": { + "billing": 0, + "encounters": 2 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 0, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 1, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 2, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 3, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 4, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 5, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 6, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 7, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "wrong_record", + "blurb": "the write landed on the wrong patient (p2)", + "i": 8, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 0 records match the target selector, expected 1 (missing / phantom / rejected write -- the screen may show success but nothing landed) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "wrong_record", + "gt_detail": "no target encounter; the write landed on the wrong record (patient 'p2')", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 0, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 1 pre-existing record(s) vanished -- collateral loss (stale / concurrent last-write-wins overwrote another actor's row) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 1, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 1 pre-existing record(s) vanished -- collateral loss (stale / concurrent last-write-wins overwrote another actor's row) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 2, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 1 pre-existing record(s) vanished -- collateral loss (stale / concurrent last-write-wins overwrote another actor's row) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 3, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 1 pre-existing record(s) vanished -- collateral loss (stale / concurrent last-write-wins overwrote another actor's row) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 4, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 1 pre-existing record(s) vanished -- collateral loss (stale / concurrent last-write-wins overwrote another actor's row) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 5, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 1 pre-existing record(s) vanished -- collateral loss (stale / concurrent last-write-wins overwrote another actor's row) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 6, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 1 pre-existing record(s) vanished -- collateral loss (stale / concurrent last-write-wins overwrote another actor's row) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 7, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 1 pre-existing record(s) vanished -- collateral loss (stale / concurrent last-write-wins overwrote another actor's row) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "stale", + "blurb": "last-write-wins destroyed a concurrent actor's row", + "i": 8, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 1 pre-existing record(s) vanished -- collateral loss (stale / concurrent last-write-wins overwrote another actor's row) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_loss", + "gt_detail": "1 pre-existing encounter row(s) destroyed (lost update)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 0, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 1 records match the target selector but exactly 0 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 1, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 1 records match the target selector but exactly 0 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 2, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 1 records match the target selector but exactly 0 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 3, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 1 records match the target selector but exactly 0 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 4, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 1 records match the target selector but exactly 0 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 5, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 1 records match the target selector but exactly 0 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 6, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 1 records match the target selector but exactly 0 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 7, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 1 records match the target selector but exactly 0 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "collateral_unaudited", + "blurb": "target row correct BUT a stray row hit an unaudited (billing) surface", + "i": 8, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "System-of-record effect verification HALTED step 'save' (save encounter): record_written refuted against the sql system of record and could not be reconciled (escalated) \u2014 [sql] record_written: 1 records match the target selector but exactly 0 was expected (duplicate / double-delivered write -- not at-most-once) -- no compensator available for an irreversible refuted effect -- durably halt and escalate \u2014 run aborted", + "gt_correct": false, + "gt_fault": "collateral_write", + "gt_detail": "target encounter is correct, but a collateral row was written to a surface the encounters record oracle does not read (billing delta +1)", + "table_deltas": { + "billing": 1, + "encounters": 1 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 0, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 1, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 2, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 3, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 4, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 5, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 6, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 7, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "optimistic", + "blurb": "server rejected (409) after the optimistic UI", + "i": 8, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=optimistic returned 409 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 0, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 1, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 2, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 3, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 4, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 5, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 6, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 7, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "session", + "blurb": "session expired (401): nothing persisted", + "i": 8, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): POST /api/encounter?fault=session returned 401 (not success) -- the write was attempted; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": false, + "gt_fault": "absent", + "gt_detail": "no target encounter persisted (phantom write)", + "table_deltas": { + "billing": 0, + "encounters": 0 + }, + "silent_wrong": false, + "caught": true, + "false_abort": false, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 0, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 1, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 2, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 3, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 4, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 5, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 6, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 7, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + }, + { + "arm": "effect_full", + "scenario": "timeout", + "blurb": "row committed, then the client timed out (unknown)", + "i": 8, + "reported_success": false, + "halted": true, + "actuation": "api", + "effect_verified": false, + "error": "API actuation HALTED step 'save' (save encounter): request sent but timed out awaiting the response (ReadTimeout) -- the write may have landed; HALT (never double-write via the GUI) -- run aborted", + "gt_correct": true, + "gt_fault": "correct", + "gt_detail": "exactly one correct target encounter", + "table_deltas": { + "billing": 0, + "encounters": 1 + }, + "silent_wrong": false, + "caught": false, + "false_abort": true, + "clean_success": false + } + ] +} diff --git a/benchmark-claims/upstream/openadapt-flow/aee094193b232f472f991be6fa9b33c3c4b3f9be/benchmark/openemr/results.json b/benchmark-claims/upstream/openadapt-flow/aee094193b232f472f991be6fa9b33c3c4b3f9be/benchmark/openemr/results.json new file mode 100644 index 0000000..368c285 --- /dev/null +++ b/benchmark-claims/upstream/openadapt-flow/aee094193b232f472f991be6fa9b33c3c4b3f9be/benchmark/openemr/results.json @@ -0,0 +1,692 @@ +{ + "agent_arm_note": null, + "cost_caps_usd": { + "per_run": 1.5, + "total": 8.0 + }, + "generated_at": "2026-07-08T22:46:39.730720+00:00", + "flow": { + "flow_version": "0.1.0", + "flow_version_qualifier": "pre-v0.2.0 development source checkout", + "exact_runtime_commit": null, + "exact_runtime_commit_recorded": false, + "flow_version_note": "Measured from a source checkout declaring version = \"0.1.0\", not a published wheel. The exact runtime HEAD was not retained. Refreshing these numbers means re-running the benchmark, not editing this field." + }, + "artifact_provenance": { + "first_committed_in": "e98975640428fdc8037430cdca81bcb68db54220", + "parent_before_artifact_commit": "099eac0759440a86eceef38530797e8d5b765a15", + "note": "These SHAs identify when the retained rows entered repository history; neither is attributed as the exact runtime HEAD." + }, + "oracle_adjudication": { + "corrected_at": "2026-07-28", + "contract": "The final screenshot must show a contiguous note fragment of at least 16 squashed characters inside a saved Patient Messages Content row aligned with its New status. Note text in the unsaved entry form is not saved-row evidence.", + "retained_final_path": "benchmark/openemr/finals/compiled_019.png", + "retained_final_sha256": "8c504ba15bab9cdca8b5987dd1d1ab7b0ba7ae77f67fac5e93ba8481492ae18f", + "old_behavior": "The legacy verifier accepted the requested note anywhere in full-frame OCR and returned success for compiled run 20.", + "new_behavior": "The saved-row verifier rejects compiled run 20 because the note remains in the open textarea and the Save as new message button remains visible; no saved row contains the note.", + "changed_runs": [ + "compiled:19" + ], + "denominators_changed": false, + "latency_aggregates_changed": false + }, + "task": "OpenEMR public demo: log in, search the demo patient, open the chart, scroll the dashboard to the Messages card, open Patient Messages, add a parameterized note, save", + "success_contract": { + "kind": "screen_saved_message_row", + "note_min_contiguous_squashed_chars": 16, + "requires": [ + "Patient Messages Content and Status headers", + "note fragment in the Content column below the headers", + "an aligned New status for the same saved row" + ], + "rejects": "note text visible only in the unsaved entry form" + }, + "target": "https://demo.openemr.io/openemr/index.php", + "workflow_steps": 18, + "model": "claude-sonnet-5", + "computer_tool": "computer_20251124", + "beta_header": "computer-use-2025-11-24", + "agent_max_actions": 40, + "pricing_usd_per_mtok": { + "input": 3.0, + "output": 15.0, + "cache_write": 3.75, + "cache_read": 0.3, + "note": "list price; an introductory $2/$10 rate applies through 2026-08-31" + }, + "platform": "macOS-15.7.3-arm64-arm-64bit", + "arms": { + "compiled": { + "n": 20, + "success_count": 19, + "success_rate": 0.95, + "wall_s_p50": 39.170917561999886, + "wall_s_p95": 41.04867044315233, + "wall_s_mean": 39.51734407485037, + "actions_mean": 18.0, + "input_tokens_total": 0, + "output_tokens_total": 0, + "cache_creation_input_tokens_total": 0, + "cache_read_input_tokens_total": 0, + "cost_usd_per_run": 0.0, + "cost_usd_total": 0.0 + }, + "agent": { + "n": 10, + "success_count": 10, + "success_rate": 1.0, + "wall_s_p50": 70.44607112499943, + "wall_s_p95": 82.59687604810188, + "wall_s_mean": 72.01063286680036, + "actions_mean": 23.8, + "input_tokens_total": 496, + "output_tokens_total": 27272, + "cache_creation_input_tokens_total": 1317803, + "cache_read_input_tokens_total": 563928, + "cost_usd_per_run": 0.552150765, + "cost_usd_total": 5.52150765 + } + }, + "runs": { + "compiled": [ + { + "arm": "compiled", + "wall_s": 40.32277429199894, + "success": true, + "replayer_success": true, + "heal_count": 1, + "actions": 18, + "first_failure": null, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 0.0, + "error": null, + "matched_ratio": 1.0, + "longest_run": 31, + "i": 0, + "note": "Renal panel ordered ahead of the next quarterly visit." + }, + { + "arm": "compiled", + "wall_s": 40.1873638750003, + "success": true, + "replayer_success": true, + "heal_count": 1, + "actions": 18, + "first_failure": null, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 0.0, + "error": null, + "matched_ratio": 1.0, + "longest_run": 33, + "i": 1, + "note": "Walking program begun, thirty minutes on weekday mornings." + }, + { + "arm": "compiled", + "wall_s": 39.49069095800223, + "success": true, + "replayer_success": true, + "heal_count": 1, + "actions": 18, + "first_failure": null, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 0.0, + "error": null, + "matched_ratio": 1.0, + "longest_run": 27, + "i": 2, + "note": "Pharmacy contact changed to the downtown branch office." + }, + { + "arm": "compiled", + "wall_s": 38.458301624999876, + "success": true, + "replayer_success": true, + "heal_count": 1, + "actions": 18, + "first_failure": null, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 0.0, + "error": null, + "matched_ratio": 1.0, + "longest_run": 32, + "i": 3, + "note": "Dizziness resolved after the evening dose adjustment." + }, + { + "arm": "compiled", + "wall_s": 39.02663908300019, + "success": true, + "replayer_success": true, + "heal_count": 1, + "actions": 18, + "first_failure": null, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 0.0, + "error": null, + "matched_ratio": 1.0, + "longest_run": 30, + "i": 4, + "note": "Low-sodium meal plan handout given and explained." + }, + { + "arm": "compiled", + "wall_s": 39.230697625000175, + "success": true, + "replayer_success": true, + "heal_count": 1, + "actions": 18, + "first_failure": null, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 0.0, + "error": null, + "matched_ratio": 1.0, + "longest_run": 32, + "i": 5, + "note": "Home blood-pressure log shows stable readings all month." + }, + { + "arm": "compiled", + "wall_s": 38.62730450000163, + "success": true, + "replayer_success": true, + "heal_count": 1, + "actions": 18, + "first_failure": null, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 0.0, + "error": null, + "matched_ratio": 1.0, + "longest_run": 31, + "i": 6, + "note": "Flu shot declined today; revisit the topic in autumn." + }, + { + "arm": "compiled", + "wall_s": 39.18036958299854, + "success": true, + "replayer_success": true, + "heal_count": 1, + "actions": 18, + "first_failure": null, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 0.0, + "error": null, + "matched_ratio": 1.0, + "longest_run": 32, + "i": 7, + "note": "Occupational therapy referral faxed this afternoon." + }, + { + "arm": "compiled", + "wall_s": 39.65608079100275, + "success": true, + "replayer_success": true, + "heal_count": 1, + "actions": 18, + "first_failure": null, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 0.0, + "error": null, + "matched_ratio": 1.0, + "longest_run": 34, + "i": 8, + "note": "Lab results within limits; no follow-up required." + }, + { + "arm": "compiled", + "wall_s": 38.909656625000935, + "success": true, + "replayer_success": true, + "heal_count": 1, + "actions": 18, + "first_failure": null, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 0.0, + "error": null, + "matched_ratio": 0.5532, + "longest_run": 18, + "i": 9, + "note": "Refill approved for ninety days of current medication." + }, + { + "arm": "compiled", + "wall_s": 38.622817999999825, + "success": true, + "replayer_success": true, + "heal_count": 1, + "actions": 18, + "first_failure": null, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 0.0, + "error": null, + "matched_ratio": 1.0, + "longest_run": 31, + "i": 10, + "note": "Cardiology consult summary scanned into the chart." + }, + { + "arm": "compiled", + "wall_s": 38.95875354099917, + "success": true, + "replayer_success": true, + "heal_count": 1, + "actions": 18, + "first_failure": null, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 0.0, + "error": null, + "matched_ratio": 1.0, + "longest_run": 31, + "i": 11, + "note": "Smoking cessation resources mailed to home address." + }, + { + "arm": "compiled", + "wall_s": 40.94058916600261, + "success": true, + "replayer_success": true, + "heal_count": 1, + "actions": 18, + "first_failure": null, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 0.0, + "error": null, + "matched_ratio": 1.0, + "longest_run": 31, + "i": 12, + "note": "Podiatry exam scheduled for early next month." + }, + { + "arm": "compiled", + "wall_s": 39.161465541001235, + "success": true, + "replayer_success": true, + "heal_count": 1, + "actions": 18, + "first_failure": null, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 0.0, + "error": null, + "matched_ratio": 1.0, + "longest_run": 28, + "i": 13, + "note": "Weight down four pounds since the spring checkup." + }, + { + "arm": "compiled", + "wall_s": 38.89333633299975, + "success": true, + "replayer_success": true, + "heal_count": 1, + "actions": 18, + "first_failure": null, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 0.0, + "error": null, + "matched_ratio": 1.0, + "longest_run": 29, + "i": 14, + "note": "Eye exam reminder sent through the patient portal." + }, + { + "arm": "compiled", + "wall_s": 39.06889295900328, + "success": true, + "replayer_success": true, + "heal_count": 1, + "actions": 18, + "first_failure": null, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 0.0, + "error": null, + "matched_ratio": 1.0, + "longest_run": 27, + "i": 15, + "note": "Colonoscopy screening due date moved to October." + }, + { + "arm": "compiled", + "wall_s": 39.76785374999963, + "success": true, + "replayer_success": true, + "heal_count": 1, + "actions": 18, + "first_failure": null, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 0.0, + "error": null, + "matched_ratio": 0.8333, + "longest_run": 23, + "i": 16, + "note": "Physical therapy completed; discharge notes attached." + }, + { + "arm": "compiled", + "wall_s": 39.63622920800117, + "success": true, + "replayer_success": true, + "heal_count": 1, + "actions": 18, + "first_failure": null, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 0.0, + "error": null, + "matched_ratio": 1.0, + "longest_run": 31, + "i": 17, + "note": "Sleep study questionnaire returned and scored." + }, + { + "arm": "compiled", + "wall_s": 39.10484933299813, + "success": true, + "replayer_success": true, + "heal_count": 1, + "actions": 18, + "first_failure": null, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 0.0, + "error": null, + "matched_ratio": 1.0, + "longest_run": 32, + "i": 18, + "note": "Hearing aid battery supply reordered by front desk." + }, + { + "arm": "compiled", + "wall_s": 43.10221470899705, + "success": false, + "replayer_success": false, + "heal_count": 1, + "actions": 18, + "first_failure": { + "step": "step_017", + "error": "Postconditions failed for step 'step_017' (click 'Save as new messag'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(0, 0, 1280, 800); text_present 'CalendarFinderFlowRecallsMessagesPatientFeesModulesInventoryProceduresAdminReportsMiscellaneousPopups' \u2014 run aborted" + }, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 0.0, + "error": null, + "matched_ratio": 0.0, + "longest_run": 0, + "legacy_screen_success": true, + "oracle_correction": "The legacy whole-frame OCR verifier matched the note in the unsaved entry textarea. The retained final frame has no saved row for this note.", + "i": 19, + "note": "Orthopedic pillow suggestion discussed for neck pain." + } + ], + "agent": [ + { + "arm": "agent", + "wall_s": 67.23859954100044, + "success": true, + "actions": 23, + "api_calls": 24, + "input_tokens": 48, + "output_tokens": 2630, + "cache_creation_input_tokens": 127719, + "cache_read_input_tokens": 52866, + "cost_usd": 0.53440005, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "matched_ratio": 1.0, + "longest_run": 31, + "i": 0, + "note": "Tetanus booster administered in the left deltoid." + }, + { + "arm": "agent", + "wall_s": 78.28020625000136, + "success": true, + "actions": 25, + "api_calls": 26, + "input_tokens": 52, + "output_tokens": 3055, + "cache_creation_input_tokens": 141609, + "cache_read_input_tokens": 59086, + "cost_usd": 0.5947405500000001, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "matched_ratio": 1.0, + "longest_run": 32, + "i": 1, + "note": "Grip strength improved at this week's therapy session." + }, + { + "arm": "agent", + "wall_s": 73.20673066699965, + "success": true, + "actions": 24, + "api_calls": 25, + "input_tokens": 50, + "output_tokens": 2798, + "cache_creation_input_tokens": 133398, + "cache_read_input_tokens": 57018, + "cost_usd": 0.5594679, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "matched_ratio": 1.0, + "longest_run": 30, + "i": 2, + "note": "Insurance card copied and coverage verified by phone." + }, + { + "arm": "agent", + "wall_s": 65.5979254169979, + "success": true, + "actions": 23, + "api_calls": 24, + "input_tokens": 48, + "output_tokens": 2446, + "cache_creation_input_tokens": 124817, + "cache_read_input_tokens": 54936, + "cost_usd": 0.52137855, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "matched_ratio": 1.0, + "longest_run": 27, + "i": 3, + "note": "Allergy list updated to include seasonal pollen." + }, + { + "arm": "agent", + "wall_s": 63.98518375000276, + "success": true, + "actions": 23, + "api_calls": 24, + "input_tokens": 48, + "output_tokens": 2671, + "cache_creation_input_tokens": 126200, + "cache_read_input_tokens": 54938, + "cost_usd": 0.5299404, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "matched_ratio": 1.0, + "longest_run": 27, + "i": 4, + "note": "Crutches returned; gait steady without assistance." + }, + { + "arm": "agent", + "wall_s": 69.03068095900016, + "success": true, + "actions": 24, + "api_calls": 25, + "input_tokens": 50, + "output_tokens": 2855, + "cache_creation_input_tokens": 134493, + "cache_read_input_tokens": 57028, + "cost_usd": 0.56443215, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "matched_ratio": 1.0, + "longest_run": 28, + "i": 5, + "note": "Dermatology biopsy site healing cleanly, no drainage." + }, + { + "arm": "agent", + "wall_s": 71.74900062499728, + "success": true, + "actions": 24, + "api_calls": 25, + "input_tokens": 50, + "output_tokens": 2713, + "cache_creation_input_tokens": 132636, + "cache_read_input_tokens": 57012, + "cost_usd": 0.5553336, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "matched_ratio": 1.0, + "longest_run": 32, + "i": 6, + "note": "Travel vaccine consult booked before the June trip." + }, + { + "arm": "agent", + "wall_s": 69.14314162500159, + "success": true, + "actions": 23, + "api_calls": 24, + "input_tokens": 48, + "output_tokens": 2601, + "cache_creation_input_tokens": 125737, + "cache_read_input_tokens": 54940, + "cost_usd": 0.5271547499999999, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "matched_ratio": 1.0, + "longest_run": 30, + "i": 7, + "note": "Glucometer readings uploaded from the home device." + }, + { + "arm": "agent", + "wall_s": 75.74616304200026, + "success": true, + "actions": 24, + "api_calls": 25, + "input_tokens": 50, + "output_tokens": 2702, + "cache_creation_input_tokens": 132264, + "cache_read_input_tokens": 57020, + "cost_usd": 0.5537759999999999, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "matched_ratio": 1.0, + "longest_run": 24, + "i": 8, + "note": "Knee brace fitted and sizing documented in chart." + }, + { + "arm": "agent", + "wall_s": 86.1286967920023, + "success": true, + "actions": 25, + "api_calls": 26, + "input_tokens": 52, + "output_tokens": 2801, + "cache_creation_input_tokens": 138930, + "cache_read_input_tokens": 59084, + "cost_usd": 0.5808836999999999, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "matched_ratio": 1.0, + "longest_run": 29, + "i": 9, + "note": "Caregiver contact number added to emergency file." + } + ] + }, + "pace_s": 30.0 +} diff --git a/benchmark-claims/upstream/openadapt-flow/aee094193b232f472f991be6fa9b33c3c4b3f9be/benchmark/results.json b/benchmark-claims/upstream/openadapt-flow/aee094193b232f472f991be6fa9b33c3c4b3f9be/benchmark/results.json new file mode 100644 index 0000000..e805392 --- /dev/null +++ b/benchmark-claims/upstream/openadapt-flow/aee094193b232f472f991be6fa9b33c3c4b3f9be/benchmark/results.json @@ -0,0 +1,2012 @@ +{ + "generated_at": "2026-07-08T15:46:40.963053+00:00", + "flow": { + "flow_version": "0.1.0", + "flow_version_qualifier": "pre-v0.2.0 development source checkout", + "exact_runtime_commit": null, + "exact_runtime_commit_recorded": false, + "flow_version_note": "Measured from a source checkout declaring version = \"0.1.0\", not a published wheel. The exact runtime HEAD was not retained. Refreshing these numbers means re-running the benchmark, not editing this field." + }, + "artifact_provenance": { + "first_committed_in": "b2eec0be7ddee1930bd45d447cee3afced8d1fd8", + "parent_before_artifact_commit": "45f5ba8a141d361420d903c0700aac7403a37d97", + "note": "These SHAs identify when the retained rows entered repository history; neither is attributed as the exact runtime HEAD." + }, + "task": "MockMed triage: login, open first referral, create a Triage encounter, enter the note, save", + "note_text": "Follow-up in 2 weeks; BP recheck.", + "model": "claude-sonnet-5", + "computer_tool": "computer_20251124", + "beta_header": "computer-use-2025-11-24", + "pricing_usd_per_mtok": { + "input": 3.0, + "output": 15.0, + "note": "list price; an introductory $2/$10 rate applies through 2026-08-31" + }, + "platform": "macOS-15.7.3-arm64-arm-64bit", + "arms": { + "compiled": { + "n": 100, + "success_count": 100, + "success_rate": 1.0, + "wall_s_p50": 4.9309526044999075, + "wall_s_p95": 5.055971199399937, + "wall_s_mean": 4.942865620329931, + "actions_mean": 11.0, + "input_tokens_total": 0, + "output_tokens_total": 0, + "cost_usd_per_run": 0.0, + "cost_usd_total": 0.0 + }, + "agent": { + "n": 20, + "success_count": 20, + "success_rate": 1.0, + "wall_s_p50": 37.492187666500286, + "wall_s_p95": 43.44962514519993, + "wall_s_mean": 37.861873618749904, + "actions_mean": 12.0, + "input_tokens_total": 1684942, + "output_tokens_total": 25085, + "cost_usd_per_run": 0.27155505, + "cost_usd_total": 5.431101 + } + }, + "drift_theme": { + "compiled": { + "arm": "compiled", + "wall_s": 9.720975041999736, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 8, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null + }, + "agent": { + "arm": "agent", + "wall_s": 87.36445195899978, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 23, + "api_calls": 24, + "input_tokens": 192086, + "output_tokens": 3711, + "cost_usd": 0.631923, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null + } + }, + "runs": { + "compiled": [ + { + "arm": "compiled", + "wall_s": 4.9552447079995545, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 0 + }, + { + "arm": "compiled", + "wall_s": 5.0017216249998455, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 1 + }, + { + "arm": "compiled", + "wall_s": 5.058394165999744, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 2 + }, + { + "arm": "compiled", + "wall_s": 4.991929374999927, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 3 + }, + { + "arm": "compiled", + "wall_s": 4.972108999999364, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 4 + }, + { + "arm": "compiled", + "wall_s": 5.059951416000331, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 5 + }, + { + "arm": "compiled", + "wall_s": 4.996505375000197, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 6 + }, + { + "arm": "compiled", + "wall_s": 4.905264083000475, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 7 + }, + { + "arm": "compiled", + "wall_s": 4.954853000000185, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 8 + }, + { + "arm": "compiled", + "wall_s": 4.9077381249999235, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 9 + }, + { + "arm": "compiled", + "wall_s": 4.902273208000224, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 10 + }, + { + "arm": "compiled", + "wall_s": 4.947772708999764, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 11 + }, + { + "arm": "compiled", + "wall_s": 4.903467666999859, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 12 + }, + { + "arm": "compiled", + "wall_s": 4.8973091249999925, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 13 + }, + { + "arm": "compiled", + "wall_s": 4.91245649999928, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 14 + }, + { + "arm": "compiled", + "wall_s": 4.897031165999579, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 15 + }, + { + "arm": "compiled", + "wall_s": 4.942904000000453, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 16 + }, + { + "arm": "compiled", + "wall_s": 4.90311841699986, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 17 + }, + { + "arm": "compiled", + "wall_s": 4.922759249999217, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 18 + }, + { + "arm": "compiled", + "wall_s": 5.055917665999914, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 19 + }, + { + "arm": "compiled", + "wall_s": 4.95709533399986, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 20 + }, + { + "arm": "compiled", + "wall_s": 4.925138874999902, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 21 + }, + { + "arm": "compiled", + "wall_s": 4.947614541999428, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 22 + }, + { + "arm": "compiled", + "wall_s": 4.904581124999822, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 23 + }, + { + "arm": "compiled", + "wall_s": 4.895782874999895, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 24 + }, + { + "arm": "compiled", + "wall_s": 4.937606624999717, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 25 + }, + { + "arm": "compiled", + "wall_s": 4.989733582999179, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 26 + }, + { + "arm": "compiled", + "wall_s": 4.9410122919998685, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 27 + }, + { + "arm": "compiled", + "wall_s": 5.014261958000134, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 28 + }, + { + "arm": "compiled", + "wall_s": 4.931548915999883, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 29 + }, + { + "arm": "compiled", + "wall_s": 4.913614125000095, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 30 + }, + { + "arm": "compiled", + "wall_s": 4.943551958999706, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 31 + }, + { + "arm": "compiled", + "wall_s": 5.0218980419995205, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 32 + }, + { + "arm": "compiled", + "wall_s": 4.923821291000422, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 33 + }, + { + "arm": "compiled", + "wall_s": 5.023044749999826, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 34 + }, + { + "arm": "compiled", + "wall_s": 4.909020125000097, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 35 + }, + { + "arm": "compiled", + "wall_s": 4.886381666000489, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 36 + }, + { + "arm": "compiled", + "wall_s": 5.05698833400038, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 37 + }, + { + "arm": "compiled", + "wall_s": 4.899967207999907, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 38 + }, + { + "arm": "compiled", + "wall_s": 4.913453250000202, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 39 + }, + { + "arm": "compiled", + "wall_s": 4.914081874999283, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 40 + }, + { + "arm": "compiled", + "wall_s": 4.951178291999895, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 41 + }, + { + "arm": "compiled", + "wall_s": 4.903782040999431, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 42 + }, + { + "arm": "compiled", + "wall_s": 4.904322832999242, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 43 + }, + { + "arm": "compiled", + "wall_s": 4.907327540999177, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 44 + }, + { + "arm": "compiled", + "wall_s": 4.920802375000676, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 45 + }, + { + "arm": "compiled", + "wall_s": 4.925570958000208, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 46 + }, + { + "arm": "compiled", + "wall_s": 4.978624041999865, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 47 + }, + { + "arm": "compiled", + "wall_s": 4.950948958000481, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 48 + }, + { + "arm": "compiled", + "wall_s": 4.931790208000166, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 49 + }, + { + "arm": "compiled", + "wall_s": 4.960389666000083, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 50 + }, + { + "arm": "compiled", + "wall_s": 4.949694582999655, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 51 + }, + { + "arm": "compiled", + "wall_s": 4.923188875000051, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 52 + }, + { + "arm": "compiled", + "wall_s": 4.9206881249992875, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 53 + }, + { + "arm": "compiled", + "wall_s": 4.896856875000594, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 54 + }, + { + "arm": "compiled", + "wall_s": 4.949743291999766, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 55 + }, + { + "arm": "compiled", + "wall_s": 4.9470230839997384, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 56 + }, + { + "arm": "compiled", + "wall_s": 5.112419915999453, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 57 + }, + { + "arm": "compiled", + "wall_s": 4.864904291999665, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 58 + }, + { + "arm": "compiled", + "wall_s": 4.955483458000344, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 59 + }, + { + "arm": "compiled", + "wall_s": 5.054122333000123, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 60 + }, + { + "arm": "compiled", + "wall_s": 4.940589707999607, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 61 + }, + { + "arm": "compiled", + "wall_s": 4.940294083000481, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 62 + }, + { + "arm": "compiled", + "wall_s": 4.992926707999686, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 63 + }, + { + "arm": "compiled", + "wall_s": 4.838405709000654, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 64 + }, + { + "arm": "compiled", + "wall_s": 4.977797958999872, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 65 + }, + { + "arm": "compiled", + "wall_s": 4.9258394579992455, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 66 + }, + { + "arm": "compiled", + "wall_s": 4.924930959000449, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 67 + }, + { + "arm": "compiled", + "wall_s": 4.981753791000301, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 68 + }, + { + "arm": "compiled", + "wall_s": 4.967452208000395, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 69 + }, + { + "arm": "compiled", + "wall_s": 4.974433458000021, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 70 + }, + { + "arm": "compiled", + "wall_s": 4.9256118339999375, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 71 + }, + { + "arm": "compiled", + "wall_s": 4.94952049999938, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 72 + }, + { + "arm": "compiled", + "wall_s": 4.9143084580000505, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 73 + }, + { + "arm": "compiled", + "wall_s": 4.999550542000179, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 74 + }, + { + "arm": "compiled", + "wall_s": 4.913648208000268, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 75 + }, + { + "arm": "compiled", + "wall_s": 4.93058208399998, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 76 + }, + { + "arm": "compiled", + "wall_s": 4.9221208750004735, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 77 + }, + { + "arm": "compiled", + "wall_s": 4.905563207999876, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 78 + }, + { + "arm": "compiled", + "wall_s": 4.944601959000465, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 79 + }, + { + "arm": "compiled", + "wall_s": 4.92488958299964, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 80 + }, + { + "arm": "compiled", + "wall_s": 4.915068166999845, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 81 + }, + { + "arm": "compiled", + "wall_s": 4.9063454580000325, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 82 + }, + { + "arm": "compiled", + "wall_s": 4.8955146250000325, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 83 + }, + { + "arm": "compiled", + "wall_s": 4.865054083000359, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 84 + }, + { + "arm": "compiled", + "wall_s": 4.891606207999757, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 85 + }, + { + "arm": "compiled", + "wall_s": 4.8825175410001975, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 86 + }, + { + "arm": "compiled", + "wall_s": 4.928078083999935, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 87 + }, + { + "arm": "compiled", + "wall_s": 4.920149333000154, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 88 + }, + { + "arm": "compiled", + "wall_s": 4.95750654099993, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 89 + }, + { + "arm": "compiled", + "wall_s": 5.009986749999371, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 90 + }, + { + "arm": "compiled", + "wall_s": 4.921199458000046, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 91 + }, + { + "arm": "compiled", + "wall_s": 4.9354450829996495, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 92 + }, + { + "arm": "compiled", + "wall_s": 4.9354730830000335, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 93 + }, + { + "arm": "compiled", + "wall_s": 4.917275415999939, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 94 + }, + { + "arm": "compiled", + "wall_s": 4.884874250000394, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 95 + }, + { + "arm": "compiled", + "wall_s": 4.974082665999958, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 96 + }, + { + "arm": "compiled", + "wall_s": 5.147877458999574, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 97 + }, + { + "arm": "compiled", + "wall_s": 4.824582332999853, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 98 + }, + { + "arm": "compiled", + "wall_s": 4.931323124999835, + "success": true, + "banner_found": true, + "note_found": true, + "replayer_success": true, + "heal_count": 0, + "actions": 11, + "api_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "error": null, + "i": 99 + } + ], + "agent": [ + { + "arm": "agent", + "wall_s": 40.45065283399981, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 12, + "api_calls": 13, + "input_tokens": 84369, + "output_tokens": 1267, + "cost_usd": 0.27211199999999997, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "i": 0 + }, + { + "arm": "agent", + "wall_s": 44.61182387499957, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 12, + "api_calls": 13, + "input_tokens": 84347, + "output_tokens": 1271, + "cost_usd": 0.272106, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "i": 1 + }, + { + "arm": "agent", + "wall_s": 38.29790462500023, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 12, + "api_calls": 13, + "input_tokens": 84208, + "output_tokens": 1259, + "cost_usd": 0.271509, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "i": 2 + }, + { + "arm": "agent", + "wall_s": 43.21879962500043, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 12, + "api_calls": 13, + "input_tokens": 84317, + "output_tokens": 1261, + "cost_usd": 0.27186600000000005, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "i": 3 + }, + { + "arm": "agent", + "wall_s": 32.94066704099987, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 12, + "api_calls": 13, + "input_tokens": 84162, + "output_tokens": 1259, + "cost_usd": 0.271371, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "i": 4 + }, + { + "arm": "agent", + "wall_s": 38.76979091599969, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 12, + "api_calls": 13, + "input_tokens": 84109, + "output_tokens": 1236, + "cost_usd": 0.270867, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "i": 5 + }, + { + "arm": "agent", + "wall_s": 43.062727291000556, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 12, + "api_calls": 13, + "input_tokens": 83976, + "output_tokens": 1185, + "cost_usd": 0.26970299999999997, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "i": 6 + }, + { + "arm": "agent", + "wall_s": 35.777499584, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 12, + "api_calls": 13, + "input_tokens": 84626, + "output_tokens": 1327, + "cost_usd": 0.27378300000000005, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "i": 7 + }, + { + "arm": "agent", + "wall_s": 41.026563124999484, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 12, + "api_calls": 13, + "input_tokens": 84233, + "output_tokens": 1278, + "cost_usd": 0.271869, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "i": 8 + }, + { + "arm": "agent", + "wall_s": 36.98442279199935, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 12, + "api_calls": 13, + "input_tokens": 84421, + "output_tokens": 1279, + "cost_usd": 0.272448, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "i": 9 + }, + { + "arm": "agent", + "wall_s": 34.22517041699939, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 12, + "api_calls": 13, + "input_tokens": 84319, + "output_tokens": 1259, + "cost_usd": 0.271842, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "i": 10 + }, + { + "arm": "agent", + "wall_s": 33.53040362499996, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 12, + "api_calls": 13, + "input_tokens": 84129, + "output_tokens": 1219, + "cost_usd": 0.27067199999999997, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "i": 11 + }, + { + "arm": "agent", + "wall_s": 34.15257279199977, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 12, + "api_calls": 13, + "input_tokens": 84124, + "output_tokens": 1251, + "cost_usd": 0.271137, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "i": 12 + }, + { + "arm": "agent", + "wall_s": 43.38845679099995, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 12, + "api_calls": 13, + "input_tokens": 84083, + "output_tokens": 1223, + "cost_usd": 0.270594, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "i": 13 + }, + { + "arm": "agent", + "wall_s": 35.37157445899993, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 12, + "api_calls": 13, + "input_tokens": 84423, + "output_tokens": 1257, + "cost_usd": 0.272124, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "i": 14 + }, + { + "arm": "agent", + "wall_s": 33.22209041699989, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 12, + "api_calls": 13, + "input_tokens": 84236, + "output_tokens": 1259, + "cost_usd": 0.27159300000000003, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "i": 15 + }, + { + "arm": "agent", + "wall_s": 37.64779133299999, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 12, + "api_calls": 13, + "input_tokens": 84211, + "output_tokens": 1279, + "cost_usd": 0.271818, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "i": 16 + }, + { + "arm": "agent", + "wall_s": 37.336584000000585, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 12, + "api_calls": 13, + "input_tokens": 84079, + "output_tokens": 1240, + "cost_usd": 0.270837, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "i": 17 + }, + { + "arm": "agent", + "wall_s": 34.55193816700012, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 12, + "api_calls": 13, + "input_tokens": 84174, + "output_tokens": 1250, + "cost_usd": 0.271272, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "i": 18 + }, + { + "arm": "agent", + "wall_s": 38.67003866599953, + "success": true, + "banner_found": true, + "note_found": true, + "actions": 12, + "api_calls": 13, + "input_tokens": 84396, + "output_tokens": 1226, + "cost_usd": 0.271578, + "stopped": "model_done", + "model_stop_reason": "end_turn", + "error": null, + "i": 19 + } + ] + } +} diff --git a/scripts/check_benchmark_claims.py b/scripts/check_benchmark_claims.py new file mode 100644 index 0000000..2a8b617 --- /dev/null +++ b/scripts/check_benchmark_claims.py @@ -0,0 +1,768 @@ +#!/usr/bin/env python3 +"""Bind every benchmark figure published on an org front page to its artifact. + +Other guards in this organization assert that a page CONTAINS an attribution +string. None of them compares a published NUMBER to the measurement it came +from, so a figure can drift from its artifact while every guard stays green. + +This checker closes that gap for `README.md` and `profile/README.md`: + +1. Each pinned upstream artifact is vendored under `benchmark-claims/upstream/` + and re-hashed on every run. A snapshot cannot be edited to bless a figure. +2. Every registered figure is rendered from the vendored artifact and must + equal the exact text published on the page. +3. The pages are swept for figure-shaped tokens. A token that no registry entry + claims is an error, so a new figure cannot land unregistered. Registering a + figure, or exempting a non-benchmark one, is a deliberate reviewable edit. +4. `--online` re-fetches each pinned path from raw.githubusercontent.com and + compares digests. An unreachable GitHub warns; a digest mismatch fails. + +Scope: this is a transcription-fidelity guard, not a validity check. A green run +certifies that the published figure matches the artifact, never that the +measurement behind the artifact is sound. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +import urllib.error +import urllib.request +from datetime import date, datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +REGISTRY_PATH = ROOT / "benchmark-claims.json" + +REGISTRY_SCHEMA = "openadapt.benchmark-claim-binding/v1" +RAW_GITHUB_ORIGIN = "https://raw.githubusercontent.com" +FETCH_TIMEOUT_SECONDS = 30 + +# The sweep patterns live in code, not in the registry, so that editing the +# registry can never weaken the sweep that guards it. +FIGURE_PATTERNS = ( + r"\d+\s*/\s*\d+", + r"\d+(?:\.\d+)?\s*%", + r"\$\d+(?:\.\d+)?", + r"\d+(?:\.\d+)?\s*s\b", + r"\b\d+\s+of\s+\d+\b", +) +FIGURE_RE = re.compile("|".join(f"(?:{pattern})" for pattern in FIGURE_PATTERNS)) + +SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$") +HEX40 = re.compile(r"^[0-9a-f]{40}$") +DATE = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}$") +IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{1,63}$") +REPOSITORY = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") + +RENDERERS = ("ratio", "count_of", "percent", "seconds", "usd", "integer") +POINTER_COUNTS = { + "ratio": 2, + "count_of": 2, + "percent": 1, + "seconds": 1, + "usd": 1, + "integer": 1, +} +DECIMAL_RENDERERS = {"percent", "seconds", "usd"} + +# A recorded drift is a dated exemption for a figure this repository publishes +# and does not own. It may never become an open-ended licence to be wrong. +MAX_DRIFT_DAYS = 60 + +REGISTRY_KEYS = { + "$comment", + "schema_version", + "upstream", + "surfaces", + "claims", + "non_benchmark_figures", +} +UPSTREAM_KEYS = {"repository", "commit", "sources"} +SOURCE_KEYS = {"id", "path", "git_blob", "sha256", "size_bytes", "snapshot"} +CLAIM_KEYS = { + "id", + "surface", + "context", + "text", + "renderer", + "decimals", + "source", + "pointers", + "status", + "drift", +} +DRIFT_KEYS = { + "upstream_text", + "recorded_on", + "expires_on", + "owner", + "reason", +} +EXEMPTION_KEYS = {"id", "surface", "context", "text", "reason"} + + +class RegistryError(Exception): + """The registry itself is malformed, so nothing can be trusted.""" + + +def normalize(text: str) -> str: + """Collapse Markdown line wrapping so a claim can span wrapped lines.""" + return " ".join(text.split()) + + +def require(condition: object, message: str) -> None: + if not condition: + raise RegistryError(message) + + +def require_relative(value: str, label: str) -> PurePosixPath: + path = PurePosixPath(value) + require( + not path.is_absolute() and ".." not in path.parts and value == str(path), + f"{label} is not a normalized relative path: {value!r}", + ) + return path + + +def load_registry(path: Path) -> dict[str, Any]: + """Parse and structurally validate the registry, rejecting unknown keys.""" + try: + registry = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise RegistryError(f"cannot read {path.name}: {exc}") from exc + + require(isinstance(registry, dict), "the registry is not a JSON object") + require( + set(registry) == REGISTRY_KEYS, + f"registry keys are {sorted(registry)}; expected {sorted(REGISTRY_KEYS)}", + ) + require( + registry["schema_version"] == REGISTRY_SCHEMA, + f"registry schema_version must be {REGISTRY_SCHEMA!r}", + ) + require( + isinstance(registry["$comment"], str) and registry["$comment"].strip(), + "the registry must carry a non-empty $comment", + ) + + upstream = registry["upstream"] + require(isinstance(upstream, dict), "upstream is not a JSON object") + require( + set(upstream) == UPSTREAM_KEYS, + f"upstream keys are {sorted(upstream)}; expected {sorted(UPSTREAM_KEYS)}", + ) + require( + isinstance(upstream["repository"], str) + and REPOSITORY.fullmatch(upstream["repository"]), + "upstream.repository must be OWNER/NAME", + ) + require( + isinstance(upstream["commit"], str) and HEX40.fullmatch(upstream["commit"]), + "upstream.commit must be an exact 40-character commit", + ) + + sources = upstream["sources"] + require( + isinstance(sources, list) and sources, + "upstream.sources must be a non-empty list", + ) + seen_ids: set[str] = set() + seen_paths: set[str] = set() + for source in sources: + require(isinstance(source, dict), "an upstream source is not a JSON object") + require( + set(source) == SOURCE_KEYS, + f"source keys are {sorted(source)}; expected {sorted(SOURCE_KEYS)}", + ) + source_id = source["id"] + require( + isinstance(source_id, str) and IDENTIFIER.fullmatch(source_id), + f"source id is not an identifier: {source_id!r}", + ) + require(source_id not in seen_ids, f"duplicate source id: {source_id}") + seen_ids.add(source_id) + require_relative(source["path"], f"source {source_id} path") + require( + source["path"] not in seen_paths, + f"duplicate upstream path: {source['path']}", + ) + seen_paths.add(source["path"]) + require( + isinstance(source["git_blob"], str) + and HEX40.fullmatch(source["git_blob"]), + f"source {source_id} git_blob must be a 40-character object name", + ) + require( + isinstance(source["sha256"], str) and SHA256.fullmatch(source["sha256"]), + f"source {source_id} sha256 must be sha256:<64 hex>", + ) + require( + isinstance(source["size_bytes"], int) and source["size_bytes"] > 0, + f"source {source_id} size_bytes must be a positive integer", + ) + snapshot = require_relative(source["snapshot"], f"source {source_id} snapshot") + require( + snapshot.parts[0] == "benchmark-claims", + f"source {source_id} snapshot must live under benchmark-claims/", + ) + + surfaces = registry["surfaces"] + require( + isinstance(surfaces, list) and surfaces, + "surfaces must be a non-empty list", + ) + for surface in surfaces: + require(isinstance(surface, str), "a surface is not a string") + require_relative(surface, "surface") + require(surface.endswith(".md"), f"surface is not Markdown: {surface}") + require(len(set(surfaces)) == len(surfaces), "surfaces contains a duplicate") + + claims = registry["claims"] + require(isinstance(claims, list), "claims must be a list") + seen_claim_ids: set[str] = set() + for claim in claims: + require(isinstance(claim, dict), "a claim is not a JSON object") + require( + set(claim) == CLAIM_KEYS, + f"claim keys are {sorted(claim)}; expected {sorted(CLAIM_KEYS)}", + ) + claim_id = claim["id"] + require( + isinstance(claim_id, str) and IDENTIFIER.fullmatch(claim_id), + f"claim id is not an identifier: {claim_id!r}", + ) + require(claim_id not in seen_claim_ids, f"duplicate claim id: {claim_id}") + seen_claim_ids.add(claim_id) + require( + claim["surface"] in surfaces, + f"claim {claim_id} names an unregistered surface: {claim['surface']!r}", + ) + require( + claim["source"] in seen_ids, + f"claim {claim_id} names an unknown source: {claim['source']!r}", + ) + require( + isinstance(claim["text"], str) and claim["text"].strip(), + f"claim {claim_id} has no published text", + ) + require( + isinstance(claim["context"], str) and claim["context"].strip(), + f"claim {claim_id} has no context", + ) + require( + claim["text"] in claim["context"], + f"claim {claim_id} context does not contain its text", + ) + renderer = claim["renderer"] + require( + renderer in RENDERERS, + f"claim {claim_id} renderer must be one of {list(RENDERERS)}", + ) + pointers = claim["pointers"] + require( + isinstance(pointers, list) + and len(pointers) == POINTER_COUNTS[renderer] + and all(isinstance(item, str) for item in pointers), + f"claim {claim_id} needs {POINTER_COUNTS[renderer]} JSON pointers", + ) + for pointer in pointers: + require( + pointer.startswith("/"), + f"claim {claim_id} pointer must start with '/': {pointer!r}", + ) + decimals = claim["decimals"] + if renderer in DECIMAL_RENDERERS: + require( + isinstance(decimals, int) and 0 <= decimals <= 6, + f"claim {claim_id} needs decimals between 0 and 6", + ) + else: + require( + decimals is None, + f"claim {claim_id} renderer {renderer} does not take decimals", + ) + require( + claim["status"] in {"bound", "drift_open"}, + f"claim {claim_id} status must be 'bound' or 'drift_open'", + ) + drift = claim["drift"] + if claim["status"] == "bound": + require(drift is None, f"claim {claim_id} is bound but records a drift") + continue + require( + isinstance(drift, dict) and set(drift) == DRIFT_KEYS, + f"claim {claim_id} drift keys must be {sorted(DRIFT_KEYS)}", + ) + for field in ("upstream_text", "owner", "reason"): + require( + isinstance(drift[field], str) and drift[field].strip(), + f"claim {claim_id} drift.{field} must be a non-empty string", + ) + for field in ("recorded_on", "expires_on"): + require( + isinstance(drift[field], str) and DATE.fullmatch(drift[field]), + f"claim {claim_id} drift.{field} must be YYYY-MM-DD", + ) + recorded_on = date.fromisoformat(drift["recorded_on"]) + expires_on = date.fromisoformat(drift["expires_on"]) + require( + expires_on > recorded_on, + f"claim {claim_id} drift expires before it was recorded", + ) + require( + (expires_on - recorded_on).days <= MAX_DRIFT_DAYS, + f"claim {claim_id} drift may not run more than {MAX_DRIFT_DAYS} days", + ) + require( + len(drift["reason"]) >= 40, + f"claim {claim_id} drift.reason must state why the figure is wrong", + ) + + exemptions = registry["non_benchmark_figures"] + require(isinstance(exemptions, list), "non_benchmark_figures must be a list") + for exemption in exemptions: + require( + isinstance(exemption, dict) and set(exemption) == EXEMPTION_KEYS, + f"exemption keys must be {sorted(EXEMPTION_KEYS)}", + ) + exemption_id = exemption["id"] + require( + isinstance(exemption_id, str) and IDENTIFIER.fullmatch(exemption_id), + f"exemption id is not an identifier: {exemption_id!r}", + ) + require( + exemption_id not in seen_claim_ids, + f"exemption {exemption_id} reuses a claim id", + ) + seen_claim_ids.add(exemption_id) + require( + exemption["surface"] in surfaces, + f"exemption {exemption_id} names an unregistered surface", + ) + for field in ("context", "text", "reason"): + require( + isinstance(exemption[field], str) and exemption[field].strip(), + f"exemption {exemption_id} {field} must be a non-empty string", + ) + require( + exemption["text"] in exemption["context"], + f"exemption {exemption_id} context does not contain its text", + ) + require( + len(exemption["reason"]) >= 20, + f"exemption {exemption_id} reason must state why it is not a figure", + ) + + return registry + + +def resolve_pointer(document: Any, pointer: str) -> Any: + """Resolve an RFC 6901 JSON pointer, raising KeyError when it is unbound.""" + current = document + for raw_token in pointer.split("/")[1:]: + token = raw_token.replace("~1", "/").replace("~0", "~") + if isinstance(current, dict): + if token not in current: + raise KeyError(pointer) + current = current[token] + elif isinstance(current, list): + if not token.isdigit() or int(token) >= len(current): + raise KeyError(pointer) + current = current[int(token)] + else: + raise KeyError(pointer) + return current + + +def render(renderer: str, values: list[Any], decimals: int | None) -> str: + """Render an upstream value the way a front page writes it.""" + if renderer in {"ratio", "count_of"}: + for value in values: + if not isinstance(value, int) or isinstance(value, bool): + raise TypeError(f"{renderer} needs integers, found {value!r}") + separator = "/" if renderer == "ratio" else " of " + return f"{values[0]}{separator}{values[1]}" + + value = values[0] + if renderer == "integer": + if not isinstance(value, int) or isinstance(value, bool): + raise TypeError(f"integer needs an integer, found {value!r}") + return str(value) + + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise TypeError(f"{renderer} needs a number, found {value!r}") + if renderer == "percent": + return f"{float(value) * 100:.{decimals}f}%" + if renderer == "seconds": + return f"{float(value):.{decimals}f}s" + return f"${float(value):.{decimals}f}" + + +def verify_snapshots( + root: Path, registry: dict[str, Any], errors: list[str] +) -> dict[str, Any]: + """Re-hash every vendored snapshot and parse the ones that survive.""" + documents: dict[str, Any] = {} + for source in registry["upstream"]["sources"]: + snapshot = root / source["snapshot"] + try: + payload = snapshot.read_bytes() + except OSError as exc: + errors.append(f"source {source['id']}: cannot read snapshot: {exc}") + continue + digest = "sha256:" + hashlib.sha256(payload).hexdigest() + if digest != source["sha256"]: + errors.append( + f"source {source['id']}: snapshot {source['snapshot']} has digest " + f"{digest}, but the registry pins {source['sha256']}. The vendored " + "bytes were edited, or the pin is wrong. Re-vendor from " + f"{registry['upstream']['repository']}@" + f"{registry['upstream']['commit']}:{source['path']}." + ) + continue + if len(payload) != source["size_bytes"]: + errors.append( + f"source {source['id']}: snapshot is {len(payload)} bytes, but the " + f"registry pins {source['size_bytes']}" + ) + continue + try: + documents[source["id"]] = json.loads(payload) + except ValueError as exc: + errors.append(f"source {source['id']}: snapshot is not valid JSON: {exc}") + return documents + + +def locate( + entry: dict[str, Any], surface_text: str, errors: list[str] +) -> tuple[int, int] | None: + """Find the exact span of a registered figure inside its surface.""" + label = f"{entry['surface']} {entry['id']}" + occurrences = surface_text.count(entry["context"]) + if occurrences != 1: + errors.append( + f"{label}: the registered context appears {occurrences} times; it must " + f"appear exactly once. Context: {entry['context']!r}" + ) + return None + context_start = surface_text.index(entry["context"]) + inner = entry["context"].count(entry["text"]) + if inner != 1: + errors.append( + f"{label}: the text {entry['text']!r} appears {inner} times inside its " + "context; it must appear exactly once" + ) + return None + start = context_start + entry["context"].index(entry["text"]) + return start, start + len(entry["text"]) + + +def check_claims( + registry: dict[str, Any], + documents: dict[str, Any], + surfaces: dict[str, str], + today: date, + errors: list[str], + drifts: list[str], +) -> dict[str, list[tuple[int, int, str]]]: + """Compare every registered figure to the artifact it is bound to.""" + spans: dict[str, list[tuple[int, int, str]]] = {name: [] for name in surfaces} + + for claim in registry["claims"]: + surface_text = surfaces[claim["surface"]] + span = locate(claim, surface_text, errors) + if span is not None: + spans[claim["surface"]].append((span[0], span[1], claim["id"])) + + document = documents.get(claim["source"]) + if document is None: + errors.append( + f"claim {claim['id']}: source {claim['source']} did not verify" + ) + continue + try: + values = [ + resolve_pointer(document, pointer) for pointer in claim["pointers"] + ] + except KeyError as exc: + errors.append( + f"claim {claim['id']}: pointer {exc.args[0]} is not present in " + f"{claim['source']}" + ) + continue + try: + upstream_text = render(claim["renderer"], values, claim["decimals"]) + except TypeError as exc: + errors.append(f"claim {claim['id']}: {exc}") + continue + + if claim["status"] == "bound": + if upstream_text != claim["text"]: + errors.append( + f"claim {claim['id']}: {claim['surface']} publishes " + f"{claim['text']!r}, but {claim['source']} " + f"{claim['pointers']} renders {upstream_text!r}" + ) + continue + + drift = claim["drift"] + if upstream_text != drift["upstream_text"]: + errors.append( + f"claim {claim['id']}: the recorded drift expects the artifact to " + f"render {drift['upstream_text']!r}, but it renders " + f"{upstream_text!r}. Re-adjudicate the drift before trusting it." + ) + continue + if upstream_text == claim["text"]: + errors.append( + f"claim {claim['id']}: the published figure now equals the artifact. " + "Change status to 'bound' and delete the drift record." + ) + continue + expires_on = date.fromisoformat(drift["expires_on"]) + if today > expires_on: + errors.append( + f"claim {claim['id']}: the recorded drift expired on " + f"{drift['expires_on']}. {claim['surface']} still publishes " + f"{claim['text']!r} where the artifact says " + f"{drift['upstream_text']!r}. Correct the published figure." + ) + continue + drifts.append( + f"{claim['id']}: {claim['surface']} publishes {claim['text']!r}; " + f"{claim['source']} says {drift['upstream_text']!r}; " + f"owner {drift['owner']}; this exemption fails after " + f"{drift['expires_on']}" + ) + + for exemption in registry["non_benchmark_figures"]: + span = locate(exemption, surfaces[exemption["surface"]], errors) + if span is not None: + spans[exemption["surface"]].append((span[0], span[1], exemption["id"])) + + for surface, entries in spans.items(): + ordered = sorted(entries) + for earlier, later in zip(ordered, ordered[1:]): + if later[0] < earlier[1]: + errors.append( + f"{surface}: registry entries {earlier[2]} and {later[2]} claim " + "overlapping text" + ) + return spans + + +def sweep( + surfaces: dict[str, str], + spans: dict[str, list[tuple[int, int, str]]], + errors: list[str], +) -> int: + """Fail on any figure-shaped token that no registry entry claims.""" + claimed_total = 0 + for surface, text in surfaces.items(): + claimed = sorted(spans[surface]) + for match in FIGURE_RE.finditer(text): + start, end = match.span() + owner = next( + ( + entry_id + for entry_start, entry_end, entry_id in claimed + if entry_start <= start and end <= entry_end + ), + None, + ) + if owner is None: + window = text[max(0, start - 60) : end + 40] + errors.append( + f"{surface}: the figure {match.group(0)!r} is not claimed by any " + "benchmark-claims.json entry. Bind it to its upstream artifact, " + "or record it in non_benchmark_figures with a reason. " + f"Context: ...{window}..." + ) + else: + claimed_total += 1 + + for entry_start, entry_end, entry_id in claimed: + fragment = text[entry_start:entry_end] + if not FIGURE_RE.fullmatch(fragment): + errors.append( + f"{surface}: registry entry {entry_id} claims {fragment!r}, which " + "the sweep does not read as a figure. Remove the stale entry." + ) + return claimed_total + + +def cross_check_comparison( + documents: dict[str, Any], errors: list[str] +) -> None: + """Confirm the derived comparison artifact still agrees with its sources.""" + comparison = documents.get("comparison") + if comparison is None: + return + fields = ( + "n", + "success_count", + "success_rate", + "wall_s_p50", + "wall_s_p95", + "cost_usd_per_run", + "cost_usd_total", + ) + for name in ("openemr", "mockmed"): + source = documents.get(name) + if source is None: + continue + try: + derived = comparison["benchmarks"][name]["arms"] + except (KeyError, TypeError): + errors.append(f"comparison artifact has no {name} arms") + continue + for arm in ("compiled", "agent"): + for field in fields: + try: + left = derived[arm][field] + right = source["arms"][arm][field] + except (KeyError, TypeError): + errors.append(f"{name} {arm} {field} is missing from an artifact") + continue + if left != right: + errors.append( + f"{name} {arm} {field}: the comparison artifact says {left!r}, " + f"but the measured results say {right!r}" + ) + + +def fetch(url: str) -> bytes: + request = urllib.request.Request( + url, headers={"User-Agent": "OpenAdapt-benchmark-claims/1"} + ) + with urllib.request.urlopen(request, timeout=FETCH_TIMEOUT_SECONDS) as response: + return response.read() + + +def check_online( + registry: dict[str, Any], errors: list[str], warnings: list[str] +) -> int: + """Re-fetch each pinned path and compare it to the vendored snapshot. + + GitHub being unreachable is an outage, not evidence of drift, so it warns. + A successful fetch whose digest differs is drift, and it fails. + """ + upstream = registry["upstream"] + verified = 0 + for source in upstream["sources"]: + url = ( + f"{RAW_GITHUB_ORIGIN}/{upstream['repository']}/" + f"{upstream['commit']}/{source['path']}" + ) + try: + payload = fetch(url) + except (OSError, urllib.error.URLError, TimeoutError, ValueError) as exc: + warnings.append(f"source {source['id']}: {url} is unreachable: {exc}") + continue + digest = "sha256:" + hashlib.sha256(payload).hexdigest() + if digest != source["sha256"]: + errors.append( + f"source {source['id']}: {url} returned digest {digest}, but the " + f"vendored snapshot is {source['sha256']}. The pin no longer " + "describes the upstream bytes." + ) + continue + verified += 1 + return verified + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--root", + type=Path, + default=ROOT, + help="repository root to check (default: this repository)", + ) + parser.add_argument( + "--online", + action="store_true", + help="also re-fetch each pinned upstream path and compare digests", + ) + parser.add_argument( + "--allow-recorded-drift", + action="store_true", + help=( + "exit 0 when the only failures are dated drift records already in the " + "registry; every such record is still printed and still expires" + ), + ) + parser.add_argument( + "--today", + type=date.fromisoformat, + default=datetime.now(timezone.utc).date(), + help="evaluate drift expiry against this date (YYYY-MM-DD)", + ) + arguments = parser.parse_args(argv) + + root: Path = arguments.root + errors: list[str] = [] + warnings: list[str] = [] + drifts: list[str] = [] + + try: + registry = load_registry(root / REGISTRY_PATH.name) + except RegistryError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + documents = verify_snapshots(root, registry, errors) + cross_check_comparison(documents, errors) + + surfaces: dict[str, str] = {} + for surface in registry["surfaces"]: + path = root / surface + try: + surfaces[surface] = normalize(path.read_text(encoding="utf-8")) + except OSError as exc: + errors.append(f"cannot read registered surface {surface}: {exc}") + + spans = check_claims( + registry, documents, surfaces, arguments.today, errors, drifts + ) + claimed = sweep(surfaces, spans, errors) + + verified_online = check_online(registry, errors, warnings) if arguments.online else 0 + + for warning in warnings: + print(f"WARNING: {warning}", file=sys.stderr) + for drift in drifts: + print(f"DRIFT: {drift}", file=sys.stderr) + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + + if errors: + return 1 + if drifts and not arguments.allow_recorded_drift: + print( + "ERROR: a published figure does not match its artifact. Pass " + "--allow-recorded-drift only where a dated correction is already " + "tracked.", + file=sys.stderr, + ) + return 1 + + summary = ( + f"Bound {claimed} published figures across {len(surfaces)} surfaces to " + f"{len(documents)} pinned artifacts" + ) + if arguments.online: + summary += f"; re-fetched {verified_online} of {len(documents)} upstream paths" + if drifts: + summary += f"; {len(drifts)} dated drift record(s) still open" + print(summary + ".") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_benchmark_claims.py b/tests/test_benchmark_claims.py new file mode 100644 index 0000000..9d4266c --- /dev/null +++ b/tests/test_benchmark_claims.py @@ -0,0 +1,550 @@ +"""Fail-closed tests for the published-benchmark-figure binding.""" + +from __future__ import annotations + +import copy +import hashlib +import io +import json +import shutil +import sys +import tempfile +import unittest +import urllib.error +from contextlib import redirect_stderr, redirect_stdout +from datetime import date +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import check_benchmark_claims as claims # noqa: E402 + +SURFACE = "profile/README.md" +SNAPSHOT = "benchmark-claims/upstream/example/results.json" +ARTIFACT = {"arms": {"compiled": {"n": 20, "success_count": 19, "wall_s_p50": 39.17}}} + + +def digest(payload: bytes) -> str: + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def run(root: Path, *arguments: str) -> tuple[int, str, str]: + out, err = io.StringIO(), io.StringIO() + with redirect_stdout(out), redirect_stderr(err): + code = claims.main(["--root", str(root), *arguments]) + return code, out.getvalue(), err.getvalue() + + +def synthetic_registry() -> dict: + payload = json.dumps(ARTIFACT, indent=2).encode() + b"\n" + return { + "$comment": "A synthetic fixture used only by the tests.", + "schema_version": claims.REGISTRY_SCHEMA, + "upstream": { + "repository": "OpenAdaptAI/openadapt-flow", + "commit": "a" * 40, + "sources": [ + { + "id": "example", + "path": "benchmark/example/results.json", + "git_blob": "b" * 40, + "sha256": digest(payload), + "size_bytes": len(payload), + "snapshot": SNAPSHOT, + } + ], + }, + "surfaces": [SURFACE], + "claims": [ + { + "id": "example-compiled-success", + "surface": SURFACE, + "context": "compiled replay went 19/20 on the demo.", + "text": "19/20", + "renderer": "ratio", + "decimals": None, + "source": "example", + "pointers": [ + "/arms/compiled/success_count", + "/arms/compiled/n", + ], + "status": "bound", + "drift": None, + } + ], + "non_benchmark_figures": [], + } + + +class Fixture: + """A throwaway repository holding one surface and one pinned artifact.""" + + def __init__(self, directory: str) -> None: + self.root = Path(directory) + self.registry = synthetic_registry() + self.surface_text = ( + "# Example\n\nThe compiled replay went 19/20 on the demo.\n" + ) + + def write(self) -> Path: + payload = json.dumps(ARTIFACT, indent=2).encode() + b"\n" + snapshot = self.root / SNAPSHOT + snapshot.parent.mkdir(parents=True, exist_ok=True) + snapshot.write_bytes(payload) + surface = self.root / SURFACE + surface.parent.mkdir(parents=True, exist_ok=True) + surface.write_text(self.surface_text, encoding="utf-8") + (self.root / "benchmark-claims.json").write_text( + json.dumps(self.registry, indent=2) + "\n", encoding="utf-8" + ) + return self.root + + +class RendererTests(unittest.TestCase): + def test_every_renderer_matches_the_way_a_front_page_writes_it(self) -> None: + self.assertEqual(claims.render("ratio", [19, 20], None), "19/20") + self.assertEqual(claims.render("count_of", [54, 90], None), "54 of 90") + self.assertEqual(claims.render("percent", [0.75], 1), "75.0%") + self.assertEqual(claims.render("percent", [0.0], 0), "0%") + self.assertEqual(claims.render("seconds", [39.170917561999886], 1), "39.2s") + self.assertEqual(claims.render("usd", [0.552150765], 2), "$0.55") + self.assertEqual(claims.render("integer", [18], None), "18") + + def test_a_boolean_is_not_a_measurement(self) -> None: + with self.assertRaises(TypeError): + claims.render("ratio", [True, 20], None) + with self.assertRaises(TypeError): + claims.render("percent", [True], 1) + + def test_every_renderer_is_reachable_from_the_pointer_contract(self) -> None: + self.assertEqual(set(claims.RENDERERS), set(claims.POINTER_COUNTS)) + + +class PointerTests(unittest.TestCase): + def test_an_unbound_pointer_raises_instead_of_returning_none(self) -> None: + with self.assertRaises(KeyError): + claims.resolve_pointer(ARTIFACT, "/arms/compiled/missing") + with self.assertRaises(KeyError): + claims.resolve_pointer(ARTIFACT, "/arms/compiled/n/deeper") + + def test_a_list_index_resolves(self) -> None: + self.assertEqual(claims.resolve_pointer({"a": [7, 8]}, "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/a/1"), 8) + + +class SyntheticGuardTests(unittest.TestCase): + def build(self, mutate=None) -> Path: + directory = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, directory, True) + fixture = Fixture(directory) + if mutate is not None: + mutate(fixture) + return fixture.write() + + def test_a_faithful_surface_passes(self) -> None: + code, out, _ = run(self.build()) + self.assertEqual(code, 0) + self.assertIn("Bound 1 published figures", out) + + def test_an_edited_snapshot_cannot_bless_a_figure(self) -> None: + root = self.build() + snapshot = root / SNAPSHOT + snapshot.write_bytes(snapshot.read_bytes().replace(b"19", b"20")) + code, _, err = run(root) + self.assertEqual(code, 1) + self.assertIn("has digest", err) + self.assertIn("the registry pins", err) + + def test_an_unregistered_figure_fails_closed(self) -> None: + def mutate(fixture: Fixture) -> None: + fixture.surface_text += "\nA later edit added 7/8 and nobody bound it.\n" + + code, _, err = run(self.build(mutate)) + self.assertEqual(code, 1) + self.assertIn("'7/8' is not claimed by any", err) + + def test_a_wrong_published_figure_fails(self) -> None: + def mutate(fixture: Fixture) -> None: + fixture.surface_text = fixture.surface_text.replace("19/20", "20/20") + claim = fixture.registry["claims"][0] + claim["context"] = claim["context"].replace("19/20", "20/20") + claim["text"] = "20/20" + + code, _, err = run(self.build(mutate)) + self.assertEqual(code, 1) + self.assertIn("publishes '20/20'", err) + self.assertIn("renders '19/20'", err) + + def test_every_sweep_pattern_is_caught_when_unregistered(self) -> None: + for figure in ("7/8", "12.5%", "$3.10", "4.9s", "9 of 90"): + with self.subTest(figure=figure): + + def mutate(fixture: Fixture, figure: str = figure) -> None: + fixture.surface_text += f"\nAn unbound {figure} slipped in.\n" + + code, _, err = run(self.build(mutate)) + self.assertEqual(code, 1) + self.assertIn(f"{figure!r} is not claimed", err) + + def test_an_exemption_covers_a_non_benchmark_figure(self) -> None: + def mutate(fixture: Fixture) -> None: + fixture.surface_text += "\nThe launcher pins version 2/3 of the schema.\n" + fixture.registry["non_benchmark_figures"].append( + { + "id": "schema-version-fragment", + "surface": SURFACE, + "context": "pins version 2/3 of the schema", + "text": "2/3", + "reason": "A schema version, not a measured benchmark result.", + } + ) + + code, out, _ = run(self.build(mutate)) + self.assertEqual(code, 0) + self.assertIn("Bound 2 published figures", out) + + def test_a_stale_exemption_does_not_rot_in_place(self) -> None: + def mutate(fixture: Fixture) -> None: + fixture.registry["non_benchmark_figures"].append( + { + "id": "removed-version-fragment", + "surface": SURFACE, + "context": "pins version 2/3 of the schema", + "text": "2/3", + "reason": "A schema version, not a measured benchmark result.", + } + ) + + code, _, err = run(self.build(mutate)) + self.assertEqual(code, 1) + self.assertIn("appears 0 times", err) + + def test_an_ambiguous_context_is_refused(self) -> None: + def mutate(fixture: Fixture) -> None: + fixture.surface_text += "\nThe compiled replay went 19/20 on the demo.\n" + + code, _, err = run(self.build(mutate)) + self.assertEqual(code, 1) + self.assertIn("appears 2 times", err) + + +class RecordedDriftTests(unittest.TestCase): + def build(self, mutate=None) -> Path: + directory = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, directory, True) + fixture = Fixture(directory) + fixture.surface_text = fixture.surface_text.replace("19/20", "20/20") + claim = fixture.registry["claims"][0] + claim["context"] = claim["context"].replace("19/20", "20/20") + claim["text"] = "20/20" + claim["status"] = "drift_open" + claim["drift"] = { + "upstream_text": "19/20", + "recorded_on": "2026-08-28", + "expires_on": "2026-09-25", + "owner": "the agent that owns org front-page corrections", + "reason": ( + "The published figure is wrong and the correction is tracked " + "elsewhere; this guard reports it rather than editing the prose." + ), + } + if mutate is not None: + mutate(fixture) + return fixture.write() + + def test_a_recorded_drift_fails_by_default(self) -> None: + code, _, err = run(self.build(), "--today", "2026-08-28") + self.assertEqual(code, 1) + self.assertIn("DRIFT:", err) + self.assertIn("--allow-recorded-drift", err) + + def test_a_recorded_drift_is_always_printed_even_when_allowed(self) -> None: + code, out, err = run( + self.build(), "--today", "2026-08-28", "--allow-recorded-drift" + ) + self.assertEqual(code, 0) + self.assertIn("DRIFT:", err) + self.assertIn("1 dated drift record(s) still open", out) + + def test_an_expired_drift_fails_even_when_allowed(self) -> None: + code, _, err = run( + self.build(), "--today", "2026-09-26", "--allow-recorded-drift" + ) + self.assertEqual(code, 1) + self.assertIn("expired on 2026-09-25", err) + + def test_a_drift_that_no_longer_drifts_must_be_promoted(self) -> None: + def mutate(fixture: Fixture) -> None: + fixture.surface_text = fixture.surface_text.replace("20/20", "19/20") + claim = fixture.registry["claims"][0] + claim["context"] = claim["context"].replace("20/20", "19/20") + claim["text"] = "19/20" + + code, _, err = run( + self.build(mutate), "--today", "2026-08-28", "--allow-recorded-drift" + ) + self.assertEqual(code, 1) + self.assertIn("Change status to 'bound'", err) + + def test_a_changed_artifact_invalidates_the_drift_record(self) -> None: + def mutate(fixture: Fixture) -> None: + fixture.registry["claims"][0]["drift"]["upstream_text"] = "18/20" + + code, _, err = run( + self.build(mutate), "--today", "2026-08-28", "--allow-recorded-drift" + ) + self.assertEqual(code, 1) + self.assertIn("Re-adjudicate the drift", err) + + def test_a_drift_may_not_run_without_an_end(self) -> None: + def mutate(fixture: Fixture) -> None: + fixture.registry["claims"][0]["drift"]["expires_on"] = "2027-08-28" + + code, _, err = run( + self.build(mutate), "--today", "2026-08-28", "--allow-recorded-drift" + ) + self.assertEqual(code, 1) + self.assertIn(f"more than {claims.MAX_DRIFT_DAYS} days", err) + + +class RegistryShapeTests(unittest.TestCase): + def load(self, mutate) -> None: + directory = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, directory, True) + fixture = Fixture(directory) + mutate(fixture) + root = fixture.write() + with self.assertRaises(claims.RegistryError): + claims.load_registry(root / "benchmark-claims.json") + + def test_an_unknown_top_level_key_is_refused(self) -> None: + self.load(lambda fixture: fixture.registry.update({"extra": True})) + + def test_a_missing_comment_is_refused(self) -> None: + self.load(lambda fixture: fixture.registry.update({"$comment": " "})) + + def test_a_short_commit_is_refused(self) -> None: + self.load(lambda fixture: fixture.registry["upstream"].update({"commit": "aee0941"})) + + def test_a_snapshot_outside_the_vendored_tree_is_refused(self) -> None: + self.load( + lambda fixture: fixture.registry["upstream"]["sources"][0].update( + {"snapshot": "../escape.json"} + ) + ) + + def test_a_claim_naming_an_unregistered_surface_is_refused(self) -> None: + self.load( + lambda fixture: fixture.registry["claims"][0].update({"surface": "OTHER.md"}) + ) + + def test_a_claim_naming_an_unknown_source_is_refused(self) -> None: + self.load( + lambda fixture: fixture.registry["claims"][0].update({"source": "nowhere"}) + ) + + def test_a_ratio_needs_two_pointers(self) -> None: + self.load( + lambda fixture: fixture.registry["claims"][0].update( + {"pointers": ["/arms/compiled/success_count"]} + ) + ) + + def test_a_bound_claim_may_not_carry_a_drift(self) -> None: + self.load( + lambda fixture: fixture.registry["claims"][0].update( + {"drift": {"upstream_text": "19/20"}} + ) + ) + + def test_a_context_that_omits_its_text_is_refused(self) -> None: + self.load( + lambda fixture: fixture.registry["claims"][0].update( + {"context": "no figure here"} + ) + ) + + def test_an_exemption_without_a_reason_is_refused(self) -> None: + self.load( + lambda fixture: fixture.registry["non_benchmark_figures"].append( + { + "id": "thin", + "surface": SURFACE, + "context": "19/20", + "text": "19/20", + "reason": "no", + } + ) + ) + + +class OnlineTests(unittest.TestCase): + def registry(self) -> dict: + return copy.deepcopy(synthetic_registry()) + + def test_an_unreachable_github_warns_and_does_not_fail(self) -> None: + errors: list[str] = [] + warnings: list[str] = [] + with mock.patch.object( + claims, "fetch", side_effect=urllib.error.URLError("down") + ): + verified = claims.check_online(self.registry(), errors, warnings) + self.assertEqual(verified, 0) + self.assertEqual(errors, []) + self.assertEqual(len(warnings), 1) + self.assertIn("unreachable", warnings[0]) + + def test_a_timeout_warns_and_does_not_fail(self) -> None: + errors: list[str] = [] + warnings: list[str] = [] + with mock.patch.object(claims, "fetch", side_effect=TimeoutError("slow")): + claims.check_online(self.registry(), errors, warnings) + self.assertEqual(errors, []) + self.assertEqual(len(warnings), 1) + + def test_a_digest_mismatch_fails(self) -> None: + errors: list[str] = [] + warnings: list[str] = [] + with mock.patch.object(claims, "fetch", return_value=b"different bytes"): + claims.check_online(self.registry(), errors, warnings) + self.assertEqual(warnings, []) + self.assertEqual(len(errors), 1) + self.assertIn("no longer describes the upstream bytes", errors[0]) + + def test_a_matching_fetch_verifies(self) -> None: + errors: list[str] = [] + warnings: list[str] = [] + payload = json.dumps(ARTIFACT, indent=2).encode() + b"\n" + with mock.patch.object(claims, "fetch", return_value=payload) as fetched: + verified = claims.check_online(self.registry(), errors, warnings) + self.assertEqual((verified, errors, warnings), (1, [], [])) + self.assertEqual( + fetched.call_args.args[0], + f"{claims.RAW_GITHUB_ORIGIN}/OpenAdaptAI/openadapt-flow/" + f"{'a' * 40}/benchmark/example/results.json", + ) + + +class PublishedRegistryTests(unittest.TestCase): + def copy_repository(self) -> Path: + directory = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, directory, True) + root = Path(directory) + shutil.copytree(ROOT / "benchmark-claims", root / "benchmark-claims") + shutil.copy2(ROOT / "benchmark-claims.json", root / "benchmark-claims.json") + registry = json.loads((ROOT / "benchmark-claims.json").read_text()) + for surface in registry["surfaces"]: + target = root / surface + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(ROOT / surface, target) + return root + + def test_the_published_registry_is_structurally_valid(self) -> None: + registry = claims.load_registry(ROOT / "benchmark-claims.json") + self.assertEqual(registry["upstream"]["repository"], "OpenAdaptAI/openadapt-flow") + self.assertTrue(registry["claims"]) + + def test_every_published_figure_is_bound_or_recorded(self) -> None: + code, out, _ = run(ROOT, "--allow-recorded-drift") + self.assertEqual(code, 0) + self.assertIn("Bound 15 published figures", out) + + def test_the_published_surfaces_cover_both_front_pages(self) -> None: + registry = claims.load_registry(ROOT / "benchmark-claims.json") + self.assertEqual(set(registry["surfaces"]), {"README.md", "profile/README.md"}) + + def test_every_open_drift_states_who_fixes_it_and_by_when(self) -> None: + registry = claims.load_registry(ROOT / "benchmark-claims.json") + for claim in registry["claims"]: + if claim["status"] != "drift_open": + continue + drift = claim["drift"] + self.assertTrue(drift["owner"].strip(), claim["id"]) + self.assertGreater(len(drift["reason"]), 80, claim["id"]) + self.assertLessEqual( + ( + date.fromisoformat(drift["expires_on"]) + - date.fromisoformat(drift["recorded_on"]) + ).days, + claims.MAX_DRIFT_DAYS, + claim["id"], + ) + + def test_a_new_unregistered_figure_on_a_real_surface_fails(self) -> None: + root = self.copy_repository() + surface = root / "README.md" + surface.write_text( + surface.read_text(encoding="utf-8") + + "\nAn unbound compiled replay went 19/20.\n", + encoding="utf-8", + ) + code, _, err = run(root, "--allow-recorded-drift") + self.assertEqual(code, 1) + self.assertIn("README.md: the figure '19/20' is not claimed", err) + + def test_a_tampered_published_snapshot_fails(self) -> None: + root = self.copy_repository() + registry = json.loads((root / "benchmark-claims.json").read_text()) + snapshot = next( + root / source["snapshot"] + for source in registry["upstream"]["sources"] + if source["id"] == "openemr" + ) + snapshot.write_bytes( + snapshot.read_bytes().replace( + b'"success_count": 19', b'"success_count": 20' + ) + ) + code, _, err = run(root, "--allow-recorded-drift") + self.assertEqual(code, 1) + self.assertIn("snapshot", err) + self.assertIn("the registry pins", err) + + def test_the_comparison_artifact_must_agree_with_its_sources(self) -> None: + root = self.copy_repository() + registry = json.loads((root / "benchmark-claims.json").read_text()) + source = next( + item for item in registry["upstream"]["sources"] if item["id"] == "comparison" + ) + snapshot = root / source["snapshot"] + document = json.loads(snapshot.read_text(encoding="utf-8")) + document["benchmarks"]["openemr"]["arms"]["compiled"]["success_count"] = 20 + payload = json.dumps(document, indent=2).encode() + b"\n" + snapshot.write_bytes(payload) + source["sha256"] = digest(payload) + source["size_bytes"] = len(payload) + (root / "benchmark-claims.json").write_text( + json.dumps(registry, indent=2) + "\n", encoding="utf-8" + ) + code, _, err = run(root, "--allow-recorded-drift") + self.assertEqual(code, 1) + self.assertIn("openemr compiled success_count", err) + + +class WorkflowWiringTests(unittest.TestCase): + def workflow(self) -> str: + return (ROOT / ".github" / "workflows" / "profile-consistency.yml").read_text( + encoding="utf-8" + ) + + def test_the_offline_half_runs_on_every_pull_request(self) -> None: + content = self.workflow() + job = content.split(" validate-profile:", 1)[1].split("\n report-", 1)[0] + self.assertIn("scripts/check_benchmark_claims.py", job) + + def test_the_online_half_runs_only_on_the_daily_schedule(self) -> None: + content = self.workflow() + step = content.split( + " - name: Re-fetch the pinned benchmark artifacts\n", 1 + )[1].split(" - name:", 1)[0] + self.assertIn("--online", step) + self.assertIn("github.event_name == 'schedule'", step) + + def test_the_daily_job_needs_no_dependency_install(self) -> None: + content = self.workflow() + self.assertNotIn("pip install", content) + + +if __name__ == "__main__": + unittest.main()