From cf290c0e1569d50c7fb2bdb96741001a9bd54556 Mon Sep 17 00:00:00 2001 From: abrichr Date: Fri, 28 Aug 2026 14:27:14 -0400 Subject: [PATCH 1/2] docs: quote the corrected 19/20 OpenEMR result on the org profile The compiled arm scored 19/20 with one safe halt, not 20/20. Source of truth is openadapt-flow benchmark/openemr/results.json (n: 20, success_count: 19) and the dated oracle correction in that directory's BENCHMARK.md. Co-Authored-By: Claude Opus 5 --- profile/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/profile/README.md b/profile/README.md index 1ea751f..5a3900a 100644 --- a/profile/README.md +++ b/profile/README.md @@ -41,10 +41,12 @@ Published head-to-head comparisons, each graded by an external success check that is independent of every arm: - **Live third-party EMR** (OpenEMR public demo, fake patients, 18-step - add-patient-note workflow): compiled replay went **20/20 at 39.2s p50 with + add-patient-note workflow): compiled replay went **19/20 at 39.2s p50 with zero model calls**; the agent went 10/10 at 70.4s p50 at about $0.55 of model - charge per run. Small sample on a shared, daily-resetting demo — not - CI-reproducible. + charge per run. Compiled run 20 didn't pass. The saved-row oracle, tightened + on 2026-07-28, refuses to count a note still sitting in the unsaved entry + form, and the replayer had already halted at step 17 rather than press on. + Small sample on a shared, daily-resetting demo — not CI-reproducible. [Methodology and caveats](https://github.com/OpenAdaptAI/openadapt-flow/blob/main/benchmark/openemr/BENCHMARK.md). - **CI-reproducible control** (bundled MockMed task): both arms passed every run (100/100 compiled, 20/20 agent), so the result is cost and latency, not From 4ed10a0c80948be66cdcee592b4dd1177831deb2 Mon Sep 17 00:00:00 2001 From: abrichr Date: Fri, 28 Aug 2026 14:56:41 -0400 Subject: [PATCH 2/2] chore: close the openemr-compiled-success-ratio drift profile/README.md now publishes 19/20, which is what the pinned artifact records, so the entry moves from drift_open to bound. Co-Authored-By: Claude Opus 5 --- benchmark-claims.json | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/benchmark-claims.json b/benchmark-claims.json index 6080d48..d482a6f 100644 --- a/benchmark-claims.json +++ b/benchmark-claims.json @@ -1,5 +1,5 @@ { - "$comment": "Binds every benchmark figure printed on an org front page to the upstream artifact that measured it. Every other claim guard in this org checks that a page contains an attribution string; none of them checks the number. openadapt-web published success_count 20 for the OpenEMR compiled arm for five weeks while the measured artifact said 19, and no guard noticed. scripts/check_benchmark_claims.py re-hashes the vendored snapshots under benchmark-claims/upstream/, renders each registered figure straight out of those bytes, and refuses any figure-shaped token on a registered surface that no entry here claims. So a new figure cannot land on the org profile without a deliberate edit to this file. profile/README.md currently publishes 20/20 for the OpenEMR compiled arm and the pinned artifact says 19 of 20; that one figure is recorded below as an open drift with an expiry date, and the checker fails once the date passes. This repository does not own that prose, so this guard reports the mismatch instead of editing it.", + "$comment": "Binds every benchmark figure printed on an org front page to the upstream artifact that measured it. Every other claim guard in this org checks that a page contains an attribution string; none of them checks the number. openadapt-web published success_count 20 for the OpenEMR compiled arm for five weeks while the measured artifact said 19, and no guard noticed. scripts/check_benchmark_claims.py re-hashes the vendored snapshots under benchmark-claims/upstream/, renders each registered figure straight out of those bytes, and refuses any figure-shaped token on a registered surface that no entry here claims. So a new figure cannot land on the org profile without a deliberate edit to this file. A figure this repository does not own can be recorded as an open drift with an owner and an expiry date; the checker prints it on every run and fails unconditionally once the date passes.", "schema_version": "openadapt.benchmark-claim-binding/v1", "upstream": { "repository": "OpenAdaptAI/openadapt-flow", @@ -47,8 +47,8 @@ { "id": "openemr-compiled-success-ratio", "surface": "profile/README.md", - "context": "compiled replay went **20/20 at 39.2s p50 with zero model calls**", - "text": "20/20", + "context": "compiled replay went **19/20 at 39.2s p50 with zero model calls**", + "text": "19/20", "renderer": "ratio", "decimals": null, "source": "openemr", @@ -56,19 +56,13 @@ "/arms/compiled/success_count", "/arms/compiled/n" ], - "status": "drift_open", - "drift": { - "upstream_text": "19/20", - "recorded_on": "2026-08-28", - "expires_on": "2026-09-25", - "owner": "the agent that owns org front-page corrections", - "reason": "profile/README.md publishes 20/20 for the OpenEMR compiled arm. The pinned artifact records success_count 19 of n 20, because the saved-row oracle adjudication of 2026-07-28 rejected compiled run 20: the note stayed in the unsaved entry form. The published figure is wrong and must become 19/20. This guard adds the binding and does not edit the prose." - } + "status": "bound", + "drift": null }, { "id": "openemr-compiled-p50", "surface": "profile/README.md", - "context": "compiled replay went **20/20 at 39.2s p50 with zero model calls**", + "context": "compiled replay went **19/20 at 39.2s p50 with zero model calls**", "text": "39.2s", "renderer": "seconds", "decimals": 1,