From d996a5c7c89938528e1a17ad1ccfb8b9b6dabdcf Mon Sep 17 00:00:00 2001 From: Obvious Date: Thu, 17 Sep 2026 17:44:05 +0000 Subject: [PATCH] feat(evaluation): shared acceptance corpus + candidate-agnostic harness Corpus as data (6 JSON fixtures, expectations independent of candidate code): multi-event narrative (count/order), relative-time across America/New_York, Pacific/Auckland (NZST + NZDT) and UTC with documented conventions and machine-computed IANA instants, malformed extraction (failure never blocks capture, raw preserved), retry/double-submit on the same captureId (exactly one event, IdempotentReplay), raw fidelity (SHA-256 byte equality across validation failure and retry), and reload persistence (timeline identical across simulated cold start). Harness: narrow CandidateAdapter interface stated in types (createEntry/readTimeline/reload), fixed per-fixture runner protocols, worked example adapter over the Effect v4 schema contract wire shapes, and a deliberately broken negative control proving the checks bite. README documents the adapter contract, conventions, and exact commands. Co-authored-by: Gilbert Polanco --- evaluation/.gitignore | 4 + evaluation/README.md | 172 +++++++++++ .../fixtures/01-multi-event-narrative.json | 14 + .../fixtures/02-relative-time-timezones.json | 78 +++++ .../fixtures/03-malformed-extraction.json | 8 + .../fixtures/04-retry-double-submit.json | 12 + evaluation/fixtures/05-raw-fidelity.json | 24 ++ .../fixtures/06-reload-persistence.json | 26 ++ evaluation/package.json | 17 ++ evaluation/src/adapter.ts | 88 ++++++ evaluation/src/example/broken-adapter.ts | 67 +++++ evaluation/src/example/example-adapter.ts | 224 ++++++++++++++ evaluation/src/fixtures.ts | 171 +++++++++++ evaluation/src/index.ts | 15 + evaluation/src/match.ts | 105 +++++++ evaluation/src/run.ts | 57 ++++ evaluation/src/runner.ts | 284 ++++++++++++++++++ evaluation/tsconfig.json | 17 ++ 18 files changed, 1383 insertions(+) create mode 100644 evaluation/.gitignore create mode 100644 evaluation/README.md create mode 100644 evaluation/fixtures/01-multi-event-narrative.json create mode 100644 evaluation/fixtures/02-relative-time-timezones.json create mode 100644 evaluation/fixtures/03-malformed-extraction.json create mode 100644 evaluation/fixtures/04-retry-double-submit.json create mode 100644 evaluation/fixtures/05-raw-fidelity.json create mode 100644 evaluation/fixtures/06-reload-persistence.json create mode 100644 evaluation/package.json create mode 100644 evaluation/src/adapter.ts create mode 100644 evaluation/src/example/broken-adapter.ts create mode 100644 evaluation/src/example/example-adapter.ts create mode 100644 evaluation/src/fixtures.ts create mode 100644 evaluation/src/index.ts create mode 100644 evaluation/src/match.ts create mode 100644 evaluation/src/run.ts create mode 100644 evaluation/src/runner.ts create mode 100644 evaluation/tsconfig.json diff --git a/evaluation/.gitignore b/evaluation/.gitignore new file mode 100644 index 0000000..424b42b --- /dev/null +++ b/evaluation/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +bun.lock +package-lock.json +pnpm-lock.yaml diff --git a/evaluation/README.md b/evaluation/README.md new file mode 100644 index 0000000..ee962a0 --- /dev/null +++ b/evaluation/README.md @@ -0,0 +1,172 @@ +# Acceptance Corpus & Evaluation Harness + +Shared acceptance/evaluation corpus plus a candidate-agnostic harness that the +arena cross-review runs **identically** against all four candidates +(`arena/candidate-a|b|c|d`). Fixtures are pure **data** — inputs plus expected +outcomes, independent of any candidate code — so per-candidate results are +directly comparable. + +Design sources: the Effect v4 schema contract (wire shapes for `Event`/`Entry`, +Unix-ms timestamps, six categories, `_tag` literals) and the product +architecture blueprint (capture flow: preserve raw source → extract typed +events → review → publish; resilient capture identifiers; duplicate protection). + +## Layout + +``` +evaluation/ + src/ + adapter.ts ← THE adapter interface (narrow, stated in types) + fixtures.ts ← fixture data types + fail-fast loader + match.ts ← canonical JSON, SHA-256 byte fidelity, event matcher + runner.ts ← fixed per-fixture protocols (identical for every candidate) + run.ts ← CLI entry + example/example-adapter.ts ← worked example adapter (in-memory, contract-shaped) + example/broken-adapter.ts ← deliberate negative control (must fail the corpus) + fixtures/ ← the corpus: 6 JSON fixtures (data only) +``` + +## The adapter contract + +Implement `CandidateAdapter` (see `src/adapter.ts`) and export a **named +factory** from one module: + +```ts +import type { CandidateAdapter, CreateEntryInput, CreateEntryResult, WireEntry } from '../../evaluation/src/adapter.ts' + +export function createAdapter(): CandidateAdapter { + return { + name: 'my-candidate', + async createEntry(input: CreateEntryInput): Promise { /* ... */ }, + async readTimeline(): Promise { /* ... */ }, + async reload(): Promise { /* ... */ }, + } +} +``` + +Semantic rules every adapter must honor (the runner checks all of them): + +- **`createEntry`** runs the full capture pipeline for one dictation: preserve + the raw `transcript` **byte-for-byte** (no trim, no Unicode normalization, no + re-encoding), extract events, resolve relative times against + `capturedAt`/`timezone`, validate against the contract, persist, and resolve + only after extraction and persistence have settled. +- **Idempotency**: `captureId` is the idempotency key. A second submission with + the same `captureId` must return `{ _tag: 'IdempotentReplay', entry }` with + the **existing** record — never a second entry, never new events. +- **Extraction failure never blocks capture** (contract rule): when no + schema-valid event can be produced, the entry is still **created** with + `events: []` — returning `Rejected` for extraction failure is a contract + violation. `Rejected` is reserved for raw-input policy refusals (e.g. an + empty transcript; the corpus never triggers it). +- **Fresh entries are `draft`** — review/publish happens later, outside corpus + scope. `readTimeline()` returns all persisted entries (corpus scope: one + child's timeline, drafts only). +- **`reload()`** simulates a cold start: after it resolves, `readTimeline()` + must equal what a fresh process would read from durable state. A + server-backed adapter whose reads are already cold-start-equivalent may + no-op. +- **Wire shapes** mirror the schema contract: `occurredAt`/`createdAt` are + Unix-ms **numbers**, `_tag` literals are required, optional fields are + **absent** — explicit `null` (e.g. `quantity: null`, `note: ""`) is a + violation. One deliberate addition: `WireEntry.captureId` carries the + capture identifier (resilient capture / duplicate-protection key), which the + Entry contract predates — map your capture identity to it. + +## The corpus (6 fixtures, data only) + +| # | File | Kind | Pins | +|---|------|------|------| +| 1 | `01-multi-event-narrative.json` | `multi-event-narrative` | 3 events from one narrative: exact count, exact categories/quantities, transcript order, absolute times | +| 2 | `02-relative-time-timezones.json` | `relative-time` | 8 single-event cases: exact absolute instants across `America/New_York`, `Pacific/Auckland` (NZST **and** NZDT), `UTC` | +| 3 | `03-malformed-extraction.json` | `malformed-extraction` | Hostile raw input (control chars, DEL, tabs): entry still created, **0** events persisted, raw preserved | +| 4 | `04-retry-double-submit.json` | `retry-double-submit` | Same `captureId` twice → `Created` then `IdempotentReplay`, exactly 1 entry / 1 event on the timeline | +| 5 | `05-raw-fidelity.json` | `raw-fidelity` | SHA-256 byte equality of the raw transcript across a validation failure and a retry (created entry, replay, timeline read) | +| 6 | `06-reload-persistence.json` | `reload-persistence` | Timeline identical (canonical JSON deep-equal) across a simulated cold start; events unchanged | + +### Time conventions (fixture 2 holds candidates to these) + +| Expression | Resolves to | +|---|---| +| `this morning` | 08:00 local on the capture date, in the capture zone | +| `yesterday 6pm` | 18:00 local on the calendar day before the capture date, in the capture zone (offset taken **at that instant** — DST-safe) | +| `an hour ago` | `capturedAt` − exactly 3 600 000 ms (zone-independent) | +| `just now` | `capturedAt` exactly (zone-independent) | + +Expected instants were machine-computed with the IANA tz database via +`Intl.DateTimeFormat` (offset at the resolved instant, iterative convergence). +Highlights that pin the hard cases: NZST→NZDT boundary (Sep 27 2026) shifts +"yesterday 6pm" from `+12:00` (`1789452000000`) to `+13:00` (`1790571600000`), +and the US DST end (Nov 1 2026) makes "yesterday 6pm" resolve in EST +(`1793574000000` = Nov 1 18:00 EST). The generator used to derive every +expected value is archived in the PR authoring notes. + +### What is (and isn't) pinned + +- **Pinned exactly**: event count and order, category, `occurredAt` (default + tolerance 0 — fixtures may allow slack via `toleranceMs`), quantity + (presence, value, unit), note keywords (`noteContains`, case-insensitive — + free text is matched by keywords, not exact strings), `authorId`, entry + status `draft`, raw byte-fidelity, idempotency, reload stability. +- **Not pinned**: `confidence` value (only finite within [0,1]) and + `createdAt` (only finite — candidates may stamp wall clock; every pinned + time is `occurredAt`). NLP quality beyond these fixtures is judged + qualitatively by the cross-review, not by this harness. + +## Exact commands + +No runtime dependencies — any TS runner works. Primary (matches the repo's +bun availability): + +```bash +cd evaluation +bun src/run.ts # corpus vs the worked example adapter +bun src/run.ts --adapter=./path/to/your-adapter.ts # corpus vs a candidate adapter +bun src/run.ts --adapter=./src/example/broken-adapter.ts --expect-failure +bun install && bun run typecheck # tsc --noEmit, strict +``` + +Node-flavored alternative (zero-dep harness, tsx fetches itself): + +```bash +npx tsx src/run.ts +``` + +Exit codes: `0` = all fixtures passed; `1` = at least one failure. With +`--expect-failure`, inverted (exit `0` iff the run **caught** a failure) — the +negative control depends on that. + +## Attaching a candidate adapter (cross-review) + +1. Author one module exporting `createAdapter(): CandidateAdapter` that wraps + the candidate's **entry-creation** and **timeline-read** entry points + (plus a `reload()`; a no-op is acceptable for always-fresh server reads). + Import interface types from `evaluation/src/adapter.ts` (or + `@journal/evaluation` once workspace-wired). +2. Run `bun src/run.ts --adapter=./your/adapter.ts` and attach the full output + as the candidate's evidence. The harness never imports candidate code and + candidate adapters never import corpus internals — only the interface. +3. Arena isolation: adapters must not read other candidates' branches; the + corpus is identical for all four. + +## Validation evidence (observed, this branch) + +Ran on `eval/corpus-harness` (Node's IANA tzdb via bun 1.3.14 / tsc 5.9.3): + +| Command | Result | +|---|---| +| `bun src/run.ts` (worked example adapter) | **6/6 fixtures PASS**, exit 0 — `multi-event-narrative` (3 events), `relative-time-timezones` (8 cases), `malformed-extraction` (failure path, raw preserved), `retry-double-submit` (exactly one entry, no duplicate events), `raw-fidelity` (2 cases, sha256-checked), `reload-persistence` (2 entries, cold-start) | +| `bun src/run.ts --adapter=./src/example/broken-adapter.ts --expect-failure` | **Faults caught** (exit 0): retry created a duplicate (`expected IdempotentReplay, got Created`), trimmed transcripts rejected by SHA-256 byte check on created/replayed/timeline surfaces, 0-event entries rejected on every event expectation — 5 of 6 fixtures flagged. `malformed-extraction` still passes, correctly (the control happens to satisfy that contract) | +| `bun run typecheck` | exit 0 — strict, `noUncheckedIndexedAccess`, `verbatimModuleSyntax` | + +Fixtures that execute: all six (1 + 8 + 1 + 1 + 2 + 2 = 15 adapter workloads). +Observed per-fixture results are printed by the runner; nothing was skipped. + +## Non-goals + +- Not an NLP benchmark: extraction quality beyond what these transcripts pin + is judged qualitatively at cross-review. +- Not the E2E/integration task (that proves one integrated flow end-to-end); + this judges all candidates identically on capture-pipeline semantics. +- No publish/review transitions, multi-child scoping, or authz — corpus scope + is one child's draft timeline. diff --git a/evaluation/fixtures/01-multi-event-narrative.json b/evaluation/fixtures/01-multi-event-narrative.json new file mode 100644 index 0000000..3ea6728 --- /dev/null +++ b/evaluation/fixtures/01-multi-event-narrative.json @@ -0,0 +1,14 @@ +{ + "id": "multi-event-narrative", + "kind": "multi-event-narrative", + "description": "One narrative dictation yields multiple events; count and transcript order must be preserved exactly.", + "capture": { "capturedAt": 1789563600000, "timezone": "America/New_York", "authorId": "caregiver-1" }, + "input": { "transcript": "Ava had 8 ounces of milk with breakfast at 8:00 AM. Then she went poop on the potty at 9:30 AM. Before lunch she was really happy and giggly at 12:45 PM." }, + "expected": { + "events": [ + { "category": "meal", "occurredAt": 1789560000000, "quantity": { "value": 8, "unit": "oz" }, "noteContains": ["milk"] }, + { "category": "potty", "occurredAt": 1789565400000, "noteContains": ["potty"] }, + { "category": "mood", "occurredAt": 1789577100000, "noteContains": ["giggly"] } + ] + } +} diff --git a/evaluation/fixtures/02-relative-time-timezones.json b/evaluation/fixtures/02-relative-time-timezones.json new file mode 100644 index 0000000..0233125 --- /dev/null +++ b/evaluation/fixtures/02-relative-time-timezones.json @@ -0,0 +1,78 @@ +{ + "id": "relative-time-timezones", + "kind": "relative-time", + "description": "Relative expressions must resolve to the same absolute instant regardless of capture zone, using the IANA offset at the resolved instant (correct across DST).", + "conventions": { + "this morning": "08:00 local time on the capture date, in the capture zone", + "yesterday 6pm": "18:00 local time on the calendar day before the capture date, in the capture zone (offset at that instant, so DST-safe)", + "an hour ago": "capturedAt minus exactly 3600000 ms (zone-independent)", + "just now": "capturedAt exactly (zone-independent)" + }, + "authorId": "caregiver-1", + "cases": [ + { + "id": "this-morning-et", + "captureId": "cap-rel-01", + "transcript": "She went poop on the potty this morning.", + "capturedAt": 1789563600000, + "timezone": "America/New_York", + "expected": { "category": "potty", "occurredAt": 1789560000000, "noteContains": ["potty"] } + }, + { + "id": "yesterday-6pm-et", + "captureId": "cap-rel-02", + "transcript": "She had a big bowl of pasta yesterday 6pm.", + "capturedAt": 1789563600000, + "timezone": "America/New_York", + "expected": { "category": "meal", "occurredAt": 1789509600000, "noteContains": ["pasta"] } + }, + { + "id": "yesterday-6pm-akl-nzst", + "captureId": "cap-rel-03", + "transcript": "She had a big bowl of pasta yesterday 6pm.", + "capturedAt": 1789520400000, + "timezone": "Pacific/Auckland", + "expected": { "category": "meal", "occurredAt": 1789452000000, "noteContains": ["pasta"] } + }, + { + "id": "yesterday-6pm-akl-nzdt", + "captureId": "cap-rel-04", + "transcript": "She had a big bowl of pasta yesterday 6pm.", + "capturedAt": 1790647200000, + "timezone": "Pacific/Auckland", + "expected": { "category": "meal", "occurredAt": 1790571600000, "noteContains": ["pasta"] } + }, + { + "id": "yesterday-6pm-et-est", + "captureId": "cap-rel-05", + "transcript": "She had a big bowl of pasta yesterday 6pm.", + "capturedAt": 1793671200000, + "timezone": "America/New_York", + "expected": { "category": "meal", "occurredAt": 1793574000000, "noteContains": ["pasta"] } + }, + { + "id": "an-hour-ago-utc", + "captureId": "cap-rel-06", + "transcript": "She went poop on the potty an hour ago.", + "capturedAt": 1789549200000, + "timezone": "UTC", + "expected": { "category": "potty", "occurredAt": 1789545600000, "noteContains": ["potty"] } + }, + { + "id": "just-now-utc", + "captureId": "cap-rel-07", + "transcript": "She went poop on the potty just now.", + "capturedAt": 1789549200000, + "timezone": "UTC", + "expected": { "category": "potty", "occurredAt": 1789549200000, "noteContains": ["potty"] } + }, + { + "id": "this-morning-akl-nzdt", + "captureId": "cap-rel-08", + "transcript": "She went poop on the potty this morning.", + "capturedAt": 1790647200000, + "timezone": "Pacific/Auckland", + "expected": { "category": "potty", "occurredAt": 1790622000000, "noteContains": ["potty"] } + } + ] +} diff --git a/evaluation/fixtures/03-malformed-extraction.json b/evaluation/fixtures/03-malformed-extraction.json new file mode 100644 index 0000000..42a7f37 --- /dev/null +++ b/evaluation/fixtures/03-malformed-extraction.json @@ -0,0 +1,8 @@ +{ + "id": "malformed-extraction", + "kind": "malformed-extraction", + "description": "Raw input from which no schema-valid event can be faithfully extracted. Extraction must fail without blocking capture (contract: events may be empty) and the raw transcript must be preserved byte-for-byte.", + "capture": { "capturedAt": 1789563600000, "timezone": "America/New_York", "authorId": "caregiver-1" }, + "input": { "captureId": "cap-malformed-001", "transcript": "!!! ??? ~~~ [unintelligible] \u0000\u0001\t \u007f" }, + "expected": { "entryCreated": true, "events": [], "rawPreserved": true } +} diff --git a/evaluation/fixtures/04-retry-double-submit.json b/evaluation/fixtures/04-retry-double-submit.json new file mode 100644 index 0000000..eb18f90 --- /dev/null +++ b/evaluation/fixtures/04-retry-double-submit.json @@ -0,0 +1,12 @@ +{ + "id": "retry-double-submit", + "kind": "retry-double-submit", + "description": "The same capture submitted twice (network retry / double tap) must produce exactly one persisted event set; the second submission is an idempotent replay of the first record, not a new entry.", + "capture": { "capturedAt": 1789563600000, "timezone": "America/New_York", "authorId": "caregiver-1" }, + "input": { "captureId": "cap-retry-001", "transcript": "Ava went poop on the potty at 10:15 AM." }, + "expected": { + "events": [ + { "category": "potty", "occurredAt": 1789568100000, "noteContains": ["potty"] } + ] + } +} diff --git a/evaluation/fixtures/05-raw-fidelity.json b/evaluation/fixtures/05-raw-fidelity.json new file mode 100644 index 0000000..35e8074 --- /dev/null +++ b/evaluation/fixtures/05-raw-fidelity.json @@ -0,0 +1,24 @@ +{ + "id": "raw-fidelity", + "kind": "raw-fidelity", + "description": "The raw transcript must survive a validation failure and a retry byte-for-byte (UTF-8): no normalization (NFC/NFD), no trimming, no re-encoding. Byte equality is checked via SHA-256 on the created entry, the replayed entry, and the timeline read.", + "capture": { "capturedAt": 1789563600000, "timezone": "America/New_York", "authorId": "caregiver-1" }, + "cases": [ + { + "id": "raw-after-validation-failure", + "captureId": "cap-raw-001", + "transcript": "∆∆∆ no events here \u0000\r\n trailing \t", + "expectCreated": true, + "expectedEvents": [] + }, + { + "id": "raw-across-retry", + "captureId": "cap-raw-002", + "transcript": "Ava drank 6 ounces of water at 11:30 AM. 🐧 nai\u0308ve \r\n\t ", + "expectCreated": true, + "expectedEvents": [ + { "category": "meal", "occurredAt": 1789572600000, "quantity": { "value": 6, "unit": "oz" }, "noteContains": ["water"] } + ] + } + ] +} diff --git a/evaluation/fixtures/06-reload-persistence.json b/evaluation/fixtures/06-reload-persistence.json new file mode 100644 index 0000000..36e4c50 --- /dev/null +++ b/evaluation/fixtures/06-reload-persistence.json @@ -0,0 +1,26 @@ +{ + "id": "reload-persistence", + "kind": "reload-persistence", + "description": "After a simulated cold start (adapter.reload()), the persisted timeline must be identical to what was observed before reload — no lost, duplicated, reordered, or mutated events.", + "capture": { "capturedAt": 1789563600000, "timezone": "America/New_York", "authorId": "caregiver-1" }, + "inputs": [ + { + "captureId": "cap-reload-001", + "transcript": "Ava had 4 ounces of water with her snack at 3:00 PM.", + "expected": { + "events": [ + { "category": "meal", "occurredAt": 1789585200000, "quantity": { "value": 4, "unit": "oz" }, "noteContains": ["water"] } + ] + } + }, + { + "captureId": "cap-reload-002", + "transcript": "Ava was super sleepy at bedtime at 7:30 PM.", + "expected": { + "events": [ + { "category": "sleep", "occurredAt": 1789601400000, "noteContains": ["sleepy"] } + ] + } + } + ] +} diff --git a/evaluation/package.json b/evaluation/package.json new file mode 100644 index 0000000..f6a80ed --- /dev/null +++ b/evaluation/package.json @@ -0,0 +1,17 @@ +{ + "name": "@journal/evaluation", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "Shared acceptance/evaluation corpus and candidate-agnostic harness for the Shared Child Journal arena cross-review.", + "scripts": { + "eval": "bun src/run.ts", + "test": "bun src/run.ts", + "typecheck": "tsc --noEmit", + "eval:broken": "bun src/run.ts --adapter=./src/example/broken-adapter.ts --expect-failure" + }, + "devDependencies": { + "@types/node": "^20.14.0", + "typescript": "^5.9.3" + } +} diff --git a/evaluation/src/adapter.ts b/evaluation/src/adapter.ts new file mode 100644 index 0000000..154c01e --- /dev/null +++ b/evaluation/src/adapter.ts @@ -0,0 +1,88 @@ +/** + * Candidate adapter interface — the ONLY integration surface between the + * acceptance corpus and a candidate implementation. + * + * The cross-review harness runs this interface IDENTICALLY against every + * candidate. Adapters must not import corpus internals; the corpus must not + * import candidate code. Everything a candidate owes the corpus is stated in + * the types below plus the semantic rules in evaluation/README.md. + * + * Wire shapes mirror the Effect v4 schema contract (art_I2TCG08V): + * timestamps are Unix milliseconds on the wire, `_tag` literals are required, + * and optional fields are absent — never explicitly `null`. + * One deliberate addition: `WireEntry.captureId` carries the capture + * identifier (resilient capture / duplicate-protection key), which the + * Entry contract predates. Candidates map their capture identity here. + */ + +export type EventCategory = 'potty' | 'meal' | 'sleep' | 'mood' | 'milestone' | 'school' +export type EntryStatus = 'draft' | 'published' + +/** Contract Event, wire format (`occurredAt` = Unix ms). */ +export interface WireEvent { + readonly _tag: 'Event' + readonly category: EventCategory + /** Absolute instant, Unix ms — relative expressions must already be resolved. */ + readonly occurredAt: number + readonly quantity?: { readonly value: number; readonly unit?: string } + /** Finite, within [0, 1]. 1 = caregiver-confirmed. */ + readonly confidence: number + readonly authorId: string + /** Optional free text; non-empty when present. */ + readonly note?: string +} + +/** Contract Entry, wire format (`createdAt` = Unix ms), plus `captureId`. */ +export interface WireEntry { + readonly _tag: 'Entry' + readonly captureId: string + /** Raw dictated text — must be preserved byte-for-byte, never normalized. */ + readonly transcript: string + readonly authorId: string + readonly createdAt: number + readonly status: EntryStatus + /** May be empty — extraction failure never blocks capture. */ + readonly events: readonly WireEvent[] +} + +export interface CreateEntryInput { + /** Idempotency key: duplicate submissions with the same `captureId` are replays. */ + readonly captureId: string + readonly transcript: string + readonly authorId: string + /** "Now" for relative-time resolution, Unix ms. */ + readonly capturedAt: number + /** IANA zone (e.g. "America/New_York") for relative expressions. */ + readonly timezone: string +} + +export type CreateEntryResult = + /** New entry persisted. */ + | { readonly _tag: 'Created'; readonly entry: WireEntry } + /** `captureId` already existed — the EXISTING entry is returned, nothing new persisted. */ + | { readonly _tag: 'IdempotentReplay'; readonly entry: WireEntry } + /** Capture refused for raw-input policy reasons (e.g. empty transcript). Never for extraction failure. */ + | { readonly _tag: 'Rejected'; readonly reason: string } + +export interface CandidateAdapter { + readonly name: string + /** + * Run the full capture pipeline for one dictation: preserve the raw + * transcript, extract events, resolve relative times against + * `capturedAt`/`timezone`, validate against the contract, and persist. + * Must resolve only after extraction and persistence have settled. + */ + createEntry(input: CreateEntryInput): Promise + /** + * All persisted entries (corpus scope: one child's timeline, drafts only). + * Must reflect every entry whose `createEntry` call has resolved. + */ + readTimeline(): Promise + /** + * Simulate a cold start: after this resolves, `readTimeline` must be + * equivalent to what a fresh process would read from durable state. + * A server-backed adapter whose reads are already cold-start-equivalent + * may implement this as a no-op. + */ + reload(): Promise +} diff --git a/evaluation/src/example/broken-adapter.ts b/evaluation/src/example/broken-adapter.ts new file mode 100644 index 0000000..38614b3 --- /dev/null +++ b/evaluation/src/example/broken-adapter.ts @@ -0,0 +1,67 @@ +/** + * Deliberately faulty adapter — the harness's negative control. + * + * Faults (each must be caught by a different fixture): + * 1. double-submit creates a DUPLICATE entry instead of replaying (breaks + * retry-double-submit and duplicate protection) + * 2. transcripts are trimmed before storage (breaks byte fidelity) + * + * Run with --expect-failure: `bun src/run.ts --adapter=./src/example/broken-adapter.ts --expect-failure` + * exits 0 iff the harness catches these faults. + */ +import type { CandidateAdapter, CreateEntryResult, EntryStatus, WireEvent } from '../adapter.ts' + +interface StoreEntry { + readonly _tag: 'Entry' + readonly captureId: string + readonly transcript: string // BUG (fault 2): trimmed before storage + readonly authorId: string + readonly createdAt: number + readonly status: EntryStatus + readonly events: readonly WireEvent[] +} + +export function createAdapter(): CandidateAdapter { + const store = new Map() + + return { + name: 'broken (deliberate negative control — must fail the corpus)', + + async createEntry(input): Promise { + const existing = store.get(input.captureId) + if (existing !== undefined) { + // BUG (fault 1): a retry persists a second entry instead of replaying. + const dupe = { + _tag: 'Entry' as const, + captureId: `${input.captureId}#${store.size + 1}`, + transcript: input.transcript.trim(), + authorId: input.authorId, + createdAt: input.capturedAt, + status: 'draft' as const, + events: existing.events, + } + store.set(dupe.captureId, dupe) + return { _tag: 'Created', entry: dupe } + } + const entry = { + _tag: 'Entry' as const, + captureId: input.captureId, + transcript: input.transcript.trim(), + authorId: input.authorId, + createdAt: input.capturedAt, + status: 'draft' as const, + events: [], + } + store.set(input.captureId, entry) + return { _tag: 'Created', entry } + }, + + async readTimeline() { + return [...store.values()] + }, + + async reload(): Promise { + // Reads are already cold-start equivalent (single Map = "durable" store). + }, + } +} diff --git a/evaluation/src/example/example-adapter.ts b/evaluation/src/example/example-adapter.ts new file mode 100644 index 0000000..384afb3 --- /dev/null +++ b/evaluation/src/example/example-adapter.ts @@ -0,0 +1,224 @@ +/** + * Worked example adapter — a complete, runnable implementation of the + * CandidateAdapter interface over the wire shapes of the Effect v4 schema + * contract (art_I2TCG08V). + * + * What it demonstrates: + * - capture pipeline shape: preserve raw transcript -> extract -> validate + * against the contract -> persist -> dedupe by captureId + * - relative-time resolution against `capturedAt` + IANA `timezone` + * (offset taken at the resolved instant, correct across DST) + * - a cold-start `reload()` that rebuilds the read index from durable state + * + * What it is NOT: a reference extractor. The rule table below is deliberately + * minimal pattern matching, fixed in code, and independent of fixture + * expectation data — it exists so the corpus has a known-good adapter to + * validate the harness itself. Candidates bring their own extraction. + * + * The in-file validator mirrors the contract tables; per contract rule 1, + * swap to the shared `packages/domain` schema import once it lands — the + * shapes are compatible by design. + */ +import type { + CandidateAdapter, + CreateEntryInput, + CreateEntryResult, + EventCategory, + WireEntry, + WireEvent, +} from '../adapter.ts' + +const CATEGORIES: readonly EventCategory[] = ['potty', 'meal', 'sleep', 'mood', 'milestone', 'school'] + +/** Mirror of the contract's Event table (art_I2TCG08V) used to gate persistence. */ +function contractViolations(event: WireEvent): readonly string[] { + const bad: string[] = [] + if (event._tag !== 'Event') bad.push('_tag must be "Event"') + if (!CATEGORIES.includes(event.category)) bad.push(`unknown category "${event.category}"`) + if (!Number.isFinite(event.occurredAt)) bad.push('occurredAt must be a finite number') + if (!Number.isFinite(event.confidence) || event.confidence < 0 || event.confidence > 1) { + bad.push('confidence must be finite within [0,1]') + } + if (typeof event.authorId !== 'string' || event.authorId.length === 0) bad.push('authorId must be non-empty') + if (event.quantity !== undefined) { + if (event.quantity === null || typeof event.quantity.value !== 'number' || !Number.isFinite(event.quantity.value)) { + bad.push('quantity must be an object with a finite numeric value (explicit null is not in the contract)') + } else if (event.quantity.unit !== undefined && (typeof event.quantity.unit !== 'string' || event.quantity.unit.length === 0)) { + bad.push('quantity.unit must be a non-empty string when present') + } + } + if (event.note !== undefined && (typeof event.note !== 'string' || event.note.length === 0)) { + bad.push('note must be non-empty when present') + } + return bad +} + +// --------------------------------------------------------------------------- +// Time resolution (IANA-correct; offset taken at the target instant). +// --------------------------------------------------------------------------- + +function zoneOffsetMs(zone: string, utcMs: number): number { + const parts = new Intl.DateTimeFormat('en-US', { + timeZone: zone, hour12: false, + year: 'numeric', month: '2-digit', day: '2-digit', + hour: '2-digit', minute: '2-digit', second: '2-digit', + }).formatToParts(new Date(utcMs)) + const get = (type: string): number => { + const part = parts.find((p) => p.type === type) + if (part === undefined) throw new Error(`Intl returned no "${type}" part for zone "${zone}"`) + return Number(part.value) + } + const asUTC = Date.UTC(get('year'), get('month') - 1, get('day'), get('hour') % 24, get('minute'), get('second')) + return asUTC - (utcMs - (utcMs % 1000)) +} + +/** Wall-clock time in `zone` -> Unix ms. Iterates until the offset at the target instant is stable. */ +function wallToUtc(zone: string, year: number, month: number, day: number, hour: number, minute: number): number { + const guess = Date.UTC(year, month - 1, day, hour, minute, 0) + let utc = guess - zoneOffsetMs(zone, guess) + for (let i = 0; i < 3; i++) { + const next = guess - zoneOffsetMs(zone, utc) + if (next === utc) return utc + utc = next + } + return utc +} + +/** The capture's local (wall) calendar date in `zone`. */ +function captureWallDate(zone: string, capturedAt: number): { year: number; month: number; day: number } { + const parts = new Intl.DateTimeFormat('en-US', { + timeZone: zone, year: 'numeric', month: '2-digit', day: '2-digit', + }).formatToParts(new Date(capturedAt)) + const get = (type: string): number => { + const part = parts.find((p) => p.type === type) + if (part === undefined) throw new Error(`Intl returned no "${type}" part for zone "${zone}"`) + return Number(part.value) + } + return { year: get('year'), month: get('month'), day: get('day') } +} + +function to24h(hour: number, minute: number, meridiem: string): { hour: number; minute: number } { + const ampm = meridiem.toLowerCase() + if (ampm === 'pm' && hour !== 12) return { hour: hour + 12, minute } + if (ampm === 'am' && hour === 12) return { hour: 0, minute } + return { hour, minute } +} + +// --------------------------------------------------------------------------- +// Minimal deterministic extraction (example only — see file header). +// --------------------------------------------------------------------------- + +const CATEGORY_RULES: readonly { readonly category: EventCategory; readonly pattern: RegExp }[] = [ + { category: 'potty', pattern: /\bpoo(?:p|ped|ping)\b|\bpotty\b/i }, + { category: 'sleep', pattern: /\b(?:woke up|nap(?:ped)?|bedtime|sleepy|asleep|sleep)\b/i }, + { category: 'mood', pattern: /\b(?:happy|giggly|grumpy|sad|fussy|cheerful)\b/i }, + { category: 'meal', pattern: /\b(?:milk|pasta|water|breakfast|lunch|dinner|snack|ate|drank|had)\b/i }, +] + +function extractEvents(transcript: string, input: CreateEntryInput): { events: WireEvent[]; dropped: readonly string[] } { + const events: WireEvent[] = [] + const dropped: string[] = [] + const sentences = transcript.split(/(?<=[.!?])\s+/) + for (const rawSentence of sentences) { + const sentence = rawSentence.trim() + if (sentence.length === 0) continue + const rule = CATEGORY_RULES.find((r) => r.pattern.test(sentence)) + if (rule === undefined) continue + + const occurredAt = resolveOccurredAt(sentence, input) + if (occurredAt === undefined) continue // no resolvable time — the example adapter skips + + const quantityMatch = /(\d+(?:\.\d+)?)\s+ounces? of/i.exec(sentence) + const event: WireEvent = { + _tag: 'Event', + category: rule.category, + occurredAt, + quantity: quantityMatch === null ? undefined : { value: Number(quantityMatch[1]), unit: 'oz' }, + confidence: 0.9, // deterministic guess — not caregiver-confirmed + authorId: input.authorId, + note: sentence, + } + const violations = contractViolations(event) + if (violations.length > 0) { + dropped.push(`${violations.join('; ')} — dropped (extraction failure never blocks capture)`) + continue + } + events.push(event) + } + return { events, dropped } +} + +/** Resolves the sentence's time expression to an absolute Unix-ms instant. */ +function resolveOccurredAt(sentence: string, input: CreateEntryInput): number | undefined { + const wall = captureWallDate(input.timezone, input.capturedAt) + + const yesterday = /\byesterday\s+(\d{1,2})(?::(\d{2}))?\s*(am|pm)\b/i.exec(sentence) + if (yesterday !== null) { + const { hour, minute } = to24h(Number(yesterday[1]), Number(yesterday[2] ?? '0'), yesterday[3] ?? 'am') + return wallToUtc(input.timezone, wall.year, wall.month, wall.day - 1, hour, minute) // Date.UTC normalizes day 0 + } + if (/\bthis morning\b/i.test(sentence)) { + // Convention: 08:00 local time on the capture date. + return wallToUtc(input.timezone, wall.year, wall.month, wall.day, 8, 0) + } + if (/\ban hour ago\b/i.test(sentence)) return input.capturedAt - 3600000 + if (/\bjust now\b/i.test(sentence)) return input.capturedAt + + const clock = /\bat\s+(\d{1,2})(?::(\d{2}))?\s*(am|pm)\b/i.exec(sentence) + if (clock !== null) { + const { hour, minute } = to24h(Number(clock[1]), Number(clock[2] ?? '0'), clock[3] ?? 'am') + // Convention: explicit AM/PM times are on the capture's local date. + return wallToUtc(input.timezone, wall.year, wall.month, wall.day, hour, minute) + } + return undefined +} + +// --------------------------------------------------------------------------- +// Adapter: append-only log (durable) + rebuildable index (hot cache). +// --------------------------------------------------------------------------- + +export function createAdapter(): CandidateAdapter { + const log: WireEntry[] = [] + let index = new Map() + + function persist(entry: WireEntry): void { + log.push(entry) + index.set(entry.captureId, entry) + } + + return { + name: 'example (worked example over the schema contract)', + + async createEntry(input: CreateEntryInput): Promise { + const existing = index.get(input.captureId) + if (existing !== undefined) return { _tag: 'IdempotentReplay', entry: existing } + + const { events, dropped } = extractEvents(input.transcript, input) + if (dropped.length > 0) { + // Contract: extraction failure never blocks capture — the entry is + // created, invalid candidates are dropped, the raw input is preserved. + console.error(`[example-adapter] capture ${input.captureId}: ${dropped.length} extracted event(s) failed contract validation and were dropped`) + } + const entry: WireEntry = { + _tag: 'Entry', + captureId: input.captureId, + transcript: input.transcript, // verbatim — never trimmed, normalized, or re-encoded + authorId: input.authorId, + createdAt: input.capturedAt, + status: 'draft', + events, + } + persist(entry) + return { _tag: 'Created', entry } + }, + + async readTimeline(): Promise { + return [...index.values()] + }, + + async reload(): Promise { + // Cold start: discard the hot cache and rebuild it from durable state. + index = new Map(log.map((entry) => [entry.captureId, entry])) + }, + } +} diff --git a/evaluation/src/fixtures.ts b/evaluation/src/fixtures.ts new file mode 100644 index 0000000..d6b339e --- /dev/null +++ b/evaluation/src/fixtures.ts @@ -0,0 +1,171 @@ +/** + * Fixture (corpus) data types and loader. + * + * Fixtures are DATA: inputs plus expected outcomes, independent of any + * candidate code. The runner interprets each `kind` with a fixed protocol; + * everything expected of a candidate lives in the `expected` blocks. + */ +import type { EventCategory } from './adapter.ts' + +export interface QuantityExpectation { + readonly value: number + readonly unit?: string +} + +export interface EventExpectation { + readonly category: EventCategory + /** Exact absolute instant, Unix ms (see `toleranceMs`). */ + readonly occurredAt: number + /** Expected quantity; when absent the event must NOT carry one (null is a contract violation). */ + readonly quantity?: QuantityExpectation + /** Every needle must appear in `note` (case-insensitive, whitespace-collapsed). */ + readonly noteContains?: readonly string[] + /** Allowed slack around `occurredAt` in ms. Default 0 (exact). */ + readonly toleranceMs?: number +} + +export interface CaptureContext { + /** "Now" for relative-time resolution, Unix ms. */ + readonly capturedAt: number + readonly timezone: string + readonly authorId: string +} + +export interface MultiEventFixture { + readonly kind: 'multi-event-narrative' + readonly id: string + readonly description: string + readonly capture: CaptureContext + readonly input: { readonly transcript: string } + /** Events in transcript order; count and order are asserted exactly. */ + readonly expected: { readonly events: readonly EventExpectation[] } +} + +export interface RelativeTimeCase { + readonly id: string + readonly captureId: string + readonly transcript: string + readonly capturedAt: number + readonly timezone: string + readonly expected: EventExpectation +} + +export interface RelativeTimeFixture { + readonly kind: 'relative-time' + readonly id: string + readonly description: string + /** Resolution conventions the corpus holds candidates to, keyed by expression. */ + readonly conventions: Readonly> + readonly authorId: string + readonly cases: readonly RelativeTimeCase[] +} + +export interface MalformedFixture { + readonly kind: 'malformed-extraction' + readonly id: string + readonly description: string + readonly capture: CaptureContext + readonly input: { readonly captureId: string; readonly transcript: string } + readonly expected: { + readonly entryCreated: true + readonly events: readonly [] + readonly rawPreserved: true + } +} + +export interface RetryDoubleSubmitFixture { + readonly kind: 'retry-double-submit' + readonly id: string + readonly description: string + readonly capture: CaptureContext + readonly input: { readonly captureId: string; readonly transcript: string } + readonly expected: { readonly events: readonly EventExpectation[] } +} + +export interface RawFidelityCase { + readonly id: string + readonly captureId: string + readonly transcript: string + /** Whether the capture must be accepted (corpus always true: capture is never blocked). */ + readonly expectCreated: boolean + readonly expectedEvents: readonly EventExpectation[] +} + +export interface RawFidelityFixture { + readonly kind: 'raw-fidelity' + readonly id: string + readonly description: string + readonly capture: CaptureContext + readonly cases: readonly RawFidelityCase[] +} + +export interface ReloadInput { + readonly captureId: string + readonly transcript: string + readonly expected: { readonly events: readonly EventExpectation[] } +} + +export interface ReloadFixture { + readonly kind: 'reload-persistence' + readonly id: string + readonly description: string + readonly capture: CaptureContext + readonly inputs: readonly ReloadInput[] +} + +export type Fixture = + | MultiEventFixture + | RelativeTimeFixture + | MalformedFixture + | RetryDoubleSubmitFixture + | RawFidelityFixture + | ReloadFixture + +const KNOWN_KINDS = [ + 'multi-event-narrative', + 'relative-time', + 'malformed-extraction', + 'retry-double-submit', + 'raw-fidelity', + 'reload-persistence', +] as const + +function assertFixture(value: unknown, file: string): Fixture { + if (typeof value !== 'object' || value === null) { + throw new Error(`${file}: fixture must be a JSON object`) + } + const f = value as { kind?: unknown; id?: unknown } + if (typeof f.id !== 'string' || f.id.length === 0) { + throw new Error(`${file}: fixture is missing a non-empty "id"`) + } + if (typeof f.kind !== 'string' || !(KNOWN_KINDS as readonly string[]).includes(f.kind)) { + throw new Error(`${file}: unknown fixture kind ${JSON.stringify(f.kind)} (known: ${KNOWN_KINDS.join(', ')})`) + } + return value as Fixture +} + +/** Load every *.json fixture from a directory, sorted by filename. Fails fast — never skips. */ +export async function loadFixtures(fixturesDir: URL): Promise { + const { readdir } = await import('node:fs/promises') + const { fileURLToPath } = await import('node:url') + const { join } = await import('node:path') + + const dir = fileURLToPath(fixturesDir) + const files = (await readdir(dir)).filter((f) => f.endsWith('.json')).sort() + if (files.length === 0) { + throw new Error(`no fixture files found in ${dir}`) + } + const fixtures: Fixture[] = [] + for (const file of files) { + const path = join(dir, file) + const raw = await import('node:fs/promises').then((fs) => fs.readFile(path, 'utf8')) + let parsed: unknown + try { + parsed = JSON.parse(raw) + } catch (err) { + throw new Error(`${file}: invalid JSON — ${err instanceof Error ? err.message : String(err)}`) + } + fixtures.push(assertFixture(parsed, file)) + } + return fixtures +} diff --git a/evaluation/src/index.ts b/evaluation/src/index.ts new file mode 100644 index 0000000..da6d5f1 --- /dev/null +++ b/evaluation/src/index.ts @@ -0,0 +1,15 @@ +/** + * Programmatic surface for cross-review adapters and CI wiring: + * adapter types, corpus loading, and the runner — without the CLI. + */ +export type { + CandidateAdapter, + CreateEntryInput, + CreateEntryResult, + EntryStatus, + EventCategory, + WireEntry, + WireEvent, +} from './adapter.ts' +export { loadFixtures, type Fixture } from './fixtures.ts' +export { runCorpus, type FixtureResult, type RunSummary } from './runner.ts' diff --git a/evaluation/src/match.ts b/evaluation/src/match.ts new file mode 100644 index 0000000..a279227 --- /dev/null +++ b/evaluation/src/match.ts @@ -0,0 +1,105 @@ +/** + * Comparison helpers: canonical JSON, byte fidelity (SHA-256 over UTF-8), + * and the single event matcher the corpus compares candidates with. + */ +import { createHash } from 'node:crypto' +import type { EventExpectation } from './fixtures.ts' +import type { WireEvent } from './adapter.ts' + +function sortedValue(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortedValue) + if (value !== null && typeof value === 'object') { + const source = value as Record + const out: Record = {} + for (const key of Object.keys(source).sort()) { + const v = source[key] + if (v !== undefined) out[key] = sortedValue(v) // undefined ≡ absent key + } + return out + } + return value +} + +/** Deterministic serialization: object keys sorted, arrays order-sensitive. */ +export function canonicalJson(value: unknown): string { + return JSON.stringify(sortedValue(value)) +} + +export function deepEqual(a: unknown, b: unknown): boolean { + return canonicalJson(a) === canonicalJson(b) +} + +/** Byte fidelity: two strings are byte-equal iff their UTF-8 digests match. */ +export function sha256Hex(input: string): string { + return createHash('sha256').update(Buffer.from(input, 'utf8')).digest('hex') +} + +/** Case-insensitive, whitespace-collapsed comparison form for note needles. */ +export function normalizeNote(text: string): string { + return text.toLowerCase().replace(/\s+/g, ' ') +} + +/** + * Compare one observed event against one expectation. Returns mismatch + * descriptions (empty array = match). Contract-shape violations + * (wrong `_tag`, null optionals, out-of-range confidence) are mismatches too. + */ +export function eventMismatches( + observed: unknown, + expected: EventExpectation, + authorId: string, +): readonly string[] { + const bad: string[] = [] + if (typeof observed !== 'object' || observed === null) { + return [`event is not an object: ${canonicalJson(observed)}`] + } + const e = observed as Partial & { quantity?: unknown; note?: unknown; confidence?: unknown } + + if (e._tag !== 'Event') bad.push(`_tag must be "Event", got ${JSON.stringify(e._tag)}`) + if (e.category !== expected.category) { + bad.push(`category: expected "${expected.category}", got ${JSON.stringify(e.category)}`) + } + if (typeof e.occurredAt !== 'number' || !Number.isFinite(e.occurredAt)) { + bad.push(`occurredAt must be a finite Unix-ms number, got ${JSON.stringify(e.occurredAt)}`) + } else { + const tolerance = expected.toleranceMs ?? 0 + const drift = Math.abs(e.occurredAt - expected.occurredAt) + if (drift > tolerance) { + bad.push( + `occurredAt: expected ${expected.occurredAt} (${new Date(expected.occurredAt).toISOString()}), ` + + `got ${e.occurredAt} (${new Date(e.occurredAt).toISOString()}), drift ${drift}ms > tolerance ${tolerance}ms`, + ) + } + } + + const q = e.quantity + if (expected.quantity === undefined) { + if (q === null) bad.push('quantity: explicit null is not in the contract — omit the field instead') + else if (q !== undefined) bad.push(`quantity must be absent, got ${canonicalJson(q)}`) + } else { + if (q === null || typeof q !== 'object' || typeof (q as { value?: unknown }).value !== 'number') { + bad.push(`quantity: expected ${canonicalJson(expected.quantity)}, got ${canonicalJson(q ?? null)}`) + } else if (!deepEqual(q, expected.quantity)) { + bad.push(`quantity: expected ${canonicalJson(expected.quantity)}, got ${canonicalJson(q)}`) + } + } + + if (typeof e.confidence !== 'number' || !Number.isFinite(e.confidence) || e.confidence < 0 || e.confidence > 1) { + bad.push(`confidence must be finite within [0,1], got ${JSON.stringify(e.confidence)}`) + } + if (e.authorId !== authorId) { + bad.push(`authorId: expected "${authorId}", got ${JSON.stringify(e.authorId)}`) + } + if (e.note !== undefined && (typeof e.note !== 'string' || e.note.length === 0)) { + bad.push('note must be a non-empty string when present (empty string is a contract violation)') + } + if (expected.noteContains !== undefined) { + const note = typeof e.note === 'string' ? normalizeNote(e.note) : '' + for (const needle of expected.noteContains) { + if (!note.includes(normalizeNote(needle))) { + bad.push(`note: expected it to contain "${needle}" (case-insensitive), got ${JSON.stringify(e.note ?? null)}`) + } + } + } + return bad +} diff --git a/evaluation/src/run.ts b/evaluation/src/run.ts new file mode 100644 index 0000000..4dd5d22 --- /dev/null +++ b/evaluation/src/run.ts @@ -0,0 +1,57 @@ +/** + * CLI entry: `bun src/run.ts [--adapter=] [--expect-failure]` + * + * --adapter= Module (relative to the evaluation package root or + * absolute) exporting `createAdapter(): CandidateAdapter`. + * Defaults to the worked example adapter. + * --expect-failure Invert the exit code: exit 0 iff the run found failures + * (used by the broken-adapter negative control). + */ +import type { CandidateAdapter } from './adapter.ts' +import { loadFixtures } from './fixtures.ts' +import { runCorpus } from './runner.ts' +import { createAdapter as createExampleAdapter } from './example/example-adapter.ts' + +async function loadAdapter(adapterArg: string | undefined): Promise { + if (adapterArg === undefined) return createExampleAdapter() + + const { pathToFileURL, fileURLToPath } = await import('node:url') + const { resolve } = await import('node:path') + const packageRoot = fileURLToPath(new URL('..', import.meta.url)) + const modulePath = pathToFileURL(resolve(packageRoot, adapterArg)).href + const mod = (await import(modulePath)) as { createAdapter?: unknown } + if (typeof mod.createAdapter !== 'function') { + throw new Error(`adapter module ${adapterArg} must export a named factory: createAdapter(): CandidateAdapter`) + } + const adapter = (mod.createAdapter as () => CandidateAdapter)() + if (typeof adapter?.name !== 'string' || adapter.name.length === 0) { + throw new Error(`adapter factory in ${adapterArg} returned an object without a non-empty "name"`) + } + return adapter +} + +async function main(): Promise { + const args = process.argv.slice(2) + const adapterArg = args.find((a) => a.startsWith('--adapter='))?.slice('--adapter='.length) + const expectFailure = args.includes('--expect-failure') + + const adapter = await loadAdapter(adapterArg) + const fixtures = await loadFixtures(new URL('../fixtures/', import.meta.url)) + const summary = await runCorpus(adapter, fixtures) + + console.log(`\nAcceptance corpus — adapter: ${summary.adapterName}`) + console.log('='.repeat(72)) + for (const result of summary.results) { + console.log(` ${result.ok ? 'PASS' : 'FAIL'} ${result.fixtureId}${result.note ? ` (${result.note})` : ''}`) + for (const failure of result.failures) { + console.log(` - ${failure}`) + } + } + console.log('='.repeat(72)) + console.log(`Summary: ${summary.passed}/${summary.results.length} fixtures passed`) + + if (expectFailure) return summary.ok ? 1 : 0 // exit 0 iff the harness caught at least one failure + return summary.ok ? 0 : 1 +} + +process.exitCode = await main() diff --git a/evaluation/src/runner.ts b/evaluation/src/runner.ts new file mode 100644 index 0000000..1ae9842 --- /dev/null +++ b/evaluation/src/runner.ts @@ -0,0 +1,284 @@ +/** + * The harness core: runs each fixture kind against a CandidateAdapter with a + * fixed protocol. The protocol is identical for every candidate; all + * expectations come from fixture data. Adapter exceptions are recorded as + * fixture failures, never swallowed and never fatal to the run. + */ +import type { CandidateAdapter, CreateEntryInput, WireEntry } from './adapter.ts' +import type { CaptureContext, EventExpectation, Fixture } from './fixtures.ts' +import { canonicalJson, deepEqual, eventMismatches, sha256Hex } from './match.ts' + +export interface FixtureResult { + readonly fixtureId: string + readonly ok: boolean + readonly failures: readonly string[] + /** Human-readable context for the summary table (e.g. case counts). */ + readonly note?: string +} + +export interface RunSummary { + readonly adapterName: string + readonly results: readonly FixtureResult[] + readonly ok: boolean + readonly passed: number + readonly failed: number +} + +function toInput( + capture: CaptureContext, + captureId: string, + transcript: string, +): CreateEntryInput { + return { + captureId, + transcript, + authorId: capture.authorId, + capturedAt: capture.capturedAt, + timezone: capture.timezone, + } +} + +/** Contract-level checks applied to every entry an adapter hands back. */ +function entryBasicsMismatches(entry: WireEntry, input: CreateEntryInput): readonly string[] { + const bad: string[] = [] + if (entry._tag !== 'Entry') bad.push(`_tag must be "Entry", got ${JSON.stringify(entry._tag)}`) + if (entry.captureId !== input.captureId) { + bad.push(`captureId: expected "${input.captureId}", got ${JSON.stringify(entry.captureId)}`) + } + if (entry.status !== 'draft') { + bad.push(`status right after creation must be "draft" (review/publish happens later), got ${JSON.stringify(entry.status)}`) + } + if (entry.authorId !== input.authorId) { + bad.push(`entry.authorId: expected "${input.authorId}", got ${JSON.stringify(entry.authorId)}`) + } + if (typeof entry.createdAt !== 'number' || !Number.isFinite(entry.createdAt)) { + bad.push(`createdAt must be a finite Unix-ms number, got ${JSON.stringify(entry.createdAt)}`) + } + return bad +} + +/** Exact count + index-by-index comparison (transcript order). */ +function checkEvents( + observed: readonly unknown[], + expected: readonly EventExpectation[], + authorId: string, + label: string, +): readonly string[] { + const bad: string[] = [] + if (observed.length !== expected.length) { + bad.push(`${label}: expected ${expected.length} event(s), got ${observed.length}`) + } + const pairs = expected.map((exp, i) => [i, exp, observed[i]] as const) + for (const [i, exp, obs] of pairs) { + if (obs === undefined) break // count mismatch already reported + for (const m of eventMismatches(obs, exp, authorId)) { + bad.push(`${label}: event[${i}] — ${m}`) + } + } + return bad +} + +function rawNotPreserved(input: string, stored: string): string | undefined { + return input === stored ? undefined : 'raw transcript is not byte-equal to the input — transcripts must be preserved verbatim (no trim/normalize/re-encode)' +} + +async function runMultiEvent(adapter: CandidateAdapter, fixture: Extract): Promise { + const failures: string[] = [] + const input = toInput(fixture.capture, `cap-${fixture.id}`, fixture.input.transcript) + const result = await adapter.createEntry(input) + if (result._tag !== 'Created') { + failures.push(`expected Created, got ${result._tag}${result._tag === 'Rejected' ? `: "${result.reason}"` : ''}`) + } else { + failures.push(...entryBasicsMismatches(result.entry, input)) + const rawIssue = rawNotPreserved(input.transcript, result.entry.transcript) + if (rawIssue) failures.push(rawIssue) + failures.push(...checkEvents(result.entry.events, fixture.expected.events, input.authorId, fixture.id)) + } + return { fixtureId: fixture.id, ok: failures.length === 0, failures, note: `${fixture.expected.events.length} events` } +} + +async function runRelativeTime(adapter: CandidateAdapter, fixture: Extract): Promise { + const failures: string[] = [] + for (const c of fixture.cases) { + const input: CreateEntryInput = { + captureId: c.captureId, + transcript: c.transcript, + authorId: fixture.authorId, + capturedAt: c.capturedAt, + timezone: c.timezone, + } + const result = await adapter.createEntry(input) + if (result._tag !== 'Created') { + failures.push(`[${c.id}] expected Created, got ${result._tag}${result._tag === 'Rejected' ? `: "${result.reason}"` : ''}`) + continue + } + failures.push(...entryBasicsMismatches(result.entry, input).map((m) => `[${c.id}] ${m}`)) + const rawIssue = rawNotPreserved(c.transcript, result.entry.transcript) + if (rawIssue) failures.push(`[${c.id}] ${rawIssue}`) + if (result.entry.events.length !== 1) { + failures.push(`[${c.id}] expected exactly 1 event, got ${result.entry.events.length}`) + } else { + failures.push(...eventMismatches(result.entry.events[0], c.expected, input.authorId).map((m) => `[${c.id}] ${m}`)) + } + } + return { fixtureId: fixture.id, ok: failures.length === 0, failures, note: `${fixture.cases.length} cases` } +} + +async function runMalformed(adapter: CandidateAdapter, fixture: Extract): Promise { + const failures: string[] = [] + const input = toInput(fixture.capture, fixture.input.captureId, fixture.input.transcript) + const result = await adapter.createEntry(input) + // Contract (art_I2TCG08V): "May be empty — extraction failure never blocks capture." + if (result._tag === 'Rejected') { + failures.push(`capture was Rejected ("${result.reason}") but the contract forbids blocking capture on extraction failure`) + } else { + const entry = result.entry + failures.push(...entryBasicsMismatches(entry, input)) + if (entry.events.length !== fixture.expected.events.length) { + failures.push(`expected ${fixture.expected.events.length} persisted events (extraction must fail without persisting anything), got ${entry.events.length}`) + } + const rawIssue = rawNotPreserved(fixture.input.transcript, entry.transcript) + if (rawIssue) failures.push(rawIssue) + } + return { fixtureId: fixture.id, ok: failures.length === 0, failures, note: 'failure path, raw preserved' } +} + +async function runRetryDoubleSubmit(adapter: CandidateAdapter, fixture: Extract): Promise { + const failures: string[] = [] + const input = toInput(fixture.capture, fixture.input.captureId, fixture.input.transcript) + + const first = await adapter.createEntry(input) + if (first._tag !== 'Created') { + failures.push(`first submission: expected Created, got ${first._tag}${first._tag === 'Rejected' ? `: "${first.reason}"` : ''}`) + } + const second = await adapter.createEntry(input) // identical retry + if (second._tag !== 'IdempotentReplay') { + failures.push(`second submission (same captureId): expected IdempotentReplay, got ${second._tag}`) + } + if (first._tag === 'Created' && second._tag === 'IdempotentReplay') { + if (!deepEqual(first.entry, second.entry)) { + failures.push('replayed entry differs from the originally created entry (must return the existing record)') + } + failures.push(...entryBasicsMismatches(first.entry, input)) + failures.push(...checkEvents(first.entry.events, fixture.expected.events, input.authorId, fixture.id)) + const rawIssue = rawNotPreserved(input.transcript, first.entry.transcript) + if (rawIssue) failures.push(rawIssue) + } + + const timeline = await adapter.readTimeline() + const forCapture = timeline.filter((e) => e.captureId === fixture.input.captureId) + if (forCapture.length !== 1) { + failures.push(`timeline: expected exactly 1 entry for captureId "${fixture.input.captureId}", got ${forCapture.length} — duplicate protection failed`) + } else { + const storedEntry = forCapture[0] + if (storedEntry !== undefined) { + failures.push(...checkEvents(storedEntry.events, fixture.expected.events, input.authorId, `${fixture.id}/timeline`)) + } + } + return { fixtureId: fixture.id, ok: failures.length === 0, failures, note: 'exactly one entry, no duplicate events' } +} + +async function runRawFidelity(adapter: CandidateAdapter, fixture: Extract): Promise { + const failures: string[] = [] + for (const c of fixture.cases) { + const input = toInput(fixture.capture, c.captureId, c.transcript) + const first = await adapter.createEntry(input) + if (c.expectCreated && first._tag !== 'Created') { + failures.push(`[${c.id}] expected Created, got ${first._tag}${first._tag === 'Rejected' ? `: "${first.reason}"` : ''}`) + continue + } + if (!c.expectCreated && first._tag !== 'Rejected') { + failures.push(`[${c.id}] expected Rejected, got ${first._tag}`) + continue + } + if (first._tag === 'Created') { + failures.push(...checkEvents(first.entry.events, c.expectedEvents, input.authorId, `[${c.id}]`)) + const shaInput = sha256Hex(c.transcript) + const shaCreated = sha256Hex(first.entry.transcript) + if (shaInput !== shaCreated) failures.push(`[${c.id}] created entry transcript is not byte-equal to the input (sha256 ${shaInput} vs ${shaCreated})`) + } + const second = await adapter.createEntry(input) // retry after the failure + if (second._tag !== 'IdempotentReplay') { + failures.push(`[${c.id}] retry after failure: expected IdempotentReplay, got ${second._tag}`) + } else if (sha256Hex(c.transcript) !== sha256Hex(second.entry.transcript)) { + failures.push(`[${c.id}] replayed entry transcript is not byte-equal to the input`) + } + const stored = (await adapter.readTimeline()).find((e) => e.captureId === c.captureId) + if (stored === undefined) { + failures.push(`[${c.id}] timeline has no entry for captureId "${c.captureId}"`) + } else if (sha256Hex(c.transcript) !== sha256Hex(stored.transcript)) { + failures.push(`[${c.id}] timeline-read transcript is not byte-equal to the input`) + } + } + return { fixtureId: fixture.id, ok: failures.length === 0, failures, note: `${fixture.cases.length} cases, sha256-checked` } +} + +function sortEntries(entries: readonly WireEntry[]): readonly WireEntry[] { + return [...entries].sort((a, b) => a.createdAt - b.createdAt || (a.captureId < b.captureId ? -1 : a.captureId > b.captureId ? 1 : 0)) +} + +async function runReload(adapter: CandidateAdapter, fixture: Extract): Promise { + const failures: string[] = [] + for (const input of fixture.inputs) { + const entryInput = toInput(fixture.capture, input.captureId, input.transcript) + const result = await adapter.createEntry(entryInput) + if (result._tag !== 'Created') { + failures.push(`[${input.captureId}] expected Created, got ${result._tag}${result._tag === 'Rejected' ? `: "${result.reason}"` : ''}`) + } else { + failures.push(...checkEvents(result.entry.events, input.expected.events, entryInput.authorId, `[${input.captureId}]`)) + const rawIssue = rawNotPreserved(input.transcript, result.entry.transcript) + if (rawIssue) failures.push(`[${input.captureId}] ${rawIssue}`) + } + } + + const before = await adapter.readTimeline() + await adapter.reload() + const after = await adapter.readTimeline() + + const beforeCanon = canonicalJson(sortEntries(before)) + const afterCanon = canonicalJson(sortEntries(after)) + if (beforeCanon !== afterCanon) { + const preview = (s: string) => (s.length > 240 ? `${s.slice(0, 240)}…` : s) + failures.push(`timeline changed across reload — before: ${preview(beforeCanon)} / after: ${preview(afterCanon)}`) + } + for (const input of fixture.inputs) { + const entry = after.find((e) => e.captureId === input.captureId) + if (entry === undefined) { + failures.push(`[${input.captureId}] entry lost after reload`) + } else { + failures.push(...checkEvents(entry.events, input.expected.events, fixture.capture.authorId, `[${input.captureId}]/after-reload`)) + } + } + return { fixtureId: fixture.id, ok: failures.length === 0, failures, note: `${fixture.inputs.length} entries, cold-start` } +} + +async function runOne(adapter: CandidateAdapter, fixture: Fixture): Promise { + try { + switch (fixture.kind) { + case 'multi-event-narrative': + return await runMultiEvent(adapter, fixture) + case 'relative-time': + return await runRelativeTime(adapter, fixture) + case 'malformed-extraction': + return await runMalformed(adapter, fixture) + case 'retry-double-submit': + return await runRetryDoubleSubmit(adapter, fixture) + case 'raw-fidelity': + return await runRawFidelity(adapter, fixture) + case 'reload-persistence': + return await runReload(adapter, fixture) + } + } catch (err) { + const message = err instanceof Error ? `${err.name}: ${err.message}` : String(err) + return { fixtureId: fixture.id, ok: false, failures: [`adapter threw an unexpected error — ${message}`] } + } +} + +export async function runCorpus(adapter: CandidateAdapter, fixtures: readonly Fixture[]): Promise { + const results: FixtureResult[] = [] + for (const fixture of fixtures) { + results.push(await runOne(adapter, fixture)) + } + const failed = results.filter((r) => !r.ok).length + return { adapterName: adapter.name, results, ok: failed === 0, passed: results.length - failed, failed } +} diff --git a/evaluation/tsconfig.json b/evaluation/tsconfig.json new file mode 100644 index 0000000..3fa1424 --- /dev/null +++ b/evaluation/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022"], + "module": "ESNext", + "moduleResolution": "bundler", + "types": ["node"], + "strict": true, + "noUncheckedIndexedAccess": true, + "noFallthroughCasesInSwitch": true, + "skipLibCheck": true, + "noEmit": true, + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true + }, + "include": ["src/**/*.ts"] +}