From 95a3db45c6b8795976811c98019f7732601220f2 Mon Sep 17 00:00:00 2001 From: nullstack65 Date: Wed, 23 Sep 2026 05:46:26 -0400 Subject: [PATCH 1/7] feat(usage): prove prompt, request, session, and PR usage attribution Public usage summaries aggregate by day/provider/model and the underlying UsageRecord rows differ per provider, so there is no single provider-request unit today. Add a pure projection that re-projects measured usage onto prompt, provider-request, native-session, and pull-request levels using only explicit existing bindings (provider_session_runtime resume cursors and imported transcripts) and existing thread/PR links. Granularity is asserted per source, never inferred: Claude exposes per-request identity, Grok per prompt, Codex only per turn. The projection reports unsupported levels as null instead of dividing a turn. A session linked to several PRs feeds a non-additive shared pool rather than being cloned onto each link, and a known session with no usage is missing (null), never zero. Adds the native request/message/prompt ids to UsageRecord and the v4 scan cache so they stay separate from dedupeKey. Model/harness: opencode-go/deepseek-v4.1-flash via opencode. --- .../server/src/usage/usageAttribution.test.ts | 444 +++++++++ apps/server/src/usage/usageAttribution.ts | 851 ++++++++++++++++++ apps/server/src/usage/usageScanCache.test.ts | 24 + apps/server/src/usage/usageScanCache.ts | 21 +- .../server/src/usage/usageTranscripts.test.ts | 27 + apps/server/src/usage/usageTranscripts.ts | 38 + docs/internals/usage-attribution.md | 83 ++ 7 files changed, 1487 insertions(+), 1 deletion(-) create mode 100644 apps/server/src/usage/usageAttribution.test.ts create mode 100644 apps/server/src/usage/usageAttribution.ts create mode 100644 docs/internals/usage-attribution.md diff --git a/apps/server/src/usage/usageAttribution.test.ts b/apps/server/src/usage/usageAttribution.test.ts new file mode 100644 index 000000000000..9d129502b930 --- /dev/null +++ b/apps/server/src/usage/usageAttribution.test.ts @@ -0,0 +1,444 @@ +import { describe, expect, it } from "@effect/vitest"; + +import type { UsageTokenTotals } from "@t3tools/contracts"; + +import { + buildUsageAttribution, + renderUsageAttributionText, + USAGE_ATTRIBUTION_VERSION, + type AttributionPullRequestLink, + type AttributionSource, + type AttributionThreadBinding, + type AttributionUsageRecord, + type UsageAttributionInput, +} from "./usageAttribution.ts"; + +const CLAUDE_SESSION = "5a128faa-8253-489e-b935-6c08e8e670c0"; +const OTHER_CLAUDE_SESSION = "11111111-2222-3333-4444-555555555555"; +const CODEX_SESSION = "019fbbc1-b12c-7360-a685-28c181f0025f"; +const GROK_SESSION = "019fec1a-12f7-72f2-9b1f-7778a00aea3c"; +const CLAUDE_FINGERPRINT = "host\u0000claude\u0000/home/u/.claude\u00000:1"; +const CODEX_FINGERPRINT = "host\u0000codex\u0000/home/u/.codex\u00000:2"; + +function totals(overrides: Partial = {}): UsageTokenTotals { + return { + uncachedInputTokens: 100, + cachedInputTokens: 10, + cacheCreationTokens: 0, + outputTokens: 20, + reasoningTokens: 0, + ...overrides, + }; +} + +function record(overrides: Partial = {}): AttributionUsageRecord { + return { + provider: "claude", + sessionId: CLAUDE_SESSION, + model: "claude-fable-5", + timestampMs: 1_786_000_000_000, + totals: totals(), + costUsd: 0.01, + dedupeKey: null, + providerRequestId: null, + providerMessageId: null, + promptId: null, + sourceFingerprint: CLAUDE_FINGERPRINT, + ...overrides, + }; +} + +function codexRecord(overrides: Partial = {}): AttributionUsageRecord { + return record({ + provider: "codex", + sessionId: CODEX_SESSION, + model: "gpt-5.6-sol", + sourceFingerprint: CODEX_FINGERPRINT, + ...overrides, + }); +} + +function binding(overrides: Partial = {}): AttributionThreadBinding { + return { + threadId: "thread-1", + provider: "claude", + providerInstanceId: "claude-default", + nativeSessionId: CLAUDE_SESSION, + origin: "runtimeCursor", + ...overrides, + }; +} + +function link(overrides: Partial = {}): AttributionPullRequestLink { + return { + threadId: "thread-1", + host: "github.com", + repository: "acme/repo", + number: 12, + source: "manual", + linkedAt: "2026-09-01T00:00:00.000Z", + ...overrides, + }; +} + +function input(overrides: Partial = {}): UsageAttributionInput { + return { + generatedAtMs: 1_786_100_000_000, + records: [], + bindings: [], + links: [], + sources: [], + ...overrides, + }; +} + +describe("prompt and request granularity", () => { + it("counts provider requests for one prompt with a retry and a tool continuation", () => { + const records = [ + record({ dedupeKey: "m1:r1", providerRequestId: "r1", providerMessageId: "m1" }), + record({ dedupeKey: "m2:r2", providerRequestId: "r2", providerMessageId: "m2" }), + record({ dedupeKey: "m3:r3", providerRequestId: "r3", providerMessageId: "m3" }), + // A resumed/forked transcript repeats the first request's record verbatim. + record({ dedupeKey: "m1:r1", providerRequestId: "r1", providerMessageId: "m1" }), + ]; + + const projection = buildUsageAttribution(input({ records, bindings: [binding()] })); + const session = projection.sessions[0]!; + + expect(session.requestCount).toBe(3); + expect(session.promptCount).toBeNull(); + expect(session.requestQuality).toBe("measured"); + expect(session.promptQuality).toBe("unsupported"); + expect(projection.requests).toHaveLength(3); + expect(projection.prompts).toHaveLength(0); + expect(session.totals?.records).toBe(3); + }); + + it("groups grok usage by prompt across several models", () => { + const records = [ + record({ + provider: "grok", + sessionId: GROK_SESSION, + model: "grok-4.5", + promptId: "p1", + dedupeKey: "s:p1:grok-4.5", + totals: totals({ outputTokens: 10 }), + }), + record({ + provider: "grok", + sessionId: GROK_SESSION, + model: "grok-fast", + promptId: "p1", + dedupeKey: "s:p1:grok-fast", + totals: totals({ outputTokens: 5 }), + }), + record({ + provider: "grok", + sessionId: GROK_SESSION, + model: "grok-4.5", + promptId: "p2", + dedupeKey: "s:p2:grok-4.5", + totals: totals({ outputTokens: 7 }), + }), + ]; + + const projection = buildUsageAttribution( + input({ + records, + bindings: [ + binding({ provider: "grok", nativeSessionId: GROK_SESSION, providerInstanceId: null }), + ], + }), + ); + const session = projection.sessions[0]!; + + expect(session.promptCount).toBe(2); + expect(session.requestCount).toBeNull(); + expect(session.promptQuality).toBe("measured"); + expect(session.requestQuality).toBe("unsupported"); + expect(projection.prompts).toHaveLength(2); + const prompt1 = projection.prompts.find((prompt) => prompt.promptId === "p1")!; + expect(prompt1.models).toEqual(["grok-4.5", "grok-fast"]); + expect(prompt1.totals.records).toBe(2); + }); + + it("never reports request or prompt counts for turn-only codex usage", () => { + const projection = buildUsageAttribution( + input({ + records: [codexRecord()], + bindings: [binding({ provider: "codex", nativeSessionId: CODEX_SESSION })], + }), + ); + const session = projection.sessions[0]!; + + expect(session.requestCount).toBeNull(); + expect(session.promptCount).toBeNull(); + expect(session.requestQuality).toBe("unsupported"); + expect(session.promptQuality).toBe("unsupported"); + expect(projection.requests).toHaveLength(0); + expect(projection.prompts).toHaveLength(0); + }); +}); + +describe("session binding and data quality", () => { + it("collapses runtime and imported bindings for one native session", () => { + const projection = buildUsageAttribution( + input({ + records: [record({ dedupeKey: "a" })], + bindings: [ + binding({ threadId: "thread-1", origin: "runtimeCursor" }), + binding({ threadId: "thread-1", origin: "importedTranscript" }), + ], + }), + ); + + const session = projection.sessions[0]!; + expect(session.boundThreadIds).toEqual(["thread-1"]); + expect(session.bindingOrigins).toEqual(["importedTranscript", "runtimeCursor"]); + expect(session.totals?.records).toBe(1); + }); + + it("reports a session whose cursor was overwritten as unbound", () => { + const projection = buildUsageAttribution( + input({ + records: [record({ sessionId: CLAUDE_SESSION, dedupeKey: "a" })], + bindings: [binding({ threadId: "thread-1", nativeSessionId: OTHER_CLAUDE_SESSION })], + }), + ); + + const old = projection.sessions.find((session) => session.sessionId === CLAUDE_SESSION)!; + expect(old.boundThreadIds).toEqual([]); + expect(old.allocation).toBe("unallocated"); + expect(projection.coverage.find((entry) => entry.provider === "claude")?.unboundSessions).toBe( + 1, + ); + }); + + it("reports a bound session with no usage as missing, never zero", () => { + const projection = buildUsageAttribution(input({ bindings: [binding()] })); + const session = projection.sessions[0]!; + + expect(session.quality).toBe("missing"); + expect(session.totals).toBeNull(); + expect(session.allocation).toBe("missing"); + expect(projection.coverage.find((entry) => entry.provider === "claude")?.missingSessions).toBe( + 1, + ); + expect(projection.unallocated.records).toBe(0); + }); + + it("marks a malformed claude session id invalid", () => { + const projection = buildUsageAttribution( + input({ records: [record({ sessionId: "not-a-uuid", dedupeKey: "a" })] }), + ); + + expect(projection.sessions[0]?.quality).toBe("invalid"); + expect(projection.coverage.find((entry) => entry.provider === "claude")?.invalidSessions).toBe( + 1, + ); + }); + + it("de-duplicates identical codex records from two scans of one source", () => { + const scanned = codexRecord({ dedupeKey: null }); + const projection = buildUsageAttribution( + input({ + records: [scanned, { ...scanned }], + bindings: [binding({ provider: "codex", nativeSessionId: CODEX_SESSION })], + }), + ); + + expect(projection.sessions[0]?.totals?.records).toBe(1); + }); + + it("notes duplicate source fingerprints without double counting", () => { + const source: AttributionSource = { + fingerprint: CLAUDE_FINGERPRINT, + provider: "claude", + status: "ok", + distinctSessions: 1, + }; + const projection = buildUsageAttribution( + input({ + records: [record({ dedupeKey: "a" })], + sources: [source, { ...source }], + }), + ); + + expect(projection.sessions[0]?.totals?.records).toBe(1); + expect( + projection.limitations.some((line) => line.includes("duplicate source fingerprint")), + ).toBe(true); + }); +}); + +describe("pull request association and attribution", () => { + it("sums several sessions onto one PR additively", () => { + const projection = buildUsageAttribution( + input({ + records: [ + record({ + sessionId: CLAUDE_SESSION, + dedupeKey: "a", + totals: totals({ outputTokens: 100 }), + }), + codexRecord({ dedupeKey: null, totals: totals({ outputTokens: 50 }) }), + ], + bindings: [ + binding({ threadId: "thread-1", provider: "claude", nativeSessionId: CLAUDE_SESSION }), + binding({ threadId: "thread-2", provider: "codex", nativeSessionId: CODEX_SESSION }), + ], + links: [ + link({ threadId: "thread-1", number: 12 }), + link({ threadId: "thread-2", number: 12, source: "agent" }), + ], + }), + ); + + const pr = projection.pullRequests[0]!; + const expected = projection.sessions.reduce( + (sum, session) => sum + (session.totals?.totalTokens ?? 0), + 0, + ); + expect(pr.key).toBe("github.com/acme/repo#12"); + expect(pr.attributed.records).toBe(2); + expect(pr.attributed.totalTokens).toBe(expected); + expect(pr.contributingSessions).toHaveLength(2); + }); + + it("keeps a session linked to two PRs shared, not cloned onto both", () => { + const projection = buildUsageAttribution( + input({ + records: [record({ dedupeKey: "a" })], + bindings: [binding({ threadId: "thread-1" })], + links: [ + link({ threadId: "thread-1", number: 12 }), + link({ threadId: "thread-1", number: 13 }), + ], + }), + ); + + const session = projection.sessions[0]!; + expect(session.allocation).toBe("shared"); + expect(projection.shared.records).toBe(1); + for (const pr of projection.pullRequests) { + expect(pr.attributed.records).toBe(0); + expect(pr.shared.records).toBe(1); + } + expect(projection.unallocated.records).toBe(0); + }); + + it("treats a stack sibling as association, not attribution", () => { + const projection = buildUsageAttribution( + input({ + records: [record({ dedupeKey: "a" })], + bindings: [binding({ threadId: "thread-1" })], + links: [ + link({ threadId: "thread-1", number: 12, source: "manual" }), + link({ threadId: "thread-1", number: 13, source: "stack" }), + link({ threadId: "thread-1", number: 14, source: "stack-dismissed" }), + ], + }), + ); + + const session = projection.sessions[0]!; + expect(session.allocation).toBe("attributed"); + expect(session.pullRequestKeys).toEqual(["github.com/acme/repo#12"]); + expect(session.stackOnlyPullRequestKeys).toEqual(["github.com/acme/repo#13"]); + + const sibling = projection.pullRequests.find((pr) => pr.number === 13)!; + expect(sibling.attributed.records).toBe(0); + expect(sibling.stackAssociationSessions).toHaveLength(1); + expect(projection.pullRequests.some((pr) => pr.number === 14)).toBe(false); + }); + + it("pools unlinked and ambiguous sessions as unallocated", () => { + const projection = buildUsageAttribution( + input({ + records: [record({ dedupeKey: "a" }), codexRecord({ dedupeKey: null })], + bindings: [ + binding({ threadId: "thread-1", provider: "claude", nativeSessionId: CLAUDE_SESSION }), + binding({ threadId: "thread-2", provider: "codex", nativeSessionId: CODEX_SESSION }), + binding({ threadId: "thread-3", provider: "codex", nativeSessionId: CODEX_SESSION }), + ], + }), + ); + + expect(projection.sessions.find((session) => session.provider === "claude")?.allocation).toBe( + "unallocated", + ); + expect(projection.sessions.find((session) => session.provider === "codex")?.allocation).toBe( + "ambiguous", + ); + expect(projection.unallocated.records).toBe(2); + expect(projection.coverage.find((entry) => entry.provider === "codex")?.ambiguousSessions).toBe( + 1, + ); + }); +}); + +describe("projection contract", () => { + it("does not mutate its inputs", () => { + const records = [record({ dedupeKey: "a" })]; + const bindings = [binding()]; + const links = [link()]; + const before = JSON.stringify({ records, bindings, links }); + + buildUsageAttribution(input({ records, bindings, links })); + + expect(JSON.stringify({ records, bindings, links })).toBe(before); + }); + + it("exposes the source capability matrix with live qualification falsy", () => { + const projection = buildUsageAttribution(input({})); + expect(USAGE_ATTRIBUTION_VERSION).toBe(1); + expect(projection.contractVersion).toBe(1); + for (const entry of projection.coverage) { + expect(entry.liveQualified).toBe(false); + } + }); + + it("renders a stable human-readable sample", () => { + const projection = buildUsageAttribution( + input({ + records: [record({ dedupeKey: "a", providerRequestId: "r1", providerMessageId: "m1" })], + bindings: [binding({ threadId: "thread-1" })], + links: [link({ threadId: "thread-1", number: 12 })], + }), + ); + + const text = renderUsageAttributionText(projection); + expect(text).toContain(`claude:${CLAUDE_SESSION}`); + expect(text).toContain("requests=1"); + expect(text).toContain("github.com/acme/repo#12 attributed="); + }); + + it("returns every level for a machine-readable fixture", () => { + const projection = buildUsageAttribution( + input({ + records: [ + record({ dedupeKey: "a", providerRequestId: "r1", providerMessageId: "m1" }), + record({ + provider: "grok", + sessionId: GROK_SESSION, + model: "grok-4.5", + promptId: "p1", + dedupeKey: "s:p1", + }), + ], + bindings: [ + binding({ threadId: "thread-1", provider: "claude", nativeSessionId: CLAUDE_SESSION }), + binding({ threadId: "thread-1", provider: "grok", nativeSessionId: GROK_SESSION }), + ], + links: [link({ threadId: "thread-1", number: 12 })], + }), + ); + + expect(projection).toMatchObject({ contractVersion: 1, generatedAtMs: 1_786_100_000_000 }); + expect(projection.sessions).toHaveLength(2); + expect(projection.prompts).toHaveLength(1); + expect(projection.requests).toHaveLength(1); + expect(projection.pullRequests).toHaveLength(1); + expect(projection.coverage.map((entry) => entry.provider)).toEqual(["claude", "grok"]); + expect(projection.pullRequests[0]?.attributed.records).toBe(2); + }); +}); diff --git a/apps/server/src/usage/usageAttribution.ts b/apps/server/src/usage/usageAttribution.ts new file mode 100644 index 000000000000..973870015cfa --- /dev/null +++ b/apps/server/src/usage/usageAttribution.ts @@ -0,0 +1,851 @@ +/** + * Usage attribution projection. + * + * Re-projects the measured usage the scan already produced onto four reporting + * levels — prompt, provider request, native session, pull request — using only + * explicit bindings and links that already exist: + * + * - native session → T3 thread: the `resume_cursor_json` identity a provider + * adapter wrote, or the `importedTranscripts` metadata an imported session + * recorded. See `ProviderSessionRuntimeRepository`. + * - T3 thread → pull request: `projection_thread_pull_requests`, canonicalized + * with `@t3tools/shared/threadPullRequests`. + * + * This module is pure: it never reads the clock, the filesystem, or the + * database, and it never sees a prompt, a response, or a tool payload. Callers + * feed it allowlisted metadata plus already-normalized measurements. It is a + * proof of what the existing sources can establish, not a storage or transport + * decision. + * + * Two rules dominate the shape of the output: + * + * 1. Granularity is asserted per source, never inferred. A source that emits + * one aggregate per turn cannot yield request or prompt counts, so those + * levels report `unsupported` instead of a fabricated number. + * 2. Association is not attribution. A session linked to several pull requests + * contributes to each of those PRs' `shared` pool — which is explicitly not + * additive — rather than its total being cloned onto every linked PR. + * + * @module usageAttribution + */ +import type { + ThreadPullRequestLinkSource, + UsageProviderKind, + UsageTokenTotals, +} from "@t3tools/contracts"; +import { + normalizeThreadPullRequestKey, + threadPullRequestKeyOf, +} from "@t3tools/shared/threadPullRequests"; + +import { EMPTY_TOTALS, addTotals, totalTokens as countTokens } from "./usageTranscripts.ts"; + +export const USAGE_ATTRIBUTION_VERSION = 1 as const; + +/** The four reporting levels this projection can speak to. */ +export type AttributionGranularity = "prompt" | "request" | "session" | "pullRequest"; + +/** + * How much of a level's measurement is actually established. + * + * - `measured` — every contributing record carried the identity this level needs. + * - `partial` — some records lacked it; totals are a lower bound, not a complete one. + * - `missing` — the source supports this level but no usable measurement exists. + * This is the absence case, and it is never a zero. + * - `invalid` — an identity was present but malformed for its provider. + * - `unsupported` — the source cannot establish this level at all. + */ +export type AttributionQuality = "measured" | "partial" | "missing" | "invalid" | "unsupported"; + +/** Whether a source can establish a level from its native records. */ +export type AttributionLevelSupport = "supported" | "unsupported"; + +/** + * What each source can and cannot establish, stated once so a caller cannot + * accidentally treat an unsupported level as a measured zero. + * + * `liveQualified` is deliberately `false` for every source: this matrix is + * derived from source, not from an installed-live capture. + */ +export interface AttributionSourceCapability { + readonly provider: UsageProviderKind; + readonly nativeSource: "transcript" | "none"; + readonly session: AttributionLevelSupport; + readonly prompt: AttributionLevelSupport; + readonly request: AttributionLevelSupport; + readonly liveQualified: boolean; + readonly note: string; +} + +export const ATTRIBUTION_SOURCE_CAPABILITIES: readonly AttributionSourceCapability[] = [ + { + provider: "claude", + nativeSource: "transcript", + session: "supported", + prompt: "unsupported", + request: "supported", + liveQualified: false, + note: "One assistant message is one provider request (message id + request id). A user prompt can span several requests through tool continuation, and no prompt id is written, so prompt totals are not derivable.", + }, + { + provider: "codex", + nativeSource: "transcript", + session: "supported", + prompt: "unsupported", + request: "unsupported", + liveQualified: false, + note: "token_count deltas are turn-level increments with no request or prompt id. Request counts must never be inferred by dividing a turn.", + }, + { + provider: "grok", + nativeSource: "transcript", + session: "supported", + prompt: "supported", + request: "unsupported", + liveQualified: false, + note: "turn_completed carries prompt_id and may split one prompt across several models. No provider request id is written.", + }, +]; + +function capabilityOf(provider: UsageProviderKind): AttributionSourceCapability { + const found = ATTRIBUTION_SOURCE_CAPABILITIES.find((entry) => entry.provider === provider); + // Unknown providers have no transcript parser, so every level is unsupported. + return ( + found ?? { + provider, + nativeSource: "none", + session: "unsupported", + prompt: "unsupported", + request: "unsupported", + liveQualified: false, + note: "No transcript source is scanned for this provider.", + } + ); +} + +/** + * One already-normalized usage record, tagged with the source that produced it. + * `costUsd` is the priced cost supplied by the existing pricing path; the + * projection never prices anything itself. + */ +export interface AttributionUsageRecord { + readonly provider: UsageProviderKind; + /** Native session id; `""` when the source record carried none. */ + readonly sessionId: string; + readonly model: string; + readonly timestampMs: number; + readonly totals: UsageTokenTotals; + readonly costUsd: number; + readonly dedupeKey: string | null; + readonly providerRequestId?: string | null; + readonly providerMessageId?: string | null; + readonly promptId?: string | null; + /** Physical source identity, so a duplicate scan can be detected. */ + readonly sourceFingerprint: string; +} + +/** + * An explicit native-session → T3-thread binding that already exists in + * persisted state. `provider` is normalized to the usage provider kind, so a + * `claudeAgent` driver is `claude`. + */ +export interface AttributionThreadBinding { + readonly threadId: string; + readonly provider: UsageProviderKind; + readonly providerInstanceId: string | null; + readonly nativeSessionId: string; + /** + * Where the binding came from. `runtimeCursor` is the single current cursor + * on `provider_session_runtime`; `importedTranscript` is the accumulated + * imported-file metadata. Nothing else preserves a historical native id. + */ + readonly origin: "runtimeCursor" | "importedTranscript"; +} + +/** An existing thread → pull-request link, already canonicalized by the caller. */ +export interface AttributionPullRequestLink { + readonly threadId: string; + readonly host: string; + readonly repository: string; + readonly number: number; + readonly source: ThreadPullRequestLinkSource; + readonly linkedAt: string; +} + +/** A declared source, used for coverage and duplicate-scan reporting. */ +export interface AttributionSource { + readonly fingerprint: string; + readonly provider: UsageProviderKind; + readonly status: "ok" | "missing" | "partial" | "failed"; + readonly distinctSessions: number; +} + +export interface UsageAttributionInput { + readonly generatedAtMs: number; + readonly records: readonly AttributionUsageRecord[]; + readonly bindings: readonly AttributionThreadBinding[]; + readonly links: readonly AttributionPullRequestLink[]; + readonly sources: readonly AttributionSource[]; +} + +export interface AttributionTotals { + readonly tokens: UsageTokenTotals; + readonly totalTokens: number; + readonly costUsd: number; + readonly records: number; +} + +export type AttributionAllocation = + | "attributed" + | "shared" + | "unallocated" + | "ambiguous" + | "missing"; + +export interface AttributionSessionReport { + readonly provider: UsageProviderKind; + readonly sessionId: string; + readonly models: readonly string[]; + /** `null` when the session is known but no usage was measured for it. */ + readonly totals: AttributionTotals | null; + readonly quality: AttributionQuality; + readonly promptQuality: AttributionQuality; + readonly requestQuality: AttributionQuality; + /** `null` when the source cannot establish this level. */ + readonly promptCount: number | null; + readonly requestCount: number | null; + readonly boundThreadIds: readonly string[]; + readonly providerInstanceIds: readonly string[]; + readonly bindingOrigins: readonly AttributionThreadBinding["origin"][]; + readonly allocation: AttributionAllocation; + /** Canonical PR keys this session is associated with, if any. */ + readonly pullRequestKeys: readonly string[]; + /** PRs reached only through a stack-sibling link; association, not attribution. */ + readonly stackOnlyPullRequestKeys: readonly string[]; +} + +export interface AttributionPromptReport { + readonly provider: UsageProviderKind; + readonly sessionId: string; + readonly promptId: string; + readonly totals: AttributionTotals; + readonly models: readonly string[]; + readonly boundThreadIds: readonly string[]; + readonly allocation: AttributionAllocation; +} + +export interface AttributionRequestReport { + readonly provider: UsageProviderKind; + readonly sessionId: string; + readonly providerRequestId: string; + readonly providerMessageId: string | null; + readonly totals: AttributionTotals; + readonly model: string; + readonly boundThreadIds: readonly string[]; + readonly allocation: AttributionAllocation; +} + +export interface AttributionPullRequestReport { + readonly key: string; + readonly host: string; + readonly repository: string; + readonly number: number; + readonly threadIds: readonly string[]; + readonly linkSources: readonly ThreadPullRequestLinkSource[]; + /** Additive: sessions bound to exactly one strong link to this PR. */ + readonly attributed: AttributionTotals; + /** Sessions also linked to another PR. Never add this into `attributed`. */ + readonly shared: AttributionTotals; + /** Sessions reaching this PR only through a stack-sibling link. */ + readonly stackAssociationSessions: readonly string[]; + readonly contributingSessions: readonly string[]; +} + +export interface AttributionCoverage { + readonly provider: UsageProviderKind; + readonly nativeSource: AttributionSourceCapability["nativeSource"]; + readonly liveQualified: boolean; + readonly session: AttributionLevelSupport; + readonly prompt: AttributionLevelSupport; + readonly request: AttributionLevelSupport; + readonly measuredSessions: number; + readonly missingSessions: number; + readonly invalidSessions: number; + readonly unboundSessions: number; + readonly ambiguousSessions: number; + readonly recordsWithoutSessionId: number; +} + +export interface UsageAttribution { + readonly contractVersion: typeof USAGE_ATTRIBUTION_VERSION; + readonly generatedAtMs: number; + readonly sessions: readonly AttributionSessionReport[]; + readonly prompts: readonly AttributionPromptReport[]; + readonly requests: readonly AttributionRequestReport[]; + readonly pullRequests: readonly AttributionPullRequestReport[]; + /** Usage on sessions linked to more than one strong PR. Not additive. */ + readonly shared: AttributionTotals; + /** Usage on sessions with no usable PR link, including missing identity. */ + readonly unallocated: AttributionTotals; + readonly coverage: readonly AttributionCoverage[]; + readonly limitations: readonly string[]; +} + +const CLAUDE_SESSION_ID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu; + +/** `stack-dismissed` is a tombstone; it mirrors `visibleThreadPullRequests`. */ +function isVisibleLink(link: AttributionPullRequestLink): boolean { + return link.source !== "stack-dismissed"; +} + +/** A strong link asserts real work; a `stack` link is display association only. */ +function isStrongLink(link: AttributionPullRequestLink): boolean { + return link.source !== "stack" && isVisibleLink(link); +} + +interface SessionAccumulator { + provider: UsageProviderKind; + sessionId: string; + records: AttributionUsageRecord[]; + requestIds: Set; + recordsWithRequestId: number; + promptIds: Set; + recordsWithPromptId: number; +} + +function addTotalsOf(left: AttributionTotals, right: AttributionTotals): AttributionTotals { + return { + tokens: addTotals(left.tokens, right.tokens), + totalTokens: left.totalTokens + right.totalTokens, + costUsd: left.costUsd + right.costUsd, + records: left.records + right.records, + }; +} + +const ZERO_TOTALS: AttributionTotals = { + tokens: EMPTY_TOTALS, + totalTokens: 0, + costUsd: 0, + records: 0, +}; + +function totalsOfRecords(records: readonly AttributionUsageRecord[]): AttributionTotals { + let tokens = EMPTY_TOTALS; + let costUsd = 0; + for (const record of records) { + tokens = addTotals(tokens, record.totals); + costUsd += record.costUsd; + } + return { tokens, totalTokens: countTokens(tokens), costUsd, records: records.length }; +} + +/** + * Identity used for de-duplication when a record carries no `dedupeKey`. + * + * Two environments scanning the same directory produce byte-identical codex + * records (which have no parser dedupe key), so a content signature stops that + * shared source from being counted twice. It is intentionally not exposed as a + * provider request id. + */ +function recordContentSignature(record: AttributionUsageRecord): string { + return [ + record.provider, + record.sessionId, + record.model, + record.timestampMs, + record.totals.uncachedInputTokens, + record.totals.cachedInputTokens, + record.totals.cacheCreationTokens, + record.totals.outputTokens, + record.totals.reasoningTokens, + record.providerRequestId ?? "", + record.providerMessageId ?? "", + record.promptId ?? "", + ].join("\u0000"); +} + +interface StrongLink { + readonly key: string; + readonly host: string; + readonly repository: string; + readonly number: number; + readonly sources: Set; + readonly threadIds: Set; +} + +interface MutableCoverage { + provider: UsageProviderKind; + nativeSource: AttributionSourceCapability["nativeSource"]; + liveQualified: boolean; + session: AttributionLevelSupport; + prompt: AttributionLevelSupport; + request: AttributionLevelSupport; + measuredSessions: number; + missingSessions: number; + invalidSessions: number; + unboundSessions: number; + ambiguousSessions: number; + recordsWithoutSessionId: number; +} + +/** + * Builds the four-level projection from already-measured usage. + * + * `generatedAtMs` is supplied rather than read so the projection stays pure and + * fixtures stay deterministic. + */ +export function buildUsageAttribution(input: UsageAttributionInput): UsageAttribution { + // 1. De-duplicate records: by declared key when present (fork copies and + // resumed history share it), otherwise by content signature. + const seen = new Set(); + const records: AttributionUsageRecord[] = []; + for (const record of input.records) { + const identity = record.dedupeKey ?? recordContentSignature(record); + if (seen.has(identity)) continue; + seen.add(identity); + records.push(record); + } + + // 2. Bind native sessions to threads, keeping every origin that points at + // the same session so a resume that reused one thread is not read as a + // second owner. + const sessionThreads = new Map< + string, + { + threadIds: Set; + instanceIds: Set; + origins: Set; + } + >(); + for (const binding of input.bindings) { + if (binding.nativeSessionId.length === 0) continue; + const key = sessionKey(binding.provider, binding.nativeSessionId); + const entry = sessionThreads.get(key) ?? { + threadIds: new Set(), + instanceIds: new Set(), + origins: new Set(), + }; + entry.threadIds.add(binding.threadId); + if (binding.providerInstanceId !== null) entry.instanceIds.add(binding.providerInstanceId); + entry.origins.add(binding.origin); + sessionThreads.set(key, entry); + } + + // 3. Index visible PR links per thread and canonical PR metadata. + const threadStrongPrs = new Map>(); + const threadStackPrs = new Map>(); + const pullRequestMeta = new Map }>(); + for (const link of input.links) { + if (!isVisibleLink(link)) continue; + const normalized = normalizeThreadPullRequestKey(link); + const key = threadPullRequestKeyOf(link); + const meta = pullRequestMeta.get(key) ?? { + key, + host: normalized.host, + repository: normalized.repository, + number: normalized.number, + sources: new Set(), + threadIds: new Set(), + }; + meta.sources.add(link.source); + meta.threadIds.add(link.threadId); + pullRequestMeta.set(key, meta as StrongLink & { threadIds: Set }); + const target = isStrongLink(link) ? threadStrongPrs : threadStackPrs; + const set = target.get(link.threadId) ?? new Set(); + set.add(key); + target.set(link.threadId, set); + } + + // 4. Accumulate per native session from measured records, then include + // bindings that have no records so "known but unmeasured" is not zero. + const sessionsByKey = new Map(); + const recordsWithoutSessionIdByProvider = new Map(); + for (const record of records) { + if (record.sessionId.length === 0) { + recordsWithoutSessionIdByProvider.set( + record.provider, + (recordsWithoutSessionIdByProvider.get(record.provider) ?? 0) + 1, + ); + continue; + } + const key = sessionKey(record.provider, record.sessionId); + const accumulator = sessionsByKey.get(key) ?? { + provider: record.provider, + sessionId: record.sessionId, + records: [], + requestIds: new Set(), + recordsWithRequestId: 0, + promptIds: new Set(), + recordsWithPromptId: 0, + }; + accumulator.records.push(record); + if (record.providerRequestId) { + accumulator.requestIds.add(record.providerRequestId); + accumulator.recordsWithRequestId += 1; + } + if (record.promptId) { + accumulator.promptIds.add(record.promptId); + accumulator.recordsWithPromptId += 1; + } + sessionsByKey.set(key, accumulator); + } + + const sessionReports: AttributionSessionReport[] = []; + const promptReports: AttributionPromptReport[] = []; + const requestReports: AttributionRequestReport[] = []; + const prAttributed = new Map(); + const prShared = new Map(); + const prStackAssociations = new Map>(); + const prContributing = new Map>(); + let shared = ZERO_TOTALS; + let unallocated = ZERO_TOTALS; + const coverageByProvider = new Map(); + + const sessionUniverse = new Map(sessionsByKey); + for (const key of sessionThreads.keys()) { + if (sessionUniverse.has(key)) continue; + // A bound session with no records is a missing measurement, not a zero. + sessionUniverse.set(key, { + provider: providerOfKey(key), + sessionId: sessionIdOfKey(key), + records: [], + requestIds: new Set(), + recordsWithRequestId: 0, + promptIds: new Set(), + recordsWithPromptId: 0, + }); + } + + for (const [key, accumulator] of sessionUniverse) { + const { provider, sessionId } = accumulator; + const capability = capabilityOf(provider); + const binding = sessionThreads.get(key); + const boundThreadIds = binding ? [...binding.threadIds].toSorted() : []; + const instanceIds = binding ? [...binding.instanceIds].toSorted() : []; + const origins = binding ? [...binding.origins].toSorted() : []; + const totals = accumulator.records.length === 0 ? null : totalsOfRecords(accumulator.records); + const models = [...new Set(accumulator.records.map((record) => record.model))].toSorted(); + + const sessionQuality: AttributionQuality = + sessionId.length === 0 + ? "missing" + : provider === "claude" && !CLAUDE_SESSION_ID_PATTERN.test(sessionId) + ? "invalid" + : totals === null + ? "missing" + : "measured"; + + const promptQuality: AttributionQuality = + capability.prompt === "unsupported" + ? "unsupported" + : accumulator.records.length === 0 + ? "missing" + : accumulator.recordsWithPromptId === accumulator.records.length + ? "measured" + : accumulator.recordsWithPromptId === 0 + ? "missing" + : "partial"; + + const requestQuality: AttributionQuality = + capability.request === "unsupported" + ? "unsupported" + : accumulator.records.length === 0 + ? "missing" + : accumulator.recordsWithRequestId === accumulator.records.length + ? "measured" + : accumulator.recordsWithRequestId === 0 + ? "missing" + : "partial"; + + const strongPrs = new Set(); + for (const threadId of boundThreadIds) { + for (const prKey of threadStrongPrs.get(threadId) ?? []) strongPrs.add(prKey); + } + const stackPrs = new Set(); + for (const threadId of boundThreadIds) { + for (const prKey of threadStackPrs.get(threadId) ?? []) { + if (!strongPrs.has(prKey)) stackPrs.add(prKey); + } + } + + let allocation: AttributionAllocation; + if (sessionQuality === "missing" && totals === null) allocation = "missing"; + else if (boundThreadIds.length === 0) allocation = "unallocated"; + else if (boundThreadIds.length > 1) allocation = "ambiguous"; + else if (strongPrs.size === 1) allocation = "attributed"; + else if (strongPrs.size > 1) allocation = "shared"; + else allocation = "unallocated"; + + if (totals !== null) { + if (allocation === "shared") shared = addTotalsOf(shared, totals); + // Ambiguous sessions are pooled with unallocated usage: neither can be + // placed on a specific pull request without inventing an owner. + if (allocation === "unallocated" || allocation === "ambiguous") { + unallocated = addTotalsOf(unallocated, totals); + } + if (allocation === "attributed") { + for (const prKey of strongPrs) { + prAttributed.set(prKey, addTotalsOf(prAttributed.get(prKey) ?? ZERO_TOTALS, totals)); + } + } + if (allocation === "shared") { + for (const prKey of strongPrs) { + prShared.set(prKey, addTotalsOf(prShared.get(prKey) ?? ZERO_TOTALS, totals)); + } + } + } + for (const prKey of strongPrs) { + const contributing = prContributing.get(prKey) ?? new Set(); + contributing.add(sessionLabel(provider, sessionId)); + prContributing.set(prKey, contributing); + } + for (const prKey of stackPrs) { + const associated = prStackAssociations.get(prKey) ?? new Set(); + associated.add(sessionLabel(provider, sessionId)); + prStackAssociations.set(prKey, associated); + } + + sessionReports.push({ + provider, + sessionId, + models, + totals, + quality: sessionQuality, + promptQuality, + requestQuality, + promptCount: capability.prompt === "supported" ? accumulator.promptIds.size : null, + requestCount: capability.request === "supported" ? accumulator.requestIds.size : null, + boundThreadIds, + providerInstanceIds: instanceIds, + bindingOrigins: origins, + allocation, + pullRequestKeys: [...strongPrs].toSorted(), + stackOnlyPullRequestKeys: [...stackPrs].toSorted(), + }); + + // 5. Level reports. Only providers whose capability supports the level + // produce rows; an unsupported source yields no rows and no counts. + if (capability.request === "supported" && totals !== null) { + requestReports.push(...requestRows(accumulator, boundThreadIds, allocation, capability)); + } + if (capability.prompt === "supported" && totals !== null) { + promptReports.push(...promptRows(accumulator, boundThreadIds, allocation)); + } + + // 6. Coverage. + const coverage = coverageByProvider.get(provider) ?? { + provider, + nativeSource: capability.nativeSource, + liveQualified: capability.liveQualified, + session: capability.session, + prompt: capability.prompt, + request: capability.request, + measuredSessions: 0, + missingSessions: 0, + invalidSessions: 0, + unboundSessions: 0, + ambiguousSessions: 0, + recordsWithoutSessionId: recordsWithoutSessionIdByProvider.get(provider) ?? 0, + }; + if (sessionQuality === "measured") coverage.measuredSessions += 1; + if (sessionQuality === "missing") coverage.missingSessions += 1; + if (sessionQuality === "invalid") coverage.invalidSessions += 1; + if (boundThreadIds.length === 0) coverage.unboundSessions += 1; + if (allocation === "ambiguous") coverage.ambiguousSessions += 1; + coverageByProvider.set(provider, coverage); + } + + for (const [provider, coverage] of coverageByProvider) { + coverageByProvider.set(provider, { + ...coverage, + recordsWithoutSessionId: recordsWithoutSessionIdByProvider.get(provider) ?? 0, + }); + } + + sessionReports.sort((left, right) => + sessionLabel(left.provider, left.sessionId).localeCompare( + sessionLabel(right.provider, right.sessionId), + ), + ); + promptReports.sort((left, right) => + `${sessionLabel(left.provider, left.sessionId)}\u0000${left.promptId}`.localeCompare( + `${sessionLabel(right.provider, right.sessionId)}\u0000${right.promptId}`, + ), + ); + requestReports.sort((left, right) => + `${sessionLabel(left.provider, left.sessionId)}\u0000${left.providerRequestId}`.localeCompare( + `${sessionLabel(right.provider, right.sessionId)}\u0000${right.providerRequestId}`, + ), + ); + + const pullRequests: AttributionPullRequestReport[] = [...pullRequestMeta.values()] + .map((meta) => ({ + key: meta.key, + host: meta.host, + repository: meta.repository, + number: meta.number, + threadIds: [...meta.threadIds].toSorted(), + linkSources: [...meta.sources].toSorted(), + attributed: prAttributed.get(meta.key) ?? ZERO_TOTALS, + shared: prShared.get(meta.key) ?? ZERO_TOTALS, + stackAssociationSessions: [...(prStackAssociations.get(meta.key) ?? [])].toSorted(), + contributingSessions: [...(prContributing.get(meta.key) ?? [])].toSorted(), + })) + .toSorted((left, right) => left.key.localeCompare(right.key)); + + const coverage: AttributionCoverage[] = [...coverageByProvider.values()].toSorted((left, right) => + left.provider.localeCompare(right.provider), + ); + + return { + contractVersion: USAGE_ATTRIBUTION_VERSION, + generatedAtMs: input.generatedAtMs, + sessions: sessionReports, + prompts: promptReports, + requests: requestReports, + pullRequests, + shared, + unallocated, + coverage, + limitations: limitationsFor(input, coverage), + }; +} + +function requestRows( + accumulator: SessionAccumulator, + boundThreadIds: readonly string[], + allocation: AttributionAllocation, + capability: AttributionSourceCapability, +): AttributionRequestReport[] { + if (capability.request !== "supported") return []; + const grouped = new Map(); + for (const record of accumulator.records) { + if (!record.providerRequestId) continue; + const rows = grouped.get(record.providerRequestId) ?? []; + rows.push(record); + grouped.set(record.providerRequestId, rows); + } + return [...grouped.entries()].map(([providerRequestId, rows]) => ({ + provider: accumulator.provider, + sessionId: accumulator.sessionId, + providerRequestId, + providerMessageId: rows.find((row) => !!row.providerMessageId)?.providerMessageId ?? null, + totals: totalsOfRecords(rows), + model: rows[0]?.model ?? "", + boundThreadIds, + allocation, + })); +} + +function promptRows( + accumulator: SessionAccumulator, + boundThreadIds: readonly string[], + allocation: AttributionAllocation, +): AttributionPromptReport[] { + const grouped = new Map(); + for (const record of accumulator.records) { + if (!record.promptId) continue; + const rows = grouped.get(record.promptId) ?? []; + rows.push(record); + grouped.set(record.promptId, rows); + } + return [...grouped.entries()].map(([promptId, rows]) => ({ + provider: accumulator.provider, + sessionId: accumulator.sessionId, + promptId, + totals: totalsOfRecords(rows), + models: [...new Set(rows.map((row) => row.model))].toSorted(), + boundThreadIds, + allocation, + })); +} + +function limitationsFor( + input: UsageAttributionInput, + coverage: readonly AttributionCoverage[], +): readonly string[] { + const limitations: string[] = [ + "A native session maps to a T3 thread only through the current resume cursor or imported-transcript metadata; a session switch, fork, or restart that overwrote the cursor leaves earlier usage unbound.", + "Provider-instance identity is not recoverable from a transcript scan, so two instances of one provider cannot be told apart at the record level.", + "Request and prompt counts are reported only where the native source writes those ids; a turn-level source reports `unsupported`, never an inferred count.", + ]; + const duplicateFingerprints = + input.sources.length - new Set(input.sources.map((source) => source.fingerprint)).size; + if (duplicateFingerprints > 0) { + limitations.push( + `${duplicateFingerprints} duplicate source fingerprint(s) were reported; identical records are de-duplicated, but a shared source still requires one environment to be dropped upstream.`, + ); + } + if (coverage.some((entry) => entry.missingSessions > 0)) { + limitations.push( + "Some known sessions have no measured usage. They are reported as `missing` with a null total; this is not a zero-cost success.", + ); + } + if (input.bindings.some((binding) => binding.origin === "runtimeCursor")) { + limitations.push( + "Only the newest native session id per thread is durable. Additive retention must land before historical re-attribution is possible.", + ); + } + return limitations; +} + +function sessionKey(provider: UsageProviderKind, sessionId: string): string { + return `${provider}\u0000${sessionId}`; +} + +function providerOfKey(key: string): UsageProviderKind { + return key.slice(0, key.indexOf("\u0000")) as UsageProviderKind; +} + +function sessionIdOfKey(key: string): string { + return key.slice(key.indexOf("\u0000") + 1); +} + +function sessionLabel(provider: UsageProviderKind, sessionId: string): string { + return `${provider}:${sessionId.length === 0 ? "" : sessionId}`; +} + +/** + * A compact human-readable rendering of the projection. Intended for logs and + * PR evidence; the JSON form is the machine-readable one. + */ +export function renderUsageAttributionText(projection: UsageAttribution): string { + const lines: string[] = ["Usage attribution", ""]; + lines.push(`Sessions: ${projection.sessions.length}`); + for (const session of projection.sessions) { + const tokens = session.totals === null ? "missing" : `${session.totals.totalTokens} tokens`; + const cost = session.totals === null ? "" : ` $${session.totals.costUsd.toFixed(4)}`; + const prompts = + session.promptCount === null ? "prompts=unsupported" : `prompts=${session.promptCount}`; + const requests = + session.requestCount === null ? "requests=unsupported" : `requests=${session.requestCount}`; + lines.push( + ` ${sessionLabel(session.provider, session.sessionId)} [${session.quality}/${session.allocation}] ${tokens}${cost} ${prompts} ${requests} threads=${ + session.boundThreadIds.length === 0 ? "" : session.boundThreadIds.join(",") + }`, + ); + } + lines.push("", "Pull requests:"); + if (projection.pullRequests.length === 0) lines.push(" "); + for (const pr of projection.pullRequests) { + lines.push( + ` ${pr.key} attributed=${pr.attributed.totalTokens} tokens shared=${pr.shared.totalTokens} tokens sessions=${ + pr.contributingSessions.length === 0 ? "" : pr.contributingSessions.join(",") + } sources=${pr.linkSources.join(",")}`, + ); + } + lines.push( + "", + `Shared (not additive): ${projection.shared.totalTokens} tokens`, + `Unallocated: ${projection.unallocated.totalTokens} tokens`, + "", + "Coverage:", + ); + for (const entry of projection.coverage) { + lines.push( + ` ${entry.provider} session=${entry.session} prompt=${entry.prompt} request=${entry.request} measured=${entry.measuredSessions} missing=${entry.missingSessions} unbound=${entry.unboundSessions} ambiguous=${entry.ambiguousSessions}`, + ); + } + return lines.join("\n"); +} diff --git a/apps/server/src/usage/usageScanCache.test.ts b/apps/server/src/usage/usageScanCache.test.ts index cc1bdbcc1626..23423dd0cd1c 100644 --- a/apps/server/src/usage/usageScanCache.test.ts +++ b/apps/server/src/usage/usageScanCache.test.ts @@ -97,6 +97,30 @@ describe("scan cache round trip", () => { expect(restored.get("/codex.jsonl")).toEqual(original.get("/codex.jsonl")); }); + it("preserves native request, message, and prompt ids", () => { + const original = cacheWith([ + [ + "/a.jsonl", + 100, + [ + record({ + providerRequestId: "r1", + providerMessageId: "m1", + promptId: "p1", + }), + ], + ], + ]); + + const restored = decodeScanCache(JSON.parse(JSON.stringify(encodeScanCache(original)))); + + expect(restored.get("/a.jsonl")?.records[0]).toMatchObject({ + providerRequestId: "r1", + providerMessageId: "m1", + promptId: "p1", + }); + }); + it("drops an entry whose persisted parse state is corrupt", () => { // Resuming with a bad reducer state would attach appended usage to the // wrong model or replay fork-copied history; that entry must cold parse. diff --git a/apps/server/src/usage/usageScanCache.ts b/apps/server/src/usage/usageScanCache.ts index 71ef25051eb6..4ab99128159a 100644 --- a/apps/server/src/usage/usageScanCache.ts +++ b/apps/server/src/usage/usageScanCache.ts @@ -23,7 +23,10 @@ import type { CodexScanState, UsageRecord } from "./usageTranscripts.ts"; // entries would keep serving double-counted records forever. // v3: entries carry the parse position and reducer state so a grown file // re-parses only its appended bytes instead of starting over. -const USAGE_SCAN_CACHE_VERSION = 3 as const; +// v4: records carry native request/message/prompt ids. Without the bump, warm +// v3 entries would silently report those levels as unsupported until the file +// next changed. +const USAGE_SCAN_CACHE_VERSION = 4 as const; export interface CachedFile { readonly size: number; @@ -58,6 +61,9 @@ type SerializedRecord = readonly [ reasoningTokens: number, dedupeKey: string | null, reportedCostUsd: number | null, + providerRequestId: string | null, + providerMessageId: string | null, + promptId: string | null, ]; interface SerializedFile { @@ -109,6 +115,9 @@ export function encodeScanCache(cache: ScanCache): SerializedCache { record.totals.reasoningTokens, record.dedupeKey, record.reportedCostUsd, + record.providerRequestId ?? null, + record.providerMessageId ?? null, + record.promptId ?? null, ]; const files: Record = {}; @@ -178,6 +187,11 @@ export function decodeScanCache(document: unknown): ScanCache { dedupeKey, reportedCostUsd, ] = row as SerializedRecord; + // Appended in v4. Absent on a hand-built or truncated row, in which case + // the identity is simply not asserted rather than defaulted to a value. + const providerRequestId = row[10]; + const providerMessageId = row[11]; + const promptId = row[12]; const model = typeof modelIndex === "number" ? models[modelIndex] : undefined; if ( @@ -207,6 +221,11 @@ export function decodeScanCache(document: unknown): ScanCache { }, reportedCostUsd: typeof reportedCostUsd === "number" ? reportedCostUsd : null, dedupeKey: typeof dedupeKey === "string" ? dedupeKey : null, + // Omitted when absent so a record round-trips identically to one the + // parser produced without the field. + ...(typeof providerRequestId === "string" ? { providerRequestId } : {}), + ...(typeof providerMessageId === "string" ? { providerMessageId } : {}), + ...(typeof promptId === "string" ? { promptId } : {}), }); } return records; diff --git a/apps/server/src/usage/usageTranscripts.test.ts b/apps/server/src/usage/usageTranscripts.test.ts index b09db613ed85..b9b582eb9de2 100644 --- a/apps/server/src/usage/usageTranscripts.test.ts +++ b/apps/server/src/usage/usageTranscripts.test.ts @@ -15,12 +15,14 @@ function claudeLine(overrides: { contentType: string; model?: string; outputTokens?: number; + requestId?: string; }): string { return JSON.stringify({ type: "assistant", timestamp: "2026-08-07T04:05:13.944Z", sessionId: "5a128faa-8253-489e-b935-6c08e8e670c0", cwd: "/home/theo/project", + ...(overrides.requestId === undefined ? {} : { requestId: overrides.requestId }), message: { id: overrides.messageId, role: "assistant", @@ -67,6 +69,19 @@ describe("parseClaudeLine", () => { expect(parseClaudeLine(JSON.stringify({ type: "user", message: {} }))).toBeNull(); expect(parseClaudeLine("not json")).toBeNull(); }); + + it("exposes the native request and message ids apart from the dedupe key", () => { + // The de-duplication key is a composite; a provider request id is a + // separately meaningful value and must not be recovered from it. + const record = parseClaudeLine( + claudeLine({ messageId: "msg_9", contentType: "text", requestId: "req_9" }), + ); + + expect(record?.dedupeKey).toBe("msg_9:req_9"); + expect(record?.providerRequestId).toBe("req_9"); + expect(record?.providerMessageId).toBe("msg_9"); + expect(record?.promptId).toBeNull(); + }); }); describe("parseCodexLine", () => { @@ -111,6 +126,10 @@ describe("parseCodexLine", () => { expect(record?.totals.uncachedInputTokens).toBe(19239 - 11008); expect(record?.totals.cachedInputTokens).toBe(11008); expect(record?.totals.reasoningTokens).toBe(116); + // Turn-level usage carries no request or prompt identity. + expect(record?.providerRequestId).toBeNull(); + expect(record?.providerMessageId).toBeNull(); + expect(record?.promptId).toBeNull(); }); it("skips a repeated token_count so deltas are not double counted", () => { @@ -498,6 +517,14 @@ describe("parseGrokLine", () => { expect(sum).toBeCloseTo(1, 12); }); + it("exposes the native prompt id apart from the dedupe key", () => { + const [record] = parseGrokLine(turnCompleted({ promptId: "prompt-7" })); + + expect(record?.dedupeKey).toBe("019fec1a-12f7-72f2-9b1f-7778a00aea3c:prompt-7:grok-4.5-build"); + expect(record?.promptId).toBe("prompt-7"); + expect(record?.providerRequestId).toBeNull(); + }); + it("does not invent a colliding dedupe key when prompt_id is missing", () => { const line = JSON.stringify({ timestamp: 1_786_372_566, diff --git a/apps/server/src/usage/usageTranscripts.ts b/apps/server/src/usage/usageTranscripts.ts index 5d909379eb10..7d0c3ba28306 100644 --- a/apps/server/src/usage/usageTranscripts.ts +++ b/apps/server/src/usage/usageTranscripts.ts @@ -20,6 +20,25 @@ export interface UsageRecord { * unique and needs no dedup. */ readonly dedupeKey: string | null; + /** + * Native provider request id, when the source exposes one. Claude Code writes + * a `requestId` per API response. `undefined`/absent must never be read as a + * request count of one: the source either has the id or it does not. + * + * Deliberately separate from {@link dedupeKey}, which is a de-duplication + * composite and not a guaranteed provider request id. + */ + readonly providerRequestId?: string | null; + /** + * Native provider message id, when the source exposes one. Claude Code's + * `message.id` identifies one assistant response; it is not a prompt id. + */ + readonly providerMessageId?: string | null; + /** + * Native prompt id, when the source exposes one. Grok Build's + * `turn_completed.prompt_id` identifies the user prompt a turn answers. + */ + readonly promptId?: string | null; } const EMPTY_TOTALS: UsageTokenTotals = { @@ -146,6 +165,12 @@ export function parseClaudeLine(line: string): UsageRecord | null { }, reportedCostUsd: typeof cost === "number" && Number.isFinite(cost) ? cost : null, dedupeKey, + // Namespaced identity, kept apart from `dedupeKey`. A user prompt can span + // several assistant messages (tool continuation), so these count provider + // requests; no prompt id exists in this source. + providerRequestId: requestId, + providerMessageId: messageId, + promptId: null, }; } @@ -307,6 +332,11 @@ export function parseCodexLine(line: string, state: CodexScanState): UsageRecord // Events surviving the fork-copy suppression above are unique to this // rollout, so they need no global dedup. dedupeKey: null, + // A `token_count` delta is a turn-level increment with no request or prompt + // id. Request counts must never be inferred from it. + providerRequestId: null, + providerMessageId: null, + promptId: null, }; } @@ -435,6 +465,10 @@ export function parseGrokLine(line: string): readonly UsageRecord[] { reportedCostUsd: grokCostTicksToUsd(topLevel.costUsdTicks), // No prompt id means we cannot tell two same-second updates apart. dedupeKey: promptId === null ? null : `${sessionId}:${promptId}:grok`, + // Grok identifies the prompt, not the API request. + providerRequestId: null, + providerMessageId: null, + promptId, }, ]; } @@ -480,6 +514,10 @@ export function parseGrokLine(line: string): readonly UsageRecord[] { totals, reportedCostUsd, dedupeKey: promptId === null ? null : `${sessionId}:${promptId}:${entry.model}`, + // Grok identifies the prompt, not the API request. + providerRequestId: null, + providerMessageId: null, + promptId, }); } return results; diff --git a/docs/internals/usage-attribution.md b/docs/internals/usage-attribution.md new file mode 100644 index 000000000000..d6fb357bf7cd --- /dev/null +++ b/docs/internals/usage-attribution.md @@ -0,0 +1,83 @@ +# Usage attribution + +[`usageAttribution.ts`](../../apps/server/src/usage/usageAttribution.ts) re-projects +the usage the transcript scan already measured onto four reporting levels — prompt, +provider request, native session, and pull request. It exists so model comparison can +be done at the level a source can actually establish, instead of dividing a turn's +tokens into invented requests. It is a pure function over allowlisted metadata: it +never reads the clock, the filesystem, or the database, and it never sees a prompt, +response, or tool payload. Ingestion is the caller's job; the projection takes records, +explicit bindings, and existing PR links. + +## Granularity is a source property + +Each provider transcript exposes a different unit, and the projection reports a level +as `unsupported` rather than estimating one: + +| Provider | Session | Prompt | Request | Native unit | +| ----------- | ------- | ----------------- | ------- | --------------------------------------------------------------------- | +| Claude Code | yes | no | yes | one assistant message = one API response (`message.id` + `requestId`) | +| Codex | yes | no | no | `token_count` deltas, one per model turn | +| Grok Build | yes | yes (`prompt_id`) | no | `turn_completed`, one per prompt per model | + +This is stated once in `ATTRIBUTION_SOURCE_CAPABILITIES` and mirrored into each +projection's `coverage`. A user prompt can span several Claude requests through tool +continuation, and no prompt id is written, so prompt totals are not derivable for +Claude. A Codex turn has no request id at all, so a request count there is never a +division of the turn: it is `null` with a `unsupported` quality. The request/message +ids were added to `UsageRecord` (and the v4 scan cache) precisely so they stay +separate from `dedupeKey`, which is a de-duplication composite and not a provider +request id. + +`liveQualified` is `false` for every row in the matrix. The capability claims come +from the parsers and adapter cursor shapes in source, not from an installed-live +capture, and code that needs live evidence should say so. + +## The identity join + +A native session reaches a T3 thread through exactly one of two durable places, both +already persisted: + +- the current `resume_cursor_json` on `provider_session_runtime` — `{threadId}` for + Codex, `{resume}` for Claude, `{sessionId}` for ACP and OpenCode; +- `runtime_payload_json.importedTranscripts`, which accumulates imported Claude/Codex + file identities (including `providerSessionId`) and is the only historical binding + that survives an upsert. + +`provider_session_runtime` is one row per thread and the upsert overwrites +`resume_cursor_json`, so after a session switch, fork, or model-change restart only the +newest native id is durable. Usage from an earlier native id therefore becomes +unbound, and the projection reports it as `unallocated` rather than guessing an owner. +A thread → PR link comes from `projection_thread_pull_requests`, canonicalized with +`@t3tools/shared/threadPullRequests`; the projection does not resolve PRs itself. + +## Association is not attribution + +A session linked to two pull requests is reported once in the `shared` pool, which is +explicitly not additive, and is never cloned onto both PRs. Only sessions bound to +exactly one strong link contribute to a PR's `attributed` total. A `stack` link is a +display association, not evidence of billed work, so it feeds +`stackAssociationSessions` and never `attributed`; `stack-dismissed` tombstones are +ignored, matching `visibleThreadPullRequests`. Link changes therefore do not rewrite +past allocations — the projection is recomputed from the links that exist at read time. + +## Data quality and duplicate scans + +The output distinguishes a measured zero from an absent measurement: a known session +with no usage has `totals: null` and quality `missing`, and contributes nothing to any +pool, so a failed turn can never read as a zero-cost success. Malformed Claude session +ids are `invalid`; a session with some records lacking the level's id is `partial`. +Records are de-duplicated by `dedupeKey`, falling back to a content signature for +sourceless records such as Codex turns, so two environments scanning one transcript +directory cannot count it twice. The projection never rescans history; it consumes the +records the append-only scan cache already produced. + +## What still needs architecture approval + +The projection proves the join and the levels with fixtures. It does not choose a +storage or transport for the result and registers no endpoint. Durable per-thread +session history (an additive cursor/identity record rather than the single current +cursor) is the one schema change that would widen coverage, and it is deliberately not +adopted here. Provider-instance identity is likewise not recoverable from a transcript +scan; correlate that when the scan starts tagging files with the instance that produced +them. From ee732edd0edc771c60ce3629569c9fe16ff9ae05 Mon Sep 17 00:00:00 2001 From: nullstack65 Date: Wed, 23 Sep 2026 05:47:55 -0400 Subject: [PATCH 2/7] fix(usage): report absent request counts as null, not zero A known session with no measured usage reported equestCount: 0 while its quality was missing, which reads as a zero-request success. Return null unless the level's quality is measured or partial, so absence stays absent. Model/harness: opencode-go/deepseek-v4.1-flash via opencode. --- apps/server/src/usage/usageAttribution.test.ts | 3 +++ apps/server/src/usage/usageAttribution.ts | 12 ++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/apps/server/src/usage/usageAttribution.test.ts b/apps/server/src/usage/usageAttribution.test.ts index 9d129502b930..2d2a0fd863f0 100644 --- a/apps/server/src/usage/usageAttribution.test.ts +++ b/apps/server/src/usage/usageAttribution.test.ts @@ -221,6 +221,9 @@ describe("session binding and data quality", () => { expect(session.quality).toBe("missing"); expect(session.totals).toBeNull(); expect(session.allocation).toBe("missing"); + // An absent measurement is null, never a zero request count. + expect(session.requestCount).toBeNull(); + expect(session.promptCount).toBeNull(); expect(projection.coverage.find((entry) => entry.provider === "claude")?.missingSessions).toBe( 1, ); diff --git a/apps/server/src/usage/usageAttribution.ts b/apps/server/src/usage/usageAttribution.ts index 973870015cfa..c5f50b863774 100644 --- a/apps/server/src/usage/usageAttribution.ts +++ b/apps/server/src/usage/usageAttribution.ts @@ -614,8 +614,16 @@ export function buildUsageAttribution(input: UsageAttributionInput): UsageAttrib quality: sessionQuality, promptQuality, requestQuality, - promptCount: capability.prompt === "supported" ? accumulator.promptIds.size : null, - requestCount: capability.request === "supported" ? accumulator.requestIds.size : null, + promptCount: + capability.prompt === "supported" && + (promptQuality === "measured" || promptQuality === "partial") + ? accumulator.promptIds.size + : null, + requestCount: + capability.request === "supported" && + (requestQuality === "measured" || requestQuality === "partial") + ? accumulator.requestIds.size + : null, boundThreadIds, providerInstanceIds: instanceIds, bindingOrigins: origins, From db1a3db830ff903e5cf298f5f8c5f56c82c904fb Mon Sep 17 00:00:00 2001 From: nullstack65 Date: Wed, 23 Sep 2026 05:57:44 -0400 Subject: [PATCH 3/7] test(usage): assert the allocation reconciliation invariant Adds the METRICS-M1 invariant sum(attributed PR totals) + shared + unallocated = distinct measured total, and records that the legacy projection_thread_sessions.provider_session_id columns are not written by the live projector. Model/harness: opencode-go/deepseek-v4.1-flash via opencode. --- .../server/src/usage/usageAttribution.test.ts | 44 +++++++++++++++++++ docs/internals/usage-attribution.md | 8 +++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/apps/server/src/usage/usageAttribution.test.ts b/apps/server/src/usage/usageAttribution.test.ts index 2d2a0fd863f0..964d2848d981 100644 --- a/apps/server/src/usage/usageAttribution.test.ts +++ b/apps/server/src/usage/usageAttribution.test.ts @@ -380,6 +380,50 @@ describe("pull request association and attribution", () => { }); describe("projection contract", () => { + it("reconciles allocated + shared + unallocated to the distinct measured total", () => { + const projection = buildUsageAttribution( + input({ + records: [ + record({ + sessionId: CLAUDE_SESSION, + dedupeKey: "a", + totals: totals({ outputTokens: 100 }), + }), + codexRecord({ dedupeKey: null }), + record({ + provider: "grok", + sessionId: GROK_SESSION, + model: "grok-4.5", + promptId: "g1", + dedupeKey: "g1", + }), + ], + bindings: [ + binding({ threadId: "thread-1", provider: "claude", nativeSessionId: CLAUDE_SESSION }), + binding({ threadId: "thread-2", provider: "codex", nativeSessionId: CODEX_SESSION }), + binding({ threadId: "thread-3", provider: "grok", nativeSessionId: GROK_SESSION }), + ], + links: [ + link({ threadId: "thread-1", number: 12 }), + link({ threadId: "thread-2", number: 13 }), + link({ threadId: "thread-2", number: 14 }), + ], + }), + ); + + const distinct = projection.sessions.reduce( + (sum, session) => sum + (session.totals?.totalTokens ?? 0), + 0, + ); + const allocated = projection.pullRequests.reduce( + (sum, pr) => sum + pr.attributed.totalTokens, + 0, + ); + expect(allocated + projection.shared.totalTokens + projection.unallocated.totalTokens).toBe( + distinct, + ); + }); + it("does not mutate its inputs", () => { const records = [record({ dedupeKey: "a" })]; const bindings = [binding()]; diff --git a/docs/internals/usage-attribution.md b/docs/internals/usage-attribution.md index d6fb357bf7cd..efb5c6ea85fd 100644 --- a/docs/internals/usage-attribution.md +++ b/docs/internals/usage-attribution.md @@ -51,11 +51,17 @@ unbound, and the projection reports it as `unallocated` rather than guessing an A thread → PR link comes from `projection_thread_pull_requests`, canonicalized with `@t3tools/shared/threadPullRequests`; the projection does not resolve PRs itself. +`projection_thread_sessions` also has `provider_session_id` and `provider_thread_id` +columns, but the live upsert in `ProjectionThreadSessions.ts` never writes them, so +they carry no current mapping and must not be used as a join. + ## Association is not attribution A session linked to two pull requests is reported once in the `shared` pool, which is explicitly not additive, and is never cloned onto both PRs. Only sessions bound to -exactly one strong link contribute to a PR's `attributed` total. A `stack` link is a +exactly one strong link contribute to a PR's `attributed` total. The projection holds +`sum(attributed PR totals) + shared + unallocated = the distinct measured total`, so a +reader can reconcile every token exactly once. A `stack` link is a display association, not evidence of billed work, so it feeds `stackAssociationSessions` and never `attributed`; `stack-dismissed` tombstones are ignored, matching `visibleThreadPullRequests`. Link changes therefore do not rewrite From d1dd6821987dab544577d01826b2a4a1a81b9c04 Mon Sep 17 00:00:00 2001 From: nullstack65 Date: Wed, 23 Sep 2026 17:07:08 -0400 Subject: [PATCH 4/7] fix(usage): retain legacy cache history and record measurement presence A v3 scan cache is read instead of discarded, so measured records from deleted transcripts survive; v3 rows decode with native ids and presence explicitly unavailable and extant files are cold re-parsed to enrich them. Parsers record whether tokens were actually observed, and the occurrence-aware dedupe key moves to a shared seam reused by the scan. --- apps/server/src/usage/UsageService.ts | 28 +++-- apps/server/src/usage/usageScanCache.test.ts | 81 +++++++++++++- apps/server/src/usage/usageScanCache.ts | 103 ++++++++++++++++-- .../server/src/usage/usageTranscripts.test.ts | 44 ++++++++ apps/server/src/usage/usageTranscripts.ts | 78 +++++++++++++ 5 files changed, 312 insertions(+), 22 deletions(-) diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index 949155c650f2..2b7cb483a80c 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -61,7 +61,7 @@ import { pruneScanCache, type ScanCache, } from "./usageScanCache.ts"; -import type { UsageRecord } from "./usageTranscripts.ts"; +import { usageEventOccurrenceBaseKey, type UsageRecord } from "./usageTranscripts.ts"; const LITELLM_RATES_URL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"; @@ -101,7 +101,6 @@ const encodeRatesCache = Schema.encodeEffect( const ScanCacheJson = Schema.fromJsonString(Schema.Unknown as unknown as Schema.Codec); const decodeScanCacheFile = Schema.decodeUnknownEffect(ScanCacheJson); const encodeScanCacheFile = Schema.encodeEffect(ScanCacheJson); -const encodeUsageRecordKey = Schema.encodeSync(ScanCacheJson); const CachedSource = Schema.Struct({ dir: Schema.String, volumeId: Schema.String }); const decodeCachedSources = Schema.decodeUnknownOption( Schema.Struct({ sources: Schema.Record(Schema.String, CachedSource) }), @@ -406,9 +405,14 @@ export const make = Effect.gen(function* () { } // Only a strictly grown file may resume. Same size with a new mtime, or - // a shrunken file, means rewritten content; re-parse it whole. + // a shrunken file, means rewritten content; re-parse it whole. A legacy + // entry (ids/presence erased) is also re-parsed whole: resuming would + // keep serving id-less records and the enrichment would never happen. const resumeFrom = - cached !== undefined && cached.provider === provider && size > cached.size + cached !== undefined && + cached.provider === provider && + cached.identity === "declared" && + size > cached.size ? cached.position : undefined; @@ -436,6 +440,7 @@ export const make = Effect.gen(function* () { records, tailRecords, position: parsed.position, + identity: "declared", }); cacheDirty = true; return tailRecords.length === 0 ? records : [...records, ...tailRecords]; @@ -572,6 +577,10 @@ export const make = Effect.gen(function* () { } let scannedFiles = 0; let skippedFiles = 0; + // A usage container with no recognised token field (Claude `usage: {}`) + // parses to a record but measured nothing; surface it rather than letting + // it read as a measured zero. + let malformedRecords = 0; // Distinct per directory. Buckets carry per-cell session counts, but a // session spans days and models, so clients total this figure instead. const sessionIds = new Set(); @@ -584,17 +593,12 @@ export const make = Effect.gen(function* () { scannedFiles += 1; const codexEventOccurrences = new Map(); for (const record of file.records) { + if (record.measurement === "empty") malformedRecords += 1; let usageRecord = record; if (record.provider === "codex" && record.sessionId.length > 0) { // Match moved rollout copies without collapsing repeated equal events // within one rollout (timestamps can have only second precision). - const key = encodeUsageRecordKey([ - record.provider, - record.sessionId, - record.timestampMs, - record.model, - record.totals, - ]); + const key = usageEventOccurrenceBaseKey(record); const occurrence = (codexEventOccurrences.get(key) ?? 0) + 1; codexEventOccurrences.set(key, occurrence); usageRecord = { ...record, dedupeKey: key + ":" + occurrence }; @@ -613,7 +617,7 @@ export const make = Effect.gen(function* () { status: files === null && scannedFiles === 0 ? "missing" : "ok", scannedFiles, skippedFiles, - malformedRecords: 0, + malformedRecords, distinctSessions: sessionIds.size, message: files === null ? "No transcript directory on this environment." : null, }); diff --git a/apps/server/src/usage/usageScanCache.test.ts b/apps/server/src/usage/usageScanCache.test.ts index 23423dd0cd1c..3eb821cef674 100644 --- a/apps/server/src/usage/usageScanCache.test.ts +++ b/apps/server/src/usage/usageScanCache.test.ts @@ -25,6 +25,7 @@ function record(overrides: Partial = {}): UsageRecord { }, reportedCostUsd: null, dedupeKey: "msg_1:", + measurement: "observed", ...overrides, }; } @@ -49,6 +50,7 @@ function cacheWith(entries: readonly [string, number, readonly UsageRecord[]][]) records, tailRecords: [], position: position(), + identity: "declared", }); } return cache; @@ -69,6 +71,7 @@ describe("scan cache round trip", () => { ], tailRecords: [record({ provider: "grok", model: "grok-4.5-build", dedupeKey: null })], position: position({ resumeOffset: 30, guardLength: 30, guardHash: 123 }), + identity: "declared", }); original.set("/codex.jsonl", { size: 80, @@ -86,6 +89,7 @@ describe("scan cache round trip", () => { forkCopyAnchorMs: 0, }, }), + identity: "declared", }); const restored = decodeScanCache(JSON.parse(JSON.stringify(encodeScanCache(original)))); @@ -147,7 +151,7 @@ describe("scan cache round trip", () => { expect(decodeScanCache(JSON.parse(JSON.stringify(poisoned))).has("/a.jsonl")).toBe(false); }); - it("rejects a document from the previous cache version", () => { + it("rejects a v1/v2 document that predates the parse position", () => { const encoded = encodeScanCache(cacheWith([["/a.jsonl", 100, [record()]]])); const previous = { ...encoded, version: 2 }; @@ -212,6 +216,81 @@ describe("scan cache round trip", () => { }); }); +describe("legacy v3 cache history", () => { + const TS = 1_786_000_000_000; + + /** One deleted transcript (unrecoverable) and one extant transcript. */ + function v3Document(): unknown { + return { + version: 3, + models: ["claude-fable-5"], + sessions: ["deleted-session", "live-session"], + files: { + "/deleted.jsonl": { + s: 100, + m: 500, + p: "claude", + r: [[TS, 0, 0, 2, 1000, 10, 50, 0, "msg_d:", null]], + t: [], + o: 90, + gl: 64, + gh: 11, + cs: null, + }, + "/live.jsonl": { + s: 40, + m: 9000, + p: "claude", + r: [[TS, 0, 1, 0, 0, 0, 0, 0, null, null]], + t: [], + o: 30, + gl: 30, + gh: 22, + cs: null, + }, + }, + }; + } + + it("reads a v3 entry instead of discarding the retained history", () => { + const decoded = decodeScanCache(JSON.parse(JSON.stringify(v3Document()))); + + expect([...decoded.keys()].toSorted()).toEqual(["/deleted.jsonl", "/live.jsonl"]); + const deleted = decoded.get("/deleted.jsonl")!; + expect(deleted.identity).toBe("unavailable"); + expect(deleted.records[0]?.totals.outputTokens).toBe(50); + // Native ids are unavailable, not asserted as absent. + expect(deleted.records[0]?.providerRequestId).toBeUndefined(); + // A nonzero v3 row is still a known measurement. + expect(deleted.records[0]?.measurement).toBe("observed"); + }); + + it("keeps an all-zero v3 row explicitly unavailable, not a measured zero", () => { + const decoded = decodeScanCache(JSON.parse(JSON.stringify(v3Document()))); + + const live = decoded.get("/live.jsonl")!; + expect(live.identity).toBe("unavailable"); + expect(live.records[0]?.measurement).toBe("unavailable"); + }); + + it("persists the legacy marker so deleted history stays unavailable across restarts", () => { + const once = decodeScanCache(JSON.parse(JSON.stringify(v3Document()))); + const again = decodeScanCache(JSON.parse(JSON.stringify(encodeScanCache(once)))); + + expect(again.get("/deleted.jsonl")?.identity).toBe("unavailable"); + expect(again.get("/deleted.jsonl")?.records[0]?.totals.outputTokens).toBe(50); + }); + + it("marks a freshly re-parsed entry declared so it can resume and enrich", () => { + const encoded = encodeScanCache( + cacheWith([["/live.jsonl", 9000, [record({ sessionId: "live-session" })]]]), + ); + const decoded = decodeScanCache(JSON.parse(JSON.stringify(encoded))); + + expect(decoded.get("/live.jsonl")?.identity).toBe("declared"); + }); +}); + describe("pruneScanCache", () => { const retentionCutoffMs = 1000; diff --git a/apps/server/src/usage/usageScanCache.ts b/apps/server/src/usage/usageScanCache.ts index 4ab99128159a..eaeb60eed49e 100644 --- a/apps/server/src/usage/usageScanCache.ts +++ b/apps/server/src/usage/usageScanCache.ts @@ -17,7 +17,12 @@ import type { UsageProviderKind } from "@t3tools/contracts"; import { GUARD_LENGTH, type TranscriptParsePosition } from "./usageTranscriptReader.ts"; -import type { CodexScanState, UsageRecord } from "./usageTranscripts.ts"; +import type { + CodexScanState, + UsageMeasurement, + UsageObservationScope, + UsageRecord, +} from "./usageTranscripts.ts"; // v2: Codex fork-copy suppression changed what a file parses to, so v1 // entries would keep serving double-counted records forever. @@ -26,7 +31,22 @@ import type { CodexScanState, UsageRecord } from "./usageTranscripts.ts"; // v4: records carry native request/message/prompt ids. Without the bump, warm // v3 entries would silently report those levels as unsupported until the file // next changed. +// +// v3 documents are still *read*: the scan retains measured records from +// transcripts that have since been deleted, and those cannot be re-parsed, so +// discarding a v3 cache would destroy 90 days of history. A v3 row decodes with +// its native ids and measurement presence explicitly `unavailable`, and an +// extant file is cold re-parsed so those fields get filled in. Only v1/v2 (no +// parse position, different fork semantics) are rejected. const USAGE_SCAN_CACHE_VERSION = 4 as const; +const LEGACY_USAGE_SCAN_CACHE_VERSION = 3 as const; + +/** + * Whether a cache entry's rows still carry their native ids and measurement + * presence. A `v3` entry erased both; the projection must report them as + * unavailable rather than as a measured zero or an absent id. + */ +export type ScanCacheIdentity = "declared" | "unavailable"; export interface CachedFile { readonly size: number; @@ -41,6 +61,11 @@ export interface CachedFile { */ readonly tailRecords: readonly UsageRecord[]; readonly position: TranscriptParsePosition; + /** + * `unavailable` for a legacy row whose ids/presence were erased. Callers must + * not resume such an entry: a cold re-parse is the only way to enrich it. + */ + readonly identity: ScanCacheIdentity; } export type ScanCache = Map; @@ -64,8 +89,23 @@ type SerializedRecord = readonly [ providerRequestId: string | null, providerMessageId: string | null, promptId: string | null, + measurementCode: number, + scopeCode: number, ]; +const MEASUREMENT_CODES: readonly UsageMeasurement[] = ["observed", "empty", "unavailable"]; +const SCOPE_CODES: readonly UsageObservationScope[] = ["delta", "snapshot"]; + +function encodeMeasurement(measurement: UsageMeasurement | undefined): number { + const index = MEASUREMENT_CODES.indexOf(measurement ?? "observed"); + return index < 0 ? 0 : index; +} + +function encodeScope(scope: UsageObservationScope | undefined): number { + const index = SCOPE_CODES.indexOf(scope ?? "delta"); + return index < 0 ? 0 : index; +} + interface SerializedFile { readonly s: number; readonly m: number; @@ -79,6 +119,12 @@ interface SerializedFile { readonly gh: number; /** Codex reducer state at `o`; `null` for stateless providers. */ readonly cs: CodexScanState | null; + /** + * `1` when this entry's rows predate native ids / measurement presence. Kept + * on the file so an erased-history entry stays `unavailable` across restarts + * until an extant file is cold re-parsed. Absent on a fresh entry. + */ + readonly li?: number; } interface SerializedCache { @@ -118,6 +164,8 @@ export function encodeScanCache(cache: ScanCache): SerializedCache { record.providerRequestId ?? null, record.providerMessageId ?? null, record.promptId ?? null, + encodeMeasurement(record.measurement), + encodeScope(record.scope), ]; const files: Record = {}; @@ -132,6 +180,7 @@ export function encodeScanCache(cache: ScanCache): SerializedCache { gl: entry.position.guardLength, gh: entry.position.guardHash, cs: entry.position.codexState, + ...(entry.identity === "unavailable" ? { li: 1 } : {}), }; } @@ -153,7 +202,15 @@ export function decodeScanCache(document: unknown): ScanCache { if (typeof document !== "object" || document === null) return cache; const root = document as Partial; - if (root.version !== USAGE_SCAN_CACHE_VERSION) return cache; + if ( + root.version !== USAGE_SCAN_CACHE_VERSION && + root.version !== LEGACY_USAGE_SCAN_CACHE_VERSION + ) { + return cache; + } + // A v3 document has no native ids or measurement presence anywhere; every + // entry it holds is erased-history. A v4 entry carries its own marker. + const legacyDocument = root.version === LEGACY_USAGE_SCAN_CACHE_VERSION; if (!isRecordArray(root.models) || !isRecordArray(root.sessions)) return cache; if (typeof root.files !== "object" || root.files === null) return cache; @@ -171,6 +228,7 @@ export function decodeScanCache(document: unknown): ScanCache { const decodeRecords = ( rows: readonly unknown[], provider: UsageProviderKind, + legacy: boolean, ): UsageRecord[] | null => { const records: UsageRecord[] = []; for (const row of rows) { @@ -192,6 +250,11 @@ export function decodeScanCache(document: unknown): ScanCache { const providerRequestId = row[10]; const providerMessageId = row[11]; const promptId = row[12]; + // Appended after v4. A v3 or early-v4 row has no presence information, so + // a nonzero total proves a measurement while an all-zero row stays + // explicitly `unavailable` rather than being read as a measured zero. + const measurementCode = row[13]; + const scopeCode = row[14]; const model = typeof modelIndex === "number" ? models[modelIndex] : undefined; if ( @@ -207,6 +270,21 @@ export function decodeScanCache(document: unknown): ScanCache { return null; } + const measurement: UsageMeasurement = + typeof measurementCode === "number" && + Number.isSafeInteger(measurementCode) && + MEASUREMENT_CODES[measurementCode] !== undefined + ? MEASUREMENT_CODES[measurementCode]! + : uncached + cached + cacheCreation + output > 0 + ? "observed" + : "unavailable"; + const scope: UsageObservationScope = + typeof scopeCode === "number" && + Number.isSafeInteger(scopeCode) && + SCOPE_CODES[scopeCode] !== undefined + ? SCOPE_CODES[scopeCode]! + : "delta"; + records.push({ provider, timestampMs, @@ -221,11 +299,16 @@ export function decodeScanCache(document: unknown): ScanCache { }, reportedCostUsd: typeof reportedCostUsd === "number" ? reportedCostUsd : null, dedupeKey: typeof dedupeKey === "string" ? dedupeKey : null, - // Omitted when absent so a record round-trips identically to one the - // parser produced without the field. - ...(typeof providerRequestId === "string" ? { providerRequestId } : {}), - ...(typeof providerMessageId === "string" ? { providerMessageId } : {}), - ...(typeof promptId === "string" ? { promptId } : {}), + // A v3 row cannot carry native ids; they are unavailable, not absent. + ...(legacy + ? {} + : { + ...(typeof providerRequestId === "string" ? { providerRequestId } : {}), + ...(typeof providerMessageId === "string" ? { providerMessageId } : {}), + ...(typeof promptId === "string" ? { promptId } : {}), + }), + measurement, + ...(scope === "delta" ? {} : { scope }), }); } return records; @@ -259,8 +342,9 @@ export function decodeScanCache(document: unknown): ScanCache { if (codexState === undefined) continue; const provider: UsageProviderKind = entry.p; - const records = decodeRecords(entry.r, provider); - const tailRecords = decodeRecords(entry.t, provider); + const legacy = legacyDocument || entry.li === 1; + const records = decodeRecords(entry.r, provider, legacy); + const tailRecords = decodeRecords(entry.t, provider, legacy); if (records === null || tailRecords === null) continue; cache.set(path, { @@ -275,6 +359,7 @@ export function decodeScanCache(document: unknown): ScanCache { guardHash: entry.gh, codexState, }, + identity: legacy ? "unavailable" : "declared", }); } diff --git a/apps/server/src/usage/usageTranscripts.test.ts b/apps/server/src/usage/usageTranscripts.test.ts index b9b582eb9de2..1a4027529d20 100644 --- a/apps/server/src/usage/usageTranscripts.test.ts +++ b/apps/server/src/usage/usageTranscripts.test.ts @@ -81,6 +81,50 @@ describe("parseClaudeLine", () => { expect(record?.providerRequestId).toBe("req_9"); expect(record?.providerMessageId).toBe("msg_9"); expect(record?.promptId).toBeNull(); + expect(record?.measurement).toBe("observed"); + }); + + it("marks an empty usage container as empty, not a measured zero", () => { + const line = JSON.stringify({ + type: "assistant", + timestamp: "2026-08-07T04:05:13.944Z", + sessionId: "5a128faa-8253-489e-b935-6c08e8e670c0", + message: { id: "msg_empty", model: "claude-fable-5", usage: {} }, + }); + + const record = parseClaudeLine(line); + + expect(record).not.toBeNull(); + expect(record?.measurement).toBe("empty"); + expect(record?.totals).toEqual({ + uncachedInputTokens: 0, + cachedInputTokens: 0, + cacheCreationTokens: 0, + outputTokens: 0, + reasoningTokens: 0, + }); + }); + + it("treats an explicit zero as an observed measurement", () => { + const line = JSON.stringify({ + type: "assistant", + timestamp: "2026-08-07T04:05:13.944Z", + sessionId: "5a128faa-8253-489e-b935-6c08e8e670c0", + message: { + id: "msg_zero", + model: "claude-fable-5", + usage: { + input_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + output_tokens: 0, + }, + }, + }); + + const record = parseClaudeLine(line); + + expect(record?.measurement).toBe("observed"); }); }); diff --git a/apps/server/src/usage/usageTranscripts.ts b/apps/server/src/usage/usageTranscripts.ts index 7d0c3ba28306..74468946e3e3 100644 --- a/apps/server/src/usage/usageTranscripts.ts +++ b/apps/server/src/usage/usageTranscripts.ts @@ -8,6 +8,31 @@ */ import type { UsageProviderKind, UsageTokenTotals } from "@t3tools/contracts"; +/** + * Whether the source actually measured tokens, as opposed to handing us a + * container we normalised to zeros. + * + * - `observed` — at least one recognised token field was present. An explicit + * `0` is a real measured zero and stays `observed`. + * - `empty` — a usage container existed but carried no recognised token field + * (for example Claude's `usage: {}`). Numeric totals are zero, but that is a + * missing measurement, not a measured zero. + * - `unavailable` — the presence information was erased before we saw the + * record (a legacy cache row). The zeros may be real or may be missing; we + * must not classify them either way. + */ +export type UsageMeasurement = "observed" | "empty" | "unavailable"; + +/** + * How a record relates to other records for the same identity. + * + * - `delta` — an additive increment (the default for every parser here). + * - `snapshot` — a cumulative observation that *replaces* an earlier value for + * the same identity rather than adding to it. A source that defines updates + * sets this; the projection then keeps the newest instead of summing. + */ +export type UsageObservationScope = "delta" | "snapshot"; + export interface UsageRecord { readonly provider: UsageProviderKind; readonly timestampMs: number; @@ -39,6 +64,37 @@ export interface UsageRecord { * `turn_completed.prompt_id` identifies the user prompt a turn answers. */ readonly promptId?: string | null; + /** + * Whether the source actually measured this record. Absent means the parser + * observed recognised fields; a legacy cache row sets `unavailable` + * explicitly. See {@link UsageMeasurement}. + */ + readonly measurement?: UsageMeasurement; + /** Additive increment or replaceable snapshot. Absent means `delta`. */ + readonly scope?: UsageObservationScope; +} + +/** + * The occurrence-aware identity seam. + * + * Two records with the same value here are the same *event shape* in the same + * session. Callers append a per-delivery occurrence index to distinguish + * repeated equal events from a re-delivery of one event: a copy of a rollout + * restarts its occurrence counter, so the copy lands on the same composite key + * and is de-duplicated, while two genuine equal events in one file land on + * different keys and are both kept. This is the identity the scan cache stamps + * onto otherwise-keyless records (see `UsageService`); it is deliberately + * separate from the native request/message/prompt ids, which are reporting + * values and not delivery identity. + */ +export function usageEventOccurrenceBaseKey(record: UsageRecord): string { + return JSON.stringify([ + record.provider, + record.sessionId, + record.timestampMs, + record.model, + record.totals, + ]); } const EMPTY_TOTALS: UsageTokenTotals = { @@ -107,6 +163,14 @@ function grokCostTicksToUsd(ticks: unknown): number | null { /* Claude Code */ /* -------------------------------------------------------------------------- */ +/** Token fields that make a Claude `usage` object an actual measurement. */ +const CLAUDE_USAGE_FIELDS = [ + "input_tokens", + "cache_read_input_tokens", + "cache_creation_input_tokens", + "output_tokens", +] as const; + /** * Parses one line of a Claude Code transcript. * @@ -150,6 +214,15 @@ export function parseClaudeLine(line: string): UsageRecord | null { const cost = record["costUSD"]; + // `usage: {}` normalises to zeros but is not a measured zero. Only a + // recognised token field makes this an observed measurement; an explicit + // `input_tokens: 0` still counts as observed. + const measurement: UsageMeasurement = CLAUDE_USAGE_FIELDS.some((field) => + Object.hasOwn(usageRecord, field), + ) + ? "observed" + : "empty"; + return { provider: "claude", timestampMs, @@ -171,6 +244,7 @@ export function parseClaudeLine(line: string): UsageRecord | null { providerRequestId: requestId, providerMessageId: messageId, promptId: null, + measurement, }; } @@ -337,6 +411,8 @@ export function parseCodexLine(line: string, state: CodexScanState): UsageRecord providerRequestId: null, providerMessageId: null, promptId: null, + // Only emitted when at least one token was measured, so this is observed. + measurement: "observed", }; } @@ -469,6 +545,7 @@ export function parseGrokLine(line: string): readonly UsageRecord[] { providerRequestId: null, providerMessageId: null, promptId, + measurement: "observed", }, ]; } @@ -518,6 +595,7 @@ export function parseGrokLine(line: string): readonly UsageRecord[] { providerRequestId: null, providerMessageId: null, promptId, + measurement: "observed", }); } return results; From e4f36af5ef279246bcb0f8463adeee8a09b7bde1 Mon Sep 17 00:00:00 2001 From: nullstack65 Date: Wed, 23 Sep 2026 17:07:18 -0400 Subject: [PATCH 5/7] fix(usage): reconcile attribution and prove persisted bindings Preserve orphan usage in an explicit bucket and reconcile attributed + shared + unallocated + orphan against the deduplicated input. Replace content-equality dedupe with a provider-namespaced declared key plus occurrence-aware scan identity, keeping unkeyed records uncertain and surfacing conflicts and snapshot replacement. Split identity, measurement, level support, and allocation; seed coverage from declared sources; retain per-model and cost provenance. Add a read-only extraction seam from provider_session_runtime and projection_thread_pull_requests. --- .../server/src/usage/usageAttribution.test.ts | 440 +++++++++++++-- apps/server/src/usage/usageAttribution.ts | 524 ++++++++++++++---- .../src/usage/usageAttributionSources.test.ts | 270 +++++++++ .../src/usage/usageAttributionSources.ts | 390 +++++++++++++ docs/internals/usage-attribution.md | 143 +++-- 5 files changed, 1569 insertions(+), 198 deletions(-) create mode 100644 apps/server/src/usage/usageAttributionSources.test.ts create mode 100644 apps/server/src/usage/usageAttributionSources.ts diff --git a/apps/server/src/usage/usageAttribution.test.ts b/apps/server/src/usage/usageAttribution.test.ts index 964d2848d981..9b0d2f96f939 100644 --- a/apps/server/src/usage/usageAttribution.test.ts +++ b/apps/server/src/usage/usageAttribution.test.ts @@ -12,12 +12,14 @@ import { type AttributionUsageRecord, type UsageAttributionInput, } from "./usageAttribution.ts"; +import { totalTokens } from "./usageTranscripts.ts"; const CLAUDE_SESSION = "5a128faa-8253-489e-b935-6c08e8e670c0"; const OTHER_CLAUDE_SESSION = "11111111-2222-3333-4444-555555555555"; const CODEX_SESSION = "019fbbc1-b12c-7360-a685-28c181f0025f"; const GROK_SESSION = "019fec1a-12f7-72f2-9b1f-7778a00aea3c"; const CLAUDE_FINGERPRINT = "host\u0000claude\u0000/home/u/.claude\u00000:1"; +const OTHER_FINGERPRINT = "host\u0000claude\u0000/home/u/.claude-copy\u00000:2"; const CODEX_FINGERPRINT = "host\u0000codex\u0000/home/u/.codex\u00000:2"; function totals(overrides: Partial = {}): UsageTokenTotals { @@ -31,6 +33,20 @@ function totals(overrides: Partial = {}): UsageTokenTotals { }; } +function zeroTotals(): UsageTokenTotals { + return { + uncachedInputTokens: 0, + cachedInputTokens: 0, + cacheCreationTokens: 0, + outputTokens: 0, + reasoningTokens: 0, + }; +} + +function tokensOf(record: AttributionUsageRecord): number { + return totalTokens(record.totals); +} + function record(overrides: Partial = {}): AttributionUsageRecord { return { provider: "claude", @@ -39,11 +55,12 @@ function record(overrides: Partial = {}): AttributionUsa timestampMs: 1_786_000_000_000, totals: totals(), costUsd: 0.01, - dedupeKey: null, + dedupeKey: "msg_1:req_1", providerRequestId: null, providerMessageId: null, promptId: null, sourceFingerprint: CLAUDE_FINGERPRINT, + measurement: "observed", ...overrides, }; } @@ -54,6 +71,7 @@ function codexRecord(overrides: Partial = {}): Attributi sessionId: CODEX_SESSION, model: "gpt-5.6-sol", sourceFingerprint: CODEX_FINGERPRINT, + dedupeKey: "codex-occurrence:1", ...overrides, }); } @@ -112,6 +130,7 @@ describe("prompt and request granularity", () => { expect(projection.requests).toHaveLength(3); expect(projection.prompts).toHaveLength(0); expect(session.totals?.records).toBe(3); + expect(projection.identity.duplicatesDropped).toBe(1); }); it("groups grok usage by prompt across several models", () => { @@ -160,6 +179,7 @@ describe("prompt and request granularity", () => { const prompt1 = projection.prompts.find((prompt) => prompt.promptId === "p1")!; expect(prompt1.models).toEqual(["grok-4.5", "grok-fast"]); expect(prompt1.totals.records).toBe(2); + expect(prompt1.modelContributions).toHaveLength(2); }); it("never reports request or prompt counts for turn-only codex usage", () => { @@ -218,7 +238,8 @@ describe("session binding and data quality", () => { const projection = buildUsageAttribution(input({ bindings: [binding()] })); const session = projection.sessions[0]!; - expect(session.quality).toBe("missing"); + expect(session.identityQuality).toBe("valid"); + expect(session.measurementQuality).toBe("missing"); expect(session.totals).toBeNull(); expect(session.allocation).toBe("missing"); // An absent measurement is null, never a zero request count. @@ -230,19 +251,67 @@ describe("session binding and data quality", () => { expect(projection.unallocated.records).toBe(0); }); - it("marks a malformed claude session id invalid", () => { + it("marks a malformed claude session id invalid on the identity axis", () => { const projection = buildUsageAttribution( input({ records: [record({ sessionId: "not-a-uuid", dedupeKey: "a" })] }), ); - expect(projection.sessions[0]?.quality).toBe("invalid"); + const session = projection.sessions[0]!; + expect(session.identityQuality).toBe("invalid"); + expect(session.measurementQuality).toBe("measured"); expect(projection.coverage.find((entry) => entry.provider === "claude")?.invalidSessions).toBe( 1, ); }); - it("de-duplicates identical codex records from two scans of one source", () => { - const scanned = codexRecord({ dedupeKey: null }); + it("notes duplicate source fingerprints without double counting", () => { + const source: AttributionSource = { + fingerprint: CLAUDE_FINGERPRINT, + provider: "claude", + status: "ok", + distinctSessions: 1, + }; + const projection = buildUsageAttribution( + input({ + records: [record({ dedupeKey: "a" })], + sources: [source, { ...source }], + }), + ); + + expect(projection.sessions[0]?.totals?.records).toBe(1); + expect(projection.coverage.find((entry) => entry.provider === "claude")).toMatchObject({ + declaredSources: 2, + distinctSourceFingerprints: 1, + sourceStatus: { ok: 2, missing: 0, partial: 0, failed: 0 }, + }); + expect( + projection.limitations.some((line) => line.includes("duplicate source fingerprint")), + ).toBe(true); + }); +}); + +describe("identity: repeated deliveries, occurrences, copies, conflicts", () => { + it("counts two equal keyless occurrences instead of collapsing them", () => { + const occurrence = codexRecord({ dedupeKey: null, timestampMs: 1_786_000_000_000 }); + const projection = buildUsageAttribution( + input({ + records: [occurrence, { ...occurrence }], + bindings: [binding({ provider: "codex", nativeSessionId: CODEX_SESSION })], + }), + ); + + const session = projection.sessions[0]!; + expect(session.totals?.records).toBe(2); + expect(session.recordIdentity).toBe("uncertain"); + expect(projection.identity.unkeyedRecords).toBe(2); + expect(projection.coverage.find((entry) => entry.provider === "codex")?.unkeyedRecords).toBe(2); + expect(projection.limitations.some((line) => line.includes("no scan/delivery identity"))).toBe( + true, + ); + }); + + it("collapses a repeated scan by its stamped delivery identity", () => { + const scanned = codexRecord({ dedupeKey: "occurrence-key:1" }); const projection = buildUsageAttribution( input({ records: [scanned, { ...scanned }], @@ -251,26 +320,247 @@ describe("session binding and data quality", () => { ); expect(projection.sessions[0]?.totals?.records).toBe(1); + expect(projection.identity.duplicatesDropped).toBe(1); + expect(projection.sessions[0]?.recordIdentity).toBe("exact"); }); - it("notes duplicate source fingerprints without double counting", () => { + it("collapses copied history from another physical source without inflating", () => { + const original = record({ dedupeKey: "m1:r1", sourceFingerprint: CLAUDE_FINGERPRINT }); + const copy = record({ dedupeKey: "m1:r1", sourceFingerprint: OTHER_FINGERPRINT }); + const projection = buildUsageAttribution( + input({ records: [original, copy], bindings: [binding()] }), + ); + + expect(projection.sessions[0]?.totals?.records).toBe(1); + expect(projection.identity.duplicatesDropped).toBe(1); + // Both physical sources are still visible in coverage. + expect( + projection.coverage.find((entry) => entry.provider === "claude")?.distinctSourceFingerprints, + ).toBe(2); + }); + + it("namespaces a declared key by provider so equal local ids do not collide", () => { + const claudeRecord = record({ provider: "claude", dedupeKey: "1", sessionId: CLAUDE_SESSION }); + const codexRecordValue = codexRecord({ dedupeKey: "1" }); + const projection = buildUsageAttribution( + input({ + records: [claudeRecord, codexRecordValue], + bindings: [ + binding({ provider: "claude", nativeSessionId: CLAUDE_SESSION }), + binding({ provider: "codex", nativeSessionId: CODEX_SESSION }), + ], + }), + ); + + expect(projection.sessions).toHaveLength(2); + expect(projection.identity.duplicatesDropped).toBe(0); + expect(projection.identity.conflicts).toBe(0); + }); + + it("exposes conflicting versions of one observation instead of dropping one", () => { + const first = record({ dedupeKey: "m1:r1", totals: totals({ outputTokens: 10 }) }); + const conflicting = record({ dedupeKey: "m1:r1", totals: totals({ outputTokens: 999 }) }); + const projection = buildUsageAttribution( + input({ records: [first, conflicting], bindings: [binding()] }), + ); + + const session = projection.sessions[0]!; + expect(session.totals?.records).toBe(1); + expect(session.totals?.tokens.outputTokens).toBe(10); + expect(session.conflict).toBe(true); + expect(projection.identity.conflicts).toBe(1); + expect( + projection.coverage.find((entry) => entry.provider === "claude")?.conflictingRecords, + ).toBe(1); + expect(projection.limitations.some((line) => line.includes("identity conflict"))).toBe(true); + }); + + it("lets a snapshot observation replace an earlier value for the same identity", () => { + const delta = record({ + dedupeKey: "snap:1", + scope: "delta", + totals: totals({ outputTokens: 10 }), + }); + const snapshot = record({ + dedupeKey: "snap:1", + scope: "snapshot", + totals: totals({ outputTokens: 99 }), + }); + const projection = buildUsageAttribution( + input({ records: [delta, snapshot], bindings: [binding()] }), + ); + + const session = projection.sessions[0]!; + expect(projection.identity.snapshotsReplaced).toBe(1); + expect(projection.identity.conflicts).toBe(0); + expect(session.totals?.records).toBe(1); + expect(session.totals?.tokens.outputTokens).toBe(99); + }); +}); + +describe("measurement quality", () => { + it("treats a Claude usage:{} record as invalid, not a measured zero", () => { + const empty = record({ dedupeKey: "m1:", measurement: "empty", totals: zeroTotals() }); + const projection = buildUsageAttribution(input({ records: [empty], bindings: [binding()] })); + + const session = projection.sessions[0]!; + expect(session.measurementQuality).toBe("invalid"); + expect(session.totals?.totalTokens).toBe(0); + // A request id was present, so the request level is not "unsupported". + expect(session.requestQuality).not.toBe("unsupported"); + }); + + it("keeps an explicit zero as measured", () => { + const explicitZero = record({ + dedupeKey: "m1:", + measurement: "observed", + totals: zeroTotals(), + }); + const projection = buildUsageAttribution( + input({ records: [explicitZero], bindings: [binding()] }), + ); + + expect(projection.sessions[0]?.measurementQuality).toBe("measured"); + }); + + it("keeps a legacy all-zero row unavailable, not missing and not measured", () => { + const legacy = record({ + dedupeKey: "legacy:1", + measurement: "unavailable", + totals: zeroTotals(), + }); + const projection = buildUsageAttribution(input({ records: [legacy], bindings: [binding()] })); + + const session = projection.sessions[0]!; + expect(session.measurementQuality).toBe("unavailable"); + expect(session.totals?.records).toBe(1); + }); + + it("marks a mix of observed and empty records partial", () => { + const projection = buildUsageAttribution( + input({ + records: [ + record({ dedupeKey: "a" }), + record({ dedupeKey: "b", measurement: "empty", totals: zeroTotals() }), + ], + bindings: [binding()], + }), + ); + + expect(projection.sessions[0]?.measurementQuality).toBe("partial"); + }); + + it("surfaces a failed declared source instead of reading it as measured", () => { const source: AttributionSource = { fingerprint: CLAUDE_FINGERPRINT, provider: "claude", - status: "ok", - distinctSessions: 1, + status: "failed", + distinctSessions: 0, }; + const projection = buildUsageAttribution(input({ sources: [source] })); + + const coverage = projection.coverage.find((entry) => entry.provider === "claude")!; + expect(coverage.declaredSources).toBe(1); + expect(coverage.sourceStatus.failed).toBe(1); + expect(coverage.measuredSessions).toBe(0); + expect(projection.limitations.some((line) => line.includes("missing or failed"))).toBe(true); + }); +}); + +describe("orphan usage and reconciliation", () => { + it("preserves usage with no session id in an explicit orphan bucket", () => { + const orphanRecord = record({ + sessionId: "", + dedupeKey: "orphan-1", + totals: totals({ outputTokens: 7 }), + }); + const projection = buildUsageAttribution(input({ records: [orphanRecord] })); + + expect(projection.orphan.records).toBe(1); + expect(projection.orphan.totalTokens).toBe(tokensOf(orphanRecord)); + expect(projection.measured.totalTokens).toBe(projection.orphan.totalTokens); + expect(projection.unallocated.totalTokens).toBe(0); + expect(projection.sessions).toHaveLength(0); + expect( + projection.coverage.find((entry) => entry.provider === "claude")?.recordsWithoutSessionId, + ).toBe(1); + expect(projection.identity.orphanRecords).toBe(1); + }); + + it("reconciles attributed + shared + unallocated + orphan to the deduplicated input", () => { + const attributed = record({ + sessionId: CLAUDE_SESSION, + dedupeKey: "a", + totals: totals({ outputTokens: 100 }), + }); + const sharedRecord = codexRecord({ dedupeKey: "b", totals: totals({ outputTokens: 50 }) }); + const unallocatedRecord = record({ + provider: "grok", + sessionId: GROK_SESSION, + model: "grok-4.5", + promptId: "g1", + dedupeKey: "g1", + totals: totals({ outputTokens: 20 }), + }); + const orphanRecord = record({ + sessionId: "", + dedupeKey: "e", + totals: totals({ outputTokens: 5 }), + }); + // A repeated delivery of `attributed` must not add to the expected total. + const duplicate = record({ + sessionId: CLAUDE_SESSION, + dedupeKey: "a", + totals: totals({ outputTokens: 100 }), + }); + const projection = buildUsageAttribution( input({ - records: [record({ dedupeKey: "a" })], - sources: [source, { ...source }], + records: [attributed, sharedRecord, unallocatedRecord, orphanRecord, duplicate], + bindings: [ + binding({ threadId: "thread-1", provider: "claude", nativeSessionId: CLAUDE_SESSION }), + binding({ threadId: "thread-2", provider: "codex", nativeSessionId: CODEX_SESSION }), + binding({ threadId: "thread-3", provider: "grok", nativeSessionId: GROK_SESSION }), + ], + links: [ + link({ threadId: "thread-1", number: 12 }), + link({ threadId: "thread-2", number: 13 }), + link({ threadId: "thread-2", number: 14 }), + ], }), ); - expect(projection.sessions[0]?.totals?.records).toBe(1); + // Independent known input truth: every distinct input record, including the + // orphan, and excluding the exact duplicate. + const expected = + tokensOf(attributed) + + tokensOf(sharedRecord) + + tokensOf(unallocatedRecord) + + tokensOf(orphanRecord); + const allocated = projection.pullRequests.reduce( + (sum, pr) => sum + pr.attributed.totalTokens, + 0, + ); + + expect(projection.measured.totalTokens).toBe(expected); expect( - projection.limitations.some((line) => line.includes("duplicate source fingerprint")), - ).toBe(true); + allocated + + projection.shared.totalTokens + + projection.unallocated.totalTokens + + projection.orphan.totalTokens, + ).toBe(expected); + expect(projection.identity.duplicatesDropped).toBe(1); + }); + + it("does not mutate its inputs", () => { + const records = [record({ dedupeKey: "a" })]; + const bindings = [binding()]; + const links = [link()]; + const before = JSON.stringify({ records, bindings, links }); + + buildUsageAttribution(input({ records, bindings, links })); + + expect(JSON.stringify({ records, bindings, links })).toBe(before); }); }); @@ -284,7 +574,7 @@ describe("pull request association and attribution", () => { dedupeKey: "a", totals: totals({ outputTokens: 100 }), }), - codexRecord({ dedupeKey: null, totals: totals({ outputTokens: 50 }) }), + codexRecord({ dedupeKey: "b", totals: totals({ outputTokens: 50 }) }), ], bindings: [ binding({ threadId: "thread-1", provider: "claude", nativeSessionId: CLAUDE_SESSION }), @@ -357,7 +647,7 @@ describe("pull request association and attribution", () => { it("pools unlinked and ambiguous sessions as unallocated", () => { const projection = buildUsageAttribution( input({ - records: [record({ dedupeKey: "a" }), codexRecord({ dedupeKey: null })], + records: [record({ dedupeKey: "a" }), codexRecord({ dedupeKey: "b" })], bindings: [ binding({ threadId: "thread-1", provider: "claude", nativeSessionId: CLAUDE_SESSION }), binding({ threadId: "thread-2", provider: "codex", nativeSessionId: CODEX_SESSION }), @@ -377,68 +667,99 @@ describe("pull request association and attribution", () => { 1, ); }); -}); -describe("projection contract", () => { - it("reconciles allocated + shared + unallocated to the distinct measured total", () => { + it("preserves per-model contributions at the session and PR level", () => { const projection = buildUsageAttribution( input({ records: [ record({ - sessionId: CLAUDE_SESSION, + model: "claude-opus-5", dedupeKey: "a", - totals: totals({ outputTokens: 100 }), + totals: totals({ outputTokens: 10 }), + costUsd: 0.1, + costSource: "modelPriced", }), - codexRecord({ dedupeKey: null }), record({ - provider: "grok", - sessionId: GROK_SESSION, - model: "grok-4.5", - promptId: "g1", - dedupeKey: "g1", + model: "claude-fable-5", + dedupeKey: "b", + totals: totals({ outputTokens: 20 }), + costUsd: 0.2, + costSource: "providerReported", }), ], - bindings: [ - binding({ threadId: "thread-1", provider: "claude", nativeSessionId: CLAUDE_SESSION }), - binding({ threadId: "thread-2", provider: "codex", nativeSessionId: CODEX_SESSION }), - binding({ threadId: "thread-3", provider: "grok", nativeSessionId: GROK_SESSION }), - ], - links: [ - link({ threadId: "thread-1", number: 12 }), - link({ threadId: "thread-2", number: 13 }), - link({ threadId: "thread-2", number: 14 }), - ], + bindings: [binding({ threadId: "thread-1" })], + links: [link({ threadId: "thread-1", number: 12 })], }), ); - const distinct = projection.sessions.reduce( - (sum, session) => sum + (session.totals?.totalTokens ?? 0), - 0, - ); - const allocated = projection.pullRequests.reduce( - (sum, pr) => sum + pr.attributed.totalTokens, - 0, + const session = projection.sessions[0]!; + expect(session.models).toEqual(["claude-fable-5", "claude-opus-5"]); + expect(session.modelContributions.map((entry) => entry.model)).toEqual([ + "claude-fable-5", + "claude-opus-5", + ]); + expect( + session.modelContributions.find((entry) => entry.model === "claude-opus-5")?.costSource, + ).toBe("modelPriced"); + const pr = projection.pullRequests[0]!; + expect(pr.attributedModelContributions).toHaveLength(2); + // 100 uncached + 10 cached + 20 output; reasoning is a subset of output. + expect( + pr.attributedModelContributions.find((entry) => entry.model === "claude-fable-5") + ?.totalTokens, + ).toBe(130); + }); + + it("recomputes allocation from the links present at read time", () => { + const records = [record({ dedupeKey: "a", totals: totals({ outputTokens: 100 }) })]; + const bindings = [binding({ threadId: "thread-1" })]; + + const only12 = buildUsageAttribution( + input({ records, bindings, links: [link({ threadId: "thread-1", number: 12 })] }), ); - expect(allocated + projection.shared.totalTokens + projection.unallocated.totalTokens).toBe( - distinct, + expect(only12.sessions[0]?.allocation).toBe("attributed"); + expect(only12.association).toMatchObject({ + basis: "links-at-read-time", + linkedAtGovernsAllocation: false, + }); + + const both = buildUsageAttribution( + input({ + records, + bindings, + links: [ + link({ threadId: "thread-1", number: 12 }), + link({ threadId: "thread-1", number: 13 }), + ], + }), ); + expect(both.sessions[0]?.allocation).toBe("shared"); + expect(both.shared.records).toBe(1); + expect(both.limitations.some((line) => line.includes("does not gate allocation"))).toBe(true); }); - it("does not mutate its inputs", () => { - const records = [record({ dedupeKey: "a" })]; - const bindings = [binding()]; - const links = [link()]; - const before = JSON.stringify({ records, bindings, links }); - - buildUsageAttribution(input({ records, bindings, links })); + it("associates pre-link implementation work with a later link", () => { + const workRanAtMs = 1_786_000_000_000; + const projection = buildUsageAttribution( + input({ + generatedAtMs: workRanAtMs + 30 * 24 * 60 * 60 * 1000, + records: [record({ dedupeKey: "a", timestampMs: workRanAtMs })], + bindings: [binding({ threadId: "thread-1" })], + // Linked long after the work ran; allocation ignores `linkedAt`. + links: [link({ threadId: "thread-1", number: 12, linkedAt: "2026-09-20T00:00:00.000Z" })], + }), + ); - expect(JSON.stringify({ records, bindings, links })).toBe(before); + expect(projection.sessions[0]?.allocation).toBe("attributed"); + expect(projection.pullRequests[0]?.attributed.records).toBe(1); }); +}); +describe("projection contract", () => { it("exposes the source capability matrix with live qualification falsy", () => { const projection = buildUsageAttribution(input({})); - expect(USAGE_ATTRIBUTION_VERSION).toBe(1); - expect(projection.contractVersion).toBe(1); + expect(USAGE_ATTRIBUTION_VERSION).toBe(2); + expect(projection.contractVersion).toBe(2); for (const entry of projection.coverage) { expect(entry.liveQualified).toBe(false); } @@ -447,7 +768,7 @@ describe("projection contract", () => { it("renders a stable human-readable sample", () => { const projection = buildUsageAttribution( input({ - records: [record({ dedupeKey: "a", providerRequestId: "r1", providerMessageId: "m1" })], + records: [record({ dedupeKey: "m1:r1", providerRequestId: "r1", providerMessageId: "m1" })], bindings: [binding({ threadId: "thread-1" })], links: [link({ threadId: "thread-1", number: 12 })], }), @@ -457,13 +778,14 @@ describe("projection contract", () => { expect(text).toContain(`claude:${CLAUDE_SESSION}`); expect(text).toContain("requests=1"); expect(text).toContain("github.com/acme/repo#12 attributed="); + expect(text).toContain("Orphan:"); }); it("returns every level for a machine-readable fixture", () => { const projection = buildUsageAttribution( input({ records: [ - record({ dedupeKey: "a", providerRequestId: "r1", providerMessageId: "m1" }), + record({ dedupeKey: "m1:r1", providerRequestId: "r1", providerMessageId: "m1" }), record({ provider: "grok", sessionId: GROK_SESSION, @@ -480,7 +802,7 @@ describe("projection contract", () => { }), ); - expect(projection).toMatchObject({ contractVersion: 1, generatedAtMs: 1_786_100_000_000 }); + expect(projection).toMatchObject({ contractVersion: 2, generatedAtMs: 1_786_100_000_000 }); expect(projection.sessions).toHaveLength(2); expect(projection.prompts).toHaveLength(1); expect(projection.requests).toHaveLength(1); diff --git a/apps/server/src/usage/usageAttribution.ts b/apps/server/src/usage/usageAttribution.ts index c5f50b863774..c79628a2a8cc 100644 --- a/apps/server/src/usage/usageAttribution.ts +++ b/apps/server/src/usage/usageAttribution.ts @@ -7,7 +7,8 @@ * * - native session → T3 thread: the `resume_cursor_json` identity a provider * adapter wrote, or the `importedTranscripts` metadata an imported session - * recorded. See `ProviderSessionRuntimeRepository`. + * recorded. See `usageAttributionSources` for the read-only extraction from + * the persisted row shapes. * - T3 thread → pull request: `projection_thread_pull_requests`, canonicalized * with `@t3tools/shared/threadPullRequests`. * @@ -17,19 +18,24 @@ * proof of what the existing sources can establish, not a storage or transport * decision. * - * Two rules dominate the shape of the output: + * Three rules dominate the shape of the output: * * 1. Granularity is asserted per source, never inferred. A source that emits * one aggregate per turn cannot yield request or prompt counts, so those * levels report `unsupported` instead of a fabricated number. * 2. Association is not attribution. A session linked to several pull requests - * contributes to each of those PRs' `shared` pool — which is explicitly not - * additive — rather than its total being cloned onto every linked PR. + * contributes to one global `shared` pool — explicitly not additive — rather + * than its total being cloned onto every linked PR. + * 3. Nothing measured is dropped. Records without a session id land in an + * explicit `orphan` bucket, and the reconciliation + * `attributed + shared + unallocated + orphan === measured` holds against + * the deduplicated input, not against a pre-filtered session list. * * @module usageAttribution */ import type { ThreadPullRequestLinkSource, + UsageCostSource, UsageProviderKind, UsageTokenTotals, } from "@t3tools/contracts"; @@ -40,7 +46,7 @@ import { import { EMPTY_TOTALS, addTotals, totalTokens as countTokens } from "./usageTranscripts.ts"; -export const USAGE_ATTRIBUTION_VERSION = 1 as const; +export const USAGE_ATTRIBUTION_VERSION = 2 as const; /** The four reporting levels this projection can speak to. */ export type AttributionGranularity = "prompt" | "request" | "session" | "pullRequest"; @@ -48,18 +54,30 @@ export type AttributionGranularity = "prompt" | "request" | "session" | "pullReq /** * How much of a level's measurement is actually established. * - * - `measured` — every contributing record carried the identity this level needs. - * - `partial` — some records lacked it; totals are a lower bound, not a complete one. + * - `measured` — every contributing record carried a real measurement (an + * explicit zero counts; an empty container does not). + * - `partial` — some records lacked the level's identity, or the presence of a + * measurement was erased (a legacy cache row). Totals are a lower bound. * - `missing` — the source supports this level but no usable measurement exists. * This is the absence case, and it is never a zero. - * - `invalid` — an identity was present but malformed for its provider. + * - `invalid` — an identity or usage container was present but malformed. + * - `unavailable` — the source erased the information before we saw it. * - `unsupported` — the source cannot establish this level at all. */ -export type AttributionQuality = "measured" | "partial" | "missing" | "invalid" | "unsupported"; +export type AttributionQuality = + | "measured" + | "partial" + | "missing" + | "invalid" + | "unavailable" + | "unsupported"; /** Whether a source can establish a level from its native records. */ export type AttributionLevelSupport = "supported" | "unsupported"; +/** Whether a native session id is usable, independent of the measurement. */ +export type AttributionIdentityQuality = "valid" | "missing" | "invalid" | "unavailable"; + /** * What each source can and cannot establish, stated once so a caller cannot * accidentally treat an unsupported level as a measured zero. @@ -125,8 +143,12 @@ function capabilityOf(provider: UsageProviderKind): AttributionSourceCapability /** * One already-normalized usage record, tagged with the source that produced it. - * `costUsd` is the priced cost supplied by the existing pricing path; the - * projection never prices anything itself. + * + * `dedupeKey` is the scan/delivery identity, kept apart from the native + * observation ids below. For keyless sources (Codex `token_count`) the scan + * stamps it with the occurrence-aware identity from `usageTranscripts`; a + * record with no key at all is treated as an unkeyed observation and surfaced + * as uncertain rather than silently merged with a content-equal neighbour. */ export interface AttributionUsageRecord { readonly provider: UsageProviderKind; @@ -136,12 +158,23 @@ export interface AttributionUsageRecord { readonly timestampMs: number; readonly totals: UsageTokenTotals; readonly costUsd: number; + /** Scan/delivery identity, or `null` when the record is unkeyed. */ readonly dedupeKey: string | null; readonly providerRequestId?: string | null; readonly providerMessageId?: string | null; readonly promptId?: string | null; - /** Physical source identity, so a duplicate scan can be detected. */ + /** Physical source identity, for source coverage and duplicate detection. */ readonly sourceFingerprint: string; + /** + * Whether tokens were actually measured. Absent defaults to `observed` when + * any total is nonzero and `unavailable` when all are zero, so a legacy row + * whose presence was erased is never read as a measured zero. + */ + readonly measurement?: "observed" | "empty" | "unavailable"; + /** Additive increment or replaceable snapshot. Absent means `delta`. */ + readonly scope?: "delta" | "snapshot"; + /** Cost provenance, preserved per record so a view can carry it. */ + readonly costSource?: UsageCostSource; } /** @@ -170,6 +203,11 @@ export interface AttributionPullRequestLink { readonly number: number; readonly source: ThreadPullRequestLinkSource; readonly linkedAt: string; + /** + * The stored URL, when the caller has it. `normalizeThreadPullRequestKey` + * uses it to recover a Forgejo HTTP port that the bare host/repository loses. + */ + readonly url?: string; } /** A declared source, used for coverage and duplicate-scan reporting. */ @@ -181,6 +219,7 @@ export interface AttributionSource { } export interface UsageAttributionInput { + /** Read cutoff. Associations are as of this instant. */ readonly generatedAtMs: number; readonly records: readonly AttributionUsageRecord[]; readonly bindings: readonly AttributionThreadBinding[]; @@ -195,20 +234,36 @@ export interface AttributionTotals { readonly records: number; } +/** Per-model contribution, so a mixed-model session is never flattened away. */ +export interface AttributionModelContribution { + readonly model: string; + readonly totals: UsageTokenTotals; + readonly totalTokens: number; + readonly costUsd: number; + /** `unknown` when the caller supplied no provenance; `mixed` when they differ. */ + readonly costSource: UsageCostSource | "unknown" | "mixed"; + readonly records: number; +} + export type AttributionAllocation = | "attributed" | "shared" | "unallocated" | "ambiguous" - | "missing"; + | "missing" + | "orphan"; export interface AttributionSessionReport { readonly provider: UsageProviderKind; readonly sessionId: string; readonly models: readonly string[]; + readonly modelContributions: readonly AttributionModelContribution[]; /** `null` when the session is known but no usage was measured for it. */ readonly totals: AttributionTotals | null; - readonly quality: AttributionQuality; + /** Session-id validity, independent of measurement. */ + readonly identityQuality: AttributionIdentityQuality; + /** Numeric completeness, independent of identity and allocation. */ + readonly measurementQuality: AttributionQuality; readonly promptQuality: AttributionQuality; readonly requestQuality: AttributionQuality; /** `null` when the source cannot establish this level. */ @@ -222,6 +277,10 @@ export interface AttributionSessionReport { readonly pullRequestKeys: readonly string[]; /** PRs reached only through a stack-sibling link; association, not attribution. */ readonly stackOnlyPullRequestKeys: readonly string[]; + /** `uncertain` when any contributing record had no scan/delivery identity. */ + readonly recordIdentity: "exact" | "uncertain"; + /** `conflict` when two versions of one identity disagreed. */ + readonly conflict: boolean; } export interface AttributionPromptReport { @@ -230,6 +289,7 @@ export interface AttributionPromptReport { readonly promptId: string; readonly totals: AttributionTotals; readonly models: readonly string[]; + readonly modelContributions: readonly AttributionModelContribution[]; readonly boundThreadIds: readonly string[]; readonly allocation: AttributionAllocation; } @@ -256,11 +316,20 @@ export interface AttributionPullRequestReport { readonly attributed: AttributionTotals; /** Sessions also linked to another PR. Never add this into `attributed`. */ readonly shared: AttributionTotals; + /** Per-model contribution of `attributed`; never additive with `shared`. */ + readonly attributedModelContributions: readonly AttributionModelContribution[]; /** Sessions reaching this PR only through a stack-sibling link. */ readonly stackAssociationSessions: readonly string[]; readonly contributingSessions: readonly string[]; } +export interface AttributionSourceStatusCounts { + readonly ok: number; + readonly missing: number; + readonly partial: number; + readonly failed: number; +} + export interface AttributionCoverage { readonly provider: UsageProviderKind; readonly nativeSource: AttributionSourceCapability["nativeSource"]; @@ -268,17 +337,55 @@ export interface AttributionCoverage { readonly session: AttributionLevelSupport; readonly prompt: AttributionLevelSupport; readonly request: AttributionLevelSupport; + readonly declaredSources: number; + readonly distinctSourceFingerprints: number; + readonly sourceStatus: AttributionSourceStatusCounts; readonly measuredSessions: number; + readonly partialSessions: number; readonly missingSessions: number; readonly invalidSessions: number; + readonly unavailableSessions: number; readonly unboundSessions: number; readonly ambiguousSessions: number; + /** Records that carried no session id at all. */ readonly recordsWithoutSessionId: number; + /** Records whose scan/delivery identity was unknown. */ + readonly unkeyedRecords: number; + /** Conflicting versions of one observation that were kept-first. */ + readonly conflictingRecords: number; +} + +/** How the projection's associations are grounded, so the claim is bounded. */ +export interface AttributionAssociationBasis { + /** Associations are read from links that exist at `cutoffMs`. */ + readonly basis: "links-at-read-time"; + readonly cutoffMs: number; + /** + * Always `false`: allocation is recomputed from the links present at read + * time, so a changed link changes the recomputed view. `linkedAt` does not + * gate allocation, so pre-link implementation work is included. + */ + readonly linkedAtGovernsAllocation: false; +} + +export interface AttributionIdentityDiagnostics { + /** Repeated deliveries of one identity that were collapsed. */ + readonly duplicatesDropped: number; + /** Snapshot observations that replaced an earlier value for the same identity. */ + readonly snapshotsReplaced: number; + /** Same identity, different content, no snapshot semantics: kept-first. */ + readonly conflicts: number; + /** Records with no scan/delivery identity; counted, not merged. */ + readonly unkeyedRecords: number; + /** Records with no session id, preserved in `orphan`. */ + readonly orphanRecords: number; } export interface UsageAttribution { readonly contractVersion: typeof USAGE_ATTRIBUTION_VERSION; readonly generatedAtMs: number; + readonly association: AttributionAssociationBasis; + readonly identity: AttributionIdentityDiagnostics; readonly sessions: readonly AttributionSessionReport[]; readonly prompts: readonly AttributionPromptReport[]; readonly requests: readonly AttributionRequestReport[]; @@ -287,6 +394,10 @@ export interface UsageAttribution { readonly shared: AttributionTotals; /** Usage on sessions with no usable PR link, including missing identity. */ readonly unallocated: AttributionTotals; + /** Usage on records with no native session id. Additive with the above. */ + readonly orphan: AttributionTotals; + /** Deduplicated input total. `attributed + shared + unallocated + orphan`. */ + readonly measured: AttributionTotals; readonly coverage: readonly AttributionCoverage[]; readonly limitations: readonly string[]; } @@ -340,18 +451,25 @@ function totalsOfRecords(records: readonly AttributionUsageRecord[]): Attributio return { tokens, totalTokens: countTokens(tokens), costUsd, records: records.length }; } +function anyTotal(record: AttributionUsageRecord): number { + return countTokens(record.totals); +} + +/** Presence of a measurement, with the legacy-erased case made explicit. */ +function effectiveMeasurement( + record: AttributionUsageRecord, +): "observed" | "empty" | "unavailable" { + if (record.measurement !== undefined) return record.measurement; + return anyTotal(record) > 0 ? "observed" : "unavailable"; +} + /** - * Identity used for de-duplication when a record carries no `dedupeKey`. - * - * Two environments scanning the same directory produce byte-identical codex - * records (which have no parser dedupe key), so a content signature stops that - * shared source from being counted twice. It is intentionally not exposed as a - * provider request id. + * Content of a measured observation, used to tell a repeated delivery from a + * conflicting version of the same identity. Deliberately excludes + * `sourceFingerprint`: a copy at another path is the same observation. */ -function recordContentSignature(record: AttributionUsageRecord): string { +function observationContent(record: AttributionUsageRecord): string { return [ - record.provider, - record.sessionId, record.model, record.timestampMs, record.totals.uncachedInputTokens, @@ -362,9 +480,47 @@ function recordContentSignature(record: AttributionUsageRecord): string { record.providerRequestId ?? "", record.providerMessageId ?? "", record.promptId ?? "", + effectiveMeasurement(record), ].join("\u0000"); } +function modelContributions( + records: readonly AttributionUsageRecord[], +): readonly AttributionModelContribution[] { + const byModel = new Map< + string, + { totals: UsageTokenTotals; costUsd: number; records: number; sources: Set } + >(); + for (const record of records) { + const entry = byModel.get(record.model) ?? { + totals: EMPTY_TOTALS, + costUsd: 0, + records: 0, + sources: new Set(), + }; + entry.totals = addTotals(entry.totals, record.totals); + entry.costUsd += record.costUsd; + entry.records += 1; + entry.sources.add(record.costSource ?? "unknown"); + byModel.set(record.model, entry); + } + return [...byModel.entries()] + .map(([model, entry]): AttributionModelContribution => { + const sources = [...entry.sources].toSorted(); + const costSource = + sources.length === 1 ? (sources[0] as UsageCostSource | "unknown") : "mixed"; + return { + model, + totals: entry.totals, + totalTokens: countTokens(entry.totals), + costUsd: entry.costUsd, + costSource, + records: entry.records, + }; + }) + .toSorted((left, right) => left.model.localeCompare(right.model)); +} + interface StrongLink { readonly key: string; readonly host: string; @@ -381,30 +537,70 @@ interface MutableCoverage { session: AttributionLevelSupport; prompt: AttributionLevelSupport; request: AttributionLevelSupport; + declaredSources: number; + fingerprints: Set; + sourceStatus: { ok: number; missing: number; partial: number; failed: number }; measuredSessions: number; + partialSessions: number; missingSessions: number; invalidSessions: number; + unavailableSessions: number; unboundSessions: number; ambiguousSessions: number; recordsWithoutSessionId: number; + unkeyedRecords: number; + conflictingRecords: number; } /** * Builds the four-level projection from already-measured usage. * * `generatedAtMs` is supplied rather than read so the projection stays pure and - * fixtures stay deterministic. + * fixtures stay deterministic. It is also the association cutoff. */ export function buildUsageAttribution(input: UsageAttributionInput): UsageAttribution { - // 1. De-duplicate records: by declared key when present (fork copies and - // resumed history share it), otherwise by content signature. - const seen = new Set(); - const records: AttributionUsageRecord[] = []; + // 1. Identity and de-duplication. + // + // A declared key is the scan/delivery identity and is namespaced by + // provider so equal local ids from two providers cannot collide. A + // repeated delivery (same key, same content) is dropped; a snapshot + // replaces the earlier value; a differing delta for the same key is a + // conflict and is exposed rather than silently discarded. A record with no + // key at all is unkeyed: it is kept and counted, never merged by content + // equality, and the session is marked uncertain. + const kept: AttributionUsageRecord[] = []; + const keptIndexByIdentity = new Map(); + const conflictSessions = new Set(); + let duplicatesDropped = 0; + let snapshotsReplaced = 0; + let conflicts = 0; + let unkeyedRecords = 0; + for (const record of input.records) { - const identity = record.dedupeKey ?? recordContentSignature(record); - if (seen.has(identity)) continue; - seen.add(identity); - records.push(record); + if (record.dedupeKey === null || record.dedupeKey.length === 0) { + unkeyedRecords += 1; + kept.push(record); + continue; + } + const identity = `${record.provider}\u0000${record.dedupeKey}`; + const existingIndex = keptIndexByIdentity.get(identity); + if (existingIndex === undefined) { + keptIndexByIdentity.set(identity, kept.length); + kept.push(record); + continue; + } + const existing = kept[existingIndex]!; + if (observationContent(existing) === observationContent(record)) { + duplicatesDropped += 1; + continue; + } + if ((record.scope ?? "delta") === "snapshot") { + kept[existingIndex] = record; + snapshotsReplaced += 1; + continue; + } + conflicts += 1; + conflictSessions.add(sessionKey(record.provider, record.sessionId)); } // 2. Bind native sessions to threads, keeping every origin that points at @@ -457,16 +653,17 @@ export function buildUsageAttribution(input: UsageAttributionInput): UsageAttrib target.set(link.threadId, set); } - // 4. Accumulate per native session from measured records, then include - // bindings that have no records so "known but unmeasured" is not zero. + // 4. Accumulate per native session from measured records. A record with no + // session id is preserved in the orphan bucket instead of being dropped. const sessionsByKey = new Map(); - const recordsWithoutSessionIdByProvider = new Map(); - for (const record of records) { + const orphanRecords: AttributionUsageRecord[] = []; + const orphanByProvider = new Map(); + for (const record of kept) { if (record.sessionId.length === 0) { - recordsWithoutSessionIdByProvider.set( - record.provider, - (recordsWithoutSessionIdByProvider.get(record.provider) ?? 0) + 1, - ); + orphanRecords.push(record); + const list = orphanByProvider.get(record.provider) ?? []; + list.push(record); + orphanByProvider.set(record.provider, list); continue; } const key = sessionKey(record.provider, record.sessionId); @@ -496,12 +693,65 @@ export function buildUsageAttribution(input: UsageAttributionInput): UsageAttrib const requestReports: AttributionRequestReport[] = []; const prAttributed = new Map(); const prShared = new Map(); + const prAttributedRecords = new Map(); const prStackAssociations = new Map>(); const prContributing = new Map>(); let shared = ZERO_TOTALS; let unallocated = ZERO_TOTALS; const coverageByProvider = new Map(); + const ensureCoverage = (provider: UsageProviderKind): MutableCoverage => { + const capability = capabilityOf(provider); + const existing = coverageByProvider.get(provider); + if (existing !== undefined) return existing; + const created: MutableCoverage = { + provider, + nativeSource: capability.nativeSource, + liveQualified: capability.liveQualified, + session: capability.session, + prompt: capability.prompt, + request: capability.request, + declaredSources: 0, + fingerprints: new Set(), + sourceStatus: { ok: 0, missing: 0, partial: 0, failed: 0 }, + measuredSessions: 0, + partialSessions: 0, + missingSessions: 0, + invalidSessions: 0, + unavailableSessions: 0, + unboundSessions: 0, + ambiguousSessions: 0, + recordsWithoutSessionId: 0, + unkeyedRecords: 0, + conflictingRecords: 0, + }; + coverageByProvider.set(provider, created); + return created; + }; + + // Coverage is seeded from declared sources first, so a missing or failed + // source still produces a row and is never read as "nothing to measure". + for (const source of input.sources) { + const coverage = ensureCoverage(source.provider); + coverage.declaredSources += 1; + coverage.sourceStatus[source.status] += 1; + if (source.fingerprint.length > 0) coverage.fingerprints.add(source.fingerprint); + } + // Fingerprints come from every input record, including a dropped duplicate: + // a copied history still proves a second physical source exists. + for (const record of input.records) { + const coverage = ensureCoverage(record.provider); + if (record.sourceFingerprint.length > 0) coverage.fingerprints.add(record.sourceFingerprint); + } + for (const record of kept) { + const coverage = ensureCoverage(record.provider); + if (record.dedupeKey === null || record.dedupeKey.length === 0) coverage.unkeyedRecords += 1; + if (conflictSessions.has(sessionKey(record.provider, record.sessionId))) { + coverage.conflictingRecords += 1; + } + } + for (const binding of input.bindings) ensureCoverage(binding.provider); + const sessionUniverse = new Map(sessionsByKey); for (const key of sessionThreads.keys()) { if (sessionUniverse.has(key)) continue; @@ -527,37 +777,44 @@ export function buildUsageAttribution(input: UsageAttributionInput): UsageAttrib const totals = accumulator.records.length === 0 ? null : totalsOfRecords(accumulator.records); const models = [...new Set(accumulator.records.map((record) => record.model))].toSorted(); - const sessionQuality: AttributionQuality = + const identityQuality: AttributionIdentityQuality = sessionId.length === 0 ? "missing" : provider === "claude" && !CLAUDE_SESSION_ID_PATTERN.test(sessionId) ? "invalid" - : totals === null - ? "missing" - : "measured"; - - const promptQuality: AttributionQuality = - capability.prompt === "unsupported" - ? "unsupported" - : accumulator.records.length === 0 - ? "missing" - : accumulator.recordsWithPromptId === accumulator.records.length - ? "measured" - : accumulator.recordsWithPromptId === 0 - ? "missing" - : "partial"; + : "valid"; - const requestQuality: AttributionQuality = - capability.request === "unsupported" - ? "unsupported" - : accumulator.records.length === 0 - ? "missing" - : accumulator.recordsWithRequestId === accumulator.records.length - ? "measured" - : accumulator.recordsWithRequestId === 0 - ? "missing" + const measurements = accumulator.records.map(effectiveMeasurement); + const observedCount = measurements.filter((value) => value === "observed").length; + const emptyCount = measurements.filter((value) => value === "empty").length; + const unavailableCount = measurements.filter((value) => value === "unavailable").length; + const measurementQuality: AttributionQuality = + accumulator.records.length === 0 + ? "missing" + : observedCount === accumulator.records.length + ? "measured" + : observedCount === 0 && emptyCount > 0 && unavailableCount === 0 + ? "invalid" + : observedCount === 0 && unavailableCount > 0 + ? "unavailable" : "partial"; + const identityLevelQuality = ( + level: "prompt" | "request", + recordsWithId: number, + ): AttributionQuality => { + if (capability[level] === "unsupported") return "unsupported"; + if (accumulator.records.length === 0) return "missing"; + if (unavailableCount > 0 && recordsWithId === 0) return "unavailable"; + if (recordsWithId === 0) return "missing"; + if (recordsWithId < accumulator.records.length) return "partial"; + if (observedCount === accumulator.records.length) return "measured"; + return "partial"; + }; + + const promptQuality = identityLevelQuality("prompt", accumulator.recordsWithPromptId); + const requestQuality = identityLevelQuality("request", accumulator.recordsWithRequestId); + const strongPrs = new Set(); for (const threadId of boundThreadIds) { for (const prKey of threadStrongPrs.get(threadId) ?? []) strongPrs.add(prKey); @@ -570,7 +827,7 @@ export function buildUsageAttribution(input: UsageAttributionInput): UsageAttrib } let allocation: AttributionAllocation; - if (sessionQuality === "missing" && totals === null) allocation = "missing"; + if (totals === null && measurementQuality === "missing") allocation = "missing"; else if (boundThreadIds.length === 0) allocation = "unallocated"; else if (boundThreadIds.length > 1) allocation = "ambiguous"; else if (strongPrs.size === 1) allocation = "attributed"; @@ -587,6 +844,9 @@ export function buildUsageAttribution(input: UsageAttributionInput): UsageAttrib if (allocation === "attributed") { for (const prKey of strongPrs) { prAttributed.set(prKey, addTotalsOf(prAttributed.get(prKey) ?? ZERO_TOTALS, totals)); + const records = prAttributedRecords.get(prKey) ?? []; + records.push(...accumulator.records); + prAttributedRecords.set(prKey, records); } } if (allocation === "shared") { @@ -610,8 +870,10 @@ export function buildUsageAttribution(input: UsageAttributionInput): UsageAttrib provider, sessionId, models, + modelContributions: modelContributions(accumulator.records), totals, - quality: sessionQuality, + identityQuality, + measurementQuality, promptQuality, requestQuality, promptCount: @@ -630,6 +892,12 @@ export function buildUsageAttribution(input: UsageAttributionInput): UsageAttrib allocation, pullRequestKeys: [...strongPrs].toSorted(), stackOnlyPullRequestKeys: [...stackPrs].toSorted(), + recordIdentity: accumulator.records.some( + (record) => record.dedupeKey === null || record.dedupeKey.length === 0, + ) + ? "uncertain" + : "exact", + conflict: conflictSessions.has(key), }); // 5. Level reports. Only providers whose capability supports the level @@ -641,34 +909,26 @@ export function buildUsageAttribution(input: UsageAttributionInput): UsageAttrib promptReports.push(...promptRows(accumulator, boundThreadIds, allocation)); } - // 6. Coverage. - const coverage = coverageByProvider.get(provider) ?? { - provider, - nativeSource: capability.nativeSource, - liveQualified: capability.liveQualified, - session: capability.session, - prompt: capability.prompt, - request: capability.request, - measuredSessions: 0, - missingSessions: 0, - invalidSessions: 0, - unboundSessions: 0, - ambiguousSessions: 0, - recordsWithoutSessionId: recordsWithoutSessionIdByProvider.get(provider) ?? 0, - }; - if (sessionQuality === "measured") coverage.measuredSessions += 1; - if (sessionQuality === "missing") coverage.missingSessions += 1; - if (sessionQuality === "invalid") coverage.invalidSessions += 1; + // 6. Coverage counts, per axis. + const coverage = ensureCoverage(provider); + if (identityQuality === "valid") { + if (measurementQuality === "measured") coverage.measuredSessions += 1; + if (measurementQuality === "partial") coverage.partialSessions += 1; + if (measurementQuality === "missing") coverage.missingSessions += 1; + if (measurementQuality === "invalid") coverage.invalidSessions += 1; + if (measurementQuality === "unavailable") coverage.unavailableSessions += 1; + } else if (identityQuality === "invalid") { + coverage.invalidSessions += 1; + } else { + coverage.missingSessions += 1; + } if (boundThreadIds.length === 0) coverage.unboundSessions += 1; if (allocation === "ambiguous") coverage.ambiguousSessions += 1; - coverageByProvider.set(provider, coverage); } - for (const [provider, coverage] of coverageByProvider) { - coverageByProvider.set(provider, { - ...coverage, - recordsWithoutSessionId: recordsWithoutSessionIdByProvider.get(provider) ?? 0, - }); + const orphan = totalsOfRecords(orphanRecords); + for (const [provider, records] of orphanByProvider) { + ensureCoverage(provider).recordsWithoutSessionId += records.length; } sessionReports.sort((left, right) => @@ -697,26 +957,67 @@ export function buildUsageAttribution(input: UsageAttributionInput): UsageAttrib linkSources: [...meta.sources].toSorted(), attributed: prAttributed.get(meta.key) ?? ZERO_TOTALS, shared: prShared.get(meta.key) ?? ZERO_TOTALS, + attributedModelContributions: modelContributions(prAttributedRecords.get(meta.key) ?? []), stackAssociationSessions: [...(prStackAssociations.get(meta.key) ?? [])].toSorted(), contributingSessions: [...(prContributing.get(meta.key) ?? [])].toSorted(), })) .toSorted((left, right) => left.key.localeCompare(right.key)); - const coverage: AttributionCoverage[] = [...coverageByProvider.values()].toSorted((left, right) => - left.provider.localeCompare(right.provider), - ); + const coverage: AttributionCoverage[] = [...coverageByProvider.values()] + .map((entry): AttributionCoverage => ({ + provider: entry.provider, + nativeSource: entry.nativeSource, + liveQualified: entry.liveQualified, + session: entry.session, + prompt: entry.prompt, + request: entry.request, + declaredSources: entry.declaredSources, + distinctSourceFingerprints: entry.fingerprints.size, + sourceStatus: { ...entry.sourceStatus }, + measuredSessions: entry.measuredSessions, + partialSessions: entry.partialSessions, + missingSessions: entry.missingSessions, + invalidSessions: entry.invalidSessions, + unavailableSessions: entry.unavailableSessions, + unboundSessions: entry.unboundSessions, + ambiguousSessions: entry.ambiguousSessions, + recordsWithoutSessionId: entry.recordsWithoutSessionId, + unkeyedRecords: entry.unkeyedRecords, + conflictingRecords: entry.conflictingRecords, + })) + .toSorted((left, right) => left.provider.localeCompare(right.provider)); + + const measured = totalsOfRecords(kept); return { contractVersion: USAGE_ATTRIBUTION_VERSION, generatedAtMs: input.generatedAtMs, + association: { + basis: "links-at-read-time", + cutoffMs: input.generatedAtMs, + linkedAtGovernsAllocation: false, + }, + identity: { + duplicatesDropped, + snapshotsReplaced, + conflicts, + unkeyedRecords, + orphanRecords: orphanRecords.length, + }, sessions: sessionReports, prompts: promptReports, requests: requestReports, pullRequests, shared, unallocated, + orphan, + measured, coverage, - limitations: limitationsFor(input, coverage), + limitations: limitationsFor(input, coverage, { + unkeyedRecords, + conflicts, + orphanRecords: orphanRecords.length, + }), }; } @@ -764,6 +1065,7 @@ function promptRows( promptId, totals: totalsOfRecords(rows), models: [...new Set(rows.map((row) => row.model))].toSorted(), + modelContributions: modelContributions(rows), boundThreadIds, allocation, })); @@ -772,12 +1074,20 @@ function promptRows( function limitationsFor( input: UsageAttributionInput, coverage: readonly AttributionCoverage[], + identity: { unkeyedRecords: number; conflicts: number; orphanRecords: number }, ): readonly string[] { const limitations: string[] = [ "A native session maps to a T3 thread only through the current resume cursor or imported-transcript metadata; a session switch, fork, or restart that overwrote the cursor leaves earlier usage unbound.", "Provider-instance identity is not recoverable from a transcript scan, so two instances of one provider cannot be told apart at the record level.", "Request and prompt counts are reported only where the native source writes those ids; a turn-level source reports `unsupported`, never an inferred count.", + "Associations are read from the links that exist at `generatedAtMs`. `linkedAt` does not gate allocation, so a link added after a session ran still associates that session's usage, and a link removed or changed rewrites the recomputed view. Pre-link implementation work is included; historical allocation as of a past instant is unavailable without temporal evidence.", + "Cost is API-equivalent list value, not subscription spend; subscription coverage is out of scope here.", ]; + if (input.records.some((record) => record.costSource === undefined)) { + limitations.push( + 'Some records carried no cost provenance. Their contribution reports `costSource: "unknown"` rather than being assumed priced or unpriced.', + ); + } const duplicateFingerprints = input.sources.length - new Set(input.sources.map((source) => source.fingerprint)).size; if (duplicateFingerprints > 0) { @@ -790,6 +1100,26 @@ function limitationsFor( "Some known sessions have no measured usage. They are reported as `missing` with a null total; this is not a zero-cost success.", ); } + if (coverage.some((entry) => entry.sourceStatus.missing + entry.sourceStatus.failed > 0)) { + limitations.push( + "At least one declared source is missing or failed; its absence is coverage, not a measured zero.", + ); + } + if (identity.unkeyedRecords > 0) { + limitations.push( + `${identity.unkeyedRecords} record(s) carried no scan/delivery identity. They are counted, not merged by content equality, and the owning session is marked "uncertain"; a repeated delivery of an unkeyed record cannot be told from a second equal occurrence.`, + ); + } + if (identity.conflicts > 0) { + limitations.push( + `${identity.conflicts} identity conflict(s) were found: the same identity appeared with different content and no snapshot semantics. The first version was kept and the conflict surfaced rather than silently resolved.`, + ); + } + if (identity.orphanRecords > 0) { + limitations.push( + `${identity.orphanRecords} record(s) carried no native session id. Their usage is preserved in \`orphan\` rather than dropped.`, + ); + } if (input.bindings.some((binding) => binding.origin === "runtimeCursor")) { limitations.push( "Only the newest native session id per thread is durable. Additive retention must land before historical re-attribution is possible.", @@ -829,7 +1159,7 @@ export function renderUsageAttributionText(projection: UsageAttribution): string const requests = session.requestCount === null ? "requests=unsupported" : `requests=${session.requestCount}`; lines.push( - ` ${sessionLabel(session.provider, session.sessionId)} [${session.quality}/${session.allocation}] ${tokens}${cost} ${prompts} ${requests} threads=${ + ` ${sessionLabel(session.provider, session.sessionId)} [${session.identityQuality}/${session.measurementQuality}/${session.allocation}] ${tokens}${cost} ${prompts} ${requests} threads=${ session.boundThreadIds.length === 0 ? "" : session.boundThreadIds.join(",") }`, ); @@ -847,12 +1177,14 @@ export function renderUsageAttributionText(projection: UsageAttribution): string "", `Shared (not additive): ${projection.shared.totalTokens} tokens`, `Unallocated: ${projection.unallocated.totalTokens} tokens`, + `Orphan: ${projection.orphan.totalTokens} tokens`, + `Measured: ${projection.measured.totalTokens} tokens`, "", "Coverage:", ); for (const entry of projection.coverage) { lines.push( - ` ${entry.provider} session=${entry.session} prompt=${entry.prompt} request=${entry.request} measured=${entry.measuredSessions} missing=${entry.missingSessions} unbound=${entry.unboundSessions} ambiguous=${entry.ambiguousSessions}`, + ` ${entry.provider} session=${entry.session} prompt=${entry.prompt} request=${entry.request} measured=${entry.measuredSessions} partial=${entry.partialSessions} missing=${entry.missingSessions} unbound=${entry.unboundSessions} ambiguous=${entry.ambiguousSessions} orphanRecords=${entry.recordsWithoutSessionId}`, ); } return lines.join("\n"); diff --git a/apps/server/src/usage/usageAttributionSources.test.ts b/apps/server/src/usage/usageAttributionSources.test.ts new file mode 100644 index 000000000000..3c839e46c85a --- /dev/null +++ b/apps/server/src/usage/usageAttributionSources.test.ts @@ -0,0 +1,270 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { + extractAttributionBindings, + extractAttributionLinks, + extractAttributionSnapshot, + type PersistedProviderSessionRuntimeRow, + type PersistedThreadPullRequestRow, +} from "./usageAttributionSources.ts"; + +function runtimeRow( + overrides: Partial = {}, +): PersistedProviderSessionRuntimeRow { + return { + threadId: "thread-1", + providerName: "claudeAgent", + providerInstanceId: "claude-default", + adapterKey: "claudeAgent", + resumeCursor: { resume: "5a128faa-8253-489e-b935-6c08e8e670c0" }, + runtimePayload: null, + ...overrides, + }; +} + +function importedSource(overrides: Record = {}): Record { + return { + provider: "claudeAgent", + providerInstanceId: "claude-original", + providerSessionId: "original-session", + filePath: "/home/u/.claude/projects/-home-u-project/original.jsonl", + size: 10, + mtimeMs: 1, + device: 1, + inode: 2, + birthtimeMs: 3, + ...overrides, + }; +} + +function linkRow( + overrides: Partial = {}, +): PersistedThreadPullRequestRow { + return { + threadId: "thread-1", + host: "github.com", + repository: "acme/repo", + number: 12, + url: "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/acme/repo/pull/12", + source: "manual", + linkedAt: "2026-09-01T00:00:00.000Z", + ...overrides, + }; +} + +describe("extractAttributionBindings", () => { + it("reads the Claude {resume} cursor shape into a usage binding", () => { + const { bindings, nativeSessions, diagnostics } = extractAttributionBindings([runtimeRow()]); + + expect(bindings).toEqual([ + { + threadId: "thread-1", + provider: "claude", + providerInstanceId: "claude-default", + nativeSessionId: "5a128faa-8253-489e-b935-6c08e8e670c0", + origin: "runtimeCursor", + }, + ]); + expect(nativeSessions[0]).toMatchObject({ + providerName: "claudeAgent", + adapterKey: "claudeAgent", + usageProvider: "claude", + }); + expect(diagnostics.runtimeCursorBindings).toBe(1); + expect(diagnostics.absentIdentityRows).toBe(0); + }); + + it("reads the Codex {threadId} cursor shape", () => { + const { bindings } = extractAttributionBindings([ + runtimeRow({ + providerName: "codex", + adapterKey: "codex", + providerInstanceId: "codex-default", + resumeCursor: { threadId: "019fbbc1-b12c-7360-a685-28c181f0025f" }, + }), + ]); + + expect(bindings[0]).toMatchObject({ + provider: "codex", + nativeSessionId: "019fbbc1-b12c-7360-a685-28c181f0025f", + }); + }); + + it("binds an imported transcript only for the thread it names", () => { + const { bindings, diagnostics } = extractAttributionBindings([ + runtimeRow({ + threadId: "import:claude-original:original-session", + resumeCursor: null, + runtimePayload: { importedTranscripts: [importedSource()] }, + }), + ]); + + expect(bindings).toEqual([ + { + threadId: "import:claude-original:original-session", + provider: "claude", + providerInstanceId: "claude-original", + nativeSessionId: "original-session", + origin: "importedTranscript", + }, + ]); + expect(diagnostics.importedTranscriptBindings).toBe(1); + }); + + it("exposes an OpenCode session without inventing a usage provider", () => { + const { bindings, nativeSessions, diagnostics } = extractAttributionBindings([ + runtimeRow({ + providerName: "opencode", + adapterKey: "opencode", + providerInstanceId: "opencode-default", + resumeCursor: { sessionId: "ses_opencode_1" }, + }), + ]); + + expect(bindings).toEqual([]); + expect(nativeSessions).toEqual([ + { + threadId: "thread-1", + providerName: "opencode", + adapterKey: "opencode", + providerInstanceId: "opencode-default", + nativeSessionId: "ses_opencode_1", + origin: "runtimeCursor", + usageProvider: null, + }, + ]); + expect(diagnostics.unsupportedProviderBindings).toBe(1); + }); + + it("leaves absent, malformed, and overwritten history visible", () => { + const { bindings, diagnostics } = extractAttributionBindings([ + // Absent: no cursor, no imported transcripts. + runtimeRow({ threadId: "thread-absent", resumeCursor: null }), + // Malformed cursor: an object with no recognised id field. + runtimeRow({ threadId: "thread-malformed", resumeCursor: { nope: true } }), + // Malformed payload: importedTranscripts is not an array. + runtimeRow({ + threadId: "thread-payload", + resumeCursor: null, + runtimePayload: { importedTranscripts: {} }, + }), + // Overwritten: the current cursor is a later session than the retained import. + runtimeRow({ + threadId: "import:claude-original:original-session", + providerName: "codex", + adapterKey: "codex", + providerInstanceId: "codex-new", + resumeCursor: { threadId: "new-session" }, + runtimePayload: { importedTranscripts: [importedSource()] }, + }), + ]); + + expect(diagnostics.absentIdentityRows).toBe(1); + expect(diagnostics.malformedResumeCursors).toBe(1); + expect(diagnostics.malformedRuntimePayloads).toBe(1); + expect(diagnostics.overwrittenThreads).toBe(1); + // The overwritten row still yields both identities. + expect(bindings.map((entry) => entry.nativeSessionId).toSorted()).toEqual([ + "new-session", + "original-session", + ]); + }); + + it("counts one native session bound to two threads as ambiguous", () => { + const { diagnostics } = extractAttributionBindings([ + runtimeRow({ threadId: "thread-1" }), + runtimeRow({ threadId: "thread-2" }), + ]); + + expect(diagnostics.ambiguousSessionIds).toBe(1); + }); + + it("skips imported entries that name a different thread or provider", () => { + const { diagnostics } = extractAttributionBindings([ + runtimeRow({ + threadId: "import:claude-original:original-session", + resumeCursor: null, + runtimePayload: { + importedTranscripts: [ + null, + {}, + importedSource({ provider: "cursor" }), + importedSource({ providerSessionId: "wrong-session" }), + importedSource(), + ], + }, + }), + ]); + + expect(diagnostics.skippedImportedTranscripts).toBe(4); + expect(diagnostics.importedTranscriptBindings).toBe(1); + }); +}); + +describe("extractAttributionLinks", () => { + it("reads allowlisted fields from a persisted projection row", () => { + const { links, diagnostics } = extractAttributionLinks([linkRow()]); + + expect(links).toEqual([ + { + threadId: "thread-1", + host: "github.com", + repository: "acme/repo", + number: 12, + source: "manual", + linkedAt: "2026-09-01T00:00:00.000Z", + url: "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/acme/repo/pull/12", + }, + ]); + expect(diagnostics).toMatchObject({ rows: 1, links: 1, dismissed: 0, malformed: 0 }); + }); + + it("keeps stack-dismissed tombstones for the projection to filter", () => { + const { links, diagnostics } = extractAttributionLinks([ + linkRow({ number: 12 }), + linkRow({ number: 13, source: "stack" }), + linkRow({ number: 14, source: "stack-dismissed" }), + ]); + + expect(links.map((entry) => entry.number)).toEqual([12, 13, 14]); + expect(diagnostics.dismissed).toBe(1); + }); + + it("drops malformed rows and counts them", () => { + const { links, diagnostics } = extractAttributionLinks([ + linkRow(), + linkRow({ number: 0 }), + linkRow({ host: "" }), + linkRow({ source: "not-a-source" as PersistedThreadPullRequestRow["source"] }), + ]); + + expect(links).toHaveLength(1); + expect(diagnostics.malformed).toBe(3); + }); +}); + +describe("extractAttributionSnapshot", () => { + it("labels the cutoff and never leaks the runtime payload", () => { + const snapshot = extractAttributionSnapshot({ + cutoffMs: 1_786_100_000_000, + runtimeRows: [ + runtimeRow({ + threadId: "import:claude-original:original-session", + resumeCursor: null, + runtimePayload: { + cwd: "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/secret/path", + importedTranscripts: [importedSource()], + marker: "do-not-leak", + }, + }), + ], + linkRows: [linkRow({ threadId: "import:claude-original:original-session" })], + }); + + expect(snapshot.cutoffMs).toBe(1_786_100_000_000); + expect(snapshot.bindings).toHaveLength(1); + expect(snapshot.links).toHaveLength(1); + expect(JSON.stringify(snapshot)).not.toContain("do-not-leak"); + expect(JSON.stringify(snapshot)).not.toContain("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/secret/path"); + }); +}); diff --git a/apps/server/src/usage/usageAttributionSources.ts b/apps/server/src/usage/usageAttributionSources.ts new file mode 100644 index 000000000000..fa97b8876882 --- /dev/null +++ b/apps/server/src/usage/usageAttributionSources.ts @@ -0,0 +1,390 @@ +/** + * Read-only extraction of the persisted identities the attribution projection + * consumes. + * + * `buildUsageAttribution` is pure and takes already-normalized bindings and + * links; this module is the smallest seam that proves those can be read from + * what the server actually writes: + * + * - native session → thread from `provider_session_runtime.resume_cursor_json` + * (the single current cursor) and `runtime_payload_json.importedTranscripts` + * (the accumulated imported-file history); + * - thread → pull request from `projection_thread_pull_requests`. + * + * It reads only allowlisted fields and never returns a runtime payload. What it + * cannot read — an absent cursor, a malformed payload, a cursor overwritten by + * a later session, or one native session bound to two threads — is reported in + * `diagnostics` rather than silently dropped. + * + * @module usageAttributionSources + */ +import type { ThreadPullRequestLinkSource, UsageProviderKind } from "@t3tools/contracts"; + +import type { AttributionPullRequestLink, AttributionThreadBinding } from "./usageAttribution.ts"; + +/** + * Allowlisted `provider_session_runtime` row. This is the shape the repository + * returns (`resumeCursor` and `runtimePayload` already JSON-decoded). + */ +export interface PersistedProviderSessionRuntimeRow { + readonly threadId: string; + readonly providerName: string; + readonly providerInstanceId: string | null; + readonly adapterKey: string; + readonly resumeCursor: unknown; + readonly runtimePayload: unknown; +} + +/** + * Allowlisted `projection_thread_pull_requests` row. `snapshot_json` and + * `stack_json` are intentionally not part of the input: the projection needs + * only the canonical key, the link source, and the link instant. + */ +export interface PersistedThreadPullRequestRow { + readonly threadId: string; + readonly host: string; + readonly repository: string; + readonly number: number; + readonly url?: string; + readonly source: ThreadPullRequestLinkSource; + readonly linkedAt: string; +} + +/** + * One native session identity that exists in persistence, including providers + * T3 does not scan for usage. M1R can consume this narrow shape for OpenCode + * without widening `UsageProviderKind` or adding a parser here. + */ +export interface ExtractedNativeSession { + readonly threadId: string; + readonly providerName: string; + readonly adapterKey: string; + readonly providerInstanceId: string | null; + readonly nativeSessionId: string; + readonly origin: "runtimeCursor" | "importedTranscript"; + /** + * `null` when the provider has no scanned usage source (OpenCode, Antigravity, + * Cursor). The identity is still exposed; it simply cannot feed token usage + * in this proof. + */ + readonly usageProvider: UsageProviderKind | null; +} + +export interface AttributionBindingDiagnostics { + readonly runtimeRows: number; + readonly runtimeCursorBindings: number; + readonly importedTranscriptBindings: number; + /** Rows with neither a cursor nor imported transcripts. */ + readonly absentIdentityRows: number; + readonly malformedResumeCursors: number; + readonly malformedRuntimePayloads: number; + readonly skippedImportedTranscripts: number; + /** Bindings for a provider T3 does not scan for usage. */ + readonly unsupportedProviderBindings: number; + /** Threads whose cursor session differs from a retained imported session. */ + readonly overwrittenThreads: number; + /** Native sessions bound to more than one thread. */ + readonly ambiguousSessionIds: number; +} + +export interface AttributionBindingExtraction { + /** Only bindings for providers with a scanned usage source. */ + readonly bindings: readonly AttributionThreadBinding[]; + /** Every extracted native identity, including unsupported providers. */ + readonly nativeSessions: readonly ExtractedNativeSession[]; + readonly diagnostics: AttributionBindingDiagnostics; +} + +export interface AttributionLinkDiagnostics { + readonly rows: number; + readonly links: number; + /** `stack-dismissed` tombstones, preserved for the projection to filter. */ + readonly dismissed: number; + readonly malformed: number; +} + +export interface AttributionLinkExtraction { + readonly links: readonly AttributionPullRequestLink[]; + readonly diagnostics: AttributionLinkDiagnostics; +} + +export interface AttributionSnapshotExtraction { + /** Read cutoff; associations are as of this instant. */ + readonly cutoffMs: number; + readonly bindings: readonly AttributionThreadBinding[]; + readonly nativeSessions: readonly ExtractedNativeSession[]; + readonly links: readonly AttributionPullRequestLink[]; + readonly diagnostics: { + readonly bindings: AttributionBindingDiagnostics; + readonly links: AttributionLinkDiagnostics; + }; +} + +const USAGE_PROVIDER_BY_DRIVER: Readonly> = { + claude: "claude", + claudeagent: "claude", + codex: "codex", + grok: "grok", +}; + +const LINK_SOURCES: ReadonlySet = new Set([ + "manual", + "created", + "agent", + "stack", + "stack-dismissed", +]); + +function usageProviderOf(...names: readonly string[]): UsageProviderKind | null { + for (const name of names) { + const mapped = USAGE_PROVIDER_BY_DRIVER[name.trim().toLowerCase()]; + if (mapped !== undefined) return mapped; + } + return null; +} + +type CursorRead = + | { readonly kind: "absent" } + | { readonly kind: "malformed" } + | { readonly kind: "id"; readonly id: string }; + +/** + * The three cursor shapes adapters actually write: `{ resume }` (Claude), + * `{ threadId }` (Codex), `{ sessionId }` (Grok, OpenCode, Antigravity). Order + * does not matter because a cursor carries exactly one of them. + */ +function readResumeCursor(cursor: unknown): CursorRead { + if (cursor === null || cursor === undefined) return { kind: "absent" }; + if (typeof cursor !== "object" || Array.isArray(cursor)) return { kind: "malformed" }; + const record = cursor as Record; + for (const field of ["resume", "threadId", "sessionId"] as const) { + const value = record[field]; + if (typeof value === "string" && value.trim().length > 0) { + return { kind: "id", id: value.trim() }; + } + } + return { kind: "malformed" }; +} + +type ImportedRead = + | { readonly kind: "absent" } + | { readonly kind: "malformed" } + | { readonly kind: "entries"; readonly entries: readonly unknown[] }; + +function readImportedTranscripts(payload: unknown): ImportedRead { + if (payload === null || payload === undefined) return { kind: "absent" }; + if (typeof payload !== "object" || Array.isArray(payload)) return { kind: "malformed" }; + const record = payload as Record; + if (!Object.hasOwn(record, "importedTranscripts")) return { kind: "absent" }; + const entries = record["importedTranscripts"]; + if (!Array.isArray(entries)) return { kind: "malformed" }; + return { kind: "entries", entries }; +} + +interface ValidImportedSource { + readonly usageProvider: UsageProviderKind; + readonly providerInstanceId: string; + readonly providerSessionId: string; +} + +/** Mirrors the `AgentSessionImportSource` schema for the fields we bind on. */ +function decodeImportedSource(entry: unknown): ValidImportedSource | null { + if (typeof entry !== "object" || entry === null || Array.isArray(entry)) return null; + const record = entry as Record; + const usageProvider = usageProviderOf( + typeof record["provider"] === "string" ? record["provider"] : "", + ); + if (usageProvider === null) return null; + const providerInstanceId = record["providerInstanceId"]; + const providerSessionId = record["providerSessionId"]; + const filePath = record["filePath"]; + if (typeof providerInstanceId !== "string" || providerInstanceId.trim().length === 0) return null; + if (typeof providerSessionId !== "string" || providerSessionId.trim().length === 0) return null; + if (typeof filePath !== "string" || filePath.trim().length === 0) return null; + return { + usageProvider, + providerInstanceId: providerInstanceId.trim(), + providerSessionId: providerSessionId.trim(), + }; +} + +/** Reads native-session → thread bindings from persisted runtime rows. */ +export function extractAttributionBindings( + rows: readonly PersistedProviderSessionRuntimeRow[], +): AttributionBindingExtraction { + const nativeSessions: ExtractedNativeSession[] = []; + const bindings: AttributionThreadBinding[] = []; + const sessionThreads = new Map>(); + const diagnostics = { + runtimeRows: rows.length, + runtimeCursorBindings: 0, + importedTranscriptBindings: 0, + absentIdentityRows: 0, + malformedResumeCursors: 0, + malformedRuntimePayloads: 0, + skippedImportedTranscripts: 0, + unsupportedProviderBindings: 0, + overwrittenThreads: 0, + ambiguousSessionIds: 0, + }; + + for (const row of rows) { + const usageProvider = usageProviderOf(row.providerName, row.adapterKey); + + const cursor = readResumeCursor(row.resumeCursor); + if (cursor.kind === "malformed") diagnostics.malformedResumeCursors += 1; + if (cursor.kind === "id") { + diagnostics.runtimeCursorBindings += 1; + if (usageProvider === null) diagnostics.unsupportedProviderBindings += 1; + nativeSessions.push({ + threadId: row.threadId, + providerName: row.providerName, + adapterKey: row.adapterKey, + providerInstanceId: row.providerInstanceId, + nativeSessionId: cursor.id, + origin: "runtimeCursor", + usageProvider, + }); + if (usageProvider !== null) { + bindings.push({ + threadId: row.threadId, + provider: usageProvider, + providerInstanceId: row.providerInstanceId, + nativeSessionId: cursor.id, + origin: "runtimeCursor", + }); + addSessionThread(sessionThreads, usageProvider, cursor.id, row.threadId); + } + } + + const imported = readImportedTranscripts(row.runtimePayload); + if (imported.kind === "malformed") diagnostics.malformedRuntimePayloads += 1; + if (imported.kind === "entries") { + const importedSessionIds = new Set(); + for (const entry of imported.entries) { + const source = decodeImportedSource(entry); + if (source === null) { + diagnostics.skippedImportedTranscripts += 1; + continue; + } + const expectedThreadId = `import:${source.providerInstanceId}:${source.providerSessionId}`; + // An imported transcript is only a binding for the thread it names. + if (row.threadId !== expectedThreadId) { + diagnostics.skippedImportedTranscripts += 1; + continue; + } + diagnostics.importedTranscriptBindings += 1; + importedSessionIds.add(source.providerSessionId); + nativeSessions.push({ + threadId: row.threadId, + providerName: row.providerName, + adapterKey: row.adapterKey, + providerInstanceId: source.providerInstanceId, + nativeSessionId: source.providerSessionId, + origin: "importedTranscript", + usageProvider: source.usageProvider, + }); + bindings.push({ + threadId: row.threadId, + provider: source.usageProvider, + providerInstanceId: source.providerInstanceId, + nativeSessionId: source.providerSessionId, + origin: "importedTranscript", + }); + addSessionThread( + sessionThreads, + source.usageProvider, + source.providerSessionId, + row.threadId, + ); + } + // A retained imported session that is not the current cursor means the + // cursor was overwritten; the earlier identity survives only here. + if (cursor.kind === "id" && importedSessionIds.size > 0) { + const differs = [...importedSessionIds].some((id) => id !== cursor.id); + if (differs) diagnostics.overwrittenThreads += 1; + } + } + + // Truly absent: no cursor and no imported-transcript property at all. A + // malformed value is counted on its own axis, not folded into absence. + if (cursor.kind === "absent" && imported.kind === "absent") { + diagnostics.absentIdentityRows += 1; + } + } + + for (const threads of sessionThreads.values()) { + if (threads.size > 1) diagnostics.ambiguousSessionIds += 1; + } + + return { bindings, nativeSessions, diagnostics }; +} + +function addSessionThread( + index: Map>, + provider: UsageProviderKind, + sessionId: string, + threadId: string, +): void { + const key = `${provider}\u0000${sessionId}`; + const threads = index.get(key) ?? new Set(); + threads.add(threadId); + index.set(key, threads); +} + +/** Reads thread → PR links from persisted projection rows, dropping payloads. */ +export function extractAttributionLinks( + rows: readonly PersistedThreadPullRequestRow[], +): AttributionLinkExtraction { + const links: AttributionPullRequestLink[] = []; + let dismissed = 0; + let malformed = 0; + + for (const row of rows) { + if ( + typeof row.threadId !== "string" || + row.threadId.length === 0 || + typeof row.host !== "string" || + row.host.trim().length === 0 || + typeof row.repository !== "string" || + row.repository.trim().length === 0 || + !Number.isSafeInteger(row.number) || + row.number <= 0 || + !LINK_SOURCES.has(row.source) || + typeof row.linkedAt !== "string" || + row.linkedAt.length === 0 + ) { + malformed += 1; + continue; + } + if (row.source === "stack-dismissed") dismissed += 1; + links.push({ + threadId: row.threadId, + host: row.host, + repository: row.repository, + number: row.number, + source: row.source, + linkedAt: row.linkedAt, + ...(typeof row.url === "string" && row.url.length > 0 ? { url: row.url } : {}), + }); + } + + return { links, diagnostics: { rows: rows.length, links: links.length, dismissed, malformed } }; +} + +/** Combined read-only snapshot the projection can be reproduced from. */ +export function extractAttributionSnapshot(input: { + readonly cutoffMs: number; + readonly runtimeRows: readonly PersistedProviderSessionRuntimeRow[]; + readonly linkRows: readonly PersistedThreadPullRequestRow[]; +}): AttributionSnapshotExtraction { + const bindings = extractAttributionBindings(input.runtimeRows); + const links = extractAttributionLinks(input.linkRows); + return { + cutoffMs: input.cutoffMs, + bindings: bindings.bindings, + nativeSessions: bindings.nativeSessions, + links: links.links, + diagnostics: { bindings: bindings.diagnostics, links: links.diagnostics }, + }; +} diff --git a/docs/internals/usage-attribution.md b/docs/internals/usage-attribution.md index efb5c6ea85fd..43149adc3c0d 100644 --- a/docs/internals/usage-attribution.md +++ b/docs/internals/usage-attribution.md @@ -6,8 +6,14 @@ provider request, native session, and pull request. It exists so model compariso be done at the level a source can actually establish, instead of dividing a turn's tokens into invented requests. It is a pure function over allowlisted metadata: it never reads the clock, the filesystem, or the database, and it never sees a prompt, -response, or tool payload. Ingestion is the caller's job; the projection takes records, -explicit bindings, and existing PR links. +response, or tool payload. + +[`usageAttributionSources.ts`](../../apps/server/src/usage/usageAttributionSources.ts) +is the read-only extraction seam that proves the pure function can be fed from what +the server actually writes. It reads the allowlisted fields of +`provider_session_runtime` (`resume_cursor_json`, `runtime_payload_json.importedTranscripts`) +and `projection_thread_pull_requests`, and returns a binding/link snapshot plus +diagnostics for what it could not read. It never returns a runtime payload. ## Granularity is a source property @@ -24,59 +30,103 @@ This is stated once in `ATTRIBUTION_SOURCE_CAPABILITIES` and mirrored into each projection's `coverage`. A user prompt can span several Claude requests through tool continuation, and no prompt id is written, so prompt totals are not derivable for Claude. A Codex turn has no request id at all, so a request count there is never a -division of the turn: it is `null` with a `unsupported` quality. The request/message -ids were added to `UsageRecord` (and the v4 scan cache) precisely so they stay -separate from `dedupeKey`, which is a de-duplication composite and not a provider -request id. +division of the turn: it is `null` with an `unsupported` quality. `liveQualified` is `false` for every row in the matrix. The capability claims come from the parsers and adapter cursor shapes in source, not from an installed-live capture, and code that needs live evidence should say so. -## The identity join - -A native session reaches a T3 thread through exactly one of two durable places, both -already persisted: - -- the current `resume_cursor_json` on `provider_session_runtime` — `{threadId}` for - Codex, `{resume}` for Claude, `{sessionId}` for ACP and OpenCode; -- `runtime_payload_json.importedTranscripts`, which accumulates imported Claude/Codex - file identities (including `providerSessionId`) and is the only historical binding - that survives an upsert. - -`provider_session_runtime` is one row per thread and the upsert overwrites -`resume_cursor_json`, so after a session switch, fork, or model-change restart only the -newest native id is durable. Usage from an earlier native id therefore becomes -unbound, and the projection reports it as `unallocated` rather than guessing an owner. -A thread → PR link comes from `projection_thread_pull_requests`, canonicalized with -`@t3tools/shared/threadPullRequests`; the projection does not resolve PRs itself. - -`projection_thread_sessions` also has `provider_session_id` and `provider_thread_id` -columns, but the live upsert in `ProjectionThreadSessions.ts` never writes them, so -they carry no current mapping and must not be used as a join. +## Four independent axes + +A level's honesty is the combination of four things that are deliberately kept apart, +because collapsing them is how a zero-cost success gets invented: + +- **identity validity** (`identityQuality`: `valid | missing | invalid`) — is the + native session id present and well-formed? A malformed Claude id is `invalid`. +- **measurement completeness** (`measurementQuality`: `measured | partial | missing | +invalid | unavailable`) — were tokens actually measured? An explicit zero is + `measured`; Claude's `usage: {}` is `invalid`; an all-zero legacy row whose presence + was erased is `unavailable`, never `missing` and never a measured zero. +- **level support** (`promptQuality` / `requestQuality`) — can the source establish + this level, and did the records carry its id? `unsupported` is a structural limit, + not a zero. +- **allocation certainty** (`allocation`: `attributed | shared | unallocated | +ambiguous | missing | orphan`). + +The M1 contract uses `exact | partial | unavailable | ambiguous` for the same ideas. +The mapping is by meaning: `measured` → `exact`, `partial` → `partial`, `missing` / +`unavailable` → `unavailable`, `invalid` stays a malformed observation, `unsupported` +stays a structural limit. No enum is renamed mechanically. + +## Identity is not content equality + +`dedupeKey` is the **scan/delivery identity**, kept apart from the native observation +ids (`providerRequestId`, `providerMessageId`, `promptId`), which are reporting values. +The projection resolves identity in three ways: + +- **declared** — a `dedupeKey` present on the record, namespaced by provider so equal + local ids from two providers cannot collide. Two deliveries of the same key with the + same content are one observation (a repeated scan or a copied/moved rollout). +- **occurrence** — for a keyless source such as Codex `token_count`, the scan stamps an + occurrence-aware key from `usageEventOccurrenceBaseKey` plus a per-delivery occurrence + index. A copied rollout restarts its counter, so the copy lands on the same key and is + de-duplicated, while two genuine equal events in one file land on different keys and + are both kept. +- **unkeyed** — no identity at all. The record is **kept and counted**, never merged by + content equality, and the owning session is marked `recordIdentity: "uncertain"`. + A repeated delivery cannot be told from a second equal occurrence, so the projection + says so instead of guessing. + +Two versions of one identity with different content are a **conflict**: the first is +kept and the conflict is surfaced (`identity.conflicts`, `session.conflict`, a +limitation). A record whose source defines snapshot semantics (`scope: "snapshot"`) +instead **replaces** the earlier value for that identity, matching a +`cumulative_snapshot`/`aggregate` observation that is non-additive. + +## Nothing measured is dropped + +A record with no native session id is preserved in an explicit `orphan` bucket rather +than being discarded, and coverage seeds from declared sources and records before any +session is built, so a provider with only a failed or missing source still produces a +coverage row. The reconciliation identity holds against the deduplicated input, not a +pre-filtered session list: + +``` +sum(PR.attributed) + shared + unallocated + orphan === measured +``` + +where `measured` is the total of every distinct input record. Ambiguous sessions (a +native session bound to more than one thread) are pooled with unallocated usage; neither +can be placed on a PR without inventing an owner. Per-model contributions are preserved +at the session and PR levels, with each model's `costUsd` and `costSource` kept separate +so an unpriced model is never hidden by a priced one. ## Association is not attribution A session linked to two pull requests is reported once in the `shared` pool, which is explicitly not additive, and is never cloned onto both PRs. Only sessions bound to -exactly one strong link contribute to a PR's `attributed` total. The projection holds -`sum(attributed PR totals) + shared + unallocated = the distinct measured total`, so a -reader can reconcile every token exactly once. A `stack` link is a +exactly one strong link contribute to a PR's `attributed` total. A `stack` link is a display association, not evidence of billed work, so it feeds `stackAssociationSessions` and never `attributed`; `stack-dismissed` tombstones are -ignored, matching `visibleThreadPullRequests`. Link changes therefore do not rewrite -past allocations — the projection is recomputed from the links that exist at read time. - -## Data quality and duplicate scans - -The output distinguishes a measured zero from an absent measurement: a known session -with no usage has `totals: null` and quality `missing`, and contributes nothing to any -pool, so a failed turn can never read as a zero-cost success. Malformed Claude session -ids are `invalid`; a session with some records lacking the level's id is `partial`. -Records are de-duplicated by `dedupeKey`, falling back to a content signature for -sourceless records such as Codex turns, so two environments scanning one transcript -directory cannot count it twice. The projection never rescans history; it consumes the -records the append-only scan cache already produced. +ignored, matching `visibleThreadPullRequests`. + +Associations are read from the links that exist at the read cutoff +(`association.basis: "links-at-read-time"`, `cutoffMs === generatedAtMs`). `linkedAt` +does **not** gate allocation, so pre-link implementation work is included, and a link +that is later added, removed, or changed rewrites the recomputed view. The projection +therefore does not claim that changed links leave past allocations untouched: +historical allocation as of a past instant is unavailable without temporal evidence. + +## Cache upgrades retain existing history + +The scan cache version is still `4`, but a `v3` document is now **read** rather than +discarded. The scan retains measured records from transcripts that have since been +deleted for 90 days, and those cannot be re-parsed, so discarding a v3 cache would +destroy that history. A v3 row decodes with its native ids and measurement presence +explicitly `unavailable` (an all-zero row stays unknown, not a measured zero; a nonzero +row is still a known measurement), and the entry is flagged so it is never resumed +incrementally. An extant file is cold re-parsed on the next scan, which enriches it with +ids and presence without double counting because the re-parse replaces the entry. ## What still needs architecture approval @@ -87,3 +137,10 @@ cursor) is the one schema change that would widen coverage, and it is deliberate adopted here. Provider-instance identity is likewise not recoverable from a transcript scan; correlate that when the scan starts tagging files with the instance that produced them. + +`UsageProviderKind` is `claude | codex | grok`. OpenCode, Antigravity, and Cursor have a +native cursor id but no transcript T3 scans, so they have no usage source here. +`usageAttributionSources` still exposes their native session → thread identity as a +narrow `ExtractedNativeSession` with `usageProvider: null` for M1R to consume; that is +the whole integration. Turn-only data remains turn-only: nothing in this module reads +`projection_turns`, and no OpenCode parser is added. From daa55eb88966665faa8c1770c35fc3c53a68f6fa Mon Sep 17 00:00:00 2001 From: nullstack65 Date: Wed, 23 Sep 2026 19:31:25 -0400 Subject: [PATCH 6/7] fix(usage): finish measurement and cache correctness Classify usage fields by validity and provider completeness instead of property presence: an invalid value is invalid, a valid known subset is partial, and a nonzero total never implies a complete measurement. Make dedupe-key scope explicit. A source-local key is qualified by its native session; a global key reused under another session is incompatible ownership surfaced as a conflict. Cost and provenance are part of the observation, so a repriced record conflicts rather than silently deduping. Require the current identity format for a warm scan-cache hit so an unchanged legacy transcript is cold re-parsed once to enrich it, keeping deleted history and read-failure fallbacks. Carry the legacy identity-erased marker through to the projection so an erased native id is unavailable, not missing. --- apps/server/src/usage/UsageService.test.ts | 129 ++++++++++ apps/server/src/usage/UsageService.ts | 12 +- .../server/src/usage/usageAttribution.test.ts | 243 ++++++++++++++++++ apps/server/src/usage/usageAttribution.ts | 127 +++++++-- .../src/usage/usageAttributionSources.test.ts | 186 ++++++++++++++ .../src/usage/usageAttributionSources.ts | 9 +- apps/server/src/usage/usageScanCache.test.ts | 57 ++++ apps/server/src/usage/usageScanCache.ts | 82 +++++- .../server/src/usage/usageTranscripts.test.ts | 68 +++++ apps/server/src/usage/usageTranscripts.ts | 195 ++++++++++++-- docs/internals/usage-attribution.md | 34 ++- 11 files changed, 1070 insertions(+), 72 deletions(-) diff --git a/apps/server/src/usage/UsageService.test.ts b/apps/server/src/usage/UsageService.test.ts index b391e213ab9a..bd23deccb353 100644 --- a/apps/server/src/usage/UsageService.test.ts +++ b/apps/server/src/usage/UsageService.test.ts @@ -32,6 +32,7 @@ import * as ServerSettings from "../serverSettings.ts"; import * as UsageService from "./UsageService.ts"; const encodeUnknownJsonString = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); +const decodeUnknownJsonString = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown)); function claudeLine(id: number, outputTokens: number, model = "claude-fable-5"): string { return `${JSON.stringify({ @@ -501,6 +502,134 @@ describe("UsageService", () => { }).pipe(Effect.scoped), ); + it.live("counts empty and invalid usage containers as malformed, not zero", () => + Effect.gen(function* () { + const { transcript, settings, home } = yield* setup; + const lines = + [ + encodeUnknownJsonString({ + type: "assistant", + timestamp: "2026-08-01T10:00:00Z", + sessionId: "session-1", + message: { id: "m_empty", model: "claude-fable-5", usage: {} }, + }), + encodeUnknownJsonString({ + type: "assistant", + timestamp: "2026-08-01T10:00:01Z", + sessionId: "session-1", + message: { id: "m_invalid", model: "claude-fable-5", usage: { input_tokens: null } }, + }), + claudeLine(3, 9), + ].join("\n") + "\n"; + yield* Effect.promise(() => NodeFSP.writeFile(transcript, lines)); + + yield* Effect.gen(function* () { + const service = yield* UsageService.make; + const summary = yield* service.readSummary(WINDOW); + assert.strictEqual( + summary.sources.find((source) => source.fingerprint.provider === "claude") + ?.malformedRecords, + 2, + ); + // Only the valid line contributes tokens. + assert.strictEqual(totalOutputTokens(summary), 9); + }).pipe( + Effect.provide(serviceLayers({ prefix: "usage-service-malformed-test", home, settings })), + ); + }).pipe(Effect.scoped), + ); + + it.live("enriches an unchanged legacy v3 entry once and keeps deleted history", () => + Effect.gen(function* () { + const { transcript, settings, home } = yield* setup; + const content = claudeLine(1, 5); + yield* Effect.promise(() => NodeFSP.writeFile(transcript, content)); + + yield* Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const liveDir = yield* Effect.promise(() => NodeFSP.realpath(NodePath.dirname(transcript))); + const livePath = NodePath.join(liveDir, NodePath.basename(transcript)); + const liveStat = yield* Effect.promise(() => NodeFSP.stat(livePath)); + const deletedPath = NodePath.join(liveDir, "deleted.jsonl"); + const scanCachePath = NodePath.join(config.stateDir, "usage-scan-cache.json"); + const TS = Date.parse("2026-08-01T10:00:00Z"); + // A v3 document: the live entry's (size, mtime) match the file exactly so + // the pre-fix warm hit would have served the erased row forever, and the + // deleted entry exists only in the cache. + yield* Effect.promise(() => + NodeFSP.writeFile( + scanCachePath, + encodeUnknownJsonString({ + version: 3, + models: ["claude-fable-5"], + sessions: ["session-1", "deleted-session"], + files: { + [livePath]: { + s: liveStat.size, + m: liveStat.mtimeMs, + p: "claude", + r: [[TS, 0, 0, 10, 0, 0, 5, 0, "msg_live:", null]], + t: [], + o: liveStat.size, + gl: 0, + gh: 0, + cs: null, + }, + [deletedPath]: { + s: 100, + m: liveStat.mtimeMs, + p: "claude", + r: [[TS, 0, 1, 10, 0, 0, 7, 0, "msg_deleted:", null]], + t: [], + o: 90, + gl: 64, + gh: 11, + cs: null, + }, + }, + }), + ), + ); + + const service = yield* UsageService.make; + const first = yield* service.readSummary(WINDOW); + // The deleted entry's 7 tokens are retained and the live file's 5 are + // counted once, whether read from the enriched cache or re-parsed. + assert.strictEqual(totalOutputTokens(first), 12); + + const afterFirst = decodeUnknownJsonString( + yield* Effect.promise(() => NodeFSP.readFile(scanCachePath, "utf8")), + ) as { + version: number; + files: Record; + }; + assert.strictEqual(afterFirst.version, 4); + const liveEntry = afterFirst.files[livePath]!; + assert.strictEqual(liveEntry.li, undefined); + // Enriched: the native request id the v3 row could not carry is present. + assert.strictEqual(liveEntry.r[0]?.[10], "req_1"); + assert.strictEqual(afterFirst.files[deletedPath]?.li, 1); + + // A second scan of the unchanged live file is a warm hit: no re-parse, + // no cache rewrite, and stable totals. + const beforeSecond = yield* Effect.promise(() => NodeFSP.readFile(scanCachePath, "utf8")); + const second = yield* service.readSummary(WINDOW); + assert.strictEqual(totalOutputTokens(second), 12); + const afterSecond = yield* Effect.promise(() => NodeFSP.readFile(scanCachePath, "utf8")); + assert.strictEqual(afterSecond, beforeSecond); + + // A restart reads the enriched cache and still keeps deleted history. + const restarted = yield* UsageService.make; + const third = yield* restarted.readSummary(WINDOW); + assert.strictEqual(totalOutputTokens(third), 12); + }).pipe( + Effect.provide( + serviceLayers({ prefix: "usage-service-legacy-enrich-test", home, settings }), + ), + ); + }).pipe(Effect.scoped), + ); + it.live("does not share an in-flight scan after custom prices change", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index 2b7cb483a80c..8bd7beac69c5 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -393,8 +393,12 @@ export const make = Effect.gen(function* () { const cached = fileCache.get(filePath); // Provider is part of the identity: if both providers were ever pointed // at one directory, a hit parsed by the other parser must not be reused. + // The cache format is part of it too: a legacy entry erased native ids and + // measurement presence, so an unchanged file must still cold re-parse once + // to enrich it instead of serving the erased row forever. if ( cached && + cached.identity === "declared" && cached.size === size && cached.mtimeMs === mtimeMs && cached.provider === provider @@ -578,8 +582,9 @@ export const make = Effect.gen(function* () { let scannedFiles = 0; let skippedFiles = 0; // A usage container with no recognised token field (Claude `usage: {}`) - // parses to a record but measured nothing; surface it rather than letting - // it read as a measured zero. + // or one whose fields are all present-but-invalid parses to a record but + // measured nothing; surface it rather than letting it read as a measured + // zero. let malformedRecords = 0; // Distinct per directory. Buckets carry per-cell session counts, but a // session spans days and models, so clients total this figure instead. @@ -593,7 +598,8 @@ export const make = Effect.gen(function* () { scannedFiles += 1; const codexEventOccurrences = new Map(); for (const record of file.records) { - if (record.measurement === "empty") malformedRecords += 1; + if (record.measurement === "empty" || record.measurement === "invalid") + malformedRecords += 1; let usageRecord = record; if (record.provider === "codex" && record.sessionId.length > 0) { // Match moved rollout copies without collapsing repeated equal events diff --git a/apps/server/src/usage/usageAttribution.test.ts b/apps/server/src/usage/usageAttribution.test.ts index 9b0d2f96f939..3d101c1349a2 100644 --- a/apps/server/src/usage/usageAttribution.test.ts +++ b/apps/server/src/usage/usageAttribution.test.ts @@ -13,6 +13,8 @@ import { type UsageAttributionInput, } from "./usageAttribution.ts"; import { totalTokens } from "./usageTranscripts.ts"; +import { parseClaudeLine, type UsageRecord } from "./usageTranscripts.ts"; +import { decodeScanCache, encodeScanCache, type ScanCache } from "./usageScanCache.ts"; const CLAUDE_SESSION = "5a128faa-8253-489e-b935-6c08e8e670c0"; const OTHER_CLAUDE_SESSION = "11111111-2222-3333-4444-555555555555"; @@ -398,6 +400,111 @@ describe("identity: repeated deliveries, occurrences, copies, conflicts", () => }); }); +describe("identity scope and cost provenance", () => { + it("keeps two source-local keys from different sessions as separate records", () => { + const first = record({ + sessionId: CLAUDE_SESSION, + dedupeKey: "1", + dedupeKeyScope: "source-local", + totals: totals({ outputTokens: 10 }), + }); + const second = record({ + sessionId: OTHER_CLAUDE_SESSION, + dedupeKey: "1", + dedupeKeyScope: "source-local", + totals: totals({ outputTokens: 20 }), + }); + const projection = buildUsageAttribution( + input({ + records: [first, second], + bindings: [ + binding({ threadId: "thread-1", nativeSessionId: CLAUDE_SESSION }), + binding({ threadId: "thread-2", nativeSessionId: OTHER_CLAUDE_SESSION }), + ], + }), + ); + + expect(projection.sessions).toHaveLength(2); + expect(projection.identity.duplicatesDropped).toBe(0); + expect(projection.identity.conflicts).toBe(0); + expect(projection.measured.tokens.outputTokens).toBe(30); + }); + + it("surfaces a global key reused under a second session as a conflict", () => { + const first = record({ + sessionId: CLAUDE_SESSION, + dedupeKey: "1", + dedupeKeyScope: "global", + }); + const reused = record({ + sessionId: OTHER_CLAUDE_SESSION, + dedupeKey: "1", + dedupeKeyScope: "global", + }); + const projection = buildUsageAttribution( + input({ + records: [first, reused], + bindings: [ + binding({ threadId: "thread-1", nativeSessionId: CLAUDE_SESSION }), + binding({ threadId: "thread-2", nativeSessionId: OTHER_CLAUDE_SESSION }), + ], + }), + ); + + // Incompatible ownership is not silently dropped as a duplicate. + expect(projection.identity.conflicts).toBe(1); + expect(projection.identity.duplicatesDropped).toBe(0); + expect(projection.measured.records).toBe(1); + const flagged = projection.sessions.filter((session) => session.conflict); + expect(flagged).toHaveLength(2); + }); + + it("still collapses a copy of a global key at another physical path", () => { + const original = record({ + dedupeKey: "m1:r1", + dedupeKeyScope: "global", + sourceFingerprint: CLAUDE_FINGERPRINT, + }); + const copy = record({ + dedupeKey: "m1:r1", + dedupeKeyScope: "global", + sourceFingerprint: OTHER_FINGERPRINT, + }); + const projection = buildUsageAttribution( + input({ records: [original, copy], bindings: [binding()] }), + ); + + expect(projection.identity.duplicatesDropped).toBe(1); + expect(projection.identity.conflicts).toBe(0); + expect(projection.measured.records).toBe(1); + }); + + it("surfaces a cost-only change as a conflict, not a duplicate", () => { + const first = record({ dedupeKey: "m1:r1", costUsd: 0.1 }); + const repriced = record({ dedupeKey: "m1:r1", costUsd: 99 }); + const projection = buildUsageAttribution( + input({ records: [first, repriced], bindings: [binding()] }), + ); + + expect(projection.identity.conflicts).toBe(1); + expect(projection.identity.duplicatesDropped).toBe(0); + // Kept-first, with the conflict surfaced rather than the change applied. + expect(projection.sessions[0]?.totals?.costUsd).toBe(0.1); + expect(projection.sessions[0]?.conflict).toBe(true); + }); + + it("treats a differing cost provenance as a conflict", () => { + const first = record({ dedupeKey: "m1:r1", costSource: "modelPriced" }); + const reported = record({ dedupeKey: "m1:r1", costSource: "providerReported" }); + const projection = buildUsageAttribution( + input({ records: [first, reported], bindings: [binding()] }), + ); + + expect(projection.identity.conflicts).toBe(1); + expect(projection.identity.duplicatesDropped).toBe(0); + }); +}); + describe("measurement quality", () => { it("treats a Claude usage:{} record as invalid, not a measured zero", () => { const empty = record({ dedupeKey: "m1:", measurement: "empty", totals: zeroTotals() }); @@ -450,6 +557,50 @@ describe("measurement quality", () => { expect(projection.sessions[0]?.measurementQuality).toBe("partial"); }); + it("keeps a partial measurement partial, not measured", () => { + const partial = record({ + dedupeKey: "m1:", + measurement: "observed", + measurementCompleteness: "partial", + invalidTokenFields: 1, + }); + const projection = buildUsageAttribution(input({ records: [partial], bindings: [binding()] })); + + expect(projection.sessions[0]?.measurementQuality).toBe("partial"); + }); + + it("keeps a present-but-invalid value invalid, not a measured zero", () => { + const invalid = record({ + dedupeKey: "m1:", + measurement: "invalid", + totals: zeroTotals(), + }); + const projection = buildUsageAttribution(input({ records: [invalid], bindings: [binding()] })); + + expect(projection.sessions[0]?.measurementQuality).toBe("invalid"); + expect(projection.coverage.find((entry) => entry.provider === "claude")?.invalidSessions).toBe( + 1, + ); + }); + + it("reports erased native identity as unavailable, not missing", () => { + // A legacy nonzero row: the tokens are a known measurement, but the native + // request id was erased, so the request level is unavailable. + const legacy = record({ + dedupeKey: "legacy:1", + measurement: "observed", + measurementCompleteness: "partial", + identityAvailable: false, + totals: totals({ outputTokens: 40 }), + }); + const projection = buildUsageAttribution(input({ records: [legacy], bindings: [binding()] })); + const session = projection.sessions[0]!; + + expect(session.measurementQuality).toBe("partial"); + expect(session.requestQuality).toBe("unavailable"); + expect(session.requestCount).toBeNull(); + }); + it("surfaces a failed declared source instead of reading it as measured", () => { const source: AttributionSource = { fingerprint: CLAUDE_FINGERPRINT, @@ -755,6 +906,98 @@ describe("pull request association and attribution", () => { }); }); +describe("parser to projection", () => { + function claudeUsageLine(usage: Record, requestId = "req_1"): string { + return JSON.stringify({ + type: "assistant", + timestamp: "2026-08-07T04:05:13.944Z", + sessionId: CLAUDE_SESSION, + requestId, + message: { id: "msg_1", model: "claude-fable-5", usage }, + }); + } + + function fromParsed(record: UsageRecord): AttributionUsageRecord { + return { + provider: record.provider, + sessionId: record.sessionId, + model: record.model, + timestampMs: record.timestampMs, + totals: record.totals, + costUsd: 0, + dedupeKey: record.dedupeKey, + sourceFingerprint: CLAUDE_FINGERPRINT, + ...(record.dedupeKeyScope === undefined ? {} : { dedupeKeyScope: record.dedupeKeyScope }), + ...(record.providerRequestId === undefined + ? {} + : { providerRequestId: record.providerRequestId }), + ...(record.providerMessageId === undefined + ? {} + : { providerMessageId: record.providerMessageId }), + ...(record.promptId === undefined ? {} : { promptId: record.promptId }), + ...(record.measurement === undefined ? {} : { measurement: record.measurement }), + ...(record.measurementCompleteness === undefined + ? {} + : { measurementCompleteness: record.measurementCompleteness }), + ...(record.invalidTokenFields === undefined + ? {} + : { invalidTokenFields: record.invalidTokenFields }), + ...(record.identityAvailable === undefined + ? {} + : { identityAvailable: record.identityAvailable }), + }; + } + + it("carries a partial usage object through to a partial session", () => { + const parsed = parseClaudeLine(claudeUsageLine({ input_tokens: 10 }))!; + const projection = buildUsageAttribution( + input({ records: [fromParsed(parsed)], bindings: [binding()] }), + ); + const session = projection.sessions[0]!; + + expect(parsed.measurementCompleteness).toBe("partial"); + expect(session.measurementQuality).toBe("partial"); + expect(session.totals?.tokens.uncachedInputTokens).toBe(10); + }); + + it("carries an invalid usage value through to an invalid session", () => { + const parsed = parseClaudeLine(claudeUsageLine({ input_tokens: null }))!; + const projection = buildUsageAttribution( + input({ records: [fromParsed(parsed)], bindings: [binding()] }), + ); + + expect(parsed.measurement).toBe("invalid"); + expect(projection.sessions[0]?.measurementQuality).toBe("invalid"); + }); + + it("survives a cache round trip without promoting partial to complete", () => { + const parsed = parseClaudeLine(claudeUsageLine({ input_tokens: 10 }))!; + const cache: ScanCache = new Map([ + [ + "/a.jsonl", + { + size: 10, + mtimeMs: 1, + provider: "claude", + records: [parsed], + tailRecords: [], + position: { resumeOffset: 0, guardLength: 0, guardHash: 0, codexState: null }, + identity: "declared", + }, + ], + ]); + const restored = decodeScanCache(JSON.parse(JSON.stringify(encodeScanCache(cache)))); + const roundTripped = restored.get("/a.jsonl")!.records[0]!; + + expect(roundTripped.measurementCompleteness).toBe("partial"); + expect(roundTripped.dedupeKeyScope).toBe("global"); + const projection = buildUsageAttribution( + input({ records: [fromParsed(roundTripped)], bindings: [binding()] }), + ); + expect(projection.sessions[0]?.measurementQuality).toBe("partial"); + }); +}); + describe("projection contract", () => { it("exposes the source capability matrix with live qualification falsy", () => { const projection = buildUsageAttribution(input({})); diff --git a/apps/server/src/usage/usageAttribution.ts b/apps/server/src/usage/usageAttribution.ts index c79628a2a8cc..4efff2a1e740 100644 --- a/apps/server/src/usage/usageAttribution.ts +++ b/apps/server/src/usage/usageAttribution.ts @@ -45,6 +45,11 @@ import { } from "@t3tools/shared/threadPullRequests"; import { EMPTY_TOTALS, addTotals, totalTokens as countTokens } from "./usageTranscripts.ts"; +import type { + DedupeKeyScope, + UsageMeasurement, + UsageMeasurementCompleteness, +} from "./usageTranscripts.ts"; export const USAGE_ATTRIBUTION_VERSION = 2 as const; @@ -160,6 +165,12 @@ export interface AttributionUsageRecord { readonly costUsd: number; /** Scan/delivery identity, or `null` when the record is unkeyed. */ readonly dedupeKey: string | null; + /** + * How far `dedupeKey` can be trusted on its own. `global` (the default) means + * the key is a globally qualified native observation id; `source-local` means + * it must be qualified by the native session. See `usageTranscripts`. + */ + readonly dedupeKeyScope?: DedupeKeyScope; readonly providerRequestId?: string | null; readonly providerMessageId?: string | null; readonly promptId?: string | null; @@ -170,7 +181,17 @@ export interface AttributionUsageRecord { * any total is nonzero and `unavailable` when all are zero, so a legacy row * whose presence was erased is never read as a measured zero. */ - readonly measurement?: "observed" | "empty" | "unavailable"; + readonly measurement?: UsageMeasurement; + /** Whether an `observed` measurement covered every required field. */ + readonly measurementCompleteness?: UsageMeasurementCompleteness; + /** Present-but-invalid recognised token fields; distinguishes invalid from absent. */ + readonly invalidTokenFields?: number; + /** + * `false` when the source erased native identity before we saw it (a legacy + * cache row). Kept apart from numeric quality so an absent native id is + * reported `unavailable`, never recovered from token magnitude. + */ + readonly identityAvailable?: boolean; /** Additive increment or replaceable snapshot. Absent means `delta`. */ readonly scope?: "delta" | "snapshot"; /** Cost provenance, preserved per record so a view can carry it. */ @@ -456,17 +477,47 @@ function anyTotal(record: AttributionUsageRecord): number { } /** Presence of a measurement, with the legacy-erased case made explicit. */ -function effectiveMeasurement( - record: AttributionUsageRecord, -): "observed" | "empty" | "unavailable" { +function effectiveMeasurement(record: AttributionUsageRecord): UsageMeasurement { if (record.measurement !== undefined) return record.measurement; return anyTotal(record) > 0 ? "observed" : "unavailable"; } +/** + * The identity a record is de-duplicated under. + * + * A `global` key is namespaced by provider only; a `source-local` key is + * qualified by the canonical native session, never by a physical path, so a + * copy of the same session at another location still collapses while two + * sessions that happen to reuse a local key stay distinct. + */ +function dedupeIdentity(record: AttributionUsageRecord): string { + return (record.dedupeKeyScope ?? "global") === "source-local" + ? `${record.provider}\u0000local\u0000${record.sessionId}\u0000${record.dedupeKey}` + : `${record.provider}\u0000${record.dedupeKey}`; +} + +/** Per-record numeric quality, keeping completeness and validity separate. */ +type RecordQuality = "measured" | "partial" | "empty" | "invalid" | "unavailable"; + +function recordQuality(record: AttributionUsageRecord): RecordQuality { + const measurement = record.measurement; + if (measurement === undefined) { + // No declared measurement: a nonzero total proves some tokens were + // measured, but never that the measurement was complete. + return anyTotal(record) > 0 ? "partial" : "unavailable"; + } + if (measurement === "observed") { + return record.measurementCompleteness === "partial" ? "partial" : "measured"; + } + return measurement; +} + /** * Content of a measured observation, used to tell a repeated delivery from a * conflicting version of the same identity. Deliberately excludes - * `sourceFingerprint`: a copy at another path is the same observation. + * `sourceFingerprint`: a copy at another path is the same observation. Cost and + * its provenance are included, so a record whose cost changed is a conflicting + * version of one observation rather than a silent duplicate. */ function observationContent(record: AttributionUsageRecord): string { return [ @@ -480,6 +531,8 @@ function observationContent(record: AttributionUsageRecord): string { record.providerRequestId ?? "", record.providerMessageId ?? "", record.promptId ?? "", + record.costUsd, + record.costSource ?? "", effectiveMeasurement(record), ].join("\u0000"); } @@ -561,13 +614,16 @@ interface MutableCoverage { export function buildUsageAttribution(input: UsageAttributionInput): UsageAttribution { // 1. Identity and de-duplication. // - // A declared key is the scan/delivery identity and is namespaced by - // provider so equal local ids from two providers cannot collide. A - // repeated delivery (same key, same content) is dropped; a snapshot - // replaces the earlier value; a differing delta for the same key is a - // conflict and is exposed rather than silently discarded. A record with no - // key at all is unkeyed: it is kept and counted, never merged by content - // equality, and the session is marked uncertain. + // A declared key is the scan/delivery identity. Its scope is explicit: a + // `global` key is a globally qualified native observation id and is + // namespaced by provider alone; a `source-local` key is qualified by the + // native session, because the same local key in two sessions is two + // observations, not one. A repeated delivery (same identity, same content) + // is dropped; a snapshot replaces the earlier value; a differing delta for + // the same identity is a conflict and is exposed rather than silently + // discarded. A record with no key at all is unkeyed: it is kept and + // counted, never merged by content equality, and the session is marked + // uncertain. const kept: AttributionUsageRecord[] = []; const keptIndexByIdentity = new Map(); const conflictSessions = new Set(); @@ -582,7 +638,7 @@ export function buildUsageAttribution(input: UsageAttributionInput): UsageAttrib kept.push(record); continue; } - const identity = `${record.provider}\u0000${record.dedupeKey}`; + const identity = dedupeIdentity(record); const existingIndex = keptIndexByIdentity.get(identity); if (existingIndex === undefined) { keptIndexByIdentity.set(identity, kept.length); @@ -590,6 +646,17 @@ export function buildUsageAttribution(input: UsageAttributionInput): UsageAttrib continue; } const existing = kept[existingIndex]!; + // A global key that shows up under a second native session is incompatible + // ownership, not a copy: one native observation cannot belong to two + // sessions. Surface it instead of silently dropping a version. A + // source-local key cannot reach here across sessions, because the session + // is part of its identity. + if (existing.sessionId !== record.sessionId) { + conflicts += 1; + conflictSessions.add(sessionKey(existing.provider, existing.sessionId)); + conflictSessions.add(sessionKey(record.provider, record.sessionId)); + continue; + } if (observationContent(existing) === observationContent(record)) { duplicatesDropped += 1; continue; @@ -784,20 +851,30 @@ export function buildUsageAttribution(input: UsageAttributionInput): UsageAttrib ? "invalid" : "valid"; - const measurements = accumulator.records.map(effectiveMeasurement); - const observedCount = measurements.filter((value) => value === "observed").length; - const emptyCount = measurements.filter((value) => value === "empty").length; - const unavailableCount = measurements.filter((value) => value === "unavailable").length; + const qualities = accumulator.records.map(recordQuality); + const hasValidMeasurement = qualities.some( + (quality) => quality === "measured" || quality === "partial", + ); const measurementQuality: AttributionQuality = accumulator.records.length === 0 ? "missing" - : observedCount === accumulator.records.length + : qualities.every((quality) => quality === "measured") ? "measured" - : observedCount === 0 && emptyCount > 0 && unavailableCount === 0 - ? "invalid" - : observedCount === 0 && unavailableCount > 0 - ? "unavailable" - : "partial"; + : qualities.every((quality) => quality === "unavailable") + ? "unavailable" + : hasValidMeasurement + ? "partial" + : qualities.some((quality) => quality === "invalid") + ? "invalid" + : qualities.every((quality) => quality === "empty") + ? "invalid" + : "unavailable"; + + // A legacy row erased native identity entirely; an absent id there is + // `unavailable`, never `missing`, and never recovered from a nonzero total. + const identityErased = + accumulator.records.length > 0 && + accumulator.records.every((record) => record.identityAvailable === false); const identityLevelQuality = ( level: "prompt" | "request", @@ -805,10 +882,10 @@ export function buildUsageAttribution(input: UsageAttributionInput): UsageAttrib ): AttributionQuality => { if (capability[level] === "unsupported") return "unsupported"; if (accumulator.records.length === 0) return "missing"; - if (unavailableCount > 0 && recordsWithId === 0) return "unavailable"; + if (identityErased) return "unavailable"; if (recordsWithId === 0) return "missing"; if (recordsWithId < accumulator.records.length) return "partial"; - if (observedCount === accumulator.records.length) return "measured"; + if (qualities.every((quality) => quality === "measured")) return "measured"; return "partial"; }; diff --git a/apps/server/src/usage/usageAttributionSources.test.ts b/apps/server/src/usage/usageAttributionSources.test.ts index 3c839e46c85a..265f90cbfbbe 100644 --- a/apps/server/src/usage/usageAttributionSources.test.ts +++ b/apps/server/src/usage/usageAttributionSources.test.ts @@ -244,6 +244,31 @@ describe("extractAttributionLinks", () => { }); describe("extractAttributionSnapshot", () => { + it("reads a Claude imported cursor that carries both threadId and resume", () => { + // Claude writes `{ threadId, resume }` together for an imported session; the + // native session id is `resume`, not the T3 thread id in `threadId`. + const { bindings, nativeSessions } = extractAttributionBindings([ + runtimeRow({ + threadId: "import:claude-original:original-session", + providerName: "claudeAgent", + adapterKey: "claudeAgent", + resumeCursor: { + threadId: "import:claude-original:original-session", + resume: "5a128faa-8253-489e-b935-6c08e8e670c0", + }, + }), + ]); + + expect(nativeSessions[0]).toMatchObject({ + nativeSessionId: "5a128faa-8253-489e-b935-6c08e8e670c0", + origin: "runtimeCursor", + usageProvider: "claude", + }); + expect(bindings[0]).toMatchObject({ + nativeSessionId: "5a128faa-8253-489e-b935-6c08e8e670c0", + }); + }); + it("labels the cutoff and never leaks the runtime payload", () => { const snapshot = extractAttributionSnapshot({ cutoffMs: 1_786_100_000_000, @@ -267,4 +292,165 @@ describe("extractAttributionSnapshot", () => { expect(JSON.stringify(snapshot)).not.toContain("do-not-leak"); expect(JSON.stringify(snapshot)).not.toContain("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/secret/path"); }); + + /** + * The deterministic interchange fixture M1C consumes. It is produced by the + * real extractor from persisted-row shapes only, so it can be regenerated + * exactly. Field semantics: + * + * - `cutoffMs` — associations are read as of this instant. + * - `bindings[]` — usage-provider native session -> T3 thread, with the + * canonical `provider` (never the adapter key) and `origin`. + * - `nativeSessions[]` — every native identity, including OpenCode with + * `usageProvider: null`; a label, never a join key. + * - `links[]` — canonical thread -> PR links; `stack-dismissed` tombstones are + * preserved for the projection to filter. + * - `diagnostics` — what could not be read, never dropped silently. + */ + it("produces a stable fixture for the M1C interchange", () => { + const snapshot = extractAttributionSnapshot({ + cutoffMs: 1_786_100_000_000, + runtimeRows: [ + runtimeRow({ + threadId: "thread-opencode", + providerName: "opencode", + adapterKey: "opencode", + providerInstanceId: "opencode-default", + resumeCursor: { sessionId: "ses_opencode_1" }, + }), + runtimeRow({ + threadId: "import:claude-original:original-session", + providerName: "claudeAgent", + adapterKey: "claudeAgent", + providerInstanceId: "claude-default", + resumeCursor: { + threadId: "import:claude-original:original-session", + resume: "5a128faa-8253-489e-b935-6c08e8e670c0", + }, + runtimePayload: { importedTranscripts: [importedSource()] }, + }), + runtimeRow({ + threadId: "thread-codex", + providerName: "codex", + adapterKey: "codex", + providerInstanceId: "codex-default", + resumeCursor: { threadId: "019fbbc1-b12c-7360-a685-28c181f0025f" }, + }), + ], + linkRows: [ + linkRow({ threadId: "thread-opencode", number: 12 }), + linkRow({ threadId: "thread-codex", number: 12, source: "agent" }), + linkRow({ threadId: "thread-codex", number: 13, source: "stack" }), + ], + }); + + expect(snapshot).toEqual({ + cutoffMs: 1_786_100_000_000, + bindings: [ + { + threadId: "import:claude-original:original-session", + provider: "claude", + providerInstanceId: "claude-default", + nativeSessionId: "5a128faa-8253-489e-b935-6c08e8e670c0", + origin: "runtimeCursor", + }, + { + threadId: "import:claude-original:original-session", + provider: "claude", + providerInstanceId: "claude-original", + nativeSessionId: "original-session", + origin: "importedTranscript", + }, + { + threadId: "thread-codex", + provider: "codex", + providerInstanceId: "codex-default", + nativeSessionId: "019fbbc1-b12c-7360-a685-28c181f0025f", + origin: "runtimeCursor", + }, + ], + nativeSessions: [ + { + threadId: "thread-opencode", + providerName: "opencode", + adapterKey: "opencode", + providerInstanceId: "opencode-default", + nativeSessionId: "ses_opencode_1", + origin: "runtimeCursor", + usageProvider: null, + }, + { + threadId: "import:claude-original:original-session", + providerName: "claudeAgent", + adapterKey: "claudeAgent", + providerInstanceId: "claude-default", + nativeSessionId: "5a128faa-8253-489e-b935-6c08e8e670c0", + origin: "runtimeCursor", + usageProvider: "claude", + }, + { + threadId: "import:claude-original:original-session", + providerName: "claudeAgent", + adapterKey: "claudeAgent", + providerInstanceId: "claude-original", + nativeSessionId: "original-session", + origin: "importedTranscript", + usageProvider: "claude", + }, + { + threadId: "thread-codex", + providerName: "codex", + adapterKey: "codex", + providerInstanceId: "codex-default", + nativeSessionId: "019fbbc1-b12c-7360-a685-28c181f0025f", + origin: "runtimeCursor", + usageProvider: "codex", + }, + ], + links: [ + { + threadId: "thread-opencode", + host: "github.com", + repository: "acme/repo", + number: 12, + source: "manual", + linkedAt: "2026-09-01T00:00:00.000Z", + url: "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/acme/repo/pull/12", + }, + { + threadId: "thread-codex", + host: "github.com", + repository: "acme/repo", + number: 12, + source: "agent", + linkedAt: "2026-09-01T00:00:00.000Z", + url: "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/acme/repo/pull/12", + }, + { + threadId: "thread-codex", + host: "github.com", + repository: "acme/repo", + number: 13, + source: "stack", + linkedAt: "2026-09-01T00:00:00.000Z", + url: "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/acme/repo/pull/12", + }, + ], + diagnostics: { + bindings: { + runtimeRows: 3, + runtimeCursorBindings: 3, + importedTranscriptBindings: 1, + absentIdentityRows: 0, + malformedResumeCursors: 0, + malformedRuntimePayloads: 0, + skippedImportedTranscripts: 0, + unsupportedProviderBindings: 1, + overwrittenThreads: 1, + ambiguousSessionIds: 0, + }, + links: { rows: 3, links: 3, dismissed: 0, malformed: 0 }, + }, + }); + }); }); diff --git a/apps/server/src/usage/usageAttributionSources.ts b/apps/server/src/usage/usageAttributionSources.ts index fa97b8876882..9c1b0be09966 100644 --- a/apps/server/src/usage/usageAttributionSources.ts +++ b/apps/server/src/usage/usageAttributionSources.ts @@ -149,9 +149,12 @@ type CursorRead = | { readonly kind: "id"; readonly id: string }; /** - * The three cursor shapes adapters actually write: `{ resume }` (Claude), - * `{ threadId }` (Codex), `{ sessionId }` (Grok, OpenCode, Antigravity). Order - * does not matter because a cursor carries exactly one of them. + * The cursor shapes adapters actually write. Claude writes `{ resume }` for a + * live session and `{ threadId, resume }` together for an imported one, so a + * cursor is not guaranteed to carry exactly one field. `resume` is the native + * session id and is preferred over `threadId` (the T3 thread id) when both are + * present; `{ threadId }` alone is Codex, `{ sessionId }` is Grok, OpenCode, + * and Antigravity. Order matters, so the preference is explicit. */ function readResumeCursor(cursor: unknown): CursorRead { if (cursor === null || cursor === undefined) return { kind: "absent" }; diff --git a/apps/server/src/usage/usageScanCache.test.ts b/apps/server/src/usage/usageScanCache.test.ts index 3eb821cef674..e5c5f278b37e 100644 --- a/apps/server/src/usage/usageScanCache.test.ts +++ b/apps/server/src/usage/usageScanCache.test.ts @@ -26,6 +26,8 @@ function record(overrides: Partial = {}): UsageRecord { reportedCostUsd: null, dedupeKey: "msg_1:", measurement: "observed", + measurementCompleteness: "complete", + dedupeKeyScope: "global", ...overrides, }; } @@ -125,6 +127,61 @@ describe("scan cache round trip", () => { }); }); + it("round-trips validity, completeness, and key-scope metadata", () => { + const original = cacheWith([ + [ + "/partial.jsonl", + 100, + [ + record({ + measurementCompleteness: "partial", + invalidTokenFields: 1, + dedupeKeyScope: "source-local", + }), + ], + ], + ["/invalid.jsonl", 100, [record({ measurement: "invalid" })]], + ]); + + const restored = decodeScanCache(JSON.parse(JSON.stringify(encodeScanCache(original)))); + + expect(restored.get("/partial.jsonl")?.records[0]).toMatchObject({ + measurement: "observed", + measurementCompleteness: "partial", + invalidTokenFields: 1, + dedupeKeyScope: "source-local", + }); + expect(restored.get("/invalid.jsonl")?.records[0]?.measurement).toBe("invalid"); + }); + + it("preserves the legacy identity-unavailable marker across a round trip", () => { + // A decoded v3 entry carries identityAvailable: false; re-encoding must not + // promote it to a declared identity. + const v3 = { + version: 3, + models: ["claude-fable-5"], + sessions: ["deleted-session"], + files: { + "/deleted.jsonl": { + s: 100, + m: 500, + p: "claude", + r: [[1_786_000_000_000, 0, 0, 2, 1000, 10, 50, 0, "msg_d:", null]], + t: [], + o: 90, + gl: 64, + gh: 11, + cs: null, + }, + }, + }; + const once = decodeScanCache(JSON.parse(JSON.stringify(v3))); + const again = decodeScanCache(JSON.parse(JSON.stringify(encodeScanCache(once)))); + + expect(again.get("/deleted.jsonl")?.records[0]?.identityAvailable).toBe(false); + expect(again.get("/deleted.jsonl")?.records[0]?.measurementCompleteness).toBe("partial"); + }); + it("drops an entry whose persisted parse state is corrupt", () => { // Resuming with a bad reducer state would attach appended usage to the // wrong model or replay fork-copied history; that entry must cold parse. diff --git a/apps/server/src/usage/usageScanCache.ts b/apps/server/src/usage/usageScanCache.ts index eaeb60eed49e..ec60cf1edf61 100644 --- a/apps/server/src/usage/usageScanCache.ts +++ b/apps/server/src/usage/usageScanCache.ts @@ -19,7 +19,9 @@ import type { UsageProviderKind } from "@t3tools/contracts"; import { GUARD_LENGTH, type TranscriptParsePosition } from "./usageTranscriptReader.ts"; import type { CodexScanState, + DedupeKeyScope, UsageMeasurement, + UsageMeasurementCompleteness, UsageObservationScope, UsageRecord, } from "./usageTranscripts.ts"; @@ -91,10 +93,25 @@ type SerializedRecord = readonly [ promptId: string | null, measurementCode: number, scopeCode: number, + /** Appended after the first v4 rows; absent means `complete` when observed. */ + completenessCode: number, + /** Appended after the first v4 rows; count of present-but-invalid fields. */ + invalidTokenFields: number, + /** Appended after the first v4 rows; absent means `global`. */ + dedupeKeyScopeCode: number, ]; -const MEASUREMENT_CODES: readonly UsageMeasurement[] = ["observed", "empty", "unavailable"]; +// `invalid` is appended last so the existing observed/empty/unavailable codes +// keep their values and older v4 rows keep decoding. +const MEASUREMENT_CODES: readonly UsageMeasurement[] = [ + "observed", + "empty", + "unavailable", + "invalid", +]; const SCOPE_CODES: readonly UsageObservationScope[] = ["delta", "snapshot"]; +const COMPLETENESS_CODES: readonly UsageMeasurementCompleteness[] = ["complete", "partial"]; +const DEDUPE_KEY_SCOPE_CODES: readonly DedupeKeyScope[] = ["global", "source-local"]; function encodeMeasurement(measurement: UsageMeasurement | undefined): number { const index = MEASUREMENT_CODES.indexOf(measurement ?? "observed"); @@ -106,6 +123,23 @@ function encodeScope(scope: UsageObservationScope | undefined): number { return index < 0 ? 0 : index; } +function encodeCompleteness(completeness: UsageMeasurementCompleteness | undefined): number { + const index = COMPLETENESS_CODES.indexOf(completeness ?? "complete"); + return index < 0 ? 0 : index; +} + +function encodeDedupeKeyScope(scope: DedupeKeyScope | undefined): number { + const index = DEDUPE_KEY_SCOPE_CODES.indexOf(scope ?? "global"); + return index < 0 ? 0 : index; +} + +function decodeCode( + value: unknown, + codes: readonly Value[], +): Value | undefined { + return typeof value === "number" && Number.isSafeInteger(value) ? codes[value] : undefined; +} + interface SerializedFile { readonly s: number; readonly m: number; @@ -166,6 +200,9 @@ export function encodeScanCache(cache: ScanCache): SerializedCache { record.promptId ?? null, encodeMeasurement(record.measurement), encodeScope(record.scope), + encodeCompleteness(record.measurementCompleteness), + record.invalidTokenFields ?? 0, + encodeDedupeKeyScope(record.dedupeKeyScope), ]; const files: Record = {}; @@ -255,6 +292,11 @@ export function decodeScanCache(document: unknown): ScanCache { // explicitly `unavailable` rather than being read as a measured zero. const measurementCode = row[13]; const scopeCode = row[14]; + // Appended after the first v4 rows: validity/completeness metadata. Absent + // on an older row, which is treated as a complete observation. + const completenessCode = row[15]; + const invalidTokenFieldsRaw = row[16]; + const dedupeKeyScopeCode = row[17]; const model = typeof modelIndex === "number" ? models[modelIndex] : undefined; if ( @@ -271,19 +313,25 @@ export function decodeScanCache(document: unknown): ScanCache { } const measurement: UsageMeasurement = - typeof measurementCode === "number" && - Number.isSafeInteger(measurementCode) && - MEASUREMENT_CODES[measurementCode] !== undefined - ? MEASUREMENT_CODES[measurementCode]! - : uncached + cached + cacheCreation + output > 0 - ? "observed" - : "unavailable"; - const scope: UsageObservationScope = - typeof scopeCode === "number" && - Number.isSafeInteger(scopeCode) && - SCOPE_CODES[scopeCode] !== undefined - ? SCOPE_CODES[scopeCode]! - : "delta"; + decodeCode(measurementCode, MEASUREMENT_CODES) ?? + (uncached + cached + cacheCreation + output > 0 ? "observed" : "unavailable"); + const scope: UsageObservationScope = decodeCode(scopeCode, SCOPE_CODES) ?? "delta"; + // A legacy row erased field presence, so a nonzero observation is only + // known-partial: we cannot prove every required field was present. + const completeness: UsageMeasurementCompleteness | undefined = + measurement === "observed" + ? (decodeCode(completenessCode, COMPLETENESS_CODES) ?? (legacy ? "partial" : "complete")) + : undefined; + const invalidTokenFields = + typeof invalidTokenFieldsRaw === "number" && + Number.isFinite(invalidTokenFieldsRaw) && + invalidTokenFieldsRaw > 0 + ? Math.trunc(invalidTokenFieldsRaw) + : 0; + const dedupeKeyScope: DedupeKeyScope | undefined = decodeCode( + dedupeKeyScopeCode, + DEDUPE_KEY_SCOPE_CODES, + ); records.push({ provider, @@ -308,6 +356,12 @@ export function decodeScanCache(document: unknown): ScanCache { ...(typeof promptId === "string" ? { promptId } : {}), }), measurement, + ...(completeness === undefined ? {} : { measurementCompleteness: completeness }), + ...(invalidTokenFields === 0 ? {} : { invalidTokenFields }), + // Legacy rows erased identity; keep that visible so the projection does + // not read an absent native id as a missing one. + ...(legacy ? { identityAvailable: false } : {}), + ...(dedupeKeyScope === undefined ? {} : { dedupeKeyScope }), ...(scope === "delta" ? {} : { scope }), }); } diff --git a/apps/server/src/usage/usageTranscripts.test.ts b/apps/server/src/usage/usageTranscripts.test.ts index 1a4027529d20..84ce39631e63 100644 --- a/apps/server/src/usage/usageTranscripts.test.ts +++ b/apps/server/src/usage/usageTranscripts.test.ts @@ -125,6 +125,71 @@ describe("parseClaudeLine", () => { const record = parseClaudeLine(line); expect(record?.measurement).toBe("observed"); + expect(record?.measurementCompleteness).toBe("complete"); + }); + + /** A Claude assistant line with an arbitrary raw `usage` object. */ + function claudeUsage(usage: Record): string { + return JSON.stringify({ + type: "assistant", + timestamp: "2026-08-07T04:05:13.944Z", + sessionId: "5a128faa-8253-489e-b935-6c08e8e670c0", + message: { id: "msg_raw", model: "claude-fable-5", usage }, + }); + } + + it("treats present-but-invalid values as invalid, not a measured zero", () => { + for (const usage of [ + { input_tokens: null }, + { input_tokens: "missing" }, + { input_tokens: -99 }, + { input_tokens: Number.NaN }, + { input_tokens: Number.POSITIVE_INFINITY }, + ]) { + const record = parseClaudeLine(claudeUsage(usage)); + expect(record?.measurement).toBe("invalid"); + expect(record?.measurementCompleteness).toBeUndefined(); + expect(record?.totals).toEqual({ + uncachedInputTokens: 0, + cachedInputTokens: 0, + cacheCreationTokens: 0, + outputTokens: 0, + reasoningTokens: 0, + }); + } + }); + + it("preserves a valid known subset as partial instead of complete", () => { + // `output_tokens` is required, so input alone is a partial measurement. + const inputOnly = parseClaudeLine(claudeUsage({ input_tokens: 10 })); + expect(inputOnly?.measurement).toBe("observed"); + expect(inputOnly?.measurementCompleteness).toBe("partial"); + expect(inputOnly?.invalidTokenFields).toBeUndefined(); + expect(inputOnly?.totals.uncachedInputTokens).toBe(10); + + // An explicit valid zero in one required field is still partial when the + // other required field is absent. + const zeroInputOnly = parseClaudeLine(claudeUsage({ input_tokens: 0 })); + expect(zeroInputOnly?.measurementCompleteness).toBe("partial"); + }); + + it("keeps a complete explicit zero complete", () => { + const record = parseClaudeLine(claudeUsage({ input_tokens: 0, output_tokens: 0 })); + expect(record?.measurement).toBe("observed"); + expect(record?.measurementCompleteness).toBe("complete"); + expect(record?.invalidTokenFields).toBeUndefined(); + }); + + it("distinguishes an invalid value from an absent field on a partial record", () => { + const record = parseClaudeLine(claudeUsage({ input_tokens: 10, output_tokens: null })); + expect(record?.measurement).toBe("observed"); + expect(record?.measurementCompleteness).toBe("partial"); + expect(record?.invalidTokenFields).toBe(1); + }); + + it("scopes a Claude message/request key as global", () => { + const record = parseClaudeLine(claudeLine({ messageId: "msg_scope", contentType: "text" })); + expect(record?.dedupeKeyScope).toBe("global"); }); }); @@ -174,6 +239,9 @@ describe("parseCodexLine", () => { expect(record?.providerRequestId).toBeNull(); expect(record?.providerMessageId).toBeNull(); expect(record?.promptId).toBeNull(); + // Its occurrence key is only meaningful within the session. + expect(record?.dedupeKeyScope).toBe("source-local"); + expect(record?.measurementCompleteness).toBe("complete"); }); it("skips a repeated token_count so deltas are not double counted", () => { diff --git a/apps/server/src/usage/usageTranscripts.ts b/apps/server/src/usage/usageTranscripts.ts index 74468946e3e3..942345efe82b 100644 --- a/apps/server/src/usage/usageTranscripts.ts +++ b/apps/server/src/usage/usageTranscripts.ts @@ -12,16 +12,51 @@ import type { UsageProviderKind, UsageTokenTotals } from "@t3tools/contracts"; * Whether the source actually measured tokens, as opposed to handing us a * container we normalised to zeros. * - * - `observed` — at least one recognised token field was present. An explicit - * `0` is a real measured zero and stays `observed`. + * - `observed` — at least one recognised token field held a valid value. An + * explicit `0` is a real measured zero and stays `observed`. Whether the + * measurement is complete is a separate axis; see + * {@link UsageMeasurementCompleteness}. * - `empty` — a usage container existed but carried no recognised token field * (for example Claude's `usage: {}`). Numeric totals are zero, but that is a * missing measurement, not a measured zero. + * - `invalid` — one or more recognised token fields were present but none held + * a valid value (`null`, a string, a negative number). This is a malformed + * observation, not an absent one. * - `unavailable` — the presence information was erased before we saw the * record (a legacy cache row). The zeros may be real or may be missing; we * must not classify them either way. */ -export type UsageMeasurement = "observed" | "empty" | "unavailable"; +export type UsageMeasurement = "observed" | "empty" | "invalid" | "unavailable"; + +/** + * Whether an `observed` measurement covered every field the provider requires. + * + * - `complete` — every required field was present and held a valid value, and + * no recognised field was invalid. An explicit all-zero usage object is a + * complete measurement. + * - `partial` — at least one required field was absent or invalid, or a + * recognised field held an invalid value. The valid subset is still a real + * measurement and its totals are a lower bound, never a complete one. + * + * Absent on a record means `complete` for backward compatibility with callers + * that predate this axis; the parsers always set it for `observed`. + */ +export type UsageMeasurementCompleteness = "complete" | "partial"; + +/** + * How far a declared `dedupeKey` can be trusted on its own. + * + * - `global` — the key is a globally qualified native observation id (Claude's + * `message.id:requestId`, Grok's `sessionId:promptId:model`). Equal keys name + * the same observation, so a copy at another path is the same event. + * - `source-local` — the key is only meaningful within its native session or + * occurrence (the scan's Codex occurrence key). It must be qualified by the + * native session before it can identify an event, so equal keys in two + * sessions are two observations, not one. + * + * Absent defaults to `global`. + */ +export type DedupeKeyScope = "global" | "source-local"; /** * How a record relates to other records for the same identity. @@ -70,6 +105,30 @@ export interface UsageRecord { * explicitly. See {@link UsageMeasurement}. */ readonly measurement?: UsageMeasurement; + /** + * Whether an `observed` measurement covered every required field. Only + * meaningful for `observed`; absent means `complete`. See + * {@link UsageMeasurementCompleteness}. + */ + readonly measurementCompleteness?: UsageMeasurementCompleteness; + /** + * Count of recognised token fields that were present but held an invalid + * value. Distinguishes a `partial` measurement with an invalid value from one + * with an absent field; absent/`0` means no invalid value was seen. + */ + readonly invalidTokenFields?: number; + /** + * `false` when the source erased native identity before we saw it (a legacy + * cache row). Kept apart from the numeric totals so identity availability is + * never recovered from token magnitude. Absent means the identity is as the + * source wrote it. + */ + readonly identityAvailable?: boolean; + /** + * How far `dedupeKey` can be trusted on its own. Absent means `global`. See + * {@link DedupeKeyScope}. + */ + readonly dedupeKeyScope?: DedupeKeyScope; /** Additive increment or replaceable snapshot. Absent means `delta`. */ readonly scope?: UsageObservationScope; } @@ -109,6 +168,54 @@ function int(value: unknown): number { return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.trunc(value) : 0; } +/** A token field is valid only as a finite, non-negative number. */ +function isValidTokenValue(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0; +} + +interface TokenFieldClassification { + readonly measurement: UsageMeasurement; + readonly completeness?: UsageMeasurementCompleteness; + readonly invalidTokenFields: number; +} + +/** + * Classifies a provider usage object by field validity and completeness. + * + * Property presence alone is not enough: a field holding `null`, a string, or + * a negative number is present but invalid, and a usage object missing a + * required field is a valid known subset rather than a complete measurement. + * The valid subset is preserved; only the classification says it is partial. + */ +function classifyTokenFields( + fields: Record, + required: readonly string[], + optional: readonly string[], +): TokenFieldClassification { + let recognized = 0; + let validRequired = 0; + let validAny = 0; + let invalidTokenFields = 0; + for (const field of [...required, ...optional]) { + if (!Object.hasOwn(fields, field)) continue; + recognized += 1; + if (isValidTokenValue(fields[field])) { + validAny += 1; + if (required.includes(field)) validRequired += 1; + } else { + invalidTokenFields += 1; + } + } + if (recognized === 0) return { measurement: "empty", invalidTokenFields: 0 }; + if (validAny === 0) return { measurement: "invalid", invalidTokenFields }; + const complete = validRequired === required.length && invalidTokenFields === 0; + return { + measurement: "observed", + completeness: complete ? "complete" : "partial", + invalidTokenFields, + }; +} + function parseTimestampMs(value: unknown): number | null { if (typeof value !== "string") return null; const parsed = Date.parse(value); @@ -163,12 +270,17 @@ function grokCostTicksToUsd(ticks: unknown): number | null { /* Claude Code */ /* -------------------------------------------------------------------------- */ -/** Token fields that make a Claude `usage` object an actual measurement. */ -const CLAUDE_USAGE_FIELDS = [ - "input_tokens", +/** + * Token fields that make a Claude `usage` object an actual measurement. + * + * `input_tokens` and `output_tokens` are the measurement; the cache fields are + * genuinely optional and Anthropic omits them when zero, so their absence does + * not make an otherwise complete record partial. + */ +const CLAUDE_REQUIRED_USAGE_FIELDS = ["input_tokens", "output_tokens"] as const; +const CLAUDE_OPTIONAL_USAGE_FIELDS = [ "cache_read_input_tokens", "cache_creation_input_tokens", - "output_tokens", ] as const; /** @@ -214,14 +326,15 @@ export function parseClaudeLine(line: string): UsageRecord | null { const cost = record["costUSD"]; - // `usage: {}` normalises to zeros but is not a measured zero. Only a - // recognised token field makes this an observed measurement; an explicit - // `input_tokens: 0` still counts as observed. - const measurement: UsageMeasurement = CLAUDE_USAGE_FIELDS.some((field) => - Object.hasOwn(usageRecord, field), - ) - ? "observed" - : "empty"; + // `usage: {}` normalises to zeros but is not a measured zero; a field holding + // `null`, a string, or a negative number is invalid; a missing required field + // leaves a valid known subset that is only `partial`. Only actual values + // decide this, never property presence or a nonzero total. + const classification = classifyTokenFields( + usageRecord, + CLAUDE_REQUIRED_USAGE_FIELDS, + CLAUDE_OPTIONAL_USAGE_FIELDS, + ); return { provider: "claude", @@ -244,7 +357,16 @@ export function parseClaudeLine(line: string): UsageRecord | null { providerRequestId: requestId, providerMessageId: messageId, promptId: null, - measurement, + measurement: classification.measurement, + ...(classification.completeness === undefined + ? {} + : { measurementCompleteness: classification.completeness }), + ...(classification.invalidTokenFields === 0 + ? {} + : { invalidTokenFields: classification.invalidTokenFields }), + // A Claude message/request pair is a globally qualified native observation + // id: the same response copied into another transcript is the same event. + dedupeKeyScope: "global", }; } @@ -395,6 +517,12 @@ export function parseCodexLine(line: string, state: CodexScanState): UsageRecord if (totalTokens(totals) === 0) return null; + const classification = classifyTokenFields( + lastRecord, + ["input_tokens", "output_tokens"], + ["cached_input_tokens", "cache_write_input_tokens", "reasoning_output_tokens"], + ); + return { provider: "codex", timestampMs, @@ -413,6 +541,15 @@ export function parseCodexLine(line: string, state: CodexScanState): UsageRecord promptId: null, // Only emitted when at least one token was measured, so this is observed. measurement: "observed", + ...(classification.completeness === undefined + ? {} + : { measurementCompleteness: classification.completeness }), + ...(classification.invalidTokenFields === 0 + ? {} + : { invalidTokenFields: classification.invalidTokenFields }), + // The scan's occurrence key is only meaningful within this session, so a + // caller stamping it must qualify it with the native session. + dedupeKeyScope: "source-local", }; } @@ -427,6 +564,7 @@ interface GrokUsageTotals { readonly cacheCreationTokens: number; readonly reasoningTokens: number; readonly costUsdTicks: number | null; + readonly classification: TokenFieldClassification; } function readGrokUsageTotals(value: unknown): GrokUsageTotals | null { @@ -442,6 +580,11 @@ function readGrokUsageTotals(value: unknown): GrokUsageTotals | null { typeof record["costUsdTicks"] === "number" && Number.isFinite(record["costUsdTicks"]) ? record["costUsdTicks"] : null, + classification: classifyTokenFields( + record, + ["inputTokens", "outputTokens"], + ["cachedReadTokens", "cacheCreationTokens", "reasoningTokens"], + ), }; } @@ -545,7 +688,16 @@ export function parseGrokLine(line: string): readonly UsageRecord[] { providerRequestId: null, providerMessageId: null, promptId, - measurement: "observed", + measurement: topLevel.classification.measurement, + ...(topLevel.classification.completeness === undefined + ? {} + : { measurementCompleteness: topLevel.classification.completeness }), + ...(topLevel.classification.invalidTokenFields === 0 + ? {} + : { invalidTokenFields: topLevel.classification.invalidTokenFields }), + // `sessionId:promptId:model` is a session-qualified native observation + // id: the same turn copied into another transcript is the same event. + dedupeKeyScope: "global", }, ]; } @@ -595,7 +747,14 @@ export function parseGrokLine(line: string): readonly UsageRecord[] { providerRequestId: null, providerMessageId: null, promptId, - measurement: "observed", + measurement: entry.totals.classification.measurement, + ...(entry.totals.classification.completeness === undefined + ? {} + : { measurementCompleteness: entry.totals.classification.completeness }), + ...(entry.totals.classification.invalidTokenFields === 0 + ? {} + : { invalidTokenFields: entry.totals.classification.invalidTokenFields }), + dedupeKeyScope: "global", }); } return results; diff --git a/docs/internals/usage-attribution.md b/docs/internals/usage-attribution.md index 43149adc3c0d..e3475349ec6c 100644 --- a/docs/internals/usage-attribution.md +++ b/docs/internals/usage-attribution.md @@ -44,12 +44,17 @@ because collapsing them is how a zero-cost success gets invented: - **identity validity** (`identityQuality`: `valid | missing | invalid`) — is the native session id present and well-formed? A malformed Claude id is `invalid`. - **measurement completeness** (`measurementQuality`: `measured | partial | missing | -invalid | unavailable`) — were tokens actually measured? An explicit zero is - `measured`; Claude's `usage: {}` is `invalid`; an all-zero legacy row whose presence - was erased is `unavailable`, never `missing` and never a measured zero. +invalid | unavailable`) — were tokens actually measured, and were the provider's + required fields present and valid? An explicit zero is `measured`; a valid known + subset (Claude `input_tokens` with no `output_tokens`) is `partial`; a field that is + present but holds `null`, a string, or a negative number makes the record `invalid`; + Claude's `usage: {}` is `invalid`; an all-zero legacy row whose presence was erased + is `unavailable`, never `missing` and never a measured zero. A nonzero total never + implies a complete measurement. - **level support** (`promptQuality` / `requestQuality`) — can the source establish this level, and did the records carry its id? `unsupported` is a structural limit, - not a zero. + not a zero. A legacy row whose native id was erased reports `unavailable`, never + `missing`, and identity availability is kept apart from token magnitude. - **allocation certainty** (`allocation`: `attributed | shared | unallocated | ambiguous | missing | orphan`). @@ -65,8 +70,16 @@ ids (`providerRequestId`, `providerMessageId`, `promptId`), which are reporting The projection resolves identity in three ways: - **declared** — a `dedupeKey` present on the record, namespaced by provider so equal - local ids from two providers cannot collide. Two deliveries of the same key with the - same content are one observation (a repeated scan or a copied/moved rollout). + local ids from two providers cannot collide. Its **scope** is explicit: + `dedupeKeyScope: "global"` is a globally qualified native observation id (Claude's + `message.id:requestId`, Grok's `sessionId:promptId:model`), so the same key at + another path is the same event; `"source-local"` is qualified by the canonical + native session (the scan's Codex occurrence key), so equal local keys in two + sessions are two observations, never one. A physical path is never used as scope. + A global key that appears under a second native session is incompatible ownership, + not a copy: it is surfaced as a conflict rather than silently dropped. Cost and its + provenance are part of the observation, so a repriced record is a conflict, not a + silent duplicate. - **occurrence** — for a keyless source such as Codex `token_count`, the scan stamps an occurrence-aware key from `usageEventOccurrenceBaseKey` plus a per-delivery occurrence index. A copied rollout restarts its counter, so the copy lands on the same key and is @@ -124,9 +137,12 @@ discarded. The scan retains measured records from transcripts that have since be deleted for 90 days, and those cannot be re-parsed, so discarding a v3 cache would destroy that history. A v3 row decodes with its native ids and measurement presence explicitly `unavailable` (an all-zero row stays unknown, not a measured zero; a nonzero -row is still a known measurement), and the entry is flagged so it is never resumed -incrementally. An extant file is cold re-parsed on the next scan, which enriches it with -ids and presence without double counting because the re-parse replaces the entry. +row is a known but only-partial measurement), and the entry is flagged so it is never +resumed incrementally. Warm-cache acceptance requires the current identity/measurement +format: an unchanged extant legacy file is cold re-parsed once to enrich it with ids and +presence without double counting because the re-parse replaces the entry. A read failure +during that re-parse keeps the retained fallback rows, and a deleted file is never +re-parsed at all, so its history survives. ## What still needs architecture approval From af70495442678a06e007d13bba477f123170c0f5 Mon Sep 17 00:00:00 2001 From: nullstack65 Date: Wed, 23 Sep 2026 21:14:30 -0400 Subject: [PATCH 7/7] fix(usage): preserve zero-total quality and refresh stale cache rows Retain eligible zero-total observations instead of dropping them before classification. Codex no longer returns null for a zero subtotal, and Grok no longer skips zero-total aggregate or per-model rows: a complete measured zero, a known-zero subset, and an all-invalid payload all reach the projection with their quality, while a container with no recognised token field is still no-usage. Grok per-model cost accounting now reduces the aggregate for every emitted ticked row, so an emitted zero-token row cannot double count. Treat missing completeness metadata as unasserted. The predecessor v4 writer emitted 15-field rows without it, and the decoder defaulted a non-legacy observed row to complete, silently promoting an unknown measurement. Missing completeness now decodes as partial, and the entry is marked qualityMetadata: "predecessor" so warm-cache acceptance and resume both require the current quality format. An extant file is cold re-parsed once; deleted and unreadable history is retained conservatively. Native-id availability is a separate axis from numeric freshness. Regressions cover Codex/Grok complete zero, partial zero, invalid, and valid-plus-invalid distinct events, the parser-to-projection path, a pinned predecessor fixture generated by the actual predecessor writer, and the service-seam refresh/restart/deleted/failed-read paths. --- apps/server/src/usage/UsageService.test.ts | 244 ++++++++++++++++++ apps/server/src/usage/UsageService.ts | 13 +- .../server/src/usage/usageAttribution.test.ts | 94 ++++++- apps/server/src/usage/usageScanCache.test.ts | 164 ++++++++++++ apps/server/src/usage/usageScanCache.ts | 75 ++++-- .../server/src/usage/usageTranscripts.test.ts | 218 ++++++++++++++-- apps/server/src/usage/usageTranscripts.ts | 50 ++-- docs/internals/usage-attribution.md | 43 ++- 8 files changed, 829 insertions(+), 72 deletions(-) diff --git a/apps/server/src/usage/UsageService.test.ts b/apps/server/src/usage/UsageService.test.ts index bd23deccb353..afd55b8ba78d 100644 --- a/apps/server/src/usage/UsageService.test.ts +++ b/apps/server/src/usage/UsageService.test.ts @@ -630,6 +630,250 @@ describe("UsageService", () => { }).pipe(Effect.scoped), ); + it.live("refreshes a predecessor v4 quality cache entry once and keeps deleted history", () => + Effect.gen(function* () { + const { transcript, settings, home } = yield* setup; + const content = + encodeUnknownJsonString({ + type: "assistant", + timestamp: "2026-08-01T10:00:00Z", + requestId: "req_live", + sessionId: "session-1", + message: { id: "msg_live", model: "claude-fable-5", usage: { input_tokens: 10 } }, + }) + "\n"; + yield* Effect.promise(() => NodeFSP.writeFile(transcript, content)); + + yield* Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const liveDir = yield* Effect.promise(() => NodeFSP.realpath(NodePath.dirname(transcript))); + const livePath = NodePath.join(liveDir, NodePath.basename(transcript)); + const liveStat = yield* Effect.promise(() => NodeFSP.stat(livePath)); + const deletedPath = NodePath.join(liveDir, "deleted.jsonl"); + const scanCachePath = NodePath.join(config.stateDir, "usage-scan-cache.json"); + const TS = Date.parse("2026-08-01T10:00:00Z"); + // A predecessor v4 document: `identity` is declared (no legacy marker) + // but the rows are 15 fields with no completeness metadata, exactly as + // the predecessor writer emitted. Independently, a nonzero total used to + // decode as `complete`, silently promoting an unknown measurement. + yield* Effect.promise(() => + NodeFSP.writeFile( + scanCachePath, + encodeUnknownJsonString({ + version: 4, + models: ["claude-fable-5"], + sessions: ["session-1", "deleted-session"], + files: { + [livePath]: { + s: liveStat.size, + m: liveStat.mtimeMs, + p: "claude", + r: [[TS, 0, 0, 10, 0, 0, 0, 0, "msg_live:", null, null, null, null, 0, 0]], + t: [], + o: liveStat.size, + gl: 0, + gh: 0, + cs: null, + }, + [deletedPath]: { + s: 100, + m: liveStat.mtimeMs, + p: "claude", + r: [[TS, 0, 1, 7, 0, 0, 0, 0, "msg_deleted:", null, null, null, null, 0, 0]], + t: [], + o: 90, + gl: 64, + gh: 11, + cs: null, + }, + }, + }), + ), + ); + + const service = yield* UsageService.make; + const first = yield* service.readSummary(WINDOW); + // Live file cold re-parsed once (10) plus retained deleted history (7). + assert.strictEqual( + first.buckets.reduce((sum, bucket) => sum + bucket.totals.uncachedInputTokens, 0), + 17, + ); + + const afterFirst = decodeUnknownJsonString( + yield* Effect.promise(() => NodeFSP.readFile(scanCachePath, "utf8")), + ) as { version: number; files: Record }; + assert.strictEqual(afterFirst.version, 4); + const liveRow = afterFirst.files[livePath]!.r[0]!; + // Enriched to the current 18-field row with the completeness code (1 = + // partial) the predecessor row could not carry. + assert.strictEqual(liveRow.length, 18); + assert.strictEqual(liveRow[15], 1); + assert.strictEqual(afterFirst.files[livePath]!.li, undefined); + // Deleted history retained conservatively; it cannot be re-parsed. + const deletedRow = afterFirst.files[deletedPath]!.r[0]!; + assert.strictEqual(deletedRow[3], 7); + assert.strictEqual(deletedRow[15], 1); + + // A second scan of the unchanged live file is a warm hit: no re-parse, + // no cache rewrite, stable totals. + const beforeSecond = yield* Effect.promise(() => NodeFSP.readFile(scanCachePath, "utf8")); + const second = yield* service.readSummary(WINDOW); + assert.strictEqual( + second.buckets.reduce((sum, bucket) => sum + bucket.totals.uncachedInputTokens, 0), + 17, + ); + const afterSecond = yield* Effect.promise(() => NodeFSP.readFile(scanCachePath, "utf8")); + assert.strictEqual(afterSecond, beforeSecond); + + // A restart reads the enriched cache and keeps deleted history. + const restarted = yield* UsageService.make; + const third = yield* restarted.readSummary(WINDOW); + assert.strictEqual( + third.buckets.reduce((sum, bucket) => sum + bucket.totals.uncachedInputTokens, 0), + 17, + ); + }).pipe( + Effect.provide( + serviceLayers({ prefix: "usage-service-predecessor-refresh-test", home, settings }), + ), + ); + }).pipe(Effect.scoped), + ); + + it.live("keeps retained fallback rows when an extant transcript cannot be read", () => + Effect.gen(function* () { + const { transcript, settings, home } = yield* setup; + const content = + encodeUnknownJsonString({ + type: "assistant", + timestamp: "2026-08-01T10:00:00Z", + sessionId: "session-1", + message: { id: "msg_unreadable", model: "claude-fable-5", usage: { input_tokens: 9 } }, + }) + "\n"; + yield* Effect.promise(() => NodeFSP.writeFile(transcript, content)); + + yield* Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const liveDir = yield* Effect.promise(() => NodeFSP.realpath(NodePath.dirname(transcript))); + const livePath = NodePath.join(liveDir, NodePath.basename(transcript)); + const liveStat = yield* Effect.promise(() => NodeFSP.stat(livePath)); + const scanCachePath = NodePath.join(config.stateDir, "usage-scan-cache.json"); + const TS = Date.parse("2026-08-01T10:00:00Z"); + // A predecessor row matching the live file exactly. The warm guard must + // reject it and attempt a read; making the file unreadable then forces + // the fallback path rather than an empty transcript. + yield* Effect.promise(() => + NodeFSP.writeFile( + scanCachePath, + encodeUnknownJsonString({ + version: 4, + models: ["claude-fable-5"], + sessions: ["session-1"], + files: { + [livePath]: { + s: liveStat.size, + m: liveStat.mtimeMs, + p: "claude", + r: [[TS, 0, 0, 9, 0, 0, 0, 0, "msg_unreadable:", null, null, null, null, 0, 0]], + t: [], + o: liveStat.size, + gl: 0, + gh: 0, + cs: null, + }, + }, + }), + ), + ); + yield* Effect.promise(() => NodeFSP.chmod(livePath, 0o000)); + + const service = yield* UsageService.make; + const summary = yield* service.readSummary(WINDOW); + // The cached 9 tokens survive the failed read; nothing is zeroed out. + assert.strictEqual( + summary.buckets.reduce((sum, bucket) => sum + bucket.totals.uncachedInputTokens, 0), + 9, + ); + }).pipe( + Effect.provide(serviceLayers({ prefix: "usage-service-failed-read-test", home, settings })), + ); + }).pipe(Effect.scoped), + ); + + it.live("counts Codex and Grok invalid events as malformed while keeping valid tokens", () => + Effect.gen(function* () { + const { settings, home } = yield* setup; + const codexDir = NodePath.join(home, "codex", "sessions"); + const grokDir = NodePath.join(home, "grok", "sessions", "session"); + yield* Effect.promise(async () => { + await NodeFSP.mkdir(codexDir, { recursive: true }); + await NodeFSP.writeFile( + NodePath.join(codexDir, "rollout.jsonl"), + [ + { type: "session_meta", payload: { id: "codex-invalid-session" } }, + { type: "turn_context", payload: { model: "gpt-5.6-sol" } }, + { + type: "event_msg", + timestamp: "2026-08-01T10:00:00Z", + payload: { + type: "token_count", + info: { last_token_usage: { input_tokens: 10, output_tokens: 2 } }, + }, + }, + { + type: "event_msg", + timestamp: "2026-08-01T10:00:01Z", + payload: { + type: "token_count", + info: { last_token_usage: { input_tokens: null, output_tokens: null } }, + }, + }, + ] + .map((line) => encodeUnknownJsonString(line)) + .join("\n") + "\n", + ); + await NodeFSP.mkdir(grokDir, { recursive: true }); + await NodeFSP.writeFile( + NodePath.join(grokDir, "updates.jsonl"), + encodeUnknownJsonString({ + timestamp: Date.parse("2026-08-01T10:00:00Z") / 1000, + method: "_x.ai/session/update", + params: { + sessionId: "grok-invalid-session", + update: { + sessionUpdate: "turn_completed", + prompt_id: "prompt-1", + usage: { + inputTokens: 10, + outputTokens: 13, + modelUsage: { + "model-invalid": { inputTokens: null, outputTokens: null }, + "model-valid": { inputTokens: 10, outputTokens: 13 }, + }, + }, + }, + }, + }) + "\n", + ); + }); + + const service = yield* UsageService.make.pipe( + Effect.provide( + serviceLayers({ prefix: "usage-service-nonclaude-malformed-test", home, settings }), + ), + ); + const summary = yield* service.readSummary(WINDOW); + + const codexSource = summary.sources.find((source) => source.fingerprint.provider === "codex"); + const grokSource = summary.sources.find((source) => source.fingerprint.provider === "grok"); + assert.strictEqual(codexSource?.malformedRecords, 1); + assert.strictEqual(grokSource?.malformedRecords, 1); + // 2 Codex output + 13 Grok output; the invalid events add no tokens. + assert.strictEqual(totalOutputTokens(summary), 15); + assert.strictEqual(codexSource?.distinctSessions, 1); + assert.strictEqual(grokSource?.distinctSessions, 1); + }).pipe(Effect.scoped), + ); + it.live("does not share an in-flight scan after custom prices change", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index 8bd7beac69c5..274daf74e354 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -394,11 +394,13 @@ export const make = Effect.gen(function* () { // Provider is part of the identity: if both providers were ever pointed // at one directory, a hit parsed by the other parser must not be reused. // The cache format is part of it too: a legacy entry erased native ids and - // measurement presence, so an unchanged file must still cold re-parse once - // to enrich it instead of serving the erased row forever. + // measurement presence, and a predecessor v4 entry never asserted numeric + // completeness. An unchanged file must still cold re-parse once to enrich + // either, instead of serving the erased or unasserted row forever. if ( cached && cached.identity === "declared" && + cached.qualityMetadata === "declared" && cached.size === size && cached.mtimeMs === mtimeMs && cached.provider === provider @@ -410,12 +412,14 @@ export const make = Effect.gen(function* () { // Only a strictly grown file may resume. Same size with a new mtime, or // a shrunken file, means rewritten content; re-parse it whole. A legacy - // entry (ids/presence erased) is also re-parsed whole: resuming would - // keep serving id-less records and the enrichment would never happen. + // or predecessor entry is also re-parsed whole: resuming would keep + // serving id-less or completeness-less records and the enrichment would + // never happen. const resumeFrom = cached !== undefined && cached.provider === provider && cached.identity === "declared" && + cached.qualityMetadata === "declared" && size > cached.size ? cached.position : undefined; @@ -445,6 +449,7 @@ export const make = Effect.gen(function* () { tailRecords, position: parsed.position, identity: "declared", + qualityMetadata: "declared", }); cacheDirty = true; return tailRecords.length === 0 ? records : [...records, ...tailRecords]; diff --git a/apps/server/src/usage/usageAttribution.test.ts b/apps/server/src/usage/usageAttribution.test.ts index 3d101c1349a2..26344de87a83 100644 --- a/apps/server/src/usage/usageAttribution.test.ts +++ b/apps/server/src/usage/usageAttribution.test.ts @@ -13,7 +13,14 @@ import { type UsageAttributionInput, } from "./usageAttribution.ts"; import { totalTokens } from "./usageTranscripts.ts"; -import { parseClaudeLine, type UsageRecord } from "./usageTranscripts.ts"; +import { + initialCodexScanState, + parseClaudeLine, + parseCodexLine, + parseGrokLine, + type CodexScanState, + type UsageRecord, +} from "./usageTranscripts.ts"; import { decodeScanCache, encodeScanCache, type ScanCache } from "./usageScanCache.ts"; const CLAUDE_SESSION = "5a128faa-8253-489e-b935-6c08e8e670c0"; @@ -983,6 +990,7 @@ describe("parser to projection", () => { tailRecords: [], position: { resumeOffset: 0, guardLength: 0, guardHash: 0, codexState: null }, identity: "declared", + qualityMetadata: "declared", }, ], ]); @@ -996,6 +1004,90 @@ describe("parser to projection", () => { ); expect(projection.sessions[0]?.measurementQuality).toBe("partial"); }); + + /** A Codex rollout primed with its session meta and model. */ + function primedCodexState(): CodexScanState { + const state = initialCodexScanState(); + parseCodexLine(JSON.stringify({ type: "session_meta", payload: { id: CODEX_SESSION } }), state); + parseCodexLine( + JSON.stringify({ type: "turn_context", payload: { model: "gpt-5.6-sol" } }), + state, + ); + return state; + } + + function codexTokenCount(lastTokenUsage: Record, timestamp: string): string { + return JSON.stringify({ + type: "event_msg", + timestamp, + payload: { type: "token_count", info: { last_token_usage: lastTokenUsage } }, + }); + } + + it("carries a Codex valid and a distinct invalid event to one partial session", () => { + const state = primedCodexState(); + const valid = parseCodexLine( + codexTokenCount({ input_tokens: 10, output_tokens: 2 }, "2026-08-01T05:17:49.919Z"), + state, + )!; + const invalid = parseCodexLine( + codexTokenCount({ input_tokens: null, output_tokens: null }, "2026-08-01T05:18:00.000Z"), + state, + )!; + const projection = buildUsageAttribution( + input({ records: [fromParsed(valid), fromParsed(invalid)] }), + ); + const session = projection.sessions[0]!; + + // The invalid event reaches the session as evidence instead of vanishing at + // the parser's zero-total gate; tokens come only from the valid event. + expect(session.totals?.records).toBe(2); + expect(session.totals?.tokens.uncachedInputTokens).toBe(10); + expect(session.measurementQuality).toBe("partial"); + }); + + it("carries a Codex complete explicit zero to a measured session", () => { + const zero = parseCodexLine( + codexTokenCount({ input_tokens: 0, output_tokens: 0 }, "2026-08-01T05:17:49.919Z"), + primedCodexState(), + )!; + const projection = buildUsageAttribution(input({ records: [fromParsed(zero)] })); + + expect(projection.sessions[0]?.measurementQuality).toBe("measured"); + expect(projection.sessions[0]?.totals?.tokens.outputTokens).toBe(0); + }); + + it("carries Grok per-model invalid and zero rows into the session", () => { + const line = JSON.stringify({ + timestamp: 1_786_372_566, + method: "_x.ai/session/update", + params: { + sessionId: GROK_SESSION, + update: { + sessionUpdate: "turn_completed", + prompt_id: "prompt-1", + usage: { + inputTokens: null, + outputTokens: null, + modelUsage: { + "model-invalid": { inputTokens: null, outputTokens: null }, + "model-zero": { inputTokens: 0, outputTokens: 0 }, + "model-valid": { inputTokens: 5, outputTokens: 5 }, + }, + }, + }, + _meta: { agentTimestampMs: 1_786_372_566_485 }, + }, + }); + const records = parseGrokLine(line).map(fromParsed); + const projection = buildUsageAttribution(input({ records })); + const session = projection.sessions[0]!; + + expect(session.totals?.records).toBe(3); + expect(session.totals?.tokens.uncachedInputTokens).toBe(5); + expect(session.measurementQuality).toBe("partial"); + expect(session.models).toEqual(["model-invalid", "model-valid", "model-zero"]); + }); }); describe("projection contract", () => { diff --git a/apps/server/src/usage/usageScanCache.test.ts b/apps/server/src/usage/usageScanCache.test.ts index e5c5f278b37e..83a1877b40b4 100644 --- a/apps/server/src/usage/usageScanCache.test.ts +++ b/apps/server/src/usage/usageScanCache.test.ts @@ -53,6 +53,7 @@ function cacheWith(entries: readonly [string, number, readonly UsageRecord[]][]) tailRecords: [], position: position(), identity: "declared", + qualityMetadata: "declared", }); } return cache; @@ -74,6 +75,7 @@ describe("scan cache round trip", () => { tailRecords: [record({ provider: "grok", model: "grok-4.5-build", dedupeKey: null })], position: position({ resumeOffset: 30, guardLength: 30, guardHash: 123 }), identity: "declared", + qualityMetadata: "declared", }); original.set("/codex.jsonl", { size: 80, @@ -92,6 +94,7 @@ describe("scan cache round trip", () => { }, }), identity: "declared", + qualityMetadata: "declared", }); const restored = decodeScanCache(JSON.parse(JSON.stringify(encodeScanCache(original)))); @@ -348,6 +351,167 @@ describe("legacy v3 cache history", () => { }); }); +describe("predecessor v4 format policy", () => { + /** + * Generated verbatim by the ACTUAL predecessor writer (its own + * `parseClaudeLine` + `encodeScanCache`) at pin + * `e4f36af5ef279246bcb0f8463adeee8a09b7bde1`, which emitted 15-field v4 rows + * with no completeness/validity/key-scope metadata. Rows: `{input_tokens:10}` + * (partial under the current parser), `{input_tokens:null}` (invalid), and + * `{input_tokens:4, output_tokens:6}` (complete). Generation receipt: a + * one-off test in a worktree at that pin parsed those three Claude lines and + * encoded one entry, then printed the document below unchanged. + */ + const PINNED_PREDECESSOR_DOCUMENT = { + version: 4, + models: ["claude-fable-5"], + sessions: ["session-pred"], + files: { + "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/pred/live.jsonl": { + s: 120, + m: 500, + p: "claude", + r: [ + [ + 1785578400000, + 0, + 0, + 10, + 0, + 0, + 0, + 0, + "msg_partial:req_msg_partial", + null, + "req_msg_partial", + "msg_partial", + null, + 0, + 0, + ], + [ + 1785578400000, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + "msg_invalid:req_msg_invalid", + null, + "req_msg_invalid", + "msg_invalid", + null, + 0, + 0, + ], + [ + 1785578400000, + 0, + 0, + 4, + 0, + 0, + 6, + 0, + "msg_valid:req_msg_valid", + null, + "req_msg_valid", + "msg_valid", + null, + 0, + 0, + ], + ], + t: [], + o: 120, + gl: 64, + gh: 12345, + cs: null, + }, + }, + }; + + it("decodes a predecessor row as partial, never as a silent complete", () => { + const decoded = decodeScanCache(JSON.parse(JSON.stringify(PINNED_PREDECESSOR_DOCUMENT))); + const entry = decoded.get("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/pred/live.jsonl")!; + + // Identity is asserted, so only the numeric-quality axis marks the entry as + // stale. Reading the missing completeness as `complete` would promote an + // unknown measurement; `partial` is the conservative floor. + expect(entry.identity).toBe("declared"); + expect(entry.qualityMetadata).toBe("predecessor"); + expect(entry.records.map((row) => row.measurement)).toEqual([ + "observed", + "observed", + "observed", + ]); + expect(entry.records.map((row) => row.measurementCompleteness)).toEqual([ + "partial", + "partial", + "partial", + ]); + // Totals are retained, not discarded for the format change. + expect(entry.records[0]?.totals.uncachedInputTokens).toBe(10); + }); + + it("marks a current-format entry qualityMetadata declared", () => { + const encoded = encodeScanCache(cacheWith([["/a.jsonl", 100, [record()]]])); + const decoded = decodeScanCache(JSON.parse(JSON.stringify(encoded))); + + expect(decoded.get("/a.jsonl")?.qualityMetadata).toBe("declared"); + }); + + it("treats a row with no appended fields as predecessor", () => { + // A hand-built or truncated row that stops after the scope code is exactly + // the predecessor shape and must be treated the same way. + const encoded = encodeScanCache(cacheWith([["/a.jsonl", 100, [record()]]])); + const truncated = { + ...encoded, + files: { + "/a.jsonl": { + ...encoded.files["/a.jsonl"]!, + r: [encoded.files["/a.jsonl"]!.r[0]!.slice(0, 15)], + }, + }, + }; + + const entry = decodeScanCache(JSON.parse(JSON.stringify(truncated))).get("/a.jsonl")!; + + expect(entry.qualityMetadata).toBe("predecessor"); + expect(entry.records[0]?.measurementCompleteness).toBe("partial"); + }); + + it("marks a legacy v3 entry predecessor as well as identity-unavailable", () => { + const decoded = decodeScanCache( + JSON.parse( + JSON.stringify({ + version: 3, + models: ["claude-fable-5"], + sessions: ["deleted-session"], + files: { + "/deleted.jsonl": { + s: 100, + m: 500, + p: "claude", + r: [[1_786_000_000_000, 0, 0, 2, 1000, 10, 50, 0, "msg_d:", null]], + t: [], + o: 90, + gl: 64, + gh: 11, + cs: null, + }, + }, + }), + ), + ); + + expect(decoded.get("/deleted.jsonl")?.identity).toBe("unavailable"); + expect(decoded.get("/deleted.jsonl")?.qualityMetadata).toBe("predecessor"); + }); +}); + describe("pruneScanCache", () => { const retentionCutoffMs = 1000; diff --git a/apps/server/src/usage/usageScanCache.ts b/apps/server/src/usage/usageScanCache.ts index ec60cf1edf61..a76536a0111a 100644 --- a/apps/server/src/usage/usageScanCache.ts +++ b/apps/server/src/usage/usageScanCache.ts @@ -32,17 +32,26 @@ import type { // re-parses only its appended bytes instead of starting over. // v4: records carry native request/message/prompt ids. Without the bump, warm // v3 entries would silently report those levels as unsupported until the file -// next changed. +// next changed. The v4 row later gained validity/completeness and dedupe-key +// scope fields appended after the first v4 rows; the version deliberately +// stayed 4 because those fields are additive. // -// v3 documents are still *read*: the scan retains measured records from -// transcripts that have since been deleted, and those cannot be re-parsed, so -// discarding a v3 cache would destroy 90 days of history. A v3 row decodes with -// its native ids and measurement presence explicitly `unavailable`, and an -// extant file is cold re-parsed so those fields get filled in. Only v1/v2 (no -// parse position, different fork semantics) are rejected. +// Supported-format policy: +// - v3 documents are still *read*. The scan retains measured records from +// transcripts that have since been deleted, and those cannot be re-parsed, so +// discarding a v3 cache would destroy 90 days of history. A v3 row decodes +// with its native ids and measurement presence explicitly `unavailable`. +// - A v4 row written by the predecessor (15 fields, no quality metadata) +// decodes conservatively: completeness is `partial`, never `complete`, and +// the entry is `qualityMetadata: "predecessor"` so an extant file is cold +// re-parsed once. No row is discarded for the format change. +// - Only v1/v2 (no parse position, different fork semantics) are rejected. const USAGE_SCAN_CACHE_VERSION = 4 as const; const LEGACY_USAGE_SCAN_CACHE_VERSION = 3 as const; +/** Index of the first appended post-v4 field (completeness code) in a row. */ +const POST_V4_FIELD_INDEX = 15; + /** * Whether a cache entry's rows still carry their native ids and measurement * presence. A `v3` entry erased both; the projection must report them as @@ -50,6 +59,17 @@ const LEGACY_USAGE_SCAN_CACHE_VERSION = 3 as const; */ export type ScanCacheIdentity = "declared" | "unavailable"; +/** + * Whether a cache entry's rows carry the current numeric quality metadata + * (validity/completeness and dedupe-key scope, appended after the first v4 + * rows). Native-id availability alone does not prove that: a `15`-field v4 row + * written before that metadata existed has identity `declared` but no + * completeness, so reading it as complete would silently promote an unknown + * measurement. Such an entry is `predecessor`: its retained rows are treated as + * partial, and an extant file is cold re-parsed once to enrich it. + */ +export type ScanCacheQualityMetadata = "declared" | "predecessor"; + export interface CachedFile { readonly size: number; readonly mtimeMs: number; @@ -68,6 +88,12 @@ export interface CachedFile { * not resume such an entry: a cold re-parse is the only way to enrich it. */ readonly identity: ScanCacheIdentity; + /** + * `predecessor` for an entry whose rows omit the current quality metadata. + * Callers must not serve it warm or resume it: the numeric quality is + * unasserted, so a cold re-parse is the only way to establish completeness. + */ + readonly qualityMetadata: ScanCacheQualityMetadata; } export type ScanCache = Map; @@ -266,10 +292,17 @@ export function decodeScanCache(document: unknown): ScanCache { rows: readonly unknown[], provider: UsageProviderKind, legacy: boolean, - ): UsageRecord[] | null => { + ): { records: UsageRecord[]; qualityDeclared: boolean } | null => { const records: UsageRecord[] = []; + // Every row must carry the appended post-v4 fields for the entry to be + // current. A 15-field row was written before completeness existed; the + // entry's numeric quality is then unasserted and must not be read as + // complete. Empty row lists are vacuously current: there is no measurement + // to promote. + let qualityDeclared = true; for (const row of rows) { if (!isRecordArray(row) || row.length < 10) return null; + if (row.length <= POST_V4_FIELD_INDEX) qualityDeclared = false; const [ timestampMs, modelIndex, @@ -293,7 +326,7 @@ export function decodeScanCache(document: unknown): ScanCache { const measurementCode = row[13]; const scopeCode = row[14]; // Appended after the first v4 rows: validity/completeness metadata. Absent - // on an older row, which is treated as a complete observation. + // on an older row, whose completeness is then unasserted (`partial`). const completenessCode = row[15]; const invalidTokenFieldsRaw = row[16]; const dedupeKeyScopeCode = row[17]; @@ -316,11 +349,15 @@ export function decodeScanCache(document: unknown): ScanCache { decodeCode(measurementCode, MEASUREMENT_CODES) ?? (uncached + cached + cacheCreation + output > 0 ? "observed" : "unavailable"); const scope: UsageObservationScope = decodeCode(scopeCode, SCOPE_CODES) ?? "delta"; - // A legacy row erased field presence, so a nonzero observation is only - // known-partial: we cannot prove every required field was present. + // A row that omits the completeness code proves nothing about coverage, + // whether it is a legacy row (presence erased) or a predecessor v4 row + // (metadata predates the field). Default to `partial`, never `complete`: + // missing quality metadata must not be silently promoted to a measured + // complete observation. The current writer always emits the code, so this + // only applies to older rows. const completeness: UsageMeasurementCompleteness | undefined = measurement === "observed" - ? (decodeCode(completenessCode, COMPLETENESS_CODES) ?? (legacy ? "partial" : "complete")) + ? (decodeCode(completenessCode, COMPLETENESS_CODES) ?? "partial") : undefined; const invalidTokenFields = typeof invalidTokenFieldsRaw === "number" && @@ -365,7 +402,7 @@ export function decodeScanCache(document: unknown): ScanCache { ...(scope === "delta" ? {} : { scope }), }); } - return records; + return { records, qualityDeclared }; }; for (const [path, raw] of Object.entries(root.files)) { @@ -397,16 +434,16 @@ export function decodeScanCache(document: unknown): ScanCache { const provider: UsageProviderKind = entry.p; const legacy = legacyDocument || entry.li === 1; - const records = decodeRecords(entry.r, provider, legacy); - const tailRecords = decodeRecords(entry.t, provider, legacy); - if (records === null || tailRecords === null) continue; + const decodedRecords = decodeRecords(entry.r, provider, legacy); + const decodedTail = decodeRecords(entry.t, provider, legacy); + if (decodedRecords === null || decodedTail === null) continue; cache.set(path, { size: entry.s, mtimeMs: entry.m, provider, - records, - tailRecords, + records: decodedRecords.records, + tailRecords: decodedTail.records, position: { resumeOffset: entry.o, guardLength: entry.gl, @@ -414,6 +451,8 @@ export function decodeScanCache(document: unknown): ScanCache { codexState, }, identity: legacy ? "unavailable" : "declared", + qualityMetadata: + decodedRecords.qualityDeclared && decodedTail.qualityDeclared ? "declared" : "predecessor", }); } diff --git a/apps/server/src/usage/usageTranscripts.test.ts b/apps/server/src/usage/usageTranscripts.test.ts index 84ce39631e63..d6140293bf02 100644 --- a/apps/server/src/usage/usageTranscripts.test.ts +++ b/apps/server/src/usage/usageTranscripts.test.ts @@ -586,14 +586,16 @@ describe("parseGrokLine", () => { }), ); - expect(records).toHaveLength(2); - expect(records.every((record) => record.model !== "empty-sibling")).toBe(true); + // The all-explicit-zero sibling is a measured zero, not no-usage, so it is + // retained with its own (zero) cost and must not take a pro-rated share. + expect(records).toHaveLength(3); const byModel = Object.fromEntries(records.map((record) => [record.model, record])); + expect(byModel["empty-sibling"]?.measurement).toBe("observed"); + expect(byModel["empty-sibling"]?.measurementCompleteness).toBe("complete"); + expect(byModel["empty-sibling"]?.reportedCostUsd).toBe(0); expect(byModel["grok-4.5"]?.reportedCostUsd).toBeCloseTo(0.75, 12); expect(byModel["grok-composer-2.5-fast"]?.reportedCostUsd).toBeCloseTo(0.25, 12); - const sum = - (byModel["grok-4.5"]?.reportedCostUsd ?? 0) + - (byModel["grok-composer-2.5-fast"]?.reportedCostUsd ?? 0); + const sum = records.reduce((total, record) => total + (record.reportedCostUsd ?? 0), 0); expect(sum).toBeCloseTo(1, 12); }); @@ -629,6 +631,30 @@ describe("parseGrokLine", () => { expect(sum).toBeCloseTo(1, 12); }); + it("does not lose aggregate cost to a row with no recognised token field", () => { + const records = parseGrokLine( + turnCompleted({ + modelUsage: { + "grok-4.5": { + inputTokens: 100, + outputTokens: 0, + cachedReadTokens: 0, + reasoningTokens: 0, + }, + "cost-only": { costUsdTicks: 0.3 * GROK_COST_USD_TICKS_PER_DOLLAR }, + }, + usage: { costUsdTicks: GROK_COST_USD_TICKS_PER_DOLLAR }, + }), + ); + + // The cost-only row carries no recognised token field, so it is no-usage and + // not emitted. Its explicit ticks must stay in the aggregate for the emitted + // sibling instead of being silently dropped with the row. + expect(records).toHaveLength(1); + expect(records[0]?.model).toBe("grok-4.5"); + expect(records[0]?.reportedCostUsd).toBeCloseTo(1, 12); + }); + it("exposes the native prompt id apart from the dedupe key", () => { const [record] = parseGrokLine(turnCompleted({ promptId: "prompt-7" })); @@ -659,24 +685,33 @@ describe("parseGrokLine", () => { expect(parseGrokLine(line)[0]?.dedupeKey).toBeNull(); }); - it("ignores non-turn lines and empty usage", () => { + it("ignores non-turn lines", () => { expect(parseGrokLine(JSON.stringify({ method: "session/update", params: {} }))).toEqual([]); expect(parseGrokLine("not json")).toEqual([]); - expect( - parseGrokLine( - turnCompleted({ - modelUsage: { - "grok-4.5-build": { - inputTokens: 0, - outputTokens: 0, - cachedReadTokens: 0, - reasoningTokens: 0, - costUsdTicks: 0, - }, + }); + + it("retains a complete explicit-zero per-model row instead of erasing it", () => { + const records = parseGrokLine( + turnCompleted({ + modelUsage: { + "grok-4.5-build": { + inputTokens: 0, + outputTokens: 0, + cachedReadTokens: 0, + reasoningTokens: 0, + costUsdTicks: 0, }, - }), - ), - ).toEqual([]); + }, + }), + ); + + // Explicit zeros are a measured zero, not missing usage: keeping the record + // lets the projection label the session measured rather than absent. + expect(records).toHaveLength(1); + expect(records[0]?.model).toBe("grok-4.5-build"); + expect(records[0]?.measurement).toBe("observed"); + expect(records[0]?.measurementCompleteness).toBe("complete"); + expect(totalTokens(records[0]!.totals)).toBe(0); }); it("falls back to the outer unix-seconds timestamp when agent meta is missing", () => { @@ -703,3 +738,146 @@ describe("parseGrokLine", () => { expect(records[0]?.timestampMs).toBe(1_786_372_566_000); }); }); + +describe("zero-total quality preservation", () => { + /** A Codex rollout primed with its session meta and model. */ + function primedCodexState() { + const state = initialCodexScanState(); + parseCodexLine( + JSON.stringify({ type: "session_meta", payload: { id: "codex-session" } }), + state, + ); + parseCodexLine( + JSON.stringify({ type: "turn_context", payload: { model: "gpt-5.6-sol" } }), + state, + ); + return state; + } + + function codexTokenCount( + lastTokenUsage: Record, + timestamp = "2026-08-01T05:17:49.919Z", + ): string { + return JSON.stringify({ + type: "event_msg", + timestamp, + payload: { type: "token_count", info: { last_token_usage: lastTokenUsage } }, + }); + } + + it("keeps a Codex all-invalid payload as invalid instead of dropping it", () => { + const record = parseCodexLine( + codexTokenCount({ input_tokens: null, output_tokens: null }), + primedCodexState(), + ); + + expect(record).not.toBeNull(); + expect(record?.measurement).toBe("invalid"); + expect(record?.measurementCompleteness).toBeUndefined(); + expect(totalTokens(record!.totals)).toBe(0); + }); + + it("keeps a Codex complete explicit zero as a measured zero", () => { + const record = parseCodexLine( + codexTokenCount({ input_tokens: 0, output_tokens: 0 }), + primedCodexState(), + ); + + expect(record?.measurement).toBe("observed"); + expect(record?.measurementCompleteness).toBe("complete"); + expect(record?.invalidTokenFields).toBeUndefined(); + expect(totalTokens(record!.totals)).toBe(0); + }); + + it("keeps a Codex known-zero subset as partial", () => { + const record = parseCodexLine(codexTokenCount({ input_tokens: 10 }), primedCodexState()); + + expect(record?.measurement).toBe("observed"); + expect(record?.measurementCompleteness).toBe("partial"); + expect(record?.totals.uncachedInputTokens).toBe(10); + }); + + it("keeps a Codex valid event and a separate invalid event distinct", () => { + const state = primedCodexState(); + const valid = parseCodexLine( + codexTokenCount({ input_tokens: 10, output_tokens: 2 }, "2026-08-01T05:17:49.919Z"), + state, + ); + const invalid = parseCodexLine( + codexTokenCount({ input_tokens: null, output_tokens: null }, "2026-08-01T05:18:00.000Z"), + state, + ); + + expect(valid).not.toBeNull(); + expect(totalTokens(valid!.totals)).toBe(12); + expect(invalid).not.toBeNull(); + expect(invalid?.measurement).toBe("invalid"); + }); + + it("treats a Codex usage container with no recognised field as no-usage", () => { + expect(parseCodexLine(codexTokenCount({}), primedCodexState())).toBeNull(); + }); + + function grokTurn(usage: Record): string { + return JSON.stringify({ + timestamp: 1_786_372_566, + method: "_x.ai/session/update", + params: { + sessionId: "grook-session", + update: { sessionUpdate: "turn_completed", prompt_id: "p1", usage }, + _meta: { agentTimestampMs: 1_786_372_566_485 }, + }, + }); + } + + it("keeps a Grok aggregate all-invalid payload as invalid", () => { + const records = parseGrokLine(grokTurn({ inputTokens: null, outputTokens: null })); + + expect(records).toHaveLength(1); + expect(records[0]?.measurement).toBe("invalid"); + expect(records[0]?.measurementCompleteness).toBeUndefined(); + expect(totalTokens(records[0]!.totals)).toBe(0); + }); + + it("keeps a Grok aggregate complete explicit zero", () => { + const records = parseGrokLine(grokTurn({ inputTokens: 0, outputTokens: 0 })); + + expect(records).toHaveLength(1); + expect(records[0]?.measurement).toBe("observed"); + expect(records[0]?.measurementCompleteness).toBe("complete"); + expect(totalTokens(records[0]!.totals)).toBe(0); + }); + + it("keeps a Grok aggregate partial known-zero subset", () => { + const records = parseGrokLine(grokTurn({ inputTokens: 0 })); + + expect(records).toHaveLength(1); + expect(records[0]?.measurement).toBe("observed"); + expect(records[0]?.measurementCompleteness).toBe("partial"); + }); + + it("treats a Grok aggregate container with no recognised field as no-usage", () => { + expect(parseGrokLine(grokTurn({}))).toEqual([]); + }); + + it("keeps Grok per-model invalid and zero rows instead of skipping them", () => { + const records = parseGrokLine( + grokTurn({ + inputTokens: null, + outputTokens: null, + modelUsage: { + "model-invalid": { inputTokens: null, outputTokens: null }, + "model-zero": { inputTokens: 0, outputTokens: 0 }, + "model-valid": { inputTokens: 5, outputTokens: 5 }, + }, + }), + ); + + expect(records).toHaveLength(3); + const byModel = Object.fromEntries(records.map((record) => [record.model, record])); + expect(byModel["model-invalid"]?.measurement).toBe("invalid"); + expect(byModel["model-zero"]?.measurement).toBe("observed"); + expect(byModel["model-zero"]?.measurementCompleteness).toBe("complete"); + expect(totalTokens(byModel["model-valid"]!.totals)).toBe(10); + }); +}); diff --git a/apps/server/src/usage/usageTranscripts.ts b/apps/server/src/usage/usageTranscripts.ts index 942345efe82b..86205865d407 100644 --- a/apps/server/src/usage/usageTranscripts.ts +++ b/apps/server/src/usage/usageTranscripts.ts @@ -500,6 +500,17 @@ export function parseCodexLine(line: string, state: CodexScanState): UsageRecord state.suppressingForkCopies = false; } + // Classify before deciding to emit. A measured zero or an all-invalid payload + // is still evidence about the session, so a zero subtotal must not drop it + // before the projection can label it. Only a container with no recognised + // token field at all is no-usage and is not emitted. + const classification = classifyTokenFields( + lastRecord, + ["input_tokens", "output_tokens"], + ["cached_input_tokens", "cache_write_input_tokens", "reasoning_output_tokens"], + ); + if (classification.measurement === "empty") return null; + const inputTokens = int(lastRecord["input_tokens"]); const cachedInputTokens = int(lastRecord["cached_input_tokens"]); const cacheCreationTokens = int(lastRecord["cache_write_input_tokens"]); @@ -515,14 +526,6 @@ export function parseCodexLine(line: string, state: CodexScanState): UsageRecord reasoningTokens: Math.min(outputTokens, int(lastRecord["reasoning_output_tokens"])), }; - if (totalTokens(totals) === 0) return null; - - const classification = classifyTokenFields( - lastRecord, - ["input_tokens", "output_tokens"], - ["cached_input_tokens", "cache_write_input_tokens", "reasoning_output_tokens"], - ); - return { provider: "codex", timestampMs, @@ -539,8 +542,7 @@ export function parseCodexLine(line: string, state: CodexScanState): UsageRecord providerRequestId: null, providerMessageId: null, promptId: null, - // Only emitted when at least one token was measured, so this is observed. - measurement: "observed", + measurement: classification.measurement, ...(classification.completeness === undefined ? {} : { measurementCompleteness: classification.completeness }), @@ -673,7 +675,10 @@ export function parseGrokLine(line: string): readonly UsageRecord[] { } if (modelEntries.length === 0) { - if (totalTokens(grokTotalsToUsage(topLevel)) === 0) return []; + // Only a usage container with no recognised token field is no-usage. A + // measured zero or an all-invalid payload is retained so its quality + // reaches the projection instead of vanishing at a zero subtotal. + if (topLevel.classification.measurement === "empty") return []; return [ { provider: "grok", @@ -709,26 +714,35 @@ export function parseGrokLine(line: string): readonly UsageRecord[] { // by token share among the unticked models only. // 3. When no model has per-model ticks, remaining equals the full // aggregate and every emitted model gets a token-share slice. - // Zero-token rows are never emitted and never count toward used ticks. + // An explicit per-model cost reduces the aggregate no matter its token total, + // for any row that is actually emitted; skipping an emitted zero-token ticked + // row would let the same ticks be pro-rated onto its siblings. A row that is + // not emitted (no recognised token field) does not reduce the aggregate, so + // its ticks stay available to the emitted rows. A zero-token unticked row has + // no share to receive. const topLevelCostUsd = grokCostTicksToUsd(topLevel.costUsdTicks); let usedTickedCostUsd = 0; let untickedTokenDenominator = 0; for (const entry of modelEntries) { - const tokens = totalTokens(grokTotalsToUsage(entry.totals)); - if (tokens === 0) continue; - if (entry.totals.costUsdTicks !== null) { + const emitted = entry.totals.classification.measurement !== "empty"; + if (emitted && entry.totals.costUsdTicks !== null) { usedTickedCostUsd += grokCostTicksToUsd(entry.totals.costUsdTicks) ?? 0; - } else { - untickedTokenDenominator += tokens; + continue; } + const tokens = totalTokens(grokTotalsToUsage(entry.totals)); + if (tokens === 0) continue; + untickedTokenDenominator += tokens; } const remainingCostUsd = topLevelCostUsd === null ? null : Math.max(0, topLevelCostUsd - usedTickedCostUsd); const results: UsageRecord[] = []; for (const entry of modelEntries) { + // Same rule as the aggregate path: retain a measured zero or an invalid + // per-model observation; only a container with no recognised field is + // no-usage. + if (entry.totals.classification.measurement === "empty") continue; const totals = grokTotalsToUsage(entry.totals); - if (totalTokens(totals) === 0) continue; let reportedCostUsd = grokCostTicksToUsd(entry.totals.costUsdTicks); if (reportedCostUsd === null && remainingCostUsd !== null && untickedTokenDenominator > 0) { diff --git a/docs/internals/usage-attribution.md b/docs/internals/usage-attribution.md index e3475349ec6c..b04dfe09e549 100644 --- a/docs/internals/usage-attribution.md +++ b/docs/internals/usage-attribution.md @@ -51,6 +51,15 @@ invalid | unavailable`) — were tokens actually measured, and were the provider Claude's `usage: {}` is `invalid`; an all-zero legacy row whose presence was erased is `unavailable`, never `missing` and never a measured zero. A nonzero total never implies a complete measurement. + A **zero subtotal is not a reason to drop a record**: every provider now retains an + eligible event whose total is zero — a complete measured zero, a known-zero subset, + or an all-invalid payload — so the classification reaches the projection instead of + disappearing at a parser gate. Only a usage container with no recognised token field + (`usage: {}`, `last_token_usage: {}`, or an absent container) is treated as + no-usage and not emitted. The parser never fabricates tokens to keep such a record, + and Codex and Grok are held to the same rule as Claude. The scan's diagnostic + `malformedRecords` count and each session's `measurementQuality` are derived from + these retained records, so the two agree and nothing is double counted. - **level support** (`promptQuality` / `requestQuality`) — can the source establish this level, and did the records carry its id? `unsupported` is a structural limit, not a zero. A legacy row whose native id was erased reports `unavailable`, never @@ -132,17 +141,29 @@ historical allocation as of a past instant is unavailable without temporal evide ## Cache upgrades retain existing history -The scan cache version is still `4`, but a `v3` document is now **read** rather than -discarded. The scan retains measured records from transcripts that have since been -deleted for 90 days, and those cannot be re-parsed, so discarding a v3 cache would -destroy that history. A v3 row decodes with its native ids and measurement presence -explicitly `unavailable` (an all-zero row stays unknown, not a measured zero; a nonzero -row is a known but only-partial measurement), and the entry is flagged so it is never -resumed incrementally. Warm-cache acceptance requires the current identity/measurement -format: an unchanged extant legacy file is cold re-parsed once to enrich it with ids and -presence without double counting because the re-parse replaces the entry. A read failure -during that re-parse keeps the retained fallback rows, and a deleted file is never -re-parsed at all, so its history survives. +The scan cache version is still `4`. Supported-format policy: + +- A `v3` document is **read** rather than discarded. The scan retains measured records + from transcripts that have since been deleted for 90 days, and those cannot be + re-parsed, so discarding a v3 cache would destroy that history. A v3 row decodes with + its native ids and measurement presence explicitly `unavailable` (an all-zero row + stays unknown, not a measured zero; a nonzero row is a known but only-partial + measurement). +- A `v4` row written by the predecessor (15 fields, before the + validity/completeness/key-scope fields were appended) is **read conservatively**: + its completeness is `partial`, never `complete`, because the writer never asserted + coverage. Missing quality metadata is never silently promoted. The entry is marked + `qualityMetadata: "predecessor"`, so an extant file is cold re-parsed once and then + accepted warm. No row is discarded for the format change. +- Only `v1`/`v2` (no parse position, different fork semantics) are rejected. + +Warm-cache acceptance therefore requires both `identity: "declared"` (native ids and +measurement presence) and `qualityMetadata: "declared"` (the current numeric quality +fields). Either being stale forces one cold re-parse that replaces the entry, so +enrichment never double counts. A read failure during that re-parse keeps the retained +fallback rows, and a deleted file is never re-parsed at all, so its history survives +with conservative `partial` quality. Native-id availability alone does **not** +establish numeric metadata freshness; the two are separate axes. ## What still needs architecture approval