From 6ce5a121d7990facd662f6b918bd2c795cc43464 Mon Sep 17 00:00:00 2001 From: tris203 Date: Sun, 16 Aug 2026 15:31:57 +0100 Subject: [PATCH 01/15] feat: add OpenCode usage statistics - Read OpenCode SQLite transcripts and report token and cost usage - Display OpenCode usage across web and mobile clients --- .../src/features/usage/usageProviders.ts | 9 +- apps/server/src/usage/UsageService.ts | 106 +++++++++++++++++- apps/server/src/usage/usageScanCache.ts | 15 ++- .../server/src/usage/usageTranscriptReader.ts | 55 +++++++++ .../server/src/usage/usageTranscripts.test.ts | 88 +++++++++++++++ apps/server/src/usage/usageTranscripts.ts | 73 ++++++++++++ .../usage/UsageProviderChart.test.ts | 1 + .../src/components/usage/usageProviders.ts | 7 +- packages/contracts/src/usage.ts | 5 +- 9 files changed, 347 insertions(+), 12 deletions(-) diff --git a/apps/mobile/src/features/usage/usageProviders.ts b/apps/mobile/src/features/usage/usageProviders.ts index 2576ac21fb07..82ee5fa873cc 100644 --- a/apps/mobile/src/features/usage/usageProviders.ts +++ b/apps/mobile/src/features/usage/usageProviders.ts @@ -5,12 +5,18 @@ import { useAppearancePreferences } from "../settings/appearance/AppearancePrefe * Series and table order. The chart stacks providers from the bottom in this * order, so it also fixes which band sits on top of the bars. */ -export const PROVIDER_ORDER: readonly UsageProviderKind[] = ["codex", "claude", "grok"]; +export const PROVIDER_ORDER: readonly UsageProviderKind[] = [ + "codex", + "claude", + "grok", + "opencode", +]; export const PROVIDER_LABEL: Record = { claude: "Claude Code", codex: "Codex", grok: "Grok Build", + opencode: "OpenCode", }; /** @@ -23,5 +29,6 @@ export function useProviderColors(): Record { claude: "#d97757", codex: scheme === "dark" ? "#e6e6e6" : "#3c3c43", grok: scheme === "dark" ? "#a1a1aa" : "#52525b", + opencode: "#8b5cf6", }; } diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index 224662e9dca7..7aff94442e5b 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -21,7 +21,7 @@ import { type UsageSummaryInput, UsageReadError, } from "@t3tools/contracts"; -import { HostProcessEnvironment } from "@t3tools/shared/hostProcess"; +import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; @@ -44,6 +44,7 @@ import { parseRateTable, type RateTable } from "./usagePricing.ts"; import { listTranscriptFiles, readDirectoryVolumeId, + readOpenCodeRecords, readTranscriptRecords, } from "./usageTranscriptReader.ts"; import { @@ -200,6 +201,32 @@ export const make = Effect.gen(function* () { return nestedExists ? nested : path.join(homePath, "projects"); }); + /** + * OpenCode keeps its transcripts in a SQLite database at + * `/opencode.db`. The data home follows XDG, so a Linux default + * install lives in `~/.local/share/opencode` while macOS uses + * `~/Library/Application Support/opencode`. `OPENCODE_DATA_HOME` wins when + * set, matching the CLI. + */ + const resolveOpenCodeDatabasePath = Effect.fn("UsageService.resolveOpenCodeDatabasePath")( + function* () { + const platform = yield* HostProcessPlatform; + const env = yield* HostProcessEnvironment; + const override = env["OPENCODE_DATA_HOME"]?.trim(); + if (override !== undefined && override.length > 0) { + return path.join(path.resolve(expandHomePath(override)), "opencode.db"); + } + const dataHome = + platform === "darwin" + ? path.join(NodeOS.homedir(), "Library", "Application Support", "opencode") + : path.join( + env["XDG_DATA_HOME"] ?? path.join(NodeOS.homedir(), ".local", "share"), + "opencode", + ); + return path.join(dataHome, "opencode.db"); + }, + ); + /** Resolves the transcript directory for each provider. */ const resolveTranscriptDirs = Effect.fn("UsageService.resolveTranscriptDirs")(function* () { // A settings failure must surface as an error: swallowing it here would @@ -228,6 +255,7 @@ export const make = Effect.gen(function* () { grokHomeEnv.length > 0 ? path.resolve(expandHomePath(grokHomeEnv)) : path.join(NodeOS.homedir(), ".grok"); + const openCodeDbPath = yield* resolveOpenCodeDatabasePath(); return [ { provider: "claude" as const, dir: claudeDir }, @@ -237,6 +265,7 @@ export const make = Effect.gen(function* () { dir: path.join(grokHome, "sessions"), fileName: "updates.jsonl", }, + { provider: "opencode" as const, dir: openCodeDbPath }, ]; }); @@ -272,18 +301,38 @@ export const make = Effect.gen(function* () { ); }); - /** Parses one transcript, reusing the cached result when it is unchanged. */ + /** + * Parses one transcript, reusing the cached result when it is unchanged. + * + * The `(size, mtime)` identity assumes a file's contents are + * window-independent, which holds for the per-session JSONL transcripts. + * OpenCode's source is one SQLite database queried with a window filter, so a + * cached entry only ever covers the window it was scanned for and a wider + * window would silently reuse it. The windowed query is fast enough that the + * cache buys nothing, so OpenCode always scans fresh. + */ const readFileRecords = ( filePath: string, size: number, mtimeMs: number, provider: UsageProviderKind, + windowStartMs?: number, ): Effect.Effect => Effect.gen(function* () { + if (provider === "opencode") { + const parsed = yield* Effect.promise(() => + readOpenCodeRecords(filePath, windowStartMs ?? 0), + ); + // A read failure is not an empty transcript: reporting zero usage + // would silently drop the source's usage. + return parsed ?? []; + } + const cached = fileCache.get(filePath); // Provider is part of the identity: if both providers were ever pointed // at one directory, a hit parsed by the other parser must not be reused. if ( + mtimeMs !== 0 && cached && cached.size === size && cached.mtimeMs === mtimeMs && @@ -300,8 +349,10 @@ export const make = Effect.gen(function* () { // duplicates. The aggregator still runs the cross-file dedupe pass. const records = dedupeWithinFile(parsed); - fileCache.set(filePath, { size, mtimeMs, provider, records }); - cacheDirty = true; + if (mtimeMs !== 0) { + fileCache.set(filePath, { size, mtimeMs, provider, records }); + cacheDirty = true; + } return records; }); @@ -387,6 +438,53 @@ export const make = Effect.gen(function* () { continue; } + if (provider === "opencode") { + // The whole source is one database. Registering the file as live and + // its parent as a walked root lets the prune pass evict any entry an + // earlier version cached under a narrower window (a stale hit would + // silently cap the visible history at that first window). + livePaths.add(dir); + walkedRoots.push(path.dirname(dir)); + const stats = yield* fileSystem.stat(dir).pipe( + Effect.map((info) => ({ + size: Number(info.size), + mtimeMs: Option.match(info.mtime, { + onNone: () => 0, + onSome: (mtime) => mtime.getTime(), + }), + })), + Effect.catchCause(() => Effect.succeed(null)), + ); + + const records = yield* readFileRecords( + dir, + stats?.size ?? 0, + stats?.mtimeMs ?? 0, + provider, + windowStartMs, + ); + // Distinct per database. Buckets carry per-cell session counts, but a + // session spans days and models, so clients total this figure instead. + const sessionIds = new Set(); + for (const record of records) { + // Only sessions that contributed in-window count: the query slack + // admits boundary rows whose timestamps fall outside the range. + if (aggregator.add(record) && record.sessionId.length > 0) { + sessionIds.add(record.sessionId); + } + } + sources.push({ + fingerprint: { hostId, provider, resolvedHomePath: dir, volumeId }, + status: stats === null ? "failed" : "ok", + scannedFiles: records.length > 0 ? 1 : 0, + skippedFiles: records.length > 0 ? 0 : 1, + malformedRecords: 0, + distinctSessions: sessionIds.size, + message: stats === null ? "Transcript database could not be read." : null, + }); + continue; + } + walkedRoots.push(dir); const files = yield* Effect.promise(() => listTranscriptFiles(dir, windowStartMs, fileName === undefined ? undefined : { fileName }), diff --git a/apps/server/src/usage/usageScanCache.ts b/apps/server/src/usage/usageScanCache.ts index 02daf5ebbd70..ed07c0439cf0 100644 --- a/apps/server/src/usage/usageScanCache.ts +++ b/apps/server/src/usage/usageScanCache.ts @@ -18,9 +18,9 @@ import type { UsageProviderKind } from "@t3tools/contracts"; import type { UsageRecord } from "./usageTranscripts.ts"; -// v2: Codex fork-copy suppression changed what a file parses to, so v1 -// entries would keep serving double-counted records forever. -export const USAGE_SCAN_CACHE_VERSION = 2 as const; +// v3: OpenCode joined the scanned providers, so v2 entries must not be +// reused under a provider set they were never parsed for. +export const USAGE_SCAN_CACHE_VERSION = 3 as const; export interface CachedFile { readonly size: number; @@ -134,7 +134,14 @@ export function decodeScanCache(document: unknown): ScanCache { if (typeof raw !== "object" || raw === null) continue; const entry = raw as Partial; if (typeof entry.s !== "number" || typeof entry.m !== "number") continue; - if (entry.p !== "claude" && entry.p !== "codex" && entry.p !== "grok") continue; + if ( + entry.p !== "claude" && + entry.p !== "codex" && + entry.p !== "grok" && + entry.p !== "opencode" + ) { + continue; + } if (!isRecordArray(entry.r)) continue; const provider: UsageProviderKind = entry.p; diff --git a/apps/server/src/usage/usageTranscriptReader.ts b/apps/server/src/usage/usageTranscriptReader.ts index 33aef8fae25c..fd9bb420e427 100644 --- a/apps/server/src/usage/usageTranscriptReader.ts +++ b/apps/server/src/usage/usageTranscriptReader.ts @@ -8,12 +8,16 @@ * magnitude cheaper than materialising each file. The equivalent Effect stream * pipeline is idiomatic but not fast enough to sit behind a page load. * + * OpenCode moved its transcripts into a SQLite database, so its scan goes + * through `node:sqlite` instead of the file walk. + * * @module usageTranscriptReader */ import * as NodeFS from "node:fs"; import * as NodeFSP from "node:fs/promises"; import * as NodePath from "node:path"; import * as NodeReadline from "node:readline"; +import * as NodeSqlite from "node:sqlite"; import type { UsageProviderKind } from "@t3tools/contracts"; @@ -23,6 +27,7 @@ import { parseClaudeLine, parseCodexLine, parseGrokLine, + parseOpenCodeMessage, type UsageRecord, } from "./usageTranscripts.ts"; @@ -156,3 +161,53 @@ export async function readTranscriptRecords( return records; } + +/** + * Reads usage records from OpenCode's SQLite transcript store. + * + * Unlike the JSONL providers, OpenCode keeps one row per message in + * `opencode.db`, so the whole source is one query. The window filter is pushed + * into SQL via `time_updated`, which covers in-progress messages that predate + * the window but complete inside it. The database is opened read-only, and + * `-wal`/`-shm` siblings are never created because no write happens. + * + * Returns `null` when the database cannot be read, so the caller reports the + * source as failed rather than zero usage. + */ +export async function readOpenCodeRecords( + dbPath: string, + sinceMs: number, +): Promise { + let database: NodeSqlite.DatabaseSync; + try { + database = new NodeSqlite.DatabaseSync(dbPath, { readOnly: true }); + } catch { + return null; + } + + try { + const rows = database + .prepare( + `SELECT data FROM message + WHERE time_updated >= ? + AND json_extract(data, '$.role') = 'assistant' + AND json_extract(data, '$.tokens.total') > 0 + AND json_extract(data, '$.modelID') IS NOT NULL + AND json_extract(data, '$.time.completed') IS NOT NULL`, + ) + .all(sinceMs); + + const records: UsageRecord[] = []; + for (const row of rows) { + const data = (row as Record)["data"]; + if (typeof data !== "string") continue; + const record = parseOpenCodeMessage(data); + if (record !== null) records.push(record); + } + return records; + } catch { + return null; + } finally { + database.close(); + } +} diff --git a/apps/server/src/usage/usageTranscripts.test.ts b/apps/server/src/usage/usageTranscripts.test.ts index b09db613ed85..424a3e57672d 100644 --- a/apps/server/src/usage/usageTranscripts.test.ts +++ b/apps/server/src/usage/usageTranscripts.test.ts @@ -6,6 +6,7 @@ import { parseClaudeLine, parseCodexLine, parseGrokLine, + parseOpenCodeMessage, totalTokens, } from "./usageTranscripts.ts"; @@ -238,6 +239,93 @@ describe("parseCodexLine", () => { }); }); +describe("parseOpenCodeMessage", () => { + /** Shaped after a real OpenCode assistant message row's `data` payload. */ + function openCodeMessage(overrides: { + id: string; + role?: string; + modelID?: string; + completed?: number | null; + cost?: number; + input?: number; + output?: number; + reasoning?: number; + cacheRead?: number; + cacheWrite?: number; + }): string { + return JSON.stringify({ + id: overrides.id, + sessionID: "ses_3a6c0a5d3ffeg7BPjptjftbHYs", + role: overrides.role ?? "assistant", + time: { + created: 1771023850034, + completed: + overrides.completed === null ? undefined : (overrides.completed ?? 1771023853436), + }, + modelID: overrides.modelID ?? "gpt-5.2-codex", + providerID: "github-copilot", + cost: overrides.cost ?? 0, + tokens: { + total: 9154, + input: overrides.input ?? 486, + output: overrides.output ?? 220, + reasoning: overrides.reasoning ?? 0, + cache: { read: overrides.cacheRead ?? 8448, write: overrides.cacheWrite ?? 0 }, + }, + finish: "stop", + }); + } + + it("extracts token totals from an assistant message", () => { + const record = parseOpenCodeMessage(openCodeMessage({ id: "msg_1", reasoning: 40 })); + + expect(record).not.toBeNull(); + expect(record?.provider).toBe("opencode"); + expect(record?.model).toBe("gpt-5.2-codex"); + expect(record?.sessionId).toBe("ses_3a6c0a5d3ffeg7BPjptjftbHYs"); + expect(record?.timestampMs).toBe(1771023853436); + // OpenCode's input is exclusive of the cached portions. + expect(record?.totals).toEqual({ + uncachedInputTokens: 486, + cachedInputTokens: 8448, + cacheCreationTokens: 0, + outputTokens: 220, + reasoningTokens: 40, + }); + expect(record?.dedupeKey).toBe("msg_1"); + }); + + it("trusts a positive reported cost, and only a positive one", () => { + // OpenCode prices against its own rate table, which covers curated and + // subscription-served models LiteLLM does not know. + const priced = parseOpenCodeMessage(openCodeMessage({ id: "msg_2", cost: 0.023 })); + expect(priced?.reportedCostUsd).toBe(0.023); + + // Subscription-backed providers leave cost at 0; those fall back to the + // LiteLLM rate table like Codex. + const subscription = parseOpenCodeMessage(openCodeMessage({ id: "msg_2b", cost: 0 })); + expect(subscription?.reportedCostUsd).toBeNull(); + }); + + it("caps reasoning at output", () => { + const record = parseOpenCodeMessage( + openCodeMessage({ id: "msg_3", output: 10, reasoning: 99 }), + ); + expect(record?.totals.reasoningTokens).toBe(10); + }); + + it("ignores user messages, unfinished turns, and token-less records", () => { + expect(parseOpenCodeMessage(openCodeMessage({ id: "msg_4", role: "user" }))).toBeNull(); + expect(parseOpenCodeMessage(openCodeMessage({ id: "msg_5", completed: null }))).toBeNull(); + expect( + parseOpenCodeMessage( + openCodeMessage({ id: "msg_6", input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }), + ), + ).toBeNull(); + expect(parseOpenCodeMessage("not json")).toBeNull(); + }); +}); + describe("totalTokens", () => { it("does not add reasoning on top of output", () => { expect( diff --git a/apps/server/src/usage/usageTranscripts.ts b/apps/server/src/usage/usageTranscripts.ts index 2aea60709666..ae4da162e616 100644 --- a/apps/server/src/usage/usageTranscripts.ts +++ b/apps/server/src/usage/usageTranscripts.ts @@ -485,4 +485,77 @@ export function parseGrokLine(line: string): readonly UsageRecord[] { return results; } +/* -------------------------------------------------------------------------- */ +/* OpenCode */ +/* -------------------------------------------------------------------------- */ + +/** + * Projects one `data` payload from OpenCode's SQLite `message` table into a + * usage record. + * + * OpenCode moved its transcripts into `~/.local/share/opencode/opencode.db`: + * each row is one message with the payload stored as JSON in `data`. Usage + * sits on assistant messages as `tokens` with a `cache` breakdown; `input` is + * exclusive of the cached portions. The message `id` is the dedupe key. + * + * The `cost` field is trusted when positive: OpenCode prices against its own + * rate table, which covers its curated and subscription-served models that + * LiteLLM does not know. A zero cost on a subscription-backed provider falls + * back to LiteLLM like Codex. + */ +export function parseOpenCodeMessage(data: string): UsageRecord | null { + let parsed: unknown; + try { + parsed = JSON.parse(data); + } catch { + return null; + } + if (typeof parsed !== "object" || parsed === null) return null; + + const record = parsed as Record; + if (record["role"] !== "assistant") return null; + + const tokens = record["tokens"]; + if (typeof tokens !== "object" || tokens === null) return null; + const tokensRecord = tokens as Record; + const cache = + typeof tokensRecord["cache"] === "object" && tokensRecord["cache"] !== null + ? (tokensRecord["cache"] as Record) + : {}; + + const time = record["time"]; + const completed = + typeof time === "object" && time !== null + ? (time as Record)["completed"] + : undefined; + const timestampMs = + typeof completed === "number" && Number.isFinite(completed) ? Math.trunc(completed) : null; + if (timestampMs === null) return null; + + const model = typeof record["modelID"] === "string" ? record["modelID"] : ""; + if (model.length === 0) return null; + + const totals: UsageTokenTotals = { + uncachedInputTokens: int(tokensRecord["input"]), + cachedInputTokens: int(cache["read"]), + cacheCreationTokens: int(cache["write"]), + outputTokens: int(tokensRecord["output"]), + reasoningTokens: Math.min(int(tokensRecord["output"]), int(tokensRecord["reasoning"])), + }; + + if (totalTokens(totals) === 0) return null; + + const cost = record["cost"]; + + return { + provider: "opencode", + timestampMs, + model, + sessionId: typeof record["sessionID"] === "string" ? record["sessionID"] : "", + totals, + reportedCostUsd: typeof cost === "number" && Number.isFinite(cost) && cost > 0 ? cost : null, + dedupeKey: typeof record["id"] === "string" ? record["id"] : null, + }; +} + export { EMPTY_TOTALS }; diff --git a/apps/web/src/components/usage/UsageProviderChart.test.ts b/apps/web/src/components/usage/UsageProviderChart.test.ts index a4114cfdfb57..9b61e918a523 100644 --- a/apps/web/src/components/usage/UsageProviderChart.test.ts +++ b/apps/web/src/components/usage/UsageProviderChart.test.ts @@ -87,6 +87,7 @@ describe("buildDayColumns", () => { { provider: "codex", value: 10 }, { provider: "claude", value: 20 }, { provider: "grok", value: 0 }, + { provider: "opencode", value: 0 }, ]); }); diff --git a/apps/web/src/components/usage/usageProviders.ts b/apps/web/src/components/usage/usageProviders.ts index efad95e531ad..176e7b6c41a2 100644 --- a/apps/web/src/components/usage/usageProviders.ts +++ b/apps/web/src/components/usage/usageProviders.ts @@ -1,6 +1,6 @@ import type { UsageProviderKind } from "@t3tools/contracts"; -import { ClaudeAI, GrokIcon, type Icon, OpenAI } from "../Icons"; +import { ClaudeAI, GrokIcon, type Icon, OpenAI, OpenCodeIcon } from "../Icons"; type UsageProviderPresentation = { readonly label: string; @@ -30,6 +30,11 @@ export const PROVIDER_PRESENTATION = { color: "color-mix(in oklab, var(--contrast-foreground) 72%, var(--background))", mark: GrokIcon, }, + opencode: { + label: "OpenCode", + color: "#8b5cf6", + mark: OpenCodeIcon, + }, } satisfies Record; /** Stable provider reading order across charts, summaries, tables, and hover rows. */ diff --git a/packages/contracts/src/usage.ts b/packages/contracts/src/usage.ts index 8c099ddb33aa..552f3c269c6a 100644 --- a/packages/contracts/src/usage.ts +++ b/packages/contracts/src/usage.ts @@ -3,7 +3,8 @@ * * Each environment scans the provider CLIs' own on-disk session transcripts * (`~/.claude/projects/**\/*.jsonl`, `~/.codex/sessions/**\/*.jsonl`, - * `~/.grok/sessions/**\/updates.jsonl`) rather than relying on T3 Code's own + * `~/.grok/sessions/**\/updates.jsonl`, and OpenCode's SQLite store at + * `~/.local/share/opencode/opencode.db`) rather than relying on T3 Code's own * orchestration projections, so usage stays complete even for turns that were * never driven through T3 Code. This mirrors the approach `ccusage` takes. * @@ -32,7 +33,7 @@ export const USAGE_CONTRACT_VERSION = 5 as const; */ export const USAGE_MERGE_COMPATIBLE_SINCE = 4 as const; -export const UsageProviderKind = Schema.Literals(["claude", "codex", "grok"]); +export const UsageProviderKind = Schema.Literals(["claude", "codex", "grok", "opencode"]); export type UsageProviderKind = typeof UsageProviderKind.Type; /** From 06de92c83f78c91c93b19271d4c0152270a37958 Mon Sep 17 00:00:00 2001 From: tris203 Date: Sun, 16 Aug 2026 15:48:32 +0100 Subject: [PATCH 02/15] fix(server): report OpenCode read failures and bump usage contract version A failed OpenCode SQLite open/query was flattened to zero usage and reported as status ok; propagate the null so the source reports failed. Adding the opencode provider kind also changes the wire schema, so bump USAGE_CONTRACT_VERSION to 5 for older clients to detect. --- apps/server/src/usage/UsageService.ts | 26 +++++++++++++------------- packages/contracts/src/usage.ts | 8 ++++---- packages/shared/src/usageMerge.test.ts | 3 ++- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index 7aff94442e5b..5e1875ecf567 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -317,15 +317,12 @@ export const make = Effect.gen(function* () { mtimeMs: number, provider: UsageProviderKind, windowStartMs?: number, - ): Effect.Effect => + ): Effect.Effect => Effect.gen(function* () { if (provider === "opencode") { - const parsed = yield* Effect.promise(() => - readOpenCodeRecords(filePath, windowStartMs ?? 0), - ); - // A read failure is not an empty transcript: reporting zero usage - // would silently drop the source's usage. - return parsed ?? []; + // A read failure is not an empty transcript: returning null lets the + // caller report the source as failed instead of zero usage. + return yield* Effect.promise(() => readOpenCodeRecords(filePath, windowStartMs ?? 0)); } const cached = fileCache.get(filePath); @@ -463,10 +460,12 @@ export const make = Effect.gen(function* () { provider, windowStartMs, ); + const failed = stats === null || records === null; + const scanned = records ?? []; // Distinct per database. Buckets carry per-cell session counts, but a // session spans days and models, so clients total this figure instead. const sessionIds = new Set(); - for (const record of records) { + for (const record of scanned) { // Only sessions that contributed in-window count: the query slack // admits boundary rows whose timestamps fall outside the range. if (aggregator.add(record) && record.sessionId.length > 0) { @@ -475,12 +474,12 @@ export const make = Effect.gen(function* () { } sources.push({ fingerprint: { hostId, provider, resolvedHomePath: dir, volumeId }, - status: stats === null ? "failed" : "ok", - scannedFiles: records.length > 0 ? 1 : 0, - skippedFiles: records.length > 0 ? 0 : 1, + status: failed ? "failed" : "ok", + scannedFiles: scanned.length > 0 ? 1 : 0, + skippedFiles: scanned.length > 0 ? 0 : 1, malformedRecords: 0, distinctSessions: sessionIds.size, - message: stats === null ? "Transcript database could not be read." : null, + message: failed ? "Transcript database could not be read." : null, }); continue; } @@ -498,7 +497,8 @@ export const make = Effect.gen(function* () { for (const file of files) { livePaths.add(file.path); const records = yield* readFileRecords(file.path, file.size, file.mtimeMs, provider); - if (records.length === 0) { + // A failed read is not an empty file: it is neither scanned nor cached. + if (records === null || records.length === 0) { skippedFiles += 1; continue; } diff --git a/packages/contracts/src/usage.ts b/packages/contracts/src/usage.ts index 552f3c269c6a..e64055d2a8a9 100644 --- a/packages/contracts/src/usage.ts +++ b/packages/contracts/src/usage.ts @@ -22,14 +22,14 @@ import { NonNegativeInt, TrimmedNonEmptyString } from "./baseSchemas.ts"; * client renders partial coverage when an environment reports an older version * rather than failing the whole page. */ -export const USAGE_CONTRACT_VERSION = 5 as const; +export const USAGE_CONTRACT_VERSION = 6 as const; /** * Oldest {@link UsageSummary} version a current client will still merge. * - * v5 only adds `grok` to {@link UsageProviderKind}; v4 Claude/Codex buckets - * remain valid, so mixed-version environments keep those totals instead of - * treating every older server as stale. + * v5 and v6 only add `grok` and `opencode` to {@link UsageProviderKind}; v4 + * Claude/Codex buckets remain valid, so mixed-version environments keep those + * totals instead of treating every older server as stale. */ export const USAGE_MERGE_COMPATIBLE_SINCE = 4 as const; diff --git a/packages/shared/src/usageMerge.test.ts b/packages/shared/src/usageMerge.test.ts index 6c706395c6ff..3342d773b8f1 100644 --- a/packages/shared/src/usageMerge.test.ts +++ b/packages/shared/src/usageMerge.test.ts @@ -1,5 +1,6 @@ import { USAGE_CONTRACT_VERSION, + USAGE_MERGE_COMPATIBLE_SINCE, type EnvironmentId, type UsageBucket, type UsageDay, @@ -158,7 +159,7 @@ describe("mergeUsage", () => { summary( [bucket()], [{ provider: "claude", hostId: "linux", homePath: "/b" }], - USAGE_CONTRACT_VERSION - 2, + USAGE_MERGE_COMPATIBLE_SINCE - 1, ), ), ], From b2037f0d5b612ceb99a348fe1ccc7ade2a414f6f Mon Sep 17 00:00:00 2001 From: tris203 Date: Sun, 16 Aug 2026 16:04:24 +0100 Subject: [PATCH 03/15] fix(server): resolve the OpenCode database where the CLI actually puts it The resolver invented a macOS Application Support branch and an OPENCODE_DATA_HOME override; the CLI resolves its data home through xdg-basedir on every platform (XDG_DATA_HOME or ~/.local/share/opencode) and the only documented override is OPENCODE_DB, which can be absolute or name a database inside the data home. Mirror that, including the channel-suffixed database caveat for dev installs. --- apps/server/src/usage/UsageService.ts | 40 +++++++++++++++------------ 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index 5e1875ecf567..cc8da337f91c 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -21,7 +21,7 @@ import { type UsageSummaryInput, UsageReadError, } from "@t3tools/contracts"; -import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import { HostProcessEnvironment } from "@t3tools/shared/hostProcess"; import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; @@ -202,27 +202,33 @@ export const make = Effect.gen(function* () { }); /** - * OpenCode keeps its transcripts in a SQLite database at - * `/opencode.db`. The data home follows XDG, so a Linux default - * install lives in `~/.local/share/opencode` while macOS uses - * `~/Library/Application Support/opencode`. `OPENCODE_DATA_HOME` wins when - * set, matching the CLI. + * OpenCode keeps its transcripts in a SQLite database under its XDG data + * home — `XDG_DATA_HOME/opencode` or `~/.local/share/opencode` on every + * platform, macOS included (the CLI resolves through `xdg-basedir`, which + * never uses `~/Library/Application Support`). + * + * Overrides mirror the CLI: an absolute `OPENCODE_DB` is the database file + * itself, and a relative `OPENCODE_DB` names a database inside the data + * home. Channel builds other than latest/beta/prod write + * `opencode-.db`; we cannot observe the channel from here, so a dev + * install's database is only found through `OPENCODE_DB`. */ const resolveOpenCodeDatabasePath = Effect.fn("UsageService.resolveOpenCodeDatabasePath")( function* () { - const platform = yield* HostProcessPlatform; const env = yield* HostProcessEnvironment; - const override = env["OPENCODE_DATA_HOME"]?.trim(); - if (override !== undefined && override.length > 0) { - return path.join(path.resolve(expandHomePath(override)), "opencode.db"); + const dataHome = path.join( + env["XDG_DATA_HOME"]?.trim() || path.join(NodeOS.homedir(), ".local", "share"), + "opencode", + ); + const dbOverride = env["OPENCODE_DB"]?.trim(); + if (dbOverride !== undefined && dbOverride.length > 0) { + // The CLI treats the value verbatim, but spawned processes get no + // shell expansion, so `OPENCODE_DB=~/...` would be read as relative; + // expand a leading `~` before the absolute check. + const expanded = expandHomePath(dbOverride); + if (expanded === ":memory:" || path.isAbsolute(expanded)) return expanded; + return path.join(dataHome, expanded); } - const dataHome = - platform === "darwin" - ? path.join(NodeOS.homedir(), "Library", "Application Support", "opencode") - : path.join( - env["XDG_DATA_HOME"] ?? path.join(NodeOS.homedir(), ".local", "share"), - "opencode", - ); return path.join(dataHome, "opencode.db"); }, ); From ff700e35944e693bbf9d3ede3004960dded8f517 Mon Sep 17 00:00:00 2001 From: tris203 Date: Sun, 16 Aug 2026 17:28:05 +0100 Subject: [PATCH 04/15] fix(server): support current OpenCode usage schema --- apps/server/src/usage/UsageService.ts | 4 +- .../server/src/usage/usageTranscriptReader.ts | 111 ++++++++++++++---- .../server/src/usage/usageTranscripts.test.ts | 100 ++++++++-------- apps/server/src/usage/usageTranscripts.ts | 87 +++++++------- docs/user/usage.md | 2 +- 5 files changed, 181 insertions(+), 123 deletions(-) diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index cc8da337f91c..1c8d78725f49 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -314,8 +314,8 @@ export const make = Effect.gen(function* () { * window-independent, which holds for the per-session JSONL transcripts. * OpenCode's source is one SQLite database queried with a window filter, so a * cached entry only ever covers the window it was scanned for and a wider - * window would silently reuse it. The windowed query is fast enough that the - * cache buys nothing, so OpenCode always scans fresh. + * window would silently reuse it. The scalar-only windowed query is fast + * enough that the cache buys nothing, so OpenCode always scans fresh. */ const readFileRecords = ( filePath: string, diff --git a/apps/server/src/usage/usageTranscriptReader.ts b/apps/server/src/usage/usageTranscriptReader.ts index fd9bb420e427..cea780d4a6c7 100644 --- a/apps/server/src/usage/usageTranscriptReader.ts +++ b/apps/server/src/usage/usageTranscriptReader.ts @@ -27,7 +27,8 @@ import { parseClaudeLine, parseCodexLine, parseGrokLine, - parseOpenCodeMessage, + parseOpenCodeUsageRow, + type OpenCodeUsageRow, type UsageRecord, } from "./usageTranscripts.ts"; @@ -162,17 +163,73 @@ export async function readTranscriptRecords( return records; } +/** + * OpenCode has kept usage on assistant messages across two projections: the + * legacy `message` table (role/model/tokens nested in `data`) and the current + * `session_message` table (a `type` column, model under `$.model.id`). Both + * select only usage scalars — message content never leaves the database. + */ +const OPEN_CODE_MESSAGE_TABLES_QUERY = ` + SELECT name FROM sqlite_master + WHERE type = 'table' AND name IN ('session_message', 'message') +`; + +const OPEN_CODE_LEGACY_USAGE_QUERY = ` + SELECT + id AS messageId, + session_id AS sessionId, + time_created AS timestampMs, + json_extract(data, '$.modelID') AS modelId, + json_extract(data, '$.tokens.input') AS inputTokens, + json_extract(data, '$.tokens.output') AS outputTokens, + json_extract(data, '$.tokens.reasoning') AS reasoningTokens, + json_extract(data, '$.tokens.cache.read') AS cacheReadTokens, + json_extract(data, '$.tokens.cache.write') AS cacheWriteTokens, + json_extract(data, '$.cost') AS costUsd + FROM message + WHERE time_updated >= ? + AND json_valid(data) + AND json_extract(data, '$.role') = 'assistant' + AND json_extract(data, '$.time.completed') IS NOT NULL +`; + +const OPEN_CODE_CURRENT_USAGE_QUERY = ` + SELECT + id AS messageId, + session_id AS sessionId, + time_created AS timestampMs, + json_extract(data, '$.model.id') AS modelId, + json_extract(data, '$.tokens.input') AS inputTokens, + json_extract(data, '$.tokens.output') AS outputTokens, + json_extract(data, '$.tokens.reasoning') AS reasoningTokens, + json_extract(data, '$.tokens.cache.read') AS cacheReadTokens, + json_extract(data, '$.tokens.cache.write') AS cacheWriteTokens, + json_extract(data, '$.cost') AS costUsd + FROM session_message + WHERE time_created >= ? + AND type = 'assistant' + AND json_valid(data) +`; + +const OPEN_CODE_TABLE_QUERIES = { + session_message: OPEN_CODE_CURRENT_USAGE_QUERY, + message: OPEN_CODE_LEGACY_USAGE_QUERY, +} as const; + +type OpenCodeMessageTable = keyof typeof OPEN_CODE_TABLE_QUERIES; + /** * Reads usage records from OpenCode's SQLite transcript store. * * Unlike the JSONL providers, OpenCode keeps one row per message in * `opencode.db`, so the whole source is one query. The window filter is pushed - * into SQL via `time_updated`, which covers in-progress messages that predate - * the window but complete inside it. The database is opened read-only, and + * into SQL (`time_updated` on the legacy table covers messages created before + * the window but completed inside it). The database is opened read-only, and * `-wal`/`-shm` siblings are never created because no write happens. * - * Returns `null` when the database cannot be read, so the caller reports the - * source as failed rather than zero usage. + * An upgraded database can carry the same message ID in both projections; the + * current `session_message` row wins. Returns `null` when the database cannot + * be read, so the caller reports the source as failed rather than zero usage. */ export async function readOpenCodeRecords( dbPath: string, @@ -186,25 +243,33 @@ export async function readOpenCodeRecords( } try { - const rows = database - .prepare( - `SELECT data FROM message - WHERE time_updated >= ? - AND json_extract(data, '$.role') = 'assistant' - AND json_extract(data, '$.tokens.total') > 0 - AND json_extract(data, '$.modelID') IS NOT NULL - AND json_extract(data, '$.time.completed') IS NOT NULL`, - ) - .all(sinceMs); - - const records: UsageRecord[] = []; - for (const row of rows) { - const data = (row as Record)["data"]; - if (typeof data !== "string") continue; - const record = parseOpenCodeMessage(data); - if (record !== null) records.push(record); + const tables = new Set(); + for (const row of database.prepare(OPEN_CODE_MESSAGE_TABLES_QUERY).all()) { + const name = (row as Record)["name"]; + if (name === "session_message" || name === "message") tables.add(name); + } + if (tables.size === 0) return null; + + const recordsByKey = new Map(); + let anonymous = 0; + for (const table of ["session_message", "message"] as const) { + if (!tables.has(table)) continue; + const rows = database.prepare(OPEN_CODE_TABLE_QUERIES[table]).all(sinceMs); + for (const row of rows) { + const record = parseOpenCodeUsageRow(row as OpenCodeUsageRow); + if (record === null) continue; + if (record.dedupeKey === null) { + // An anonymous record can still be unique; it just cannot dedupe + // across the two projections. + recordsByKey.set(`${table}#${anonymous++}`, record); + continue; + } + if (!recordsByKey.has(record.dedupeKey)) { + recordsByKey.set(record.dedupeKey, record); + } + } } - return records; + return [...recordsByKey.values()]; } catch { return null; } finally { diff --git a/apps/server/src/usage/usageTranscripts.test.ts b/apps/server/src/usage/usageTranscripts.test.ts index 424a3e57672d..3859a0c3e8ff 100644 --- a/apps/server/src/usage/usageTranscripts.test.ts +++ b/apps/server/src/usage/usageTranscripts.test.ts @@ -6,7 +6,7 @@ import { parseClaudeLine, parseCodexLine, parseGrokLine, - parseOpenCodeMessage, + parseOpenCodeUsageRow, totalTokens, } from "./usageTranscripts.ts"; @@ -239,45 +239,37 @@ describe("parseCodexLine", () => { }); }); -describe("parseOpenCodeMessage", () => { - /** Shaped after a real OpenCode assistant message row's `data` payload. */ - function openCodeMessage(overrides: { - id: string; - role?: string; - modelID?: string; - completed?: number | null; - cost?: number; - input?: number; - output?: number; - reasoning?: number; - cacheRead?: number; - cacheWrite?: number; - }): string { - return JSON.stringify({ - id: overrides.id, - sessionID: "ses_3a6c0a5d3ffeg7BPjptjftbHYs", - role: overrides.role ?? "assistant", - time: { - created: 1771023850034, - completed: - overrides.completed === null ? undefined : (overrides.completed ?? 1771023853436), - }, - modelID: overrides.modelID ?? "gpt-5.2-codex", - providerID: "github-copilot", - cost: overrides.cost ?? 0, - tokens: { - total: 9154, - input: overrides.input ?? 486, - output: overrides.output ?? 220, - reasoning: overrides.reasoning ?? 0, - cache: { read: overrides.cacheRead ?? 8448, write: overrides.cacheWrite ?? 0 }, - }, - finish: "stop", - }); +describe("parseOpenCodeUsageRow", () => { + /** Shaped after the scalar row the reader's SQL projects out of `data`. */ + function openCodeRow(overrides: { + messageId?: string | null; + sessionId?: string; + timestampMs?: number | null; + modelId?: string | null; + inputTokens?: number; + outputTokens?: number; + reasoningTokens?: number; + cacheReadTokens?: number; + cacheWriteTokens?: number; + costUsd?: number; + }): Record { + return { + messageId: overrides.messageId === null ? undefined : (overrides.messageId ?? "msg_1"), + sessionId: overrides.sessionId ?? "ses_3a6c0a5d3ffeg7BPjptjftbHYs", + timestampMs: + overrides.timestampMs === null ? undefined : (overrides.timestampMs ?? 1771023853436), + modelId: overrides.modelId === null ? undefined : (overrides.modelId ?? "gpt-5.2-codex"), + inputTokens: overrides.inputTokens ?? 486, + outputTokens: overrides.outputTokens ?? 220, + reasoningTokens: overrides.reasoningTokens ?? 0, + cacheReadTokens: overrides.cacheReadTokens ?? 8448, + cacheWriteTokens: overrides.cacheWriteTokens ?? 0, + costUsd: overrides.costUsd ?? 0, + }; } - it("extracts token totals from an assistant message", () => { - const record = parseOpenCodeMessage(openCodeMessage({ id: "msg_1", reasoning: 40 })); + it("extracts token totals from an assistant usage row", () => { + const record = parseOpenCodeUsageRow(openCodeRow({ messageId: "msg_1", reasoningTokens: 40 })); expect(record).not.toBeNull(); expect(record?.provider).toBe("opencode"); @@ -295,34 +287,38 @@ describe("parseOpenCodeMessage", () => { expect(record?.dedupeKey).toBe("msg_1"); }); - it("trusts a positive reported cost, and only a positive one", () => { - // OpenCode prices against its own rate table, which covers curated and - // subscription-served models LiteLLM does not know. - const priced = parseOpenCodeMessage(openCodeMessage({ id: "msg_2", cost: 0.023 })); + it("trusts a positive reported cost, and reprices a zero one", () => { + // OpenCode prices tokens against its own rate table, which covers curated + // and subscription-served models LiteLLM does not know; the figure is + // API-equivalent arithmetic, not plan billing. + const priced = parseOpenCodeUsageRow(openCodeRow({ costUsd: 0.023 })); expect(priced?.reportedCostUsd).toBe(0.023); // Subscription-backed providers leave cost at 0; those fall back to the // LiteLLM rate table like Codex. - const subscription = parseOpenCodeMessage(openCodeMessage({ id: "msg_2b", cost: 0 })); + const subscription = parseOpenCodeUsageRow(openCodeRow({ costUsd: 0 })); expect(subscription?.reportedCostUsd).toBeNull(); }); it("caps reasoning at output", () => { - const record = parseOpenCodeMessage( - openCodeMessage({ id: "msg_3", output: 10, reasoning: 99 }), - ); + const record = parseOpenCodeUsageRow(openCodeRow({ outputTokens: 10, reasoningTokens: 99 })); expect(record?.totals.reasoningTokens).toBe(10); }); - it("ignores user messages, unfinished turns, and token-less records", () => { - expect(parseOpenCodeMessage(openCodeMessage({ id: "msg_4", role: "user" }))).toBeNull(); - expect(parseOpenCodeMessage(openCodeMessage({ id: "msg_5", completed: null }))).toBeNull(); + it("ignores rows without a timestamp, model, or tokens", () => { + expect(parseOpenCodeUsageRow(openCodeRow({ timestampMs: null }))).toBeNull(); + expect(parseOpenCodeUsageRow(openCodeRow({ modelId: null }))).toBeNull(); expect( - parseOpenCodeMessage( - openCodeMessage({ id: "msg_6", input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }), + parseOpenCodeUsageRow( + openCodeRow({ inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }), ), ).toBeNull(); - expect(parseOpenCodeMessage("not json")).toBeNull(); + }); + + it("survives a missing message id with a null dedupe key", () => { + const record = parseOpenCodeUsageRow(openCodeRow({ messageId: null })); + expect(record).not.toBeNull(); + expect(record?.dedupeKey).toBeNull(); }); }); diff --git a/apps/server/src/usage/usageTranscripts.ts b/apps/server/src/usage/usageTranscripts.ts index ae4da162e616..f560ac6b1459 100644 --- a/apps/server/src/usage/usageTranscripts.ts +++ b/apps/server/src/usage/usageTranscripts.ts @@ -490,71 +490,68 @@ export function parseGrokLine(line: string): readonly UsageRecord[] { /* -------------------------------------------------------------------------- */ /** - * Projects one `data` payload from OpenCode's SQLite `message` table into a - * usage record. + * One usage row out of OpenCode's SQLite transcript store. * - * OpenCode moved its transcripts into `~/.local/share/opencode/opencode.db`: - * each row is one message with the payload stored as JSON in `data`. Usage - * sits on assistant messages as `tokens` with a `cache` breakdown; `input` is - * exclusive of the cached portions. The message `id` is the dedupe key. - * - * The `cost` field is trusted when positive: OpenCode prices against its own - * rate table, which covers its curated and subscription-served models that - * LiteLLM does not know. A zero cost on a subscription-backed provider falls - * back to LiteLLM like Codex. + * The reader selects only these scalars via `json_extract`; message content + * never leaves the database. */ -export function parseOpenCodeMessage(data: string): UsageRecord | null { - let parsed: unknown; - try { - parsed = JSON.parse(data); - } catch { - return null; - } - if (typeof parsed !== "object" || parsed === null) return null; +export interface OpenCodeUsageRow { + readonly messageId?: unknown; + readonly sessionId?: unknown; + readonly timestampMs?: unknown; + readonly modelId?: unknown; + readonly inputTokens?: unknown; + readonly outputTokens?: unknown; + readonly reasoningTokens?: unknown; + readonly cacheReadTokens?: unknown; + readonly cacheWriteTokens?: unknown; + readonly costUsd?: unknown; +} - const record = parsed as Record; - if (record["role"] !== "assistant") return null; - - const tokens = record["tokens"]; - if (typeof tokens !== "object" || tokens === null) return null; - const tokensRecord = tokens as Record; - const cache = - typeof tokensRecord["cache"] === "object" && tokensRecord["cache"] !== null - ? (tokensRecord["cache"] as Record) - : {}; - - const time = record["time"]; - const completed = - typeof time === "object" && time !== null - ? (time as Record)["completed"] - : undefined; +/** + * Projects one OpenCode usage row into a usage record. + * + * OpenCode moved its transcripts into `~/.local/share/opencode/opencode.db`, + * with one row per message. `input` is exclusive of the cached portions and + * `reasoning` is a subset of `output`, matching the shared token convention. + * The message `id` is the dedupe key. + * + * A positive `cost` is trusted: OpenCode prices tokens against its own rate + * table, which covers the curated and subscription-served models LiteLLM does + * not know, and the figure is API-equivalent arithmetic, not plan billing. A + * zero cost (subscription-backed providers leave it at 0) falls back to the + * LiteLLM rate table like Codex. + */ +export function parseOpenCodeUsageRow(row: OpenCodeUsageRow): UsageRecord | null { const timestampMs = - typeof completed === "number" && Number.isFinite(completed) ? Math.trunc(completed) : null; + typeof row.timestampMs === "number" && Number.isFinite(row.timestampMs) + ? Math.trunc(row.timestampMs) + : null; if (timestampMs === null) return null; - const model = typeof record["modelID"] === "string" ? record["modelID"] : ""; + const model = typeof row.modelId === "string" ? row.modelId : ""; if (model.length === 0) return null; const totals: UsageTokenTotals = { - uncachedInputTokens: int(tokensRecord["input"]), - cachedInputTokens: int(cache["read"]), - cacheCreationTokens: int(cache["write"]), - outputTokens: int(tokensRecord["output"]), - reasoningTokens: Math.min(int(tokensRecord["output"]), int(tokensRecord["reasoning"])), + uncachedInputTokens: int(row.inputTokens), + cachedInputTokens: int(row.cacheReadTokens), + cacheCreationTokens: int(row.cacheWriteTokens), + outputTokens: int(row.outputTokens), + reasoningTokens: Math.min(int(row.outputTokens), int(row.reasoningTokens)), }; if (totalTokens(totals) === 0) return null; - const cost = record["cost"]; + const cost = row.costUsd; return { provider: "opencode", timestampMs, model, - sessionId: typeof record["sessionID"] === "string" ? record["sessionID"] : "", + sessionId: typeof row.sessionId === "string" ? row.sessionId : "", totals, reportedCostUsd: typeof cost === "number" && Number.isFinite(cost) && cost > 0 ? cost : null, - dedupeKey: typeof record["id"] === "string" ? record["id"] : null, + dedupeKey: typeof row.messageId === "string" ? row.messageId : null, }; } diff --git a/docs/user/usage.md b/docs/user/usage.md index ff38c730c1cd..729d73ba025f 100644 --- a/docs/user/usage.md +++ b/docs/user/usage.md @@ -1,6 +1,6 @@ # Review usage -The Usage page combines Codex, Claude Code, and Grok Build activity from your connected +The Usage page combines Codex, Claude Code, Grok Build, and OpenCode activity from your connected environments. It reads the providers' local session history and shows API-equivalent token cost, processed tokens, cache savings, provider shares, and model breakdowns. Subscription billing is separate from the raw token cost shown here. From cbb3ebb244c3f9ce4a7691dc1f5d78831fbf3a8c Mon Sep 17 00:00:00 2001 From: tris203 Date: Sun, 16 Aug 2026 17:43:07 +0100 Subject: [PATCH 05/15] fix(server): bucket OpenCode usage by completion time --- .../src/usage/usageTranscriptReader.test.ts | 99 +++++++++++++++++++ .../server/src/usage/usageTranscriptReader.ts | 15 +-- .../server/src/usage/usageTranscripts.test.ts | 1 + apps/server/src/usage/usageTranscripts.ts | 6 +- 4 files changed, 112 insertions(+), 9 deletions(-) create mode 100644 apps/server/src/usage/usageTranscriptReader.test.ts diff --git a/apps/server/src/usage/usageTranscriptReader.test.ts b/apps/server/src/usage/usageTranscriptReader.test.ts new file mode 100644 index 000000000000..5d7617098804 --- /dev/null +++ b/apps/server/src/usage/usageTranscriptReader.test.ts @@ -0,0 +1,99 @@ +// @effect-diagnostics nodeBuiltinImport:off +import { mkdtempSync, rmSync } from "node:fs"; +import * as NodePath from "node:path"; +import * as NodeSqlite from "node:sqlite"; +import { tmpdir } from "node:os"; + +import { describe, expect, it } from "@effect/vitest"; + +import { readOpenCodeRecords } from "./usageTranscriptReader.ts"; + +function createDatabase(): string { + const directory = mkdtempSync(NodePath.join(tmpdir(), "t3-opencode-")); + const dbPath = NodePath.join(directory, "opencode.db"); + const database = new NodeSqlite.DatabaseSync(dbPath); + database.exec(` + CREATE TABLE message ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + time_created INTEGER NOT NULL, + time_updated INTEGER NOT NULL, + data TEXT NOT NULL + ); + CREATE TABLE session_message ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + type TEXT NOT NULL, + time_created INTEGER NOT NULL, + time_updated INTEGER NOT NULL, + data TEXT NOT NULL + ); + `); + + const legacy = database.prepare( + "INSERT INTO message (id, session_id, time_created, time_updated, data) VALUES (?, ?, ?, ?, ?)", + ); + legacy.run( + "legacy-completed", + "session-1", + 1_000, + 2_500, + JSON.stringify({ + role: "assistant", + modelID: "legacy-model", + time: { created: 1_000, completed: 2_500 }, + tokens: { input: 10, output: 5, reasoning: 0, cache: { read: 0, write: 0 } }, + cost: 0, + }), + ); + + const current = database.prepare( + "INSERT INTO session_message (id, session_id, type, time_created, time_updated, data) VALUES (?, ?, ?, ?, ?, ?)", + ); + current.run( + "current-incomplete", + "session-1", + "assistant", + 3_000, + 3_000, + JSON.stringify({ + model: { id: "current-model" }, + time: { created: 3_000 }, + tokens: { input: 20, output: 10, reasoning: 0, cache: { read: 0, write: 0 } }, + cost: 0, + }), + ); + current.run( + "current-completed", + "session-1", + "assistant", + 4_000, + 4_500, + JSON.stringify({ + model: { id: "current-model" }, + time: { created: 4_000, completed: 4_500 }, + tokens: { input: 30, output: 15, reasoning: 0, cache: { read: 0, write: 0 } }, + cost: 0, + }), + ); + database.close(); + return dbPath; +} + +describe("readOpenCodeRecords", () => { + it("uses completion time and excludes incomplete current assistant rows", async () => { + const dbPath = createDatabase(); + try { + const records = await readOpenCodeRecords(dbPath, 2_000); + + expect(records).toHaveLength(2); + if (records === null) throw new Error("Expected the OpenCode database to be readable"); + expect(records.map((record) => [record.dedupeKey, record.timestampMs])).toEqual([ + ["current-completed", 4_500], + ["legacy-completed", 2_500], + ]); + } finally { + rmSync(NodePath.dirname(dbPath), { recursive: true, force: true }); + } + }); +}); diff --git a/apps/server/src/usage/usageTranscriptReader.ts b/apps/server/src/usage/usageTranscriptReader.ts index cea780d4a6c7..731653aec86a 100644 --- a/apps/server/src/usage/usageTranscriptReader.ts +++ b/apps/server/src/usage/usageTranscriptReader.ts @@ -167,7 +167,8 @@ export async function readTranscriptRecords( * OpenCode has kept usage on assistant messages across two projections: the * legacy `message` table (role/model/tokens nested in `data`) and the current * `session_message` table (a `type` column, model under `$.model.id`). Both - * select only usage scalars — message content never leaves the database. + * select only usage scalars and the completed timestamp — message content + * never leaves the database. */ const OPEN_CODE_MESSAGE_TABLES_QUERY = ` SELECT name FROM sqlite_master @@ -178,7 +179,7 @@ const OPEN_CODE_LEGACY_USAGE_QUERY = ` SELECT id AS messageId, session_id AS sessionId, - time_created AS timestampMs, + json_extract(data, '$.time.completed') AS timestampMs, json_extract(data, '$.modelID') AS modelId, json_extract(data, '$.tokens.input') AS inputTokens, json_extract(data, '$.tokens.output') AS outputTokens, @@ -187,7 +188,7 @@ const OPEN_CODE_LEGACY_USAGE_QUERY = ` json_extract(data, '$.tokens.cache.write') AS cacheWriteTokens, json_extract(data, '$.cost') AS costUsd FROM message - WHERE time_updated >= ? + WHERE json_extract(data, '$.time.completed') >= ? AND json_valid(data) AND json_extract(data, '$.role') = 'assistant' AND json_extract(data, '$.time.completed') IS NOT NULL @@ -197,7 +198,7 @@ const OPEN_CODE_CURRENT_USAGE_QUERY = ` SELECT id AS messageId, session_id AS sessionId, - time_created AS timestampMs, + json_extract(data, '$.time.completed') AS timestampMs, json_extract(data, '$.model.id') AS modelId, json_extract(data, '$.tokens.input') AS inputTokens, json_extract(data, '$.tokens.output') AS outputTokens, @@ -206,7 +207,7 @@ const OPEN_CODE_CURRENT_USAGE_QUERY = ` json_extract(data, '$.tokens.cache.write') AS cacheWriteTokens, json_extract(data, '$.cost') AS costUsd FROM session_message - WHERE time_created >= ? + WHERE json_extract(data, '$.time.completed') >= ? AND type = 'assistant' AND json_valid(data) `; @@ -223,8 +224,8 @@ type OpenCodeMessageTable = keyof typeof OPEN_CODE_TABLE_QUERIES; * * Unlike the JSONL providers, OpenCode keeps one row per message in * `opencode.db`, so the whole source is one query. The window filter is pushed - * into SQL (`time_updated` on the legacy table covers messages created before - * the window but completed inside it). The database is opened read-only, and + * into SQL using the completed timestamp, so usage is attributed to the hour + * or day the turn finished. The database is opened read-only, and * `-wal`/`-shm` siblings are never created because no write happens. * * An upgraded database can carry the same message ID in both projections; the diff --git a/apps/server/src/usage/usageTranscripts.test.ts b/apps/server/src/usage/usageTranscripts.test.ts index 3859a0c3e8ff..a8e4f15a0452 100644 --- a/apps/server/src/usage/usageTranscripts.test.ts +++ b/apps/server/src/usage/usageTranscripts.test.ts @@ -244,6 +244,7 @@ describe("parseOpenCodeUsageRow", () => { function openCodeRow(overrides: { messageId?: string | null; sessionId?: string; + /** The assistant turn's completed timestamp, not its creation time. */ timestampMs?: number | null; modelId?: string | null; inputTokens?: number; diff --git a/apps/server/src/usage/usageTranscripts.ts b/apps/server/src/usage/usageTranscripts.ts index f560ac6b1459..fdc0e09139eb 100644 --- a/apps/server/src/usage/usageTranscripts.ts +++ b/apps/server/src/usage/usageTranscripts.ts @@ -512,8 +512,10 @@ export interface OpenCodeUsageRow { * Projects one OpenCode usage row into a usage record. * * OpenCode moved its transcripts into `~/.local/share/opencode/opencode.db`, - * with one row per message. `input` is exclusive of the cached portions and - * `reasoning` is a subset of `output`, matching the shared token convention. + * with one row per message. The timestamp is the assistant turn's completion + * time, so usage is attributed to when the work finished. `input` is exclusive + * of the cached portions and `reasoning` is a subset of `output`, matching the + * shared token convention. * The message `id` is the dedupe key. * * A positive `cost` is trusted: OpenCode prices tokens against its own rate From 88f69d27f31fc73399156afddca8f168d80be34a Mon Sep 17 00:00:00 2001 From: tris203 Date: Sun, 16 Aug 2026 17:47:10 +0100 Subject: [PATCH 06/15] test(server): follow namespace import lint convention --- apps/server/src/usage/usageTranscriptReader.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/server/src/usage/usageTranscriptReader.test.ts b/apps/server/src/usage/usageTranscriptReader.test.ts index 5d7617098804..75d96787b441 100644 --- a/apps/server/src/usage/usageTranscriptReader.test.ts +++ b/apps/server/src/usage/usageTranscriptReader.test.ts @@ -1,15 +1,15 @@ // @effect-diagnostics nodeBuiltinImport:off -import { mkdtempSync, rmSync } from "node:fs"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import * as NodeSqlite from "node:sqlite"; -import { tmpdir } from "node:os"; import { describe, expect, it } from "@effect/vitest"; import { readOpenCodeRecords } from "./usageTranscriptReader.ts"; function createDatabase(): string { - const directory = mkdtempSync(NodePath.join(tmpdir(), "t3-opencode-")); + const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-opencode-")); const dbPath = NodePath.join(directory, "opencode.db"); const database = new NodeSqlite.DatabaseSync(dbPath); database.exec(` @@ -93,7 +93,7 @@ describe("readOpenCodeRecords", () => { ["legacy-completed", 2_500], ]); } finally { - rmSync(NodePath.dirname(dbPath), { recursive: true, force: true }); + NodeFS.rmSync(NodePath.dirname(dbPath), { recursive: true, force: true }); } }); }); From 9be28dfd46b73dc38fcb1ebb4292abc4e883d037 Mon Sep 17 00:00:00 2001 From: tris203 Date: Sun, 23 Aug 2026 10:12:34 +0100 Subject: [PATCH 07/15] fix(usage): prefer healthy shared transcript sources --- apps/server/src/usage/UsageService.ts | 18 +++++-- .../src/usage/usageTranscriptReader.test.ts | 31 ++++++++++- .../server/src/usage/usageTranscriptReader.ts | 20 +++++-- packages/shared/src/usageMerge.test.ts | 52 ++++++++++++++++++- packages/shared/src/usageMerge.ts | 42 ++++++++++++--- 5 files changed, 147 insertions(+), 16 deletions(-) diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index 1c8d78725f49..9b32e5245b21 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -491,11 +491,13 @@ export const make = Effect.gen(function* () { } walkedRoots.push(dir); - const files = yield* Effect.promise(() => + const listing = yield* Effect.promise(() => listTranscriptFiles(dir, windowStartMs, fileName === undefined ? undefined : { fileName }), ); + const files = listing.files; let scannedFiles = 0; let skippedFiles = 0; + let failedFiles = 0; // Distinct per directory. Buckets carry per-cell session counts, but a // session spans days and models, so clients total this figure instead. const sessionIds = new Set(); @@ -504,7 +506,12 @@ export const make = Effect.gen(function* () { livePaths.add(file.path); const records = yield* readFileRecords(file.path, file.size, file.mtimeMs, provider); // A failed read is not an empty file: it is neither scanned nor cached. - if (records === null || records.length === 0) { + if (records === null) { + failedFiles += 1; + skippedFiles += 1; + continue; + } + if (records.length === 0) { skippedFiles += 1; continue; } @@ -520,12 +527,15 @@ export const make = Effect.gen(function* () { sources.push({ fingerprint: { hostId, provider, resolvedHomePath: dir, volumeId }, - status: "ok", + status: listing.hadReadError || failedFiles > 0 ? "partial" : "ok", scannedFiles, skippedFiles, malformedRecords: 0, distinctSessions: sessionIds.size, - message: null, + message: + listing.hadReadError || failedFiles > 0 + ? "Some transcript files could not be read." + : null, }); } diff --git a/apps/server/src/usage/usageTranscriptReader.test.ts b/apps/server/src/usage/usageTranscriptReader.test.ts index 75d96787b441..63d66987bf53 100644 --- a/apps/server/src/usage/usageTranscriptReader.test.ts +++ b/apps/server/src/usage/usageTranscriptReader.test.ts @@ -1,12 +1,13 @@ // @effect-diagnostics nodeBuiltinImport:off import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; +import * as NodePerformance from "node:perf_hooks"; import * as NodePath from "node:path"; import * as NodeSqlite from "node:sqlite"; import { describe, expect, it } from "@effect/vitest"; -import { readOpenCodeRecords } from "./usageTranscriptReader.ts"; +import { listTranscriptFiles, readOpenCodeRecords } from "./usageTranscriptReader.ts"; function createDatabase(): string { const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-opencode-")); @@ -96,4 +97,32 @@ describe("readOpenCodeRecords", () => { NodeFS.rmSync(NodePath.dirname(dbPath), { recursive: true, force: true }); } }); + + it("waits briefly for a writer before reporting a locked database", async () => { + const dbPath = createDatabase(); + const writer = new NodeSqlite.DatabaseSync(dbPath); + writer.exec("BEGIN EXCLUSIVE"); + const startedAt = NodePerformance.performance.now(); + try { + expect(await readOpenCodeRecords(dbPath, 2_000)).toBeNull(); + expect(NodePerformance.performance.now() - startedAt).toBeGreaterThanOrEqual(500); + } finally { + writer.exec("ROLLBACK"); + writer.close(); + NodeFS.rmSync(NodePath.dirname(dbPath), { recursive: true, force: true }); + } + }); +}); + +describe("listTranscriptFiles", () => { + it("reports a traversal error separately from an empty transcript set", async () => { + const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-transcripts-")); + try { + const listing = await listTranscriptFiles(NodePath.join(directory, "missing"), 0); + expect(listing.files).toEqual([]); + expect(listing.hadReadError).toBe(true); + } finally { + NodeFS.rmSync(directory, { recursive: true, force: true }); + } + }); }); diff --git a/apps/server/src/usage/usageTranscriptReader.ts b/apps/server/src/usage/usageTranscriptReader.ts index 731653aec86a..9b83f49b60f4 100644 --- a/apps/server/src/usage/usageTranscriptReader.ts +++ b/apps/server/src/usage/usageTranscriptReader.ts @@ -38,6 +38,11 @@ export interface TranscriptFile { readonly mtimeMs: number; } +export interface TranscriptFileListing { + readonly files: readonly TranscriptFile[]; + readonly hadReadError: boolean; +} + /** * Lists `.jsonl` transcripts under `root` last modified at or after `sinceMs`. * @@ -53,8 +58,9 @@ export async function listTranscriptFiles( root: string, sinceMs: number, options?: { readonly fileName?: string }, -): Promise { +): Promise { const found: TranscriptFile[] = []; + let hadReadError = false; const fileName = options?.fileName; const walk = async (dir: string): Promise => { @@ -62,6 +68,7 @@ export async function listTranscriptFiles( try { entries = await NodeFSP.readdir(dir, { withFileTypes: true }); } catch { + hadReadError = true; return; } for (const entry of entries) { @@ -87,7 +94,7 @@ export async function listTranscriptFiles( }; await walk(root); - return found; + return { files: found, hadReadError }; } /** @@ -194,6 +201,10 @@ const OPEN_CODE_LEGACY_USAGE_QUERY = ` AND json_extract(data, '$.time.completed') IS NOT NULL `; +// OpenCode writes this database while usage is being read. A short busy +// timeout avoids treating a normal WAL transaction as a failed source. +const OPEN_CODE_SQLITE_BUSY_TIMEOUT_MS = 1_000; + const OPEN_CODE_CURRENT_USAGE_QUERY = ` SELECT id AS messageId, @@ -238,7 +249,10 @@ export async function readOpenCodeRecords( ): Promise { let database: NodeSqlite.DatabaseSync; try { - database = new NodeSqlite.DatabaseSync(dbPath, { readOnly: true }); + database = new NodeSqlite.DatabaseSync(dbPath, { + readOnly: true, + timeout: OPEN_CODE_SQLITE_BUSY_TIMEOUT_MS, + }); } catch { return null; } diff --git a/packages/shared/src/usageMerge.test.ts b/packages/shared/src/usageMerge.test.ts index 3342d773b8f1..656ee1c643f2 100644 --- a/packages/shared/src/usageMerge.test.ts +++ b/packages/shared/src/usageMerge.test.ts @@ -5,6 +5,7 @@ import { type UsageBucket, type UsageDay, type UsageProviderKind, + type UsageSourceStatus, type UsageSummary, } from "@t3tools/contracts"; import { describe, expect, it } from "vite-plus/test"; @@ -41,6 +42,7 @@ function summary( homePath: string; volumeId?: string; distinctSessions?: number; + status?: UsageSourceStatus; }[], contractVersion: number = USAGE_CONTRACT_VERSION, ): UsageSummary { @@ -58,7 +60,7 @@ function summary( resolvedHomePath: source.homePath, volumeId: source.volumeId ?? `vol-${source.hostId}`, }, - status: "ok" as const, + status: source.status ?? "ok", scannedFiles: 1, skippedFiles: 0, malformedRecords: 0, @@ -257,6 +259,54 @@ describe("mergeUsage", () => { expect(merged.duplicateSources).toHaveLength(1); }); + it("does not let a failed source hide a healthy shared source", () => { + const shared = { + provider: "opencode" as const, + hostId: "mac", + homePath: "/Users/theo/.local/share/opencode/opencode.db", + }; + const merged = mergeUsage( + [ + environment("env-a", summary([], [{ ...shared, status: "failed" }])), + environment("env-b", summary([bucket({ provider: "opencode" })], [shared])), + ], + USAGE_CONTRACT_VERSION, + ); + + expect(merged.costUsd).toBe(10); + expect(merged.records).toBe(5); + expect(merged.duplicateSources).toHaveLength(0); + expect(merged.contributingEnvironments).toEqual(["env-b"]); + }); + + it("prefers a healthy source over a partial duplicate", () => { + const shared = { + provider: "opencode" as const, + hostId: "mac", + homePath: "/Users/theo/.local/share/opencode/opencode.db", + }; + const merged = mergeUsage( + [ + environment( + "env-a", + summary( + [bucket({ provider: "opencode", costUsd: 1 })], + [{ ...shared, status: "partial" }], + ), + ), + environment("env-b", summary([bucket({ provider: "opencode" })], [shared])), + ], + USAGE_CONTRACT_VERSION, + ); + + expect(merged.costUsd).toBe(10); + expect(merged.records).toBe(5); + expect(merged.duplicateSources).toEqual([ + "env-a: /Users/theo/.local/share/opencode/opencode.db", + ]); + expect(merged.contributingEnvironments).toEqual(["env-b"]); + }); + it("totals sessions from per-directory distinct counts, not per-bucket sums", () => { // One session that spans two days appears in two buckets. Summing bucket // sessions would say 2; the source's distinct count says 1. diff --git a/packages/shared/src/usageMerge.ts b/packages/shared/src/usageMerge.ts index 428599d51c74..ecab625a9838 100644 --- a/packages/shared/src/usageMerge.ts +++ b/packages/shared/src/usageMerge.ts @@ -11,6 +11,7 @@ import { type EnvironmentId, type UsageBucket, type UsageProviderKind, + type UsageSource, type UsageSourceFingerprint, type UsageSummary, } from "@t3tools/contracts"; @@ -100,13 +101,26 @@ function fingerprintKey(fingerprint: UsageSourceFingerprint): string { ].join(" "); } +/** Only sources with usable data may claim a shared transcript directory. */ +function sourceClaimRank(source: UsageSource): number | null { + switch (source.status) { + case "ok": + return 2; + case "partial": + return 1; + case "missing": + case "failed": + return null; + } +} + /** * Decides which environment owns each physical transcript directory. * * Several environments on one machine (worktree servers, for instance) resolve * the same provider home and would otherwise double count every token. The - * first environment in a stable order claims a fingerprint; the rest have that - * provider's buckets dropped. Environments are sorted by id so the winner does + * The healthiest source claims a fingerprint; the rest have that provider's + * buckets dropped. Ties are resolved by environment ID so the winner does * not change between renders. */ function claimSources(environments: readonly EnvironmentUsage[]): { @@ -114,19 +128,33 @@ function claimSources(environments: readonly EnvironmentUsage[]): { readonly duplicates: readonly string[]; } { const ownerByFingerprint = new Map(); + const ownerLabelByFingerprint = new Map(); + const ownerRankByFingerprint = new Map(); const duplicates: string[] = []; const ordered = [...environments].sort((a, b) => a.environmentId.localeCompare(b.environmentId)); for (const environment of ordered) { for (const source of environment.summary.sources) { - if (source.status === "missing") continue; + const rank = sourceClaimRank(source); + if (rank === null) continue; const key = fingerprintKey(source.fingerprint); - if (ownerByFingerprint.has(key)) { + const ownerRank = ownerRankByFingerprint.get(key); + if (ownerRank === undefined) { + ownerByFingerprint.set(key, environment.environmentId); + ownerLabelByFingerprint.set(key, environment.label); + ownerRankByFingerprint.set(key, rank); + } else if (rank > ownerRank) { + const previousOwnerLabel = ownerLabelByFingerprint.get(key); + if (previousOwnerLabel !== undefined) { + duplicates.push(`${previousOwnerLabel}: ${source.fingerprint.resolvedHomePath}`); + } + ownerByFingerprint.set(key, environment.environmentId); + ownerLabelByFingerprint.set(key, environment.label); + ownerRankByFingerprint.set(key, rank); + } else { duplicates.push(`${environment.label}: ${source.fingerprint.resolvedHomePath}`); - continue; } - ownerByFingerprint.set(key, environment.environmentId); } } @@ -144,7 +172,7 @@ function ownedContribution( const ownedProviders = new Set(); const sessionsByProvider = new Map(); for (const source of environment.summary.sources) { - if (source.status === "missing") continue; + if (sourceClaimRank(source) === null) continue; const key = fingerprintKey(source.fingerprint); if (ownerByFingerprint.get(key) === environment.environmentId) { const provider = source.fingerprint.provider; From 409ac1131e3ccd60cfa75ac7819575c8460c5f01 Mon Sep 17 00:00:00 2001 From: tris203 Date: Sun, 23 Aug 2026 10:39:49 +0100 Subject: [PATCH 08/15] fix(usage): preserve partial transcript scans --- apps/server/src/usage/UsageService.test.ts | 71 +++++++++++++++++++ apps/server/src/usage/UsageService.ts | 2 +- .../src/usage/usageTranscriptReader.test.ts | 36 +++++++++- .../server/src/usage/usageTranscriptReader.ts | 15 ++-- 4 files changed, 114 insertions(+), 10 deletions(-) create mode 100644 apps/server/src/usage/UsageService.test.ts diff --git a/apps/server/src/usage/UsageService.test.ts b/apps/server/src/usage/UsageService.test.ts new file mode 100644 index 000000000000..cd0f78c90636 --- /dev/null +++ b/apps/server/src/usage/UsageService.test.ts @@ -0,0 +1,71 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeFS from "node:fs"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { UsageDay } from "@t3tools/contracts"; +import { HostProcessEnvironment } from "@t3tools/shared/hostProcess"; +import { expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as HttpClient from "effect/unstable/http/HttpClient"; +import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"; + +import * as ServerConfig from "../config.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import * as UsageService from "./UsageService.ts"; + +const EmptyRatesHttpClient = Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.succeed(HttpClientResponse.fromWeb(request, Response.json({}))), + ), +); + +it.effect("reports a JSONL source as partial when a transcript cannot be read", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-usage-service-" }); + const claudeHome = path.join(root, "claude"); + const claudeProjects = path.join(claudeHome, "projects"); + const unreadableTranscript = path.join(claudeProjects, "session.jsonl"); + yield* fileSystem.makeDirectory(claudeProjects, { recursive: true }); + yield* fileSystem.writeFileString(unreadableTranscript, "{}"); + NodeFS.chmodSync(unreadableTranscript, 0); + + const usageService = yield* UsageService.make.pipe( + Effect.provide( + Layer.mergeAll( + ServerSettings.layerTest({ + providers: { + claudeAgent: { homePath: claudeHome }, + codex: { homePath: path.join(root, "codex") }, + }, + }), + ServerConfig.layerTest(process.cwd(), path.join(root, "t3-home")), + EmptyRatesHttpClient, + Layer.succeed(HostProcessEnvironment, { + OPENCODE_DB: path.join(root, "missing-opencode.db"), + }), + ), + ), + ); + + const summary = yield* usageService.readSummary({ + sinceDay: UsageDay.make("2026-08-22"), + untilDay: UsageDay.make("2026-08-23"), + timeZone: "UTC", + }); + + expect( + summary.sources.find((source) => source.fingerprint.provider === "claude"), + ).toMatchObject({ + status: "partial", + scannedFiles: 0, + skippedFiles: 1, + message: "Some transcript files could not be read.", + }); + }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), +); diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index 9b32e5245b21..7c8bf10c80f3 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -347,7 +347,7 @@ export const make = Effect.gen(function* () { const parsed = yield* Effect.promise(() => readTranscriptRecords(filePath, provider)); // A read failure is not an empty transcript: caching it under this // (size, mtime) would silently drop the file's usage until it changes. - if (parsed === null) return []; + if (parsed === null) return null; // Stored already de-duplicated within the file, which is 99% of all // duplicates. The aggregator still runs the cross-file dedupe pass. const records = dedupeWithinFile(parsed); diff --git a/apps/server/src/usage/usageTranscriptReader.test.ts b/apps/server/src/usage/usageTranscriptReader.test.ts index 63d66987bf53..3e444b84f509 100644 --- a/apps/server/src/usage/usageTranscriptReader.test.ts +++ b/apps/server/src/usage/usageTranscriptReader.test.ts @@ -1,8 +1,8 @@ // @effect-diagnostics nodeBuiltinImport:off import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; -import * as NodePerformance from "node:perf_hooks"; import * as NodePath from "node:path"; +import * as NodePerfHooks from "node:perf_hooks"; import * as NodeSqlite from "node:sqlite"; import { describe, expect, it } from "@effect/vitest"; @@ -98,14 +98,44 @@ describe("readOpenCodeRecords", () => { } }); + it("skips malformed JSON rows without discarding valid usage", async () => { + const dbPath = createDatabase(); + const database = new NodeSqlite.DatabaseSync(dbPath); + try { + database + .prepare( + "INSERT INTO message (id, session_id, time_created, time_updated, data) VALUES (?, ?, ?, ?, ?)", + ) + .run("legacy-malformed", "session-1", 5_000, 5_000, "{not-json"); + database + .prepare( + "INSERT INTO session_message (id, session_id, type, time_created, time_updated, data) VALUES (?, ?, ?, ?, ?, ?)", + ) + .run("current-malformed", "session-1", "assistant", 5_000, 5_000, "{not-json"); + } finally { + database.close(); + } + + try { + const records = await readOpenCodeRecords(dbPath, 2_000); + + expect(records?.map((record) => record.dedupeKey)).toEqual([ + "current-completed", + "legacy-completed", + ]); + } finally { + NodeFS.rmSync(NodePath.dirname(dbPath), { recursive: true, force: true }); + } + }); + it("waits briefly for a writer before reporting a locked database", async () => { const dbPath = createDatabase(); const writer = new NodeSqlite.DatabaseSync(dbPath); writer.exec("BEGIN EXCLUSIVE"); - const startedAt = NodePerformance.performance.now(); + const startedAt = NodePerfHooks.performance.now(); try { expect(await readOpenCodeRecords(dbPath, 2_000)).toBeNull(); - expect(NodePerformance.performance.now() - startedAt).toBeGreaterThanOrEqual(500); + expect(NodePerfHooks.performance.now() - startedAt).toBeGreaterThanOrEqual(500); } finally { writer.exec("ROLLBACK"); writer.close(); diff --git a/apps/server/src/usage/usageTranscriptReader.ts b/apps/server/src/usage/usageTranscriptReader.ts index 9b83f49b60f4..588247185b7b 100644 --- a/apps/server/src/usage/usageTranscriptReader.ts +++ b/apps/server/src/usage/usageTranscriptReader.ts @@ -195,10 +195,12 @@ const OPEN_CODE_LEGACY_USAGE_QUERY = ` json_extract(data, '$.tokens.cache.write') AS cacheWriteTokens, json_extract(data, '$.cost') AS costUsd FROM message - WHERE json_extract(data, '$.time.completed') >= ? - AND json_valid(data) - AND json_extract(data, '$.role') = 'assistant' - AND json_extract(data, '$.time.completed') IS NOT NULL + WHERE CASE + WHEN json_valid(data) THEN json_extract(data, '$.time.completed') + END >= ? + AND CASE + WHEN json_valid(data) THEN json_extract(data, '$.role') + END = 'assistant' `; // OpenCode writes this database while usage is being read. A short busy @@ -218,9 +220,10 @@ const OPEN_CODE_CURRENT_USAGE_QUERY = ` json_extract(data, '$.tokens.cache.write') AS cacheWriteTokens, json_extract(data, '$.cost') AS costUsd FROM session_message - WHERE json_extract(data, '$.time.completed') >= ? + WHERE CASE + WHEN json_valid(data) THEN json_extract(data, '$.time.completed') + END >= ? AND type = 'assistant' - AND json_valid(data) `; const OPEN_CODE_TABLE_QUERIES = { From e8bf7e06681df3f461697dfd04bc9372c2683000 Mon Sep 17 00:00:00 2001 From: tris203 Date: Mon, 24 Aug 2026 19:39:06 +0100 Subject: [PATCH 09/15] fix(server): report transcript stat failures --- apps/server/src/usage/usageTranscriptReader.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/server/src/usage/usageTranscriptReader.ts b/apps/server/src/usage/usageTranscriptReader.ts index 588247185b7b..9ce99a770532 100644 --- a/apps/server/src/usage/usageTranscriptReader.ts +++ b/apps/server/src/usage/usageTranscriptReader.ts @@ -89,6 +89,7 @@ export async function listTranscriptFiles( } } catch { // Vanished between readdir and stat. + hadReadError = true; } } }; From 93bf709ad209dc1719af223362e70170f6355eda Mon Sep 17 00:00:00 2001 From: tris203 Date: Mon, 24 Aug 2026 19:48:36 +0100 Subject: [PATCH 10/15] fix(server): tolerate rotated transcripts --- .../src/usage/usageTranscriptReader.test.ts | 35 +++++++++++++++++++ .../server/src/usage/usageTranscriptReader.ts | 9 +++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/apps/server/src/usage/usageTranscriptReader.test.ts b/apps/server/src/usage/usageTranscriptReader.test.ts index 3e444b84f509..548b2a50c33f 100644 --- a/apps/server/src/usage/usageTranscriptReader.test.ts +++ b/apps/server/src/usage/usageTranscriptReader.test.ts @@ -155,4 +155,39 @@ describe("listTranscriptFiles", () => { NodeFS.rmSync(directory, { recursive: true, force: true }); } }); + + it.skipIf(process.platform === "win32")( + "ignores transcripts that vanish while the directory is being listed", + async () => { + const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-transcripts-")); + try { + NodeFS.symlinkSync( + NodePath.join(directory, "vanished"), + NodePath.join(directory, "gone.jsonl"), + ); + + const listing = await listTranscriptFiles(directory, 0); + + expect(listing.files).toEqual([]); + expect(listing.hadReadError).toBe(false); + } finally { + NodeFS.rmSync(directory, { recursive: true, force: true }); + } + }, + ); + + it.skipIf(process.platform === "win32")("reports non-missing stat failures", async () => { + const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-transcripts-")); + const transcript = NodePath.join(directory, "loop.jsonl"); + try { + NodeFS.symlinkSync(transcript, transcript); + + const listing = await listTranscriptFiles(directory, 0); + + expect(listing.files).toEqual([]); + expect(listing.hadReadError).toBe(true); + } finally { + NodeFS.rmSync(directory, { recursive: true, force: true }); + } + }); }); diff --git a/apps/server/src/usage/usageTranscriptReader.ts b/apps/server/src/usage/usageTranscriptReader.ts index 9ce99a770532..2210bbf8151c 100644 --- a/apps/server/src/usage/usageTranscriptReader.ts +++ b/apps/server/src/usage/usageTranscriptReader.ts @@ -87,9 +87,12 @@ export async function listTranscriptFiles( if (stats.mtimeMs >= sinceMs) { found.push({ path: child, size: stats.size, mtimeMs: stats.mtimeMs }); } - } catch { - // Vanished between readdir and stat. - hadReadError = true; + } catch (cause) { + // Rotating transcripts can vanish between readdir and stat. Other + // failures mean the listing may have silently omitted usable data. + if (!(cause instanceof Error && "code" in cause && cause.code === "ENOENT")) { + hadReadError = true; + } } } }; From f8d386f8eda27aa07bb9ca4713432a32bc0316ad Mon Sep 17 00:00:00 2001 From: tris203 Date: Mon, 24 Aug 2026 19:50:47 +0100 Subject: [PATCH 11/15] test(server): keep transcript tests platform-neutral --- .../src/usage/usageTranscriptReader.test.ts | 37 +++++++++---------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/apps/server/src/usage/usageTranscriptReader.test.ts b/apps/server/src/usage/usageTranscriptReader.test.ts index 548b2a50c33f..d3493b2d1b52 100644 --- a/apps/server/src/usage/usageTranscriptReader.test.ts +++ b/apps/server/src/usage/usageTranscriptReader.test.ts @@ -156,27 +156,24 @@ describe("listTranscriptFiles", () => { } }); - it.skipIf(process.platform === "win32")( - "ignores transcripts that vanish while the directory is being listed", - async () => { - const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-transcripts-")); - try { - NodeFS.symlinkSync( - NodePath.join(directory, "vanished"), - NodePath.join(directory, "gone.jsonl"), - ); - - const listing = await listTranscriptFiles(directory, 0); - - expect(listing.files).toEqual([]); - expect(listing.hadReadError).toBe(false); - } finally { - NodeFS.rmSync(directory, { recursive: true, force: true }); - } - }, - ); + it("ignores transcripts that vanish while the directory is being listed", async () => { + const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-transcripts-")); + try { + NodeFS.symlinkSync( + NodePath.join(directory, "vanished"), + NodePath.join(directory, "gone.jsonl"), + ); + + const listing = await listTranscriptFiles(directory, 0); + + expect(listing.files).toEqual([]); + expect(listing.hadReadError).toBe(false); + } finally { + NodeFS.rmSync(directory, { recursive: true, force: true }); + } + }); - it.skipIf(process.platform === "win32")("reports non-missing stat failures", async () => { + it("reports non-missing stat failures", async () => { const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-transcripts-")); const transcript = NodePath.join(directory, "loop.jsonl"); try { From a1cf5973ca93f70976f233918e713a5815c65e2f Mon Sep 17 00:00:00 2001 From: tris203 Date: Mon, 24 Aug 2026 20:00:17 +0100 Subject: [PATCH 12/15] empty From e13bc62d776392b19da5151453b6a2ca90c3ffe1 Mon Sep 17 00:00:00 2001 From: tris203 Date: Fri, 28 Aug 2026 13:28:07 +0100 Subject: [PATCH 13/15] style(mobile): format usage provider order --- apps/mobile/src/features/usage/usageProviders.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/apps/mobile/src/features/usage/usageProviders.ts b/apps/mobile/src/features/usage/usageProviders.ts index 82ee5fa873cc..a9f95921ac3b 100644 --- a/apps/mobile/src/features/usage/usageProviders.ts +++ b/apps/mobile/src/features/usage/usageProviders.ts @@ -5,12 +5,7 @@ import { useAppearancePreferences } from "../settings/appearance/AppearancePrefe * Series and table order. The chart stacks providers from the bottom in this * order, so it also fixes which band sits on top of the bars. */ -export const PROVIDER_ORDER: readonly UsageProviderKind[] = [ - "codex", - "claude", - "grok", - "opencode", -]; +export const PROVIDER_ORDER: readonly UsageProviderKind[] = ["codex", "claude", "grok", "opencode"]; export const PROVIDER_LABEL: Record = { claude: "Claude Code", From 42cdcfe2fe9a396f0d03e762ddbbb1dd9c58be6a Mon Sep 17 00:00:00 2001 From: tris203 Date: Fri, 28 Aug 2026 13:34:12 +0100 Subject: [PATCH 14/15] fix(server): move OpenCode usage scans off event loop --- .../src/usage/usageTranscriptReader.test.ts | 12 +- .../server/src/usage/usageTranscriptReader.ts | 151 +++++++++++++----- 2 files changed, 121 insertions(+), 42 deletions(-) diff --git a/apps/server/src/usage/usageTranscriptReader.test.ts b/apps/server/src/usage/usageTranscriptReader.test.ts index d3493b2d1b52..0fda8f2f22b7 100644 --- a/apps/server/src/usage/usageTranscriptReader.test.ts +++ b/apps/server/src/usage/usageTranscriptReader.test.ts @@ -4,6 +4,7 @@ import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import * as NodePerfHooks from "node:perf_hooks"; import * as NodeSqlite from "node:sqlite"; +import * as NodeTimersPromises from "node:timers/promises"; import { describe, expect, it } from "@effect/vitest"; @@ -128,13 +129,20 @@ describe("readOpenCodeRecords", () => { } }); - it("waits briefly for a writer before reporting a locked database", async () => { + it("waits for a writer off the event loop before reporting a locked database", async () => { const dbPath = createDatabase(); const writer = new NodeSqlite.DatabaseSync(dbPath); writer.exec("BEGIN EXCLUSIVE"); const startedAt = NodePerfHooks.performance.now(); try { - expect(await readOpenCodeRecords(dbPath, 2_000)).toBeNull(); + const read = readOpenCodeRecords(dbPath, 2_000); + const first = await Promise.race([ + read.then(() => "read" as const), + NodeTimersPromises.setTimeout(50, "timer" as const), + ]); + + expect(first).toBe("timer"); + expect(await read).toBeNull(); expect(NodePerfHooks.performance.now() - startedAt).toBeGreaterThanOrEqual(500); } finally { writer.exec("ROLLBACK"); diff --git a/apps/server/src/usage/usageTranscriptReader.ts b/apps/server/src/usage/usageTranscriptReader.ts index 2210bbf8151c..fe77eb17051d 100644 --- a/apps/server/src/usage/usageTranscriptReader.ts +++ b/apps/server/src/usage/usageTranscriptReader.ts @@ -17,7 +17,7 @@ import * as NodeFS from "node:fs"; import * as NodeFSP from "node:fs/promises"; import * as NodePath from "node:path"; import * as NodeReadline from "node:readline"; -import * as NodeSqlite from "node:sqlite"; +import * as NodeWorkerThreads from "node:worker_threads"; import type { UsageProviderKind } from "@t3tools/contracts"; @@ -237,6 +237,99 @@ const OPEN_CODE_TABLE_QUERIES = { type OpenCodeMessageTable = keyof typeof OPEN_CODE_TABLE_QUERIES; +interface OpenCodeWorkerRows { + readonly table: OpenCodeMessageTable; + readonly rows: readonly OpenCodeUsageRow[]; +} + +type OpenCodeWorkerResult = + | { readonly status: "ok"; readonly groups: readonly OpenCodeWorkerRows[] } + | { readonly status: "failed" }; + +/** + * Kept inline so server bundles do not need a second worker entrypoint. Only + * usage scalars cross back to the main thread; message content stays in SQLite. + */ +const OPEN_CODE_WORKER_SOURCE = String.raw` + const { parentPort, workerData } = require("node:worker_threads"); + const { DatabaseSync } = require("node:sqlite"); + + let database; + let result = { status: "failed" }; + try { + database = new DatabaseSync(workerData.dbPath, { + readOnly: true, + timeout: workerData.busyTimeoutMs, + }); + + const tables = new Set(); + for (const row of database.prepare(workerData.tablesQuery).all()) { + if (row.name === "session_message" || row.name === "message") tables.add(row.name); + } + if (tables.size > 0) { + const groups = []; + for (const table of ["session_message", "message"]) { + if (!tables.has(table)) continue; + const rows = database + .prepare(workerData.tableQueries[table]) + .all(workerData.sinceMs) + .map((row) => ({ ...row })); + groups.push({ table, rows }); + } + result = { status: "ok", groups }; + } + } catch { + result = { status: "failed" }; + } finally { + try { + database?.close(); + } catch {} + } + + parentPort.postMessage(result); +`; + +function readOpenCodeRows( + dbPath: string, + sinceMs: number, +): Promise { + return new Promise((resolve) => { + const worker = (() => { + try { + return new NodeWorkerThreads.Worker(OPEN_CODE_WORKER_SOURCE, { + eval: true, + workerData: { + dbPath, + sinceMs, + busyTimeoutMs: OPEN_CODE_SQLITE_BUSY_TIMEOUT_MS, + tablesQuery: OPEN_CODE_MESSAGE_TABLES_QUERY, + tableQueries: OPEN_CODE_TABLE_QUERIES, + }, + }); + } catch { + return null; + } + })(); + if (worker === null) { + resolve(null); + return; + } + + let settled = false; + const finish = (value: readonly OpenCodeWorkerRows[] | null) => { + if (settled) return; + settled = true; + resolve(value); + }; + + worker.once("message", (message: OpenCodeWorkerResult) => { + finish(message.status === "ok" ? message.groups : null); + }); + worker.once("error", () => finish(null)); + worker.once("exit", () => finish(null)); + }); +} + /** * Reads usage records from OpenCode's SQLite transcript store. * @@ -254,47 +347,25 @@ export async function readOpenCodeRecords( dbPath: string, sinceMs: number, ): Promise { - let database: NodeSqlite.DatabaseSync; - try { - database = new NodeSqlite.DatabaseSync(dbPath, { - readOnly: true, - timeout: OPEN_CODE_SQLITE_BUSY_TIMEOUT_MS, - }); - } catch { - return null; - } - - try { - const tables = new Set(); - for (const row of database.prepare(OPEN_CODE_MESSAGE_TABLES_QUERY).all()) { - const name = (row as Record)["name"]; - if (name === "session_message" || name === "message") tables.add(name); - } - if (tables.size === 0) return null; + const groups = await readOpenCodeRows(dbPath, sinceMs); + if (groups === null) return null; - const recordsByKey = new Map(); - let anonymous = 0; - for (const table of ["session_message", "message"] as const) { - if (!tables.has(table)) continue; - const rows = database.prepare(OPEN_CODE_TABLE_QUERIES[table]).all(sinceMs); - for (const row of rows) { - const record = parseOpenCodeUsageRow(row as OpenCodeUsageRow); - if (record === null) continue; - if (record.dedupeKey === null) { - // An anonymous record can still be unique; it just cannot dedupe - // across the two projections. - recordsByKey.set(`${table}#${anonymous++}`, record); - continue; - } - if (!recordsByKey.has(record.dedupeKey)) { - recordsByKey.set(record.dedupeKey, record); - } + const recordsByKey = new Map(); + let anonymous = 0; + for (const { table, rows } of groups) { + for (const row of rows) { + const record = parseOpenCodeUsageRow(row); + if (record === null) continue; + if (record.dedupeKey === null) { + // An anonymous record can still be unique; it just cannot dedupe + // across the two projections. + recordsByKey.set(`${table}#${anonymous++}`, record); + continue; + } + if (!recordsByKey.has(record.dedupeKey)) { + recordsByKey.set(record.dedupeKey, record); } } - return [...recordsByKey.values()]; - } catch { - return null; - } finally { - database.close(); } + return [...recordsByKey.values()]; } From 8e7316be1540e2cfdf431cb97f8e43cd1b514cc5 Mon Sep 17 00:00:00 2001 From: tris203 Date: Fri, 28 Aug 2026 13:46:05 +0100 Subject: [PATCH 15/15] fix(server): await OpenCode worker result on normal exit --- apps/server/src/usage/usageTranscriptReader.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/server/src/usage/usageTranscriptReader.ts b/apps/server/src/usage/usageTranscriptReader.ts index fe77eb17051d..6c51ae3d4e82 100644 --- a/apps/server/src/usage/usageTranscriptReader.ts +++ b/apps/server/src/usage/usageTranscriptReader.ts @@ -326,7 +326,9 @@ function readOpenCodeRows( finish(message.status === "ok" ? message.groups : null); }); worker.once("error", () => finish(null)); - worker.once("exit", () => finish(null)); + worker.once("exit", (code) => { + if (code !== 0) finish(null); + }); }); }