diff --git a/docs/adr/0004-unified-session-database.md b/docs/adr/0004-unified-session-database.md new file mode 100644 index 0000000000..db3d06a4f1 --- /dev/null +++ b/docs/adr/0004-unified-session-database.md @@ -0,0 +1,27 @@ +# Unified session database: remove channel-based DB path splitting + +Status: proposed (2026-08-12) + +Tracking: harmoniqs/opencode#188 · Glossary: `CONTEXT.md` (Installation Channel) + +All opencode installations on a single machine write to one database file (`opencode.db`) regardless of their build channel. The channel-based path splitting (`opencode-dev.db`, `opencode-local.db`, branch-named DBs) is removed. On first startup after the change, a one-time consolidation migrates rows from any legacy channel-named DB files into the canonical `opencode.db`, deduplicating by primary key, then renames the sources to `*.db.merged` as backups. + +**Why:** the channel-DB split was introduced defensively — to prevent a dev build from corrupting a release user's session history. In practice, sessions are append-only, schema-migrated identically across all channels, and the split only fragments session history for users who switch between dev and release builds on the same machine (the common case for contributors). The existing `consolidateLocalDb` function already acknowledges this by merging per-branch DBs into one — this change extends the same logic to its natural conclusion. + +**Mechanism:** + +1. `path()` in `database.ts` reduces to: if `OPENCODE_DB` is set, use it; otherwise return `join(Global.Path.data, "opencode.db")` with a one-time consolidation pass. +2. The consolidation function scans `Global.Path.data` for any `opencode-*.db` files (matching `opencode-dev.db`, `opencode-local.db`, `opencode-beta.db`, and any branch-named variants). For each found file, it ATTACHes it via SQLite, iterates all user tables, and runs `INSERT OR IGNORE INTO main. SELECT * FROM source.
` (primary-key dedup). Foreign key checks are disabled during the merge. Successfully merged sources are renamed to `*.db.merged`; their WAL/SHM sidecars are deleted. +3. If `opencode.db` does not yet exist but channel DBs do, the largest channel DB is renamed to become `opencode.db` (the base), then remaining channel DBs are merged into it. This avoids a full copy of the largest file. +4. The `OPENCODE_DB` environment variable override is retained for CI, testing, and `:memory:` use. +5. `OPENCODE_DISABLE_CHANNEL_DB`, `STABLE_CHANNELS`, and all channel-branching logic are deleted. + +**Standalone merge script:** a `merge-opencode-dbs.sh` bash script (distributed separately, not in the repo) provides the same consolidation for users who haven't upgraded yet. It requires only `sqlite3` on PATH, auto-detects the data directory, and performs the same ATTACH + INSERT OR IGNORE + backup flow. + +**Conditions of merge:** the consolidation must be idempotent (re-running after a partial failure picks up where it left off — already-merged files have been renamed, un-merged ones are retried). The merge must not block startup for more than ~5 seconds on a 500 MB combined DB size. WAL mode must be checkpointed on source DBs before ATTACH (to avoid attaching a DB mid-transaction). + +**Flip condition:** if opencode ever needs true multi-tenant isolation (e.g. separate DB per workspace for portability), revisit with a workspace-scoped model rather than channel-scoped. + +**Accepted costs:** users who deliberately kept channel DBs separate (for A/B testing session behavior across versions) lose that separation. The `*.db.merged` backups consume disk until manually deleted. + +**Considered:** env-var-only patch (`OPENCODE_DISABLE_CHANNEL_DB=1` in build config — runner-up: zero code change, but doesn't merge existing data and doesn't fix upstream); explicit CLI command (`opencode db merge` — rejected: the right behavior should be automatic, most users won't discover or run a manual command and will just lose history); keep channel split but add cross-DB search (rejected: complexity for a problem that shouldn't exist). diff --git a/packages/core/src/database/database.ts b/packages/core/src/database/database.ts index 8720725205..62455690e6 100644 --- a/packages/core/src/database/database.ts +++ b/packages/core/src/database/database.ts @@ -8,8 +8,8 @@ import { Flag } from "../flag/flag" import { isAbsolute, join } from "path" import { existsSync, readdirSync, renameSync, statSync, unlinkSync } from "fs" import { DatabaseMigration } from "./migration" -import { InstallationChannel } from "../installation/version" import { makeGlobalNode } from "../effect/app-node" +import { Database as BunDatabase } from "bun:sqlite" const makeDatabase = EffectDrizzleSqlite.makeWithDefaults() type DatabaseShape = Effect.Success @@ -42,64 +42,182 @@ export function layerFromPath(filename: string) { } /** - * Known stable channels that get their own dedicated DB file. - * Everything else (dev branches like "tab-drag-reorder", "local/amicode", etc.) - * collapses to "opencode-local.db" so branch switches don't strand sessions. + * All tables to merge via INSERT OR IGNORE. Order matters: parents before + * children to satisfy foreign key relationships on re-enable. */ -const STABLE_CHANNELS = new Set(["dev", "latest", "beta", "prod"]) +const MERGE_TABLES = [ + "migration", + "data_migration", + "project", + "project_directory", + "workspace", + "account", + "account_state", + "control_account", + "credential", + "event_sequence", + "event", + "permission", + "session", + "session_message", + "session_input", + "session_context_epoch", + "session_share", + "message", + "part", + "todo", +] /** - * On first access, if `opencode-local.db` doesn't exist or is smaller than an - * available branch-named DB, rename the largest branch DB to become the local - * DB. This makes the migration from per-branch DBs fully transparent — no - * manual intervention needed when users pull this change. + * One-time consolidation of legacy channel-named DBs into the unified + * `opencode.db`. Runs synchronously at startup before the Effect DB layer + * initializes. + * + * Strategy: + * 1. Find all `opencode-*.db` files (skip already-merged `*.db.merged`). + * 2. If `opencode.db` doesn't exist, rename the largest source to become it + * (avoids a full copy of the biggest file). + * 3. For each remaining source: checkpoint WAL, ATTACH to target, INSERT OR + * IGNORE per table, DETACH, rename source to `*.db.merged`. + * 4. Clean up WAL/SHM sidecars of merged sources. + * + * Idempotent: partially-merged sources remain un-renamed and will be retried + * on next startup. Already `.db.merged` files are ignored. */ -function consolidateLocalDb(targetPath: string) { +function consolidateChannelDbs(targetPath: string) { const dir = Global.Path.data - let candidates: { path: string; size: number; mtime: number }[] + + // Collect candidate channel DBs + let candidates: { path: string; name: string; size: number }[] try { - const stableFiles = new Set([...STABLE_CHANNELS].map((c) => `opencode-${c}.db`)) - stableFiles.add("opencode.db") - stableFiles.add("opencode-local.db") candidates = readdirSync(dir) - .filter((f) => f.startsWith("opencode-") && f.endsWith(".db") && !stableFiles.has(f)) + .filter((f) => f.startsWith("opencode-") && f.endsWith(".db") && !f.endsWith(".db.merged")) .map((f) => { const full = join(dir, f) try { const st = statSync(full) - return { path: full, size: st.size, mtime: st.mtimeMs } + return { path: full, name: f, size: st.size } } catch { return null } }) - .filter((x): x is { path: string; size: number; mtime: number } => x !== null) + .filter((x): x is { path: string; name: string; size: number } => x !== null) } catch { return } + if (candidates.length === 0) return - // Pick the largest branch DB (most session data) + + // Sort largest first — the biggest becomes the base if target doesn't exist candidates.sort((a, b) => b.size - a.size) - const best = candidates[0] - // Only consolidate if target doesn't exist or is smaller (i.e. has less data) - const targetSize = existsSync(targetPath) ? (statSync(targetPath).size ?? 0) : 0 - if (targetSize >= best.size) return - try { - // Remove the smaller target if it exists (schema-only or empty) - if (existsSync(targetPath)) { - unlinkSync(targetPath) + + // If target doesn't exist, promote the largest candidate by rename + if (!existsSync(targetPath)) { + const largest = candidates.shift()! + try { + renameSync(largest.path, targetPath) + // Move sidecars too for (const ext of ["-wal", "-shm"]) { - const sidecar = targetPath + ext - if (existsSync(sidecar)) unlinkSync(sidecar) + const sidecar = largest.path + ext + if (existsSync(sidecar)) renameSync(sidecar, targetPath + ext) } + } catch { + // If rename fails, put it back in the list to be merged normally + candidates.unshift(largest) } - renameSync(best.path, targetPath) - // Also rename WAL/SHM sidecars if they exist - for (const ext of ["-wal", "-shm"]) { - const sidecar = best.path + ext - if (existsSync(sidecar)) renameSync(sidecar, targetPath + ext) - } + } + + if (candidates.length === 0) return + + // Open the target DB for merging + let db: InstanceType + try { + db = new BunDatabase(targetPath, { create: true, readwrite: true }) } catch { - // Non-fatal: worst case, a fresh DB is created + return + } + + try { + // Disable foreign keys for the duration of the merge + db.run("PRAGMA foreign_keys = OFF") + db.run("PRAGMA journal_mode = WAL") + + for (const candidate of candidates) { + try { + // Checkpoint the source WAL so ATTACH sees a clean state + let sourceDb: InstanceType | undefined + try { + sourceDb = new BunDatabase(candidate.path, { readwrite: true }) + sourceDb.run("PRAGMA wal_checkpoint(TRUNCATE)") + sourceDb.close() + sourceDb = undefined + } catch { + sourceDb?.close() + // If we can't checkpoint, try the merge anyway — ATTACH will read + // whatever is committed in the main file. + } + + // Attach source and merge each table + const alias = "source_db" + db.run(`ATTACH DATABASE '${candidate.path.replace(/'/g, "''")}' AS ${alias}`) + + try { + db.run("BEGIN") + + // Get table list from the source to handle schema differences gracefully + const sourceTables = new Set( + ( + db.query(`SELECT name FROM ${alias}.sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'`).all() as { + name: string + }[] + ).map((r) => r.name), + ) + + for (const table of MERGE_TABLES) { + if (!sourceTables.has(table)) continue + db.run(`INSERT OR IGNORE INTO main."${table}" SELECT * FROM ${alias}."${table}"`) + } + + db.run("COMMIT") + } catch { + try { + db.run("ROLLBACK") + } catch { + // Rollback may fail if transaction wasn't started + } + // Leave this source un-renamed for retry on next startup + try { + db.run(`DETACH DATABASE ${alias}`) + } catch { + // If detach fails too, bail on this source + } + continue + } + + db.run(`DETACH DATABASE ${alias}`) + + // Successfully merged — rename source to *.db.merged + try { + renameSync(candidate.path, candidate.path + ".merged") + // Clean up WAL/SHM sidecars + for (const ext of ["-wal", "-shm"]) { + const sidecar = candidate.path + ext + if (existsSync(sidecar)) unlinkSync(sidecar) + } + } catch { + // Non-fatal: the source stays as-is and will be skipped next time + // (it's already fully merged, so INSERT OR IGNORE is a no-op) + } + } catch { + // Non-fatal per source: skip this one and try the rest + continue + } + } + + // Re-enable foreign keys + db.run("PRAGMA foreign_keys = ON") + } finally { + db.close() } } @@ -108,18 +226,8 @@ export function path() { if (Flag.OPENCODE_DB === ":memory:" || isAbsolute(Flag.OPENCODE_DB)) return Flag.OPENCODE_DB return join(Global.Path.data, Flag.OPENCODE_DB) } - if ( - ["latest", "beta", "prod"].includes(InstallationChannel) || - process.env.OPENCODE_DISABLE_CHANNEL_DB === "1" || - process.env.OPENCODE_DISABLE_CHANNEL_DB === "true" - ) - return join(Global.Path.data, "opencode.db") - if (STABLE_CHANNELS.has(InstallationChannel)) - return join(Global.Path.data, `opencode-${InstallationChannel}.db`) - // All non-stable channels (local dev branches) share a single DB to avoid - // session fragmentation when switching branches. - const target = join(Global.Path.data, "opencode-local.db") - consolidateLocalDb(target) + const target = join(Global.Path.data, "opencode.db") + consolidateChannelDbs(target) return target }