fix(server): the SQLite WAL file shrinks back after large writes

apps/server/src/persistence/Layers/Sqlite.test.ts

@@ -10,7 +10,11 @@ import * as Effect from "effect/Effect";
 import * as Layer from "effect/Layer";
 import * as SqlClient from "effect/unstable/sql/SqlClient";
 
-import { SqlitePersistenceMemory, makeSqlitePersistenceLive } from "./Sqlite.ts";
+import {
+  SqlitePersistenceMemory,
+  WAL_SIZE_LIMIT_BYTES,
+  makeSqlitePersistenceLive,
+} from "./Sqlite.ts";
 
 const lockHolderSource = `
 const { DatabaseSync } = require("node:sqlite");
@@ -57,6 +61,32 @@ it.effect("waits out a concurrent writer instead of failing with SQLITE_BUSY", (
   );
 });
 
+it.effect("shrinks the WAL file back to the size limit after a large write", () => {
+  const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-sqlite-wal-"));
+  const dbPath = NodePath.join(tempDir, "state.sqlite");
+  const walFileSize = () => NodeFS.statSync(`${dbPath}-wal`).size;
+  // About 25% more 4 KB rows than the limit holds, in one transaction.
+  const rowCount = Math.ceil((WAL_SIZE_LIMIT_BYTES * 1.25) / 4000);
+
+  return Effect.gen(function* () {
+    const sql = yield* SqlClient.SqlClient;
+    yield* sql`CREATE TABLE wal_probe(payload BLOB)`;
+    yield* sql`
+      WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < ${rowCount})
+      INSERT INTO wal_probe(payload) SELECT randomblob(4000) FROM n
+    `;
+    assert.isAbove(walFileSize(), WAL_SIZE_LIMIT_BYTES);
+
+    // The auto-checkpoint after the large commit copied every frame into the
+    // database, so the next commit restarts the WAL and cuts the file back.
+    yield* sql`INSERT INTO wal_probe(payload) VALUES (x'00')`;
+    assert.isAtMost(walFileSize(), WAL_SIZE_LIMIT_BYTES);
+  }).pipe(
+    Effect.provide(makeSqlitePersistenceLive(dbPath).pipe(Layer.provide(NodeServices.layer))),
+    Effect.ensuring(Effect.sync(() => NodeFS.rmSync(tempDir, { recursive: true, force: true }))),
+  );
+});
+
 it.effect("applies busy_timeout in the shared persistence setup", () =>
   Effect.gen(function* () {
     const sql = yield* SqlClient.SqlClient;

apps/server/src/persistence/Layers/Sqlite.ts

@@ -8,13 +8,19 @@ import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
 import { runMigrations } from "../Migrations.ts";
 import { ServerConfig } from "../../config.ts";
 
+// Size the -wal file is cut back to on the first commit after a WAL reset.
+export const WAL_SIZE_LIMIT_BYTES = 32 * 1024 * 1024;
+
 const setup = Layer.effectDiscard(
   Effect.gen(function* () {
     const sql = yield* SqlClient.SqlClient;
     // CLI and server write from separate processes; wait rather than fail with SQLITE_BUSY.
     yield* sql`PRAGMA busy_timeout = 5000;`;
     yield* sql`PRAGMA foreign_keys = ON;`;
     yield* sql`PRAGMA journal_mode = WAL;`;
+    // PASSIVE checkpoints never shrink the -wal file, so it otherwise keeps its
+    // largest size until the last connection closes.
+    yield* sql.unsafe(`PRAGMA journal_size_limit = ${WAL_SIZE_LIMIT_BYTES};`);
     yield* runMigrations();
   }),
 );