From 0eee3d7100d1bdb8e266cf99eeb65956184b4d72 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 17 Nov 2026 01:04:17 -0800 Subject: [PATCH] improvement(ci): reuse bounded compiler state for acceptance builds --- .github/CI.md | 4 + .github/workflows/checks.yml | 38 ++++++- apps/docs/package.json | 2 +- apps/sim/lib/core/config/env.ts | 1 + apps/sim/next.config.ts | 24 +---- apps/sim/package.json | 2 +- apps/sim/turbo.json | 6 +- scripts/prune-turbopack-cache.mts | 141 ++++++++++++++++++++++++ scripts/prune-turbopack-cache.test.ts | 150 ++++++++++++++++++++++++++ scripts/prune-turbopack-cache.ts | 114 -------------------- 10 files changed, 344 insertions(+), 138 deletions(-) create mode 100644 scripts/prune-turbopack-cache.mts create mode 100644 scripts/prune-turbopack-cache.test.ts delete mode 100644 scripts/prune-turbopack-cache.ts diff --git a/.github/CI.md b/.github/CI.md index a355fc71c6f..5461465a43f 100644 --- a/.github/CI.md +++ b/.github/CI.md @@ -34,6 +34,10 @@ Database integration uses 4-vCPU Blacksmith runners with sticky dependency disks GitHub runners install dependencies directly rather than transferring Bun and `node_modules` archives. In a same-source paired comparison, those restores took 26.2 seconds before an 82-millisecond install, while a clean install took 12.42 seconds. This is one paired observation; retain the policy only while repeated CI timings support it. Blacksmith keeps its sticky dependency disks. GitHub Turbo output caches remain enabled for jobs that request them, with a commit-specific key and a task/event/fork-scoped restore prefix, so a rerun can reuse an exact entry without creating another archive. Push jobs never restore pull-request entries. +Trusted Blacksmith E2E builders can reuse Next's incremental compiler state when a complete Turbo build is not already cached. The compiler cache has its own mode, event, OS/architecture, resolved toolchain, lockfile and build-recipe namespace; fork PRs and GitHub fallback keep the compiler cache off. The build command always runs, and an uncertain cache probe restores state rather than skipping compilation. + +Compiler maintenance enforces a 20 GiB regular-file working-state limit before and after compilation, preserves the mounted root, and fails closed on access or ownership errors. Dev maintenance remains separate and nonblocking. Sticky disks are optional last-good compiler snapshots, not cross-job artifact handoff or validation receipts; no freshest-snapshot or physical-storage eviction guarantee is assumed. Hosted/deployment builds remain cache-off until separately qualified. A cold E2E qualification job took 259 seconds versus 225 uncached; fresh-runner warm and changed-source jobs took 148 and 159 seconds versus 227 and 229 uncached. These paired observations do not establish a universal CI speedup or justify smaller runners. Compilation and all existing acceptance coverage remain required. + Closing a same-repository pull request deletes only its GitHub merge-ref caches after verifying that it remains closed or merged. The regular `pull_request` workflow skips forks and Dependabot because their tokens cannot delete caches; those entries retain GitHub's normal cache eviction. Cleanup checks out no code and never selects a branch-wide or default-branch cache. Blacksmith sticky disks retain their separate lifecycle. Memory-heavy builds and browser jobs retain appropriately sized Blacksmith runners. Electron uses 4-vCPU Blacksmith: both shards passed within 14 seconds of the 8-vCPU control; the free GitHub rest shard was slower and its lease run stalled. diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index c9d8c57df4b..cacb1bd9e8d 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -963,6 +963,8 @@ jobs: if: needs.changes.outputs.e2e_build == 'true' runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-16vcpu-ubuntu-2404' || 'linux-x64-8-core' }} timeout-minutes: 25 + env: + NEXT_BUILD_PERSISTENT_CACHE: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) && 'true' || 'false' }} steps: - *checkout-mirror @@ -982,8 +984,10 @@ jobs: echo "::warning::Runner has ${TOTAL_GB} GB. A cold-cache build peaks ~51 GB, so this run may be OOM-killed (reported only as 'the runner has received a shutdown signal'). Warm/partial builds should still fit." fi - - name: Build application - env: + - name: Plan incremental compiler state + id: compiler-cache-plan + if: env.NEXT_BUILD_PERSISTENT_CACHE == 'true' + env: &e2e-build-env NODE_OPTIONS: '--no-warnings --max-old-space-size=8192' NEXT_PUBLIC_APP_URL: 'http://localhost:3000' DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432/simstudio' @@ -998,8 +1002,38 @@ jobs: # already enforce. NEXT_BUILD_SKIP_TYPE_CHECK: 'true' NEXT_BUILD_SKIP_SOURCE_MAPS: 'true' + run: | + echo "toolchain=bun$(bun --version)-node$(node --version)" >> "$GITHUB_OUTPUT" + if bunx turbo run build --filter=@sim/app --dry=json > "$RUNNER_TEMP/e2e-build-plan.json" && + jq -e 'type == "object" and (.tasks | type) == "array" and ([.tasks[] | select(.taskId == "@sim/app#build")] | length == 1 and .[0].cache.status == "HIT")' "$RUNNER_TEMP/e2e-build-plan.json" > /dev/null; then + echo 'restore=false' >> "$GITHUB_OUTPUT" + else + echo 'restore=true' >> "$GITHUB_OUTPUT" + fi + + - name: Mount incremental compiler state + id: compiler-cache + if: steps.compiler-cache-plan.outputs.restore == 'true' + uses: ./.github/actions/cache + with: + provider: ${{ vars.CI_PROVIDER }} + key: ${{ github.repository }}-next-build-e2e-v1-${{ runner.os }}-${{ runner.arch }}-${{ github.event_name }}-${{ steps.compiler-cache-plan.outputs.toolchain }}-${{ hashFiles('bun.lock', 'package.json', 'patches/**', 'apps/sim/package.json', 'apps/sim/next.config.ts', 'apps/sim/lib/core/config/env.ts', '.github/workflows/checks.yml', '.github/actions/setup/action.yml') }} + path: ./apps/sim/.next/cache + + - name: Bound restored compiler state + if: steps.compiler-cache.outcome == 'success' + working-directory: apps/sim + run: node ../../scripts/prune-turbopack-cache.mts --build + + - name: Build application + env: *e2e-build-env run: bunx turbo run build --filter=@sim/app + - name: Bound compiler state before persistence + if: always() && !cancelled() && steps.compiler-cache.outcome == 'success' + working-directory: apps/sim + run: node ../../scripts/prune-turbopack-cache.mts --build + # Everything `next start` reads, including the spreadsheet parse worker it spawns (named, so a # missing bundle fails packaging as it fails docker/app.Dockerfile); node_modules and the source # tree come from e2e-production's own checkout and setup, at the same workspace path and diff --git a/apps/docs/package.json b/apps/docs/package.json index 47d6aa4edd2..414a177528a 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -5,7 +5,7 @@ "license": "Apache-2.0", "scripts": { "dev": "bun run dev:cache:cap && bun run content:generate && bunx concurrently --kill-others --names Next,Content \"next dev --port 3001\" \"bun scripts/compile-content.ts --watch\"", - "dev:cache:cap": "bun run ../../scripts/prune-turbopack-cache.ts", + "dev:cache:cap": "bun run ../../scripts/prune-turbopack-cache.mts", "dev:clean": "rm -rf .next/dev/cache", "build": "bun scripts/fetch-git-history.ts && bun run content:generate --prune && NODE_OPTIONS='--max-old-space-size=8192' next build", "start": "next start", diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 712a558b8c7..5a757490398 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -347,6 +347,7 @@ export const env = createEnv({ // Infrastructure & Deployment NEXT_RUNTIME: z.string().optional(), // Next.js runtime environment DOCKER_BUILD: z.boolean().optional(), // Flag indicating Docker build environment + NEXT_BUILD_PERSISTENT_CACHE: z.boolean().optional(), // Enables incremental compiler state for qualified CI builds NEXT_BUILD_SKIP_SOURCE_MAPS: z.boolean().optional(), // Omits unused browser source maps in acceptance builds NEXT_BUILD_SKIP_TYPE_CHECK: z.boolean().optional(), // Skips `next build`'s TypeScript pass where another CI job type-checks diff --git a/apps/sim/next.config.ts b/apps/sim/next.config.ts index e7c5a52ead2..f3a331098ad 100644 --- a/apps/sim/next.config.ts +++ b/apps/sim/next.config.ts @@ -182,15 +182,8 @@ const nextConfig: NextConfig = { * 5.4x faster restarts and ~1.9x less memory. Cold compile with an empty * cache is unchanged (~32s either way) — the cache only pays back on restart. * - * This is deliberately NOT the same decision as `turbopackFileSystemCacheForBuild` - * below. That one is measured-harmful for `next build`; this one is - * measured-beneficial for `next dev`. It was previously `false`, but that was - * incidental — it was introduced by a landing-page redesign (#5408) whose - * description never mentions Turbopack, caching, or dev performance, and it - * is not covered by the #6078 build A/B cited below. - * * The cache is unbounded on disk (an abandoned one reached 78 GB here), so - * `scripts/prune-turbopack-cache.ts` is chained into every `dev` script to cap it. + * `scripts/prune-turbopack-cache.mts` is chained into every `dev` script to cap it. * A *corrupted* cache can abort Turbopack outright ("Cache corruption * detected: checksum mismatch") rather than falling back — it depends whether * the damaged region is read. `bun run dev:clean` and restart is the fix. @@ -203,18 +196,11 @@ const nextConfig: NextConfig = { */ turbopackFileSystemCacheForDev: true, /** - * Turbopack's persistent build cache (beta) stays off — it is a net loss at - * this app's size. A controlled A/B on a byte-identical module graph (PR - * #6078) measured compile at 113s with it off, 162s cold with it on, and 360s - * warm: the cache made the same build 3.2x slower. It also grew 5.1 GB -> - * 12 GB across two runs of an unchanged tree, so a cache degrades the longer - * it lives. Restoring across commits is separately undocumented-as-supported - * (vercel/next.js#87283 reports stale HTML from a cache built elsewhere). - * - * The explicit pin is load-bearing: 16.3.0 flipped this default to true for - * stable (vercel/next.js#94616), so dropping it re-enables the slower cache. + * Only qualified CI builds opt into persistent compiler state. Other builds + * remain cache-off until their own memory, invalidation and restore costs are + * qualified. CI keeps this state separate from complete build-output caches. */ - turbopackFileSystemCacheForBuild: false, + turbopackFileSystemCacheForBuild: isTruthy(env.NEXT_BUILD_PERSISTENT_CACHE), /** * TypeScript 7 ships no JavaScript compiler API until 7.1, so Next's default * checker cannot load it — this shells out to the project-local `tsc` instead. diff --git a/apps/sim/package.json b/apps/sim/package.json index 31a53480a53..97eef71c8f1 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -10,7 +10,7 @@ "scripts": { "dev": "bun run build:cli-runtime && bun run dev:cache:cap && next dev --port 3000", "dev:capped": "bun run build:cli-runtime && bun run dev:cache:cap && NODE_OPTIONS='--max-old-space-size=4096' next dev --port 3000", - "dev:cache:cap": "bun run ../../scripts/prune-turbopack-cache.ts", + "dev:cache:cap": "bun run ../../scripts/prune-turbopack-cache.mts", "dev:clean": "rm -rf .next/dev/cache", "dev:webpack": "bun run build:cli-runtime && next dev --webpack", "load:workflow": "bun run load:workflow:baseline", diff --git a/apps/sim/turbo.json b/apps/sim/turbo.json index 4320f58fd80..61dbbe1da2b 100644 --- a/apps/sim/turbo.json +++ b/apps/sim/turbo.json @@ -207,7 +207,11 @@ ] }, "build": { - "env": ["NEXT_BUILD_SKIP_SOURCE_MAPS", "NEXT_BUILD_SKIP_TYPE_CHECK"], + "env": [ + "NEXT_BUILD_PERSISTENT_CACHE", + "NEXT_BUILD_SKIP_SOURCE_MAPS", + "NEXT_BUILD_SKIP_TYPE_CHECK" + ], "inputs": [ "$TURBO_DEFAULT$", ".env*", diff --git a/scripts/prune-turbopack-cache.mts b/scripts/prune-turbopack-cache.mts new file mode 100644 index 00000000000..f8162ef56fc --- /dev/null +++ b/scripts/prune-turbopack-cache.mts @@ -0,0 +1,141 @@ +#!/usr/bin/env node +/** + * Caps each app's Turbopack cache from its working directory. + * + * Dev maintenance is best effort and never blocks startup. `--build` selects + * `.next/cache`, preserves its mounted root, and fails on maintenance errors; + * run it only between compiler processes. Oversized state is discarded so the + * next compiler starts cold. Child symlinks are neither counted nor followed. + * + * Build maintenance uses Node's bounded directory iterator and stops counting + * above the cap. Dev commands keep Bun and its existing best-effort behavior. + * + * CI build maintenance runs with Node 24: `node scripts/prune-turbopack-cache.mts [--build] [--force] [--dry-run]`. + * `SIM_TURBOPACK_CACHE_MAX_GB` overrides the default 20 GiB cap. + */ +import { existsSync, lstatSync, statSync } from 'node:fs' +import { opendir, rm, rmdir, stat } from 'node:fs/promises' +import path from 'node:path' + +const DEFAULT_MAX_GB = 20 +const BYTES_PER_GB = 1024 ** 3 +const BUILD_CACHE = process.argv.includes('--build') + +/** + * Each app owns its cache under its working directory. Build maintenance selects + * a separate root, so it cannot prune a live dev server's state. + */ +const CACHE_DIR = path.resolve( + process.cwd(), + BUILD_CACHE ? '.next/cache' : '.next/dev/cache/turbopack' +) + +function isMissingPath(error: unknown): boolean { + return error instanceof Error && 'code' in error && error.code === 'ENOENT' +} + +/** Counts regular files without following symlinks or retaining the tree in memory. */ +async function directorySize(dir: string, limit: number): Promise { + let total = 0 + let entries: Awaited> + try { + entries = await opendir(dir, { bufferSize: 32 }) + } catch (error) { + if (BUILD_CACHE && isMissingPath(error)) return 0 + throw error + } + for await (const entry of entries) { + const entryPath = path.join(dir, entry.name) + if (entry.isDirectory()) { + total += await directorySize(entryPath, limit - total) + } else if (entry.isFile()) { + try { + total += (await stat(entryPath)).size + } catch (error) { + if (BUILD_CACHE && !isMissingPath(error)) throw error + } + } + if (total > limit) break + } + return total +} + +async function clearDirectory(dir: string): Promise { + for await (const entry of await opendir(dir, { bufferSize: 32 })) { + const entryPath = path.join(dir, entry.name) + try { + if (entry.isDirectory()) { + await clearDirectory(entryPath) + await rmdir(entryPath) + } else { + await rm(entryPath, { force: true }) + } + } catch (error) { + if (!isMissingPath(error)) throw error + } + } +} + +function parseMaxGb(): number { + const raw = process.env.SIM_TURBOPACK_CACHE_MAX_GB + if (!raw) return DEFAULT_MAX_GB + const parsed = Number(raw) + if (!Number.isFinite(parsed) || parsed <= 0) { + console.warn( + `[turbopack-cache] ignoring invalid SIM_TURBOPACK_CACHE_MAX_GB=${raw}, using ${DEFAULT_MAX_GB}` + ) + return DEFAULT_MAX_GB + } + return parsed +} + +async function main() { + const force = process.argv.includes('--force') + const dryRun = process.argv.includes('--dry-run') + + if (BUILD_CACHE) { + let root: ReturnType + try { + root = lstatSync(CACHE_DIR) + } catch (error) { + if (isMissingPath(error)) return + throw error + } + if (root.isSymbolicLink() || !root.isDirectory()) + throw new Error('Build cache root must be a directory without symlinks') + } else if (!existsSync(CACHE_DIR) || !statSync(CACHE_DIR).isDirectory()) { + return + } + + const maxGb = parseMaxGb() + const limit = BUILD_CACHE ? maxGb * BYTES_PER_GB : Number.POSITIVE_INFINITY + const bytes = force ? 0 : await directorySize(CACHE_DIR, limit) + const gb = bytes / BYTES_PER_GB + const sizeLabel = force + ? 'cache' + : `${BUILD_CACHE && bytes > limit ? 'at least ' : ''}${gb.toFixed(1)} GB` + + if (!force && gb <= maxGb) { + if (dryRun) console.log(`[turbopack-cache] ${sizeLabel} (cap ${maxGb} GB) — keeping`) + return + } + + const reason = force ? 'forced' : `over the ${maxGb} GB cap` + if (dryRun) { + console.log(`[turbopack-cache] ${sizeLabel} is ${reason} — would prune (dry run)`) + return + } + + console.log(`[turbopack-cache] ${sizeLabel} is ${reason} — pruning; next start recompiles cold`) + if (!BUILD_CACHE && lstatSync(CACHE_DIR).isSymbolicLink()) { + await rm(CACHE_DIR, { force: true }) + return + } + await clearDirectory(CACHE_DIR) + if (!BUILD_CACHE) await rmdir(CACHE_DIR) +} + +main().catch((error) => { + console.warn('[turbopack-cache] prune skipped:', error) + if (BUILD_CACHE) process.exitCode = 1 +}) diff --git a/scripts/prune-turbopack-cache.test.ts b/scripts/prune-turbopack-cache.test.ts new file mode 100644 index 00000000000..57cd68bfa37 --- /dev/null +++ b/scripts/prune-turbopack-cache.test.ts @@ -0,0 +1,150 @@ +import { spawnSync } from 'node:child_process' +import { + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' + +const script = resolve(import.meta.dirname, 'prune-turbopack-cache.mts') +const fixtures: string[] = [] + +function fixture() { + const root = mkdtempSync(join(tmpdir(), 'sim-compiler-cache-')) + fixtures.push(root) + const build = join(root, '.next/cache') + const dev = join(root, '.next/dev/cache/turbopack') + const outside = join(root, 'unrelated') + for (const dir of [build, dev, outside]) { + mkdirSync(dir, { recursive: true }) + writeFileSync(join(dir, 'state'), Buffer.alloc(4_096)) + } + const nested = join(build, 'nested', 'state') + mkdirSync(nested, { recursive: true }) + for (let index = 0; index < 40; index++) writeFileSync(join(nested, String(index)), 'x') + symlinkSync(outside, join(build, 'linked'), process.platform === 'win32' ? 'junction' : 'dir') + return { root, build, dev, outside } +} + +function run(root: string, args: string[], maximum = '0.000001') { + return spawnSync(args.includes('--build') ? 'node' : 'bun', [script, ...args], { + cwd: root, + encoding: 'utf8', + timeout: 10_000, + env: { ...process.env, SIM_TURBOPACK_CACHE_MAX_GB: maximum }, + }) +} + +afterEach(() => { + for (const root of fixtures.splice(0)) rmSync(root, { recursive: true, force: true }) +}) + +describe('compiler cache maintenance through the real CLI', () => { + it.each(['size', 'force'])( + 'clears only build contents by %s while preserving its mount', + (mode) => { + const { root, build, dev, outside } = fixture() + const inode = statSync(build).ino + const result = run( + root, + mode === 'force' ? ['--build', '--force'] : ['--build'], + mode === 'force' ? '1' : undefined + ) + expect(result.status, result.stderr).toBe(0) + expect(existsSync(build)).toBe(true) + expect(statSync(build).ino).toBe(inode) + expect(readdirSync(build)).toEqual([]) + expect(existsSync(join(dev, 'state'))).toBe(true) + expect(existsSync(join(outside, 'state'))).toBe(true) + } + ) + + it('retains useful build state without counting external symlink targets', () => { + const { root, build, dev, outside } = fixture() + const inode = statSync(build).ino + const buildBytes = readFileSync(join(build, 'state')) + const devBytes = readFileSync(join(dev, 'state')) + const outsideBytes = readFileSync(join(outside, 'state')) + expect(run(root, ['--build'], '0.000006').status).toBe(0) + expect(statSync(build).ino).toBe(inode) + expect(readFileSync(join(build, 'state'))).toEqual(buildBytes) + expect(readFileSync(join(dev, 'state'))).toEqual(devBytes) + expect(readFileSync(join(outside, 'state'))).toEqual(outsideBytes) + }) + + it('does not substitute dev state when the build cache is absent', () => { + const { root, build, dev } = fixture() + rmSync(build, { recursive: true }) + expect(run(root, ['--build']).status).toBe(0) + expect(existsSync(join(dev, 'state'))).toBe(true) + }) + + it('rejects a non-directory ancestor instead of treating it as an absent build cache', () => { + const { root, outside } = fixture() + const parent = join(root, '.next') + rmSync(parent, { recursive: true }) + writeFileSync(parent, 'unrelated parent file') + const result = run(root, ['--build']) + expect(result.status, result.stderr).not.toBe(0) + expect(readFileSync(parent, 'utf8')).toBe('unrelated parent file') + expect(existsSync(join(outside, 'state'))).toBe(true) + }) + + it('keeps the existing dev command scoped to dev state', () => { + const { root, build, dev, outside } = fixture() + expect(run(root, []).status).toBe(0) + expect(existsSync(dev)).toBe(false) + expect(existsSync(join(build, 'state'))).toBe(true) + expect(existsSync(join(outside, 'state'))).toBe(true) + }) + + it.each(['size', 'force'])( + 'unlinks a dev root symlink by %s without traversing its target', + (mode) => { + const { root, build, dev, outside } = fixture() + rmSync(dev, { recursive: true }) + symlinkSync(outside, dev, process.platform === 'win32' ? 'junction' : 'dir') + const outsideBytes = readFileSync(join(outside, 'state')) + const result = run( + root, + mode === 'force' ? ['--force'] : [], + mode === 'force' ? '1' : undefined + ) + expect(result.status, result.stderr).toBe(0) + expect(readFileSync(join(outside, 'state'))).toEqual(outsideBytes) + expect(existsSync(dev)).toBe(false) + expect(existsSync(join(build, 'state'))).toBe(true) + } + ) + + it('leaves both caches intact for a build dry run', () => { + const { root, build, dev } = fixture() + expect(run(root, ['--build', '--dry-run']).status).toBe(0) + expect(existsSync(join(build, 'state'))).toBe(true) + expect(existsSync(join(dev, 'state'))).toBe(true) + }) + + it.each([true, false])( + 'refuses a symlink build root when target existence is %s', + (targetExists) => { + const { root, build, dev, outside } = fixture() + rmSync(build, { recursive: true }) + symlinkSync(outside, build, process.platform === 'win32' ? 'junction' : 'dir') + if (!targetExists) rmSync(outside, { recursive: true }) + const result = run(root, ['--build', '--force']) + expect(result.status, result.stderr).not.toBe(0) + expect(lstatSync(build).isSymbolicLink()).toBe(true) + expect(existsSync(join(outside, 'state'))).toBe(targetExists) + expect(existsSync(join(dev, 'state'))).toBe(true) + } + ) +}) diff --git a/scripts/prune-turbopack-cache.ts b/scripts/prune-turbopack-cache.ts deleted file mode 100644 index 2437a0dd216..00000000000 --- a/scripts/prune-turbopack-cache.ts +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env bun -/** - * Caps the size of Turbopack's dev filesystem cache. - * - * `experimental.turbopackFileSystemCacheForDev` is ON (see `apps/sim/next.config.ts`) - * because it makes dev-server restarts 5.4x faster and cuts RSS ~1.9x. The tradeoff - * is that the cache is an append-heavy LSM store with no upstream GC: it grows with - * every route compiled and every code change, and it is never pruned. An abandoned - * cache in this repo reached 78 GB across 1,848 SST files. - * - * A stale cache is also slower to read back, so an unbounded one eventually erodes - * the win it exists to provide. This script drops the whole cache once it crosses a - * threshold; Turbopack rebuilds it on the next run (one cold compile, ~32s). - * - * Also the remedy when Turbopack aborts with `Cache corruption detected: - * checksum mismatch` — it does not self-heal from a corrupted cache, so prune - * and restart. (An ordinary hard kill does not corrupt it; Turbopack discards a - * partial write and rebuilds silently.) - * - * Runs automatically before every `dev` script. Also invokable directly: - * bun run scripts/prune-turbopack-cache.ts # prune if over the cap - * bun run scripts/prune-turbopack-cache.ts --force # always prune - * bun run scripts/prune-turbopack-cache.ts --dry-run # report only - * - * Override the cap with `SIM_TURBOPACK_CACHE_MAX_GB` (default 20). - * - * The cap is a backstop, not routine maintenance. A normal session's cache sits - * around 1-2 GB; the 78 GB case was a month of accumulation while the feature was - * effectively abandoned. Measured cost of the size walk: ~30ms on a real cache, - * ~85ms at 2,000 files — under 2% of a warm restart. - * - * Safe to run while a dev server is live, which happens if a second server is - * started from the same checkout: the running server keeps its in-memory state - * and continues serving (verified — no crash, no corruption). It does stop - * persisting for the rest of that session, so its *next* start is cold once, then - * back to normal. Nothing to clean up. - * - * Note for anyone benchmarking the cache: stop the dev server with SIGINT, not - * `kill -9`. Turbopack discards a partially-written cache and rebuilds silently, - * so a hard kill makes a real cache win read as no win at all. - */ -import { existsSync, statSync } from 'node:fs' -import { readdir, rm, stat } from 'node:fs/promises' -import path from 'node:path' - -const DEFAULT_MAX_GB = 20 -const BYTES_PER_GB = 1024 ** 3 - -/** - * Resolved from the working directory, not hardcoded to one app: every Next app - * in the monorepo has its own dev cache (`apps/docs` runs `next dev` too and, with - * no override, uses the Next default where the cache is on). Each app caps its - * own, so one app's dev start never prunes a cache another app is holding open. - */ -const CACHE_DIR = path.resolve(process.cwd(), '.next', 'dev', 'cache', 'turbopack') - -/** Sums the size of every regular file under `dir`, skipping symlinks. */ -async function directorySize(dir: string): Promise { - let total = 0 - const entries = await readdir(dir, { withFileTypes: true, recursive: true }) - for (const entry of entries) { - if (!entry.isFile()) continue - try { - total += (await stat(path.join(entry.parentPath, entry.name))).size - } catch { - // Turbopack compacts while we walk; a file vanishing mid-scan is expected. - } - } - return total -} - -function parseMaxGb(): number { - const raw = process.env.SIM_TURBOPACK_CACHE_MAX_GB - if (!raw) return DEFAULT_MAX_GB - const parsed = Number(raw) - if (!Number.isFinite(parsed) || parsed <= 0) { - console.warn( - `[turbopack-cache] ignoring invalid SIM_TURBOPACK_CACHE_MAX_GB=${raw}, using ${DEFAULT_MAX_GB}` - ) - return DEFAULT_MAX_GB - } - return parsed -} - -async function main() { - const force = process.argv.includes('--force') - const dryRun = process.argv.includes('--dry-run') - - if (!existsSync(CACHE_DIR) || !statSync(CACHE_DIR).isDirectory()) return - - const maxGb = parseMaxGb() - const bytes = await directorySize(CACHE_DIR) - const gb = bytes / BYTES_PER_GB - const sizeLabel = `${gb.toFixed(1)} GB` - - if (!force && gb <= maxGb) { - if (dryRun) console.log(`[turbopack-cache] ${sizeLabel} (cap ${maxGb} GB) — keeping`) - return - } - - const reason = force ? 'forced' : `over the ${maxGb} GB cap` - if (dryRun) { - console.log(`[turbopack-cache] ${sizeLabel} is ${reason} — would prune (dry run)`) - return - } - - console.log(`[turbopack-cache] ${sizeLabel} is ${reason} — pruning; next start recompiles cold`) - await rm(CACHE_DIR, { recursive: true, force: true }) -} - -main().catch((error) => { - // Never block `next dev` on a cache-maintenance failure. - console.warn('[turbopack-cache] prune skipped:', error) -})