From fdaf478592e50631d1bf38c41f83b9b0ffd24f7d Mon Sep 17 00:00:00 2001 From: Benchance Date: Wed, 5 Aug 2026 23:50:00 +1000 Subject: [PATCH 1/3] fix(server): stop idle private-memory growth from immortal parent spans Long-lived forkParked roots and PortDiscovery poll loops were inheriting Effect ParentSpan from short-lived startup/make spans. Nested hot-path spans (secret reads, idle poll ticks) kept parenting under those immortal spans and drove ~25-30MB/h private growth while idle. Detach ambient ParentSpan in forkParked, untrace idle port polls and secret gets, and drop PortDiscovery.make's wrapping span. Co-authored-by: Cursor --- apps/server/src/auth/ServerSecretStore.ts | 5 +++- apps/server/src/preview/PortScanner.ts | 17 +++++++++--- apps/server/src/serverActivation.test.ts | 32 ++++++++++++++++++++++- apps/server/src/serverActivation.ts | 27 +++++++++++++++++-- apps/server/src/serverRuntimeStartup.ts | 3 +++ 5 files changed, 76 insertions(+), 8 deletions(-) diff --git a/apps/server/src/auth/ServerSecretStore.ts b/apps/server/src/auth/ServerSecretStore.ts index 5e9890c1ea28..a124e5493768 100644 --- a/apps/server/src/auth/ServerSecretStore.ts +++ b/apps/server/src/auth/ServerSecretStore.ts @@ -168,6 +168,9 @@ export const make = Effect.gen(function* () { const resolveSecretPath = (name: string) => path.join(serverConfig.secretsDir, `${name}.bin`); + // Hot path: called every few seconds from parked runtime fibers. Spans here + // previously nested under immortal startup ParentSpans and dominated idle + // trace volume; keep reads cheap and untraced. const get: ServerSecretStore["Service"]["get"] = (name) => fileSystem.readFile(resolveSecretPath(name)).pipe( Effect.map((bytes) => Option.some(Uint8Array.from(bytes))), @@ -181,7 +184,7 @@ export const make = Effect.gen(function* () { }), ), ), - Effect.withSpan("ServerSecretStore.get"), + Effect.withTracerEnabled(false), ); const set: ServerSecretStore["Service"]["set"] = (name, value) => { diff --git a/apps/server/src/preview/PortScanner.ts b/apps/server/src/preview/PortScanner.ts index 4571aeef4c6b..065cc8a9e6b1 100644 --- a/apps/server/src/preview/PortScanner.ts +++ b/apps/server/src/preview/PortScanner.ts @@ -39,6 +39,7 @@ import * as Semaphore from "effect/Semaphore"; import { FetchHttpClient, HttpClient } from "effect/unstable/http"; import * as ProcessRunner from "../processRunner.ts"; +import { withoutAmbientParentSpan } from "../serverActivation.ts"; export class PortDiscovery extends Context.Service< PortDiscovery, @@ -579,9 +580,17 @@ export const make = Effect.gen(function* PortDiscoveryMake() { ); // Single layer-scoped polling fiber. Ticks are no-ops when no client is - // currently retained, so the cost is one Ref.get every POLL_INTERVAL. - yield* Effect.forkScoped(pollTick().pipe(Effect.repeat(Schedule.spaced(POLL_INTERVAL)))); - + // currently retained. Run detached from any ambient ParentSpan (e.g. + // PortDiscovery.make) and without tracing idle no-ops — otherwise every + // 3s tick parents under an immortal span and grows server private memory. + yield* Effect.forkScoped( + withoutAmbientParentSpan( + pollTick().pipe( + Effect.repeat(Schedule.spaced(POLL_INTERVAL)), + Effect.withTracerEnabled(false), + ), + ), + ); const acquireRetention = Effect.fn("PortDiscovery.retain")(function* () { const wasIdle = yield* Ref.modify(stateRef, (state) => [ state.retainCount === 0, @@ -659,6 +668,6 @@ export const make = Effect.gen(function* PortDiscoveryMake() { registerTerminalProcesses, unregisterTerminal, }); -}).pipe(Effect.withSpan("PortDiscovery.make")); +}); export const layer = Layer.effect(PortDiscovery, make); diff --git a/apps/server/src/serverActivation.test.ts b/apps/server/src/serverActivation.test.ts index a4f942a95b5d..83c8017838fc 100644 --- a/apps/server/src/serverActivation.test.ts +++ b/apps/server/src/serverActivation.test.ts @@ -1,8 +1,11 @@ import { expect, it } from "@effect/vitest"; +import * as Context from "effect/Context"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Tracer from "effect/Tracer"; -import { forkParked, ServerActivation } from "./serverActivation.ts"; +import { forkParked, ServerActivation, withoutAmbientParentSpan } from "./serverActivation.ts"; it.effect("proves a root is parked before returning and releases it with one gate", () => Effect.scoped( @@ -21,3 +24,30 @@ it.effect("proves a root is parked before returning and releases it with one gat }), ), ); + +it.effect("withoutAmbientParentSpan drops inherited ParentSpan for long-lived roots", () => + Effect.gen(function* () { + const ambient = Tracer.externalSpan({ + traceId: "00000000000000000000000000000001", + spanId: "0000000000000001", + sampled: true, + }); + + const sawParent = yield* Effect.serviceOption(Tracer.ParentSpan).pipe( + Effect.map(Option.isSome), + withoutAmbientParentSpan, + Effect.provideService(Tracer.ParentSpan, ambient), + ); + + expect(sawParent).toBe(false); + + const stillHasParent = yield* Effect.serviceOption(Tracer.ParentSpan).pipe( + Effect.map(Option.isSome), + Effect.provideService(Tracer.ParentSpan, ambient), + ); + expect(stillHasParent).toBe(true); + + const stripped = Context.omit(Tracer.ParentSpan)(Context.make(Tracer.ParentSpan, ambient)); + expect(Context.getOption(stripped, Tracer.ParentSpan)._tag).toBe("None"); + }), +); diff --git a/apps/server/src/serverActivation.ts b/apps/server/src/serverActivation.ts index c068d55e7e76..c178b7303949 100644 --- a/apps/server/src/serverActivation.ts +++ b/apps/server/src/serverActivation.ts @@ -2,25 +2,48 @@ import * as Context from "effect/Context"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import type * as Scope from "effect/Scope"; +import * as Tracer from "effect/Tracer"; export class ServerActivation extends Context.Reference | undefined>( "t3/serverActivation", { defaultValue: () => undefined }, ) {} +/** + * Drop any ambient `Tracer.ParentSpan` before running long-lived roots. + * + * Startup phases (and other short-lived `Effect.withSpan` scopes) often call + * `forkParked` while a parent span is still current. Forked fibers inherit + * that `ParentSpan` for their entire lifetime, so every nested span + * (`ServerSecretStore.get`, RPC handlers, …) keeps parenting under an + * immortal startup span and pins the `LocalFileSpan` object graph — steady + * private-memory growth while "idle". + */ +export const withoutAmbientParentSpan = ( + effect: Effect.Effect, +): Effect.Effect => + Effect.updateContext( + effect, + (context) => Context.omit(Tracer.ParentSpan)(context) as Context.Context>, + ); + /** Forks a long-running root before commit and proves it is parked at the activation boundary. */ export const forkParked = ( effect: Effect.Effect, ): Effect.Effect => Effect.gen(function* () { const activation = yield* ServerActivation; + const detached = withoutAmbientParentSpan(effect); if (activation === undefined) { - yield* Effect.forkScoped(effect); + yield* Effect.forkScoped(detached); return; } const parked = yield* Deferred.make(); yield* Effect.forkScoped( - Deferred.succeed(parked, undefined).pipe(Effect.andThen(activation), Effect.andThen(effect)), + Deferred.succeed(parked, undefined).pipe( + Effect.andThen(activation), + Effect.andThen(detached), + ), ); yield* Deferred.await(parked); }); diff --git a/apps/server/src/serverRuntimeStartup.ts b/apps/server/src/serverRuntimeStartup.ts index 5db2b75556ee..598decd7dd74 100644 --- a/apps/server/src/serverRuntimeStartup.ts +++ b/apps/server/src/serverRuntimeStartup.ts @@ -346,6 +346,9 @@ export const make = (options?: StartupOptions) => ); yield* Effect.logDebug("startup phase: parking orchestration roots at activation"); + // `forkParked` inside these starts detaches ambient ParentSpan so the + // short-lived `server.startup.reactors.start` span can end instead of + // staying pinned on long-lived reactor fibers for the process lifetime. yield* runStartupPhase( "reactors.start", Effect.gen(function* () { From 17d1479669521e0661e5e18cffc7324b4399a49a Mon Sep 17 00:00:00 2001 From: Benchance Date: Wed, 5 Aug 2026 23:51:45 +1000 Subject: [PATCH 2/3] chore(server): trim comments on idle memory-span fix Co-authored-by: Cursor --- apps/server/src/auth/ServerSecretStore.ts | 3 --- apps/server/src/preview/PortScanner.ts | 5 +---- apps/server/src/serverActivation.ts | 11 +---------- apps/server/src/serverRuntimeStartup.ts | 3 --- 4 files changed, 2 insertions(+), 20 deletions(-) diff --git a/apps/server/src/auth/ServerSecretStore.ts b/apps/server/src/auth/ServerSecretStore.ts index a124e5493768..1af1ff9336d7 100644 --- a/apps/server/src/auth/ServerSecretStore.ts +++ b/apps/server/src/auth/ServerSecretStore.ts @@ -168,9 +168,6 @@ export const make = Effect.gen(function* () { const resolveSecretPath = (name: string) => path.join(serverConfig.secretsDir, `${name}.bin`); - // Hot path: called every few seconds from parked runtime fibers. Spans here - // previously nested under immortal startup ParentSpans and dominated idle - // trace volume; keep reads cheap and untraced. const get: ServerSecretStore["Service"]["get"] = (name) => fileSystem.readFile(resolveSecretPath(name)).pipe( Effect.map((bytes) => Option.some(Uint8Array.from(bytes))), diff --git a/apps/server/src/preview/PortScanner.ts b/apps/server/src/preview/PortScanner.ts index 065cc8a9e6b1..361d26f4ce38 100644 --- a/apps/server/src/preview/PortScanner.ts +++ b/apps/server/src/preview/PortScanner.ts @@ -579,10 +579,7 @@ export const make = Effect.gen(function* PortDiscoveryMake() { ), ); - // Single layer-scoped polling fiber. Ticks are no-ops when no client is - // currently retained. Run detached from any ambient ParentSpan (e.g. - // PortDiscovery.make) and without tracing idle no-ops — otherwise every - // 3s tick parents under an immortal span and grows server private memory. + // Idle ticks are no-ops; keep them untraced and detached from ambient ParentSpan (#5410). yield* Effect.forkScoped( withoutAmbientParentSpan( pollTick().pipe( diff --git a/apps/server/src/serverActivation.ts b/apps/server/src/serverActivation.ts index c178b7303949..0b60f9e82f33 100644 --- a/apps/server/src/serverActivation.ts +++ b/apps/server/src/serverActivation.ts @@ -9,16 +9,7 @@ export class ServerActivation extends Context.Reference | un { defaultValue: () => undefined }, ) {} -/** - * Drop any ambient `Tracer.ParentSpan` before running long-lived roots. - * - * Startup phases (and other short-lived `Effect.withSpan` scopes) often call - * `forkParked` while a parent span is still current. Forked fibers inherit - * that `ParentSpan` for their entire lifetime, so every nested span - * (`ServerSecretStore.get`, RPC handlers, …) keeps parenting under an - * immortal startup span and pins the `LocalFileSpan` object graph — steady - * private-memory growth while "idle". - */ +// Clear inherited ParentSpan so long-lived forks don't pin short-lived startup spans (#5410). export const withoutAmbientParentSpan = ( effect: Effect.Effect, ): Effect.Effect => diff --git a/apps/server/src/serverRuntimeStartup.ts b/apps/server/src/serverRuntimeStartup.ts index 598decd7dd74..5db2b75556ee 100644 --- a/apps/server/src/serverRuntimeStartup.ts +++ b/apps/server/src/serverRuntimeStartup.ts @@ -346,9 +346,6 @@ export const make = (options?: StartupOptions) => ); yield* Effect.logDebug("startup phase: parking orchestration roots at activation"); - // `forkParked` inside these starts detaches ambient ParentSpan so the - // short-lived `server.startup.reactors.start` span can end instead of - // staying pinned on long-lived reactor fibers for the process lifetime. yield* runStartupPhase( "reactors.start", Effect.gen(function* () { From 655748bf3a0c3eabec1ceef50b731c51e97ee506 Mon Sep 17 00:00:00 2001 From: Benchance Date: Thu, 6 Aug 2026 00:05:54 +1000 Subject: [PATCH 3/3] fix(server): address PR review on idle span-leak fix Keep active PortDiscovery polls traced, tighten withoutAmbientParentSpan's R, and use Option.isNone in the test. Co-authored-by: Cursor --- apps/server/src/preview/PortScanner.ts | 19 +++++++++---------- apps/server/src/serverActivation.test.ts | 2 +- apps/server/src/serverActivation.ts | 7 +++++-- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/apps/server/src/preview/PortScanner.ts b/apps/server/src/preview/PortScanner.ts index 361d26f4ce38..e9ecd9e66d16 100644 --- a/apps/server/src/preview/PortScanner.ts +++ b/apps/server/src/preview/PortScanner.ts @@ -548,9 +548,8 @@ export const make = Effect.gen(function* PortDiscoveryMake() { ); }; - const pollTick = Effect.fn("PortDiscovery.pollTick")( + const pollTickActive = Effect.fn("PortDiscovery.pollTick")( function* () { - if ((yield* Ref.get(stateRef)).retainCount <= 0) return; const configuredUrls = [ ...new Set( [...(yield* Ref.get(stateRef)).listeners.values()].flatMap( @@ -579,14 +578,14 @@ export const make = Effect.gen(function* PortDiscoveryMake() { ), ); - // Idle ticks are no-ops; keep them untraced and detached from ambient ParentSpan (#5410). + // Idle early-return stays outside Effect.fn so no-op ticks create no span; detach ParentSpan (#5410). + const pollTick = Effect.gen(function* () { + if ((yield* Ref.get(stateRef)).retainCount <= 0) return; + yield* pollTickActive(); + }); + yield* Effect.forkScoped( - withoutAmbientParentSpan( - pollTick().pipe( - Effect.repeat(Schedule.spaced(POLL_INTERVAL)), - Effect.withTracerEnabled(false), - ), - ), + withoutAmbientParentSpan(pollTick.pipe(Effect.repeat(Schedule.spaced(POLL_INTERVAL)))), ); const acquireRetention = Effect.fn("PortDiscovery.retain")(function* () { const wasIdle = yield* Ref.modify(stateRef, (state) => [ @@ -596,7 +595,7 @@ export const make = Effect.gen(function* PortDiscoveryMake() { if (wasIdle) { // Run an immediate scan + broadcast so the new retainer doesn't have // to wait up to POLL_INTERVAL for the first emission. - yield* pollTick(); + yield* pollTick; } }); diff --git a/apps/server/src/serverActivation.test.ts b/apps/server/src/serverActivation.test.ts index 83c8017838fc..f07cfff319c6 100644 --- a/apps/server/src/serverActivation.test.ts +++ b/apps/server/src/serverActivation.test.ts @@ -48,6 +48,6 @@ it.effect("withoutAmbientParentSpan drops inherited ParentSpan for long-lived ro expect(stillHasParent).toBe(true); const stripped = Context.omit(Tracer.ParentSpan)(Context.make(Tracer.ParentSpan, ambient)); - expect(Context.getOption(stripped, Tracer.ParentSpan)._tag).toBe("None"); + expect(Option.isNone(Context.getOption(stripped, Tracer.ParentSpan))).toBe(true); }), ); diff --git a/apps/server/src/serverActivation.ts b/apps/server/src/serverActivation.ts index 0b60f9e82f33..edabe2f77144 100644 --- a/apps/server/src/serverActivation.ts +++ b/apps/server/src/serverActivation.ts @@ -12,10 +12,13 @@ export class ServerActivation extends Context.Reference | un // Clear inherited ParentSpan so long-lived forks don't pin short-lived startup spans (#5410). export const withoutAmbientParentSpan = ( effect: Effect.Effect, -): Effect.Effect => +): Effect.Effect> => Effect.updateContext( effect, - (context) => Context.omit(Tracer.ParentSpan)(context) as Context.Context>, + (context) => + Context.omit(Tracer.ParentSpan)(context) as Context.Context< + Exclude, Tracer.ParentSpan> + >, ); /** Forks a long-running root before commit and proves it is parked at the activation boundary. */