diff --git a/apps/desktop/src/backend/DesktopBackendConfiguration.ts b/apps/desktop/src/backend/DesktopBackendConfiguration.ts index b3bec8830ec6..c2012299802d 100644 --- a/apps/desktop/src/backend/DesktopBackendConfiguration.ts +++ b/apps/desktop/src/backend/DesktopBackendConfiguration.ts @@ -118,6 +118,9 @@ const WSL_FORWARDED_ENV_NAMES = [ "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL", "OTEL_EXPORTER_OTLP_LOGS_PROTOCOL", + "OTEL_TRACES_EXPORTER", + "OTEL_METRICS_EXPORTER", + "OTEL_LOGS_EXPORTER", ] as const; const WSL_SERVER_SYSTEM_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; diff --git a/apps/mobile/src/connection/environment-cache-store.ts b/apps/mobile/src/connection/environment-cache-store.ts index 44a1d3114532..6e56524770b6 100644 --- a/apps/mobile/src/connection/environment-cache-store.ts +++ b/apps/mobile/src/connection/environment-cache-store.ts @@ -1,6 +1,7 @@ import { ConnectionPersistenceError, EnvironmentCacheStore, + encodeShellSnapshotForCache, } from "@t3tools/client-runtime/platform"; import { type EnvironmentId, @@ -56,7 +57,6 @@ const StoredVcsRefs = Schema.Struct({ const decodeStoredShellSnapshot = Schema.decodeUnknownEffect( Schema.fromJsonString(StoredShellSnapshot), ); -const encodeStoredShellSnapshot = Schema.encodeEffect(Schema.fromJsonString(StoredShellSnapshot)); const decodeStoredThreadSnapshot = Schema.decodeUnknownEffect( Schema.fromJsonString(StoredThreadSnapshot), ); @@ -136,11 +136,18 @@ export const make = Effect.fn("MobileEnvironmentCacheStore.make")(function* () { }).pipe(Effect.tap(() => Effect.promise(() => projectFaviconDatabaseCache.hydrate()))), ), saveShell: Effect.fn("MobileEnvironmentCache.saveShell")(function* (environmentId, snapshot) { - const payload = yield* encodeStoredShellSnapshot({ - schemaVersion: SHELL_SNAPSHOT_CACHE_SCHEMA_VERSION, - environmentId, - snapshot, - }).pipe(Effect.mapError((cause) => persistenceError("save-shell", cause))); + const encodedSnapshot = yield* encodeShellSnapshotForCache(snapshot).pipe( + Effect.mapError((cause) => persistenceError("save-shell", cause)), + ); + const payload = yield* Effect.try({ + try: () => + JSON.stringify({ + schemaVersion: SHELL_SNAPSHOT_CACHE_SCHEMA_VERSION, + environmentId, + snapshot: encodedSnapshot, + } satisfies typeof StoredShellSnapshot.Encoded), + catch: (cause) => persistenceError("save-shell", cause), + }); yield* database .saveCache(environmentId, "shell", "snapshot", SHELL_SNAPSHOT_CACHE_SCHEMA_VERSION, payload) .pipe(Effect.mapError(mapDatabaseError("save-shell"))); diff --git a/apps/mobile/src/features/connection/ConnectionFormField.test.tsx b/apps/mobile/src/features/connection/ConnectionFormField.test.tsx new file mode 100644 index 000000000000..343d44cf376b --- /dev/null +++ b/apps/mobile/src/features/connection/ConnectionFormField.test.tsx @@ -0,0 +1,42 @@ +import type { ReactNode } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { describe, expect, it, vi } from "vite-plus/test"; + +vi.mock("react-native", () => ({ + View: ({ children }: { children: ReactNode }) =>
{children}
, +})); +vi.mock("../../components/AppText", () => ({ + AppText: ({ + accessibilityElementsHidden, + importantForAccessibility, + children, + }: { + accessibilityElementsHidden?: boolean; + importantForAccessibility?: string; + children: ReactNode; + }) => ( + + {children} + + ), + AppTextInput: ({ accessibilityLabel }: { accessibilityLabel?: string }) => ( + + ), +})); + +import { ConnectionFormField } from "./ConnectionFormField"; + +describe("ConnectionFormField accessibility", () => { + it.each(["Host", "Pairing code"])("exposes %s once as the input name", (label) => { + const markup = renderToStaticMarkup( + , + ); + + expect(markup).toContain(``); + expect(markup).toContain(``); + }); +}); diff --git a/apps/mobile/src/features/connection/ConnectionFormField.tsx b/apps/mobile/src/features/connection/ConnectionFormField.tsx index 0d5e21148e54..b492dd9ce43c 100644 --- a/apps/mobile/src/features/connection/ConnectionFormField.tsx +++ b/apps/mobile/src/features/connection/ConnectionFormField.tsx @@ -3,7 +3,7 @@ import { View } from "react-native"; import { AppText, AppTextInput, type AppTextInputProps } from "../../components/AppText"; import { cn } from "../../lib/cn"; -type ConnectionFormFieldProps = Omit & { +type ConnectionFormFieldProps = Omit & { readonly label: string; readonly className?: string; }; @@ -12,12 +12,16 @@ type ConnectionFormFieldProps = Omit & { export function ConnectionFormField({ label, className, ...inputProps }: ConnectionFormFieldProps) { return ( - + {label} diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index 0c3b93219a60..0f3f506b4b16 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -683,10 +683,12 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread const isSplitLayout = layoutVariant === "split"; const contentMaxWidth = isSplitLayout ? CHAT_CONTENT_MAX_WIDTH : undefined; const workspaceContentWidth = useWorkspaceContentWidth(); + // Clearing animated width can retain the unfolded width after Android resumes folded. + // Assign both layouts explicitly so the dock always follows its current parent. const composerWidthStyle = useAnimatedStyle(() => isSplitLayout && workspaceContentWidth !== null - ? { width: workspaceContentWidth.value, right: undefined } - : { width: undefined, right: 0 }, + ? { width: workspaceContentWidth.value } + : { width: "100%" }, ); const selectedInstanceId = props.selectedThread.modelSelection.instanceId; useStreamingHaptics(props.selectedThread.id, props.selectedThreadFeed); @@ -968,7 +970,7 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread {/* No paddingTop here: the overlay's measured height becomes the list's bottom inset, so any padding above the pill/composer diff --git a/apps/mobile/src/features/threads/ThreadFeed.tsx b/apps/mobile/src/features/threads/ThreadFeed.tsx index 20848cd2ad10..83b0bef9c022 100644 --- a/apps/mobile/src/features/threads/ThreadFeed.tsx +++ b/apps/mobile/src/features/threads/ThreadFeed.tsx @@ -2477,7 +2477,10 @@ export const ThreadFeed = memo(function ThreadFeed(props: ThreadFeedProps) { // content-inset override. Seed the fresh instance synchronously with the // current overlay height before the scroll integration's next reaction; // on Android the declarative contentInset floor covers this same window. - const listMountKey = `${feedThreadKey}:${presentedFeed.length === 0 ? "empty" : "filled"}`; + // The thinking row a running thread shows while its messages load is not + // content: the list must still remount, and so open at the end, when they + // arrive. + const listMountKey = `${feedThreadKey}:${presentedFeed.some((entry) => entry.type !== "thinking") ? "filled" : "empty"}`; useLayoutEffect(() => { const bottom = props.contentInsetEndAdjustment.value; if (bottom > 0) { diff --git a/apps/mobile/src/features/threads/thread-list-v2-items.tsx b/apps/mobile/src/features/threads/thread-list-v2-items.tsx index eb089b95060a..09ffe404ee79 100644 --- a/apps/mobile/src/features/threads/thread-list-v2-items.tsx +++ b/apps/mobile/src/features/threads/thread-list-v2-items.tsx @@ -1028,27 +1028,25 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { )} {pr ? ( - {pr.kind === "stack" || pr.others > 0 ? ( - - ) : null} + - {pr.kind === "stack" || pr.others > 0 ? pr.label : `#${pr.label}`} + {pr.label} ) : null} diff --git a/apps/mobile/src/features/threads/threadListV2.ts b/apps/mobile/src/features/threads/threadListV2.ts index 9a876c899ac4..8c4dd0b0f097 100644 --- a/apps/mobile/src/features/threads/threadListV2.ts +++ b/apps/mobile/src/features/threads/threadListV2.ts @@ -14,6 +14,7 @@ import { sortActiveThreadsByOrderKey, resolveSettledThreadTimestamp, sortPinnedThreadsByOrderKey, + sortSettledThreads, } from "@t3tools/client-runtime/state/thread-sort"; import type { EnvironmentId, ProjectId } from "@t3tools/contracts"; @@ -606,11 +607,7 @@ export function buildThreadListV2Items(input: { : orderedSnoozed.filter( (thread) => `${thread.environmentId}:${thread.id}` === selectedThreadKey, ); - const orderedSettled = [...settled].sort( - (left, right) => - parseTimestampMs(resolveSettledThreadTimestamp(right) ?? "") - - parseTimestampMs(resolveSettledThreadTimestamp(left) ?? ""), - ); + const orderedSettled = sortSettledThreads(settled); const settledLimit = input.settledLimit ?? Number.POSITIVE_INFINITY; const pagedSettled = orderedSettled.length > settledLimit ? orderedSettled.slice(0, settledLimit) : orderedSettled; diff --git a/apps/mobile/src/features/usage/UsageRouteScreen.tsx b/apps/mobile/src/features/usage/UsageRouteScreen.tsx index 2d5e5398bc51..4582c3135690 100644 --- a/apps/mobile/src/features/usage/UsageRouteScreen.tsx +++ b/apps/mobile/src/features/usage/UsageRouteScreen.tsx @@ -1,6 +1,7 @@ import { ScreenScrollView as ScrollView } from "../../components/ScreenScrollView"; import { EnvironmentId, USAGE_CONTRACT_VERSION } from "@t3tools/contracts"; import { type RouteProp, useIsFocused, useNavigation, useRoute } from "@react-navigation/native"; +import { cursorKeychainAccessEnvironments } from "@t3tools/client-runtime/state/usage"; import { isCompatibleUsageContractVersion, isModelCostUnknown, @@ -15,6 +16,7 @@ import { formatHourShort, formatPercent, formatTokens, + formatUsageContractMismatch, formatUsd, makeWindow, } from "@t3tools/shared/usageFormat"; @@ -101,9 +103,7 @@ export function UsageRouteScreen() { ); const isFocused = useIsFocused(); const limits = useRefreshLimits(selectedEnvironmentIds, isFocused && tab === "limits"); - const cursorAccessEnvironments = selectedEnvironments.filter((environment) => - environment.summary?.sources.some((source) => source.action === "enableCursorKeychain"), - ); + const cursorAccessEnvironments = cursorKeychainAccessEnvironments(selectedEnvironments); const refreshAfterCursorEnable = () => { void refresh(); void limits.refreshAfterEnable(); @@ -753,7 +753,12 @@ function usageEnvironmentStatus(environment: EnvironmentUsageStatus): string { environment.summary && !isCompatibleUsageContractVersion(environment.summary.contractVersion, USAGE_CONTRACT_VERSION) ) { - return "Older server · excluded from usage totals"; + return formatUsageContractMismatch(environment.label, { + direction: + environment.summary.contractVersion < USAGE_CONTRACT_VERSION + ? "serverBehind" + : "clientBehind", + }); } if (!environment.isConnected) return environment.summary ? "Disconnected · showing saved usage" : "Waiting for connection…"; diff --git a/apps/mobile/src/state/usage.ts b/apps/mobile/src/state/usage.ts index d49c26a40a44..c5895d353fef 100644 --- a/apps/mobile/src/state/usage.ts +++ b/apps/mobile/src/state/usage.ts @@ -16,7 +16,7 @@ import { type UsageSummary, type UsageSummaryInput, } from "@t3tools/contracts"; -import { refreshUsage } from "@t3tools/client-runtime/state/usage"; +import { needsCursorKeychainAccess, refreshUsage } from "@t3tools/client-runtime/state/usage"; import { mergeUsage, type EnvironmentUsage, type MergedUsage } from "@t3tools/shared/usageMerge"; import * as Option from "effect/Option"; import { AsyncResult, Atom } from "effect/unstable/reactivity"; @@ -33,6 +33,7 @@ export interface EnvironmentUsageStatus { readonly isConnected: boolean; readonly error: string | null; readonly summary: UsageSummary | null; + readonly needsCursorKeychainAccess: boolean; } /** @@ -50,13 +51,18 @@ const usageByWindowAtom = Atom.family((windowKey: string) => const statuses: EnvironmentUsageStatus[] = []; for (const [environmentId, presentation] of presentations) { const result = get(serverEnvironment.usageSummary({ environmentId, input })); + const summary = Option.getOrNull(AsyncResult.value(result)); statuses.push({ environmentId, label: presentation.entry.target.label, isPending: result.waiting, isConnected: presentation.connection.phase === "connected", error: result._tag === "Failure" ? "This environment could not report usage." : null, - summary: Option.getOrNull(AsyncResult.value(result)), + summary, + needsCursorKeychainAccess: needsCursorKeychainAccess( + summary, + get(serverEnvironment.providersValueAtom(environmentId)), + ), }); } return statuses; diff --git a/apps/server/integration/OrchestrationEngineHarness.integration.ts b/apps/server/integration/OrchestrationEngineHarness.integration.ts index 1e116a91fea7..5e3b4c6e4afe 100644 --- a/apps/server/integration/OrchestrationEngineHarness.integration.ts +++ b/apps/server/integration/OrchestrationEngineHarness.integration.ts @@ -420,6 +420,7 @@ export const makeOrchestrationIntegrationHarness = ( Layer.provideMerge( Layer.succeed(AgentAwarenessRelay.AgentAwarenessRelay, { publishThread: () => Effect.void, + requestCatchUp: () => Effect.void, start: () => Effect.void, }), ), diff --git a/apps/server/package.json b/apps/server/package.json index 56f9629e631e..75a0f1a068f8 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -33,7 +33,7 @@ "ai": "7.0.111", "diff": "8.0.3", "effect": "catalog:", - "node-pty": "^1.1.0", + "node-pty": "^1.2.0-beta.15", "stream-chain": "^4.2.5", "stream-json": "3.6.0", "yaml": "catalog:", diff --git a/apps/server/src/auth/dpop.ts b/apps/server/src/auth/dpop.ts index a841f4fd444c..5e026f584273 100644 --- a/apps/server/src/auth/dpop.ts +++ b/apps/server/src/auth/dpop.ts @@ -18,6 +18,9 @@ import { } from "./EnvironmentAuth.ts"; import * as ServerSecretStore from "./ServerSecretStore.ts"; +/** Secret store name prefix of DPoP replay markers. The server prunes expired ones. */ +export const DPOP_REPLAY_MARKER_PREFIX = "dpop-proof-"; + export const mapDpopFailureReason = (code: DpopVerificationFailureCodeType): DpopFailureReason => { switch (code) { case "time_window": @@ -96,7 +99,7 @@ export const verifyRequestDpopProof = (input: { ); yield* secretStore .create( - `dpop-proof-${replayKey}`, + `${DPOP_REPLAY_MARKER_PREFIX}${replayKey}`, new TextEncoder().encode( [ `thumbprint=${result.thumbprint}`, diff --git a/apps/server/src/auth/replayMarkers.test.ts b/apps/server/src/auth/replayMarkers.test.ts new file mode 100644 index 000000000000..76c31b0162bd --- /dev/null +++ b/apps/server/src/auth/replayMarkers.test.ts @@ -0,0 +1,92 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as TestClock from "effect/testing/TestClock"; + +import * as ServerConfig from "../config.ts"; +import { pruneExpiredReplayMarkers, REPLAY_MARKER_MAX_AGE } from "./replayMarkers.ts"; +import * as ServerSecretStore from "./ServerSecretStore.ts"; + +// Every secret name the server stores today. The last three stand for names +// built from an id at runtime. +const REAL_SECRET_NAMES = [ + "server-signing-key", + "asset-access-signing-key", + "cloud-cli-oauth-token", + "cloud-cli-desired-link", + "cloud-link-ed25519-key-pair", + "cloud-link-ed25519-private-key", + "cloud-link-ed25519-public-key", + "cloud-mint-ed25519-public-key", + "cloud-endpoint-runtime-config", + "cloud-endpoint-confirmed-origin", + "cloud-linked-user-id", + "cloud-relay-url", + "cloud-relay-issuer", + "cloud-relay-environment-credential", + "cloud-publish-agent-activity", + "provider-env-Y29kZXg-T1BFTkFJX0FQSV9LRVk", + "provider-auth-0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0", + "usage-limit-source-aHVi", +]; + +it.layer(NodeServices.layer)("replayMarkers", (it) => { + it.effect("prunes only replay markers older than the max age", () => + Effect.gen(function* () { + const secretStore = yield* ServerSecretStore.ServerSecretStore; + const { secretsDir } = yield* ServerConfig.ServerConfig; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const now = DateTime.makeUnsafe("2026-01-01T00:00:00Z"); + const setAge = (fileName: string, age: Duration.Duration) => { + const mtime = DateTime.toDateUtc(DateTime.subtractDuration(now, age)); + return fileSystem.utimes(path.join(secretsDir, fileName), mtime, mtime); + }; + const writeAged = (name: string, age: Duration.Duration) => + secretStore + .create(name, Uint8Array.from([1])) + .pipe(Effect.andThen(setAge(`${name}.bin`, age))); + + const justExpired = Duration.sum(REPLAY_MARKER_MAX_AGE, Duration.seconds(1)); + const expiredMarkers = [ + "dpop-proof-old", + "cloud-mint-jti-old", + "cloud-mint-nonce-old", + "cloud-health-jti-old", + "cloud-health-nonce-old", + ]; + for (const name of expiredMarkers) yield* writeAged(name, justExpired); + yield* writeAged("dpop-proof-at-max-age", REPLAY_MARKER_MAX_AGE); + for (const name of REAL_SECRET_NAMES) yield* writeAged(name, Duration.days(30)); + const pendingSetFile = "dpop-proof-pending.bin.0000.tmp"; + yield* fileSystem.writeFile(path.join(secretsDir, pendingSetFile), Uint8Array.from([1])); + yield* setAge(pendingSetFile, Duration.days(30)); + yield* TestClock.setTime(DateTime.toEpochMillis(now)); + + yield* pruneExpiredReplayMarkers(); + + const remaining = yield* fileSystem.readDirectory(secretsDir); + assert.deepStrictEqual( + remaining.toSorted(), + [ + ...REAL_SECRET_NAMES.map((name) => `${name}.bin`), + "dpop-proof-at-max-age.bin", + pendingSetFile, + ].toSorted(), + ); + }).pipe( + Effect.provide( + ServerSecretStore.layer.pipe( + Layer.provideMerge( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-replay-markers-test-" }), + ), + ), + ), + ), + ); +}); diff --git a/apps/server/src/auth/replayMarkers.ts b/apps/server/src/auth/replayMarkers.ts new file mode 100644 index 000000000000..67b56b722334 --- /dev/null +++ b/apps/server/src/auth/replayMarkers.ts @@ -0,0 +1,77 @@ +import * as Clock from "effect/Clock"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Schedule from "effect/Schedule"; + +import { CLOUD_REPLAY_MARKER_PREFIXES } from "../cloud/http.ts"; +import * as ServerConfig from "../config.ts"; +import { forkParked } from "../serverActivation.ts"; +import { DPOP_REPLAY_MARKER_PREFIX } from "./dpop.ts"; + +const REPLAY_MARKER_PREFIXES = [DPOP_REPLAY_MARKER_PREFIX, ...CLOUD_REPLAY_MARKER_PREFIXES]; + +/** + * How long a replay marker stays on disk. A marker only matters while its proof + * can pass the time check (about 5 minutes for DPoP, 7 for cloud proofs). After + * that, the time check rejects a replay by itself. The sweep and the time check + * both use the wall clock, so a pruned marker can let a replay through only if + * the clock moves back by almost a day, or if the filesystem stamps mtimes almost + * a day behind. Markers are files, so a restart does not reset them. + */ +export const REPLAY_MARKER_MAX_AGE = Duration.days(1); + +/** + * Deletes replay markers whose mtime is older than `REPLAY_MARKER_MAX_AGE`. + * `ServerSecretStore` saves each secret as `.bin`, so only + * `*.bin` names match. Other secrets and the `*.bin..tmp` + * files that `set` writes are never touched. + */ +export const pruneExpiredReplayMarkers = Effect.fn("replayMarkers.pruneExpired")(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { secretsDir } = yield* ServerConfig.ServerConfig; + const cutoff = (yield* Clock.currentTimeMillis) - Duration.toMillis(REPLAY_MARKER_MAX_AGE); + const markers = (yield* fileSystem.readDirectory(secretsDir)).filter( + (name) => + name.endsWith(".bin") && REPLAY_MARKER_PREFIXES.some((prefix) => name.startsWith(prefix)), + ); + // `partition` visits every marker, so one locked file does not stop the sweep. + const [failures, removed] = yield* Effect.partition(markers, (name) => { + const markerPath = path.join(secretsDir, name); + return fileSystem.stat(markerPath).pipe( + Effect.flatMap((info) => + Option.exists(info.mtime, (mtime) => mtime.getTime() < cutoff) + ? fileSystem.remove(markerPath).pipe(Effect.as(true)) + : Effect.succeed(false), + ), + Effect.catchReason("PlatformError", "NotFound", () => Effect.succeed(false)), + ); + }); + yield* Effect.annotateCurrentSpan({ + "replay_markers.matched": markers.length, + "replay_markers.removed": removed.filter(Boolean).length, + "replay_markers.failed": failures.length, + }); + if (failures.length > 0) { + yield* Effect.logWarning("Failed to prune some replay markers", { + failed: failures.length, + cause: failures[0], + }); + } +}); + +/** Prunes expired replay markers after server activation, then every hour. */ +export const layer = Layer.effectDiscard( + forkParked( + pruneExpiredReplayMarkers().pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to prune expired replay markers", { cause }), + ), + Effect.repeat(Schedule.spaced(Duration.hours(1))), + ), + ), +); diff --git a/apps/server/src/cli/trace.ts b/apps/server/src/cli/trace.ts index 7b8a83592ef8..61a209471d6e 100644 --- a/apps/server/src/cli/trace.ts +++ b/apps/server/src/cli/trace.ts @@ -13,11 +13,10 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; -import * as Stream from "effect/Stream"; import { Command, Flag } from "effect/unstable/cli"; import * as ServerConfig from "../config.ts"; -import { toRotatedTracePaths, TraceFileReadError } from "../diagnostics/TraceDiagnostics.ts"; +import { streamTraceFileLines, toRotatedTracePaths } from "../diagnostics/TraceDiagnostics.ts"; import { resolveBaseDir } from "../os-jank.ts"; import { baseDirFlag, DurationFromString, traceFileConfig, traceMaxFilesConfig } from "./config.ts"; @@ -187,27 +186,9 @@ const traceSummaryCommand = Command.make("summary", { ? (yield* Clock.currentTimeMillis) - Duration.toMillis(flags.since.value) : undefined; const summarizer = makeTraceSpanSummary(sinceMs); - // Stream each file so only one chunk of text is in memory at a time. yield* Effect.forEach( toRotatedTracePaths(traceFilePath, yield* traceMaxFilesConfig), - (path) => - fs.stream(path).pipe( - Stream.decodeText, - Stream.splitLines, - Stream.runForEachArray((lines) => Effect.sync(() => lines.forEach(summarizer.addLine))), - Effect.catchTags({ - PlatformError: (cause) => - cause.reason._tag === "NotFound" - ? Effect.void - : Effect.fail( - new TraceFileReadError({ - traceFilePath: path, - causeTag: cause.reason._tag, - cause, - }), - ), - }), - ), + (path) => streamTraceFileLines(fs, path, summarizer.addLine), { discard: true }, ); const summary = summarizer.finish(); diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index 64a512a33aba..1d6b78e56cef 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -33,6 +33,7 @@ import { type ServiceUpdateRecord, } from "./serviceProtocol.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts"; import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts"; import * as CliTokenManager from "./CliTokenManager.ts"; import { @@ -81,6 +82,12 @@ const storeFailure = (tag: "AlreadyExists" | "PermissionDenied") => }); const unusedSecretStoreOperation = () => Effect.die("unused secret-store operation"); +// Linking wakes the awareness relay; these tests do not run it. +const idleAwarenessRelay = AgentAwarenessRelay.AgentAwarenessRelay.of({ + publishThread: () => Effect.void, + requestCatchUp: () => Effect.void, + start: () => Effect.void, +}); const decodeManagedTunnelRecoveryRegistration = Schema.decodeUnknownEffect( Schema.fromJsonString(RelayManagedEndpointRecoveryRegistrationRequest), ); @@ -262,6 +269,7 @@ describe("reconcileDesiredCloudLink", () => { HttpClient.HttpClient, HttpClient.make(() => unusedSecretStoreOperation()), ), + Effect.provideService(AgentAwarenessRelay.AgentAwarenessRelay, idleAwarenessRelay), Effect.provide(NodeServices.layer), ), ); @@ -375,6 +383,7 @@ describe("releaseManagedTunnelOnShutdown", () => { (effect: Effect.Effect) => effect.pipe( Effect.provideService(ServerSecretStore.ServerSecretStore, harness.store), + Effect.provideService(AgentAwarenessRelay.AgentAwarenessRelay, idleAwarenessRelay), Effect.provideService( ServerEnvironment.ServerEnvironment, ServerEnvironment.ServerEnvironment.of({ diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index b4c366a500c1..a7e3de8c7283 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -69,6 +69,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import { requireEnvironmentScope } from "../auth/http.ts"; import * as ServerConfig from "../config.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; import { SERVICE_STATE_FILE, @@ -104,6 +105,13 @@ const CLOUD_MINT_NONCE_PREFIX = "cloud-mint-nonce-"; const CLOUD_MINT_JTI_PREFIX = "cloud-mint-jti-"; const CLOUD_HEALTH_NONCE_PREFIX = "cloud-health-nonce-"; const CLOUD_HEALTH_JTI_PREFIX = "cloud-health-jti-"; +/** Secret store name prefixes of cloud replay markers. The server prunes expired ones. */ +export const CLOUD_REPLAY_MARKER_PREFIXES = [ + CLOUD_MINT_NONCE_PREFIX, + CLOUD_MINT_JTI_PREFIX, + CLOUD_HEALTH_NONCE_PREFIX, + CLOUD_HEALTH_JTI_PREFIX, +] as const; const CLOUD_PROOF_MAX_LIFETIME_SECONDS = 5 * 60; const CLOUD_PROOF_CLOCK_SKEW_SECONDS = 60; // The desktop app stops its backends within seconds of writing the marker. @@ -404,6 +412,7 @@ interface CloudHttpDependencies { readonly environmentAuth: EnvironmentAuth.EnvironmentAuth["Service"]; readonly cliTokenManager: CliTokenManager.CloudCliTokenManager["Service"]; readonly httpClient: HttpClient.HttpClient; + readonly awarenessRelay: AgentAwarenessRelay.AgentAwarenessRelay["Service"]; } const cloudHttpDependencies = Effect.gen(function* () { @@ -414,6 +423,7 @@ const cloudHttpDependencies = Effect.gen(function* () { environmentAuth: yield* EnvironmentAuth.EnvironmentAuth, cliTokenManager: yield* CliTokenManager.CloudCliTokenManager, httpClient: yield* HttpClient.HttpClient, + awarenessRelay: yield* AgentAwarenessRelay.AgentAwarenessRelay, } satisfies CloudHttpDependencies; }); @@ -661,6 +671,7 @@ const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(fu CLOUD_MINT_PUBLIC_KEY, stringToBytes(payload.cloudMintPublicKey), ); + yield* dependencies.awarenessRelay.requestCatchUp(); if (payload.endpointRuntime) { const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime); yield* dependencies.secrets.set( @@ -1342,6 +1353,7 @@ const cloudPreferencesHandler = Effect.fn("environment.cloud.preferences")( PUBLISH_AGENT_ACTIVITY_SECRET, stringToBytes(String(payload.publishAgentActivity)), ); + yield* dependencies.awarenessRelay.requestCatchUp(); return yield* readCloudLinkState(dependencies); }, Effect.catchIf( diff --git a/apps/server/src/diagnostics/TraceDiagnostics.test.ts b/apps/server/src/diagnostics/TraceDiagnostics.test.ts index 70bb4dc815c3..3b86841b99ba 100644 --- a/apps/server/src/diagnostics/TraceDiagnostics.test.ts +++ b/apps/server/src/diagnostics/TraceDiagnostics.test.ts @@ -7,6 +7,7 @@ import * as Logger from "effect/Logger"; import * as Option from "effect/Option"; import * as PlatformError from "effect/PlatformError"; import * as References from "effect/References"; +import * as Stream from "effect/Stream"; import * as TraceDiagnostics from "./TraceDiagnostics.ts"; @@ -40,72 +41,78 @@ function record(input: { }); } +const traceFilePath = "/tmp/server.trace.ndjson"; +const readAt = DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"); + +/** Aggregates whole lines in memory, the way diagnostics read traces before streaming. */ +function aggregateLines(lines: ReadonlyArray) { + const aggregator = TraceDiagnostics.makeTraceDiagnosticsAggregator(); + lines.forEach(aggregator.addLine); + return aggregator.finish({ + traceFilePath, + scannedFilePaths: TraceDiagnostics.toRotatedTracePaths(traceFilePath, 1), + readAt, + }); +} + +/** Reads the trace file and one rotated backup through a fake file system. */ +function readTraces(fileSystem: Partial) { + return TraceDiagnostics.readTraceDiagnostics({ traceFilePath, maxFiles: 1, readAt }).pipe( + Effect.provide(TraceDiagnostics.layer.pipe(Layer.provide(FileSystem.layerNoop(fileSystem)))), + ); +} + describe("TraceDiagnostics", () => { it.effect("aggregates failures, slow spans, log levels, and parse errors", () => Effect.sync(() => { - const diagnostics = TraceDiagnostics.aggregateTraceDiagnostics({ - traceFilePath: "/tmp/server.trace.ndjson", - readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"), - slowSpanThresholdMs: 1_000, - files: [ - { - path: "/tmp/server.trace.ndjson.1", - text: [ - record({ - name: "server.getConfig", - traceId: "trace-a", - spanId: "span-a", - startMs: 1_000, - durationMs: 50, - }), - "not-json", - ].join("\n"), - }, - { - path: "/tmp/server.trace.ndjson", - text: [ - record({ - name: "orchestration.dispatch", - traceId: "trace-b", - spanId: "span-b", - startMs: 2_000, - durationMs: 1_500, - exit: { _tag: "Failure", cause: "Provider crashed" }, - events: [ - { - name: "provider failed", - timeUnixNano: ns(3_400), - attributes: { "effect.logLevel": "Error" }, - }, - ], - }), - record({ - name: "orchestration.dispatch", - traceId: "trace-c", - spanId: "span-c", - startMs: 4_000, - durationMs: 250, - exit: { _tag: "Failure", cause: "Provider crashed" }, - }), - record({ - name: "git.status", - traceId: "trace-d", - spanId: "span-d", - startMs: 5_000, - durationMs: 25, - exit: { _tag: "Interrupted", cause: "Interrupted" }, - events: [ - { - name: "status delayed", - timeUnixNano: ns(5_010), - attributes: { "effect.logLevel": "Warning" }, - }, - ], - }), - ].join("\n"), - }, - ], - }); + const diagnostics = aggregateLines([ + record({ + name: "server.getConfig", + traceId: "trace-a", + spanId: "span-a", + startMs: 1_000, + durationMs: 50, + }), + "not-json", + record({ + name: "orchestration.dispatch", + traceId: "trace-b", + spanId: "span-b", + startMs: 2_000, + durationMs: 1_500, + exit: { _tag: "Failure", cause: "Provider crashed" }, + events: [ + { + name: "provider failed", + timeUnixNano: ns(3_400), + attributes: { "effect.logLevel": "Error" }, + }, + ], + }), + record({ + name: "orchestration.dispatch", + traceId: "trace-c", + spanId: "span-c", + startMs: 4_000, + durationMs: 250, + exit: { _tag: "Failure", cause: "Provider crashed" }, + }), + record({ + name: "git.status", + traceId: "trace-d", + spanId: "span-d", + startMs: 5_000, + durationMs: 25, + exit: { _tag: "Interrupted", cause: "Interrupted" }, + events: [ + { + name: "status delayed", + timeUnixNano: ns(5_010), + attributes: { "effect.logLevel": "Warning" }, + }, + ], + }), + ]); assert.equal(diagnostics.recordCount, 4); assert.equal(DateTime.formatIso(diagnostics.readAt), "2026-05-05T10:00:00.000Z"); @@ -139,47 +146,110 @@ describe("TraceDiagnostics", () => { ); it.effect("returns a not-found diagnostic when no files are available", () => - Effect.sync(() => { - const diagnostics = TraceDiagnostics.aggregateTraceDiagnostics({ - traceFilePath: "/tmp/missing.trace.ndjson", - readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"), - files: [], - }); + Effect.gen(function* () { + const diagnostics = yield* readTraces({}); assert.equal(diagnostics.recordCount, 0); assert.equal(Option.getOrUndefined(diagnostics.error)?.kind, "trace-file-not-found"); }), ); - it.effect("preserves full failure causes and log messages", () => - Effect.sync(() => { - const longCause = `VcsProcessSpawnError: ${"missing executable ".repeat(80)}`.trim(); - const longMessage = `provider warning: ${"retrying command ".repeat(80)}`.trim(); - const diagnostics = TraceDiagnostics.aggregateTraceDiagnostics({ - traceFilePath: "/tmp/server.trace.ndjson", - readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"), - files: [ - { - path: "/tmp/server.trace.ndjson", - text: record({ - name: "VcsProcess.run", - traceId: "trace-long", - spanId: "span-long", + it.effect("streams rotated files into the same result as reading them whole", () => + Effect.gen(function* () { + // CRLF and LF endings plus multi-byte text, served one byte per chunk so + // chunks split lines, line endings, and characters. + const files = new Map([ + [ + `${traceFilePath}.1`, + [ + record({ + name: "server.getConfig", + traceId: "trace-a", + spanId: "span-a", startMs: 1_000, + durationMs: 50, + }), + "not-json", + record({ + name: "orchestration.dispatch", + traceId: "trace-b", + spanId: "span-b", + startMs: 2_000, + durationMs: 1_500, + exit: { _tag: "Failure", cause: "Provider crashed: café 🔥" }, + }), + "", + ].join("\r\n"), + ], + [ + traceFilePath, + [ + record({ + name: "git.status", + traceId: "trace-c", + spanId: "span-c", + startMs: 3_000, durationMs: 25, - exit: { _tag: "Failure", cause: longCause }, + exit: { _tag: "Interrupted", cause: "Interrupted" }, events: [ { - name: longMessage, - timeUnixNano: ns(1_010), + name: "status delayed ⏳", + timeUnixNano: ns(3_010), attributes: { "effect.logLevel": "Warning" }, }, ], }), - }, + "", + record({ + name: "orchestration.dispatch", + traceId: "trace-d", + spanId: "span-d", + startMs: 4_000, + durationMs: 250, + exit: { _tag: "Failure", cause: "Provider crashed: café 🔥" }, + }), + ].join("\n"), ], + ]); + const encoder = new TextEncoder(); + + const diagnostics = yield* readTraces({ + stream: (path) => + Stream.fromIterable( + Array.from(encoder.encode(files.get(path)), (byte) => Uint8Array.of(byte)), + ), }); + assert.equal(diagnostics.recordCount, 4); + assert.deepStrictEqual( + diagnostics, + aggregateLines([...files.values()].flatMap((text) => text.split(/\r?\n/))), + ); + }), + ); + + it.effect("preserves full failure causes and log messages", () => + Effect.sync(() => { + const longCause = `VcsProcessSpawnError: ${"missing executable ".repeat(80)}`.trim(); + const longMessage = `provider warning: ${"retrying command ".repeat(80)}`.trim(); + const diagnostics = aggregateLines([ + record({ + name: "VcsProcess.run", + traceId: "trace-long", + spanId: "span-long", + startMs: 1_000, + durationMs: 25, + exit: { _tag: "Failure", cause: longCause }, + events: [ + { + name: longMessage, + timeUnixNano: ns(1_010), + attributes: { "effect.logLevel": "Warning" }, + }, + ], + }), + ]); + assert.equal(diagnostics.latestFailures[0]?.cause, longCause); assert.equal(diagnostics.commonFailures[0]?.cause, longCause); assert.equal(diagnostics.latestWarningAndErrorLogs[0]?.message, longMessage); @@ -188,45 +258,34 @@ describe("TraceDiagnostics", () => { it.effect("keeps loaded trace data when one rotated trace file fails to read", () => Effect.gen(function* () { - const traceFilePath = "/tmp/server.trace.ndjson"; const readFailure = PlatformError.systemError({ _tag: "PermissionDenied", module: "FileSystem", - method: "readFileString", + method: "open", description: "permission denied", pathOrDescriptor: `${traceFilePath}.1`, }); - const fileSystemLayer = FileSystem.layerNoop({ - readFileString: (path) => - path === `${traceFilePath}.1` - ? Effect.fail(readFailure) - : Effect.succeed( - record({ - name: "server.getConfig", - traceId: "trace-a", - spanId: "span-a", - startMs: 1_000, - durationMs: 50, - }), - ), - }); const logAnnotations: Array> = []; const logger = Logger.make((options) => { logAnnotations.push({ ...options.fiber.getRef(References.CurrentLogAnnotations) }); }); - const diagnostics = yield* TraceDiagnostics.readTraceDiagnostics({ - traceFilePath, - maxFiles: 1, - readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"), - }).pipe( - Effect.provide( - Layer.mergeAll( - TraceDiagnostics.layer.pipe(Layer.provide(fileSystemLayer)), - Logger.layer([logger], { mergeWithExisting: false }), - ), - ), - ); + const diagnostics = yield* readTraces({ + stream: (path) => + path === `${traceFilePath}.1` + ? Stream.fail(readFailure) + : Stream.make( + new TextEncoder().encode( + record({ + name: "server.getConfig", + traceId: "trace-a", + spanId: "span-a", + startMs: 1_000, + durationMs: 50, + }), + ), + ), + }).pipe(Effect.provide(Logger.layer([logger], { mergeWithExisting: false }))); assert.equal(diagnostics.recordCount, 1); assert.equal( @@ -251,32 +310,43 @@ describe("TraceDiagnostics", () => { }), ); - it.effect("keeps only the slowest span occurrences while aggregating large inputs", () => + it.effect("keeps only the top spans, failures, and warning logs from large inputs", () => Effect.sync(() => { - const diagnostics = TraceDiagnostics.aggregateTraceDiagnostics({ - traceFilePath: "/tmp/server.trace.ndjson", - readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"), - files: [ - { - path: "/tmp/server.trace.ndjson", - text: Array.from({ length: 25 }, (_, index) => - record({ - name: `span-${index}`, - traceId: `trace-${index}`, - spanId: `span-${index}`, - startMs: index * 1_000, - durationMs: index, - }), - ).join("\n"), - }, - ], - }); + // Shuffled, so some older records arrive after the lists are full. + const indexes = Array.from({ length: 30 }, (_, step) => (step * 7) % 30); + const diagnostics = aggregateLines( + indexes.map((index) => + record({ + name: `span-${index}`, + traceId: `trace-${index}`, + spanId: `span-${index}`, + startMs: index * 1_000, + durationMs: index, + exit: { _tag: "Failure", cause: "Provider crashed" }, + events: [ + { + name: `warning ${index}`, + timeUnixNano: ns(index * 1_000), + attributes: { "effect.logLevel": "Warning" }, + }, + ], + }), + ), + ); + const newestTwenty = Array.from({ length: 20 }, (_, rank) => `trace-${29 - rank}`); - assert.equal(diagnostics.recordCount, 25); - assert.equal(diagnostics.slowestSpans.length, 10); + assert.equal(diagnostics.recordCount, 30); assert.deepStrictEqual( diagnostics.slowestSpans.map((span) => span.durationMs), - [24, 23, 22, 21, 20, 19, 18, 17, 16, 15], + [29, 28, 27, 26, 25, 24, 23, 22, 21, 20], + ); + assert.deepStrictEqual( + diagnostics.latestFailures.map((failure) => failure.traceId), + newestTwenty, + ); + assert.deepStrictEqual( + diagnostics.latestWarningAndErrorLogs.map((log) => log.traceId), + newestTwenty, ); }), ); diff --git a/apps/server/src/diagnostics/TraceDiagnostics.ts b/apps/server/src/diagnostics/TraceDiagnostics.ts index afb7b4e923bb..6dbc4afdec93 100644 --- a/apps/server/src/diagnostics/TraceDiagnostics.ts +++ b/apps/server/src/diagnostics/TraceDiagnostics.ts @@ -16,6 +16,7 @@ import * as Option from "effect/Option"; import * as PlatformError from "effect/PlatformError"; import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; interface TraceRecordLike { readonly name?: unknown; @@ -63,16 +64,6 @@ export class TraceDiagnostics extends Context.Service< } >()("t3/diagnostics/TraceDiagnostics") {} -interface TraceDiagnosticsInput { - readonly traceFilePath: string; - readonly files: ReadonlyArray<{ readonly path: string; readonly text: string }>; - readonly scannedFilePaths?: ReadonlyArray; - readonly slowSpanThresholdMs?: number; - readonly readAt: DateTime.Utc; - readonly error?: TraceDiagnosticsErrorSummary; - readonly partialFailure?: boolean; -} - interface TraceDiagnosticsErrorSummary { readonly kind: ServerTraceDiagnosticsErrorKind; readonly message: string; @@ -173,44 +164,50 @@ function isNotFoundError(error: PlatformError.PlatformError): boolean { return error.reason._tag === "NotFound"; } -function insertBoundedSlowestSpan( - slowestSpans: ServerTraceDiagnosticsSpanOccurrence[], - span: ServerTraceDiagnosticsSpanOccurrence, +/** + * Adds `item` to `items`, which stays sorted by `order` and holds at most + * `limit` entries. Same result as a stable sort and slice over every item, but + * memory stays bounded however many items stream in. + */ +function insertBounded( + items: A[], + item: A, + limit: number, + order: (left: A, right: A) => number, ): void { - if ( - slowestSpans.length >= TOP_LIMIT && - span.durationMs <= slowestSpans[slowestSpans.length - 1]!.durationMs - ) { + if (items.length >= limit && order(item, items[items.length - 1]!) >= 0) { return; } - slowestSpans.push(span); - slowestSpans.sort((left, right) => right.durationMs - left.durationMs); - if (slowestSpans.length > TOP_LIMIT) { - slowestSpans.length = TOP_LIMIT; + items.push(item); + items.sort(order); + if (items.length > limit) { + items.length = limit; } } -export function aggregateTraceDiagnostics( - input: TraceDiagnosticsInput, -): ServerTraceDiagnosticsResult { - const readAt = input.readAt; - const slowSpanThresholdMs = input.slowSpanThresholdMs ?? DEFAULT_SLOW_SPAN_THRESHOLD_MS; - const scannedFilePaths = input.scannedFilePaths ?? input.files.map((file) => file.path); - if (input.files.length === 0) { - return makeEmptyDiagnostics({ - traceFilePath: input.traceFilePath, - scannedFilePaths, - readAt, - slowSpanThresholdMs, - error: input.error ?? { - kind: "trace-file-not-found", - message: "No local trace files were found.", - }, - ...(input.partialFailure ? { partialFailure: true } : {}), - }); - } - +const slowestFirst = ( + left: ServerTraceDiagnosticsSpanOccurrence, + right: ServerTraceDiagnosticsSpanOccurrence, +) => right.durationMs - left.durationMs; + +const latestEndedFirst = ( + left: ServerTraceDiagnosticsRecentFailure, + right: ServerTraceDiagnosticsRecentFailure, +) => DateTime.toEpochMillis(right.endedAt) - DateTime.toEpochMillis(left.endedAt); + +const latestSeenFirst = ( + left: ServerTraceDiagnosticsLogEvent, + right: ServerTraceDiagnosticsLogEvent, +) => DateTime.toEpochMillis(right.seenAt) - DateTime.toEpochMillis(left.seenAt); + +/** + * Folds trace NDJSON into diagnostics. Call `addLine` once per line as the + * rotated files stream in, then `finish` for the result. + */ +export function makeTraceDiagnosticsAggregator( + slowSpanThresholdMs = DEFAULT_SLOW_SPAN_THRESHOLD_MS, +) { let parseErrorCount = 0; let recordCount = 0; let failureCount = 0; @@ -229,182 +226,184 @@ export function aggregateTraceDiagnostics( const latestWarningAndErrorLogs: ServerTraceDiagnosticsLogEvent[] = []; const logLevelCounts: Record = {}; - for (const file of input.files) { - const lines = file.text.split(/\r?\n/); - for (const line of lines) { - if (line.trim().length === 0) continue; - - let parsed: unknown; - try { - parsed = JSON.parse(line); - } catch { - parseErrorCount += 1; - continue; - } + const addLine = (line: string) => { + if (line.trim().length === 0) return; - if (!isRecordObject(parsed)) { - parseErrorCount += 1; - continue; - } + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + parseErrorCount += 1; + return; + } - const name = toStringValue(parsed.name); - const traceId = toStringValue(parsed.traceId); - const spanId = toStringValue(parsed.spanId); - const durationMs = toNumberValue(parsed.durationMs); - const endedAt = unixNanoToDateTime(parsed.endTimeUnixNano); - const startedAt = unixNanoToDateTime(parsed.startTimeUnixNano); + if (!isRecordObject(parsed)) { + parseErrorCount += 1; + return; + } - if (!name || !traceId || !spanId || durationMs === null || !endedAt) { - parseErrorCount += 1; - continue; - } + const name = toStringValue(parsed.name); + const traceId = toStringValue(parsed.traceId); + const spanId = toStringValue(parsed.spanId); + const durationMs = toNumberValue(parsed.durationMs); + const endedAt = unixNanoToDateTime(parsed.endTimeUnixNano); + const startedAt = unixNanoToDateTime(parsed.startTimeUnixNano); - recordCount += 1; - firstSpanAt = - startedAt && (firstSpanAt === null || DateTime.isLessThan(startedAt, firstSpanAt)) - ? startedAt - : firstSpanAt; - lastSpanAt = - lastSpanAt === null || DateTime.isGreaterThan(endedAt, lastSpanAt) ? endedAt : lastSpanAt; - - const exitTag = readExitTag(parsed.exit); - const isFailure = exitTag === "Failure"; - const isInterrupted = exitTag === "Interrupted"; - if (isFailure) failureCount += 1; - if (isInterrupted) interruptionCount += 1; - - const spanSummary = spansByName.get(name) ?? { - count: 0, - failureCount: 0, - totalDurationMs: 0, - maxDurationMs: 0, - }; - spanSummary.count += 1; - spanSummary.totalDurationMs += durationMs; - spanSummary.maxDurationMs = Math.max(spanSummary.maxDurationMs, durationMs); - if (isFailure) spanSummary.failureCount += 1; - spansByName.set(name, spanSummary); - - const spanItem = { name, durationMs, endedAt, traceId, spanId }; - if (durationMs >= slowSpanThresholdMs) { - slowSpanCount += 1; - } - insertBoundedSlowestSpan(slowestSpans, spanItem); - - if (isFailure) { - const cause = readExitCause(parsed.exit); - latestFailures.push({ ...spanItem, cause }); - - const failureKey = `${name}\0${cause}`; - const existing = failuresByKey.get(failureKey); - const isLatestFailure = !existing || DateTime.isGreaterThan(endedAt, existing.lastSeenAt); - failuresByKey.set(failureKey, { - name, - cause, - count: (existing?.count ?? 0) + 1, - lastSeenAt: isLatestFailure ? endedAt : existing!.lastSeenAt, - traceId: isLatestFailure ? traceId : existing!.traceId, - spanId: isLatestFailure ? spanId : existing!.spanId, - }); - } + if (!name || !traceId || !spanId || durationMs === null || !endedAt) { + parseErrorCount += 1; + return; + } - if (Array.isArray(parsed.events)) { - for (const rawEvent of parsed.events) { - if (!isTraceEvent(rawEvent)) continue; - const attributes = readEventAttributes(rawEvent); - const level = toStringValue(attributes["effect.logLevel"]); - if (!level) continue; - - logLevelCounts[level] = (logLevelCounts[level] ?? 0) + 1; - const normalizedLevel = level.toLowerCase(); - if ( - normalizedLevel !== "warning" && - normalizedLevel !== "warn" && - normalizedLevel !== "error" && - normalizedLevel !== "fatal" - ) { - continue; - } - - const seenAt = unixNanoToDateTime(rawEvent.timeUnixNano) ?? endedAt; - const message = toStringValue(rawEvent.name)?.trim() ?? "Log event"; - latestWarningAndErrorLogs.push({ - spanName: name, - level, - message, - seenAt, - traceId, - spanId, - }); + recordCount += 1; + firstSpanAt = + startedAt && (firstSpanAt === null || DateTime.isLessThan(startedAt, firstSpanAt)) + ? startedAt + : firstSpanAt; + lastSpanAt = + lastSpanAt === null || DateTime.isGreaterThan(endedAt, lastSpanAt) ? endedAt : lastSpanAt; + + const exitTag = readExitTag(parsed.exit); + const isFailure = exitTag === "Failure"; + const isInterrupted = exitTag === "Interrupted"; + if (isFailure) failureCount += 1; + if (isInterrupted) interruptionCount += 1; + + const spanSummary = spansByName.get(name) ?? { + count: 0, + failureCount: 0, + totalDurationMs: 0, + maxDurationMs: 0, + }; + spanSummary.count += 1; + spanSummary.totalDurationMs += durationMs; + spanSummary.maxDurationMs = Math.max(spanSummary.maxDurationMs, durationMs); + if (isFailure) spanSummary.failureCount += 1; + spansByName.set(name, spanSummary); + + const spanItem = { name, durationMs, endedAt, traceId, spanId }; + if (durationMs >= slowSpanThresholdMs) { + slowSpanCount += 1; + } + insertBounded(slowestSpans, spanItem, TOP_LIMIT, slowestFirst); + + if (isFailure) { + const cause = readExitCause(parsed.exit); + insertBounded(latestFailures, { ...spanItem, cause }, RECENT_LIMIT, latestEndedFirst); + + const failureKey = `${name}\0${cause}`; + const existing = failuresByKey.get(failureKey); + const isLatestFailure = !existing || DateTime.isGreaterThan(endedAt, existing.lastSeenAt); + failuresByKey.set(failureKey, { + name, + cause, + count: (existing?.count ?? 0) + 1, + lastSeenAt: isLatestFailure ? endedAt : existing!.lastSeenAt, + traceId: isLatestFailure ? traceId : existing!.traceId, + spanId: isLatestFailure ? spanId : existing!.spanId, + }); + } + + if (Array.isArray(parsed.events)) { + for (const rawEvent of parsed.events) { + if (!isTraceEvent(rawEvent)) continue; + const attributes = readEventAttributes(rawEvent); + const level = toStringValue(attributes["effect.logLevel"]); + if (!level) continue; + + logLevelCounts[level] = (logLevelCounts[level] ?? 0) + 1; + const normalizedLevel = level.toLowerCase(); + if ( + normalizedLevel !== "warning" && + normalizedLevel !== "warn" && + normalizedLevel !== "error" && + normalizedLevel !== "fatal" + ) { + continue; } + + const seenAt = unixNanoToDateTime(rawEvent.timeUnixNano) ?? endedAt; + const message = toStringValue(rawEvent.name)?.trim() ?? "Log event"; + insertBounded( + latestWarningAndErrorLogs, + { spanName: name, level, message, seenAt, traceId, spanId }, + RECENT_LIMIT, + latestSeenFirst, + ); } } - } - - const topSpansByCount: ServerTraceDiagnosticsSpanSummary[] = [...spansByName.entries()] - .map(([name, span]) => ({ - name, - count: span.count, - failureCount: span.failureCount, - totalDurationMs: span.totalDurationMs, - averageDurationMs: span.count > 0 ? span.totalDurationMs / span.count : 0, - maxDurationMs: span.maxDurationMs, - })) - .toSorted((left, right) => right.count - left.count || right.maxDurationMs - left.maxDurationMs) - .slice(0, TOP_LIMIT); + }; - return { - traceFilePath: input.traceFilePath, - scannedFilePaths, - readAt, - recordCount, - parseErrorCount, - firstSpanAt: Option.fromNullishOr(firstSpanAt), - lastSpanAt: Option.fromNullishOr(lastSpanAt), - failureCount, - interruptionCount, - slowSpanThresholdMs, - slowSpanCount, - logLevelCounts, - topSpansByCount, - slowestSpans, - commonFailures: [...failuresByKey.values()] - .toSorted( - (left, right) => - right.count - left.count || - DateTime.toEpochMillis(right.lastSeenAt) - DateTime.toEpochMillis(left.lastSeenAt), - ) - .slice(0, TOP_LIMIT), - latestFailures: latestFailures + const finish = (input: { + readonly traceFilePath: string; + readonly scannedFilePaths: ReadonlyArray; + readonly readAt: DateTime.Utc; + readonly error?: TraceDiagnosticsErrorSummary; + readonly partialFailure?: boolean; + }): ServerTraceDiagnosticsResult => { + const topSpansByCount: ServerTraceDiagnosticsSpanSummary[] = [...spansByName.entries()] + .map(([name, span]) => ({ + name, + count: span.count, + failureCount: span.failureCount, + totalDurationMs: span.totalDurationMs, + averageDurationMs: span.count > 0 ? span.totalDurationMs / span.count : 0, + maxDurationMs: span.maxDurationMs, + })) .toSorted( - (left, right) => - DateTime.toEpochMillis(right.endedAt) - DateTime.toEpochMillis(left.endedAt), + (left, right) => right.count - left.count || right.maxDurationMs - left.maxDurationMs, ) - .slice(0, RECENT_LIMIT), - latestWarningAndErrorLogs: latestWarningAndErrorLogs - .toSorted( - (left, right) => DateTime.toEpochMillis(right.seenAt) - DateTime.toEpochMillis(left.seenAt), - ) - .slice(0, RECENT_LIMIT), - partialFailure: input.partialFailure ? Option.some(true) : Option.none(), - error: Option.fromNullishOr(input.error), + .slice(0, TOP_LIMIT); + + return { + traceFilePath: input.traceFilePath, + scannedFilePaths: input.scannedFilePaths, + readAt: input.readAt, + recordCount, + parseErrorCount, + firstSpanAt: Option.fromNullishOr(firstSpanAt), + lastSpanAt: Option.fromNullishOr(lastSpanAt), + failureCount, + interruptionCount, + slowSpanThresholdMs, + slowSpanCount, + logLevelCounts, + topSpansByCount, + slowestSpans, + commonFailures: [...failuresByKey.values()] + .toSorted( + (left, right) => + right.count - left.count || + DateTime.toEpochMillis(right.lastSeenAt) - DateTime.toEpochMillis(left.lastSeenAt), + ) + .slice(0, TOP_LIMIT), + latestFailures, + latestWarningAndErrorLogs, + partialFailure: input.partialFailure ? Option.some(true) : Option.none(), + error: Option.fromNullishOr(input.error), + }; }; -} -type TraceFileReadResult = - | { readonly _tag: "Loaded"; readonly path: string; readonly text: string } - | { readonly _tag: "Missing"; readonly path: string }; + return { addLine, finish }; +} -function readTraceFile( +/** + * Feeds each line of one trace file to `onLine`, streaming so only one chunk of + * text is in memory at a time. Succeeds with false when the file does not exist. + */ +export function streamTraceFileLines( fileSystem: FileSystem.FileSystem, path: string, -): Effect.Effect { - return fileSystem.readFileString(path).pipe( - Effect.map((text): TraceFileReadResult => ({ _tag: "Loaded", path, text })), + onLine: (line: string) => void, +): Effect.Effect { + return fileSystem.stream(path).pipe( + Stream.decodeText, + Stream.splitLines, + Stream.runForEachArray((lines) => Effect.sync(() => lines.forEach(onLine))), + Effect.as(true), Effect.catchTags({ PlatformError: (cause) => isNotFoundError(cause) - ? Effect.succeed({ _tag: "Missing", path }) + ? Effect.succeed(false) : Effect.fail( new TraceFileReadError({ traceFilePath: path, @@ -425,10 +424,11 @@ export const make = Effect.gen(function* () { const readAt = options.readAt ?? (yield* DateTime.now); const slowSpanThresholdMs = options.slowSpanThresholdMs ?? DEFAULT_SLOW_SPAN_THRESHOLD_MS; const paths = toRotatedTracePaths(options.traceFilePath, options.maxFiles); + const aggregator = makeTraceDiagnosticsAggregator(slowSpanThresholdMs); const results = yield* Effect.forEach( paths, (path) => - readTraceFile(fileSystem, path).pipe( + streamTraceFileLines(fileSystem, path, aggregator.addLine).pipe( Effect.tapError((cause) => Effect.logWarning("Failed to read local trace file.").pipe( Effect.annotateLogs({ @@ -440,15 +440,10 @@ export const make = Effect.gen(function* () { ), Effect.result, ), - { - concurrency: 1, - }, - ); - const files = results.flatMap((result) => - Result.isSuccess(result) && result.success._tag === "Loaded" - ? [{ path: result.success.path, text: result.success.text }] - : [], + // Every file feeds one aggregator, so read them one at a time, oldest first. + { concurrency: 1 }, ); + const foundFile = results.some((result) => Result.isSuccess(result) && result.success); const readFailure = results.find(Result.isFailure); const readFailureError = readFailure ? ({ @@ -457,7 +452,7 @@ export const make = Effect.gen(function* () { } satisfies TraceDiagnosticsErrorSummary) : undefined; - if (files.length === 0) { + if (!foundFile) { return makeEmptyDiagnostics({ traceFilePath: options.traceFilePath, scannedFilePaths: paths, @@ -472,12 +467,10 @@ export const make = Effect.gen(function* () { }); } - return aggregateTraceDiagnostics({ + return aggregator.finish({ traceFilePath: options.traceFilePath, - files, scannedFilePaths: paths, readAt, - slowSpanThresholdMs, ...(readFailureError ? { partialFailure: true, error: readFailureError } : {}), }); }, diff --git a/apps/server/src/http.ts b/apps/server/src/http.ts index 3505e86b0c48..dd2bc675b0e2 100644 --- a/apps/server/src/http.ts +++ b/apps/server/src/http.ts @@ -316,9 +316,12 @@ export const serverEnvironmentHttpApiLayer = HttpApiBuilder.group( class DecodeOtlpTraceRecordsError extends Data.TaggedError("DecodeOtlpTraceRecordsError")<{ readonly cause: unknown; - readonly bodyJson: OtlpTracer.TraceData; }> {} +// Renderers export up to once a second while they have spans buffered, so +// tracing this proxy would add more server spans than it forwards. +// withTracerEnabled(false) drops the handler's spans, including the forward. +// untracedRequestsLayer drops the HTTP server span. export const otlpTracesProxyRouteLayer = HttpRouter.add( "POST", OTLP_TRACES_PROXY_PATH, @@ -335,15 +338,10 @@ export const otlpTracesProxyRouteLayer = HttpRouter.add( yield* Effect.try({ try: () => decodeOtlpTraceRecords(bodyJson), - catch: (cause) => new DecodeOtlpTraceRecordsError({ cause, bodyJson }), + catch: (cause) => new DecodeOtlpTraceRecordsError({ cause }), }).pipe( Effect.flatMap((records) => browserTraceCollector.record(records)), - Effect.catch((cause) => - Effect.logWarning("Failed to decode browser OTLP traces", { - cause, - bodyJson, - }), - ), + Effect.catch((cause) => Effect.logWarning("Failed to decode browser OTLP traces", { cause })), ); if (otlpTracesUrl === undefined) { @@ -374,9 +372,25 @@ export const otlpTracesProxyRouteLayer = HttpRouter.add( EnvironmentInternalError: HttpServerRespondable.toResponse, EnvironmentScopeRequiredError: HttpServerRespondable.toResponse, }), + Effect.withTracerEnabled(false), ), ); +const UNTRACED_REQUEST_PATHS: ReadonlySet = new Set([OTLP_TRACES_PROXY_PATH]); + +// Skips the HTTP server span for UNTRACED_REQUEST_PATHS. That span starts +// before routing, so a route handler cannot skip it. TracerDisabledWhen is one +// predicate for the whole server and the last layer to provide it wins, so +// makeRoutesLayer provides this one last. Add paths here instead of providing +// TracerDisabledWhen again; server.test.ts fails if a later layer replaces it. +// The query string is ignored, as in routing. +export const untracedRequestsLayer = Layer.succeed(HttpMiddleware.TracerDisabledWhen)((request) => { + const queryIndex = request.url.indexOf("?"); + return UNTRACED_REQUEST_PATHS.has( + queryIndex === -1 ? request.url : request.url.slice(0, queryIndex), + ); +}); + export const assetRouteLayer = HttpRouter.add( "GET", `${ASSET_ROUTE_PREFIX}/*`, diff --git a/apps/server/src/mcp/toolkits/device/handlers.ts b/apps/server/src/mcp/toolkits/device/handlers.ts index d2c23860c9b7..b6a8ed3d6323 100644 --- a/apps/server/src/mcp/toolkits/device/handlers.ts +++ b/apps/server/src/mcp/toolkits/device/handlers.ts @@ -59,7 +59,7 @@ export function agentDeviceQuickStart( ` ${executable} screenshot /tmp/shot.png ${target} # or call device_screenshot`, ` ${executable} install ${target}`, `Prefer snapshot refs over coordinates. Run ${executable} help for workflow guides and ${executable} --help for flags.`, - "Do not call simctl, adb, xcrun, or serve-sim directly while these tools are attached; use agent-device.", + "Prefer agent-device for driving this device. simctl, adb, and xcrun remain available for anything it does not cover.", "For remote hosts, arrange builds, app installation, and any Metro reverse forwarding yourself. T3 provides discovery, streaming, and control only.", "Keep the returned --config and --session flags on every command. Other hosts can be used concurrently; opening one does not switch these commands.", platformNotes, diff --git a/apps/server/src/observability/EventLoopMonitor.test.ts b/apps/server/src/observability/EventLoopMonitor.test.ts new file mode 100644 index 000000000000..fbe30157aec2 --- /dev/null +++ b/apps/server/src/observability/EventLoopMonitor.test.ts @@ -0,0 +1,77 @@ +import { assert, describe, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Tracer from "effect/Tracer"; +import * as TestClock from "effect/testing/TestClock"; + +import { type EventLoopReadings, layerWith, stallMs } from "./EventLoopMonitor.ts"; + +const ms = (value: number) => value * 1e6; + +// Node reports a stall of S as a gap of up to S + 1 s, the histogram resolution. +const stalled: EventLoopReadings = { + delayMaxNs: ms(5_950), + activeMs: 6_200, + utilization: 0.176, + usage: { + userCPUTime: 310_400, + systemCPUTime: 95_600, + majorPageFault: 8_412, + minorPageFault: 20_031, + involuntaryContextSwitches: 57, + }, + rssBytes: 1536 * 1024 * 1024, +}; +// Over the threshold as read, but not once the resolution is subtracted. +const quiet: EventLoopReadings = { ...stalled, delayMaxNs: ms(2_950) }; + +describe("EventLoopMonitor", () => { + it.effect("records a warning span only for samples that saw a stall", () => + Effect.gen(function* () { + const spans: Array = []; + const tracer = Tracer.make({ + span: (options) => { + const span = new Tracer.NativeSpan(options); + spans.push(span); + return span; + }, + }); + // The first sample covers startup, so the monitor discards it. + const samples = [stalled, quiet, stalled]; + + yield* Effect.gen(function* () { + yield* Layer.build(layerWith(Effect.succeed(Effect.sync(() => samples.shift() ?? quiet)))); + yield* TestClock.adjust("60 seconds"); + assert.lengthOf(spans, 0); + yield* TestClock.adjust("30 seconds"); + }).pipe(Effect.scoped, Effect.withTracer(tracer)); + + assert.deepStrictEqual( + spans.map((span) => span.name), + ["server.eventLoop.stall"], + ); + const [span] = spans; + assert.deepStrictEqual(Object.fromEntries(span!.attributes), { + delayMaxMs: 4_950, + utilization: 0.18, + cpuUserMs: 310, + cpuSystemMs: 96, + majorPageFaults: 8_412, + minorPageFaults: 20_031, + involuntaryContextSwitches: 57, + rssMb: 1536, + }); + assert.deepStrictEqual( + span!.events.map(([name, , attributes]) => [name, attributes["effect.logLevel"]]), + [["event loop stalled for 4950 ms", "WARN"]], + ); + }), + ); + + it("ignores delay the loop spent idle, such as a system sleep", () => { + // Waking from sleep reads as a long gap, but the loop was idle in poll for it. + const asleep: EventLoopReadings = { ...stalled, delayMaxNs: ms(600_000), activeMs: 900 }; + assert.isUndefined(stallMs(asleep)); + assert.strictEqual(stallMs({ ...asleep, activeMs: 600_000 }), 599_000); + }); +}); diff --git a/apps/server/src/observability/EventLoopMonitor.ts b/apps/server/src/observability/EventLoopMonitor.ts new file mode 100644 index 000000000000..13b7b48c4cf7 --- /dev/null +++ b/apps/server/src/observability/EventLoopMonitor.ts @@ -0,0 +1,135 @@ +// @effect-diagnostics nodeBuiltinImport:off - only node:perf_hooks exposes the event loop delay histogram. +import * as NodePerfHooks from "node:perf_hooks"; + +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import type * as Scope from "effect/Scope"; + +// Node's delay histogram wakes a native timer every RESOLUTION_MS and records the +// gap between wakeups, so an idle loop reads about RESOLUTION_MS and a stall of S +// reads between S and S + RESOLUTION_MS. We subtract the resolution, so a delay can +// undercount a stall by up to RESOLUTION_MS. With these values every stall over 3 s +// is caught, at 1 wakeup per second that never enters JS. +const RESOLUTION_MS = 1000; +const STALL_THRESHOLD_MS = 2000; +const SAMPLE_INTERVAL = "30 seconds"; + +/** One sample interval as Node reports it. Delay in ns, active time in ms, CPU in µs. */ +export interface EventLoopReadings { + readonly delayMaxNs: number; + readonly activeMs: number; + readonly utilization: number; + readonly usage: Pick< + NodeJS.ResourceUsage, + | "userCPUTime" + | "systemCPUTime" + | "majorPageFault" + | "minorPageFault" + | "involuntaryContextSwitches" + >; + readonly rssBytes: number; +} + +// Enables the delay histogram for the layer's lifetime. Each read returns the +// readings since the previous read and resets the histogram. Node skips the first +// gap after a reset, so a stall right at a sample boundary can be missed. +const makeNodeSampler = Effect.gen(function* () { + const histogram = yield* Effect.acquireRelease( + Effect.sync(() => { + const histogram = NodePerfHooks.monitorEventLoopDelay({ resolution: RESOLUTION_MS }); + histogram.enable(); + return histogram; + }), + (histogram) => Effect.sync(() => histogram.disable()), + ); + let elu = NodePerfHooks.performance.eventLoopUtilization(); + let usage = process.resourceUsage(); + + // @effect-diagnostics-next-line returnEffectInGen:off - the read effect is the result. + return Effect.sync(() => { + const nextElu = NodePerfHooks.performance.eventLoopUtilization(); + const nextUsage = process.resourceUsage(); + const loop = NodePerfHooks.performance.eventLoopUtilization(nextElu, elu); + const readings: EventLoopReadings = { + delayMaxNs: histogram.max, + activeMs: loop.active, + utilization: loop.utilization, + usage: { + userCPUTime: nextUsage.userCPUTime - usage.userCPUTime, + systemCPUTime: nextUsage.systemCPUTime - usage.systemCPUTime, + majorPageFault: nextUsage.majorPageFault - usage.majorPageFault, + minorPageFault: nextUsage.minorPageFault - usage.minorPageFault, + involuntaryContextSwitches: + nextUsage.involuntaryContextSwitches - usage.involuntaryContextSwitches, + }, + rssBytes: process.memoryUsage.rss(), + }; + histogram.reset(); + elu = nextElu; + usage = nextUsage; + return readings; + }); +}); + +/** + * Returns the stall to report for one sample in ms, or undefined when there was none. + */ +export const stallMs = ({ delayMaxNs, activeMs }: EventLoopReadings) => { + const delayMs = Math.round(delayMaxNs / 1e6) - RESOLUTION_MS; + // A stall is time the loop spent running code, so it counts as active time. libuv's + // clock keeps running while the system sleeps on macOS and Windows, so a sleep also + // reads as delay, but the loop spent it idle in poll. + if (delayMs <= STALL_THRESHOLD_MS || activeMs < delayMs) return undefined; + return delayMs; +}; + +/** + * Samples event loop health every 30 s and records a `server.eventLoop.stall` span + * with a warning when the loop stalled for more than 2 s, so stalls land in + * the local trace file and Settings > Diagnostics without OTLP. Takes the sampler + * so tests can inject readings. + */ +export const layerWith = ( + makeSampler: Effect.Effect, never, Scope.Scope>, +) => + Layer.effectDiscard( + Effect.gen(function* () { + const sample = yield* makeSampler; + const tick = Effect.gen(function* () { + const readings = yield* sample; + const delayMaxMs = stallMs(readings); + if (delayMaxMs === undefined) return; + const { utilization, usage, rssBytes } = readings; + // Root, as the stall has no caller to attach to. Warn level keeps it when + // T3CODE_TRACE_MIN_LEVEL is raised to cut trace noise. + yield* Effect.logWarning(`event loop stalled for ${delayMaxMs} ms`).pipe( + Effect.withSpan("server.eventLoop.stall", { + root: true, + level: "Warn", + attributes: { + delayMaxMs, + utilization: Math.round(utilization * 100) / 100, + cpuUserMs: Math.round(usage.userCPUTime / 1000), + cpuSystemMs: Math.round(usage.systemCPUTime / 1000), + majorPageFaults: usage.majorPageFault, + minorPageFaults: usage.minorPageFault, + involuntaryContextSwitches: usage.involuntaryContextSwitches, + rssMb: Math.round(rssBytes / 1024 / 1024), + }, + }), + ); + }); + const wait = Effect.sleep(SAMPLE_INTERVAL); + // The layer builds before the rest of the server, so the first sample covers + // startup work such as migrations and projection bootstrap. That can block the + // loop for seconds on a large database, so skip it rather than warn at every + // launch. Layers build outside any span, so this fiber retains no parent span. + yield* wait.pipe( + Effect.andThen(sample), + Effect.andThen(wait.pipe(Effect.andThen(tick), Effect.forever)), + Effect.forkScoped, + ); + }), + ); + +export const layer = layerWith(makeNodeSampler); diff --git a/apps/server/src/observability/HeapSnapshot.test.ts b/apps/server/src/observability/HeapSnapshot.test.ts new file mode 100644 index 000000000000..0beb466e5019 --- /dev/null +++ b/apps/server/src/observability/HeapSnapshot.test.ts @@ -0,0 +1,36 @@ +// @effect-diagnostics nodeBuiltinImport:off - tests fake a failed write at the native v8 boundary. +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; +import * as NodeV8 from "node:v8"; +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import { vi } from "vite-plus/test"; + +import { writeHeapSnapshot } from "./HeapSnapshot.ts"; + +vi.mock("node:v8", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, writeHeapSnapshot: vi.fn(actual.writeHeapSnapshot) }; +}); + +it.layer(NodeServices.layer)("writeHeapSnapshot", (it) => { + it.effect("removes the partial file when the write fails", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const logsDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-heap-snapshot-test-" }); + let partialPath: string | undefined; + vi.mocked(NodeV8.writeHeapSnapshot).mockImplementationOnce((path) => { + partialPath = path; + if (path) NodeFS.writeFileSync(path, "partial"); + throw new Error("ENOSPC: no space left on device"); + }); + + yield* writeHeapSnapshot(logsDir); + + assert.strictEqual(NodePath.dirname(partialPath ?? ""), logsDir); + assert.deepEqual(yield* fs.readDirectory(logsDir), []); + }), + ); +}); diff --git a/apps/server/src/observability/HeapSnapshot.ts b/apps/server/src/observability/HeapSnapshot.ts new file mode 100644 index 000000000000..827dfab2c6d6 --- /dev/null +++ b/apps/server/src/observability/HeapSnapshot.ts @@ -0,0 +1,52 @@ +// @effect-diagnostics nodeBuiltinImport:off - v8.writeHeapSnapshot has no Effect equivalent. +import * as NodePath from "node:path"; +import * as NodeV8 from "node:v8"; + +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; + +import * as ServerConfig from "../config.ts"; + +/** + * Writes one V8 heap snapshot into `logsDir` and logs its path. A failed write + * logs a warning and removes any partial file, because that file can hold + * secrets and the failure is often a full disk. + */ +export const writeHeapSnapshot = Effect.fn("server.heapSnapshot", { root: true })( + function* (logsDir: string) { + const fs = yield* FileSystem.FileSystem; + const timestamp = DateTime.formatIso(yield* DateTime.now).replaceAll(":", "-"); + const path = NodePath.join(logsDir, `server-${process.pid}-${timestamp}.heapsnapshot`); + yield* Effect.annotateCurrentSpan({ path }); + yield* Effect.try(() => NodeV8.writeHeapSnapshot(path)).pipe( + Effect.tapError(() => fs.remove(path, { force: true }).pipe(Effect.ignore)), + ); + yield* Effect.logInfo("Wrote heap snapshot.", { path }); + }, + Effect.catch((cause) => Effect.logWarning("Failed to write heap snapshot.", { cause })), +); + +/** + * Writes a heap snapshot when the process gets SIGUSR2 (`kill -USR2 `), + * so a maintainer can see what a long-running server holds. See "Heap + * Snapshots" in docs/operations/observability.md. + * + * The write blocks the event loop, so two snapshots never overlap: a signal + * sent during a write waits until it finishes. Windows has no SIGUSR2, so the + * layer does nothing there. + */ +export const layer = Layer.effectDiscard( + Effect.gen(function* () { + if ((yield* HostProcessPlatform) === "win32") return; + const { logsDir } = yield* ServerConfig.ServerConfig; + const runFork = Effect.runForkWith(yield* Effect.context()); + const onSignal = () => void runFork(writeHeapSnapshot(logsDir)); + yield* Effect.acquireRelease( + Effect.sync(() => process.on("SIGUSR2", onSignal)), + () => Effect.sync(() => process.off("SIGUSR2", onSignal)), + ); + }), +); diff --git a/apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts b/apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts index d25c442ac8b2..207ad9a20d34 100644 --- a/apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts +++ b/apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts @@ -108,6 +108,7 @@ describe("OrchestrationReactor", () => { Layer.provideMerge( Layer.succeed(AgentAwarenessRelay.AgentAwarenessRelay, { publishThread: () => Effect.void, + requestCatchUp: () => Effect.void, start: () => { started.push("agent-awareness-relay"); return Effect.void; diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts index 179d04843c7e..e8c7154f38e7 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts @@ -21,6 +21,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import * as Tracer from "effect/Tracer"; import * as SqlClient from "effect/unstable/sql/SqlClient"; import { makeSqlStatementCounter } from "../../../integration/SqlStatementCounter.integration.ts"; @@ -114,6 +115,66 @@ it.layer(Layer.fresh(makeProjectionPipelinePrefixedTestLayer("t3-projection-curs }, ); +it.layer(Layer.fresh(makeProjectionPipelinePrefixedTestLayer("t3-projection-cleanup-span-")))( + "OrchestrationProjectionPipeline attachment cleanup span", + (it) => { + it.effect("runs attachment cleanup only for events that remove attachments", () => + Effect.gen(function* () { + const projectionPipeline = yield* OrchestrationProjectionPipeline; + const eventStore = yield* OrchestrationEventStore; + let cleanupSpans = 0; + const tracer = Tracer.make({ + span: (options) => { + if (options.name === "applyAttachmentSideEffects") cleanupSpans += 1; + return new Tracer.NativeSpan(options); + }, + }); + const now = "2026-01-01T00:00:00.000Z"; + const projectId = ProjectId.make("project-cleanup-span"); + const threadId = ThreadId.make("thread-cleanup-span"); + + const projectCreated = yield* eventStore.append({ + type: "project.created", + eventId: EventId.make("evt-cleanup-span-project"), + aggregateKind: "project", + aggregateId: projectId, + occurredAt: now, + commandId: CommandId.make("cmd-cleanup-span-project"), + causationEventId: null, + correlationId: null, + metadata: {}, + payload: { + projectId, + title: "Cleanup span project", + workspaceRoot: "/tmp/project-cleanup-span", + defaultModelSelection: null, + scripts: [], + createdAt: now, + updatedAt: now, + }, + }); + yield* projectionPipeline.projectEvent(projectCreated).pipe(Effect.withTracer(tracer)); + assert.strictEqual(cleanupSpans, 0); + + const threadDeleted = yield* eventStore.append({ + type: "thread.deleted", + eventId: EventId.make("evt-cleanup-span-thread-delete"), + aggregateKind: "thread", + aggregateId: threadId, + occurredAt: now, + commandId: CommandId.make("cmd-cleanup-span-thread-delete"), + causationEventId: null, + correlationId: null, + metadata: {}, + payload: { threadId, deletedAt: now }, + }); + yield* projectionPipeline.projectEvent(threadDeleted).pipe(Effect.withTracer(tracer)); + assert.strictEqual(cleanupSpans, 1); + }), + ); + }, +); + it.layer(Layer.fresh(makeProjectionPipelinePrefixedTestLayer("t3-import-shell-")))( "imported thread shell projection", (it) => { diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts index 4163168157e7..1acb3c360b15 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts @@ -1991,13 +1991,6 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti const applyAttachmentSideEffects = Effect.fn("applyAttachmentSideEffects")( function* (event: OrchestrationEvent, sideEffects: AttachmentSideEffects) { - if ( - sideEffects.deletedThreadIds.size === 0 && - sideEffects.prunedThreadRelativePaths.size === 0 - ) { - return; - } - const deletedThreadIds = new Set(); for (const threadId of sideEffects.deletedThreadIds) { const recreatedLater = yield* eventStore.hasEventAfter({ @@ -2113,9 +2106,15 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti ); }), ); + const hasCleanup = + attachmentSideEffects.deletedThreadIds.size > 0 || + attachmentSideEffects.prunedThreadRelativePaths.size > 0; // Return the cleanup effect so the caller runs it after the outer transaction commits. + // Most events have no cleanup, so they skip the call and write no cleanup span. // @effect-diagnostics-next-line returnEffectInGen:off - return applyAttachmentSideEffects(event, attachmentSideEffects).pipe(Effect.asVoid); + return hasCleanup + ? applyAttachmentSideEffects(event, attachmentSideEffects).pipe(Effect.asVoid) + : Effect.void; }, Effect.provideService(FileSystem.FileSystem, fileSystem), Effect.provideService(Path.Path, path), diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts index 6fa2c370badf..cfba90297633 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts @@ -30,6 +30,7 @@ import * as ThreadPlanProgress from "../ThreadPlanProgress.ts"; import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts"; import { encodeThreadDetailPageCursor } from "../threadDetailCursor.ts"; import { projectThreadDetailSnapshot } from "../ActivityPayloadProjection.ts"; +import { readSweepSnapshot } from "../ThreadPullRequestReactor.ts"; import { makeSqlStatementCounter } from "../../../integration/SqlStatementCounter.integration.ts"; const asProjectId = (value: string): ProjectId => ProjectId.make(value); @@ -3574,6 +3575,153 @@ it.effect( }, ); +it.effect("reads one sweep thread and its projects like the shell snapshot", () => { + const layer = OrchestrationProjectionSnapshotQueryLive.pipe( + Layer.provide(ThreadBackgroundLiveness.layer), + Layer.provide(ThreadPlanProgress.layer), + Layer.provide( + Layer.succeed(RepositoryIdentityResolver.RepositoryIdentityResolver, { + resolve: () => + Effect.succeed({ + canonicalKey: "github.com/acme/web", + provider: "github", + displayName: "acme/web", + locator: { + source: "git-remote" as const, + remoteName: "origin", + remoteUrl: "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/acme/web.git", + }, + }), + }), + ), + Layer.provideMerge(SqlitePersistenceMemory), + ); + return Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const query = yield* ProjectionSnapshotQuery; + yield* sql`INSERT INTO projection_projects (project_id, title, workspace_root, scripts_json, created_at, updated_at) + VALUES ('p1', 'One', '/one', '[]', '2026-09-01T00:00:00Z', '2026-09-01T00:00:00Z'), + ('p2', 'Two', '/two', '[]', '2026-09-02T00:00:00Z', '2026-09-02T00:00:00Z'), + ('p3', 'Three', '/three', '[]', '2026-09-03T00:00:00Z', '2026-09-03T00:00:00Z')`; + yield* sql`INSERT INTO projection_threads (thread_id, project_id, title, model_selection_json, runtime_mode, interaction_mode, branch, worktree_path, branch_pull_request_json, latest_turn_id, latest_user_message_at, pending_approval_count, snoozed_until, snoozed_at, created_at, updated_at, settled_override, settled_at) + VALUES + ('t-linked', 'p1', 'Linked', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'feature', '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/one/wt', NULL, 'turn-1', '2026-09-02T00:00:00Z', 1, NULL, NULL, '2026-09-01T00:00:00Z', '2026-09-02T00:00:00Z', NULL, NULL), + ('t-branch', 'p1', 'Branch', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'other', NULL, + '{"projectId":"p2","repository":"acme/web","number":8,"url":"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/acme/web/pull/8"}', + NULL, NULL, 0, '2026-09-10T00:00:00Z', '2026-09-02T00:00:00Z', '2026-09-01T00:00:00Z', '2026-09-02T00:00:00Z', 'settled', '2026-09-03T00:00:00Z'), + ('t-other', 'p3', 'Other', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', NULL, NULL, NULL, NULL, NULL, 0, NULL, NULL, '2026-09-01T00:00:00Z', '2026-09-01T00:00:00Z', NULL, NULL)`; + yield* sql`INSERT INTO projection_thread_pull_requests (thread_id, host, repository, number, url, source, linked_at) + VALUES ('t-linked', 'github.com', 'acme/web', 7, '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/acme/web/pull/7', 'agent', '2026-09-02T00:00:00Z')`; + yield* sql`INSERT INTO projection_turns (thread_id, turn_id, state, requested_at, started_at, completed_at, checkpoint_files_json) + VALUES ('t-linked', 'turn-1', 'completed', '2026-09-02T00:00:00Z', '2026-09-02T00:00:01Z', '2026-09-02T00:00:02Z', '[]')`; + yield* sql`INSERT INTO projection_thread_sessions (thread_id, status, provider_name, active_turn_id, last_error, updated_at) + VALUES ('t-linked', 'ready', 'codex', NULL, NULL, '2026-09-02T00:00:03Z')`; + for (const projector of Object.values(ORCHESTRATION_PROJECTOR_NAMES)) { + yield* sql`INSERT INTO projection_state (projector, last_applied_sequence, updated_at) + VALUES (${projector}, 9, '2026-09-02T00:00:03Z')`; + } + + const full = yield* query.getShellSnapshot(); + // The seeded fields must reach the snapshot, or the parity check is empty. + const linked = full.threads.find((thread) => thread.id === ThreadId.make("t-linked")); + assert.strictEqual(full.snapshotSequence, 9); + assert.strictEqual(linked?.linkedPullRequest?.number, 7); + assert.strictEqual(linked?.latestTurn?.turnId, asTurnId("turn-1")); + assert.strictEqual(linked?.session?.status, "ready"); + + for (const [threadId, projectIds] of [ + [ThreadId.make("t-linked"), [asProjectId("p1")]], + // Settlement also needs the project that the saved branch PR names. + [ThreadId.make("t-branch"), [asProjectId("p1"), asProjectId("p2")]], + ] as const) { + assert.deepStrictEqual(yield* readSweepSnapshot(query, threadId), { + snapshotSequence: full.snapshotSequence, + projects: full.projects.filter((project) => projectIds.includes(project.id)), + threads: full.threads.filter((thread) => thread.id === threadId), + }); + } + }).pipe(Effect.provide(layer)); +}); + +it.effect("reads a full sweep from unsettled threads and every project", () => { + const resolved: string[] = []; + const layer = OrchestrationProjectionSnapshotQueryLive.pipe( + Layer.provide(ThreadBackgroundLiveness.layer), + Layer.provide(ThreadPlanProgress.layer), + Layer.provide( + Layer.succeed(RepositoryIdentityResolver.RepositoryIdentityResolver, { + resolve: (root) => + Effect.sync(() => { + resolved.push(root); + return null; + }), + }), + ), + Layer.provideMerge(SqlitePersistenceMemory), + ); + return Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const query = yield* ProjectionSnapshotQuery; + yield* sql`INSERT INTO projection_projects (project_id, title, workspace_root, scripts_json, created_at, updated_at) + VALUES ('p1', 'One', '/one', '[]', '2026-09-01T00:00:00Z', '2026-09-01T00:00:00Z'), + ('p2', 'Two', '/two', '[]', '2026-09-02T00:00:00Z', '2026-09-02T00:00:00Z'), + ('p3', 'Three', '/three', '[]', '2026-09-03T00:00:00Z', '2026-09-03T00:00:00Z'), + ('p4', 'Four', '/four', '[]', '2026-09-04T00:00:00Z', '2026-09-04T00:00:00Z')`; + yield* sql`INSERT INTO projection_threads (thread_id, project_id, title, model_selection_json, runtime_mode, interaction_mode, branch_pull_request_json, latest_turn_id, created_at, updated_at, archived_at, settled_override, settled_at) + VALUES + ('t-open', 'p1', 'Open', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', NULL, 'turn-open', '2026-09-01T00:00:00Z', '2026-09-01T00:00:00Z', NULL, NULL, NULL), + ('t-resumed', 'p1', 'Resumed', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', NULL, NULL, '2026-09-01T00:00:00Z', '2026-09-01T00:00:00Z', NULL, 'active', NULL), + ('t-branch', 'p1', 'Branch', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', + '{"projectId":"p2","repository":"acme/web","number":8,"url":"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/acme/web/pull/8"}', + NULL, '2026-09-01T00:00:00Z', '2026-09-01T00:00:00Z', NULL, NULL, NULL), + ('t-settled', 'p3', 'Settled', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', NULL, 'turn-settled', '2026-09-01T00:00:00Z', '2026-09-01T00:00:00Z', NULL, 'settled', '2026-09-03T00:00:00Z'), + ('t-archived', 'p4', 'Archived', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', NULL, NULL, '2026-09-01T00:00:00Z', '2026-09-01T00:00:00Z', '2026-09-04T00:00:00Z', NULL, NULL)`; + // The open and the settled thread both have a row in each joined table. The + // settled thread's turn and session are the newest rows, so updatedAt shows + // whether those two reads skip it. + yield* sql`INSERT INTO projection_thread_pull_requests (thread_id, host, repository, number, url, source, linked_at) + VALUES ('t-open', 'github.com', 'acme/web', 7, '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/acme/web/pull/7', 'agent', '2026-09-02T00:00:00Z'), + ('t-settled', 'github.com', 'acme/web', 9, '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/acme/web/pull/9', 'agent', '2026-09-02T00:00:00Z')`; + yield* sql`INSERT INTO projection_turns (thread_id, turn_id, state, requested_at, checkpoint_files_json) + VALUES ('t-open', 'turn-open', 'completed', '2026-09-02T00:00:00Z', '[]'), + ('t-settled', 'turn-settled', 'completed', '2026-09-09T00:00:00Z', '[]')`; + yield* sql`INSERT INTO projection_thread_sessions (thread_id, status, provider_name, active_turn_id, last_error, updated_at) + VALUES ('t-open', 'ready', 'codex', NULL, NULL, '2026-09-02T00:00:00Z'), + ('t-settled', 'stopped', 'codex', NULL, NULL, '2026-09-10T00:00:00Z')`; + + const full = yield* query.getShellSnapshot(); + // The settled thread's rows must reach the full read, or skipping them proves nothing. + const settled = full.threads.find((thread) => thread.id === ThreadId.make("t-settled")); + assert.strictEqual(settled?.pullRequests[0]?.number, 9); + assert.strictEqual(settled?.latestTurn?.turnId, asTurnId("turn-settled")); + assert.strictEqual(settled?.session?.status, "stopped"); + assert.strictEqual(full.updatedAt, "2026-09-10T00:00:00Z"); + resolved.length = 0; + + const sweep = yield* readSweepSnapshot(query, null); + assert.strictEqual(sweep.snapshotSequence, full.snapshotSequence); + assert.deepStrictEqual( + sweep.threads, + full.threads.filter((thread) => thread.id !== ThreadId.make("t-settled")), + ); + assert.deepStrictEqual( + sweep.threads.map((thread) => thread.id), + ["t-branch", "t-open", "t-resumed"], + ); + // Like the full read, the sweep resolves every project, so it keeps the + // repository identity cache warm for client connects. + assert.deepStrictEqual(sweep.projects, full.projects); + assert.deepStrictEqual(resolved.toSorted(), ["/four", "/one", "/three", "/two"]); + // A settled thread's link that no longer decodes breaks the full read, but + // not the sweep, which never reads it. + yield* sql`UPDATE projection_thread_pull_requests SET snapshot_json = 'invalid-json' WHERE thread_id = 't-settled'`; + assert.strictEqual((yield* Effect.exit(query.getShellSnapshot()))._tag, "Failure"); + const unsettled = yield* query.getShellSnapshot({ unsettledOnly: true }); + assert.deepStrictEqual(unsettled.threads, sweep.threads); + assert.strictEqual(unsettled.updatedAt, "2026-09-04T00:00:00Z"); + }).pipe(Effect.provide(layer)); +}); + projectionSnapshotLayer("ProjectionSnapshotQuery activities by kind", (it) => { it.effect("lists one kind across active threads only, without hydrating the threads", () => Effect.gen(function* () { diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts index 52b9f75c69a4..07d30e47e27b 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts @@ -12,7 +12,7 @@ import { OrchestrationProposedPlanId, OrchestrationReadModel, OrchestrationThreadSearchSource, - OrchestrationShellSnapshot, + type OrchestrationShellSnapshot, OrchestrationThread, OrchestrationThreadDetailSnapshot, ProjectScript, @@ -82,7 +82,6 @@ import { } from "../Services/ProjectionSnapshotQuery.ts"; const decodeReadModel = Schema.decodeUnknownEffect(OrchestrationReadModel); -const decodeShellSnapshot = Schema.decodeUnknownEffect(OrchestrationShellSnapshot); const decodeThread = Schema.decodeUnknownEffect(OrchestrationThread); const decodeImportedTranscriptsPayload = Schema.decodeUnknownOption( Schema.fromJsonString( @@ -186,6 +185,7 @@ const EventReplayStatsRowSchema = Schema.Struct({ eventCount: Schema.Number, payloadBytes: Schema.Number, }); +const ActiveThreadRowsRequest = Schema.Struct({ unsettledOnly: Schema.Boolean }); const ProjectionThreadSearchRequest = Schema.Struct({ pattern: Schema.String, limit: Schema.Int, @@ -601,10 +601,16 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { `, }); + // Background sweeps skip settled threads, the same check PR discovery makes. + const unsettledThreadsFilter = (unsettledOnly: boolean) => + unsettledOnly + ? sql`AND threads.settled_at IS NULL AND threads.settled_override IS NOT 'settled'` + : sql``; + const listActiveThreadRows = SqlSchema.findAll({ - Request: Schema.Void, + Request: ActiveThreadRowsRequest, Result: ProjectionThreadDbRowSchema, - execute: () => + execute: (request) => sql` SELECT thread_id AS "threadId", @@ -638,9 +644,10 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { pending_user_input_count AS "pendingUserInputCount", has_actionable_proposed_plan AS "hasActionableProposedPlan", deleted_at AS "deletedAt" - FROM projection_threads + FROM projection_threads threads WHERE deleted_at IS NULL AND archived_at IS NULL + ${unsettledThreadsFilter(request.unsettledOnly)} ORDER BY project_id ASC, created_at ASC, thread_id ASC `, }); @@ -780,9 +787,9 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { }); const listActiveThreadPullRequestRows = SqlSchema.findAll({ - Request: Schema.Void, + Request: ActiveThreadRowsRequest, Result: ProjectionThreadPullRequestDbRowSchema, - execute: () => + execute: (request) => sql` SELECT links.thread_id AS "threadId", @@ -799,6 +806,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { ON threads.thread_id = links.thread_id WHERE threads.deleted_at IS NULL AND threads.archived_at IS NULL + ${unsettledThreadsFilter(request.unsettledOnly)} ORDER BY links.thread_id ASC, links.linked_at ASC, links.number ASC `, }); @@ -908,9 +916,9 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { }); const listActiveThreadSessionRows = SqlSchema.findAll({ - Request: Schema.Void, + Request: ActiveThreadRowsRequest, Result: ProjectionThreadSessionDbRowSchema, - execute: () => + execute: (request) => sql` SELECT sessions.thread_id AS "threadId", @@ -928,6 +936,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { ON threads.thread_id = sessions.thread_id WHERE threads.deleted_at IS NULL AND threads.archived_at IS NULL + ${unsettledThreadsFilter(request.unsettledOnly)} ORDER BY sessions.thread_id ASC `, }); @@ -1002,9 +1011,9 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { }); const listActiveLatestTurnRows = SqlSchema.findAll({ - Request: Schema.Void, + Request: ActiveThreadRowsRequest, Result: ProjectionLatestTurnDbRowSchema, - execute: () => + execute: (request) => sql` SELECT turns.thread_id AS "threadId", @@ -1023,6 +1032,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { WHERE threads.deleted_at IS NULL AND threads.archived_at IS NULL AND threads.latest_turn_id IS NOT NULL + ${unsettledThreadsFilter(request.unsettledOnly)} ORDER BY turns.thread_id ASC `, }); @@ -2655,8 +2665,9 @@ pending_approval_requests AS ( }), ); - const getShellSnapshot: ProjectionSnapshotQueryShape["getShellSnapshot"] = () => - sql + const getShellSnapshot: ProjectionSnapshotQueryShape["getShellSnapshot"] = (options) => { + const unsettledOnly = options?.unsettledOnly === true; + return sql .withTransaction( Effect.all([ listProjectRows(undefined).pipe( @@ -2667,7 +2678,7 @@ pending_approval_requests AS ( ), ), ), - listActiveThreadRows(undefined).pipe( + listActiveThreadRows({ unsettledOnly }).pipe( Effect.mapError( toPersistenceSqlOrDecodeError( "ProjectionSnapshotQuery.getShellSnapshot:listThreads:query", @@ -2675,7 +2686,7 @@ pending_approval_requests AS ( ), ), ), - listActiveThreadSessionRows(undefined).pipe( + listActiveThreadSessionRows({ unsettledOnly }).pipe( Effect.mapError( toPersistenceSqlOrDecodeError( "ProjectionSnapshotQuery.getShellSnapshot:listThreadSessions:query", @@ -2683,7 +2694,7 @@ pending_approval_requests AS ( ), ), ), - listActiveThreadPullRequestRows(undefined).pipe( + listActiveThreadPullRequestRows({ unsettledOnly }).pipe( Effect.mapError( toPersistenceSqlOrDecodeError( "ProjectionSnapshotQuery.getShellSnapshot:listThreadPullRequests:query", @@ -2691,7 +2702,7 @@ pending_approval_requests AS ( ), ), ), - listActiveLatestTurnRows(undefined).pipe( + listActiveLatestTurnRows({ unsettledOnly }).pipe( Effect.mapError( toPersistenceSqlOrDecodeError( "ProjectionSnapshotQuery.getShellSnapshot:listLatestTurns:query", @@ -2746,7 +2757,10 @@ pending_approval_requests AS ( ); const pullRequestsByThread = groupPullRequestRowsByThread(pullRequestRows); - const snapshot = { + // Built from schema-decoded rows, so no second decode here. The HTTP + // and RPC layers encode it against OrchestrationShellSnapshot on the + // way out, like the per-item shells from getThreadShellById. + return { snapshotSequence: computeSnapshotSequence(stateRows), projects: Arr.filterMap(projectRows, (row) => row.deletedAt === null @@ -2800,15 +2814,7 @@ pending_approval_requests AS ( : Result.failVoid, ), updatedAt: updatedAt ?? "1970-01-01T00:00:00.000Z", - }; - - return yield* decodeShellSnapshot(snapshot).pipe( - Effect.mapError( - toPersistenceDecodeError( - "ProjectionSnapshotQuery.getShellSnapshot:decodeShellSnapshot", - ), - ), - ); + } satisfies OrchestrationShellSnapshot; }), ), Effect.mapError((error) => { @@ -2818,6 +2824,7 @@ pending_approval_requests AS ( return toPersistenceSqlError("ProjectionSnapshotQuery.getShellSnapshot:query")(error); }), ); + }; const listThreadsWithPullRequests: ProjectionSnapshotQueryShape["listThreadsWithPullRequests"] = () => @@ -2941,7 +2948,7 @@ pending_approval_requests AS ( sessionRows.map((row) => [row.threadId, mapSessionRow(row)] as const), ); - const snapshot = { + return { snapshotSequence: computeSnapshotSequence(stateRows), projects: Arr.filterMap(projectRows, (row) => row.deletedAt === null && activeProjectIds.has(row.projectId) @@ -2991,15 +2998,7 @@ pending_approval_requests AS ( planProgress: threadPlanProgress.getThreadPlanProgress(row.threadId), })), updatedAt: updatedAt ?? "1970-01-01T00:00:00.000Z", - }; - - return yield* decodeShellSnapshot(snapshot).pipe( - Effect.mapError( - toPersistenceDecodeError( - "ProjectionSnapshotQuery.getArchivedShellSnapshot:decodeShellSnapshot", - ), - ), - ); + } satisfies OrchestrationShellSnapshot; }), ), Effect.mapError((error) => { diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts index 254aec4bde7e..87f9bd03d46c 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts @@ -891,8 +891,92 @@ describe("ProviderCommandReactor", () => { expect(thread?.session?.threadId).toBe("thread-1"); expect(thread?.session?.status).toBe("starting"); expect(thread?.session?.runtimeMode).toBe("approval-required"); + expect(harness.startSession.mock.calls[0]?.[1]).not.toHaveProperty("title"); }); + effectIt.effect("forwards only a user-renamed title when starting a provider session", () => + Effect.gen(function* () { + const harness = yield* Effect.promise(() => + createHarness({ initialTitle: "Add a progressive blur as you scroll" }), + ); + const now = "2026-01-01T00:00:00.000Z"; + const modelSelection = { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", + }; + const startTurn = (threadId: string, text: string, titleSeed: string) => + harness.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make(`cmd-title-${threadId}`), + threadId: ThreadId.make(threadId), + message: { + messageId: asMessageId(`message-${threadId}`), + role: "user", + text, + attachments: [], + }, + titleSeed, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + createdAt: now, + }); + + yield* startTurn( + "thread-1", + "Add a progressive blur as you scroll", + "Add a progressive blur as you scroll", + ); + yield* Effect.promise(() => waitFor(() => harness.startSession.mock.calls.length === 1)); + expect(harness.startSession.mock.calls[0]?.[1]).not.toHaveProperty("title"); + + yield* harness.engine.dispatch({ + type: "thread.create", + commandId: CommandId.make("cmd-thread-create-renamed"), + threadId: ThreadId.make("thread-renamed"), + projectId: asProjectId("project-1"), + title: "New thread", + modelSelection, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + branch: null, + worktreePath: null, + createdAt: now, + }); + yield* harness.engine.dispatch({ + type: "thread.meta.update", + commandId: CommandId.make("cmd-thread-rename"), + threadId: ThreadId.make("thread-renamed"), + title: "Keep this name", + }); + yield* startTurn("thread-renamed", "hello there", "hello there"); + yield* Effect.promise(() => waitFor(() => harness.startSession.mock.calls.length === 2)); + expect(harness.startSession.mock.calls[1]?.[1]).toMatchObject({ title: "Keep this name" }); + + yield* harness.engine.dispatch({ + type: "thread.create", + commandId: CommandId.make("cmd-thread-create-seeded"), + threadId: ThreadId.make("thread-seeded"), + projectId: asProjectId("project-1"), + title: "New thread", + modelSelection, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + branch: null, + worktreePath: null, + createdAt: now, + }); + yield* harness.engine.dispatch({ + type: "thread.meta.update", + commandId: CommandId.make("cmd-thread-autotitle"), + threadId: ThreadId.make("thread-seeded"), + title: "hello there", + }); + yield* startTurn("thread-seeded", "hello there", "hello there"); + yield* Effect.promise(() => waitFor(() => harness.startSession.mock.calls.length === 3)); + expect(harness.startSession.mock.calls[2]?.[1]).not.toHaveProperty("title"); + }), + ); + effectIt.effect("projects inline context before sending the provider turn", () => Effect.gen(function* () { const harness = yield* Effect.promise(() => createHarness()); diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts index f6794135a925..77022a8518d7 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts @@ -575,6 +575,9 @@ const make = Effect.gen(function* () { options?: { readonly modelSelection?: ModelSelection; readonly pendingTurnStart?: boolean; + // First-turn prompt seed. A manual title that still equals this seed was + // written by the client's auto-title, not a user rename. + readonly titleSeed?: string; }, ) { const thread = yield* resolveThreadShell(threadId); @@ -712,6 +715,15 @@ const make = Effect.gen(function* () { .refreshWorkspaceSnapshot({ instanceId: desiredInstanceId, cwd: effectiveCwd }) .pipe(Effect.forkDetach) : Effect.void; + // OpenCode skips SessionPrompt.ensureTitle when session.create already has + // a title. Prompt seeds and "New thread" are not user titles, so omit them + // and let the provider generate one. A real rename is source "manual" and + // differs from the first-turn prompt seed (the web client writes that seed + // through thread.meta.update, which also marks the title manual). + const manualTitle = thread.titleState?.source === "manual" ? thread.title.trim() : ""; + const promptSeed = options?.titleSeed?.trim(); + const sessionTitle = + manualTitle.length > 0 && manualTitle !== promptSeed ? thread.title : undefined; const startProviderSession = (input?: { readonly resumeCursor?: unknown; @@ -723,7 +735,7 @@ const make = Effect.gen(function* () { ...(preferredProvider ? { provider: preferredProvider } : {}), providerInstanceId: desiredInstanceId, ...(effectiveCwd ? { cwd: effectiveCwd } : {}), - ...(thread.title ? { title: thread.title } : {}), + ...(sessionTitle ? { title: sessionTitle } : {}), modelSelection: desiredModelSelection, ...(input?.resumeCursor !== undefined ? { resumeCursor: input.resumeCursor } : {}), runtimeMode: desiredRuntimeMode, @@ -839,6 +851,7 @@ const make = Effect.gen(function* () { readonly modelSelection?: ModelSelection; readonly interactionMode?: "default" | "plan"; readonly createdAt: string; + readonly titleSeed?: string; }) { const thread = yield* resolveThreadShell(input.threadId); if (!thread) { @@ -848,6 +861,7 @@ const make = Effect.gen(function* () { } yield* ensureSessionForThread(input.threadId, input.createdAt, { ...(input.modelSelection !== undefined ? { modelSelection: input.modelSelection } : {}), + ...(input.titleSeed !== undefined ? { titleSeed: input.titleSeed } : {}), pendingTurnStart: true, }); if (input.modelSelection !== undefined) { @@ -1488,6 +1502,11 @@ const make = Effect.gen(function* () { : {}), interactionMode: event.payload.interactionMode, createdAt: event.payload.createdAt, + // Later turns must not reuse the current title as titleSeed. Only the + // first prompt seed should suppress a not-yet-renamed session title. + ...(!hasOtherUserMessages && event.payload.titleSeed !== undefined + ? { titleSeed: event.payload.titleSeed } + : {}), }).pipe( Effect.asSome, Effect.catchCause((cause) => handleTurnStartFailure(cause).pipe(Effect.as(Option.none()))), diff --git a/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts index 48e63d6fc8b0..08da6e00940c 100644 --- a/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts @@ -120,11 +120,15 @@ export interface ProjectionSnapshotQueryShape { * * Returns only projects and thread shell summaries so clients can bootstrap * lightweight navigation state without hydrating every thread body. + * + * `unsettledOnly` is for background sweeps, not clients. It skips settled + * threads and their sessions, PR links, and turns, and its `updatedAt` + * ignores those rows. It still resolves every project, which keeps + * repository identities cached for client connects. */ - readonly getShellSnapshot: () => Effect.Effect< - OrchestrationShellSnapshot, - ProjectionRepositoryError - >; + readonly getShellSnapshot: (options?: { + readonly unsettledOnly?: boolean; + }) => Effect.Effect; /** * Read archived thread shell summaries for the archive page. diff --git a/apps/server/src/orchestration/ThreadPullRequestReactor.test.ts b/apps/server/src/orchestration/ThreadPullRequestReactor.test.ts index d6572083dad2..2a08dd03bc54 100644 --- a/apps/server/src/orchestration/ThreadPullRequestReactor.test.ts +++ b/apps/server/src/orchestration/ThreadPullRequestReactor.test.ts @@ -21,18 +21,27 @@ import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; import { TestClock } from "effect/testing"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; import { GitManager, type GitBranchPullRequest } from "../git/GitManager.ts"; +import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts"; import { PullRequestService } from "../pullRequest/PullRequestService.ts"; import { RepositoryIdentityResolver } from "../project/RepositoryIdentityResolver.ts"; import { ServerActivation } from "../serverActivation.ts"; +import { OrchestrationProjectionSnapshotQueryLive } from "./Layers/ProjectionSnapshotQuery.ts"; import { OrchestrationEngineService } from "./Services/OrchestrationEngine.ts"; -import { ProjectionSnapshotQuery } from "./Services/ProjectionSnapshotQuery.ts"; +import { + ProjectionSnapshotQuery, + type ProjectionSnapshotQueryShape, +} from "./Services/ProjectionSnapshotQuery.ts"; +import * as ThreadBackgroundLiveness from "./ThreadBackgroundLiveness.ts"; +import * as ThreadPlanProgress from "./ThreadPlanProgress.ts"; import * as ThreadPullRequestReactor from "./ThreadPullRequestReactor.ts"; const NOW = "2026-09-01T12:00:00.000Z"; @@ -134,6 +143,8 @@ const makeHarness = Effect.fn("makeThreadPullRequestHarness")(function* (options readonly existingWorktrees?: ReadonlyArray; readonly project?: OrchestrationProjectShell; readonly resolveRepositoryIdentity?: RepositoryIdentityResolver["Service"]["resolve"]; + /** Serve full sweep reads from this instead of `threads`. */ + readonly getShellSnapshot?: ProjectionSnapshotQueryShape["getShellSnapshot"]; }) { const activation = yield* Deferred.make(); const snapshots = yield* Ref.make({ @@ -142,7 +153,8 @@ const makeHarness = Effect.fn("makeThreadPullRequestHarness")(function* (options threads: options.threads, updatedAt: NOW, }); - const reads = yield* Queue.unbounded(); + // Each shell read: a thread id for a one-thread read, null for a full read. + const reads = yield* Queue.unbounded(); const events = yield* PubSub.unbounded(); const commands = yield* Ref.make>([]); const branchCalls = yield* Ref.make< @@ -152,8 +164,27 @@ const makeHarness = Effect.fn("makeThreadPullRequestHarness")(function* (options let uuid = 0; const dependencies = Layer.mergeAll( Layer.mock(ProjectionSnapshotQuery)({ - getShellSnapshot: () => - Ref.get(snapshots).pipe(Effect.tap(() => Queue.offer(reads, undefined))), + getShellSnapshot: (readOptions) => + (options.getShellSnapshot?.(readOptions) ?? Ref.get(snapshots)).pipe( + Effect.tap(() => Queue.offer(reads, null)), + ), + getSnapshotSequence: () => + Ref.get(snapshots).pipe(Effect.map(({ snapshotSequence }) => ({ snapshotSequence }))), + getThreadShellById: (threadId) => + Ref.get(snapshots).pipe( + Effect.map(({ threads }) => + Option.fromUndefinedOr( + threads.find((thread) => thread.id === threadId && thread.archivedAt === null), + ), + ), + Effect.tap(() => Queue.offer(reads, threadId)), + ), + getProjectShells: (projectIds) => + Ref.get(snapshots).pipe( + Effect.map(({ projects }) => + projects.filter((project) => projectIds?.includes(project.id) ?? true), + ), + ), }), Layer.mock(GitManager)({ branchPullRequest: (input, readOptions) => @@ -376,7 +407,7 @@ describe("ThreadPullRequestReactor", () => { : [checkpointEvent, sessionEvent]; for (const event of events) { yield* fixture.publish(event); - yield* Queue.take(fixture.reads); + expect(yield* Queue.take(fixture.reads)).toBe(current.id); yield* reactor.drain; } expect((yield* Ref.get(fixture.commands))[0]?.branchPullRequest).toEqual(reference(42)); @@ -388,6 +419,51 @@ describe("ThreadPullRequestReactor", () => { ), ); + it.effect("refreshes the project identity when a turn adds the remote", () => + Effect.scoped( + Effect.gen(function* () { + const current = thread("new-remote"); + const fixture = yield* makeHarness({ + threads: [current], + project: { ...project, repositoryIdentity: null }, + branchPullRequest: () => Effect.succeed(branchPullRequest()), + resolveRepositoryIdentity: (_cwd, options) => + Effect.succeed(options?.refresh ? project.repositoryIdentity : null), + }); + yield* Effect.gen(function* () { + const reactor = yield* fixture.start(); + expect(yield* Ref.get(fixture.commands)).toHaveLength(0); + + yield* fixture.publish({ + type: "thread.turn-diff-completed", + sequence: 2, + eventId: EventId.make("checkpoint-finished"), + aggregateKind: "thread", + aggregateId: current.id, + occurredAt: NOW, + commandId: null, + causationEventId: null, + correlationId: null, + metadata: {}, + payload: { + threadId: current.id, + turnId: TurnId.make("turn"), + checkpointTurnCount: 1, + checkpointRef: CheckpointRef.make("checkpoint"), + status: "ready", + files: [], + assistantMessageId: null, + completedAt: NOW, + }, + }); + yield* Queue.take(fixture.reads); + yield* reactor.drain; + expect((yield* Ref.get(fixture.commands))[0]?.branchPullRequest).toEqual(reference(42)); + }).pipe(Effect.provide(fixture.layer)); + }), + ), + ); + it.effect("uses live worktrees and falls back to the project for removed worktrees", () => Effect.scoped( Effect.gen(function* () { @@ -516,6 +592,23 @@ describe("ThreadPullRequestReactor", () => { yield* Effect.gen(function* () { const reactor = yield* fixture.start(); expect(yield* Ref.get(fixture.commands)).toHaveLength(0); + // A one-thread read cannot show that other pending threads are gone. + const gone = ThreadId.make("gone"); + yield* fixture.publish({ + type: "thread.unarchived", + sequence: 2, + eventId: EventId.make("gone-unarchived"), + aggregateKind: "thread", + aggregateId: gone, + occurredAt: NOW, + commandId: null, + causationEventId: null, + correlationId: null, + metadata: {}, + payload: { threadId: gone, updatedAt: NOW }, + }); + expect(yield* Queue.take(fixture.reads)).toBe(gone); + yield* reactor.drain; yield* Ref.set(online, true); yield* TestClock.adjust("1 minute"); yield* Queue.take(fixture.reads); @@ -564,6 +657,113 @@ describe("ThreadPullRequestReactor", () => { ), ); + it.effect("discovers the same PRs from the unsettled read as from the full read", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const query = yield* ProjectionSnapshotQuery; + yield* sql`INSERT INTO projection_projects (project_id, title, workspace_root, scripts_json, created_at, updated_at) + VALUES ('project', 'Project', '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/workspace/project', '[]', ${NOW}, ${NOW}), + ('dormant', 'Dormant', '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/workspace/dormant', '[]', ${NOW}, ${NOW})`; + yield* sql`INSERT INTO projection_threads (thread_id, project_id, title, model_selection_json, runtime_mode, interaction_mode, branch, branch_pull_request_json, created_at, updated_at, archived_at, settled_override, settled_at) + VALUES + ('open', 'project', 'Open', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'open', NULL, ${NOW}, ${NOW}, NULL, NULL, NULL), + ('resumed', 'project', 'Resumed', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'resumed', NULL, ${NOW}, ${NOW}, NULL, 'active', NULL), + ('linked', 'project', 'Linked', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'linked', '{"projectId":"project","repository":"owner/repository","number":3,"url":"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/owner/repository/pull/3"}', ${NOW}, ${NOW}, NULL, NULL, NULL), + ('settled', 'project', 'Settled', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'settled', '{"projectId":"project","repository":"owner/repository","number":4,"url":"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/owner/repository/pull/4"}', ${NOW}, ${NOW}, NULL, 'settled', ${NOW}), + ('backfill', 'project', 'Backfill', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'backfill', NULL, ${NOW}, ${NOW}, NULL, 'settled', ${NOW}), + ('imported', 'dormant', 'Imported', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', NULL, NULL, ${NOW}, ${NOW}, NULL, 'settled', ${NOW}), + ('archived', 'project', 'Archived', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'archived', NULL, ${NOW}, ${NOW}, ${NOW}, NULL, NULL)`; + const numbers = new Map([ + ["open", 1], + ["resumed", 2], + ["linked", 3], + ["settled", 4], + ["backfill", 5], + ["archived", 6], + ]); + + // Startup, then two periodic passes. The startup backfill lookup fails, + // so the first periodic pass retries it from the full read. + const discover = (read: ProjectionSnapshotQueryShape["getShellSnapshot"]) => + Effect.gen(function* () { + const online = yield* Ref.make(false); + const reads: Array> = []; + const fixture = yield* makeHarness({ + threads: [], + getShellSnapshot: (options) => + read(options).pipe( + Effect.tap((snapshot) => + Effect.sync(() => reads.push(snapshot.threads.map(({ id }) => id).toSorted())), + ), + ), + branchPullRequest: ({ cwd, branch }) => + Effect.gen(function* () { + if (branch === "backfill" && !(yield* Ref.get(online))) { + return yield* new GitManagerError({ + operation: "branchPullRequest", + cwd, + detail: "Offline", + }); + } + const number = numbers.get(branch); + return number === undefined ? null : branchPullRequest(number); + }), + }); + return yield* Effect.gen(function* () { + const reactor = yield* fixture.start(); + const startupCalls = (yield* Ref.get(fixture.branchCalls)).length; + const startupCommands = (yield* Ref.get(fixture.commands)).length; + yield* Ref.set(online, true); + for (let pass = 0; pass < 2; pass++) { + yield* TestClock.adjust("1 minute"); + yield* Queue.take(fixture.reads); + yield* reactor.drain; + } + return { + reads, + branchCalls: (yield* Ref.get(fixture.branchCalls)) + .slice(startupCalls) + .map(({ branch }) => branch) + .toSorted(), + commands: (yield* Ref.get(fixture.commands)) + .slice(startupCommands) + .map( + ({ threadId, branchPullRequest }) => `${threadId} ${branchPullRequest?.number}`, + ) + .toSorted(), + }; + }).pipe(Effect.provide(fixture.layer)); + }).pipe(Effect.scoped); + + const { reads: unsettledReads, ...unsettled } = yield* discover(query.getShellSnapshot); + const { reads: fullReads, ...full } = yield* discover(() => query.getShellSnapshot()); + expect(unsettled).toEqual(full); + expect(unsettled.commands).toEqual([ + "backfill 5", + "open 1", + "open 1", + "resumed 2", + "resumed 2", + ]); + // The last pass has no backfill left, so it reads no settled thread. + expect(fullReads.at(-1)).toContain("imported"); + expect(unsettledReads.at(-1)).toEqual(["linked", "open", "resumed"]); + }).pipe( + Effect.provide( + OrchestrationProjectionSnapshotQueryLive.pipe( + Layer.provide(ThreadBackgroundLiveness.layer), + Layer.provide(ThreadPlanProgress.layer), + Layer.provide( + Layer.succeed(RepositoryIdentityResolver, { + resolve: () => Effect.succeed(project.repositoryIdentity), + }), + ), + Layer.provideMerge(SqlitePersistenceMemory), + ), + ), + ), + ); + it.effect("matches Azure SSH projects to HTTPS PRs with the provider repository selector", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/orchestration/ThreadPullRequestReactor.ts b/apps/server/src/orchestration/ThreadPullRequestReactor.ts index 17efc74e4f40..66ff81d5a9e7 100644 --- a/apps/server/src/orchestration/ThreadPullRequestReactor.ts +++ b/apps/server/src/orchestration/ThreadPullRequestReactor.ts @@ -6,6 +6,7 @@ import { CommandId, type OrchestrationEvent, type OrchestrationProjectShell, + type OrchestrationShellSnapshot, type ThreadId, type ThreadLinkedPullRequest, } from "@t3tools/contracts"; @@ -16,11 +17,13 @@ import * as Crypto from "effect/Crypto"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Schedule from "effect/Schedule"; import type * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as GitManager from "../git/GitManager.ts"; +import type { ProjectionRepositoryError } from "../persistence/Errors.ts"; import * as PullRequestService from "../pullRequest/PullRequestService.ts"; import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts"; import { forkParked } from "../serverActivation.ts"; @@ -69,6 +72,37 @@ export function pullRequestMatchesProject( ); } +/** + * Read the shell state for a discovery or settlement sweep. A sweep for one + * thread reads that thread and the projects it names, not every thread. A + * sweep over all threads reads only unsettled threads, since both sweeps skip + * settled ones. Discovery's backfill does its own full read. + */ +export const readSweepSnapshot = ( + snapshots: ProjectionSnapshotQuery.ProjectionSnapshotQueryShape, + threadId: ThreadId | null, +): Effect.Effect< + Pick, + ProjectionRepositoryError +> => + threadId === null + ? snapshots.getShellSnapshot({ unsettledOnly: true }) + : Effect.gen(function* () { + // Read the sequence first. The thread is then at least this new, so a + // command guarded by the sequence is rejected rather than missing a change. + const { snapshotSequence } = yield* snapshots.getSnapshotSequence(); + const thread = yield* snapshots.getThreadShellById(threadId); + if (Option.isNone(thread)) return { snapshotSequence, projects: [], threads: [] }; + // Settlement also checks the project a saved pull request names. + const reference = thread.value.linkedPullRequest ?? thread.value.branchPullRequest; + const projects = yield* snapshots.getProjectShells( + reference == null + ? [thread.value.projectId] + : [thread.value.projectId, reference.projectId], + ); + return { snapshotSequence, projects, threads: [thread.value] }; + }); + /** @public Service construction is part of the canonical Effect module API. */ export const make = Effect.gen(function* () { const engine = yield* OrchestrationEngine.OrchestrationEngineService; @@ -97,7 +131,11 @@ export const make = Effect.gen(function* () { const synchronize = Effect.fn("ThreadPullRequestReactor.synchronize")(function* ( request: RefreshRequest, ) { - const snapshot = yield* snapshots.getShellSnapshot(); + // Backfill looks up settled threads, so its passes read every thread. + const snapshot = + request.threadId === null && (request.backfill || pendingBackfill.size > 0) + ? yield* snapshots.getShellSnapshot() + : yield* readSweepSnapshot(snapshots, request.threadId); const projects = new Map(snapshot.projects.map((project) => [project.id, project])); if (request.backfill) { for (const thread of snapshot.threads) { @@ -109,14 +147,19 @@ export const make = Effect.gen(function* () { } } } - const threadIds = new Set(snapshot.threads.map((thread) => thread.id)); - for (const threadId of pendingBackfill.keys()) { - if (!threadIds.has(threadId)) pendingBackfill.delete(threadId); + // A single-thread read only shows whether its own thread is gone. A thread + // with no branch has nothing to look up, and its entry would keep every + // periodic pass on the full read. + const branchThreadIds = new Set( + snapshot.threads.filter((thread) => thread.branch !== null).map((thread) => thread.id), + ); + const checkedIds = request.threadId === null ? pendingBackfill.keys() : [request.threadId]; + for (const threadId of checkedIds) { + if (!branchThreadIds.has(threadId)) pendingBackfill.delete(threadId); } const threads = snapshot.threads.filter( (thread) => thread.archivedAt === null && - (request.threadId === null || thread.id === request.threadId) && ((thread.settledOverride !== "settled" && thread.settledAt === null) || request.threadId !== null || pendingBackfill.has(thread.id)) && @@ -131,8 +174,19 @@ export const make = Effect.gen(function* () { (group) => Effect.gen(function* () { const first = group[0]!; - const project = projects.get(first.projectId); - if (project === undefined) return finishBackfill(group); + const snapshotProject = projects.get(first.projectId); + if (snapshotProject === undefined) return finishBackfill(group); + // A finished turn may have added the remote this PR lives on. A failed + // refresh resolves to null, so keep the snapshot's identity then. + const project = request.refresh + ? { + ...snapshotProject, + repositoryIdentity: + (yield* repositoryIdentities.resolve(snapshotProject.workspaceRoot, { + refresh: true, + })) ?? snapshotProject.repositoryIdentity, + } + : snapshotProject; const repository = sourceControlRepositorySelector(project.repositoryIdentity); if (first.branch !== null && repository === null) return finishBackfill(group); const worktreeExists = diff --git a/apps/server/src/orchestration/ThreadSettlementReactor.test.ts b/apps/server/src/orchestration/ThreadSettlementReactor.test.ts index 07afbaf05e6a..cad45007665c 100644 --- a/apps/server/src/orchestration/ThreadSettlementReactor.test.ts +++ b/apps/server/src/orchestration/ThreadSettlementReactor.test.ts @@ -24,13 +24,17 @@ import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; import { TestClock } from "effect/testing"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; import { GitManager, type GitBranchPullRequest } from "../git/GitManager.ts"; +import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts"; +import { RepositoryIdentityResolver } from "../project/RepositoryIdentityResolver.ts"; import { PullRequestService, type PullRequestMergeEvent, @@ -38,11 +42,17 @@ import { import { ServerActivation } from "../serverActivation.ts"; import { ServerSettingsService } from "../serverSettings.ts"; import { OrchestrationCommandInvariantError } from "./Errors.ts"; +import { OrchestrationProjectionSnapshotQueryLive } from "./Layers/ProjectionSnapshotQuery.ts"; import { OrchestrationEngineService, type OrchestrationEngineShape, } from "./Services/OrchestrationEngine.ts"; -import { ProjectionSnapshotQuery } from "./Services/ProjectionSnapshotQuery.ts"; +import { + ProjectionSnapshotQuery, + type ProjectionSnapshotQueryShape, +} from "./Services/ProjectionSnapshotQuery.ts"; +import * as ThreadBackgroundLiveness from "./ThreadBackgroundLiveness.ts"; +import * as ThreadPlanProgress from "./ThreadPlanProgress.ts"; import * as ThreadSettlementReactor from "./ThreadSettlementReactor.ts"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Path from "effect/Path"; @@ -168,6 +178,8 @@ function makeBranchPullRequest( interface HarnessOptions { readonly snapshot: OrchestrationShellSnapshot; + /** Serve full sweep reads from this instead of `snapshot`. */ + readonly getShellSnapshot?: ProjectionSnapshotQueryShape["getShellSnapshot"]; readonly settings?: ServerSettings; readonly branchPullRequest?: GitManager["Service"]["branchPullRequest"]; readonly pullRequestSummary?: PullRequestService["Service"]["summary"]; @@ -181,7 +193,8 @@ const makeHarness = Effect.fn("makeThreadSettlementHarness")(function* (options: const activation = yield* Deferred.make(); const snapshots = yield* Ref.make(options.snapshot); const snapshotReadCount = yield* Ref.make(0); - const snapshotReads = yield* Queue.unbounded(); + // Each shell read: a thread id for a one-thread read, null for a full read. + const snapshotReads = yield* Queue.unbounded(); const settings = yield* Ref.make(options.settings ?? DEFAULT_SERVER_SETTINGS); const settingsReads = yield* Queue.unbounded(); const settingsChanges = yield* PubSub.unbounded(); @@ -254,10 +267,27 @@ const makeHarness = Effect.fn("makeThreadSettlementHarness")(function* (options: const dependencies = Layer.mergeAll( Layer.mock(ProjectionSnapshotQuery)({ - getShellSnapshot: () => - Ref.updateAndGet(snapshotReadCount, (count) => count + 1).pipe( - Effect.tap((count) => Queue.offer(snapshotReads, count)), - Effect.andThen(Ref.get(snapshots)), + getShellSnapshot: (readOptions) => + Ref.update(snapshotReadCount, (count) => count + 1).pipe( + Effect.andThen(Queue.offer(snapshotReads, null)), + Effect.andThen(options.getShellSnapshot?.(readOptions) ?? Ref.get(snapshots)), + ), + getSnapshotSequence: () => + Ref.get(snapshots).pipe(Effect.map(({ snapshotSequence }) => ({ snapshotSequence }))), + getThreadShellById: (threadId) => + Ref.get(snapshots).pipe( + Effect.map(({ threads }) => + Option.fromUndefinedOr( + threads.find((thread) => thread.id === threadId && thread.archivedAt === null), + ), + ), + Effect.tap(() => Queue.offer(snapshotReads, threadId)), + ), + getProjectShells: (projectIds) => + Ref.get(snapshots).pipe( + Effect.map(({ projects }) => + projects.filter((project) => projectIds?.includes(project.id) ?? true), + ), ), }), Layer.mock(GitManager)({ @@ -313,7 +343,7 @@ const makeHarness = Effect.fn("makeThreadSettlementHarness")(function* (options: const startHarness = Effect.fn("startThreadSettlementHarness")(function* ( reactor: ThreadSettlementReactor.ThreadSettlementReactor["Service"], activation: Deferred.Deferred, - snapshotReads: Queue.Queue, + snapshotReads: Queue.Queue, ) { yield* reactor.start(); yield* Deferred.succeed(activation, undefined); @@ -443,7 +473,7 @@ describe("ThreadSettlementReactor", () => { updatedAt: NOW, }, }); - yield* Queue.take(fixture.snapshotReads); + assert.strictEqual(yield* Queue.take(fixture.snapshotReads), thread.id); yield* reactor.drain; } assert.deepStrictEqual( @@ -463,7 +493,7 @@ describe("ThreadSettlementReactor", () => { aggregateId: readySession.threadId, payload: { threadId: readySession.threadId, session: readySession }, }); - yield* Queue.take(fixture.snapshotReads); + assert.strictEqual(yield* Queue.take(fixture.snapshotReads), readySession.threadId); yield* reactor.drain; assert.deepStrictEqual( (yield* Ref.get(fixture.commands)).map(({ threadId }) => threadId), @@ -1418,6 +1448,88 @@ describe("ThreadSettlementReactor", () => { }), ), ); + + it.effect("settles the same threads from the unsettled read as from the full read", () => + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse(NOW)); + const sql = yield* SqlClient.SqlClient; + const query = yield* ProjectionSnapshotQuery; + yield* sql`INSERT INTO projection_projects (project_id, title, workspace_root, scripts_json, created_at, updated_at) + VALUES ('settlement-project', 'Project', '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/workspace/project', '[]', ${NOW}, ${NOW}), + ('linked-settlement-project', 'Linked', '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/workspace/linked', '[]', ${NOW}, ${NOW}), + ('dormant-project', 'Dormant', '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/workspace/dormant', '[]', ${NOW}, ${NOW})`; + yield* sql`INSERT INTO projection_threads (thread_id, project_id, title, model_selection_json, runtime_mode, interaction_mode, branch, branch_pull_request_json, latest_user_message_at, created_at, updated_at, archived_at, settled_override, settled_at) + VALUES + ('idle', 'settlement-project', 'Idle', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', NULL, NULL, '2026-08-20T00:00:00.000Z', '2026-08-01T00:00:00.000Z', ${NOW}, NULL, NULL, NULL), + ('merged', 'settlement-project', 'Merged', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'feature', + '{"projectId":"linked-settlement-project","repository":"owner/repository","number":42,"url":"https://example.test/owner/repository/pull/42"}', + '2026-08-27T00:00:00.000Z', '2026-08-01T00:00:00.000Z', ${NOW}, NULL, NULL, NULL), + ('linked', 'settlement-project', 'Linked', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', NULL, NULL, '2026-08-27T00:00:00.000Z', '2026-08-01T00:00:00.000Z', ${NOW}, NULL, NULL, NULL), + ('open', 'settlement-project', 'Open', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'open-feature', NULL, '2026-08-27T00:00:00.000Z', '2026-08-01T00:00:00.000Z', ${NOW}, NULL, NULL, NULL), + ('resumed', 'settlement-project', 'Resumed', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', NULL, NULL, '2026-08-20T00:00:00.000Z', '2026-08-01T00:00:00.000Z', ${NOW}, NULL, 'active', NULL), + ('settled', 'dormant-project', 'Settled', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', 'done', NULL, '2026-08-20T00:00:00.000Z', '2026-08-01T00:00:00.000Z', ${NOW}, NULL, 'settled', '2026-08-21T00:00:00.000Z'), + ('archived', 'settlement-project', 'Archived', '{"provider":"codex","model":"gpt-5"}', 'full-access', 'default', NULL, NULL, '2026-08-20T00:00:00.000Z', '2026-08-01T00:00:00.000Z', ${NOW}, ${NOW}, NULL, NULL)`; + yield* sql`INSERT INTO projection_thread_pull_requests (thread_id, host, repository, number, url, source, linked_at, snapshot_json) + VALUES ('linked', 'example.test', 'owner/repository', 7, 'https://example.test/owner/repository/pull/7', 'manual', ${NOW}, + '{"state":"merged","title":"Review","headBranch":"linked","baseBranch":"main","isDraft":false,"updatedAt":"2026-08-28T12:00:00.000Z","syncedAt":"2026-08-28T12:00:00.000Z","mergedAt":"2026-08-28T12:00:00.000Z","closedAt":null}')`; + + const sweep = (read: ProjectionSnapshotQueryShape["getShellSnapshot"]) => + Effect.gen(function* () { + const readThreadIds: Array = []; + const fixture = yield* makeHarness({ + snapshot: makeSnapshot([]), + getShellSnapshot: (options) => + read(options).pipe( + Effect.tap((snapshot) => + Effect.sync(() => readThreadIds.push(...snapshot.threads.map(({ id }) => id))), + ), + ), + branchPullRequest: ({ branch }) => + Effect.succeed(branch === "open-feature" ? makeBranchPullRequest("open") : null), + pullRequestSummary: (input) => + Effect.succeed(makePullRequestSummary({ ...input, state: "merged" })), + }); + return yield* Effect.gen(function* () { + const reactor = yield* ThreadSettlementReactor.ThreadSettlementReactor; + yield* startHarness(reactor, fixture.activation, fixture.snapshotReads); + return { + readThreadIds: readThreadIds.toSorted(), + commands: (yield* Ref.get(fixture.commands)) + .map(({ threadId, settledAt }) => `${threadId} ${settledAt}`) + .toSorted(), + branchCalls: (yield* Ref.get(fixture.branchCalls)) + .map(({ branch }) => branch) + .toSorted(), + summaryCalls: yield* Ref.get(fixture.summaryCalls), + }; + }).pipe(Effect.provide(fixture.layer)); + }).pipe(Effect.scoped); + + const { readThreadIds: unsettledReads, ...unsettled } = yield* sweep(query.getShellSnapshot); + const { readThreadIds: fullReads, ...full } = yield* sweep(() => query.getShellSnapshot()); + assert.deepStrictEqual(unsettled, full); + // A settle for inactivity, for a synced merged link, and for a saved + // branch PR whose project only that PR names. + assert.deepStrictEqual(unsettled.commands, [ + "idle 2026-08-20T00:00:00.000Z", + "linked 2026-08-27T00:00:00.000Z", + "merged 2026-08-27T00:00:00.000Z", + ]); + assert.deepStrictEqual(fullReads, [...unsettledReads, "settled"].toSorted()); + assert.deepStrictEqual(unsettledReads, ["idle", "linked", "merged", "open", "resumed"]); + }).pipe( + Effect.provide( + OrchestrationProjectionSnapshotQueryLive.pipe( + Layer.provide(ThreadBackgroundLiveness.layer), + Layer.provide(ThreadPlanProgress.layer), + Layer.provide( + Layer.succeed(RepositoryIdentityResolver, { resolve: () => Effect.succeed(null) }), + ), + Layer.provideMerge(SqlitePersistenceMemory), + ), + ), + ), + ); }); describe("storage cleanup", () => { diff --git a/apps/server/src/orchestration/ThreadSettlementReactor.ts b/apps/server/src/orchestration/ThreadSettlementReactor.ts index 4192896efed5..22d21ff27996 100644 --- a/apps/server/src/orchestration/ThreadSettlementReactor.ts +++ b/apps/server/src/orchestration/ThreadSettlementReactor.ts @@ -23,7 +23,7 @@ import * as ServerSettings from "../serverSettings.ts"; import { forkParked } from "../serverActivation.ts"; import * as OrchestrationEngine from "./Services/OrchestrationEngine.ts"; import * as ProjectionSnapshotQuery from "./Services/ProjectionSnapshotQuery.ts"; -import { pullRequestMatchesProject } from "./ThreadPullRequestReactor.ts"; +import { pullRequestMatchesProject, readSweepSnapshot } from "./ThreadPullRequestReactor.ts"; import { isAutoSettlementCandidate, resolveAutoSettlementAt, @@ -95,20 +95,16 @@ export const make = Effect.gen(function* () { if (!autoSettlementConfigured(settings)) { return; } - const snapshot = yield* snapshots.getShellSnapshot(); + const snapshot = yield* readSweepSnapshot(snapshots, threadId ?? null); const now = DateTime.formatIso(yield* DateTime.now); const projects = new Map(snapshot.projects.map((project) => [project.id, project])); // A merge rechecks all candidates, including branches that discovery has // not linked yet. Those lookups can still have cached the PR as open. - const candidates = snapshot.threads.filter( - (thread) => - (threadId === undefined || thread.id === threadId) && - isAutoSettlementCandidate(thread, now), - ); + const candidates = snapshot.threads.filter((thread) => isAutoSettlementCandidate(thread, now)); // Return the thread when it still needs a pull request decision. A rejected // dispatch skips it for this snapshot instead of retrying through a lookup. - const settleThread = Effect.fn("ThreadSettlementReactor.settleThread")( + const settleThread = Effect.fnUntraced( function* (thread: (typeof candidates)[number], pullRequest: SettlementPullRequest | null) { const settings = resolveProjectSettings( yield* settingsService.getSettings, diff --git a/apps/server/src/persistence/ProviderSessionRuntime.ts b/apps/server/src/persistence/ProviderSessionRuntime.ts index 80588f58858f..1ec7adcd623b 100644 --- a/apps/server/src/persistence/ProviderSessionRuntime.ts +++ b/apps/server/src/persistence/ProviderSessionRuntime.ts @@ -102,11 +102,14 @@ export class ProviderSessionRuntimeRepository extends Context.Service< >; /** - * List all provider runtime rows. + * List provider runtime rows. * - * Returned in ascending last-seen order. + * Returned in ascending last-seen order. `excludeStopped` filters stopped + * rows in SQL. Long-lived installs keep thousands for their resume cursors. */ - readonly list: () => Effect.Effect< + readonly list: (options?: { + readonly excludeStopped?: boolean; + }) => Effect.Effect< ReadonlyArray, ProviderSessionRuntimeRepositoryError >; @@ -336,9 +339,9 @@ export const make = Effect.gen(function* () { }); const listRuntimeRows = SqlSchema.findAll({ - Request: Schema.Void, + Request: Schema.Struct({ excludeStopped: Schema.Boolean }), Result: ProviderSessionRuntimeRawDbRowSchema, - execute: () => + execute: ({ excludeStopped }) => sql` SELECT thread_id AS "threadId", @@ -351,6 +354,7 @@ export const make = Effect.gen(function* () { resume_cursor_json AS "resumeCursor", runtime_payload_json AS "runtimePayload" FROM provider_session_runtime + ${excludeStopped ? sql`WHERE status != 'stopped'` : sql``} ORDER BY last_seen_at ASC, thread_id ASC `, }); @@ -414,8 +418,8 @@ export const make = Effect.gen(function* () { ), ); - const list: ProviderSessionRuntimeRepository["Service"]["list"] = () => - listRuntimeRows(undefined).pipe( + const list: ProviderSessionRuntimeRepository["Service"]["list"] = (options) => + listRuntimeRows({ excludeStopped: options?.excludeStopped === true }).pipe( Effect.mapError( toPersistenceSqlOrDecodeError( "ProviderSessionRuntimeRepository.list:query", diff --git a/apps/server/src/preview/PortScanner.test.ts b/apps/server/src/preview/PortScanner.test.ts index 7fa15defeca9..d790002f95f1 100644 --- a/apps/server/src/preview/PortScanner.test.ts +++ b/apps/server/src/preview/PortScanner.test.ts @@ -18,6 +18,7 @@ import * as Layer from "effect/Layer"; import * as PlatformError from "effect/PlatformError"; import * as Scope from "effect/Scope"; import * as TestClock from "effect/testing/TestClock"; +import * as Tracer from "effect/Tracer"; import { expect } from "vite-plus/test"; import { FetchHttpClient } from "effect/unstable/http"; @@ -441,6 +442,26 @@ effectIt.effect("stops probing a subscriber's configured paths after its scope c }).pipe(Effect.scoped, Effect.provide(layer)); }); +effectIt.effect("writes no poll span while no client retains the scanner", () => { + let pollSpans = 0; + const tracer = Tracer.make({ + span: (options) => { + if (options.name === "PortDiscovery.pollTick") pollSpans += 1; + return new Tracer.NativeSpan(options); + }, + }); + const layer = makeProbeFailureLayer(processProbeFailure); + + return Effect.gen(function* () { + const scanner = yield* PortScanner.PortDiscovery; + yield* TestClock.adjust(Duration.seconds(15)); + expect(pollSpans).toBe(0); + + yield* scanner.retain; + expect(pollSpans).toBe(1); + }).pipe(Effect.scoped, Effect.provide(layer), Effect.withTracer(tracer)); +}); + effectIt.effect("uses the current configured fragment when readiness comes from cache", () => { const requests: string[] = []; const fetchFn = ((input: Parameters[0]) => { diff --git a/apps/server/src/preview/PortScanner.ts b/apps/server/src/preview/PortScanner.ts index f4d73d62320d..9eee1a3e215e 100644 --- a/apps/server/src/preview/PortScanner.ts +++ b/apps/server/src/preview/PortScanner.ts @@ -550,7 +550,6 @@ export const make = Effect.gen(function* PortDiscoveryMake() { const pollTick = Effect.fn("PortDiscovery.pollTick")( function* () { - if ((yield* Ref.get(stateRef)).retainCount <= 0) return; const configuredUrls = [ ...new Set( [...(yield* Ref.get(stateRef)).listeners.values()].flatMap( @@ -579,9 +578,12 @@ export const make = Effect.gen(function* PortDiscoveryMake() { ), ); - // Single layer-scoped polling fiber. Ticks are no-ops when no client is - // currently retained, so the cost is one Ref.get every POLL_INTERVAL. - yield* Effect.forkScoped(pollTick().pipe(Effect.repeat(Schedule.spaced(POLL_INTERVAL)))); + // Single layer-scoped polling fiber. Ticks skip the scan and its span when no + // client is currently retained, so the cost is one Ref.get every POLL_INTERVAL. + const pollIfRetained = Ref.get(stateRef).pipe( + Effect.flatMap((state) => (state.retainCount > 0 ? pollTick() : Effect.void)), + ); + yield* Effect.forkScoped(pollIfRetained.pipe(Effect.repeat(Schedule.spaced(POLL_INTERVAL)))); const acquireRetention = Effect.fn("PortDiscovery.retain")(function* () { const wasIdle = yield* Ref.modify(stateRef, (state) => [ diff --git a/apps/server/src/process/externalLauncher.test.ts b/apps/server/src/process/externalLauncher.test.ts index f714a70f783d..aab4d78f4d71 100644 --- a/apps/server/src/process/externalLauncher.test.ts +++ b/apps/server/src/process/externalLauncher.test.ts @@ -923,6 +923,18 @@ it.effect("discovers editors through the service API", () => ); for (const { platform, installPath, editor, args } of [ + { + platform: "darwin", + installPath: "Applications/Antigravity IDE.app/Contents/Resources/app/bin/antigravity-ide", + editor: "antigravity", + args: ["--goto", "/workspace with spaces/file.ts:12:4"], + }, + { + platform: "linux", + installPath: ".local/bin/antigravity-ide", + editor: "antigravity", + args: ["--goto", "/workspace with spaces/file.ts:12:4"], + }, { platform: "darwin", installPath: "Applications/Cursor.app/Contents/Resources/app/bin/code", @@ -1028,6 +1040,45 @@ for (const { platform, installPath, editor, args } of [ ); } +// `agy` is the standalone Antigravity CLI, which installs to ~/.local/bin on +// macOS and Linux and to its own bin folder on Windows. It is not the IDE. +for (const { platform, installPath, onPath } of [ + { platform: "darwin", installPath: ".local/bin/agy", onPath: true }, + { platform: "linux", installPath: ".local/bin/agy", onPath: false }, + { platform: "win32", installPath: "agy/bin/agy.cmd", onPath: true }, +] as const) { + it.effect.skipIf(windowsHost && platform !== "win32")( + `does not report the agy CLI as the Antigravity IDE on ${platform}`, + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-agy-cli-" }); + const executable = path.join(home, installPath); + yield* fs.makeDirectory(path.dirname(executable), { recursive: true }); + yield* fs.writeFileString(executable, "#!/bin/sh\n"); + yield* fs.chmod(executable, 0o755); + const editors = yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + return yield* launcher.resolveAvailableEditors(); + }).pipe( + Effect.provide( + testLayer({ + platform, + env: { + HOME: home, + LOCALAPPDATA: home, + PATH: onPath ? path.dirname(executable) : path.join(home, "empty"), + PATHEXT: ".COM;.EXE;.BAT;.CMD", + }, + }), + ), + ); + assert.notInclude(editors, "antigravity"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +} + it.effect.skipIf(windowsHost)("ignores unusable app bundles and keeps PATH launchers first", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/process/externalLauncher.ts b/apps/server/src/process/externalLauncher.ts index 29c25e790c61..ab287e4d78ee 100644 --- a/apps/server/src/process/externalLauncher.ts +++ b/apps/server/src/process/externalLauncher.ts @@ -20,7 +20,11 @@ import { } from "@t3tools/contracts"; import { resolveEditorCommand } from "@t3tools/shared/editor"; import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; -import { isCommandAvailable, resolveSpawnCommand } from "@t3tools/shared/shell"; +import { + isCommandAvailable, + resolveSpawnCommand, + withPathDirectoryListings, +} from "@t3tools/shared/shell"; import * as Clock from "effect/Clock"; import * as Config from "effect/Config"; import * as Context from "effect/Context"; @@ -442,7 +446,7 @@ const resolveBrowserLaunch = Effect.fn("externalLauncher.resolveBrowserLaunch")( const resolveAvailableEditors = Effect.fn("externalLauncher.resolveAvailableEditors")(function* () { const platform = yield* HostProcessPlatform; const env = { ...(yield* readBrowserLaunchEnv), ...(yield* readCommandLookupEnv) }; - return yield* buildAvailableEditors(platform, env); + return yield* buildAvailableEditors(platform, env).pipe(withPathDirectoryListings); }); const resolveFileManagerRevealKind = Effect.fn("externalLauncher.resolveFileManagerRevealKind")( diff --git a/apps/server/src/processRunner.ts b/apps/server/src/processRunner.ts index 36bb5b649f06..049125de3fbf 100644 --- a/apps/server/src/processRunner.ts +++ b/apps/server/src/processRunner.ts @@ -171,7 +171,8 @@ export const isWindowsCommandNotFound = Effect.fn("processRunner.isWindowsComman }, ); -const collectText = Effect.fn("processRunner.collectText")(function* (input: { +// Untraced: no attributes, and its time is the runProcessCore span. Errors fail that span. +const collectText = Effect.fnUntraced(function* (input: { readonly command: string; readonly args: ReadonlyArray; readonly cwd?: string | undefined; diff --git a/apps/server/src/project/RepositoryIdentityResolver.test.ts b/apps/server/src/project/RepositoryIdentityResolver.test.ts index d6ddb0b9263f..58f199b834e2 100644 --- a/apps/server/src/project/RepositoryIdentityResolver.test.ts +++ b/apps/server/src/project/RepositoryIdentityResolver.test.ts @@ -94,6 +94,8 @@ it.layer(NodeServices.layer)("RepositoryIdentityResolverLive", (it) => { const resolver = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; const first = yield* resolver.resolve("/repo/packages/web"); rootPath = "/repo/packages/web"; + // Longer than the one-minute cadence of the background sweeps. + yield* TestClock.adjust(Duration.minutes(10)); const second = yield* resolver.resolve("/repo/packages/web"); expect(first?.canonicalKey).toBe("github.com/t3tools/t3code"); @@ -123,10 +125,10 @@ it.layer(NodeServices.layer)("RepositoryIdentityResolverLive", (it) => { const unavailable = yield* resolver.resolve(rootPath, { refresh: true }); expect(unavailable?.webUrl).toBeUndefined(); expect(unavailable?.canonicalKey).toBe("ssh.forge.test/team/repo"); - }).pipe(Effect.provide(resolverLayer)); + }).pipe(Effect.provide(Layer.merge(TestClock.layer(), resolverLayer))); }); - it.effect("retries Git root discovery after a failed lookup", () => { + it.effect("retries Git root discovery after the negative TTL", () => { const calls: Array> = []; let rootAttempts = 0; const processRunner = Layer.succeed(ProcessRunner.ProcessRunner, { @@ -159,7 +161,9 @@ it.layer(NodeServices.layer)("RepositoryIdentityResolverLive", (it) => { return Effect.gen(function* () { const resolver = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; expect(yield* resolver.resolve("/repo/packages/web")).toBeNull(); + expect(yield* resolver.resolve("/repo/packages/web")).toBeNull(); + yield* TestClock.adjust(Duration.minutes(1)); const recovered = yield* resolver.resolve("/repo/packages/web"); expect(recovered?.rootPath).toBe("/repo"); expect(calls).toEqual([ @@ -167,7 +171,7 @@ it.layer(NodeServices.layer)("RepositoryIdentityResolverLive", (it) => { ["-C", "/repo/packages/web", "rev-parse", "--show-toplevel"], ["-C", "/repo", "remote", "-v"], ]); - }).pipe(Effect.provide(resolverLayer)); + }).pipe(Effect.provide(Layer.merge(TestClock.layer(), resolverLayer))); }); it.effect("normalizes equivalent GitHub remotes into a stable repository identity", () => diff --git a/apps/server/src/project/RepositoryIdentityResolver.ts b/apps/server/src/project/RepositoryIdentityResolver.ts index 2d7f5d02d02e..5acafa47e2e2 100644 --- a/apps/server/src/project/RepositoryIdentityResolver.ts +++ b/apps/server/src/project/RepositoryIdentityResolver.ts @@ -13,7 +13,11 @@ import * as Layer from "effect/Layer"; import * as ProcessRunner from "../processRunner.ts"; const DEFAULT_REPOSITORY_IDENTITY_CACHE_CAPACITY = 512; -const DEFAULT_POSITIVE_CACHE_TTL = Duration.minutes(1); +// Background sweeps resolve every project each minute. A long TTL keeps them +// from spawning git each time. Clone, publish, and PR discovery (after a turn +// and before it saves links) resolve with `refresh: true`. +const DEFAULT_POSITIVE_CACHE_TTL = Duration.minutes(15); +// Short, so a folder that gains a repository or a remote shows up quickly. const DEFAULT_NEGATIVE_CACHE_TTL = Duration.minutes(1); export interface RepositoryIdentityResolverOptions { @@ -142,20 +146,23 @@ export const make = Effect.fn("RepositoryIdentityResolver.make")(function* ( const processRunner = yield* ProcessRunner.ProcessRunner; const cacheCapacity = options.cacheCapacity ?? DEFAULT_REPOSITORY_IDENTITY_CACHE_CAPACITY; const refine = options.refine ?? Effect.succeed; + // Git errors and timeouts resolve to null, so they use the negative TTL like + // "no repository" or "no remote". Only interrupts and defects skip the cache. + const timeToLive = (exit: Exit.Exit) => + Exit.match(exit, { + onSuccess: (value) => + value === null + ? (options.negativeCacheTtl ?? DEFAULT_NEGATIVE_CACHE_TTL) + : (options.positiveCacheTtl ?? DEFAULT_POSITIVE_CACHE_TTL), + onFailure: () => Duration.zero, + }); const repositoryRootCache = yield* Cache.makeWith( (cwd) => resolveRepositoryIdentityCacheKey(cwd).pipe( Effect.provideService(ProcessRunner.ProcessRunner, processRunner), ), - { - capacity: cacheCapacity, - timeToLive: Exit.match({ - onSuccess: (value) => - value === null ? Duration.zero : (options.positiveCacheTtl ?? DEFAULT_POSITIVE_CACHE_TTL), - onFailure: () => Duration.zero, - }), - }, + { capacity: cacheCapacity, timeToLive }, ); const repositoryIdentityCache = yield* Cache.makeWith( @@ -167,27 +174,20 @@ export const make = Effect.fn("RepositoryIdentityResolver.make")(function* ( (identity) => refine(identity).pipe(Effect.orElseSucceed(() => identity)), ), ), - { - capacity: cacheCapacity, - timeToLive: Exit.match({ - onSuccess: (value) => - value === null - ? (options.negativeCacheTtl ?? DEFAULT_NEGATIVE_CACHE_TTL) - : (options.positiveCacheTtl ?? DEFAULT_POSITIVE_CACHE_TTL), - onFailure: () => Duration.zero, - }), - }, + { capacity: cacheCapacity, timeToLive }, ); - const resolve: RepositoryIdentityResolver["Service"]["resolve"] = Effect.fn( - "RepositoryIdentityResolver.resolve", - )(function* (cwd, options) { - if (options?.refresh) yield* Cache.invalidate(repositoryRootCache, cwd); - const cacheKey = yield* Cache.get(repositoryRootCache, cwd); - if (cacheKey === null) return null; - if (options?.refresh) yield* Cache.invalidate(repositoryIdentityCache, cacheKey); - return yield* Cache.get(repositoryIdentityCache, cacheKey); - }); + // Untraced because almost every call is a cache hit. The lookups that spawn + // git keep their own spans. + const resolve: RepositoryIdentityResolver["Service"]["resolve"] = Effect.fnUntraced( + function* (cwd, options) { + if (options?.refresh) yield* Cache.invalidate(repositoryRootCache, cwd); + const cacheKey = yield* Cache.get(repositoryRootCache, cwd); + if (cacheKey === null) return null; + if (options?.refresh) yield* Cache.invalidate(repositoryIdentityCache, cacheKey); + return yield* Cache.get(repositoryIdentityCache, cacheKey); + }, + ); return RepositoryIdentityResolver.of({ resolve }); }); diff --git a/apps/server/src/provider/CodexDeveloperInstructions.ts b/apps/server/src/provider/CodexDeveloperInstructions.ts index 2ed593d159bc..e134ebfc68cb 100644 --- a/apps/server/src/provider/CodexDeveloperInstructions.ts +++ b/apps/server/src/provider/CodexDeveloperInstructions.ts @@ -12,7 +12,7 @@ Do not switch to global browser skills, Chrome, Node REPL browser automation, st const T3_CODE_DEVICE_TOOL_INSTRUCTIONS = `## T3 Code devices -The \`t3-code\` MCP server also exposes \`device_*\` tools for iOS Simulators and Android Emulators on this environment. For mobile verification, call \`device_list\`, then \`device_open\` so the user can watch the device in their Device panel; its result explains how to drive the device. Driving happens through the \`agent-device\` CLI, which is on PATH. Keep the host config and session flags returned by \`device_open\` on every command so concurrent devices stay independent: prefer \`agent-device snapshot -i\` refs over coordinates, and use \`device_screenshot\` when you need to see the screen. Do not call simctl, adb, xcrun, or serve-sim directly while these tools are present. If \`device_list\` reports a platform as unavailable, say so instead of trying another route.`; +The \`t3-code\` MCP server also exposes \`device_*\` tools for iOS Simulators and Android Emulators on this environment. For mobile verification, call \`device_list\`, then \`device_open\` so the user can watch the device in their Device panel; its result explains how to drive the device. Driving happens through the \`agent-device\` CLI, which is on PATH. Keep the host config and session flags returned by \`device_open\` on every command so concurrent devices stay independent: prefer \`agent-device snapshot -i\` refs over coordinates, and use \`device_screenshot\` when you need to see the screen. Prefer these tools and \`agent-device\` for opening and driving devices. Platform tools such as \`xcrun simctl\` and \`adb\` remain available for anything they do not cover, such as builds, logs, or port forwarding. If \`device_list\` reports a platform as unavailable, say so.`; export interface T3CodeToolAvailability { readonly browser: boolean; diff --git a/apps/server/src/provider/Layers/AntigravityAdapter.ts b/apps/server/src/provider/Layers/AntigravityAdapter.ts index e12a320c4d48..c5e6400d11b4 100644 --- a/apps/server/src/provider/Layers/AntigravityAdapter.ts +++ b/apps/server/src/provider/Layers/AntigravityAdapter.ts @@ -786,9 +786,9 @@ export const makeAntigravityAdapter = Effect.fn("makeAntigravityAdapter")(functi stopOwned, Effect.gen(function* () { const mcp = McpProviderSession.readMcpProviderSession(input.threadId); - // The attachments dir grant lets the agent read pasted files at - // the paths ProviderService injects into the turn text. It is a - // leaf directory holding only uploads. + // The attachments dir grant lets the agent read path-only uploads + // at the paths ProviderService injects into the turn text. It is + // a leaf directory holding only uploads. const runtime = yield* options.makeRuntime({ cwd, clientInfo: { name: "t3-code", version: "0.0.0" }, diff --git a/apps/server/src/provider/Layers/ProviderService.test.ts b/apps/server/src/provider/Layers/ProviderService.test.ts index 547935a51e3c..7f0465b9c401 100644 --- a/apps/server/src/provider/Layers/ProviderService.test.ts +++ b/apps/server/src/provider/Layers/ProviderService.test.ts @@ -648,6 +648,78 @@ it.effect("ProviderServiceLive catches stopAll failures during shutdown", () => }), ); +it.effect("ProviderServiceLive shutdown leaves settled session rows untouched", () => + Effect.gen(function* () { + const recordedAnalytics = makeRecordingAnalytics(); + const codex = makeFakeCodexAdapter(); + const persistence = yield* Layer.build( + ProviderSessionDirectoryLive.pipe( + Layer.provide(ProviderSessionRuntime.layer.pipe(Layer.provide(SqlitePersistenceMemory))), + ), + ); + const directory = yield* ProviderSessionDirectory.ProviderSessionDirectory.pipe( + Effect.provide(persistence), + ); + const seed = (threadId: ThreadId, status: "running" | "stopped", activeTurnId: TurnId | null) => + directory.upsert({ + threadId, + provider: CODEX_DRIVER, + providerInstanceId: codexInstanceId, + status, + runtimePayload: { cwd: "/repo", activeTurnId }, + }); + const readBindings = directory + .listBindings() + .pipe( + Effect.map((bindings) => new Map(bindings.map((binding) => [binding.threadId, binding]))), + ); + const settledId = asThreadId("shutdown-settled"); + const runningId = asThreadId("shutdown-running"); + const stoppedWithTurnId = asThreadId("shutdown-stopped-with-turn"); + yield* seed(settledId, "stopped", null); + yield* seed(runningId, "running", asTurnId("running-turn")); + yield* seed(stoppedWithTurnId, "stopped", asTurnId("stale-turn")); + const settledBefore = (yield* readBindings).get(settledId); + assert(settledBefore !== undefined); + + const scope = yield* Scope.make(); + yield* Layer.build( + makeProviderServiceLive().pipe( + Layer.provide(NodeServices.layer), + Layer.provide(Layer.succeed(ProviderSessionDirectory.ProviderSessionDirectory, directory)), + Layer.provide( + Layer.succeed( + ProviderAdapterRegistry.ProviderAdapterRegistry, + makeStaticInstanceRegistry([[codexInstanceId, codex.adapter]]), + ), + ), + Layer.provide(defaultServerSettingsLayer), + Layer.provide(serverConfigTestLayer), + Layer.provide(recordedAnalytics.layer), + Layer.provide( + Layer.succeed( + ProviderEventLoggers.ProviderEventLoggers, + ProviderEventLoggers.NoOpProviderEventLoggers, + ), + ), + ), + ).pipe(Scope.provide(scope)); + yield* TestClock.adjust("1 minute"); + yield* Scope.close(scope, Exit.void); + + const byThread = yield* readBindings; + assert.deepStrictEqual(byThread.get(settledId), settledBefore); + for (const threadId of [runningId, stoppedWithTurnId]) { + const binding = byThread.get(threadId); + assert.equal(binding?.status, "stopped"); + assert.propertyVal(binding?.runtimePayload, "activeTurnId", null); + assert.propertyVal(binding?.runtimePayload, "lastRuntimeEvent", "provider.stopAll"); + } + const [stoppedAll] = recordedAnalytics.eventsByName("provider.sessions.stopped_all"); + assert.equal(stoppedAll?.properties?.stoppedSessionCount, 2); + }).pipe(Effect.provide(NodeServices.layer)), +); + it.effect("ProviderServiceLive flushes deferred completions during shutdown", () => Effect.gen(function* () { const recordedAnalytics = makeRecordingAnalytics(); @@ -4729,6 +4801,57 @@ validation.layer("ProviderServiceLive validation", (it) => { }), ); + it.effect("rejects a file when its path cannot fit in the prompt", () => + Effect.gen(function* () { + const provider = yield* ProviderService.ProviderService; + validation.codex.sendTurn.mockClear(); + const failure = yield* provider + .sendTurn({ + threadId: asThreadId("thread-file-path-context-limit"), + input: "x".repeat(PROVIDER_SEND_TURN_MAX_INPUT_CHARS), + attachments: [ + { + type: "file", + id: "thread-attach-12345678-1234-1234-1234-123456789abc-zip", + name: "archive.zip", + mimeType: "application/zip", + sizeBytes: 1024, + }, + ], + }) + .pipe(Effect.flip); + + assert.instanceOf(failure, ProviderValidationError); + assert.include(failure.issue, String(PROVIDER_SEND_TURN_MAX_INPUT_CHARS)); + assert.equal(validation.codex.sendTurn.mock.calls.length, 0); + }), + ); + + it.effect("sends a native image when its path cannot fit in the prompt", () => + Effect.gen(function* () { + const provider = yield* ProviderService.ProviderService; + const threadId = asThreadId("thread-image-path-context-limit"); + yield* provider.startSession(threadId, { + provider: ProviderDriverKind.make("codex"), + providerInstanceId: codexInstanceId, + threadId, + runtimeMode: "full-access", + }); + validation.codex.sendTurn.mockClear(); + const attachment = { + type: "image" as const, + id: "thread-attach-12345678-1234-1234-1234-123456789abc-png", + name: "screen.png", + mimeType: "image/png", + sizeBytes: 1024, + }; + const input = "x".repeat(PROVIDER_SEND_TURN_MAX_INPUT_CHARS); + yield* provider.sendTurn({ threadId, input, attachments: [attachment] }); + assert.equal(validation.codex.sendTurn.mock.calls[0]?.[0].input, input); + assert.deepEqual(validation.codex.sendTurn.mock.calls[0]?.[0].attachments, [attachment]); + }), + ); + it.effect("rejects citation-expanded input over the provider character limit", () => Effect.gen(function* () { const provider = yield* ProviderService.ProviderService; diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index 5e5052d5ef23..f4e7b0b39bdb 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -427,6 +427,14 @@ function readPersistedCwd( return trimmed.length > 0 ? trimmed : undefined; } +/** Stopped rows with no active turn are settled; shutdown leaves them untouched. */ +function isSettledBinding(binding: ProviderSessionDirectory.ProviderRuntimeBinding): boolean { + if (binding.status !== "stopped") return false; + const payload = binding.runtimePayload; + if (!payload || typeof payload !== "object" || Array.isArray(payload)) return true; + return !("activeTurnId" in payload) || payload.activeTurnId == null; +} + const dieOnMissingBindingInstanceId = ( operation: string, payload: { @@ -1626,10 +1634,12 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ? `[Pasted text "${attachment.name}" is saved at: ${attachmentPath}. Inspect it as needed.]` : `[Attached ${attachment.type} "${attachment.name}" is saved at: ${attachmentPath}]`, ); - if (isPastedText && !appended) { + // Most adapters see generic files only through this path line, so a file + // without one would be silently dropped. Images still go natively. + if (!appended && attachment.type === "file") { return yield* toValidationError( "ProviderService.sendTurn", - `Input plus pasted-text attachment context exceeds the ${PROVIDER_SEND_TURN_MAX_INPUT_CHARS} character limit`, + `Input plus attachment context exceeds the ${PROVIDER_SEND_TURN_MAX_INPUT_CHARS} character limit`, ); } } @@ -2331,7 +2341,6 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( return [completed, state] as const; }); yield* recordCompletedTurnProperties(properties); - const threadIds = yield* directory.listThreadIds(); const currentAdapters = yield* getAdapterEntries; const activeSessions = yield* Effect.forEach(currentAdapters, ([instanceId, adapter]) => adapter.listSessions().pipe( @@ -2362,7 +2371,12 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( yield* Effect.forEach(currentAdapters, ([, adapter]) => adapter.stopAll()).pipe(Effect.asVoid); yield* McpSessionRegistry.revokeAllActiveMcpCredentials(); McpProviderSession.clearAllMcpProviderSessions(); - const bindings = yield* directory.listBindings().pipe(Effect.orElseSucceed(() => [])); + // Stopped rows stay for their resume cursors, so long-lived installs hold + // thousands. Only rewrite the ones this shutdown actually stops. + const bindings = yield* directory.listBindings().pipe( + Effect.map((all) => all.filter((binding) => !isSettledBinding(binding))), + Effect.orElseSucceed(() => []), + ); yield* Effect.forEach(bindings, (binding) => Effect.gen(function* () { const providerInstanceId = dieOnMissingBindingInstanceId( @@ -2382,8 +2396,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( }); }), ).pipe(Effect.asVoid); + // Not `sessionCount`: that older property counted every row, so a new name + // keeps the two meanings in separate series. yield* analytics.record("provider.sessions.stopped_all", { - sessionCount: threadIds.length, + stoppedSessionCount: bindings.length, }); yield* analytics.flush; }); diff --git a/apps/server/src/provider/Layers/ProviderSessionDirectory.test.ts b/apps/server/src/provider/Layers/ProviderSessionDirectory.test.ts index 8b41bd3e518c..45615fe7b440 100644 --- a/apps/server/src/provider/Layers/ProviderSessionDirectory.test.ts +++ b/apps/server/src/provider/Layers/ProviderSessionDirectory.test.ts @@ -372,6 +372,41 @@ it.layer(makeDirectoryLayer(SqlitePersistenceMemory))("ProviderSessionDirectoryL }), ); + it.effect("lists only bindings that are not stopped when asked", () => + Effect.gen(function* () { + const directory = yield* ProviderSessionDirectory; + const runtimeRepository = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository; + const statuses = ["running", "starting", "error", "stopped"] as const; + const threadIds = new Set(); + + for (const status of statuses) { + const threadId = ThreadId.make(`thread-exclude-stopped-${status}`); + threadIds.add(threadId); + yield* runtimeRepository.upsert({ + threadId, + providerName: "codex", + providerInstanceId: ProviderInstanceId.make("codex"), + adapterKey: "codex", + runtimeMode: "full-access", + status, + lastSeenAt: "2026-04-14T12:00:00.000Z", + resumeCursor: null, + runtimePayload: null, + }); + } + + const liveStatuses = (yield* directory.listBindings({ excludeStopped: true })) + .filter((binding) => threadIds.has(binding.threadId)) + .map((binding) => binding.status); + const allStatuses = (yield* directory.listBindings()) + .filter((binding) => threadIds.has(binding.threadId)) + .map((binding) => binding.status); + + assert.deepEqual(liveStatuses.toSorted(), ["error", "running", "starting"]); + assert.deepEqual(allStatuses.toSorted(), ["error", "running", "starting", "stopped"]); + }), + ); + it.effect( "resets adapterKey to the new provider when provider changes without an explicit adapter key", () => diff --git a/apps/server/src/provider/Layers/ProviderSessionDirectory.ts b/apps/server/src/provider/Layers/ProviderSessionDirectory.ts index 03f1ece0b5e0..0e9de8fb868f 100644 --- a/apps/server/src/provider/Layers/ProviderSessionDirectory.ts +++ b/apps/server/src/provider/Layers/ProviderSessionDirectory.ts @@ -182,8 +182,8 @@ const makeProviderSessionDirectory = Effect.gen(function* () { Effect.map((rows) => rows.map((row) => row.threadId)), ); - const listBindings: ProviderSessionDirectoryShape["listBindings"] = () => - repository.list().pipe( + const listBindings: ProviderSessionDirectoryShape["listBindings"] = (options) => + repository.list(options).pipe( Effect.mapError(toPersistenceError("ProviderSessionDirectory.listBindings:list")), Effect.flatMap((rows) => Effect.forEach( diff --git a/apps/server/src/provider/Layers/ProviderSessionReaper.ts b/apps/server/src/provider/Layers/ProviderSessionReaper.ts index 8f04bd1821fd..bf8199f80eac 100644 --- a/apps/server/src/provider/Layers/ProviderSessionReaper.ts +++ b/apps/server/src/provider/Layers/ProviderSessionReaper.ts @@ -35,15 +35,13 @@ const makeProviderSessionReaper = (options?: ProviderSessionReaperLiveOptions) = const sweepIntervalMs = Math.max(1, options?.sweepIntervalMs ?? DEFAULT_SWEEP_INTERVAL_MS); const sweep = Effect.gen(function* () { - const bindings = yield* directory.listBindings(); + // Stopped rows stay for their resume cursors and far outnumber live + // ones, so the query skips them. + const bindings = yield* directory.listBindings({ excludeStopped: true }); const now = yield* Clock.currentTimeMillis; let reapedCount = 0; for (const binding of bindings) { - if (binding.status === "stopped") { - continue; - } - const lastSeenMs = Date.parse(binding.lastSeenAt); if (Number.isNaN(lastSeenMs)) { yield* Effect.logWarning("provider.session.reaper.invalid-last-seen", { @@ -122,7 +120,7 @@ const makeProviderSessionReaper = (options?: ProviderSessionReaperLiveOptions) = if (reapedCount > 0) { yield* Effect.logInfo("provider.session.reaper.sweep-complete", { reapedCount, - totalBindings: bindings.length, + liveBindings: bindings.length, }); } }); diff --git a/apps/server/src/provider/Services/ProviderSessionDirectory.ts b/apps/server/src/provider/Services/ProviderSessionDirectory.ts index 9dbafd3e804e..1b5c47ec1d62 100644 --- a/apps/server/src/provider/Services/ProviderSessionDirectory.ts +++ b/apps/server/src/provider/Services/ProviderSessionDirectory.ts @@ -70,7 +70,10 @@ export interface ProviderSessionDirectoryShape { ProviderSessionDirectoryPersistenceError >; - readonly listBindings: () => Effect.Effect< + /** `excludeStopped` skips stopped rows in the query, not after decoding. */ + readonly listBindings: (options?: { + readonly excludeStopped?: boolean; + }) => Effect.Effect< ReadonlyArray, ProviderSessionDirectoryPersistenceError >; diff --git a/apps/server/src/provider/acp/AntigravityAcpSupport.test.ts b/apps/server/src/provider/acp/AntigravityAcpSupport.test.ts index aee10782a613..7d6cfb1e2620 100644 --- a/apps/server/src/provider/acp/AntigravityAcpSupport.test.ts +++ b/apps/server/src/provider/acp/AntigravityAcpSupport.test.ts @@ -2,7 +2,6 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; import { ANTIGRAVITY_DEFAULT_MODEL, - PROVIDER_SEND_TURN_MAX_FILE_BYTES, PROVIDER_SEND_TURN_MAX_IMAGE_BYTES, type ChatAttachment, type RuntimeMode, @@ -15,7 +14,6 @@ import type * as EffectAcpSchema from "effect-acp/schema"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import { - ANTIGRAVITY_MAX_TEXT_ATTACHMENT_BYTES, antigravityPermissionMode, applyAntigravityAcpModelSelection, buildAntigravityPrompt, @@ -324,6 +322,41 @@ it.layer(NodeServices.layer)("buildAntigravityPrompt", (it) => { }), ); + it.effect.each([ + { name: "archive.zip", mimeType: "application/zip" }, + { name: "clip.mp4", mimeType: "video/mp4" }, + { name: "recording.aiff", mimeType: "audio/aiff" }, + { name: "large.pdf", mimeType: "application/pdf", sizeBytes: 75_000_000 }, + { name: "large.txt", mimeType: "text/plain", sizeBytes: 1024 * 1024 + 1 }, + { name: "large.wav", mimeType: "audio/wav", sizeBytes: 20 * 1024 * 1024 + 1 }, + ])("keeps $name as a file path without reading or spending the native media budget", (file) => + Effect.gen(function* () { + const fixture = yield* makeAttachmentFixture(); + const attachment = { ...textAttachment, sizeBytes: 50 * 1024 * 1024, ...file }; + const upload = yield* fixture.write(attachment, ""); + yield* fixture.fs.truncate(upload.filePath, attachment.sizeBytes); + const pdf = yield* fixture.write(pdfAttachment, ""); + yield* fixture.fs.truncate(pdf.filePath, 50 * 1024 * 1024); + const input = `Inspect the file at ${upload.filePath}`; + const prompt = yield* buildAntigravityPrompt({ + input, + attachments: [attachment, pdfAttachment], + attachmentsDir: fixture.attachmentsDir, + }).pipe( + Effect.provideService(FileSystem.FileSystem, { + ...fixture.fs, + stream: () => { + throw new Error("Path attachments must not be read into the prompt"); + }, + }), + ); + expect(prompt).toEqual([ + { type: "text", text: input }, + { type: "resource_link", uri: pdf.uri, name: "report.pdf", mimeType: "application/pdf" }, + ]); + }), + ); + it.effect("sends supported audio files as native audio content", () => Effect.gen(function* () { const fixture = yield* makeAttachmentFixture(); @@ -378,8 +411,6 @@ it.layer(NodeServices.layer)("buildAntigravityPrompt", (it) => { it.effect.each([ { ...imageAttachment, name: "animation.gif", mimeType: "image/gif" }, - { ...textAttachment, name: "archive.zip", mimeType: "application/zip" }, - { ...textAttachment, name: "recording.aiff", mimeType: "audio/aiff" }, ] satisfies ReadonlyArray)( "rejects $name instead of silently dropping it from a valid prompt", (attachment) => @@ -400,32 +431,25 @@ it.layer(NodeServices.layer)("buildAntigravityPrompt", (it) => { }), ); - it.effect.each([ - { attachment: textAttachment, bytes: ANTIGRAVITY_MAX_TEXT_ATTACHMENT_BYTES + 1 }, - { attachment: imageAttachment, bytes: PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + 1 }, - { attachment: pdfAttachment, bytes: PROVIDER_SEND_TURN_MAX_FILE_BYTES + 1 }, - ])( - "rejects oversized $attachment.name using file size instead of upload metadata", - ({ attachment, bytes }) => - Effect.gen(function* () { - const fixture = yield* makeAttachmentFixture(); - const upload = yield* fixture.write(attachment, ""); - yield* fixture.fs.truncate(upload.filePath, bytes); - const error = yield* buildAntigravityPrompt({ - input: "Read this attachment.", - attachments: [attachment], - attachmentsDir: fixture.attachmentsDir, - }).pipe(Effect.flip); - - expect(error).toMatchObject({ - _tag: "AcpRequestError", - code: -32602, - errorMessage: expect.stringContaining(`'${attachment.name}' is too large`), - }); - }), + it.effect("rejects oversized images using file size instead of upload metadata", () => + Effect.gen(function* () { + const fixture = yield* makeAttachmentFixture(); + const upload = yield* fixture.write(imageAttachment, ""); + yield* fixture.fs.truncate(upload.filePath, PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + 1); + const error = yield* buildAntigravityPrompt({ + input: "Read this attachment.", + attachments: [imageAttachment], + attachmentsDir: fixture.attachmentsDir, + }).pipe(Effect.flip); + expect(error).toMatchObject({ + _tag: "AcpRequestError", + code: -32602, + errorMessage: expect.stringContaining("'screen.png' is too large"), + }); + }), ); - it.effect("accepts 50 MiB in total but rejects one byte more across files", () => + it.effect("keeps PDF overflow on the file-path route", () => Effect.gen(function* () { const fixture = yield* makeAttachmentFixture(); const secondAttachment = { @@ -435,25 +459,57 @@ it.layer(NodeServices.layer)("buildAntigravityPrompt", (it) => { }; const first = yield* fixture.write(pdfAttachment, ""); const second = yield* fixture.write(secondAttachment, ""); - yield* fixture.fs.truncate(first.filePath, PROVIDER_SEND_TURN_MAX_FILE_BYTES / 2); - yield* fixture.fs.truncate(second.filePath, PROVIDER_SEND_TURN_MAX_FILE_BYTES / 2); + yield* fixture.fs.truncate(first.filePath, (50 * 1024 * 1024) / 2); + yield* fixture.fs.truncate(second.filePath, (50 * 1024 * 1024) / 2); const input = { - input: undefined, + input: `Read ${first.filePath} and ${second.filePath}`, attachments: [pdfAttachment, secondAttachment], attachmentsDir: fixture.attachmentsDir, }; const prompt = yield* buildAntigravityPrompt(input); expect(prompt).toEqual([ - { type: "resource_link", uri: first.uri, name: "report.pdf", mimeType: "application/pdf" }, - { type: "resource_link", uri: second.uri, name: "second.pdf", mimeType: "application/pdf" }, + { type: "text", text: input.input }, + { + type: "resource_link", + uri: first.uri, + name: "report.pdf", + mimeType: "application/pdf", + }, + { + type: "resource_link", + uri: second.uri, + name: "second.pdf", + mimeType: "application/pdf", + }, ]); - yield* fixture.fs.truncate(second.filePath, PROVIDER_SEND_TURN_MAX_FILE_BYTES / 2 + 1); - const error = yield* buildAntigravityPrompt(input).pipe(Effect.flip); + yield* fixture.fs.truncate(second.filePath, (50 * 1024 * 1024) / 2 + 1); + expect(yield* buildAntigravityPrompt(input)).toEqual([ + { type: "text", text: input.input }, + { + type: "resource_link", + uri: first.uri, + name: "report.pdf", + mimeType: "application/pdf", + }, + ]); + }), + ); + + it.effect("still rejects images when the native budget is full", () => + Effect.gen(function* () { + const fixture = yield* makeAttachmentFixture(); + const pdf = yield* fixture.write(pdfAttachment, ""); + yield* fixture.fs.truncate(pdf.filePath, 50 * 1024 * 1024); + yield* fixture.write(imageAttachment, new Uint8Array([1])); + const error = yield* buildAntigravityPrompt({ + input: "Inspect both attachments.", + attachments: [pdfAttachment, imageAttachment], + attachmentsDir: fixture.attachmentsDir, + }).pipe(Effect.flip); expect(error).toMatchObject({ _tag: "AcpRequestError", - code: -32602, - errorMessage: expect.stringContaining("'second.pdf' is too large"), + errorMessage: expect.stringContaining("'screen.png' is too large"), }); }), ); @@ -463,7 +519,7 @@ it.layer(NodeServices.layer)("buildAntigravityPrompt", (it) => { const fixture = yield* makeAttachmentFixture(); const pdf = yield* fixture.write(pdfAttachment, ""); const text = yield* fixture.write(textAttachment, "a"); - yield* fixture.fs.truncate(pdf.filePath, PROVIDER_SEND_TURN_MAX_FILE_BYTES - 1); + yield* fixture.fs.truncate(pdf.filePath, 50 * 1024 * 1024 - 1); const error = yield* buildAntigravityPrompt({ input: undefined, attachments: [pdfAttachment, textAttachment], diff --git a/apps/server/src/provider/acp/AntigravityAcpSupport.ts b/apps/server/src/provider/acp/AntigravityAcpSupport.ts index f2f370068181..8f2877330bbe 100644 --- a/apps/server/src/provider/acp/AntigravityAcpSupport.ts +++ b/apps/server/src/provider/acp/AntigravityAcpSupport.ts @@ -243,10 +243,14 @@ const TEXT_FILE_EXTENSIONS = new Set([ ".ini", ".conf", ]); -export const ANTIGRAVITY_MAX_TEXT_ATTACHMENT_BYTES = 1024 * 1024; +const ANTIGRAVITY_MAX_TEXT_ATTACHMENT_BYTES = 1024 * 1024; const MAX_TOTAL_ATTACHMENT_BYTES = PROVIDER_SEND_TURN_MAX_FILE_BYTES; -/** Sends uploads as native ACP content instead of workspace path hints. */ +/** + * Sends supported uploads as native ACP content. Other files, and native + * candidates over their limits, reach the agent through the saved path + * ProviderService puts in the text block. + */ export const buildAntigravityPrompt = Effect.fn("buildAntigravityPrompt")(function* (input: { readonly input: ProviderSendTurnInput["input"]; readonly attachments: ProviderSendTurnInput["attachments"]; @@ -280,7 +284,9 @@ export const buildAntigravityPrompt = Effect.fn("buildAntigravityPrompt")(functi (mimeType.startsWith("text/") || TEXT_MIME_TYPES.has(mimeType) || TEXT_FILE_EXTENSIONS.has(path.extname(attachment.name).toLowerCase())); - if (!image && !audio && !pdf && !textFile) { + const isPathOnly = + attachment.type === "file" && (isPastedText || (!audio && !pdf && !textFile)); + if (attachment.type === "image" && !image) { return yield* EffectAcpErrors.AcpRequestError.invalidParams( `Antigravity does not support '${attachment.name}' (${attachment.mimeType}). Attach a BMP, JPEG, PNG, WebP, PDF, audio, or text file.`, ); @@ -303,26 +309,30 @@ export const buildAntigravityPrompt = Effect.fn("buildAntigravityPrompt")(functi ), ), ); - if (isPastedText) { - if (info.type !== "File") { - return yield* EffectAcpErrors.AcpRequestError.invalidParams( - `Could not read attachment '${attachment.name}'.`, - ); - } - continue; + if (info.type !== "File") { + return yield* EffectAcpErrors.AcpRequestError.invalidParams( + `Could not read attachment '${attachment.name}'.`, + ); } + if (isPathOnly) continue; const size = Number(info.size); const limit = image ? PROVIDER_SEND_TURN_MAX_IMAGE_BYTES : audio ? ANTIGRAVITY_MAX_AUDIO_ATTACHMENT_BYTES : pdf - ? PROVIDER_SEND_TURN_MAX_FILE_BYTES + ? MAX_TOTAL_ATTACHMENT_BYTES : ANTIGRAVITY_MAX_TEXT_ATTACHMENT_BYTES; + if ( + attachment.type === "file" && + (size > limit || totalBytes + size > MAX_TOTAL_ATTACHMENT_BYTES) + ) { + continue; + } totalBytes += size; - if (info.type !== "File" || size > limit || totalBytes > MAX_TOTAL_ATTACHMENT_BYTES) { + if (size > limit || totalBytes > MAX_TOTAL_ATTACHMENT_BYTES) { return yield* EffectAcpErrors.AcpRequestError.invalidParams( - `Attachment '${attachment.name}' is too large. Antigravity accepts text files up to 1 MiB, images up to 10 MiB, audio up to 20 MiB, and 50 MiB total attachments.`, + `Image '${attachment.name}' is too large. Antigravity accepts images up to 10 MiB and 50 MiB of native attachments per message.`, ); } const uri = yield* path.toFileUrl(attachmentPath).pipe( diff --git a/apps/server/src/provider/cursorCredentialStore.test.ts b/apps/server/src/provider/cursorCredentialStore.test.ts index 051cb5425c69..f212ce110ea6 100644 --- a/apps/server/src/provider/cursorCredentialStore.test.ts +++ b/apps/server/src/provider/cursorCredentialStore.test.ts @@ -1,6 +1,9 @@ import { assert, describe, it } from "@effect/vitest"; -import { makeCachedCursorAccessTokenReader } from "./cursorCredentialStore.ts"; +import { + CursorKeychainTimeoutError, + makeCachedCursorAccessTokenReader, +} from "./cursorCredentialStore.ts"; describe("Cursor Keychain reader", () => { it("shares concurrent reads and rechecks after the cache expires", async () => { @@ -19,4 +22,25 @@ describe("Cursor Keychain reader", () => { time = 5 * 60_000; assert.strictEqual(await read(), "token-2"); }); + + it("gives up on an unanswered prompt and reuses it on the next read", async () => { + let reads = 0; + let allow: (token: string) => void = () => {}; + const read = makeCachedCursorAccessTokenReader( + () => { + reads++; + return new Promise((resolve) => { + allow = resolve; + }); + }, + () => 0, + 1, + ); + const error = await read().catch((cause: unknown) => cause); + assert.instanceOf(error, CursorKeychainTimeoutError); + const retry = read(); + allow("token"); + assert.strictEqual(await retry, "token"); + assert.strictEqual(reads, 1); + }); }); diff --git a/apps/server/src/provider/cursorCredentialStore.ts b/apps/server/src/provider/cursorCredentialStore.ts index 9d7d1c3bccff..2857683e4821 100644 --- a/apps/server/src/provider/cursorCredentialStore.ts +++ b/apps/server/src/provider/cursorCredentialStore.ts @@ -4,17 +4,31 @@ const CACHE_MS = 5 * 60_000; const requireForKeyring = NodeModule.createRequire(import.meta.url); -/** Share one Keychain request across usage history and limits in this server process. */ +/** Rejected when nobody answers the macOS Keychain prompt in time. */ +export class CursorKeychainTimeoutError extends Error { + constructor() { + super("Timed out waiting for Keychain access."); + } +} + +/** + * Share one Keychain request across usage history and limits in this server process. + * + * macOS shows the access prompt on the server's own screen, which a remote + * client cannot answer, so callers give up after `timeoutMs`. The read stays in + * flight: the next call reuses it instead of stacking a second prompt, and picks + * up the token once someone allows access. + */ export function makeCachedCursorAccessTokenReader( read: () => Promise, now: () => number = Date.now, + timeoutMs = 30_000, ): () => Promise { let cached: { token: string; until: number } | null = null; let pending: Promise | null = null; return () => { if (cached && cached.until > now()) return Promise.resolve(cached.token); - if (pending) return pending; - pending = read() + pending ??= read() .then((token) => { cached = token ? { token, until: now() + CACHE_MS } : null; return token; @@ -22,7 +36,15 @@ export function makeCachedCursorAccessTokenReader( .finally(() => { pending = null; }); - return pending; + const deadline = AbortSignal.timeout(timeoutMs); + return Promise.race([ + pending, + new Promise((_, reject) => { + deadline.addEventListener("abort", () => reject(new CursorKeychainTimeoutError()), { + once: true, + }); + }), + ]); }; } diff --git a/apps/server/src/provider/opencodeRuntime.ts b/apps/server/src/provider/opencodeRuntime.ts index e87f758e0e3d..d8319bc44a71 100644 --- a/apps/server/src/provider/opencodeRuntime.ts +++ b/apps/server/src/provider/opencodeRuntime.ts @@ -78,7 +78,6 @@ export function resolveOpenCodeServerPassword( : input.environment.OPENCODE_SERVER_PASSWORD; } -const OPENCODE_SERVER_READY_PREFIX = "opencode server listening"; const DEFAULT_OPENCODE_SERVER_TIMEOUT_MS = 30_000; const DEFAULT_HOSTNAME = "127.0.0.1"; const OPENCODE_SERVER_STARTUP_MAX_OUTPUT_CHARS = 64 * 1024; @@ -289,11 +288,8 @@ export interface OpenCodeRuntimeShape { function parseServerUrlFromOutput(output: string): string | null { for (const line of output.split("\n")) { - if (!line.startsWith(OPENCODE_SERVER_READY_PREFIX)) { - continue; - } - const match = line.match(/on\s+(https?:\/\/[^\s]+)/); - return match?.[1] ?? null; + const match = line.match(/server listening on\s+(https?:\/\/[^\s]+)/i); + if (match?.[1]) return match[1]; } return null; } diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index db4f1f53b710..71ec0719325d 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -28,6 +28,7 @@ import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; import * as Tracer from "effect/Tracer"; +import * as TestClock from "effect/testing/TestClock"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; @@ -975,3 +976,114 @@ describe.sequential("signRelayAgentActivityPublishProof", () => { ), ); }); + +describe.sequential("startup catch-up", () => { + // An unlinked relay with publishing off. `link` writes the link secrets and + // `enablePublishing` the opt-in. Counts link checks (relay URL reads) and + // catch-up publishes (shell snapshot reads). + function makeUnlinkedRelay() { + const secrets = makeMemorySecretStore(); + const counts = { linkChecks: 0, catchUpPublishes: 0 }; + const countingStore = { + ...secrets.store, + get: (name: string) => + Effect.suspend(() => { + if (name === RELAY_URL_SECRET) counts.linkChecks += 1; + return secrets.store.get(name); + }), + } satisfies ServerSecretStore.ServerSecretStore["Service"]; + + const layer = AgentAwarenessRelay.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.succeed(ServerSecretStore.ServerSecretStore, countingStore), + Layer.succeed(ServerEnvironment.ServerEnvironment, { + getEnvironmentId: Effect.succeed("env-1" as EnvironmentId), + getDescriptor: Effect.die("unused descriptor"), + }), + Layer.succeed(OrchestrationEngineService, { + streamDomainEvents: Stream.never, + } as unknown as OrchestrationEngineShape), + Layer.succeed(ProjectionSnapshotQuery, { + getShellSnapshot: () => + Effect.sync(() => { + counts.catchUpPublishes += 1; + return { + snapshotSequence: 1, + projects: [], + threads: [], + updatedAt: "2026-05-25T00:00:00.000Z", + } satisfies OrchestrationShellSnapshot; + }), + } as unknown as ProjectionSnapshotQueryShape), + ), + ), + Layer.provideMerge(NodeServices.layer), + ); + const link = Effect.all( + [ + secrets.setString(RELAY_URL_SECRET, "https://relay.example.test"), + secrets.setString(RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "relay-credential"), + ], + { discard: true }, + ); + const enablePublishing = secrets.setString(PUBLISH_AGENT_ACTIVITY_SECRET, "true"); + return { counts, layer, link, enablePublishing }; + } + + it.effect("checks an unlinked environment once a minute and still catches up once linked", () => { + const { counts, layer, link, enablePublishing } = makeUnlinkedRelay(); + return Effect.gen(function* () { + const relay = yield* AgentAwarenessRelay.AgentAwarenessRelay; + yield* enablePublishing; + yield* relay.start(); + + // Get past the backoff ramp, then count checks in a steady window. + yield* TestClock.adjust("10 minutes"); + const checksBeforeWindow = counts.linkChecks; + yield* TestClock.adjust("10 minutes"); + expect(counts.linkChecks - checksBeforeWindow).toBe(10); + expect(counts.catchUpPublishes).toBe(0); + + yield* link; + yield* TestClock.adjust("1 minute"); + expect(counts.catchUpPublishes).toBe(1); + }).pipe(Effect.provide(layer), Effect.scoped); + }); + + it.effect("publishes at once when this process links while the check is backed off", () => { + const { counts, layer, link, enablePublishing } = makeUnlinkedRelay(); + return Effect.gen(function* () { + const relay = yield* AgentAwarenessRelay.AgentAwarenessRelay; + yield* enablePublishing; + yield* relay.start(); + + // Backed off to 60 s: the next check is still seconds away. + yield* TestClock.adjust("10 minutes"); + yield* link; + yield* TestClock.adjust("1 second"); + expect(counts.catchUpPublishes).toBe(0); + + yield* relay.requestCatchUp(); + yield* TestClock.adjust("1 second"); + expect(counts.catchUpPublishes).toBe(1); + }).pipe(Effect.provide(layer), Effect.scoped); + }); + + it.effect("catches up within 5 s when another process enables publishing on a link", () => { + const { counts, layer, link, enablePublishing } = makeUnlinkedRelay(); + return Effect.gen(function* () { + const relay = yield* AgentAwarenessRelay.AgentAwarenessRelay; + yield* link; + yield* relay.start(); + + yield* TestClock.adjust("10 minutes"); + expect(counts.catchUpPublishes).toBe(0); + + // `t3 connect publish` writes the opt-in without waking this process. + yield* enablePublishing; + yield* TestClock.adjust("5 seconds"); + expect(counts.catchUpPublishes).toBe(1); + }).pipe(Effect.provide(layer), Effect.scoped); + }); +}); diff --git a/apps/server/src/relay/AgentAwarenessRelay.ts b/apps/server/src/relay/AgentAwarenessRelay.ts index 29d2abecebb7..c9acf6b510fc 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.ts @@ -25,6 +25,7 @@ import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import type * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; @@ -51,6 +52,8 @@ export class AgentAwarenessRelay extends Context.Service< AgentAwarenessRelay, { readonly publishThread: (threadId: ThreadId) => Effect.Effect; + /** Retries a pending catch-up publish now. Call after this process links or enables publishing. */ + readonly requestCatchUp: () => Effect.Effect; readonly start: () => Effect.Effect; } >()("t3/relay/AgentAwarenessRelay") {} @@ -309,6 +312,8 @@ export const make = Effect.gen(function* () { const cloudLinkKeyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(secrets); const startedAt = (yield* DateTime.now).epochMilliseconds; const activeSnapshotPublishedRef = yield* Ref.make(false); + // Holds at most one pending wake, so a burst of requests costs one retry. + const catchUpRequests = yield* Queue.dropping(1); const publishedStateByThreadRef = yield* Ref.make(new Map()); const readSecretString = (name: string) => @@ -534,18 +539,29 @@ export const make = Effect.gen(function* () { withRelayClientTracing, ); + // Publishes the active threads once. Returns why it did not, so the retry + // knows whether it is waiting on a link or on the publish setting. const publishActiveThreadsUnsafe = Effect.gen(function* () { + // One secret read settles the common never-linked case; the full link + // config is read only once publishing is on. + const relayUrl = yield* readSecretString(RELAY_URL_SECRET).pipe( + Effect.orElseSucceed(() => null), + ); + if (!relayUrl) { + yield* Effect.logDebug("agent activity snapshot skipped; relay link credentials unavailable"); + return "unlinked" as const; + } const publishAgentActivity = yield* readPublishAgentActivityEnabled.pipe( Effect.orElseSucceed(() => false), ); if (!publishAgentActivity) { yield* Effect.logDebug("agent activity snapshot skipped; publication disabled"); - return false; + return "disabled" as const; } const relayConfig = yield* readRelayConfig.pipe(Effect.orElseSucceed(() => null)); if (!relayConfig) { yield* Effect.logDebug("agent activity snapshot skipped; relay link credentials unavailable"); - return false; + return "unlinked" as const; } const environmentId = yield* serverEnvironment.getEnvironmentId; const snapshot = yield* snapshotQuery.getShellSnapshot(); @@ -557,20 +573,29 @@ export const make = Effect.gen(function* () { }); if (activeThreadIds.length === 0) { yield* Effect.logDebug("agent activity snapshot has no publishable threads"); - return true; + return "published" as const; } yield* Effect.logInfo("publishing active agent activity snapshot", { count: activeThreadIds.length, }); yield* Effect.forEach(activeThreadIds, publishThread, { concurrency: 4, discard: true }); - return true; + return "published" as const; }); + // Publishes the catch-up snapshot of active threads once the environment is + // linked and publishing is enabled. Many environments never link, so while + // unlinked the retry backs off from 5 s to 60 s. Only this process writes + // the link, and it calls `requestCatchUp`, which ends the wait early. A + // linked environment keeps the 5 s retry, because `t3 connect publish` can + // turn publishing on from another process. const publishActiveThreadsOnceWhenConfigured = (logEnabledWhenReady: boolean) => Effect.gen(function* () { + let unlinkedRetryDelayMs = 5_000; while (!(yield* Ref.get(activeSnapshotPublishedRef))) { - const published = yield* publishActiveThreadsUnsafe.pipe(Effect.orElseSucceed(() => false)); - if (published) { + const result = yield* publishActiveThreadsUnsafe.pipe( + Effect.orElseSucceed(() => "failed" as const), + ); + if (result === "published") { yield* Ref.set(activeSnapshotPublishedRef, true); if (logEnabledWhenReady) { const relayConfig = yield* readRelayConfig.pipe(Effect.orElseSucceed(() => null)); @@ -580,7 +605,11 @@ export const make = Effect.gen(function* () { } return; } - yield* Effect.sleep("5 seconds"); + const retryDelayMs = result === "unlinked" ? unlinkedRetryDelayMs : 5_000; + yield* Effect.race(Effect.sleep(retryDelayMs), Queue.take(catchUpRequests)); + if (result === "unlinked") { + unlinkedRetryDelayMs = Math.min(unlinkedRetryDelayMs * 2, 60_000); + } } }); @@ -657,6 +686,7 @@ export const make = Effect.gen(function* () { return AgentAwarenessRelay.of({ publishThread, + requestCatchUp: () => Queue.offer(catchUpRequests, undefined).pipe(Effect.asVoid), start, }); }); diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 071374f3addb..50c24cbd64ae 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -176,10 +176,12 @@ import * as VcsProcess from "./vcs/VcsProcess.ts"; import * as GitWorkflowService from "./git/GitWorkflowService.ts"; import * as ReviewService from "./review/ReviewService.ts"; import * as SourceControlRepositoryService from "./sourceControl/SourceControlRepositoryService.ts"; +import { REPLAY_MARKER_MAX_AGE } from "./auth/replayMarkers.ts"; import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; import * as PairingGrantStore from "./auth/PairingGrantStore.ts"; import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts"; +import * as AgentAwarenessRelay from "./relay/AgentAwarenessRelay.ts"; import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts"; import * as ProcessDiagnostics from "./diagnostics/ProcessDiagnostics.ts"; import * as HostResources from "./resourceTelemetry/HostResources.ts"; @@ -563,6 +565,7 @@ const buildAppUnderTest = (options?: { CloudManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"] >; relayClient?: Partial; + agentAwarenessRelay?: Partial; cloudCliTokenManager?: Partial; httpClient?: HttpClient.HttpClient; nativeTelemetryClient?: Partial; @@ -1184,14 +1187,20 @@ const buildAppUnderTest = (options?: { }), ), Layer.provide( - Layer.succeed( - CloudManagedEndpointRuntime.CloudManagedEndpointRuntime, - CloudManagedEndpointRuntime.CloudManagedEndpointRuntime.of({ - applyConfig: () => Effect.succeed({ status: "disabled" }), - recoveryRequests: Stream.empty, - requestRecovery: () => Effect.void, - withLinkStateLock: (effect) => effect, - ...options?.layers?.cloudManagedEndpointRuntime, + Layer.mergeAll( + Layer.succeed( + CloudManagedEndpointRuntime.CloudManagedEndpointRuntime, + CloudManagedEndpointRuntime.CloudManagedEndpointRuntime.of({ + applyConfig: () => Effect.succeed({ status: "disabled" }), + recoveryRequests: Stream.empty, + requestRecovery: () => Effect.void, + withLinkStateLock: (effect) => effect, + ...options?.layers?.cloudManagedEndpointRuntime, + }), + ), + Layer.mock(AgentAwarenessRelay.AgentAwarenessRelay)({ + requestCatchUp: () => Effect.void, + ...options?.layers?.agentAwarenessRelay, }), ), ), @@ -2655,6 +2664,40 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + it.effect("rejects a DPoP replay by time alone once its marker can be pruned", () => + Effect.gen(function* () { + yield* buildAppUnderTest(); + + const ownerCookie = yield* getAuthenticatedSessionCookieHeader(); + const credentialResponse = yield* HttpClient.post("/api/auth/pairing-token", { + headers: { cookie: ownerCookie }, + body: yield* HttpBody.json({}), + }); + const credential = (yield* credentialResponse.json) as { readonly credential: string }; + const tokenUrl = yield* getHttpServerUrl("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/oauth/token"); + const acceptedAt = yield* DateTime.now; + // The longest-lived proof: `iat` at the 5 s future skew the verifier allows. + const dpop = makeDpopProof({ + method: "POST", + url: tokenUrl, + iat: Math.floor(acceptedAt.epochMilliseconds / 1_000) + 5, + }); + const exchange = exchangeAccessToken(credential.credential, { + headers: { dpop: dpop.proof }, + scope: "orchestration:read orchestration:operate terminal:operate review:write", + }); + + assert.equal((yield* exchange).response.status, 200); + // While the proof is fresh, only the replay marker rejects it. + assert.equal((yield* exchange).body.dpopFailureReason, "replay"); + // Once the marker can be pruned, the time check rejects the proof by itself. + yield* TestClock.setTime( + acceptedAt.epochMilliseconds + Duration.toMillis(REPLAY_MARKER_MAX_AGE), + ); + assert.equal((yield* exchange).body.dpopFailureReason, "time_window"); + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ); + it.effect("ignores forwarded host headers when validating token exchange DPoP URLs", () => Effect.gen(function* () { yield* buildAppUnderTest(); @@ -3091,6 +3134,55 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + it.effect("wakes the agent awareness relay when this server links or changes publishing", () => + Effect.gen(function* () { + let catchUpRequests = 0; + yield* buildAppUnderTest({ + layers: { + agentAwarenessRelay: { + requestCatchUp: () => + Effect.sync(() => { + catchUpRequests += 1; + }), + }, + }, + }); + + const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, + }); + const ownerCookie = yield* getAuthenticatedSessionCookieHeader(); + const relayConfigResponse = yield* fetchEffect( + yield* getHttpServerUrl("/api/connect/relay-config"), + { + method: "POST", + headers: { cookie: ownerCookie, "content-type": "application/json" }, + body: jsonRequestBody({ + relayUrl: "https://relay.example.test", + cloudUserId: "user_123", + environmentCredential: "t3env_test_credential", + cloudMintPublicKey: cloudKeyPair.publicKey, + endpointRuntime: null, + }), + }, + ); + assert.equal(relayConfigResponse.status, 200); + assert.equal(catchUpRequests, 1); + + const preferencesResponse = yield* fetchEffect( + yield* getHttpServerUrl("/api/connect/preferences"), + { + method: "POST", + headers: { cookie: ownerCookie, "content-type": "application/json" }, + body: jsonRequestBody({ publishAgentActivity: true }), + }, + ); + assert.equal(preferencesResponse.status, 200); + assert.equal(catchUpRequests, 2); + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ); + it.effect("rejects relay config with an invalid cloud mint public key", () => Effect.gen(function* () { yield* buildAppUnderTest(); @@ -3722,6 +3814,82 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + it.effect("rejects cloud replays by time alone once their markers can be pruned", () => + Effect.gen(function* () { + yield* buildAppUnderTest(); + + const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, + }); + const ownerCookie = yield* getAuthenticatedSessionCookieHeader(); + const relayConfigResponse = yield* fetchEffect( + yield* getHttpServerUrl("/api/connect/relay-config"), + { + method: "POST", + headers: { cookie: ownerCookie, "content-type": "application/json" }, + body: jsonRequestBody({ + relayUrl: "https://relay.example.test", + cloudUserId: "user_123", + environmentCredential: "t3env_test_credential", + cloudMintPublicKey: cloudKeyPair.publicKey, + endpointRuntime: null, + }), + }, + ); + assert.equal(relayConfigResponse.status, 200); + + const acceptedAt = yield* DateTime.now; + // The longest-lived proofs: `iat` at the 60 s future skew the handlers + // allow, and the 5 minute maximum lifetime. + const issuedAt = DateTime.add(acceptedAt, { minutes: 1 }); + const proofTimes = { + issuedAt: DateTime.formatIso(issuedAt), + expiresAt: DateTime.formatIso(DateTime.add(issuedAt, { minutes: 5 })), + }; + const requests = [ + [ + "/api/t3-connect/health", + makeCloudEnvironmentHealthRequest({ + privateKey: cloudKeyPair.privateKey, + environmentId: testEnvironmentDescriptor.environmentId, + nonce: "cloud-health-nonce-pruned", + ...proofTimes, + }), + ], + [ + "/api/t3-connect/mint-credential", + makeCloudMintCredentialRequest({ + privateKey: cloudKeyPair.privateKey, + environmentId: testEnvironmentDescriptor.environmentId, + clientProofKeyThumbprint: "client-proof-key-thumbprint", + nonce: "cloud-mint-nonce-pruned", + ...proofTimes, + }), + ], + ] as const; + const postAll = Effect.forEach(requests, ([pathname, request]) => + Effect.gen(function* () { + const response = yield* fetchEffect(yield* getHttpServerUrl(pathname), { + method: "POST", + headers: { "content-type": "application/json" }, + body: jsonRequestBody(request), + }); + return response.status; + }), + ); + + assert.deepStrictEqual(yield* postAll, [200, 200]); + // While the proofs are fresh, only the replay markers reject them (409). + assert.deepStrictEqual(yield* postAll, [409, 409]); + // Once the markers can be pruned, the time checks reject the proofs by themselves (401). + yield* TestClock.setTime( + acceptedAt.epochMilliseconds + Duration.toMillis(REPLAY_MARKER_MAX_AGE), + ); + assert.deepStrictEqual(yield* postAll, [401, 401]); + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ); + it.effect( "validates cloud proofs against the configured relay issuer, not the transport URL", () => @@ -5673,6 +5841,56 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + it.effect("does not trace browser OTLP trace exports on the server", () => + Effect.gen(function* () { + const spanNames: Array = []; + const forwardedUrls: Array = []; + yield* buildAppUnderTest({ + config: { otlpTracesUrl: "http://collector.test/v1/traces" }, + layers: { + httpClient: HttpClient.make((request) => + Effect.sync(() => { + forwardedUrls.push(request.url); + return HttpClientResponse.fromWeb(request, new Response(null, { status: 204 })); + }), + ), + }, + }).pipe( + Effect.provideService( + Tracer.Tracer, + Tracer.make({ + span: (options) => { + spanNames.push(options.name); + return new Tracer.NativeSpan(options); + }, + }), + ), + ); + const cookie = yield* getAuthenticatedSessionCookieHeader(); + spanNames.length = 0; + + // The query string must not bring back the HTTP server span. + for (const url of ["/api/observability/v1/traces", "/api/observability/v1/traces?x=1"]) { + const response = yield* HttpClient.post(url, { + headers: { cookie, "content-type": "application/json" }, + body: yield* HttpBody.json({ resourceSpans: [] }), + }); + assert.equal(response.status, 204); + } + + assert.deepEqual(forwardedUrls, [ + "http://collector.test/v1/traces", + "http://collector.test/v1/traces", + ]); + assert.deepEqual(spanNames, []); + + // Other routes keep their HTTP server span. + const session = yield* HttpClient.get("/api/auth/session", { headers: { cookie } }); + assert.equal(session.status, 200); + assert.include(spanNames, "http.server GET"); + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ); + it.effect("routes websocket rpc server.upsertKeybinding", () => Effect.gen(function* () { const rule: KeybindingRule = { diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 0f619d33229e..bb165acdada3 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -35,6 +35,7 @@ import { staticAndDevRouteLayer, browserApiCorsLayer, httpCompressionLayer, + untracedRequestsLayer, } from "./http.ts"; import { guardHttpResponseWriteErrors } from "./httpResponseErrorGuard.ts"; import { fixPath } from "./os-jank.ts"; @@ -123,9 +124,12 @@ import * as SourceControlRepositoryService from "./sourceControl/SourceControlRe import * as ProjectSetupScriptRunner from "./project/ProjectSetupScriptRunner.ts"; import * as WorktreeSetupTracker from "./project/WorktreeSetupTracker.ts"; import { ObservabilityLive } from "./observability/Layers/Observability.ts"; +import * as HeapSnapshot from "./observability/HeapSnapshot.ts"; +import * as EventLoopMonitor from "./observability/EventLoopMonitor.ts"; import * as ServerEnvironment from "./environment/ServerEnvironment.ts"; import * as RemoteOpenTargets from "./environment/RemoteOpenTargets.ts"; import { authHttpApiLayer, environmentAuthenticatedAuthLayer } from "./auth/http.ts"; +import * as ReplayMarkers from "./auth/replayMarkers.ts"; import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; import { @@ -186,7 +190,8 @@ export const HTTP_ROUTER_CONFIG = { // those finalizers get a chance to run. const HTTP_PREEMPTIVE_SHUTDOWN_GRACE_MS = 0; const ResourceAttributionLayerLive = ResourceAttribution.layer; -const ApplicationObservabilityLive = ObservabilityLive.pipe( +const ApplicationObservabilityLive = EventLoopMonitor.layer.pipe( + Layer.provideMerge(ObservabilityLive), Layer.provideMerge(ResourceAttributionLayerLive), ); @@ -511,6 +516,7 @@ const AntigravityInstallationRefreshLive = Layer.effectDiscard( const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe( Layer.provideMerge(AntigravityInstallationRefreshLive), + Layer.provideMerge(ReplayMarkers.layer), Layer.provideMerge(ProviderAuthServiceLive), // Core Services Layer.provideMerge(ServerSettingsLayerLive), @@ -615,6 +621,8 @@ export const makeRoutesLayer = Layer.mergeAll( websocketRpcRouteLayer, ), McpHttpServer.layer.pipe(Layer.provide(McpSessionRegistry.layer)), + // Last, so no route layer can replace the server's one TracerDisabledWhen. + untracedRequestsLayer, ).pipe( // Both transports consume the same service instance, so caches single-flight across clients // and mutations observed on WebSocket invalidate patches subsequently read over HTTP. @@ -972,6 +980,7 @@ const makeServerLayer = Layer.unwrap( runtimeStateLayer.pipe(Layer.provide(launcherLayer)), tailscaleServeLayer, cloudDesiredLinkReconcileLayer, + HeapSnapshot.layer, ); return serverApplicationLayer.pipe( diff --git a/apps/server/src/usage/UsageService.test.ts b/apps/server/src/usage/UsageService.test.ts index a80a31f48ae6..15ea4b673f22 100644 --- a/apps/server/src/usage/UsageService.test.ts +++ b/apps/server/src/usage/UsageService.test.ts @@ -121,6 +121,46 @@ function totalOutputTokens(summary: { buckets: readonly { totals: { outputTokens } describe("UsageService", () => { + it.live("omits Cursor account usage when no file login is saved", () => + Effect.gen(function* () { + const { settings, home } = yield* setup; + for (const platform of ["linux", "win32", "darwin"] as const) { + const service = yield* UsageService.make.pipe( + Effect.provide( + serviceLayers({ + prefix: `usage-service-cursor-no-login-${platform}`, + home, + settings, + platform, + environment: { AGENT_CLI_CREDENTIAL_STORE: "file" }, + }), + ), + ); + const summary = yield* service.readSummary(WINDOW); + assert.isFalse(summary.sources.some((source) => source.fingerprint.provider === "cursor")); + } + }).pipe(Effect.scoped), + ); + + it.live("keeps Cursor credential errors visible when a saved login cannot be read", () => + Effect.gen(function* () { + const { settings, home } = yield* setup; + const authPath = NodePath.join(home, "config", "cursor", "auth.json"); + yield* Effect.promise(async () => { + await NodeFSP.mkdir(NodePath.dirname(authPath), { recursive: true }); + await NodeFSP.writeFile(authPath, "invalid json"); + }); + const service = yield* UsageService.make.pipe( + Effect.provide( + serviceLayers({ prefix: "usage-service-cursor-invalid-login", home, settings }), + ), + ); + const summary = yield* service.readSummary(WINDOW); + const cursor = summary.sources.find((source) => source.fingerprint.provider === "cursor"); + assert.strictEqual(cursor?.message, "Cursor credentials could not be read."); + }).pipe(Effect.scoped), + ); + it.live("does not read the macOS Cursor Keychain before account usage is enabled", () => Effect.gen(function* () { const { settings, home } = yield* setup; @@ -233,10 +273,7 @@ describe("UsageService", () => { const summary = yield* service.readSummary(WINDOW); assert.strictEqual(summary.buckets[0]?.provider, "opencode"); assert.isFalse(summary.buckets.some((bucket) => bucket.provider === "cursor")); - assert.strictEqual( - summary.sources.find((source) => source.fingerprint.provider === "cursor")?.status, - "missing", - ); + assert.isFalse(summary.sources.some((source) => source.fingerprint.provider === "cursor")); assert.strictEqual( summary.buckets[0]?.sourcePath, yield* Effect.promise(() => NodeFSP.realpath(root)), @@ -247,10 +284,6 @@ describe("UsageService", () => { ?.distinctSessions, 1, ); - assert.include( - summary.sources.find((source) => source.fingerprint.provider === "cursor")?.message ?? "", - "Cursor account history needs a Cursor CLI login", - ); }).pipe(Effect.scoped), ); diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index 6930c7307d8c..1e5cb6db20fe 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -635,6 +635,8 @@ export const make = Effect.gen(function* () { cursorUntilMs, ), ); + // No saved login means there is no account source to report, not a setup error. + if (account.missing && account.error === null) return scanned; if (account.accountKey !== null && account.error === null && !account.missing) { // The same account includes CLI and desktop history from every machine. // A stable remote fingerprint prevents connected environments counting it twice. diff --git a/apps/server/src/usage/cursorUsageReader.test.ts b/apps/server/src/usage/cursorUsageReader.test.ts new file mode 100644 index 000000000000..4bfa647c0aff --- /dev/null +++ b/apps/server/src/usage/cursorUsageReader.test.ts @@ -0,0 +1,22 @@ +import { assert, describe, it } from "@effect/vitest"; + +import { CursorKeychainTimeoutError } from "../provider/cursorCredentialStore.ts"; +import { readCursorAccountUsage } from "./cursorUsageReader.ts"; + +describe("readCursorAccountUsage", () => { + it("asks for Keychain approval when the prompt goes unanswered", async () => { + const result = await readCursorAccountUsage( + { kind: "keychain" }, + 0, + 1, + () => Promise.reject(new Error("no network expected")), + () => Promise.reject(new CursorKeychainTimeoutError()), + ); + assert.deepStrictEqual(result, { + accountKey: null, + records: [], + missing: false, + error: "Allow Keychain access on the Mac running T3 Code, then refresh.", + }); + }); +}); diff --git a/apps/server/src/usage/cursorUsageReader.ts b/apps/server/src/usage/cursorUsageReader.ts index 1e07fcc7a825..573506d818a1 100644 --- a/apps/server/src/usage/cursorUsageReader.ts +++ b/apps/server/src/usage/cursorUsageReader.ts @@ -5,7 +5,10 @@ import * as NodeCrypto from "node:crypto"; import * as NodeTimersPromises from "node:timers/promises"; import type { UsageRecord } from "./usageTranscripts.ts"; -import { readMacCursorAccessToken } from "../provider/cursorCredentialStore.ts"; +import { + CursorKeychainTimeoutError, + readMacCursorAccessToken, +} from "../provider/cursorCredentialStore.ts"; function object(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) @@ -86,7 +89,9 @@ export async function readCursorAccountUsage( ? null : typeof credentialSource === "string" ? "Cursor credentials could not be read." - : "Cursor Keychain credentials could not be read.", + : cause instanceof CursorKeychainTimeoutError + ? "Allow Keychain access on the Mac running T3 Code, then refresh." + : "Cursor Keychain credentials could not be read.", }; } if (typeof accessToken !== "string" || !accessToken) { diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index 47ca4e178700..85c4d1a59d1d 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -1918,6 +1918,35 @@ it.layer(TestLayer)("GitVcsDriver core integration", (it) => { }), ); + it.effect("does not start Git auto-maintenance from background upstream fetches", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-vcs-driver-remote-"); + const { initialBranch } = yield* initRepoWithCommit(cwd); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["push", "-u", "origin", initialBranch]); + yield* git(cwd, ["repack", "-d"]); + yield* writeTextFile(cwd, "second.txt", "second\n"); + yield* git(cwd, ["add", "second.txt"]); + yield* git(cwd, ["commit", "-m", "second commit"]); + yield* git(cwd, ["push"]); + yield* git(cwd, ["repack", "-d"]); + // Two packs make `git gc --auto` due, and without detaching it would run inside the fetch. + yield* git(cwd, ["config", "gc.autoPackLimit", "1"]); + yield* git(cwd, ["config", "gc.autoDetach", "false"]); + yield* git(cwd, ["config", "maintenance.autoDetach", "false"]); + const packCount = git(cwd, ["count-objects", "-v"]).pipe( + Effect.map((stdout) => stdout.match(/^packs: (\d+)$/m)?.[1]), + ); + assert.equal(yield* packCount, "2"); + + yield* (yield* GitVcsDriver.GitVcsDriver).statusDetailsRemote(cwd); + + assert.equal(yield* packCount, "2"); + }), + ); + it.effect("uses origin HEAD for default-branch detection with a non-origin upstream", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 8ec274a46611..0dd73af687f9 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -549,7 +549,9 @@ function trace2ChildKey(record: Record): string | null { const Trace2Record = Schema.Record(Schema.String, Schema.Unknown); const decodeTrace2Record = decodeJsonResult(Trace2Record); -const createTrace2Monitor = Effect.fn("createTrace2Monitor")(function* ( +// Untraced because it runs on every git spawn and returns at once without hook +// callbacks. Its errors fail the runGitCommand span. +const createTrace2Monitor = Effect.fnUntraced(function* ( input: Pick, progress: GitVcsDriver.ExecuteGitProgress | undefined, ): Effect.fn.Return< @@ -1125,10 +1127,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ): Effect.Effect => { const fetchCwd = path.basename(gitCommonDir) === ".git" ? path.dirname(gitCommonDir) : gitCommonDir; + // `--no-auto-gc` (a synonym of `--no-auto-maintenance` that older Git also knows) keeps + // this poll from starting `git gc --auto`. When that gc fails, for example on a repository + // with missing objects, Git retries it on every fetch and leaves a full-size `tmp_pack_*` + // behind each time, so a background poll could fill the disk. return executeGit( "GitVcsDriver.fetchRemoteForStatus", fetchCwd, - ["--git-dir", gitCommonDir, "fetch", "--quiet", "--no-tags", remoteName], + ["--git-dir", gitCommonDir, "fetch", "--quiet", "--no-tags", "--no-auto-gc", remoteName], { env: STATUS_UPSTREAM_REFRESH_ENV, fallbackErrorDetail: "Background Git fetch exited with a non-zero status.", diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 10b7ac533a55..403c761f5d9b 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -3042,9 +3042,13 @@ const makeWsRpcLayer = ( [WS_METHODS.sourceControlPublishRepository]: (input) => observeRpcEffect( WS_METHODS.sourceControlPublishRepository, - sourceControlRepositories - .publishRepository(input) - .pipe(Effect.tap(() => refreshGitStatus(input.cwd))), + sourceControlRepositories.publishRepository(input).pipe( + // A new remote can change the cached identity. Only the `cwd` entry + // refreshes, so after a publish from a linked worktree the project + // root entry waits for its TTL. + Effect.tap(() => repositoryIdentityResolver.resolve(input.cwd, { refresh: true })), + Effect.tap(() => refreshGitStatus(input.cwd)), + ), { "rpc.aggregate": "source-control", }, diff --git a/apps/web/package.json b/apps/web/package.json index 466e73226e56..622b1e3efc6d 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -77,6 +77,7 @@ "@vitejs/plugin-react": "^6.0.0", "babel-plugin-react-compiler": "1.0.0", "compression": "^1.8.1", + "jsdom": "^30.0.1", "react-test-renderer": "19.2.6", "tailwindcss": "^4.0.0", "unified": "^11.0.5", diff --git a/apps/web/src/components/AppSidebarLayout.tsx b/apps/web/src/components/AppSidebarLayout.tsx index 4716797704fe..a1e8884807a0 100644 --- a/apps/web/src/components/AppSidebarLayout.tsx +++ b/apps/web/src/components/AppSidebarLayout.tsx @@ -91,6 +91,7 @@ function readInitialThreadSidebarWidth(): number { } function SidebarControl() { + const usagePageOpen = useLocation({ select: (location) => location.pathname === "/usage" }); const keybindings = useAtomValue(primaryServerKeybindingsAtom); const { toggleSidebar } = useSidebar(); const isSidebarVisible = useSidebarVisibility(); @@ -98,7 +99,9 @@ function SidebarControl() { const stageBackdropVariant = useSidebarStageBackdropVariant( environmentIdentificationMode === "artwork", ); - const shortcutLabel = shortcutLabelForCommand(keybindings, "sidebar.toggle"); + const shortcutLabel = shortcutLabelForCommand(keybindings, "sidebar.toggle", { + context: { usagePageOpen }, + }); useEffect(() => { const onKeyDown = (event: KeyboardEvent) => { @@ -118,7 +121,11 @@ function SidebarControl() { // available everywhere else, including the plain-text composer. return; } - if (resolveShortcutCommand(event, keybindings) !== "sidebar.toggle") return; + if ( + resolveShortcutCommand(event, keybindings, { context: { usagePageOpen } }) !== + "sidebar.toggle" + ) + return; event.preventDefault(); event.stopPropagation(); @@ -128,7 +135,7 @@ function SidebarControl() { // Capture before focused editors consume commands such as Mod+B for rich-text formatting. window.addEventListener("keydown", onKeyDown, true); return () => window.removeEventListener("keydown", onKeyDown, true); - }, [keybindings, toggleSidebar]); + }, [keybindings, toggleSidebar, usagePageOpen]); return ( // The right-side layout controls carry mr-px (border compensation inside @@ -312,6 +319,8 @@ export function AppSidebarLayout({ children }: { children: ReactNode }) { side="left" collapsible="offcanvas" data-app-sidebar="" + role="navigation" + aria-label={isOnSettings ? "Settings" : "Threads"} resizable={{ maxWidth: sidebarMaximumWidth, minWidth: THREAD_SIDEBAR_MIN_WIDTH, diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index a7b9a66ebdce..84c04b895498 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -325,12 +325,13 @@ import { terminalContextReference, } from "../lib/composerContextRecords"; import { - isQueuedMessageDue, latestCompletedToolActivityId, type QueuedComposerMessage, + type QueuedMessageSendSettings, useQueuedMessages, useQueuedMessageStore, } from "../queuedMessageStore"; +import { sendQueuedMessage } from "./chat/sendQueuedMessage"; import { type ReviewCommentContext } from "../reviewCommentContext"; import { environmentCatalog } from "../connection/catalog"; import { isDesktopLocalConnectionTarget } from "../connection/desktopLocal"; @@ -918,7 +919,14 @@ const PersistentThreadTerminalDrawer = memo(function PersistentThreadTerminalDra const writeTerminal = useAtomCommand(terminalEnvironment.write, "terminal write"); const closeTerminalMutation = useAtomCommand(terminalEnvironment.close, "terminal close"); const draftThread = useComposerDraftStore((store) => store.getDraftThreadByRef(threadRef)); - const serverThread = useThread(threadRef, { waitForShell: draftThread !== null }); + // Hidden drawers stay mounted (see MAX_HIDDEN_MOUNTED_TERMINAL_THREADS), so they read only + // the shell: a detail subscription would keep each hidden thread's history in memory. The + // active drawer shares ChatView's detail, which also covers archived threads (no shell). + const activeServerThread = useThread(active ? threadRef : null, { + waitForShell: draftThread !== null, + }); + const serverThreadShell = useThreadShell(threadRef); + const serverThread = activeServerThread ?? serverThreadShell; const projectRef = serverThread ? scopeProjectRef(serverThread.environmentId, serverThread.projectId) : draftThread @@ -2698,68 +2706,40 @@ export default function ChatView(props: ChatViewProps) { unavailableConnection !== null && (unavailableConnection.phase === "connecting" || unavailableConnection.phase === "reconnecting"); - // Reconnecting to a version-skewed server with no update in flight - // usually means the server is restarting mid-update and a refresh wiped - // the in-memory update state. Fold the reconnect and version banners - // into one calm line instead of stacking "Failed to connect" on - // "versions differ". A failed update never folds: its error and retry - // action must stay visible. - const reconnectingThroughVersionSkew = - serverUpdateState.status === "idle" && environmentReconnecting && versionMismatch !== null; // While an update runs, transient connect blips are expected (the server // restarts) and the update banner already shows progress. Hard failure // phases still surface so the Reconnect action stays reachable. const suppressUnavailableBanner = - environmentReconnecting && - (updateRunning || (!reconnectingThroughVersionSkew && !reconnectWarningGraceElapsed)); + environmentReconnecting && (updateRunning || !reconnectWarningGraceElapsed); if (activeEnvironmentUnavailableState && unavailableConnection && !suppressUnavailableBanner) { - if (reconnectingThroughVersionSkew) { - items.push({ - id: `environment-unavailable:${activeEnvironmentUnavailableState.environmentId}`, - variant: "default", - // Prioritize live connection progress among the notices. - priority: "urgent", - icon: ( -
{isDraftHeroState ? (
diff --git a/apps/web/src/components/NoProjectsHero.tsx b/apps/web/src/components/NoProjectsHero.tsx index 833167d5026d..9a33bb116eff 100644 --- a/apps/web/src/components/NoProjectsHero.tsx +++ b/apps/web/src/components/NoProjectsHero.tsx @@ -3,9 +3,11 @@ import { useCallback } from "react"; import { openCommandPalette } from "../commandPaletteBus"; import { CustomBackground } from "./CustomBackground"; +import { isElectron } from "../env"; import { Button } from "./ui/button"; import { Empty, EmptyDescription, EmptyHeader, EmptyTitle } from "./ui/empty"; import { SidebarInset } from "./ui/sidebar"; +import { WorkspacePageHeader } from "./WorkspacePageHeader"; export function NoProjectsHero() { const openAddProject = useCallback(() => openCommandPalette({ open: "add-project" }), []); @@ -14,6 +16,8 @@ export function NoProjectsHero() {
+ {/* The desktop window only moves where CSS opts in, so keep a titlebar strip. */} + {isElectron ? : null}
diff --git a/apps/web/src/components/QueuedMessageSender.test.tsx b/apps/web/src/components/QueuedMessageSender.test.tsx new file mode 100644 index 000000000000..dc31694f8b96 --- /dev/null +++ b/apps/web/src/components/QueuedMessageSender.test.tsx @@ -0,0 +1,231 @@ +import { scopeThreadRef, scopedThreadKey } from "@t3tools/client-runtime/environment"; +import { EnvironmentId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import { act, createElement } from "react"; +import { create, type ReactTestRenderer } from "react-test-renderer"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +import { useQueuedMessageStore, type QueuedComposerMessage } from "../queuedMessageStore"; +import { sendQueuedMessage } from "./chat/sendQueuedMessage"; +import { QueuedMessageSender } from "./QueuedMessageSender"; + +const io = vi.hoisted(() => ({ + run: vi.fn(), + upload: vi.fn(), + toast: vi.fn(), + thread: null as unknown, + shell: { runtimeMode: "full-access", interactionMode: "default" } as Record, +})); +const config = { + environment: { capabilities: { attachmentUploads: true, inlineMessageContext: true } }, +}; +vi.mock("@t3tools/client-runtime/state/runtime", async (load) => ({ + ...(await load()), + runAtomCommand: (...args: unknown[]) => io.run(...args), +})); +vi.mock("../rpc/atomRegistry", () => ({ + appAtomRegistry: { get: () => new Map([["env-a", config]]) }, +})); +vi.mock("../state/server", () => ({ environmentServerConfigsAtom: {} })); +vi.mock("../state/threads", () => ({ + threadEnvironment: { + updateMetadata: "metadata", + setRuntimeMode: "runtime", + setInteractionMode: "interaction", + startTurn: "start", + }, +})); +vi.mock("../state/environments", () => ({ + useEnvironment: () => ({ connection: { phase: "connected" } }), +})); +vi.mock("../state/entities", () => ({ + useThread: () => io.thread, + useThreadStatus: () => "live", + useServerConfigs: () => new Map([["env-a", config]]), + readThreadShell: () => io.shell, + readThread: () => io.thread, +})); +vi.mock("./ui/toast", () => ({ toastManager: { add: (...args: unknown[]) => io.toast(...args) } })); +vi.mock("../lib/attachmentUploadQueue", () => ({ + startAttachmentUpload: vi.fn(), + awaitAttachmentUploads: (...args: unknown[]) => io.upload(...args), + getUploadedAttachments: () => [ + { type: "image", id: "uploaded", name: "a.png", mimeType: "image/png", sizeBytes: 4 }, + ], + releaseDraftAttachments: vi.fn(), +})); + +const threadRef = scopeThreadRef(EnvironmentId.make("env-a"), ThreadId.make("thread-a")); +const threadKey = scopedThreadKey(threadRef); +const modelSelection = { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5" }; + +function enqueue(overrides: Partial = {}) { + return useQueuedMessageStore.getState().enqueue(threadKey, { + prompt: "follow up", + images: [], + files: [], + terminalContexts: [], + previewAnnotations: [], + reviewComments: [], + sendSettings: { + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + promptEffort: null, + }, + queuedAfterToolActivityId: null, + createdAt: "2026-09-25T00:00:00Z", + ...overrides, + }); +} + +const commandsRun = () => io.run.mock.calls.map((call) => call[1]); +const queue = () => useQueuedMessageStore.getState().queuesByThreadKey[threadKey]; + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + useQueuedMessageStore.setState({ queuesByThreadKey: {}, lastDispatchByThreadKey: {} }); + io.thread = null; + io.run.mockReset().mockResolvedValue({ _tag: "Success", value: undefined }); + io.upload.mockReset().mockResolvedValue(undefined); + io.toast.mockReset(); + io.shell = { + modelSelection, + branch: null, + runtimeMode: "full-access", + interactionMode: "default", + }; +}); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("QueuedMessageSender", () => { + const thread = ( + status: string, + { toolActivityIds = [] as string[], userMessageIds = [] as string[] } = {}, + ) => ({ + session: { status, activeTurnId: null, updatedAt: status }, + activities: toolActivityIds.map((id, index) => ({ + id, + kind: "tool.completed", + sequence: index, + createdAt: "2026-09-25T00:00:01Z", + })), + messages: userMessageIds.map((id) => ({ id, role: "user" })), + latestTurn: null, + }); + let root: ReactTestRenderer | null = null; + const render = () => + act(() => { + if (root) root.update(createElement(QueuedMessageSender)); + else root = create(createElement(QueuedMessageSender)); + }); + afterEach(async () => { + await act(() => root?.unmount()); + root = null; + }); + + it("sends a queued message when the turn ends, with no chat view open", async () => { + enqueue(); + io.thread = thread("running"); + await render(); + expect(commandsRun()).toEqual([]); + + io.thread = thread("ready"); + await render(); + + expect(commandsRun()).toEqual(["start"]); + expect(io.run.mock.calls[0]?.[2]).toMatchObject({ + environmentId: "env-a", + input: { threadId: "thread-a", message: { text: "follow up" }, modelSelection }, + }); + expect(queue()).toBeUndefined(); + }); + + it("holds the next message until the server picks up the one before it", async () => { + enqueue({ prompt: "first" }); + enqueue({ prompt: "second" }); + io.thread = thread("ready"); + await render(); + await render(); + expect(commandsRun()).toEqual(["start"]); + + // The first message started a turn; the second waits for its next tool call. + io.thread = thread("running", { userMessageIds: ["first"] }); + await render(); + expect(commandsRun()).toEqual(["start"]); + io.thread = thread("running", { userMessageIds: ["first"], toolActivityIds: ["tool-1"] }); + await render(); + expect(commandsRun()).toEqual(["start", "start"]); + }); + + it("moves on to the next message after a failed one is cancelled", async () => { + io.run.mockResolvedValueOnce({ _tag: "Failure", cause: Cause.fail(new Error("offline")) }); + const first = enqueue({ prompt: "first" }); + enqueue({ prompt: "second" }); + io.thread = thread("ready"); + await render(); + expect(queue()?.[0]).toMatchObject({ prompt: "first", holdUntilUserAction: true }); + + await act(() => { + useQueuedMessageStore.getState().remove(threadKey, first.id); + }); + await render(); + + expect(commandsRun()).toEqual(["start", "start"]); + expect(io.run.mock.calls[1]?.[2]).toMatchObject({ input: { message: { text: "second" } } }); + }); +}); + +describe("sendQueuedMessage", () => { + it("saves a mode changed before queueing, then starts the turn", async () => { + io.shell = { ...io.shell, runtimeMode: "approval-required" }; + const message = enqueue(); + + await sendQueuedMessage(threadRef, message.id); + + expect(commandsRun()).toEqual(["runtime", "start"]); + expect(io.run.mock.calls[1]?.[2]).toMatchObject({ input: { runtimeMode: "full-access" } }); + expect(queue()).toBeUndefined(); + }); + + it("gives a message back to Stop while its upload runs, without starting a turn", async () => { + let finishUpload!: () => void; + io.upload.mockReturnValue(new Promise((resolve) => (finishUpload = resolve))); + const image = { + type: "image" as const, + id: "image-1", + name: "a.png", + mimeType: "image/png", + sizeBytes: 4, + previewUrl: "data:image/png;base64,AAAA", + file: new File(["AAAA"], "a.png", { type: "image/png" }), + }; + const message = enqueue({ images: [image] }); + + const sending = sendQueuedMessage(threadRef, message.id); + expect(useQueuedMessageStore.getState().drain(threadKey)).toHaveLength(1); + finishUpload(); + await sending; + + expect(commandsRun()).toEqual([]); + expect(io.toast).not.toHaveBeenCalled(); + expect(queue()).toBeUndefined(); + }); + + it("holds a message at the head when the turn start fails", async () => { + io.run.mockResolvedValue({ _tag: "Failure", cause: Cause.fail(new Error("offline")) }); + enqueue({ prompt: "first" }); + const second = enqueue({ prompt: "second" }); + + await sendQueuedMessage(threadRef, second.id); + + expect(queue()?.map((entry) => [entry.prompt, entry.holdUntilUserAction])).toEqual([ + ["second", true], + ["first", undefined], + ]); + expect(io.toast).toHaveBeenCalledWith(expect.objectContaining({ description: "offline" })); + }); +}); diff --git a/apps/web/src/components/QueuedMessageSender.tsx b/apps/web/src/components/QueuedMessageSender.tsx new file mode 100644 index 000000000000..86dfa8a265fe --- /dev/null +++ b/apps/web/src/components/QueuedMessageSender.tsx @@ -0,0 +1,100 @@ +import { parseScopedThreadKey } from "@t3tools/client-runtime/environment"; +import { derivePendingRequests } from "@t3tools/client-runtime/pending-requests"; +import { useEffect, useMemo } from "react"; +import { useShallow } from "zustand/react/shallow"; + +import { useComposerDraftStore } from "../composerDraftStore"; +import { + isQueuedMessageDue, + latestCompletedToolActivityId, + useQueuedMessageStore, + useQueuedMessages, +} from "../queuedMessageStore"; +import { derivePhase } from "../session-logic"; +import { useServerConfigs, useThread, useThreadStatus } from "../state/entities"; +import { useEnvironment } from "../state/environments"; +import { hasServerAcknowledgedLocalDispatch, latestTurnStartFailureId } from "./ChatView.logic"; +import { sendQueuedMessage } from "./chat/sendQueuedMessage"; + +/** + * Sends queued messages when they are due, for every thread with a queue, + * whether or not the thread is on screen. Mounted once at the root. + */ +export function QueuedMessageSender() { + const threadKeys = useQueuedMessageStore( + useShallow((state) => Object.keys(state.queuesByThreadKey)), + ); + return threadKeys.map((threadKey) => ); +} + +/** + * Watches one thread while it has queued messages. Reading the thread keeps + * its detail subscribed, so tool boundaries and the end of the turn are + * visible while the user is elsewhere. + */ +function ThreadQueueSender({ threadKey }: { threadKey: string }) { + const threadRef = useMemo(() => parseScopedThreadKey(threadKey), [threadKey]); + const thread = useThread(threadRef); + const threadStatus = useThreadStatus(threadRef); + const environmentId = threadRef?.environmentId ?? null; + const environment = useEnvironment(environmentId); + const serverConfigs = useServerConfigs(); + const serverConfigLoaded = environmentId !== null && serverConfigs.has(environmentId); + const rewinding = useComposerDraftStore((store) => store.rewindingThreadKeys.has(threadKey)); + const queue = useQueuedMessages(threadKey); + const next = queue[0]; + const sending = queue.some((message) => message.sending); + const activities = thread?.activities; + const latestToolActivityId = useMemo( + () => latestCompletedToolActivityId(activities ?? []), + [activities], + ); + const pendingRequests = useMemo(() => derivePendingRequests(activities ?? []), [activities]); + const phase = derivePhase(thread?.session ?? null); + + // A send that starts a new turn leaves the thread idle until the server + // picks it up. Hold the next message until then, as the composer does for + // its own sends. + const lastDispatch = useQueuedMessageStore( + (state) => state.lastDispatchByThreadKey[threadKey]?.thread ?? null, + ); + const latestUserMessageId = thread?.messages.findLast((m) => m.role === "user")?.id ?? null; + const waitingForServer = + lastDispatch !== null && + !hasServerAcknowledgedLocalDispatch({ + localDispatch: lastDispatch, + phase, + latestTurn: thread?.latestTurn ?? null, + latestUserMessageId, + session: thread?.session ?? null, + hasPendingApproval: pendingRequests.approvals.length > 0, + hasPendingUserInput: pendingRequests.userInputs.length > 0, + latestTurnStartFailureId: latestTurnStartFailureId(thread ?? undefined, latestUserMessageId), + threadError: null, + }); + + // Approvals and questions block the agent; a steer landing on top of them + // would answer nothing and confuse the turn, so the queue holds until the + // user resolves them. + const blocked = + threadRef === null || + thread === null || + threadStatus !== "live" || + (environment !== null && environment.connection.phase !== "connected") || + !serverConfigLoaded || + rewinding || + sending || + waitingForServer || + pendingRequests.approvals.length > 0 || + pendingRequests.userInputs.length > 0; + const due = + next !== undefined && + !blocked && + isQueuedMessageDue({ message: next, phase, latestToolActivityId }); + const nextId = next?.id; + useEffect(() => { + if (!due || !threadRef || nextId === undefined) return; + void sendQueuedMessage(threadRef, nextId); + }, [due, nextId, threadRef]); + return null; +} diff --git a/apps/web/src/components/Sidebar.logic.test.ts b/apps/web/src/components/Sidebar.logic.test.ts index 30044e641137..fede3c183448 100644 --- a/apps/web/src/components/Sidebar.logic.test.ts +++ b/apps/web/src/components/Sidebar.logic.test.ts @@ -23,6 +23,7 @@ import { isTrailingDoubleClick, orderItemsByPreferredIds, resolveProjectStatusIndicator, + resolveSidebarRowAccessibility, resolveSidebarThreadStatus, resolveThreadStatusPill, resolveWorkingStartedAt, @@ -31,7 +32,6 @@ import { shouldClearThreadSelectionOnMouseDown, shouldRecedeSidebarThread, sortLogicalProjectsForSidebar, - sortSettledThreadsForSidebar, resolveSidebarDropTarget, pinOrderKeyBetween, planPinnedReorder, @@ -51,6 +51,7 @@ import { resolveSidebarDropVerb, } from "./Sidebar.logic"; import { threadSearchMatchKey } from "@t3tools/client-runtime/state/thread-search"; +import { sortSettledThreads } from "@t3tools/client-runtime/state/thread-sort"; import { EnvironmentId, OrchestrationLatestTurn, @@ -69,6 +70,35 @@ import { const localEnvironmentId = EnvironmentId.make("environment-local"); +describe("resolveSidebarRowAccessibility", () => { + it.each([ + { + title: "Can you audit the UI?", + statusLabel: "Working", + projectDisplayName: "T3 Code", + isActive: true, + expected: { label: "Can you audit the UI?, Working, T3 Code", current: "page" }, + }, + { + title: "The audit is done", + statusLabel: null, + projectDisplayName: "T3 Code", + isActive: false, + expected: { label: "The audit is done, T3 Code", current: undefined }, + }, + { + title: "Untitled task", + statusLabel: null, + projectDisplayName: null, + isActive: false, + expected: { label: "Untitled task", current: undefined }, + }, + ])("leads with the title without folding row actions into its name: %j", (input) => { + const { expected, ...state } = input; + expect(resolveSidebarRowAccessibility(state)).toEqual(expected); + }); +}); + describe("animateSidebarLayoutChanges", () => { const baseArgs: Parameters[0] = { active: null, @@ -1690,12 +1720,12 @@ describe("applySidebarThreadDrop", () => { }; const existing = { ...newer, settledOverride: "settled" as const, settledAt: newer.createdAt }; expect(preview).toEqual({ ...final, settledAt: now }); - expect(sortSettledThreadsForSidebar([existing, preview]).map((row) => row.id)).toEqual([ + expect(sortSettledThreads([existing, preview]).map((row) => row.id)).toEqual([ "dragged", "newer", ]); - expect(sortSettledThreadsForSidebar([existing, preview]).map((row) => row.id)).toEqual( - sortSettledThreadsForSidebar([existing, final]).map((row) => row.id), + expect(sortSettledThreads([existing, preview]).map((row) => row.id)).toEqual( + sortSettledThreads([existing, final]).map((row) => row.id), ); }); @@ -1710,7 +1740,7 @@ describe("applySidebarThreadDrop", () => { const final = { ...source, snoozedAt: null, snoozedUntil: null }; const existing = { ...newer, settledOverride: "settled" as const, settledAt: newer.createdAt }; expect(preview).toEqual(final); - expect(sortSettledThreadsForSidebar([existing, preview]).map((row) => row.id)).toEqual([ + expect(sortSettledThreads([existing, preview]).map((row) => row.id)).toEqual([ "newer", "dragged", ]); @@ -1820,74 +1850,6 @@ describe("sortPinnedThreadsForSidebar", () => { }); }); -describe("sortSettledThreadsForSidebar", () => { - const settled = (input: { - id: string; - settledAt?: string | null; - latestUserMessageAt?: string | null; - latestTurn?: OrchestrationLatestTurn | null; - updatedAt?: string; - }) => ({ - id: input.id, - settledAt: input.settledAt ?? null, - latestUserMessageAt: input.latestUserMessageAt ?? null, - latestTurn: input.latestTurn ?? null, - updatedAt: input.updatedAt ?? "2026-03-09T09:00:00.000Z", - }); - - it("orders by settle time, most recently settled first", () => { - const sorted = sortSettledThreadsForSidebar([ - settled({ - id: "settled-first", - settledAt: "2026-03-09T10:00:00.000Z", - // Created/active later than the other thread: settle time must win. - latestUserMessageAt: "2026-03-09T09:59:00.000Z", - }), - settled({ - id: "settled-last", - settledAt: "2026-03-09T12:00:00.000Z", - latestUserMessageAt: "2026-03-09T08:00:00.000Z", - }), - ]); - - expect(sorted.map((thread) => thread.id)).toEqual(["settled-last", "settled-first"]); - }); - - it("falls back to last activity for auto-settled threads without a settledAt stamp", () => { - const sorted = sortSettledThreadsForSidebar([ - settled({ id: "auto-old", latestUserMessageAt: "2026-03-09T08:00:00.000Z" }), - settled({ id: "explicit", settledAt: "2026-03-09T10:00:00.000Z" }), - settled({ id: "auto-recent", latestUserMessageAt: "2026-03-09T11:00:00.000Z" }), - ]); - - expect(sorted.map((thread) => thread.id)).toEqual(["auto-recent", "explicit", "auto-old"]); - }); - - it("counts a turn completion as activity for auto-settled threads", () => { - // The message came in before the other thread's, but its turn finished - // after: completion time is the real "work ended" moment. - const sorted = sortSettledThreadsForSidebar([ - settled({ id: "message-only", latestUserMessageAt: "2026-03-09T10:04:00.000Z" }), - settled({ - id: "completed-later", - latestUserMessageAt: "2026-03-09T10:00:00.000Z", - latestTurn: makeLatestTurn({ completedAt: "2026-03-09T10:30:00.000Z" }), - }), - ]); - - expect(sorted.map((thread) => thread.id)).toEqual(["completed-later", "message-only"]); - }); - - it("breaks timestamp ties by id so the order is stable", () => { - const sorted = sortSettledThreadsForSidebar([ - settled({ id: "b", settledAt: "2026-03-09T10:00:00.000Z" }), - settled({ id: "a", settledAt: "2026-03-09T10:00:00.000Z" }), - ]); - - expect(sorted.map((thread) => thread.id)).toEqual(["a", "b"]); - }); -}); - describe("resolveWorkingStartedAt", () => { const session = { threadId: ThreadId.make("thread-1"), @@ -2375,6 +2337,49 @@ describe("sortProjectsForSidebar", () => { ]); }); + it.each(["updated_at", "created_at"] as const)( + "matches the per-comparison %s order on a shuffled list with ties", + (sortOrder) => { + const minute = (value: number) => `2026-03-09T10:0${value}:00.000Z`; + // (index * 7) % 24 scrambles the input order. Titles repeat, and + // projects 16-23 have no threads, so they use their own stamps. + const projects = Array.from({ length: 24 }, (_, index) => { + const n = (index * 7) % 24; + return makeProject({ + id: ProjectId.make(`project-${n}`), + title: n % 2 === 0 ? "Alpha" : "Beta", + createdAt: minute(n % 3), + updatedAt: n % 5 === 0 ? "invalid" : minute(n % 2), + }); + }); + const threads = Array.from({ length: 48 }, (_, n) => ({ + projectId: ProjectId.make(`project-${n % 16}`), + createdAt: minute(n % 6), + updatedAt: minute(n % 3), + latestUserMessageAt: n % 4 === 0 ? null : minute(n % 5), + })); + // The comparator this sort replaced: it walked each project's threads + // on every call. + const timestamp = (project: Project) => + getProjectSortTimestamp( + project, + threads.filter((thread) => thread.projectId === project.id), + sortOrder, + ); + const expected = projects.toSorted((left, right) => { + const rightTimestamp = timestamp(right); + const leftTimestamp = timestamp(left); + const byTimestamp = + rightTimestamp === leftTimestamp ? 0 : rightTimestamp > leftTimestamp ? 1 : -1; + return ( + byTimestamp || left.title.localeCompare(right.title) || left.id.localeCompare(right.id) + ); + }); + + expect(sortProjectsForSidebar(projects, threads, sortOrder)).toEqual(expected); + }, + ); + it("returns the project timestamp when no threads are present", () => { const timestamp = getProjectSortTimestamp( makeProject({ updatedAt: "2026-03-09T10:10:00.000Z" }), diff --git a/apps/web/src/components/Sidebar.logic.ts b/apps/web/src/components/Sidebar.logic.ts index f4b13a5000bc..2796b2f7f885 100644 --- a/apps/web/src/components/Sidebar.logic.ts +++ b/apps/web/src/components/Sidebar.logic.ts @@ -16,10 +16,8 @@ import { } from "@t3tools/client-runtime/state/thread-settled"; import { getThreadSortTimestamp, - resolveSettledThreadTimestamp, sortThreads, toSortableTimestamp, - type SettledThreadTimestampInput, type ThreadSortInput, } from "../lib/threadSort"; import type { SidebarThreadSummary, Thread } from "../types"; @@ -44,6 +42,21 @@ export function shouldNavigateAfterThreadPark(input: { const THREAD_SELECTION_SAFE_SELECTOR = "[data-thread-item], [data-thread-selection-safe]"; export const THREAD_JUMP_HINT_SHOW_DELAY_MS = 200; + +export function resolveSidebarRowAccessibility(input: { + readonly title: string; + readonly statusLabel: string | null; + readonly projectDisplayName: string | null; + readonly isActive: boolean; +}): { readonly label: string; readonly current: "page" | undefined } { + return { + // The title is the row's identity and must lead when users scan tasks. + // Only static context belongs here; nested action labels remain separate controls. + label: [input.title, input.statusLabel, input.projectDisplayName].filter(Boolean).join(", "), + current: input.isActive ? "page" : undefined, + }; +} + // Visible sidebar rows are prewarmed into the thread-detail cache so opening a // nearby thread usually reuses an already-hot subscription. Each prewarmed // thread holds a live, fully hydrated detail subscription (all messages and @@ -937,20 +950,6 @@ export function reduceSidebarProjectScopeMenuState( } } -// Settled rows are history, so they order by when the work ENDED, not when -// the thread was created or last touched. -export function sortSettledThreadsForSidebar< - T extends SettledThreadTimestampInput & { readonly id: string }, ->(threads: readonly T[]): T[] { - const timestampMs = (thread: T) => { - const timestamp = resolveSettledThreadTimestamp(thread); - return timestamp === null ? 0 : Date.parse(timestamp); - }; - return [...threads].toSorted( - (left, right) => timestampMs(right) - timestampMs(left) || left.id.localeCompare(right.id), - ); -} - /** The timestamp a working thread's elapsed label counts from: the running turn's start (request time until adoption), falling back to the session's last transition when the turn projection lags behind. Malformed @@ -1136,13 +1135,19 @@ function sortProjectsByActivity( return [...projects]; } - return [...projects].toSorted((left, right) => { - const rightTimestamp = getProjectSortTimestamp(right, getProjectThreads(right), sortOrder); - const leftTimestamp = getProjectSortTimestamp(left, getProjectThreads(left), sortOrder); - const byTimestamp = - rightTimestamp === leftTimestamp ? 0 : rightTimestamp > leftTimestamp ? 1 : -1; - return byTimestamp || compareTies(left, right); - }); + // Each project's timestamp walks all of its threads, so compute it once + // per project instead of once per comparison. + return projects + .map((project) => ({ + project, + timestamp: getProjectSortTimestamp(project, getProjectThreads(project), sortOrder), + })) + .sort((left, right) => { + const byTimestamp = + right.timestamp === left.timestamp ? 0 : right.timestamp > left.timestamp ? 1 : -1; + return byTimestamp || compareTies(left.project, right.project); + }) + .map(({ project }) => project); } export function sortProjectsForSidebar< diff --git a/apps/web/src/components/Sidebar.tsx b/apps/web/src/components/Sidebar.tsx index 4b8027c14108..90f65ea2b941 100644 --- a/apps/web/src/components/Sidebar.tsx +++ b/apps/web/src/components/Sidebar.tsx @@ -21,7 +21,10 @@ import { effectiveSnoozed, threadWokeAt, } from "@t3tools/client-runtime/state/thread-settled"; -import { resolveSettledThreadTimestamp } from "@t3tools/client-runtime/state/thread-sort"; +import { + resolveSettledThreadTimestamp, + sortSettledThreads, +} from "@t3tools/client-runtime/state/thread-sort"; import { threadSearchMatchKey, type EnvironmentThreadSearchMatch, @@ -170,6 +173,7 @@ import { resolveAdjacentThreadId, resolveSidebarDropTarget, resolveSidebarDropVerb, + resolveSidebarRowAccessibility, type SidebarDropVerb, resolveSidebarThreadStatus, searchSidebarThreads, @@ -181,7 +185,6 @@ import { sidebarMarkerId, sortLogicalProjectsForSidebar, sortPinnedThreadsForSidebar, - sortSettledThreadsForSidebar, sortThreadsForSidebar, useRetainedValue, useSidebarRowSubscriptionLease, @@ -726,6 +729,12 @@ const SidebarDraftRow = memo(function SidebarDraftRow(props: { promptPreview.length > 0 ? promptPreview : `${attachmentCount} attachment${attachmentCount === 1 ? "" : "s"}`; + const accessibility = resolveSidebarRowAccessibility({ + title: preview, + statusLabel: "Unsent draft", + projectDisplayName: props.projectDisplayName, + isActive: props.isActive, + }); const handleActivate = useCallback(() => onNavigate(draftId), [draftId, onNavigate]); const handleKeyDown = useCallback( (event: ReactKeyboardEvent) => { @@ -753,14 +762,17 @@ const SidebarDraftRow = memo(function SidebarDraftRow(props: {
+ {preview}
@@ -788,7 +800,9 @@ const SidebarDraftRow = memo(function SidebarDraftRow(props: {
-
{preview}
+
+ {preview} +
@@ -1390,7 +1404,7 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { // a useful hierarchy nor a reliable hover cue. Status now lives in the row // content; surface is reserved for interaction (hover, multi-select, route). const rowSurfaceClassName = cn( - "group/sidebar-row relative w-full cursor-pointer overflow-hidden rounded-md text-left outline-none select-none", + "group/sidebar-row relative w-full cursor-pointer overflow-hidden rounded-md text-left outline-none select-none focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-ring", variantAction === "unsettle" && "[&:not(:hover):not(:focus-within)_*]:text-secondary-label/70", props.isActive ? "bg-sidebar-row-active text-sidebar-foreground" @@ -1445,6 +1459,13 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { ) : null; + const accessibility = resolveSidebarRowAccessibility({ + title: thread.title, + statusLabel: topStatus?.label ?? null, + projectDisplayName: props.projectDisplayName, + isActive: props.isActive, + }); + const title = isRenaming ? ( ) : ( ); + const accessibleTitle = isRenaming ? null : {thread.title}; // Stacks show their layer count; multiple unrelated links show their total count. // Plain clicks open T3; individual PR links also support opening the host in a new tab. @@ -1591,6 +1614,8 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { ref={rowRef} role="button" tabIndex={0} + aria-label={accessibility.label} + aria-current={accessibility.current} data-testid="sidebar-row-slim" aria-busy={isRegeneratingTitle || undefined} className={cn(rowSurfaceClassName, "flex h-9 items-center gap-2.5 px-2.5")} @@ -1601,6 +1626,7 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { /> } > + {accessibleTitle} {/* Settled history recedes: dimmed favicon at rest, restored on hover so the tail stays scannable when you're hunting. */} } > + {accessibleTitle}
{draftIndicator} @@ -2006,6 +2035,12 @@ const SidebarSearchResultRow = memo(function SidebarSearchResultRow(props: { onFileDropThreads: (threadRef: ScopedThreadRef, files: File[]) => void; }) { const { thread } = props; + const accessibility = resolveSidebarRowAccessibility({ + title: thread.title, + statusLabel: null, + projectDisplayName: props.projectDisplayName, + isActive: props.isRouteActive, + }); const threadRef = useMemo( () => scopeThreadRef(thread.environmentId, thread.id), [thread.environmentId, thread.id], @@ -2082,12 +2117,8 @@ const SidebarSearchResultRow = memo(function SidebarSearchResultRow(props: { // which owns all keyboard interaction for the listbox. tabIndex={-1} aria-selected={props.isHighlighted} - aria-current={props.isRouteActive ? "page" : undefined} - aria-label={ - props.projectDisplayName - ? `${thread.title}, ${props.projectDisplayName}` - : thread.title - } + aria-current={accessibility.current} + aria-label={accessibility.label} onMouseMove={props.onHighlight} onClick={props.onSelect} className={cn( @@ -2626,7 +2657,7 @@ export default function Sidebar() { firstValidTimestampMs(left.snoozedUntil ?? null) - firstValidTimestampMs(right.snoozedUntil ?? null), ), - settledThreads: sortSettledThreadsForSidebar(settled), + settledThreads: sortSettledThreads(settled), snoozeNow: preciseNow, }; }, [nowMinute, optimisticDrop, scopedProjectKeys, serverConfigs, snoozeWakeTick, threads]); @@ -3452,7 +3483,7 @@ export default function Sidebar() { if (dragState === null || thread === undefined) return []; const key = (candidate: EnvironmentThreadShell) => scopedThreadKey(scopeThreadRef(candidate.environmentId, candidate.id)); - return sortSettledThreadsForSidebar([ + return sortSettledThreads([ ...settledThreads.filter((candidate) => key(candidate) !== dragState.activeKey), applySidebarThreadDrop(thread, "settled", dragState.occurredAt), ]).map(key); @@ -4571,7 +4602,7 @@ export default function Sidebar() { } > - + {isSearchingThreads ? ( threadSearchResults.length > 0 ? (