diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.search.test.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.search.test.ts index 3e865b82e259..e8a635177261 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.search.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.search.test.ts @@ -1,7 +1,9 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; +import { MessageId, ProjectId, ThreadId } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; import * as SqlClient from "effect/unstable/sql/SqlClient"; import { ServerConfig } from "../../config.ts"; @@ -9,30 +11,388 @@ import { SqlitePersistenceMemory } from "../../persistence/Layers/Sqlite.ts"; import * as ProjectEnrichment from "../../project/ProjectEnrichmentService.ts"; import * as ProjectFaviconResolver from "../../project/ProjectFaviconResolver.ts"; import * as RepositoryIdentityResolver from "../../project/RepositoryIdentityResolver.ts"; -import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts"; +import { + PROJECTION_THREAD_CONTENT_SEARCH_LIMITS, + ProjectionSnapshotQuery, +} from "../Services/ProjectionSnapshotQuery.ts"; import * as ThreadBackgroundLiveness from "../ThreadBackgroundLiveness.ts"; import * as ThreadPlanProgress from "../ThreadPlanProgress.ts"; import { OrchestrationProjectionSnapshotQueryLive } from "./ProjectionSnapshotQuery.ts"; const metadataLayer = Layer.merge( Layer.succeed(RepositoryIdentityResolver.RepositoryIdentityResolver, { - resolve: () => Effect.succeed(null), + resolve: (workspaceRoot) => + Effect.succeed({ + canonicalKey: `test:${workspaceRoot}`, + locator: { + source: "git-remote" as const, + remoteName: "origin", + remoteUrl: "https://example.test/search.git", + }, + rootPath: workspaceRoot, + }), }), Layer.succeed(ProjectFaviconResolver.ProjectFaviconResolver, { resolvePath: () => Effect.succeed(null), }), ); -const TestLayer = OrchestrationProjectionSnapshotQueryLive.pipe( +const encodeUnknownJsonString = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); + +const testLayer = OrchestrationProjectionSnapshotQueryLive.pipe( Layer.provideMerge(ThreadBackgroundLiveness.layer), Layer.provideMerge(ThreadPlanProgress.layer), Layer.provideMerge(ProjectEnrichment.layer), Layer.provideMerge(metadataLayer), Layer.provideMerge(SqlitePersistenceMemory), - Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "projection-search-test-" })), + Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "thread-content-search-test-" })), Layer.provide(NodeServices.layer), ); +it.effect("searches bounded durable thread content without changing the legacy search RPC", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const query = yield* ProjectionSnapshotQuery; + const projectId = ProjectId.make("project:search-current"); + const otherProjectId = ProjectId.make("project:search-other"); + const activeThreadId = ThreadId.make("thread:search-active"); + const archivedThreadId = ThreadId.make("thread:search-archived"); + const deletedThreadId = ThreadId.make("thread:search-deleted"); + const otherThreadId = ThreadId.make("thread:search-other"); + const legacyOnlyThreadId = ThreadId.make("thread:search-legacy-only"); + const longText = `${"prefix ".repeat(2_000)}literal %_! alpha\tbeta needle😀終 ${"suffix ".repeat(2_000)}`; + const longTitle = `Needle active title ${"x".repeat(2_000)} hidden-title-match`; + + yield* sql` + INSERT INTO projection_projects ( + project_id, title, workspace_root, default_model_selection_json, scripts_json, + created_at, updated_at, deleted_at + ) VALUES + (${projectId}, 'Current', '/tmp/search-current', NULL, '[]', + '2026-01-01T00:00:00.000Z', '2026-01-10T00:00:00.000Z', NULL), + (${otherProjectId}, 'Other', '/tmp/search-other', NULL, '[]', + '2026-01-01T00:00:00.000Z', '2026-01-10T00:00:00.000Z', NULL) + `; + yield* sql` + INSERT INTO projection_threads ( + thread_id, project_id, title, model_selection_json, runtime_mode, interaction_mode, + branch, worktree_path, latest_turn_id, created_at, updated_at, archived_at, + settled_override, settled_at, deleted_at + ) VALUES + (${activeThreadId}, ${projectId}, 'Legacy shadow title', '{}', 'full-access', 'default', + 'main', '/tmp/search-current', NULL, '2026-01-01T00:00:00.000Z', + '2026-01-10T00:00:00.000Z', NULL, NULL, NULL, NULL), + (${archivedThreadId}, ${projectId}, 'Archived history', '{}', 'full-access', 'default', + 'main', '/tmp/search-current', NULL, '2026-01-01T00:00:00.000Z', + '2026-01-09T00:00:00.000Z', NULL, NULL, NULL, NULL), + (${deletedThreadId}, ${projectId}, 'Deleted needle', '{}', 'full-access', 'default', + 'main', '/tmp/search-current', NULL, '2026-01-01T00:00:00.000Z', + '2026-01-08T00:00:00.000Z', NULL, NULL, NULL, NULL), + (${otherThreadId}, ${otherProjectId}, 'Other needle', '{}', 'full-access', 'default', + 'main', '/tmp/search-other', NULL, '2026-01-01T00:00:00.000Z', + '2026-01-07T00:00:00.000Z', NULL, NULL, NULL, NULL), + (${legacyOnlyThreadId}, ${projectId}, 'Legacy only history', '{}', 'full-access', 'default', + 'main', '/tmp/search-current', NULL, '2026-01-01T00:00:00.000Z', + '2026-01-02T00:00:00.000Z', NULL, NULL, NULL, NULL) + `; + yield* sql` + INSERT INTO orchestration_v2_projection_threads ( + thread_id, project_id, title, default_provider, provider_instance_id, + runtime_mode, interaction_mode, active_provider_thread_id, created_at, updated_at, + archived_at, deleted_at, payload_json + ) VALUES + (${activeThreadId}, ${projectId}, ${longTitle}, 'codex', 'codex', + 'full-access', 'default', NULL, '2026-01-01T00:00:00.000Z', + '2026-01-10T00:00:00.000Z', NULL, NULL, '{}'), + (${archivedThreadId}, ${projectId}, 'Archived history', 'codex', 'codex', + 'full-access', 'default', NULL, '2026-01-01T00:00:00.000Z', + '2026-01-09T00:00:00.000Z', '2026-01-09T00:00:00.000Z', NULL, '{}'), + (${deletedThreadId}, ${projectId}, 'Deleted needle', 'codex', 'codex', + 'full-access', 'default', NULL, '2026-01-01T00:00:00.000Z', + '2026-01-08T00:00:00.000Z', NULL, '2026-01-08T00:00:00.000Z', '{}'), + (${otherThreadId}, ${otherProjectId}, 'Other needle', 'codex', 'codex', + 'full-access', 'default', NULL, '2026-01-01T00:00:00.000Z', + '2026-01-07T00:00:00.000Z', NULL, NULL, '{}') + `; + yield* sql` + INSERT INTO projection_thread_messages ( + message_id, thread_id, turn_id, role, text, attachments_json, is_streaming, + created_at, updated_at + ) VALUES + ('message:legacy-active', ${activeThreadId}, NULL, 'user', 'legacy needle', '[]', 0, + '2026-01-03T00:00:00.000Z', '2026-01-03T00:00:00.000Z'), + ('message:legacy-archived', ${archivedThreadId}, NULL, 'user', 'archived needle', '[]', 0, + '2026-01-04T00:00:00.000Z', '2026-01-04T00:00:00.000Z'), + ('message:legacy-other', ${otherThreadId}, NULL, 'user', 'secret needle', '[]', 0, + '2026-01-05T00:00:00.000Z', '2026-01-05T00:00:00.000Z'), + ('message:legacy-only', ${legacyOnlyThreadId}, NULL, 'user', 'legacy-only-marker', '[]', 0, + '2026-01-02T00:00:00.000Z', '2026-01-02T00:00:00.000Z') + `; + yield* sql` + INSERT INTO orchestration_v2_projection_runs ( + run_id, thread_id, ordinal, provider, provider_thread_id, status, + requested_at, completed_at, payload_json + ) VALUES + ('run:visible', ${activeThreadId}, 1, 'codex', NULL, 'completed', + '2026-01-06T00:00:00.000Z', '2026-01-06T00:01:00.000Z', '{}'), + ('run:rolled-back', ${activeThreadId}, 2, 'codex', NULL, 'rolled_back', + '2026-01-07T00:00:00.000Z', '2026-01-07T00:01:00.000Z', '{}'), + ('run:cancelled', ${activeThreadId}, 3, 'codex', NULL, 'cancelled', + '2026-01-08T00:00:00.000Z', '2026-01-08T00:01:00.000Z', '{}') + `; + yield* sql` + INSERT INTO orchestration_v2_projection_messages ( + message_id, thread_id, run_id, node_id, role, streaming, created_at, updated_at, payload_json + ) VALUES + ('message:v2-visible', ${activeThreadId}, 'run:visible', NULL, 'assistant', 0, + '2026-01-06T00:00:00.000Z', '2026-01-06T00:00:00.000Z', + ${encodeUnknownJsonString({ text: longText })}), + ('message:v2-user', ${activeThreadId}, 'run:visible', NULL, 'user', 0, + '2026-01-05T00:00:00.000Z', '2026-01-05T00:00:00.000Z', + ${encodeUnknownJsonString({ text: "visible user needle" })}), + ('message:v2-streaming', ${activeThreadId}, 'run:visible', NULL, 'assistant', 1, + '2026-01-05T00:00:01.000Z', '2026-01-05T00:00:01.000Z', + ${encodeUnknownJsonString({ text: "streaming needle" })}), + ('message:v2-rolled-back', ${activeThreadId}, 'run:rolled-back', NULL, 'assistant', 0, + '2026-01-07T00:00:00.000Z', '2026-01-07T00:00:00.000Z', + ${encodeUnknownJsonString({ text: "rolled-back needle" })}), + ('message:v2-cancelled', ${activeThreadId}, 'run:cancelled', NULL, 'user', 0, + '2026-01-08T00:00:00.000Z', '2026-01-08T00:00:00.000Z', + ${encodeUnknownJsonString({ text: "cancelled needle" })}), + ('message:v2-queued-null-run', ${activeThreadId}, NULL, NULL, 'user', 0, + '2026-01-08T00:00:01.000Z', '2026-01-08T00:00:01.000Z', + ${encodeUnknownJsonString({ text: "queued-null-marker" })}), + ('message:v2-queued-missing-run', ${activeThreadId}, 'run:missing', NULL, 'user', 0, + '2026-01-08T00:00:02.000Z', '2026-01-08T00:00:02.000Z', + ${encodeUnknownJsonString({ text: "queued-missing-marker" })}) + `; + yield* sql` + INSERT INTO orchestration_v2_projection_turn_items ( + turn_item_id, thread_id, run_id, node_id, provider_thread_id, provider_turn_id, + parent_item_id, ordinal, type, status, updated_at, payload_json + ) VALUES + ('item:v2-visible', ${activeThreadId}, 'run:visible', NULL, NULL, NULL, NULL, 1, + 'assistant_message', 'completed', '2026-01-06T00:00:00.000Z', + ${encodeUnknownJsonString({ messageId: "message:v2-visible", text: longText, streaming: false })}), + ('item:v2-user', ${activeThreadId}, 'run:visible', NULL, NULL, NULL, NULL, 2, + 'user_message', 'completed', '2026-01-05T00:00:00.000Z', + ${encodeUnknownJsonString({ messageId: "message:v2-user", text: "visible user needle", inputIntent: "turn_start" })}), + ('item:v2-streaming', ${activeThreadId}, 'run:visible', NULL, NULL, NULL, NULL, 3, + 'assistant_message', 'streaming', '2026-01-05T00:00:01.000Z', + ${encodeUnknownJsonString({ messageId: "message:v2-streaming", text: "streaming needle", streaming: true })}), + ('item:v2-rolled-back', ${activeThreadId}, 'run:rolled-back', NULL, NULL, NULL, NULL, 2, + 'assistant_message', 'completed', '2026-01-07T00:00:00.000Z', + ${encodeUnknownJsonString({ messageId: "message:v2-rolled-back", text: "rolled-back needle", streaming: false })}), + ('item:v2-cancelled', ${activeThreadId}, 'run:cancelled', NULL, NULL, NULL, NULL, 3, + 'user_message', 'completed', '2026-01-08T00:00:00.000Z', + ${encodeUnknownJsonString({ messageId: "message:v2-cancelled", text: "cancelled needle", inputIntent: "queued_turn" })}), + ('item:v2-queued-null-run', ${activeThreadId}, NULL, NULL, NULL, NULL, NULL, 4, + 'user_message', 'completed', '2026-01-08T00:00:01.000Z', + ${encodeUnknownJsonString({ messageId: "message:v2-queued-null-run", text: "queued-null-marker", inputIntent: "queued_turn" })}), + ('item:v2-queued-missing-run', ${activeThreadId}, 'run:missing', NULL, NULL, NULL, NULL, 5, + 'user_message', 'completed', '2026-01-08T00:00:02.000Z', + ${encodeUnknownJsonString({ messageId: "message:v2-queued-missing-run", text: "queued-missing-marker", inputIntent: "queued_turn" })}) + `; + + const firstPage = yield* query.searchThreadContent({ + projectId, + query: "needle", + includeArchived: true, + offset: 0, + limit: 2, + snippetChars: 80, + }); + assert.lengthOf(firstPage.hits, 2); + assert.isTrue(firstPage.hasMore); + assert.equal(firstPage.nextOffset, 2); + assert.deepEqual( + firstPage.hits.map((hit) => [hit.source, hit.origin]), + [ + ["title", "v2"], + ["assistant", "v2"], + ], + ); + const anchored = firstPage.hits[1]; + assert.equal(anchored?.sourceThreadId, activeThreadId); + assert.equal(anchored?.messageId, "message:v2-visible"); + assert.equal(anchored?.itemId, "item:v2-visible"); + assert.match(anchored?.snippet ?? "", /needle/); + assert.isAtMost(Array.from(anchored?.snippet ?? "").length, 80); + assert.isTrue(anchored?.snippetTruncated ?? false); + assert.isAtMost( + Array.from(anchored?.threadTitle ?? "").length, + PROJECTION_THREAD_CONTENT_SEARCH_LIMITS.titleMaxChars, + ); + assert.isTrue(anchored?.threadTitleTruncated ?? false); + + const secondPage = yield* query.searchThreadContent({ + projectId, + query: "needle", + includeArchived: true, + offset: firstPage.nextOffset ?? 0, + limit: 10, + snippetChars: 80, + }); + assert.deepEqual( + secondPage.hits.map((hit) => hit.threadId), + [activeThreadId, archivedThreadId, activeThreadId], + ); + assert.equal(secondPage.hits[0]?.messageId, "message:v2-user"); + assert.equal(secondPage.hits[0]?.source, "user"); + assert.equal(secondPage.hits[0]?.origin, "v2"); + assert.isNull(secondPage.hits[1]?.sourceThreadId ?? null); + assert.isTrue(secondPage.hits[1]?.archived ?? false); + assert.notInclude( + secondPage.hits.map((hit) => hit.messageId), + MessageId.make("message:v2-streaming"), + ); + assert.notInclude( + secondPage.hits.map((hit) => hit.threadId), + otherThreadId, + ); + assert.notInclude( + secondPage.hits.map((hit) => hit.threadId), + deletedThreadId, + ); + + const activeOnly = yield* query.searchThreadContent({ + projectId, + query: "needle", + includeArchived: false, + offset: 0, + limit: 10, + snippetChars: 80, + }); + assert.isTrue(activeOnly.hits.every((hit) => !hit.archived)); + + const queuedWithoutRun = yield* query.searchThreadContent({ + projectId, + query: "queued-", + includeArchived: false, + offset: 0, + limit: 10, + snippetChars: 64, + }); + assert.deepEqual( + queuedWithoutRun.hits.map((hit) => hit.messageId), + [ + MessageId.make("message:v2-queued-missing-run"), + MessageId.make("message:v2-queued-null-run"), + ], + ); + + const legacyOnly = yield* query.searchThreadContent({ + projectId, + query: "legacy-only-marker", + includeArchived: false, + offset: 0, + limit: 10, + snippetChars: 64, + }); + assert.equal(legacyOnly.hits[0]?.threadId, legacyOnlyThreadId); + assert.equal(legacyOnly.hits[0]?.origin, "legacy"); + assert.isNull(legacyOnly.hits[0]?.sourceThreadId ?? null); + + const titleTail = yield* query.searchThreadContent({ + projectId, + threadId: activeThreadId, + query: "hidden-title-match", + includeArchived: false, + offset: 0, + limit: 10, + snippetChars: 64, + }); + assert.equal(titleTail.hits[0]?.source, "title"); + assert.isTrue(titleTail.hits[0]?.threadTitleTruncated ?? false); + assert.isAtMost( + Array.from(titleTail.hits[0]?.threadTitle ?? "").length, + PROJECTION_THREAD_CONTENT_SEARCH_LIMITS.titleMaxChars, + ); + assert.match(titleTail.hits[0]?.snippet ?? "", /hidden-title-match/); + + const literal = yield* query.searchThreadContent({ + projectId, + threadId: activeThreadId, + query: "%_!", + includeArchived: false, + offset: 0, + limit: 10, + snippetChars: 64, + }); + assert.deepEqual( + literal.hits.map((hit) => hit.messageId), + [MessageId.make("message:v2-visible")], + ); + + const literalWhitespace = yield* query.searchThreadContent({ + projectId, + threadId: activeThreadId, + query: "alpha\tbeta", + includeArchived: false, + offset: 0, + limit: 10, + snippetChars: 64, + }); + assert.include(literalWhitespace.hits[0]?.snippet ?? "", "alpha\tbeta"); + + const nonAscii = yield* query.searchThreadContent({ + projectId, + threadId: activeThreadId, + query: "needle😀終", + includeArchived: false, + offset: 0, + limit: 10, + snippetChars: 64, + }); + assert.match(nonAscii.hits[0]?.snippet ?? "", /needle😀終/); + assert.isAtMost(Array.from(nonAscii.hits[0]?.snippet ?? "").length, 64); + + const legacyResult = yield* query.searchThreads({ query: "needle", limit: 50 }); + assert.deepEqual(Object.keys(legacyResult), ["matches"]); + assert.isTrue(legacyResult.matches.every((match) => match.source !== undefined)); + assert.isTrue(legacyResult.matches.every((match) => !("itemId" in match))); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("rejects out-of-bounds lower-layer requests before querying", () => + Effect.gen(function* () { + const query = yield* ProjectionSnapshotQuery; + const failure = yield* Effect.flip( + query.searchThreadContent({ + projectId: ProjectId.make("project:unused"), + query: "needle", + includeArchived: false, + offset: PROJECTION_THREAD_CONTENT_SEARCH_LIMITS.offsetMax + 1, + limit: 1, + snippetChars: 64, + }), + ); + assert.equal(failure._tag, "ProjectionThreadContentSearchInputError"); + if (failure._tag !== "ProjectionThreadContentSearchInputError") { + return yield* Effect.die("Expected bounded input failure."); + } + assert.equal(failure.field, "offset"); + assert.equal(failure.constraint.type, "range"); + + const nulFailure = yield* Effect.flip( + query.searchThreadContent({ + projectId: ProjectId.make("project:unused"), + query: "unrelated\0needle", + includeArchived: false, + offset: 0, + limit: 1, + snippetChars: 64, + }), + ); + assert.equal(nulFailure._tag, "ProjectionThreadContentSearchInputError"); + if (nulFailure._tag !== "ProjectionThreadContentSearchInputError") { + return yield* Effect.die("Expected NUL input failure."); + } + assert.equal(nulFailure.field, "query"); + assert.equal(nulFailure.constraint.type, "no_nul"); + }).pipe(Effect.provide(testLayer)), +); + it.effect("search uses v2 visibility while legacy transcripts are still lazy", () => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; @@ -98,5 +458,5 @@ it.effect("search uses v2 visibility while legacy transcripts are still lazy", ( ["thread:assistant", "assistant", "migration needle answer"], ], ); - }).pipe(Effect.provide(TestLayer)), + }).pipe(Effect.provide(testLayer)), ); diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts index 778c4d3d7226..0a31c11bdd5f 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts @@ -11,6 +11,8 @@ import { ProjectId, ProjectIconOverride, ThreadId, + RunId, + TurnItemId, } from "@t3tools/contracts"; import { OrchestrationCheckpointFile, @@ -70,6 +72,9 @@ import { type ProjectionThreadCheckpointContext, type ProjectionThreadDetailQuery, type ProjectionSnapshotQueryShape, + PROJECTION_THREAD_CONTENT_SEARCH_LIMITS, + ProjectionThreadContentSearchInputError, + type ProjectionThreadContentSearchHit, } from "../Services/ProjectionSnapshotQuery.ts"; const decodeReadModel = Schema.decodeUnknownEffect(OrchestrationReadModel); @@ -151,6 +156,37 @@ const ProjectionThreadSearchRow = Schema.Struct({ matchText: Schema.String, messageCreatedAt: Schema.NullOr(IsoDateTime), }); +const ProjectionThreadContentSearchRequest = Schema.Struct({ + projectId: ProjectId, + threadId: Schema.NullOr(ThreadId), + pattern: Schema.String, + queryText: Schema.String, + includeArchived: Schema.Boolean, + limit: Schema.Int, + offset: Schema.Int, + snippetBodyChars: Schema.Int, + snippetContextChars: Schema.Int, + titleChars: Schema.Int, +}); +const ProjectionThreadContentSearchSource = Schema.Literals(["title", "user", "assistant"]); +const ProjectionThreadContentSearchOrigin = Schema.Literals(["legacy", "v2"]); +const ProjectionThreadContentSearchRow = Schema.Struct({ + threadId: ThreadId, + projectId: ProjectId, + threadTitle: Schema.String, + threadTitleTruncated: Schema.Number, + archived: Schema.Number, + source: ProjectionThreadContentSearchSource, + origin: ProjectionThreadContentSearchOrigin, + matchText: Schema.String, + matchTextChars: Schema.Int, + fragmentStart: Schema.Int, + matchedAt: IsoDateTime, + sourceThreadId: Schema.NullOr(ThreadId), + messageId: Schema.NullOr(MessageId), + runId: Schema.NullOr(RunId), + itemId: Schema.NullOr(TurnItemId), +}); const WorkspaceRootLookupInput = Schema.Struct({ workspaceRoot: Schema.String, }); @@ -259,6 +295,18 @@ function buildSearchSnippet(text: string, query: string): string { }`; } +function buildBoundedSearchSnippet(input: { + readonly fragment: string; + readonly fragmentStart: number; + readonly totalChars: number; +}): string { + const normalizedFragment = input.fragment.trim(); + const fragmentChars = Array.from(input.fragment).length; + const leading = input.fragmentStart > 1 ? "…" : ""; + const trailing = input.fragmentStart - 1 + fragmentChars < input.totalChars ? "…" : ""; + return `${leading}${normalizedFragment}${trailing}`; +} + function computeSnapshotSequence( stateRows: ReadonlyArray>, ): number { @@ -932,6 +980,219 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { `, }); + const searchThreadContentRows = SqlSchema.findAll({ + Request: ProjectionThreadContentSearchRequest, + Result: ProjectionThreadContentSearchRow, + execute: ({ + projectId, + threadId, + pattern, + queryText, + includeArchived, + limit, + offset, + snippetBodyChars, + snippetContextChars, + titleChars, + }) => + sql` + WITH v2_thread_meta AS ( + SELECT + threads.thread_id AS thread_id, + threads.project_id AS project_id, + threads.title AS thread_title, + threads.updated_at AS thread_updated_at, + CASE WHEN threads.archived_at IS NULL THEN 0 ELSE 1 END AS archived, + 'v2' AS thread_origin + FROM orchestration_v2_projection_threads AS threads + INNER JOIN projection_projects AS projects + ON projects.project_id = threads.project_id + WHERE threads.project_id = ${projectId} + AND threads.deleted_at IS NULL + AND projects.deleted_at IS NULL + AND (${includeArchived ? 1 : 0} = 1 OR threads.archived_at IS NULL) + ${threadId === null ? sql`` : sql`AND threads.thread_id = ${threadId}`} + ), + legacy_thread_meta AS ( + SELECT + threads.thread_id AS thread_id, + threads.project_id AS project_id, + threads.title AS thread_title, + threads.updated_at AS thread_updated_at, + CASE WHEN threads.archived_at IS NULL THEN 0 ELSE 1 END AS archived, + 'legacy' AS thread_origin + FROM projection_threads AS threads + INNER JOIN projection_projects AS projects + ON projects.project_id = threads.project_id + WHERE threads.project_id = ${projectId} + AND threads.deleted_at IS NULL + AND projects.deleted_at IS NULL + AND (${includeArchived ? 1 : 0} = 1 OR threads.archived_at IS NULL) + ${threadId === null ? sql`` : sql`AND threads.thread_id = ${threadId}`} + AND NOT EXISTS ( + SELECT 1 + FROM orchestration_v2_projection_threads AS v2_thread + WHERE v2_thread.thread_id = threads.thread_id + ) + ), + thread_meta AS ( + SELECT * FROM v2_thread_meta + UNION ALL + SELECT * FROM legacy_thread_meta + ), + candidate AS ( + SELECT + meta.thread_id, + meta.project_id, + meta.thread_title, + meta.archived, + 'title' AS source, + meta.thread_origin AS origin, + meta.thread_title AS match_text, + meta.thread_updated_at AS matched_at, + NULL AS source_thread_id, + NULL AS message_id, + NULL AS run_id, + NULL AS item_id, + 0 AS source_rank, + '' AS identity_key + FROM thread_meta AS meta + WHERE meta.thread_title LIKE ${pattern} ESCAPE '!' + + UNION ALL + + SELECT + meta.thread_id, + meta.project_id, + meta.thread_title, + meta.archived, + CASE messages.role WHEN 'user' THEN 'user' ELSE 'assistant' END AS source, + 'legacy' AS origin, + messages.text AS match_text, + messages.created_at AS matched_at, + NULL AS source_thread_id, + messages.message_id AS message_id, + NULL AS run_id, + NULL AS item_id, + CASE messages.role WHEN 'user' THEN 1 ELSE 2 END AS source_rank, + messages.message_id AS identity_key + FROM projection_thread_messages AS messages + INNER JOIN thread_meta AS meta + ON meta.thread_id = messages.thread_id + WHERE messages.is_streaming = 0 + AND ( + messages.role = 'user' + OR ( + messages.role = 'assistant' + AND messages.message_id IN ( + SELECT turns.assistant_message_id + FROM projection_turns AS turns + WHERE turns.assistant_message_id IS NOT NULL + ) + ) + ) + AND messages.text LIKE ${pattern} ESCAPE '!' + AND NOT EXISTS ( + SELECT 1 + FROM orchestration_v2_projection_turn_items AS duplicate_item + WHERE duplicate_item.thread_id = messages.thread_id + AND duplicate_item.type IN ('user_message', 'assistant_message') + AND json_extract(duplicate_item.payload_json, '$.messageId') = messages.message_id + ) + + UNION ALL + + SELECT + meta.thread_id, + meta.project_id, + meta.thread_title, + meta.archived, + CASE items.type WHEN 'user_message' THEN 'user' ELSE 'assistant' END AS source, + 'v2' AS origin, + json_extract(items.payload_json, '$.text') AS match_text, + messages.created_at AS matched_at, + items.thread_id AS source_thread_id, + messages.message_id AS message_id, + items.run_id AS run_id, + items.turn_item_id AS item_id, + CASE items.type WHEN 'user_message' THEN 1 ELSE 2 END AS source_rank, + items.turn_item_id AS identity_key + FROM orchestration_v2_projection_turn_items AS items + INNER JOIN thread_meta AS meta + ON meta.thread_id = items.thread_id + INNER JOIN orchestration_v2_projection_messages AS messages + ON messages.thread_id = items.thread_id + AND messages.message_id = json_extract(items.payload_json, '$.messageId') + LEFT JOIN orchestration_v2_projection_runs AS runs + ON runs.run_id = items.run_id + WHERE items.type IN ('user_message', 'assistant_message') + AND messages.role IN ('user', 'assistant') + AND messages.streaming = 0 + AND (items.run_id IS NULL OR runs.status IS NULL OR runs.status <> 'rolled_back') + AND NOT ( + items.type = 'user_message' + AND json_extract(items.payload_json, '$.inputIntent') = 'queued_turn' + AND EXISTS ( + SELECT 1 + FROM orchestration_v2_projection_runs AS cancelled_run + WHERE cancelled_run.run_id = items.run_id + AND cancelled_run.status = 'cancelled' + ) + ) + AND json_extract(items.payload_json, '$.text') LIKE ${pattern} ESCAPE '!' + ), + located AS ( + SELECT + candidate.*, + length(match_text) AS match_text_chars, + instr(lower(match_text), lower(${queryText})) AS match_index + FROM candidate + ), + windowed AS ( + SELECT + located.*, + min( + max( + 1, + CASE + WHEN match_index > ${snippetContextChars + 1} + THEN match_index - ${snippetContextChars} + ELSE 1 + END + ), + max(1, match_text_chars - ${snippetBodyChars} + 1) + ) AS fragment_start + FROM located + ) + SELECT + thread_id AS "threadId", + project_id AS "projectId", + substr(thread_title, 1, ${titleChars}) AS "threadTitle", + CASE WHEN length(thread_title) > ${titleChars} THEN 1 ELSE 0 END + AS "threadTitleTruncated", + archived, + source, + origin, + substr(match_text, fragment_start, ${snippetBodyChars}) AS "matchText", + match_text_chars AS "matchTextChars", + fragment_start AS "fragmentStart", + matched_at AS "matchedAt", + source_thread_id AS "sourceThreadId", + message_id AS "messageId", + run_id AS "runId", + item_id AS "itemId" + FROM windowed + ORDER BY + matched_at DESC, + thread_id ASC, + source_rank ASC, + origin ASC, + identity_key ASC + LIMIT ${limit} + OFFSET ${offset} + `, + }); + const getActiveProjectRowByWorkspaceRoot = SqlSchema.findOneOption({ Request: WorkspaceRootLookupInput, Result: ProjectionProjectLookupRowSchema, @@ -2520,6 +2781,129 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { }; }); + const boundedIntegerSearchInputError = ( + field: "limit" | "offset" | "snippetChars", + actual: number, + minimum: number, + maximum: number, + ): ProjectionThreadContentSearchInputError | null => { + if (!Number.isInteger(actual)) { + return new ProjectionThreadContentSearchInputError({ + field, + constraint: { + type: "integer", + ...(Number.isFinite(actual) ? { actual } : {}), + }, + }); + } + return actual < minimum || actual > maximum + ? new ProjectionThreadContentSearchInputError({ + field, + constraint: { type: "range", minimum, maximum, actual }, + }) + : null; + }; + + const searchThreadContent: ProjectionSnapshotQueryShape["searchThreadContent"] = Effect.fn( + "ProjectionSnapshotQuery.searchThreadContent", + )(function* (input) { + const limits = PROJECTION_THREAD_CONTENT_SEARCH_LIMITS; + if (input.query !== input.query.trim()) { + return yield* new ProjectionThreadContentSearchInputError({ + field: "query", + constraint: { type: "trimmed" }, + }); + } + if (input.query.includes("\0")) { + return yield* new ProjectionThreadContentSearchInputError({ + field: "query", + constraint: { type: "no_nul" }, + }); + } + const queryCodeUnits = input.query.length; + const queryChars = + queryCodeUnits <= limits.queryMaxChars * 2 ? Array.from(input.query).length : null; + if ( + queryChars === null || + queryChars < limits.queryMinChars || + queryChars > limits.queryMaxChars + ) { + return yield* new ProjectionThreadContentSearchInputError({ + field: "query", + constraint: { + type: "range", + minimum: limits.queryMinChars, + maximum: limits.queryMaxChars, + ...(queryChars === null ? {} : { actual: queryChars }), + }, + }); + } + const numericInputError = + boundedIntegerSearchInputError("limit", input.limit, 1, limits.pageMax) ?? + boundedIntegerSearchInputError("offset", input.offset, 0, limits.offsetMax) ?? + boundedIntegerSearchInputError( + "snippetChars", + input.snippetChars, + limits.snippetMinChars, + limits.snippetMaxChars, + ); + if (numericInputError !== null) { + return yield* numericInputError; + } + + const escapedQuery = escapeLikePattern(input.query); + const rows = yield* searchThreadContentRows({ + projectId: input.projectId, + threadId: input.threadId ?? null, + pattern: `%${escapedQuery}%`, + queryText: input.query, + includeArchived: input.includeArchived, + limit: input.limit + 1, + offset: input.offset, + snippetBodyChars: input.snippetChars - 2, + snippetContextChars: Math.floor((input.snippetChars - 2) / 3), + titleChars: limits.titleMaxChars, + }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.searchThreadContent:query", + "ProjectionSnapshotQuery.searchThreadContent:decodeRows", + ), + ), + ); + const hasMore = rows.length > input.limit; + const hits: ReadonlyArray = rows + .slice(0, input.limit) + .map((row) => ({ + threadId: row.threadId, + projectId: row.projectId, + threadTitle: row.threadTitle, + threadTitleTruncated: row.threadTitleTruncated === 1, + archived: row.archived === 1, + source: row.source, + origin: row.origin, + snippet: buildBoundedSearchSnippet({ + fragment: row.matchText, + fragmentStart: row.fragmentStart, + totalChars: row.matchTextChars, + }), + snippetTruncated: row.matchTextChars > input.snippetChars - 2, + matchedAt: row.matchedAt, + sourceThreadId: row.sourceThreadId, + messageId: row.messageId, + runId: row.runId, + itemId: row.itemId, + })); + return { + hits, + hasMore, + nextOffset: + hasMore && input.offset + hits.length <= limits.offsetMax + ? input.offset + hits.length + : null, + }; + }); + const getActiveProjectByWorkspaceRoot: ProjectionSnapshotQueryShape["getActiveProjectByWorkspaceRoot"] = (workspaceRoot) => getActiveProjectRowByWorkspaceRoot({ workspaceRoot }).pipe( @@ -3113,6 +3497,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { getShellSnapshotWithoutEnrichment, getArchivedShellSnapshot, searchThreads, + searchThreadContent, getSnapshotSequence, getCounts, getEventReplayStats, diff --git a/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts index caa8d618b37a..55a2cb34dcca 100644 --- a/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts @@ -6,7 +6,15 @@ * * @module ProjectionSnapshotQuery */ -import type { ApprovalRequestId, CheckpointRef, ProjectId, ThreadId } from "@t3tools/contracts"; +import type { + ApprovalRequestId, + CheckpointRef, + MessageId, + ProjectId, + RunId, + ThreadId, + TurnItemId, +} from "@t3tools/contracts"; import type { OrchestrationCheckpointSummary, OrchestrationProject, @@ -23,6 +31,7 @@ import type { } from "@t3tools/contracts/legacy-orchestration"; import * as Context from "effect/Context"; import type * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; import type * as Effect from "effect/Effect"; import type { ProjectionRepositoryError } from "../../persistence/Errors.ts"; @@ -67,6 +76,107 @@ export interface ProjectionThreadDetailQuery { readonly activityKinds?: ReadonlyArray; } +export type ProjectionThreadContentSearchSource = "title" | "user" | "assistant"; +export type ProjectionThreadContentSearchOrigin = "legacy" | "v2"; + +/** Text limits are Unicode code points, matching SQLite length/substr semantics. */ +export const PROJECTION_THREAD_CONTENT_SEARCH_LIMITS = { + queryMinChars: 2, + queryMaxChars: 200, + pageMax: 50, + offsetMax: 10_000, + snippetMinChars: 64, + snippetMaxChars: 1_000, + titleMaxChars: 500, +} as const; + +function unexpectedSearchConstraint(value: never): never { + value satisfies never; + throw new Error("Unexpected thread content search constraint."); +} + +export class ProjectionThreadContentSearchInputError extends Schema.TaggedErrorClass()( + "ProjectionThreadContentSearchInputError", + { + field: Schema.Literals(["query", "limit", "offset", "snippetChars"]), + constraint: Schema.Union([ + Schema.Struct({ type: Schema.Literal("trimmed") }), + Schema.Struct({ type: Schema.Literal("no_nul") }), + Schema.Struct({ + type: Schema.Literal("integer"), + actual: Schema.optional(Schema.Number), + }), + Schema.Struct({ + type: Schema.Literal("range"), + minimum: Schema.Number, + maximum: Schema.Number, + actual: Schema.optional(Schema.Number), + }), + ]), + }, +) { + override get message(): string { + const constraint = this.constraint; + switch (constraint.type) { + case "trimmed": + return "Thread content search query must not have leading or trailing whitespace."; + case "no_nul": + return "Thread content search query must not contain NUL."; + case "integer": + return `Thread content search ${this.field} must be an integer${ + constraint.actual === undefined ? "." : `; received ${constraint.actual}.` + }`; + case "range": + return `Thread content search ${this.field} must be between ${constraint.minimum} and ${constraint.maximum}${ + constraint.actual === undefined ? "." : `; received ${constraint.actual}.` + }`; + } + return unexpectedSearchConstraint(constraint); + } +} + +export interface ProjectionThreadContentSearchInput { + readonly projectId: ProjectId; + readonly threadId?: ThreadId; + /** Trimmed literal query, bounded in Unicode code points. */ + readonly query: string; + readonly includeArchived: boolean; + readonly offset: number; + readonly limit: number; + /** Maximum returned snippet length in Unicode code points. */ + readonly snippetChars: number; +} + +export interface ProjectionThreadContentSearchHit { + readonly threadId: ThreadId; + readonly projectId: ProjectId; + /** SQL-bounded display title measured in Unicode code points. */ + readonly threadTitle: string; + readonly threadTitleTruncated: boolean; + readonly archived: boolean; + readonly source: ProjectionThreadContentSearchSource; + readonly origin: ProjectionThreadContentSearchOrigin; + readonly snippet: string; + readonly snippetTruncated: boolean; + readonly matchedAt: string; + readonly sourceThreadId: ThreadId | null; + readonly messageId: MessageId | null; + readonly runId: RunId | null; + readonly itemId: TurnItemId | null; +} + +export interface ProjectionThreadContentSearchPage { + readonly hits: ReadonlyArray; + /** Whether more rows existed when this live query page was read. */ + readonly hasMore: boolean; + /** + * Offset for the next live query page, or null when complete or when the + * bounded traversal limit has been reached. This is not a snapshot cursor; + * concurrent projection changes can shift later pages. + */ + readonly nextOffset: number | null; +} + /** * ProjectionSnapshotQueryShape - Service API for read-model snapshots. */ @@ -135,6 +245,18 @@ export interface ProjectionSnapshotQueryShape { input: OrchestrationSearchThreadsInput, ) => Effect.Effect; + /** + * Search a bounded page of durable content in one project without hydrating + * thread projections. Unlike searchThreads, this opt-in query can include + * archived threads and returns stable V2 message anchors where available. + */ + readonly searchThreadContent: ( + input: ProjectionThreadContentSearchInput, + ) => Effect.Effect< + ProjectionThreadContentSearchPage, + ProjectionRepositoryError | ProjectionThreadContentSearchInputError + >; + /** * Read the latest projection snapshot sequence without hydrating read-model * entities.