Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions apps/sim/lib/knowledge/search/diagnostics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,8 @@ type SearchProviderSignal =
| 'slack.channel_filter_unresolved'
/** The channel lookup failed; the search kept its results without a note. */
| 'slack.channel_lookup_failed'
/** A Slack file read returned the preview instead of the file's text. */
| 'slack.file_preview'

const FAILURE_REASONS = [
'provider_validation',
Expand Down
4 changes: 2 additions & 2 deletions apps/sim/lib/sim-search/live/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ Each event must exist under the source's delegated token, be in an allowed calen

### Slack

Member mode only. Search uses the connected user's Slack real-time search grant and provider search syntax. Date bounds are sent as numeric provider timestamps; date ordering selects keyword retrieval and timestamp sort, because Slack semantic retrieval uses relevance ordering. Channels, messages, direct messages, and files are limited by that grant and provider scopes; there is no admin channel filter in member mode. Missing required search scopes produce a reconnect status. Thread/file reads use the same connected identity. Real-time Search returns at most 20 results a call and counts every page against the member's rate limit, so a search across every source reads one page; a Slack-scoped or native Slack search follows the cursor for up to three pages to fill its limit, and a later page that fails keeps the earlier pages and returns the unread cursor.
Member mode only. Search uses the connected user's Slack real-time search grant and provider search syntax. Date bounds are sent as numeric provider timestamps; date ordering selects keyword retrieval and timestamp sort, because Slack semantic retrieval uses relevance ordering. Channels, messages, direct messages, and files are limited by that grant and provider scopes; there is no admin channel filter in member mode. Missing required search scopes produce a reconnect status. Thread/file reads use the same connected identity. A file read downloads the full text of a canvas (HTML), text snippet, PDF or DOCX from `files.slack.com` only, never a remote file's own URL, within the shared response-size limit; when Slack will not serve it, the read returns the `files.info` preview. Real-time Search returns at most 20 results a call and counts every page against the member's rate limit, so a search across every source reads one page; a Slack-scoped or native Slack search follows the cursor for up to three pages to fill its limit, and a later page that fails keeps the earlier pages and returns the unread cursor.

### GitHub

Expand Down Expand Up @@ -131,7 +131,7 @@ Self-managed GitLab is resolved from the saved source's validated host/project i
| Drive | `GET /drive/v3/files` with Drive `q` | File metadata, Docs/Slides exports, Sheets values, or supported text media | Delegated Drive file visibility and configured folders/types |
| Gmail | `GET /gmail/v1/users/me/messages` with Gmail operators | Anchor plus up to seven independently authorized conversation messages | Same-mailbox delegation, labels, date range, category and custom-query checks |
| Calendar | CalendarList then `/calendars/{id}/events` | `/calendars/{id}/events/{eventId}` | Same-user delegation, selected calendars, event window/query |
| Slack | `POST /api/assistant.search.context` | `conversations.replies` or `files.info` preview | Member only; Slack enforces the connected user's grant |
| Slack | `POST /api/assistant.search.context` | `conversations.replies`, or the full text of a Slack-hosted canvas, text, PDF or DOCX file (`files.info` preview otherwise) | Member only; Slack enforces the connected user's grant |
| Jira | `POST /ex/jira/{cloudId}/rest/api/3/search/jql` | `/rest/api/3/issue/{key}` under that cloud site | Member only |
| Confluence | `/ex/confluence/{cloudId}/wiki/rest/api/search` with CQL | v2 `/wiki/api/v2/pages/{id}` or `/blogposts/{id}` (`body-format=view`); a space reads as its homepage | Same site, spaces, current type/status/labels, source readability |
| GitHub | `/search/issues`, `/search/code`, `/search/repositories`, `/search/commits` | Issue, repository, commit, or contents endpoint for returned kind | Added repositories; installation coverage/stable IDs and code filters |
Expand Down
94 changes: 3 additions & 91 deletions apps/sim/lib/sim-search/live/drive-content.ts
Original file line number Diff line number Diff line change
@@ -1,24 +1,10 @@
import { toStringOrNull } from '@sim/utils/coerce'
import { toRecord } from '@sim/utils/object'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { FileParserError, getFileParserErrorCode } from '@/lib/file-parsers/errors'
import { sniffFileKind } from '@/lib/file-parsers/sniff'
import type { FileParseResult } from '@/lib/file-parsers/types'
import {
assertOoxmlArchiveWithinLimits,
DEFAULT_OOXML_SIZE_LIMITS,
} from '@/lib/file-parsers/zip-guard'
import { documentExtension, extractDocumentText } from '@/lib/sim-search/live/file-text'
import { NATIVE_RESPONSE_MAX_BYTES, NativeSearchError, segment } from '@/lib/sim-search/live/http'
import type { NativeClient } from '@/lib/sim-search/live/types'

const MAX_DRIVE_TEXT_BYTES = 1024 * 1024
const MAX_DRIVE_PDF_PAGES = 100
const DRIVE_DOCX_LIMITS = {
...DEFAULT_OOXML_SIZE_LIMITS,
maxTotalUncompressedBytes: 10 * 1024 * 1024,
maxEntryUncompressedBytes: 4 * 1024 * 1024,
}

/** Extracts bounded text from the exact downloadable PDF or DOCX identified by Drive metadata. */
export async function readDriveFileContent(
client: NativeClient,
Expand All @@ -37,13 +23,7 @@ export async function readDriveFileContent(
'unavailable',
'Drive does not allow downloading this file. Ask the owner to allow downloads or open the source.'
)
const mimeType = toStringOrNull(row.mimeType)
const extension =
mimeType === 'application/pdf'
? 'pdf'
: mimeType === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
? 'docx'
: undefined
const extension = documentExtension(toStringOrNull(row.mimeType))
if (!extension)
throw new NativeSearchError(
'unavailable',
Expand Down Expand Up @@ -78,73 +58,5 @@ export async function readDriveFileContent(
)
throw error
}
signal?.throwIfAborted()
try {
if (bytes.byteLength === 0) throw new FileParserError('empty_input', 'Empty file')
if (bytes.byteLength > NATIVE_RESPONSE_MAX_BYTES)
throw new FileParserError('complexity_limit', 'File exceeds the binary read limit')
const kind = sniffFileKind(bytes, extension)
if (kind === 'encrypted-ooxml')
throw new FileParserError('encrypted_file', 'Encrypted document')
if (kind !== extension)
throw new FileParserError('invalid_format', 'File content does not match its Drive MIME type')

let parsed: FileParseResult
if (extension === 'pdf') {
const { PdfParser } = await import('@/lib/file-parsers/pdf-parser')
parsed = await new PdfParser().parseBuffer(bytes, {
signal,
pdfTextMode: 'complete',
pdfMaxPages: MAX_DRIVE_PDF_PAGES,
maxTextBytes: MAX_DRIVE_TEXT_BYTES,
})
} else {
assertOoxmlArchiveWithinLimits(bytes, DRIVE_DOCX_LIMITS)
const { DocxParser } = await import('@/lib/file-parsers/docx-parser')
parsed = await new DocxParser().parseBuffer(bytes, {
signal,
docxTextMode: 'complete',
maxTextBytes: MAX_DRIVE_TEXT_BYTES,
})
}
signal?.throwIfAborted()
if (parsed.metadata?.degraded || parsed.metadata?.truncated)
throw new FileParserError('complexity_limit', 'Complete document text was not extracted')
if (!parsed.content.trim())
throw new FileParserError('no_extractable_text', 'No document text was extracted')
if (Buffer.byteLength(parsed.content, 'utf8') > MAX_DRIVE_TEXT_BYTES)
throw new FileParserError('complexity_limit', 'Extracted text exceeds the live-read limit')
return parsed.content
} catch (error) {
signal?.throwIfAborted()
switch (getFileParserErrorCode(error)) {
case 'encrypted_file':
throw new NativeSearchError(
'unavailable',
'This file is password-protected. Remove the password from a copy or open the source.'
)
case 'empty_input':
case 'no_extractable_text':
throw new NativeSearchError(
'unavailable',
'No text could be extracted from this file. Image-only scans require OCR; open the source or provide a searchable copy.'
)
case 'complexity_limit':
throw new NativeSearchError(
'unavailable',
'This file exceeds live text extraction limits. Split it into smaller files or open the source.'
)
case 'invalid_format':
case 'unsupported_type':
throw new NativeSearchError(
'unavailable',
'The file content is malformed or does not match its PDF or DOCX type. Re-save it in a supported format or open the source.'
)
default:
throw new NativeSearchError(
'unavailable',
'The file could not be read as text. Retry or open the source.'
)
}
}
return extractDocumentText(bytes, extension, signal)
}
105 changes: 105 additions & 0 deletions apps/sim/lib/sim-search/live/file-text.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
import { FileParserError, getFileParserErrorCode } from '@/lib/file-parsers/errors'
import { sniffFileKind } from '@/lib/file-parsers/sniff'
import type { FileParseResult } from '@/lib/file-parsers/types'
import {
assertOoxmlArchiveWithinLimits,
DEFAULT_OOXML_SIZE_LIMITS,
} from '@/lib/file-parsers/zip-guard'
import { NATIVE_RESPONSE_MAX_BYTES, NativeSearchError } from '@/lib/sim-search/live/http'

/** Extracted text one live file read may return. */
export const MAX_FILE_TEXT_BYTES = 1024 * 1024
const MAX_FILE_PDF_PAGES = 100
const FILE_DOCX_LIMITS = {
...DEFAULT_OOXML_SIZE_LIMITS,
maxTotalUncompressedBytes: 10 * 1024 * 1024,
maxEntryUncompressedBytes: 4 * 1024 * 1024,
}

/** The document format `extractDocumentText` reads for a provider-declared MIME type. */
export function documentExtension(mimeType: string | null | undefined): 'pdf' | 'docx' | undefined {
if (mimeType === 'application/pdf') return 'pdf'
if (mimeType === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document')
return 'docx'
return undefined
}

/**
* Extracts the complete, bounded text of a downloaded PDF or DOCX whose bytes must match the type
* its provider declared. Every failure is a NativeSearchError naming what the reader can do.
*/
export async function extractDocumentText(
bytes: Buffer,
extension: 'pdf' | 'docx',
signal?: AbortSignal
): Promise<string> {
signal?.throwIfAborted()
try {
if (bytes.byteLength === 0) throw new FileParserError('empty_input', 'Empty file')
if (bytes.byteLength > NATIVE_RESPONSE_MAX_BYTES)
throw new FileParserError('complexity_limit', 'File exceeds the binary read limit')
const kind = sniffFileKind(bytes, extension)
if (kind === 'encrypted-ooxml')
throw new FileParserError('encrypted_file', 'Encrypted document')
if (kind !== extension)
throw new FileParserError('invalid_format', 'File content does not match its declared type')

let parsed: FileParseResult
if (extension === 'pdf') {
const { PdfParser } = await import('@/lib/file-parsers/pdf-parser')
parsed = await new PdfParser().parseBuffer(bytes, {
signal,
pdfTextMode: 'complete',
pdfMaxPages: MAX_FILE_PDF_PAGES,
maxTextBytes: MAX_FILE_TEXT_BYTES,
})
} else {
assertOoxmlArchiveWithinLimits(bytes, FILE_DOCX_LIMITS)
const { DocxParser } = await import('@/lib/file-parsers/docx-parser')
parsed = await new DocxParser().parseBuffer(bytes, {
signal,
docxTextMode: 'complete',
maxTextBytes: MAX_FILE_TEXT_BYTES,
})
}
signal?.throwIfAborted()
if (parsed.metadata?.degraded || parsed.metadata?.truncated)
throw new FileParserError('complexity_limit', 'Complete document text was not extracted')
if (!parsed.content.trim())
throw new FileParserError('no_extractable_text', 'No document text was extracted')
if (Buffer.byteLength(parsed.content, 'utf8') > MAX_FILE_TEXT_BYTES)
throw new FileParserError('complexity_limit', 'Extracted text exceeds the live-read limit')
return parsed.content
} catch (error) {
signal?.throwIfAborted()
switch (getFileParserErrorCode(error)) {
case 'encrypted_file':
throw new NativeSearchError(
'unavailable',
'This file is password-protected. Remove the password from a copy or open the source.'
)
case 'empty_input':
case 'no_extractable_text':
throw new NativeSearchError(
'unavailable',
'No text could be extracted from this file. Image-only scans require OCR; open the source or provide a searchable copy.'
)
case 'complexity_limit':
throw new NativeSearchError(
'unavailable',
'This file exceeds live text extraction limits. Split it into smaller files or open the source.'
)
case 'invalid_format':
case 'unsupported_type':
throw new NativeSearchError(
'unavailable',
'The file content is malformed or does not match its PDF or DOCX type. Re-save it in a supported format or open the source.'
)
default:
throw new NativeSearchError(
'unavailable',
'The file could not be read as text. Retry or open the source.'
)
}
}
}
4 changes: 2 additions & 2 deletions apps/sim/lib/sim-search/live/google.ts
Original file line number Diff line number Diff line change
Expand Up @@ -124,14 +124,14 @@ export async function readDrive(
} else if (document.kind === 'application/vnd.google-apps.spreadsheet') {
const metadata = object(
await client.json(`/v4/spreadsheets/${segment(id)}`, {
googleService: 'sheets',
service: 'sheets',
query: { fields: 'sheets.properties.title' },
})
)
const titles = array(metadata.sheets).map((sheet) => string(object(sheet.properties).title))
const values = object(
await client.json(`/v4/spreadsheets/${segment(id)}/values:batchGet`, {
googleService: 'sheets',
service: 'sheets',
query: {
ranges: titles.slice(0, 20).map((title) => `'${title.replace(/'/g, "''")}'!A1:AZ1000`),
valueRenderOption: 'FORMATTED_VALUE',
Expand Down
25 changes: 17 additions & 8 deletions apps/sim/lib/sim-search/live/http.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,16 +63,25 @@ describe('native search network boundary', () => {
await expect(client.json('https://evil.test')).rejects.toThrow('Invalid provider path')
expect(inputValidationMockFns.mockSecureFetchWithValidation).not.toHaveBeenCalled()
})
it('cannot move a non-Google token to the Sheets API', async () => {
const client = createNativeClient({
origin: 'https://api.github.com',
accessToken: 'private',
signal: new AbortController().signal,
it('only moves a token to the sibling host of its own provider', async () => {
const clientFor = (origin: string) =>
createNativeClient({ origin, accessToken: 'private', signal: new AbortController().signal })
await expect(
clientFor('https://api.github.com').json('/v4/spreadsheets/id', { service: 'sheets' })
).rejects.toThrow('Invalid provider service')
await expect(
clientFor('https://www.googleapis.com').text('/files-pri/T1-F1/plan', undefined, {
service: 'slackFiles',
})
).rejects.toThrow('Invalid provider service')
expect(inputValidationMockFns.mockSecureFetchWithValidation).not.toHaveBeenCalled()
await clientFor('https://slack.com').text('/files-pri/T1-F1/plan', undefined, {
service: 'slackFiles',
})
await expect(client.json('/v4/spreadsheets/id', { googleService: 'sheets' })).rejects.toThrow(
'Invalid Google service'
expect(inputValidationMockFns.mockSecureFetchWithValidation).toHaveBeenCalledWith(
'https://files.slack.com/files-pri/T1-F1/plan',
expect.objectContaining({ maxRedirects: 0 })
)
expect(inputValidationMockFns.mockSecureFetchWithValidation).not.toHaveBeenCalled()
})
it('keeps the HTTP status of a provider failure so callers can tell a missing item apart', async () => {
const client = createNativeClient({
Expand Down
Loading
Loading