diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier-error.mjs b/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier-error.mjs new file mode 100644 index 000000000000..7a70b22c7527 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier-error.mjs @@ -0,0 +1,48 @@ +import * as Sentry from '@sentry/node'; +import { Mastra } from '@mastra/core'; +import { Classifier } from '@mastra/core/classifier'; +import { Observability } from '@mastra/observability'; +import { SentryMastraExporter } from '@sentry/node'; + +// Inlined `EvaluationModelV4` mock: the ESM/CJS runner copies only this file into its temp dir. +const evaluationModel = { + specificationVersion: 'v4', + provider: 'typesafe-ai.evaluation', + modelId: 'jev-latest', + supportedQuestionTypes: ['choice', 'score', 'boolean'], + async doEvaluate() { + throw new Error('evaluation failed'); + }, +}; + +async function run() { + const classifier = new Classifier({ + id: 'request-classifier', + model: evaluationModel, + questions: { urgent: { type: 'boolean', instructions: 'Does this request need an immediate response?' } }, + }); + + const mastra = new Mastra({ + classifiers: { classifier }, + logger: false, + observability: new Observability({ + configs: { + default: { + serviceName: 'mastra-test', + exporters: [new SentryMastraExporter()], + }, + }, + }), + }); + + await Sentry.startSpan({ op: 'function', name: 'mastra-test' }, async () => { + await mastra + .getClassifier('classifier') + .evaluate({ state: { message: 'I was charged twice.' }, maxRetries: 0 }) + .catch(() => undefined); + }); + + await mastra.observability.shutdown(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier.mjs b/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier.mjs new file mode 100644 index 000000000000..e4e5efb15074 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier.mjs @@ -0,0 +1,51 @@ +import * as Sentry from '@sentry/node'; +import { Mastra } from '@mastra/core'; +import { Classifier } from '@mastra/core/classifier'; +import { Observability } from '@mastra/observability'; +import { SentryMastraExporter } from '@sentry/node'; + +// Inlined `EvaluationModelV4` mock: the ESM/CJS runner copies only this file into its temp dir. +const evaluationModel = { + specificationVersion: 'v4', + provider: 'typesafe-ai.evaluation', + modelId: 'jev-latest', + supportedQuestionTypes: ['choice', 'score', 'boolean'], + async doEvaluate() { + return { + answers: { urgent: { type: 'boolean', probability: 0.9 } }, + usage: { inputTokens: 30, outputTokens: 2 }, + warnings: [], + }; + }, +}; + +async function run() { + const classifier = new Classifier({ + id: 'request-classifier', + model: evaluationModel, + questions: { urgent: { type: 'boolean', instructions: 'Does this request need an immediate response?' } }, + }); + + const mastra = new Mastra({ + classifiers: { classifier }, + logger: false, + observability: new Observability({ + configs: { + default: { + serviceName: 'mastra-test', + exporters: [new SentryMastraExporter()], + }, + }, + }), + }); + + await Sentry.startSpan({ op: 'function', name: 'mastra-test' }, async () => { + await mastra + .getClassifier('classifier') + .evaluate({ state: { message: 'I was charged twice.', apiKey: 'sk-test-123' } }); + }); + + await mastra.observability.shutdown(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts index 281362c074f4..d26ea2281428 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts @@ -20,6 +20,7 @@ import { SENTRY_ORIGIN, URL_FULL, } from '@sentry/conventions/attributes'; +import { GEN_AI_CHAT, GEN_AI_EVALUATE, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; import { afterAll, expect } from 'vitest'; import { conditionalTest } from '../../../utils'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; @@ -43,6 +44,14 @@ const MASTRA_NESTING_DEPENDENCIES = { }, }; +// `Classifier` (the `@mastra/core/classifier` entry) ships in newer `@mastra/core` releases only. +const MASTRA_CLASSIFIER_DEPENDENCIES = { + additionalDependencies: { + '@mastra/core': '1.74.0', + '@mastra/observability': '1.18.3', + }, +}; + conditionalTest({ min: 22 })('Mastra integration', () => { afterAll(() => { cleanupChildProcesses(); @@ -63,7 +72,7 @@ conditionalTest({ min: 22 })('Mastra integration', () => { const agentSpan = spans.find(span => span.name === 'invoke_agent weather_agent')!; expect(agentSpan.status).toBe('ok'); - expect(agentSpan.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(agentSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_INVOKE_AGENT); expect(agentSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); expect(agentSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('invoke_agent'); expect(agentSpan.attributes[GEN_AI_AGENT_NAME].value).toBe('weather_agent'); @@ -75,7 +84,7 @@ conditionalTest({ min: 22 })('Mastra integration', () => { const chatSpan = spans.find(span => span.name === 'chat gpt-4o-mini')!; expect(chatSpan.status).toBe('ok'); - expect(chatSpan.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(chatSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_CHAT); expect(chatSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); expect(chatSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); expect(chatSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gpt-4o-mini'); @@ -157,7 +166,7 @@ conditionalTest({ min: 22 })('Mastra integration', () => { const toolSpan = spans.find(span => span.name === 'execute_tool get_weather')!; expect(toolSpan.status).toBe('ok'); - expect(toolSpan.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); + expect(toolSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EXECUTE_TOOL); expect(toolSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); expect(toolSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('execute_tool'); expect(toolSpan.attributes[GEN_AI_TOOL_NAME].value).toBe('get_weather'); @@ -196,7 +205,7 @@ conditionalTest({ min: 22 })('Mastra integration', () => { expect(spans.map(span => span.name)).toEqual(['invoke_agent math_workflow']); const workflowSpan = spans[0]!; - expect(workflowSpan.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(workflowSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_INVOKE_AGENT); expect(workflowSpan.attributes[GEN_AI_PIPELINE_NAME].value).toBe('math_workflow'); }, }) @@ -207,6 +216,98 @@ conditionalTest({ min: 22 })('Mastra integration', () => { MASTRA_DEPENDENCIES, ); + createEsmAndCjsTests( + __dirname, + 'scenario-classifier.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('maps a classifier evaluation to a gen_ai.evaluate span', async () => { + await createRunner() + .expect({ + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra'); + expect(spans.map(span => span.name)).toEqual(['evaluate jev-latest']); + + const evaluateSpan = spans[0]!; + expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE); + expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); + expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('evaluate'); + expect(evaluateSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('jev-latest'); + expect(evaluateSpan.attributes[GEN_AI_PROVIDER_NAME].value).toBe('typesafe-ai.evaluation'); + expect(evaluateSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(30); + expect(evaluateSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(2); + expect(evaluateSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(32); + expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); + expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); + }, + }) + .start() + .completed(); + }); + }, + MASTRA_CLASSIFIER_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-classifier.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('records the evaluated state, questions and answers when genAI recording is on', async () => { + await createRunner() + .expect({ + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); + const evaluateSpan = container.items.find(span => span.name === 'evaluate jev-latest')!; + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ + { + type: 'evaluation', + state: { message: 'I was charged twice.', apiKey: '[REDACTED]' }, + questions: { + urgent: { type: 'boolean', instructions: 'Does this request need an immediate response?' }, + }, + }, + ]); + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([ + { type: 'evaluation', answers: { urgent: { type: 'boolean', probability: 0.9 } } }, + ]); + }, + }) + .start() + .completed(); + }); + }, + MASTRA_CLASSIFIER_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-classifier-error.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('ends a failed classifier evaluation span with an error status', async () => { + await createRunner() + .ignore('event') + .expect({ + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra'); + expect(spans.map(span => span.name)).toEqual(['evaluate jev-latest']); + + const evaluateSpan = spans[0]!; + expect(evaluateSpan.status).toBe('error'); + expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value).toContain('I was charged twice.'); + expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); + }, + }) + .start() + .completed(); + }); + }, + MASTRA_CLASSIFIER_DEPENDENCIES, + ); + createEsmAndCjsTests( __dirname, 'scenario-auto.mjs', @@ -302,14 +403,14 @@ conditionalTest({ min: 22 })('Mastra integration', () => { ); expect(loadSpans.length).toBeGreaterThan(0); const cacheGetParentIds = loadSpans.map(span => span.parent_span_id); - const chat = container.items.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.chat')!; + const chat = container.items.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_CHAT)!; expect(chat).toBeDefined(); const chatSpanId = chat.span_id; const executeTool = container.items.find( span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'count_items', )!; - expect(executeTool.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); + expect(executeTool.attributes[SENTRY_OP]?.value).toBe(GEN_AI_EXECUTE_TOOL); const executeToolSpanId = executeTool.span_id; // The `executeWithContext` bridge makes the exporter spans active during the real work, so the @@ -355,14 +456,14 @@ conditionalTest({ min: 22 })('Mastra integration', () => { ); expect(loadSpans.length).toBeGreaterThan(0); const cacheGetParentIds = loadSpans.map(span => span.parent_span_id); - const chat = container.items.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.chat')!; + const chat = container.items.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_CHAT)!; expect(chat).toBeDefined(); const chatSpanId = chat.span_id; const executeTool = container.items.find( span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'count_items', )!; - expect(executeTool.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); + expect(executeTool.attributes[SENTRY_OP]?.value).toBe(GEN_AI_EXECUTE_TOOL); const executeToolSpanId = executeTool.span_id; expect(chatSpanId).toBeDefined(); diff --git a/packages/server-utils/src/ai/core/utils.ts b/packages/server-utils/src/ai/core/utils.ts index 49f76fb18998..93949ec40eca 100644 --- a/packages/server-utils/src/ai/core/utils.ts +++ b/packages/server-utils/src/ai/core/utils.ts @@ -136,6 +136,16 @@ export function getTokenUsageAttributes( return attributes; } +/** Serialize the `state` and `questions` of an evaluation request (TypeSafe, Workers AI, Mastra classifiers). */ +export function getEvaluationInputMessages(request: Record): string | undefined { + return stringify([{ type: 'evaluation', state: request.state, questions: request.questions }]); +} + +/** Serialize the `answers` of an evaluation result (TypeSafe, Workers AI, Mastra classifiers). */ +export function getEvaluationOutputMessages(answers: unknown): string | undefined { + return stringify([{ type: 'evaluation', answers }]); +} + /** One assistant turn for {@link setOutputMessagesAttribute}. */ export interface GenAiOutputMessage { /** The message's text content, already flattened out of any content-part array. */ diff --git a/packages/server-utils/src/ai/mastra/classifier-evaluation.ts b/packages/server-utils/src/ai/mastra/classifier-evaluation.ts new file mode 100644 index 000000000000..952748b89512 --- /dev/null +++ b/packages/server-utils/src/ai/mastra/classifier-evaluation.ts @@ -0,0 +1,58 @@ +import type { Span, SpanTimeInput } from '@sentry/core'; +import type { MastraExportedSpan, MastraSpanOutputProcessor } from './types'; + +/** + * Links a `Classifier.evaluate()` call to the Sentry span the exporter opens for its + * `classifier_evaluation` span. Mastra's span carries no input or output, so the integration adds + * them from the call. Mastra ends its span before `evaluate()` returns the answers, so the exporter + * leaves the Sentry span open and the integration ends it once the call settles. + */ +export interface ClassifierEvaluationCall { + span?: Span; + /** Mastra's span and the processors of its observability instance, to filter the call's data. */ + mastraSpan?: MastraExportedSpan; + spanOutputProcessors?: MastraSpanOutputProcessor[]; + /** The recording options of the exporter that opened the span, so the call's data follows them. */ + recordInputs?: boolean; + recordOutputs?: boolean; + /** Mastra's end time, set when Mastra ended its span before the call settled. */ + endTime?: SpanTimeInput; + settled?: boolean; +} + +// Mastra emits `span_started` synchronously inside `evaluate()`, before its first `await`, so at +// most one call is ever in this window. +let startingCall: ClassifierEvaluationCall | undefined; + +export function setStartingClassifierEvaluation(call: ClassifierEvaluationCall | undefined): void { + startingCall = call; +} + +export function takeStartingClassifierEvaluation(): ClassifierEvaluationCall | undefined { + const call = startingCall; + startingCall = undefined; + return call; +} + +/** + * Run the call's data through Mastra's span output processors (for example its default + * `SensitiveDataFilter`), as Mastra does for the input and output of its own spans. Returns `undefined` + * when a processor drops the span or throws, so unfiltered data is never recorded. + */ +export function processClassifierEvaluationData( + call: ClassifierEvaluationCall, + data: { input?: unknown; output?: unknown }, +): { input?: unknown; output?: unknown } | undefined { + let span: MastraExportedSpan | undefined = { ...call.mastraSpan, ...data } as MastraExportedSpan; + for (const processor of call.spanOutputProcessors ?? []) { + try { + span = processor.process(span); + } catch { + return undefined; + } + if (!span) { + return undefined; + } + } + return span; +} diff --git a/packages/server-utils/src/ai/mastra/constants.ts b/packages/server-utils/src/ai/mastra/constants.ts index 26a05910c45d..ab55142a2bf5 100644 --- a/packages/server-utils/src/ai/mastra/constants.ts +++ b/packages/server-utils/src/ai/mastra/constants.ts @@ -1,4 +1,10 @@ -import { GEN_AI_CHAT, GEN_AI_EMBEDDINGS, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; +import { + GEN_AI_CHAT, + GEN_AI_EMBEDDINGS, + GEN_AI_EVALUATE, + GEN_AI_EXECUTE_TOOL, + GEN_AI_INVOKE_AGENT, +} from '@sentry/conventions/op'; import type { MastraSpanType } from './types'; export const MASTRA_INTEGRATION_NAME = 'Mastra' as const; @@ -36,6 +42,7 @@ export const SPAN_TYPE_OPS: Readonly = new Set([ diff --git a/packages/server-utils/src/ai/mastra/index.ts b/packages/server-utils/src/ai/mastra/index.ts index 0bfcdd712777..7943275fe6bd 100644 --- a/packages/server-utils/src/ai/mastra/index.ts +++ b/packages/server-utils/src/ai/mastra/index.ts @@ -7,6 +7,7 @@ import { LRUMap, SPAN_STATUS_ERROR, startInactiveSpan, + timestampInSeconds, } from '@sentry/core'; import { GEN_AI_RESPONSE_MODEL, SENTRY_OP, SENTRY_ORIGIN } from '@sentry/conventions/attributes'; import { DEBUG_BUILD } from '../../debug-build'; @@ -19,6 +20,7 @@ import { getUsageAttributes, isExportedSpanType, mergeUsageAttributes, + type AttributeRecordingOptions, type SpanAttributes, } from './utils'; import { @@ -29,8 +31,17 @@ import { MAX_TRACKED_MASTRA_SPANS, MODEL_SPAN_TYPES, } from './constants'; +import type { ClassifierEvaluationCall } from './classifier-evaluation'; +import { takeStartingClassifierEvaluation } from './classifier-evaluation'; import { registerMastraSpan, unregisterMastraSpan } from './span-registry'; -import type { MastraExportedSpan, MastraObservabilityExporter, MastraSpanType, MastraTracingEvent } from './types'; +import type { + MastraExportedSpan, + MastraExporterInitOptions, + MastraObservabilityExporter, + MastraSpanOutputProcessor, + MastraSpanType, + MastraTracingEvent, +} from './types'; export type MastraExporterOptions = GenAiOptions; @@ -38,6 +49,7 @@ interface TrackedSpan { span: Span; spanType: MastraSpanType; usage: SpanAttributes; + evaluation?: ClassifierEvaluationCall; } const FLUSH_TIMEOUT_MS = 2000; @@ -64,11 +76,17 @@ export class SentryMastraExporter implements MastraObservabilityExporter { */ private readonly _skipped = new LRUMap(MAX_TRACKED_MASTRA_SPANS); private readonly _options: MastraExporterOptions; + private _spanOutputProcessors: MastraSpanOutputProcessor[] = []; public constructor(options: MastraExporterOptions = {}) { this._options = options; } + /** Called by Mastra with the config of the observability instance this exporter belongs to. */ + public init(options: MastraExporterInitOptions): void { + this._spanOutputProcessors = options.config?.spanOutputProcessors ?? []; + } + /** Mastra's interface is async; the work is synchronous. */ public async exportTracingEvent(event: MastraTracingEvent): Promise { if (!getClient()) { @@ -117,20 +135,22 @@ export class SentryMastraExporter implements MastraObservabilityExporter { return; } + // Per event: the exporter can be constructed before `Sentry.init()`, when `dataCollection.genAI` does not exist yet. + const recordingOptions = resolveAIRecordingOptions(this._options); switch (event.type) { case 'span_started': - this._onSpanStarted(span); + this._onSpanStarted(span, recordingOptions); break; case 'span_updated': - this._onSpanUpdated(span); + this._onSpanUpdated(span, recordingOptions); break; case 'span_ended': - this._onSpanEnded(span); + this._onSpanEnded(span, recordingOptions); break; } } - private _onSpanStarted(span: MastraExportedSpan): void { + private _onSpanStarted(span: MastraExportedSpan, recordingOptions: AttributeRecordingOptions): void { const parentId = this._resolveParentId(span.parentSpanId); const parentSpan = parentId ? this._spans.get(parentId)?.span : undefined; const activeSpan = getActiveSpan(); @@ -141,13 +161,24 @@ export class SentryMastraExporter implements MastraObservabilityExporter { // Prefer the Mastra parent so the tree stays together; else the active request span. parentSpan: parentSpan ?? activeSpan, attributes: { - ...this._attributesFor(span), + ...getSpanAttributes(span, recordingOptions), [SENTRY_OP]: getOperation(span.type)?.op, [SENTRY_ORIGIN]: MASTRA_ORIGIN, }, }); - this._trackSpan(span.id, { span: sentrySpan, spanType: span.type, usage: {} }); + const evaluation = span.type === 'classifier_evaluation' ? takeStartingClassifierEvaluation() : undefined; + if (evaluation) { + Object.assign(evaluation, { + span: sentrySpan, + mastraSpan: span, + spanOutputProcessors: this._spanOutputProcessors, + recordInputs: recordingOptions.recordInputs, + recordOutputs: recordingOptions.recordOutputs, + }); + } + + this._trackSpan(span.id, { span: sentrySpan, spanType: span.type, usage: {}, evaluation }); } /** Track a started span, ending any Sentry span that would otherwise be dropped without `end()`. */ @@ -174,14 +205,14 @@ export class SentryMastraExporter implements MastraObservabilityExporter { return this._spans.remove(id); } - private _onSpanUpdated(span: MastraExportedSpan): void { + private _onSpanUpdated(span: MastraExportedSpan, recordingOptions: AttributeRecordingOptions): void { const tracked = this._spans.get(span.id); if (tracked) { - tracked.span.setAttributes(this._attributesFor(span)); + tracked.span.setAttributes(getSpanAttributes(span, recordingOptions)); } } - private _onSpanEnded(span: MastraExportedSpan): void { + private _onSpanEnded(span: MastraExportedSpan, recordingOptions: AttributeRecordingOptions): void { const tracked = this._spans.get(span.id); if (!tracked) { DEBUG_BUILD && debug.warn(`[Mastra] no Sentry span open for ended span ${span.id} (${span.name})`); @@ -189,7 +220,7 @@ export class SentryMastraExporter implements MastraObservabilityExporter { } const { span: sentrySpan } = tracked; - sentrySpan.setAttributes(this._attributesFor(span)); + sentrySpan.setAttributes(getSpanAttributes(span, recordingOptions)); sentrySpan.updateName(getSpanName(span)); if (MODEL_SPAN_TYPES.has(span.type)) { @@ -201,7 +232,13 @@ export class SentryMastraExporter implements MastraObservabilityExporter { sentrySpan.setStatus({ code: SPAN_STATUS_ERROR, message: span.errorInfo.message }); } - sentrySpan.end(span.endTime); + const { evaluation } = tracked; + if (evaluation && !evaluation.settled) { + // The integration adds the call's input and answers once `evaluate()` settles, then ends the span. + evaluation.endTime = span.endTime ?? timestampInSeconds(); + } else { + sentrySpan.end(span.endTime); + } this._removeTracked(span.id); } @@ -240,9 +277,4 @@ export class SentryMastraExporter implements MastraObservabilityExporter { } return undefined; } - - private _attributesFor(span: MastraExportedSpan): ReturnType { - // Per event: the exporter can be constructed before `Sentry.init()`, when `dataCollection.genAI` does not exist yet. - return getSpanAttributes(span, resolveAIRecordingOptions(this._options)); - } } diff --git a/packages/server-utils/src/ai/mastra/types.ts b/packages/server-utils/src/ai/mastra/types.ts index 35d125644346..5de6ad91efcc 100644 --- a/packages/server-utils/src/ai/mastra/types.ts +++ b/packages/server-utils/src/ai/mastra/types.ts @@ -15,6 +15,7 @@ export type MastraSpanType = | 'provider_tool_call' | 'client_tool_call' | 'rag_embedding' + | 'classifier_evaluation' | (string & {}); export type MastraTracingEventType = 'span_started' | 'span_updated' | 'span_ended'; @@ -58,6 +59,8 @@ export interface MastraSpanAttributes { stopSequences?: string[]; seed?: number; }; + // classifier_evaluation + modelId?: string; // tool_call / mcp_tool_call / provider_tool_call / client_tool_call toolDescription?: string; // agent_run / workflow_run @@ -69,6 +72,7 @@ export interface MastraSpanAttributes { export interface MastraExportedSpan { id: string; + traceId?: string; parentSpanId?: string; /** Mastra's own name; unused for Sentry span names. */ name: string; @@ -91,6 +95,17 @@ export interface MastraTracingEvent { exportedSpan: MastraExportedSpan; } +/** Mastra's `SpanOutputProcessor`, such as its default `SensitiveDataFilter`. */ +export interface MastraSpanOutputProcessor { + name: string; + process(span: MastraExportedSpan): MastraExportedSpan | undefined; +} + +/** Subset of what Mastra passes to `ObservabilityExporter.init()`. */ +export interface MastraExporterInitOptions { + config?: { spanOutputProcessors?: MastraSpanOutputProcessor[] }; +} + /** Subset of Mastra's `ObservabilityExporter`. The rest of that interface is optional. */ export interface MastraObservabilityExporter { name: string; diff --git a/packages/server-utils/src/ai/mastra/utils.ts b/packages/server-utils/src/ai/mastra/utils.ts index 0cdb20e1b80f..5fa9f1b62316 100644 --- a/packages/server-utils/src/ai/mastra/utils.ts +++ b/packages/server-utils/src/ai/mastra/utils.ts @@ -61,14 +61,21 @@ export function getOperation(spanType: MastraSpanType): { op: string; operationN */ export function getSpanName(span: MastraExportedSpan): string { const operationName = getOperation(span.type)?.operationName ?? span.type; - const identifier = - MODEL_SPAN_TYPES.has(span.type) || span.type === 'rag_embedding' - ? span.attributes?.model - : (span.entityName ?? span.entityId); + const identifier = getSpanIdentifier(span); return identifier ? `${operationName} ${identifier}` : operationName; } +function getSpanIdentifier(span: MastraExportedSpan): string | undefined { + if (span.type === 'classifier_evaluation') { + return span.attributes?.modelId; + } + if (MODEL_SPAN_TYPES.has(span.type) || span.type === 'rag_embedding') { + return span.attributes?.model; + } + return span.entityName ?? span.entityId; +} + function serialize(value: unknown): string | undefined { if (value === undefined || value === null) { return undefined; @@ -157,6 +164,10 @@ export function getSpanAttributes(span: MastraExportedSpan, options: AttributeRe addToolAttributes(attributes, span, options); } + if (span.type === 'classifier_evaluation') { + addEvaluationAttributes(attributes, span); + } + if (span.errorInfo) { attributes[ERROR_TYPE] = span.errorInfo.name ?? span.errorInfo.id; } @@ -243,6 +254,15 @@ function addToolAttributes( } } +/** Mastra's `Classifier` sets no input or output on this span, so only the model and usage are known. */ +function addEvaluationAttributes(attributes: SpanAttributes, span: MastraExportedSpan): void { + const attrs = span.attributes ?? {}; + + attributes[GEN_AI_REQUEST_MODEL] = attrs.modelId; + attributes[GEN_AI_PROVIDER_NAME] = attrs.provider; + Object.assign(attributes, getUsageAttributes(attrs.usage)); +} + function responseText(output: unknown): string | undefined { if (typeof output === 'string') { return output; diff --git a/packages/server-utils/src/ai/typesafe/index.ts b/packages/server-utils/src/ai/typesafe/index.ts index 9552e5e96a4d..99280c6843ad 100644 --- a/packages/server-utils/src/ai/typesafe/index.ts +++ b/packages/server-utils/src/ai/typesafe/index.ts @@ -12,9 +12,9 @@ import { } from '@sentry/conventions/attributes'; import { GEN_AI_EVALUATE } from '@sentry/conventions/op'; import type { Span, SpanAttributes } from '@sentry/core'; -import { isObjectLike, SPAN_STATUS_ERROR, startInactiveSpan, stringify, withActiveSpan } from '@sentry/core'; +import { isObjectLike, SPAN_STATUS_ERROR, startInactiveSpan, withActiveSpan } from '@sentry/core'; import type { GenAiOptions } from '../core/utils'; -import { resolveAIRecordingOptions } from '../core/utils'; +import { getEvaluationInputMessages, getEvaluationOutputMessages, resolveAIRecordingOptions } from '../core/utils'; import { TYPESAFE_ORIGIN, TYPESAFE_PROVIDER_NAME } from './constants'; /** @@ -54,11 +54,6 @@ function getRequestAttributes( }; } -/** Serialize the `state` and `questions` of an evaluation request. Also used for TypeSafe models on Workers AI. */ -export function getEvaluationInputMessages(request: Record): string | undefined { - return stringify([{ type: 'evaluation', state: request.state, questions: request.questions }]); -} - /** * Add the response model, token usage and (optionally) the answers of a `systemOne` result. * Also used for TypeSafe models on Workers AI. @@ -86,7 +81,7 @@ export function addResponseAttributes(span: Span, result: unknown, recordOutputs } if (recordOutputs && result.answers !== undefined) { - span.setAttribute(GEN_AI_OUTPUT_MESSAGES, stringify([{ type: 'evaluation', answers: result.answers }])); + span.setAttribute(GEN_AI_OUTPUT_MESSAGES, getEvaluationOutputMessages(result.answers)); } } diff --git a/packages/server-utils/src/ai/workers-ai/utils.ts b/packages/server-utils/src/ai/workers-ai/utils.ts index 66dd3d567c8c..73da7883b12e 100644 --- a/packages/server-utils/src/ai/workers-ai/utils.ts +++ b/packages/server-utils/src/ai/workers-ai/utils.ts @@ -20,8 +20,13 @@ import { GEN_AI_CHAT, GEN_AI_EMBEDDINGS, GEN_AI_EVALUATE } from '@sentry/convent import { isObjectLike, stringify } from '@sentry/core'; import type { Span, SpanAttributeValue } from '@sentry/core'; import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../core/gen-ai-attributes'; -import { extractSystemInstructions, getTokenUsageAttributes, setOutputMessagesAttribute } from '../core/utils'; -import { addResponseAttributes as addEvaluateResponseAttributes, getEvaluationInputMessages } from '../typesafe'; +import { + extractSystemInstructions, + getEvaluationInputMessages, + getTokenUsageAttributes, + setOutputMessagesAttribute, +} from '../core/utils'; +import { addResponseAttributes as addEvaluateResponseAttributes } from '../typesafe'; // Re-exported so `workers-ai/streaming.ts` keeps importing it from this module. export { setOutputMessagesAttribute }; import { WORKERS_AI_ORIGIN, WORKERS_AI_PROVIDER_NAME } from './constants'; diff --git a/packages/server-utils/src/integrations/mastra-classifier.ts b/packages/server-utils/src/integrations/mastra-classifier.ts new file mode 100644 index 000000000000..d25434bb2c9d --- /dev/null +++ b/packages/server-utils/src/integrations/mastra-classifier.ts @@ -0,0 +1,73 @@ +import { GEN_AI_INPUT_MESSAGES, GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventions/attributes'; +import { isObjectLike } from '@sentry/core'; +import type { ClassifierEvaluationCall } from '../ai/mastra/classifier-evaluation'; +import { processClassifierEvaluationData, setStartingClassifierEvaluation } from '../ai/mastra/classifier-evaluation'; +import { getEvaluationInputMessages, getEvaluationOutputMessages } from '../ai/core/utils'; +import { CHANNELS } from '../orchestrion/channels'; +import { safeChannelCallback } from '../tracing-channel'; +import * as diagnosticsChannel from '../utils/diagnosticsChannel'; + +interface ClassifierEvaluateChannelContext { + // `evaluate({ state, questions? })`; `self` is the `Classifier`. + arguments: unknown[]; + self?: unknown; + result?: unknown; +} + +/** + * Add the evaluated state, questions and answers of a `Classifier.evaluate()` call to the exporter's + * `classifier_evaluation` span, which Mastra leaves without input or output. + */ +export function recordClassifierEvaluations(): void { + const channel = diagnosticsChannel.tracingChannel( + CHANNELS.MASTRA_CLASSIFIER_EVALUATE, + ); + const calls = new WeakMap(); + + channel.start.subscribe(message => { + safeChannelCallback(() => { + const call: ClassifierEvaluationCall = {}; + calls.set(message as object, call); + setStartingClassifierEvaluation(call); + }); + }); + // `evaluate()` has reached its first `await`, so Mastra has already started its span. + channel.end.subscribe(() => { + safeChannelCallback(() => setStartingClassifierEvaluation(undefined)); + }); + channel.asyncEnd.subscribe(message => { + safeChannelCallback(() => { + const call = calls.get(message as object); + if (call) { + finishClassifierEvaluation(call, message as ClassifierEvaluateChannelContext); + } + }); + }); +} + +function finishClassifierEvaluation(call: ClassifierEvaluationCall, message: ClassifierEvaluateChannelContext): void { + call.settled = true; + const { span, recordInputs, recordOutputs } = call; + if (!span) { + return; + } + + const params = isObjectLike(message.arguments[0]) ? message.arguments[0] : {}; + // Questions given to the constructor take precedence over the ones passed to `evaluate()`, as in Mastra. + const questions = (isObjectLike(message.self) ? message.self.questions : undefined) ?? params.questions; + const data = processClassifierEvaluationData(call, { + input: recordInputs ? { state: params.state, questions } : undefined, + output: recordOutputs && isObjectLike(message.result) ? message.result.answers : undefined, + }); + + if (isObjectLike(data?.input)) { + span.setAttribute(GEN_AI_INPUT_MESSAGES, getEvaluationInputMessages(data.input)); + } + if (data?.output !== undefined) { + span.setAttribute(GEN_AI_OUTPUT_MESSAGES, getEvaluationOutputMessages(data.output)); + } + + if (call.endTime) { + span.end(call.endTime); + } +} diff --git a/packages/server-utils/src/integrations/mastra.ts b/packages/server-utils/src/integrations/mastra.ts index 9856ef51651d..1f541fe83303 100644 --- a/packages/server-utils/src/integrations/mastra.ts +++ b/packages/server-utils/src/integrations/mastra.ts @@ -24,6 +24,7 @@ import type { MastraObservabilityExporter } from '../ai/mastra/types'; import { DEBUG_BUILD } from '../debug-build'; import { CHANNELS } from '../orchestrion/channels'; import { mastraModuleNames } from '../orchestrion/config/mastra'; +import { recordClassifierEvaluations } from './mastra-classifier'; import { invokeOrchestrionInstrumentation } from '../orchestrion/instrumentation'; import { bindSpanToChannelStore, safeChannelCallback } from '../tracing-channel'; @@ -96,6 +97,7 @@ function instrumentExporter(options: MastraOptions): void { }); captureExecuteWithContextErrors(); + recordClassifierEvaluations(); } /** diff --git a/packages/server-utils/src/orchestrion/config/mastra.ts b/packages/server-utils/src/orchestrion/config/mastra.ts index dfa6ee151cbf..3e5c0d53c658 100644 --- a/packages/server-utils/src/orchestrion/config/mastra.ts +++ b/packages/server-utils/src/orchestrion/config/mastra.ts @@ -41,11 +41,30 @@ const mastraContextConfig: InstrumentationConfig[] = ['executeWithContext', 'exe }), ); -export const mastraConfig = [...mastraConstructorConfig, ...mastraContextConfig] satisfies InstrumentationConfig[]; +// Mastra's `classifier_evaluation` span carries no input or output, so we read them from the +// `Classifier.evaluate()` call. `Classifier` lives in the stable, unhashed `classifier` entry. +const mastraClassifierConfig: InstrumentationConfig[] = [ + { + channelName: 'mastraClassifierEvaluate', + module: { + name: '@mastra/core', + versionRange: '>=1.63.2 <2.0.0', + filePath: /^dist\/classifier\/index\.(?:cjs|mjs|js)$/, + }, + functionQuery: { className: 'Classifier', methodName: 'evaluate', kind: 'Async' }, + }, +]; + +export const mastraConfig = [ + ...mastraConstructorConfig, + ...mastraContextConfig, + ...mastraClassifierConfig, +] satisfies InstrumentationConfig[]; export const mastraModuleNames = getModuleNames(mastraConfig); export const mastraChannels = { MASTRA_CONSTRUCTOR: 'orchestrion:@mastra/core:mastraConstructor', MASTRA_EXECUTE_WITH_CONTEXT: 'orchestrion:@mastra/core:mastraExecuteWithContext', + MASTRA_CLASSIFIER_EVALUATE: 'orchestrion:@mastra/core:mastraClassifierEvaluate', } as const; diff --git a/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts b/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts index 5265272ffa0f..844cc9e82514 100644 --- a/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts @@ -1,4 +1,37 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { + ERROR_TYPE, + GEN_AI_AGENT_NAME, + GEN_AI_CONVERSATION_ID, + GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PIPELINE_NAME, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_REQUEST_STOP_SEQUENCES, + GEN_AI_RESPONSE_MODEL, + GEN_AI_RESPONSE_TEXT, + GEN_AI_RESPONSE_TOOL_CALLS, + GEN_AI_SYSTEM_INSTRUCTIONS, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_NAME, + GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, + GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { + GEN_AI_CHAT, + GEN_AI_EMBEDDINGS, + GEN_AI_EVALUATE, + GEN_AI_EXECUTE_TOOL, + GEN_AI_INVOKE_AGENT, +} from '@sentry/conventions/op'; import type { Span } from '@sentry/core'; import { _INTERNAL_clearAiProviderSkips, @@ -8,6 +41,8 @@ import { spanToStaticSpanJSON, } from '@sentry/core'; import { SentryMastraExporter } from '../../../../src/ai/mastra'; +import type { ClassifierEvaluationCall } from '../../../../src/ai/mastra/classifier-evaluation'; +import { setStartingClassifierEvaluation } from '../../../../src/ai/mastra/classifier-evaluation'; import type { MastraExportedSpan, MastraSpanType, MastraTracingEvent } from '../../../../src/ai/mastra/types'; import { OPENAI_INTEGRATION_NAME } from '../../../../src/ai/openai/constants'; import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; @@ -81,11 +116,11 @@ describe('SentryMastraExporter', () => { const json = spanToStaticSpanJSON(endedSpans[0]!); expect(json.description).toBe('invoke_agent weather_agent'); - expect(json.data['sentry.op']).toBe('gen_ai.invoke_agent'); - expect(json.data['sentry.origin']).toBe('auto.ai.mastra'); - expect(json.data['gen_ai.operation.name']).toBe('invoke_agent'); - expect(json.data['gen_ai.agent.name']).toBe('weather_agent'); - expect(json.data['gen_ai.pipeline.name']).toBe('weather_agent'); + expect(json.data[SENTRY_OP]).toBe(GEN_AI_INVOKE_AGENT); + expect(json.data[SENTRY_ORIGIN]).toBe('auto.ai.mastra'); + expect(json.data[GEN_AI_OPERATION_NAME]).toBe('invoke_agent'); + expect(json.data[GEN_AI_AGENT_NAME]).toBe('weather_agent'); + expect(json.data[GEN_AI_PIPELINE_NAME]).toBe('weather_agent'); }); it('names generation spans `chat {model}` and maps usage to the current conventions', async () => { @@ -108,17 +143,61 @@ describe('SentryMastraExporter', () => { const json = spanToStaticSpanJSON(endedSpans[0]!); expect(json.description).toBe('chat gpt-5'); - expect(json.data['sentry.op']).toBe('gen_ai.chat'); - expect(json.data['gen_ai.provider.name']).toBe('openai'); - expect(json.data['gen_ai.usage.input_tokens']).toBe(10); - expect(json.data['gen_ai.usage.output_tokens']).toBe(4); - expect(json.data['gen_ai.usage.total_tokens']).toBe(14); - expect(json.data['gen_ai.usage.reasoning.output_tokens']).toBe(3); - expect(json.data['gen_ai.usage.cache_read.input_tokens']).toBe(2); - expect(json.data['gen_ai.usage.cache_creation.input_tokens']).toBe(5); + expect(json.data[SENTRY_OP]).toBe(GEN_AI_CHAT); + expect(json.data[GEN_AI_PROVIDER_NAME]).toBe('openai'); + expect(json.data[GEN_AI_USAGE_INPUT_TOKENS]).toBe(10); + expect(json.data[GEN_AI_USAGE_OUTPUT_TOKENS]).toBe(4); + expect(json.data[GEN_AI_USAGE_TOTAL_TOKENS]).toBe(14); + expect(json.data[GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]).toBe(3); + expect(json.data[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]).toBe(2); + expect(json.data[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]).toBe(5); expect(json.data['gen_ai.usage.reasoning_tokens']).toBeUndefined(); - expect(json.data['gen_ai.response.text']).toBeUndefined(); - expect(json.data['gen_ai.response.tool_calls']).toBeUndefined(); + expect(json.data[GEN_AI_RESPONSE_TEXT]).toBeUndefined(); + expect(json.data[GEN_AI_RESPONSE_TOOL_CALLS]).toBeUndefined(); + }); + + it('maps classifier evaluations to `evaluate {model}` spans', async () => { + const attributes = { classifierId: 'sentiment', modelId: 'jev-1', provider: 'typesafe.evaluation' }; + const span = makeSpan({ id: 'eval-1', type: 'classifier_evaluation', name: "classifier evaluate: 'sentiment'" }); + await run( + started({ ...span, attributes }), + ended({ ...span, attributes: { ...attributes, usage: { inputTokens: 30, outputTokens: 2 } } }), + ); + + const json = spanToStaticSpanJSON(endedSpans[0]!); + expect(json.description).toBe('evaluate jev-1'); + expect(json.data).toMatchObject({ + [SENTRY_OP]: GEN_AI_EVALUATE, + [SENTRY_ORIGIN]: 'auto.ai.mastra', + [GEN_AI_OPERATION_NAME]: 'evaluate', + [GEN_AI_REQUEST_MODEL]: 'jev-1', + [GEN_AI_PROVIDER_NAME]: 'typesafe.evaluation', + [GEN_AI_USAGE_INPUT_TOKENS]: 30, + [GEN_AI_USAGE_OUTPUT_TOKENS]: 2, + [GEN_AI_USAGE_TOTAL_TOKENS]: 32, + }); + }); + + it('leaves a classifier evaluation span open for the call to end once it settles', async () => { + const call: ClassifierEvaluationCall = {}; + const span = makeSpan({ id: 'eval-1', type: 'classifier_evaluation', attributes: { modelId: 'jev-1' } }); + + setStartingClassifierEvaluation(call); + await run(started(span), ended(span)); + + expect(endedSpans).toHaveLength(0); + expect(call.endTime).toEqual(span.endTime); + expect(spanToStaticSpanJSON(call.span!).description).toBe('evaluate jev-1'); + }); + + it('passes its own recording options to the classifier evaluation call', async () => { + exporter = new SentryMastraExporter({ recordInputs: false, recordOutputs: false }); + const call: ClassifierEvaluationCall = {}; + + setStartingClassifierEvaluation(call); + await run(started(makeSpan({ id: 'eval-1', type: 'classifier_evaluation' }))); + + expect(call).toMatchObject({ recordInputs: false, recordOutputs: false }); }); it('records the agent-level prompt and response as gen_ai messages', async () => { @@ -131,10 +210,10 @@ describe('SentryMastraExporter', () => { await run(started(span), ended(span)); const json = spanToStaticSpanJSON(endedSpans[0]!); - expect(json.data['gen_ai.input.messages']).toBe('[{"role":"user","content":"hi"}]'); - expect(json.data['gen_ai.output.messages']).toBe('{"text":"hello"}'); - expect(json.data['gen_ai.response.text']).toBe('hello'); - expect(json.data['gen_ai.system_instructions']).toBe('be brief'); + expect(json.data[GEN_AI_INPUT_MESSAGES]).toBe('[{"role":"user","content":"hi"}]'); + expect(json.data[GEN_AI_OUTPUT_MESSAGES]).toBe('{"text":"hello"}'); + expect(json.data[GEN_AI_RESPONSE_TEXT]).toBe('hello'); + expect(json.data[GEN_AI_SYSTEM_INSTRUCTIONS]).toBe('be brief'); }); it.each([ @@ -150,7 +229,7 @@ describe('SentryMastraExporter', () => { }); await run(started(span), ended(span)); - expect(spanToStaticSpanJSON(endedSpans[0]!).data['gen_ai.response.text']).toBe(expected); + expect(spanToStaticSpanJSON(endedSpans[0]!).data[GEN_AI_RESPONSE_TEXT]).toBe(expected); }); it.each([ @@ -158,28 +237,28 @@ describe('SentryMastraExporter', () => { 'workflow_run', { type: 'workflow_run' as const, entityName: 'math_workflow' }, 'invoke_agent math_workflow', - 'gen_ai.invoke_agent', + GEN_AI_INVOKE_AGENT, ], [ 'rag_embedding', { type: 'rag_embedding' as const, attributes: { model: 'text-embedding-3' } }, 'embeddings text-embedding-3', - 'gen_ai.embeddings', + GEN_AI_EMBEDDINGS, ], [ 'mcp_tool_call', { type: 'mcp_tool_call' as const, entityName: 'search' }, 'execute_tool search', - 'gen_ai.execute_tool', + GEN_AI_EXECUTE_TOOL, ], - ['entityId when entityName is missing', { entityId: 'agent-42' }, 'invoke_agent agent-42', 'gen_ai.invoke_agent'], + ['entityId when entityName is missing', { entityId: 'agent-42' }, 'invoke_agent agent-42', GEN_AI_INVOKE_AGENT], ])('names a %s span', async (_label, overrides, description, op) => { const span = makeSpan(overrides); await run(started(span), ended(span)); const json = spanToStaticSpanJSON(endedSpans[0]!); expect(json.description).toBe(description); - expect(json.data['sentry.op']).toBe(op); + expect(json.data[SENTRY_OP]).toBe(op); }); it('omits inputs and outputs when recording is disabled', async () => { @@ -188,8 +267,8 @@ describe('SentryMastraExporter', () => { await run(started(span), ended(span)); const json = spanToStaticSpanJSON(endedSpans[0]!); - expect(json.data['gen_ai.input.messages']).toBeUndefined(); - expect(json.data['gen_ai.output.messages']).toBeUndefined(); + expect(json.data[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); + expect(json.data[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); expect(JSON.stringify(json.data)).not.toContain('pii'); }); @@ -226,9 +305,9 @@ describe('SentryMastraExporter', () => { const chatJson = json.find(span => span.description === 'chat gpt-5'); expect(toolJson).toBeDefined(); expect(chatJson).toBeDefined(); - expect(toolJson!.data['sentry.op']).toBe('gen_ai.execute_tool'); - expect(toolJson!.data['gen_ai.tool.name']).toBe('get_weather'); - expect(toolJson!.data['gen_ai.tool.call.arguments']).toBe('{"city":"Berlin"}'); + expect(toolJson!.data[SENTRY_OP]).toBe(GEN_AI_EXECUTE_TOOL); + expect(toolJson!.data[GEN_AI_TOOL_NAME]).toBe('get_weather'); + expect(toolJson!.data[GEN_AI_TOOL_CALL_ARGUMENTS]).toBe('{"city":"Berlin"}'); expect(toolJson!.parent_span_id).toBe(chatJson!.span_id); }); @@ -251,10 +330,10 @@ describe('SentryMastraExporter', () => { .map(span => spanToStaticSpanJSON(span)) .find(span => span.description === 'invoke_agent agent'); expect(agentJson).toBeDefined(); - expect(agentJson!.data['gen_ai.usage.input_tokens']).toBe(10); - expect(agentJson!.data['gen_ai.usage.output_tokens']).toBe(4); - expect(agentJson!.data['gen_ai.usage.total_tokens']).toBe(14); - expect(agentJson!.data['gen_ai.response.model']).toBe('gpt-5-2026'); + expect(agentJson!.data[GEN_AI_USAGE_INPUT_TOKENS]).toBe(10); + expect(agentJson!.data[GEN_AI_USAGE_OUTPUT_TOKENS]).toBe(4); + expect(agentJson!.data[GEN_AI_USAGE_TOTAL_TOKENS]).toBe(14); + expect(agentJson!.data[GEN_AI_RESPONSE_MODEL]).toBe('gpt-5-2026'); }); it('sums usage across multiple generations onto the parent agent', async () => { @@ -285,11 +364,11 @@ describe('SentryMastraExporter', () => { .map(span => spanToStaticSpanJSON(span)) .find(span => span.description === 'invoke_agent agent'); expect(agentJson).toBeDefined(); - expect(agentJson!.data['gen_ai.usage.input_tokens']).toBe(40); - expect(agentJson!.data['gen_ai.usage.output_tokens']).toBe(12); - expect(agentJson!.data['gen_ai.usage.total_tokens']).toBe(52); - expect(agentJson!.data['gen_ai.usage.cache_read.input_tokens']).toBe(3); - expect(agentJson!.data['gen_ai.response.model']).toBe('gpt-5-2026'); + expect(agentJson!.data[GEN_AI_USAGE_INPUT_TOKENS]).toBe(40); + expect(agentJson!.data[GEN_AI_USAGE_OUTPUT_TOKENS]).toBe(12); + expect(agentJson!.data[GEN_AI_USAGE_TOTAL_TOKENS]).toBe(52); + expect(agentJson!.data[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]).toBe(3); + expect(agentJson!.data[GEN_AI_RESPONSE_MODEL]).toBe('gpt-5-2026'); }); it('marks the span errored without capturing a reconstructed exception', async () => { @@ -298,7 +377,7 @@ describe('SentryMastraExporter', () => { const json = spanToStaticSpanJSON(endedSpans[0]!); expect(json.status).toBe('internal_error'); - expect(json.data['error.type']).toBe('ToolError'); + expect(json.data[ERROR_TYPE]).toBe('ToolError'); }); it.each([ @@ -334,7 +413,7 @@ describe('SentryMastraExporter', () => { const json = spanToStaticSpanJSON(endedSpans[0]!); expect(Object.keys(json.data).filter(key => key.startsWith('mastra.'))).toEqual([]); - expect(json.data['gen_ai.conversation.id']).toBe('thread-9'); + expect(json.data[GEN_AI_CONVERSATION_ID]).toBe('thread-9'); }); it.each([ @@ -348,14 +427,14 @@ describe('SentryMastraExporter', () => { const span = makeSpan({ entityName: 'agent', ...overrides }); await run(started(span), ended(span)); - expect(spanToStaticSpanJSON(endedSpans[0]!).data['gen_ai.conversation.id']).toBe(expected); + expect(spanToStaticSpanJSON(endedSpans[0]!).data[GEN_AI_CONVERSATION_ID]).toBe(expected); }); it('does not use runId as gen_ai.conversation.id', async () => { const span = makeSpan({ entityName: 'agent', metadata: { runId: 'run-3' } }); await run(started(span), ended(span)); - expect(spanToStaticSpanJSON(endedSpans[0]!).data['gen_ai.conversation.id']).toBeUndefined(); + expect(spanToStaticSpanJSON(endedSpans[0]!).data[GEN_AI_CONVERSATION_ID]).toBeUndefined(); }); it('falls back to the bare operation name when there is no identifier', async () => { @@ -411,7 +490,7 @@ describe('SentryMastraExporter', () => { }); await run(started(span), ended(span)); - expect(spanToStaticSpanJSON(endedSpans[0]!).data['gen_ai.request.stop_sequences']).toEqual(['\n\n', 'END']); + expect(spanToStaticSpanJSON(endedSpans[0]!).data[GEN_AI_REQUEST_STOP_SEQUENCES]).toEqual(['\n\n', 'END']); }); // Parentless dropped spans used to be stored under a falsy sentinel that `LRUMap.remove` never