diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/package.json b/dev-packages/e2e-tests/test-applications/node-mastra/package.json index 20340b7e81a6..592a00860591 100644 --- a/dev-packages/e2e-tests/test-applications/node-mastra/package.json +++ b/dev-packages/e2e-tests/test-applications/node-mastra/package.json @@ -15,11 +15,11 @@ "test:dev": "TEST_ENV=development playwright test" }, "dependencies": { - "@mastra/core": "~1.65.0", + "@mastra/core": "~1.74.0", "@mastra/libsql": "~1.22.4", "@mastra/memory": "~1.28.3", "@mastra/observability": "~1.17.6", - "@openrouter/ai-sdk-provider": "~3.0.0", + "@openrouter/ai-sdk-provider": "~3.1.0", "@sentry/node": "file:../../packed/sentry-node-packed.tgz", "ai": "~7.0.97", "dataloader": "~2.2.3", @@ -28,6 +28,7 @@ "devDependencies": { "@playwright/test": "~1.56.0", "@sentry-internal/test-utils": "link:../../../test-utils", + "@sentry/conventions": "0.26.0", "@types/node": "24.x", "mastra": "~1.28.0", "typescript": "~5.9.0" diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/agents/weather-agent.ts b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/agents/weather-agent.ts index 8b33b4cc4d5e..2b56591e4ec5 100644 --- a/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/agents/weather-agent.ts +++ b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/agents/weather-agent.ts @@ -1,19 +1,10 @@ -import { createOpenRouter } from '@openrouter/ai-sdk-provider'; import { Agent } from '@mastra/core/agent'; import { Memory } from '@mastra/memory'; import { LibSQLStore } from '@mastra/libsql'; import { countItemsTool } from '../tools/count-items.js'; import { failNowTool } from '../tools/fail-now.js'; import { getWeatherTool } from '../tools/get-weather.js'; - -const apiKey = process.env.E2E_OPENROUTER_API_KEY; -if (!apiKey) { - throw new Error('E2E_OPENROUTER_API_KEY is not set'); -} - -// Call OpenRouter directly (rather than the default Vercel AI Gateway) so the -// e2e test needs only a single OpenRouter key, reusing `E2E_OPENROUTER_API_KEY`. -const openrouter = createOpenRouter({ apiKey }); +import { openrouter } from '../openrouter.js'; // An in-memory libsql store is enough for the test: Mastra requires a storage // provider before `generate(..., { memory: { thread, resource } })` is accepted, diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/classifiers/ticket-classifier.ts b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/classifiers/ticket-classifier.ts new file mode 100644 index 000000000000..6a66654bb7e9 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/classifiers/ticket-classifier.ts @@ -0,0 +1,17 @@ +import { Classifier } from '@mastra/core/classifier'; +import { openrouter } from '../openrouter.js'; + +export const TICKET_CLASSIFIER = 'ticketClassifier'; + +// A Jev evaluation through OpenRouter's Decisions API. `Classifier` calls the model's `doEvaluate()` +// directly, so its span comes from the Mastra exporter, not from the Vercel AI integration. +export const ticketClassifier = new Classifier({ + id: TICKET_CLASSIFIER, + model: openrouter.evaluationModel('typesafe/jev-1.13'), + questions: { + is_bug: { + type: 'boolean', + instructions: 'Is the customer reporting a software defect?', + }, + }, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/index.ts b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/index.ts index b6f2dd975b39..c5528dbd6ba9 100644 --- a/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/index.ts +++ b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/index.ts @@ -2,6 +2,7 @@ import { Mastra } from '@mastra/core'; import { registerApiRoute } from '@mastra/core/server'; import { LibSQLStore } from '@mastra/libsql'; import { WEATHER_AGENT, weatherAgent } from './agents/weather-agent.js'; +import { TICKET_CLASSIFIER, ticketClassifier } from './classifiers/ticket-classifier.js'; // The agent is driven through Mastra's built-in `POST /api/agents/:id/generate` // endpoint (see tests/utils.ts). `dataloader` (orchestrion-instrumented) is exercised @@ -19,12 +20,24 @@ const manualRoute = registerApiRoute('/manual-route', { }, }); +// Evaluates the registered classifier outside of any agent run. Registration lets Mastra start a root +// `classifier_evaluation` span when no Mastra span is active. +const classifyRoute = registerApiRoute('/classify', { + method: 'POST', + async handler(c) { + const { ticket } = await c.req.json<{ ticket: string }>(); + const result = await ticketClassifier.evaluate({ state: { ticket } }); + return c.json({ answers: result.answers }); + }, +}); + export const mastra = new Mastra({ agents: { [WEATHER_AGENT]: weatherAgent }, + classifiers: { [TICKET_CLASSIFIER]: ticketClassifier }, storage: new LibSQLStore({ id: 'mastra-storage', url: ':memory:' }), server: { port: 4111, - apiRoutes: [manualRoute], + apiRoutes: [manualRoute, classifyRoute], }, // No `bundler.externals` override: `mastra build` defaults to externalizing all non-workspace deps, // so Hono (and the other instrumented modules) stay unbundled and the `--import` orchestrion hook diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/openrouter.ts b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/openrouter.ts new file mode 100644 index 000000000000..b04e3e9e5358 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/openrouter.ts @@ -0,0 +1,10 @@ +import { createOpenRouter } from '@openrouter/ai-sdk-provider'; + +const apiKey = process.env.E2E_OPENROUTER_API_KEY; +if (!apiKey) { + throw new Error('E2E_OPENROUTER_API_KEY is not set'); +} + +// Call OpenRouter directly (rather than the default Vercel AI Gateway) so the +// e2e test needs only a single OpenRouter key, reusing `E2E_OPENROUTER_API_KEY`. +export const openrouter = createOpenRouter({ apiKey }); diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts b/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts new file mode 100644 index 000000000000..6eee9bb68359 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts @@ -0,0 +1,65 @@ +import { expect, test } from '@playwright/test'; +import { + GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_USAGE_INPUT_TOKENS, + SENTRY_ORIGIN, + URL_FULL, +} from '@sentry/conventions/attributes'; +import { GEN_AI_EVALUATE, HTTP_SERVER } from '@sentry/conventions/op'; +import { collectStreamedSpans, getSpanOp, SerializedStreamedSpan } from '@sentry-internal/test-utils'; + +const APP = 'node-mastra'; +const TICKET = 'My checkout page shows a blank screen after I click Pay.'; + +const isEvaluateSpan = (span: SerializedStreamedSpan): boolean => getSpanOp(span) === GEN_AI_EVALUATE; +const isClassifyServerSpan = (span: SerializedStreamedSpan): boolean => + getSpanOp(span) === HTTP_SERVER && String(span.attributes?.[URL_FULL]?.value ?? '').includes('/classify'); + +test('captures a Mastra classifier (Jev) evaluation as a gen_ai.evaluate span', async ({ baseURL }) => { + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => spansOfTrace.some(isEvaluateSpan) && spansOfTrace.some(isClassifyServerSpan), + ); + + const res = await fetch(`${baseURL}/classify`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ ticket: TICKET }), + }); + expect(res.status).toBe(200); + await res.json(); + + const spans = await spansPromise; + const serverSpan = spans.find(isClassifyServerSpan)!; + + // Only the Mastra exporter reports the evaluation: `Classifier` calls `doEvaluate()` directly, so the + // Vercel AI integration does not see it. + const evaluateSpans = spans.filter(isEvaluateSpan); + expect(evaluateSpans.map(span => ({ name: span.name, origin: span.attributes?.[SENTRY_ORIGIN]?.value }))).toEqual([ + { name: 'evaluate typesafe/jev-1.13', origin: 'auto.ai.mastra' }, + ]); + + const evaluateSpan = evaluateSpans[0]!; + expect(evaluateSpan.parent_span_id).toBe(serverSpan.span_id); + expect(evaluateSpan.attributes?.[GEN_AI_OPERATION_NAME]?.value).toBe('evaluate'); + expect(evaluateSpan.attributes?.[GEN_AI_REQUEST_MODEL]?.value).toBe('typesafe/jev-1.13'); + expect(evaluateSpan.attributes?.[GEN_AI_PROVIDER_NAME]?.value).toBe('openrouter'); + expect(evaluateSpan.attributes?.[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBeGreaterThan(0); + expect(JSON.parse(String(evaluateSpan.attributes?.[GEN_AI_INPUT_MESSAGES]?.value))).toEqual([ + { + type: 'evaluation', + state: { ticket: TICKET }, + questions: { + is_bug: { type: 'boolean', instructions: 'Is the customer reporting a software defect?' }, + }, + }, + ]); + // The probability comes from the live model, so only its shape is stable. + expect(JSON.parse(String(evaluateSpan.attributes?.[GEN_AI_OUTPUT_MESSAGES]?.value))).toEqual([ + { type: 'evaluation', answers: { is_bug: { type: 'boolean', probability: expect.any(Number) } } }, + ]); +});