From bf3f6ea023c0f4c6dac3d0879af5a9516df872e7 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 7 Oct 2026 10:14:30 +0200 Subject: [PATCH 1/3] Add Jev classifier e2e test to node-mastra --- .../node-mastra/package.json | 5 +- .../src/mastra/agents/weather-agent.ts | 11 +--- .../mastra/classifiers/ticket-classifier.ts | 17 ++++++ .../node-mastra/src/mastra/index.ts | 15 ++++- .../node-mastra/src/mastra/openrouter.ts | 10 ++++ .../node-mastra/tests/classifier.test.ts | 56 +++++++++++++++++++ 6 files changed, 101 insertions(+), 13 deletions(-) create mode 100644 dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/classifiers/ticket-classifier.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/openrouter.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/package.json b/dev-packages/e2e-tests/test-applications/node-mastra/package.json index 20340b7e81a6..592a00860591 100644 --- a/dev-packages/e2e-tests/test-applications/node-mastra/package.json +++ b/dev-packages/e2e-tests/test-applications/node-mastra/package.json @@ -15,11 +15,11 @@ "test:dev": "TEST_ENV=development playwright test" }, "dependencies": { - "@mastra/core": "~1.65.0", + "@mastra/core": "~1.74.0", "@mastra/libsql": "~1.22.4", "@mastra/memory": "~1.28.3", "@mastra/observability": "~1.17.6", - "@openrouter/ai-sdk-provider": "~3.0.0", + "@openrouter/ai-sdk-provider": "~3.1.0", "@sentry/node": "file:../../packed/sentry-node-packed.tgz", "ai": "~7.0.97", "dataloader": "~2.2.3", @@ -28,6 +28,7 @@ "devDependencies": { "@playwright/test": "~1.56.0", "@sentry-internal/test-utils": "link:../../../test-utils", + "@sentry/conventions": "0.26.0", "@types/node": "24.x", "mastra": "~1.28.0", "typescript": "~5.9.0" diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/agents/weather-agent.ts b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/agents/weather-agent.ts index 8b33b4cc4d5e..2b56591e4ec5 100644 --- a/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/agents/weather-agent.ts +++ b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/agents/weather-agent.ts @@ -1,19 +1,10 @@ -import { createOpenRouter } from '@openrouter/ai-sdk-provider'; import { Agent } from '@mastra/core/agent'; import { Memory } from '@mastra/memory'; import { LibSQLStore } from '@mastra/libsql'; import { countItemsTool } from '../tools/count-items.js'; import { failNowTool } from '../tools/fail-now.js'; import { getWeatherTool } from '../tools/get-weather.js'; - -const apiKey = process.env.E2E_OPENROUTER_API_KEY; -if (!apiKey) { - throw new Error('E2E_OPENROUTER_API_KEY is not set'); -} - -// Call OpenRouter directly (rather than the default Vercel AI Gateway) so the -// e2e test needs only a single OpenRouter key, reusing `E2E_OPENROUTER_API_KEY`. -const openrouter = createOpenRouter({ apiKey }); +import { openrouter } from '../openrouter.js'; // An in-memory libsql store is enough for the test: Mastra requires a storage // provider before `generate(..., { memory: { thread, resource } })` is accepted, diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/classifiers/ticket-classifier.ts b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/classifiers/ticket-classifier.ts new file mode 100644 index 000000000000..6a66654bb7e9 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/classifiers/ticket-classifier.ts @@ -0,0 +1,17 @@ +import { Classifier } from '@mastra/core/classifier'; +import { openrouter } from '../openrouter.js'; + +export const TICKET_CLASSIFIER = 'ticketClassifier'; + +// A Jev evaluation through OpenRouter's Decisions API. `Classifier` calls the model's `doEvaluate()` +// directly, so its span comes from the Mastra exporter, not from the Vercel AI integration. +export const ticketClassifier = new Classifier({ + id: TICKET_CLASSIFIER, + model: openrouter.evaluationModel('typesafe/jev-1.13'), + questions: { + is_bug: { + type: 'boolean', + instructions: 'Is the customer reporting a software defect?', + }, + }, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/index.ts b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/index.ts index b6f2dd975b39..c5528dbd6ba9 100644 --- a/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/index.ts +++ b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/index.ts @@ -2,6 +2,7 @@ import { Mastra } from '@mastra/core'; import { registerApiRoute } from '@mastra/core/server'; import { LibSQLStore } from '@mastra/libsql'; import { WEATHER_AGENT, weatherAgent } from './agents/weather-agent.js'; +import { TICKET_CLASSIFIER, ticketClassifier } from './classifiers/ticket-classifier.js'; // The agent is driven through Mastra's built-in `POST /api/agents/:id/generate` // endpoint (see tests/utils.ts). `dataloader` (orchestrion-instrumented) is exercised @@ -19,12 +20,24 @@ const manualRoute = registerApiRoute('/manual-route', { }, }); +// Evaluates the registered classifier outside of any agent run. Registration lets Mastra start a root +// `classifier_evaluation` span when no Mastra span is active. +const classifyRoute = registerApiRoute('/classify', { + method: 'POST', + async handler(c) { + const { ticket } = await c.req.json<{ ticket: string }>(); + const result = await ticketClassifier.evaluate({ state: { ticket } }); + return c.json({ answers: result.answers }); + }, +}); + export const mastra = new Mastra({ agents: { [WEATHER_AGENT]: weatherAgent }, + classifiers: { [TICKET_CLASSIFIER]: ticketClassifier }, storage: new LibSQLStore({ id: 'mastra-storage', url: ':memory:' }), server: { port: 4111, - apiRoutes: [manualRoute], + apiRoutes: [manualRoute, classifyRoute], }, // No `bundler.externals` override: `mastra build` defaults to externalizing all non-workspace deps, // so Hono (and the other instrumented modules) stay unbundled and the `--import` orchestrion hook diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/openrouter.ts b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/openrouter.ts new file mode 100644 index 000000000000..b04e3e9e5358 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/openrouter.ts @@ -0,0 +1,10 @@ +import { createOpenRouter } from '@openrouter/ai-sdk-provider'; + +const apiKey = process.env.E2E_OPENROUTER_API_KEY; +if (!apiKey) { + throw new Error('E2E_OPENROUTER_API_KEY is not set'); +} + +// Call OpenRouter directly (rather than the default Vercel AI Gateway) so the +// e2e test needs only a single OpenRouter key, reusing `E2E_OPENROUTER_API_KEY`. +export const openrouter = createOpenRouter({ apiKey }); diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts b/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts new file mode 100644 index 000000000000..1ee8ac493c96 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts @@ -0,0 +1,56 @@ +import { expect, test } from '@playwright/test'; +import { + GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_USAGE_INPUT_TOKENS, + SENTRY_ORIGIN, + URL_FULL, +} from '@sentry/conventions/attributes'; +import { GEN_AI_EVALUATE, HTTP_SERVER } from '@sentry/conventions/op'; +import { collectStreamedSpans, getSpanOp, SerializedStreamedSpan } from '@sentry-internal/test-utils'; + +const APP = 'node-mastra'; +const TICKET = 'My checkout page shows a blank screen after I click Pay.'; + +const attrValue = (span: SerializedStreamedSpan, key: string): unknown => span.attributes?.[key]?.value; + +const isEvaluateSpan = (span: SerializedStreamedSpan): boolean => getSpanOp(span) === GEN_AI_EVALUATE; +const isClassifyServerSpan = (span: SerializedStreamedSpan): boolean => + getSpanOp(span) === HTTP_SERVER && String(attrValue(span, URL_FULL) ?? '').includes('/classify'); + +test('captures a Mastra classifier (Jev) evaluation as a gen_ai.evaluate span', async ({ baseURL }) => { + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => spansOfTrace.some(isEvaluateSpan) && spansOfTrace.some(isClassifyServerSpan), + ); + + const res = await fetch(`${baseURL}/classify`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ ticket: TICKET }), + }); + expect(res.status).toBe(200); + await res.json(); + + const spans = await spansPromise; + const serverSpan = spans.find(isClassifyServerSpan)!; + + // Only the Mastra exporter reports the evaluation: `Classifier` calls `doEvaluate()` directly, so the + // Vercel AI integration does not see it. + const evaluateSpans = spans.filter(isEvaluateSpan); + expect(evaluateSpans.map(span => ({ name: span.name, origin: attrValue(span, SENTRY_ORIGIN) }))).toEqual([ + { name: 'evaluate typesafe/jev-1.13', origin: 'auto.ai.mastra' }, + ]); + + const evaluateSpan = evaluateSpans[0]!; + expect(evaluateSpan.parent_span_id).toBe(serverSpan.span_id); + expect(attrValue(evaluateSpan, GEN_AI_OPERATION_NAME)).toBe('evaluate'); + expect(attrValue(evaluateSpan, GEN_AI_REQUEST_MODEL)).toBe('typesafe/jev-1.13'); + expect(attrValue(evaluateSpan, GEN_AI_PROVIDER_NAME)).toBe('openrouter'); + expect(attrValue(evaluateSpan, GEN_AI_USAGE_INPUT_TOKENS)).toBeGreaterThan(0); + expect(String(attrValue(evaluateSpan, GEN_AI_INPUT_MESSAGES))).toContain(TICKET); + expect(String(attrValue(evaluateSpan, GEN_AI_OUTPUT_MESSAGES))).toContain('"is_bug"'); +}); From 21aeecc4b88cc602c69de31017f35a7f2490bbcd Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 7 Oct 2026 10:43:00 +0200 Subject: [PATCH 2/3] Assert exact classifier input and answer shape --- .../node-mastra/tests/classifier.test.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts b/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts index 1ee8ac493c96..d8e1e177b8f5 100644 --- a/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts @@ -51,6 +51,17 @@ test('captures a Mastra classifier (Jev) evaluation as a gen_ai.evaluate span', expect(attrValue(evaluateSpan, GEN_AI_REQUEST_MODEL)).toBe('typesafe/jev-1.13'); expect(attrValue(evaluateSpan, GEN_AI_PROVIDER_NAME)).toBe('openrouter'); expect(attrValue(evaluateSpan, GEN_AI_USAGE_INPUT_TOKENS)).toBeGreaterThan(0); - expect(String(attrValue(evaluateSpan, GEN_AI_INPUT_MESSAGES))).toContain(TICKET); - expect(String(attrValue(evaluateSpan, GEN_AI_OUTPUT_MESSAGES))).toContain('"is_bug"'); + expect(JSON.parse(String(attrValue(evaluateSpan, GEN_AI_INPUT_MESSAGES)))).toEqual([ + { + type: 'evaluation', + state: { ticket: TICKET }, + questions: { + is_bug: { type: 'boolean', instructions: 'Is the customer reporting a software defect?' }, + }, + }, + ]); + // The probability comes from the live model, so only its shape is stable. + expect(JSON.parse(String(attrValue(evaluateSpan, GEN_AI_OUTPUT_MESSAGES)))).toEqual([ + { type: 'evaluation', answers: { is_bug: { type: 'boolean', probability: expect.any(Number) } } }, + ]); }); From fe214505286c8403602d772bc8331822df35d4a4 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 7 Oct 2026 16:44:21 +0200 Subject: [PATCH 3/3] Inline span attribute access in classifier test --- .../node-mastra/tests/classifier.test.ts | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts b/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts index d8e1e177b8f5..6eee9bb68359 100644 --- a/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts @@ -15,11 +15,9 @@ import { collectStreamedSpans, getSpanOp, SerializedStreamedSpan } from '@sentry const APP = 'node-mastra'; const TICKET = 'My checkout page shows a blank screen after I click Pay.'; -const attrValue = (span: SerializedStreamedSpan, key: string): unknown => span.attributes?.[key]?.value; - const isEvaluateSpan = (span: SerializedStreamedSpan): boolean => getSpanOp(span) === GEN_AI_EVALUATE; const isClassifyServerSpan = (span: SerializedStreamedSpan): boolean => - getSpanOp(span) === HTTP_SERVER && String(attrValue(span, URL_FULL) ?? '').includes('/classify'); + getSpanOp(span) === HTTP_SERVER && String(span.attributes?.[URL_FULL]?.value ?? '').includes('/classify'); test('captures a Mastra classifier (Jev) evaluation as a gen_ai.evaluate span', async ({ baseURL }) => { const spansPromise = collectStreamedSpans( @@ -41,17 +39,17 @@ test('captures a Mastra classifier (Jev) evaluation as a gen_ai.evaluate span', // Only the Mastra exporter reports the evaluation: `Classifier` calls `doEvaluate()` directly, so the // Vercel AI integration does not see it. const evaluateSpans = spans.filter(isEvaluateSpan); - expect(evaluateSpans.map(span => ({ name: span.name, origin: attrValue(span, SENTRY_ORIGIN) }))).toEqual([ + expect(evaluateSpans.map(span => ({ name: span.name, origin: span.attributes?.[SENTRY_ORIGIN]?.value }))).toEqual([ { name: 'evaluate typesafe/jev-1.13', origin: 'auto.ai.mastra' }, ]); const evaluateSpan = evaluateSpans[0]!; expect(evaluateSpan.parent_span_id).toBe(serverSpan.span_id); - expect(attrValue(evaluateSpan, GEN_AI_OPERATION_NAME)).toBe('evaluate'); - expect(attrValue(evaluateSpan, GEN_AI_REQUEST_MODEL)).toBe('typesafe/jev-1.13'); - expect(attrValue(evaluateSpan, GEN_AI_PROVIDER_NAME)).toBe('openrouter'); - expect(attrValue(evaluateSpan, GEN_AI_USAGE_INPUT_TOKENS)).toBeGreaterThan(0); - expect(JSON.parse(String(attrValue(evaluateSpan, GEN_AI_INPUT_MESSAGES)))).toEqual([ + expect(evaluateSpan.attributes?.[GEN_AI_OPERATION_NAME]?.value).toBe('evaluate'); + expect(evaluateSpan.attributes?.[GEN_AI_REQUEST_MODEL]?.value).toBe('typesafe/jev-1.13'); + expect(evaluateSpan.attributes?.[GEN_AI_PROVIDER_NAME]?.value).toBe('openrouter'); + expect(evaluateSpan.attributes?.[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBeGreaterThan(0); + expect(JSON.parse(String(evaluateSpan.attributes?.[GEN_AI_INPUT_MESSAGES]?.value))).toEqual([ { type: 'evaluation', state: { ticket: TICKET }, @@ -61,7 +59,7 @@ test('captures a Mastra classifier (Jev) evaluation as a gen_ai.evaluate span', }, ]); // The probability comes from the live model, so only its shape is stable. - expect(JSON.parse(String(attrValue(evaluateSpan, GEN_AI_OUTPUT_MESSAGES)))).toEqual([ + expect(JSON.parse(String(evaluateSpan.attributes?.[GEN_AI_OUTPUT_MESSAGES]?.value))).toEqual([ { type: 'evaluation', answers: { is_bug: { type: 'boolean', probability: expect.any(Number) } } }, ]); });