Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 27 additions & 2 deletions apps/agent-forge/e2e/a11y.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,20 @@ test.beforeEach(async ({ page }) => {
await expect(page.locator('.app')).toBeVisible();
});

/**
* Open the builder on the unsaved default agent. A hash change (not a reload)
* keeps the already-open `untitled-agent` document, which is the only way the
* id resolves once other agents are saved - an unknown id falls back to the list.
*/
async function openBuilder(page: Page) {
await page.evaluate(() => {
window.location.hash = '/agents/untitled-agent/canvas';
});
await expect(page.locator('.canvas-wrap')).toBeVisible();
}

test('the page has landmarks, an h1 and a live run status', async ({ page }) => {
await openBuilder(page);
await expect(page.getByRole('banner')).toHaveCount(1);
await expect(page.getByRole('navigation', { name: 'Primary' })).toHaveCount(1);
await expect(page.getByRole('navigation', { name: 'Agents and node palette' })).toHaveCount(1);
Expand All @@ -33,6 +46,7 @@ test('the page has landmarks, an h1 and a live run status', async ({ page }) =>
});

test('drawer and rail tabs are tablists driven by the arrow keys', async ({ page }) => {
await openBuilder(page);
const drawerTabs = page.getByRole('tablist', { name: 'Run panels' });
await expect(drawerTabs.getByRole('tab')).toHaveCount(6);
const chat = drawerTabs.getByRole('tab', { name: 'Chat' });
Expand All @@ -51,11 +65,13 @@ test('drawer and rail tabs are tablists driven by the arrow keys', async ({ page
});

test('chat controls are labelled', async ({ page }) => {
await openBuilder(page);
await expect(page.getByRole('combobox', { name: 'Past conversations' })).toBeVisible();
await expect(page.getByRole('textbox', { name: 'Message' })).toBeVisible();
});

test('palette items add nodes from the keyboard; hook switch and toggles respond to Enter/Space', async ({ page }) => {
await openBuilder(page);
const nodes = page.locator('.react-flow__node');
const before = await nodes.count();

Expand Down Expand Up @@ -162,6 +178,14 @@ const ITEM_H1: Record<string, string> = {
};
for (const { label, path } of SCANS) {
test(`${label} keeps the landmarks and passes axe`, async ({ page }) => {
// The agent routes land on the builder, which needs the id to be saved -
// an unknown one falls back to the list.
if (path.startsWith('/agents/')) {
await page.request.put('/agents/e2e-agent', {
headers: { 'x-lousho-studio-token': 'e2e-studio-token' },
data: { name: 'e2e-agent', prompt: 'You are a helpful agent.', provider: { type: 'mock', model: 'mock-1' } },
});
}
await page.goto(`/?token=e2e-studio-token#${path}`);
await expect(page.locator('.app')).toBeVisible();
await page.waitForLoadState('networkidle');
Expand All @@ -171,9 +195,10 @@ for (const { label, path } of SCANS) {
await expect(page.getByRole('navigation', { name: 'Primary' })).toHaveCount(1);
const current = page.getByRole('navigation', { name: 'Primary' }).locator('[aria-current="page"]');
await expect(current).toHaveCount(1);
// Agents keeps its visually hidden "Agent Forge" h1; every other screen's h1 is its PageHeader title (the nav
// The builder (`/agents/<id>` and its tabs, which land on canvas) keeps its visually hidden "Agent Forge"
// h1; `/agents` is the agents list whose h1 is the PageHeader title like every other screen's (the nav
// label, or the item's name on an ITEM_H1 detail screen).
const h1 = path.startsWith('/agents') ? 'Agent Forge' : (ITEM_H1[label] ?? (await current.innerText()));
const h1 = path.startsWith('/agents/') ? 'Agent Forge' : (ITEM_H1[label] ?? (await current.innerText()));
await expect(page.getByRole('heading', { level: 1 })).toHaveText(h1);
await expectNoSeriousAxeViolations(page, path);
});
Expand Down
16 changes: 13 additions & 3 deletions apps/agent-forge/e2e/docShots.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,16 @@ async function open(page: Page, hash: string) {
await expect(page.locator('.app')).toBeVisible();
}

/**
* A builder URL that survives the unknown-id fallback to `#/agents`: the first
* saved agent's canvas tab, or the unsaved default's when none are saved yet.
*/
async function builderHash(page: Page): Promise<string> {
const res = await page.request.get('/agents', { headers: { 'x-lousho-studio-token': 'e2e-studio-token' } });
const ids = ((await res.json()) as { id: string }[]).map((a) => a.id);
return `/agents/${ids[0] ?? 'untitled-agent'}/canvas`;
}

/** The docs set shot in dark too, written as `studio-<screen>-dark.png`. */
const DARK_DOCS = new Set(['agents', 'traces', 'trace', 'monitoring']);

Expand Down Expand Up @@ -314,7 +324,7 @@ async function shoot(page: Page, screen: string) {
const rows = (page: Page) => page.getByRole('main').locator('table.data-table tbody tr:not(.skeleton-row)');

test('agents: the builder with an agent', async ({ page }) => {
await open(page, '/agents');
await open(page, await builderHash(page));
await page.getByRole('button', { name: 'New agent', exact: true }).click();
await page.locator('#new-agent-name').fill('support-bot');
await page.locator('#new-agent-template').selectOption('support-bot');
Expand Down Expand Up @@ -625,7 +635,7 @@ test('x: 375 wide, Traces with the nav drawer closed and open', async ({ page })
* agent's stream socket, so nodes and edges hold their running state.
*/
async function newAgent(page: Page, name: string, template?: 'support-bot') {
await open(page, '/agents');
await open(page, await builderHash(page));
await page.getByRole('button', { name: 'New agent', exact: true }).click();
await page.locator('#new-agent-name').fill(name);
if (template) await page.locator('#new-agent-template').selectOption(template);
Expand Down Expand Up @@ -981,7 +991,7 @@ test('agents x-rail-status: the unsaved default agent, and an errored agent with
await page.route(/\/agents(\?|$)/, (route) =>
route.request().method() === 'GET' ? route.fulfill({ status: 200, contentType: 'application/json', body: '[]' }) : route.continue(),
);
await open(page, '/agents');
await open(page, '/agents/untitled-agent/canvas');
await expect(page.locator('.agent-card.selected')).toBeVisible();
await shoot(page, 'x-agents-rail-unsaved');
await page.unroute(/\/agents(\?|$)/);
Expand Down
6 changes: 5 additions & 1 deletion apps/agent-forge/e2e/narrow.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,11 @@ import { test, expect, type Page } from '@playwright/test';
*/
async function open(page: Page, width: number, height: number) {
await page.setViewportSize({ width, height });
await page.goto('/?token=e2e-studio-token');
// The builder is `#/agents/:id/canvas` (bare `#/agents` is the agents list);
// an id that is not saved falls back there, so open a saved agent when one exists.
const res = await page.request.get('/agents', { headers: { 'x-lousho-studio-token': 'e2e-studio-token' } });
const ids = ((await res.json()) as { id: string }[]).map((a) => a.id);
await page.goto(`/?token=e2e-studio-token#/agents/${ids[0] ?? 'untitled-agent'}/canvas`);
await expect(page.locator('.app')).toBeVisible();
}

Expand Down
19 changes: 14 additions & 5 deletions apps/agent-forge/e2e/revamp.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,8 @@ const TO = Date.now();
*/
const FILTERS: Record<ScreenId, string> = {
agents: '',
agentList: '',
agentDetail: '',
playground: '?trace=e2e-trace&fromObs=e2e-obs',
traces: `?agent=orders-bot&status=error&q=order&from=${FROM}&to=${TO}&sort=duration&order=desc`,
trace: '?obs=e2e-obs',
Expand Down Expand Up @@ -227,13 +229,17 @@ const CONTROLS: Record<string, (main: Locator, page: Page) => Promise<void>> = {
},
};
CONTROLS['/hub/:namespace'] = CONTROLS['/hub'];
// `/agents/:id` redirects to the canvas tab, and an unbuilt `/agents/:id/:tab`
// does too - both land on the same builder view the crumbs check covers.
CONTROLS['/agents/:id/canvas'] = CONTROLS['/agents/:id'];
CONTROLS['/agents/:id/:tab'] = CONTROLS['/agents/:id'];

/**
* Routes with no filter controls of their own to re-check (the reload still
* checks the hash and the screen). A route in neither CONTROLS nor here fails.
*/
const NO_CONTROLS = new Set([
'/agents', // canonicalises to the open agent; nothing to filter
'/agents', // the agents list has no filters
'/threads/:sessionId', // a thread's turns; the list's filters only ride along in the URL
'/evals/queues/:id', // one queue's review pane; no URL-backed controls beyond the id
'/prompts', // the registry list has no filters
Expand All @@ -256,7 +262,7 @@ const API = { headers: { 'x-lousho-studio-token': 'e2e-studio-token' } };
async function ensureSavedAgent(page: Page) {
const ids = ((await (await page.request.get('/agents', API)).json()) as { id: string }[]).map((a) => a.id);
if (ids.includes(SAVED_AGENT)) return;
await openApp(page);
await openApp(page, '/agents/untitled-agent/canvas');
await page.getByRole('button', { name: 'New agent', exact: true }).click();
await page.locator('#new-agent-name').fill(SAVED_AGENT);
await page.getByRole('button', { name: 'Create', exact: true }).click();
Expand All @@ -275,15 +281,18 @@ for (const { pattern, screen, example } of ROUTES) {
).toBe(true);
expect(controls !== undefined && NO_CONTROLS.has(pattern), `route ${pattern} is in both CONTROLS and NO_CONTROLS`).toBe(false);

if (screen === 'agents') await ensureSavedAgent(page);
if (screen === 'agents' || screen === 'agentDetail') await ensureSavedAgent(page);
const hash = `#${example}${filters}`;
await openApp(page, `${example}${filters}`);
const main = page.getByRole('main');
await controls?.(main, page);
const before = await screenIdentity(page);
const settled = new URL(page.url()).hash;
// Bare #/agents is replaced by the open agent's #/agents/:id (its canonical URL); every other URL stays as given.
if (pattern === '/agents') expect(settled).toMatch(/^#\/agents(\/[^/?]+)?$/);
// `/agents/:id` (the legacy builder URL) and an unbuilt `/agents/:id/:tab` redirect
// (replace) to the canvas tab; every other URL stays as given, filters kept.
if (pattern === '/agents/:id') expect(settled).toBe(`#${example}/canvas${filters}`);
else if (pattern === '/agents/:id/:tab')
expect(settled).toBe(`#${example.slice(0, example.lastIndexOf('/'))}/canvas${filters}`);
else expect(settled).toBe(hash);

await page.reload();
Expand Down
59 changes: 41 additions & 18 deletions apps/agent-forge/e2e/studio.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,24 @@ import { test, expect, type Page } from '@playwright/test';
* - so this test chats on the freshly created agent straight
* away, with no Run first.
*/
const API = { headers: { 'x-lousho-studio-token': 'e2e-studio-token' } };

/**
* A builder URL that survives the unknown-id fallback to `#/agents`: the first
* saved agent's canvas tab, or the unsaved default's when none are saved yet.
*/
async function builderHash(page: Page): Promise<string> {
const res = await page.request.get('/agents', API);
const ids = ((await res.json()) as { id: string }[]).map((a) => a.id);
return `#/agents/${ids[0] ?? 'untitled-agent'}/canvas`;
}

test('create an agent, hit an approval gate via chat, approve it, and complete the run', async ({ page }) => {
const agentId = `e2e-approval-agent-${Date.now()}`;

// The studio URL carries the per-launch token (see playwright.config.ts).
await page.goto('/?token=e2e-studio-token');
// The studio URL carries the per-launch token (see playwright.config.ts);
// the builder lives at `#/agents/:id/canvas`.
await page.goto(`/?token=e2e-studio-token${await builderHash(page)}`);
await expect(page.locator('.app')).toBeVisible();

// --- Create an agent from the "Support bot" template (trigger -> llm ->
Expand Down Expand Up @@ -106,7 +119,7 @@ async function expectInside(inner: ReturnType<Page['locator']>, outer: ReturnTyp
test('a pending approval keeps "Open trace" and the MOCK badge in the toolbar at 1280 and 1024 wide', async ({ page }) => {
const agentId = `e2e-pending-trace-${Date.now()}`;
await page.setViewportSize({ width: 1280, height: 800 });
await page.goto('/?token=e2e-studio-token');
await page.goto(`/?token=e2e-studio-token${await builderHash(page)}`);
await page.getByRole('button', { name: 'New agent', exact: true }).click();
await page.locator('#new-agent-name').fill(agentId);
await page.locator('#new-agent-template').selectOption('support-bot');
Expand All @@ -130,8 +143,15 @@ test('a pending approval keeps "Open trace" and the MOCK badge in the toolbar at
}
});

/** F5: the builder is the Agents screen, and `#/agents/:id` names the open agent. */
/** F5: the builder lives at `#/agents/:id/canvas`; the hash names the open agent. */
async function createAgent(page: Page, id: string) {
// Landing on `#/agents` (the list) first? Open the builder through it:
// an agent's link, or the empty state's builder link when there are none.
if ((await page.getByRole('button', { name: 'New agent', exact: true }).count()) === 0) {
const agentLink = page.locator('.agent-list-placeholder a').first();
if ((await agentLink.count()) > 0) await agentLink.click();
else await page.getByRole('link', { name: 'Open the builder' }).click();
}
await page.getByRole('button', { name: 'New agent', exact: true }).click();
await page.locator('#new-agent-name').fill(id);
await page.getByRole('button', { name: 'Create', exact: true }).click();
Expand All @@ -145,39 +165,42 @@ test('selecting an agent updates the URL, a reload restores it, an unknown id fa
await expect(page.locator('.app')).toBeVisible();

await createAgent(page, first);
await expect(page).toHaveURL(new RegExp(`#/agents/${first}$`));
await expect(page).toHaveURL(new RegExp(`#/agents/${first}/canvas$`));
await createAgent(page, second);
await expect(page).toHaveURL(new RegExp(`#/agents/${second}$`));
await expect(page).toHaveURL(new RegExp(`#/agents/${second}/canvas$`));

await page.locator('.agent-card-main', { hasText: first }).click();
await expect(page).toHaveURL(new RegExp(`#/agents/${first}$`));
await expect(page).toHaveURL(new RegExp(`#/agents/${first}/canvas$`));
await expect(page.locator('.crumbs b')).toHaveText(first);

// The token moved to sessionStorage on the first load, so the reload has none in the URL.
await page.reload();
await expect(page.locator('.crumbs b')).toHaveText(first);
await expect(page.locator('.agent-card.selected')).toContainText(first);
await expect(page).toHaveURL(new RegExp(`#/agents/${first}$`));
await expect(page).toHaveURL(new RegExp(`#/agents/${first}/canvas$`));

// The legacy `#/agents/:id` URL still opens the builder, redirected to its canvas tab.
await page.goto(`/?token=e2e-studio-token#/agents/${second}`);
await expect(page).toHaveURL(new RegExp(`#/agents/${second}/canvas$`));
await expect(page.locator('.crumbs b')).toHaveText(second);

// Unknown id: no error screen; falls back to #/agents, which then names the open agent.
// Unknown id: no error screen; falls back to the agents list at #/agents.
await page.goto('/#/agents/no-such-agent');
await expect(page.locator('.canvas-wrap')).toBeVisible();
await expect(page).toHaveURL(/#\/agents\/(?!no-such-agent)[^/]+$/);
await expect(page).toHaveURL(/#\/agents$/);
await expect(page.locator('.agent-list-screen')).toBeVisible();

// Agents -> Traces -> Agents (the nav link is bare #/agents) -> reload keeps the open agent.
await page.locator('.agent-card-main', { hasText: second }).click();
// From the list an agent link opens its builder; Agents -> Traces -> Agents lands back on the list.
await page.locator(`.agent-list-placeholder a[href="#/agents/${second}/overview"]`).click();
await expect(page).toHaveURL(new RegExp(`#/agents/${second}/canvas$`));
await expect(page.locator('.crumbs b')).toHaveText(second);
const nav = page.getByRole('navigation', { name: 'Primary' });
await nav.getByRole('link', { name: 'Traces' }).click();
await expect(page).toHaveURL(/#\/traces$/);
await nav.getByRole('link', { name: 'Agents' }).click();
await expect(page).toHaveURL(new RegExp(`#/agents/${second}$`));
await expect(page).toHaveURL(/#\/agents$/);
await page.reload();
await expect(page.locator('.crumbs b')).toHaveText(second);
await expect(page).toHaveURL(new RegExp(`#/agents/${second}$`));
await expect(page.getByRole('heading', { name: 'Agents' })).toBeVisible();
await expect(page).toHaveURL(/#\/agents$/);
});

test('unsaved changes ask before switching agents or leaving Agents; cancelling keeps the URL', async ({ page }) => {
Expand All @@ -203,13 +226,13 @@ test('unsaved changes ask before switching agents or leaving Agents; cancelling
await page.getByRole('navigation', { name: 'Primary' }).getByRole('link', { name: 'Traces' }).click();
await expect.poll(() => dialogs.length).toBe(1);
expect(dialogs[0]).toContain(second);
await expect(page).toHaveURL(new RegExp(`#/agents/${second}$`));
await expect(page).toHaveURL(new RegExp(`#/agents/${second}/canvas$`));
await expect(page.locator('.canvas-wrap')).toBeVisible();

await page.getByRole('tab', { name: 'Agents' }).click();
page.once('dialog', (d) => void d.dismiss());
await page.locator('.agent-card-main', { hasText: first }).click();
await expect(page).toHaveURL(new RegExp(`#/agents/${second}$`));
await expect(page).toHaveURL(new RegExp(`#/agents/${second}/canvas$`));
await expect(page.locator('.crumbs b')).toHaveText(second);

page.once('dialog', (d) => void d.accept());
Expand Down
Loading
Loading