From aaf69f120effbbcf4256d71fe0299c31717d7c7d Mon Sep 17 00:00:00 2001 From: hsliu Date: Sun, 18 Oct 2026 00:32:26 +0800 Subject: [PATCH] Add the real-model browser E2E suite of #204 Four app-independent scenario checks (fixture CSV in, summary.json / summary-east.json out, reload survival, sidebar previews) with one FlyBot web adapter, a real-model bench launcher and a fail-closed runner (guards, budgets, outcome contract, evidence sanitizer), run nightly on main by a dedicated secret-bearing workflow. Scenario logic is covered keyless in verify through a fake adapter, following the dev/interop pattern; Orbit reuses the scenarios through its own adapter (#209). --- .github/workflows/live-e2e.yml | 158 ++++++++++++++++++++++++ dev/live-e2e/README.md | 56 +++++++++ dev/live-e2e/adapter.mjs | 163 +++++++++++++++++++++++++ dev/live-e2e/bench.mjs | 151 +++++++++++++++++++++++ dev/live-e2e/fixtures/orders.csv | 6 + dev/live-e2e/report.mjs | 135 +++++++++++++++++++++ dev/live-e2e/run.mjs | 147 ++++++++++++++++++++++ dev/live-e2e/sanitize.mjs | 37 ++++++ dev/live-e2e/scenarios.mjs | 174 +++++++++++++++++++++++++++ dev/live-e2e/test/fake-adapter.mjs | 63 ++++++++++ dev/live-e2e/test/scenarios.test.mjs | 151 +++++++++++++++++++++++ docs/interop/live-e2e-adapter.md | 62 ++++++++++ docs/subsystems/README.md | 1 + docs/subsystems/live-e2e.md | 35 ++++++ package.json | 6 +- scripts/lint-live-e2e.mjs | 12 ++ 16 files changed, 1355 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/live-e2e.yml create mode 100644 dev/live-e2e/README.md create mode 100644 dev/live-e2e/adapter.mjs create mode 100644 dev/live-e2e/bench.mjs create mode 100644 dev/live-e2e/fixtures/orders.csv create mode 100644 dev/live-e2e/report.mjs create mode 100644 dev/live-e2e/run.mjs create mode 100644 dev/live-e2e/sanitize.mjs create mode 100644 dev/live-e2e/scenarios.mjs create mode 100644 dev/live-e2e/test/fake-adapter.mjs create mode 100644 dev/live-e2e/test/scenarios.test.mjs create mode 100644 docs/interop/live-e2e-adapter.md create mode 100644 docs/subsystems/live-e2e.md create mode 100644 scripts/lint-live-e2e.mjs diff --git a/.github/workflows/live-e2e.yml b/.github/workflows/live-e2e.yml new file mode 100644 index 000000000..5763f27ea --- /dev/null +++ b/.github/workflows/live-e2e.yml @@ -0,0 +1,158 @@ +name: Live E2E + +# The real-model browser suite of #204: four task checks driven through the FlyBot web +# UI on a fresh isolated bench, against the real DeepSeek endpoint. Separate from the +# compatibility matrix (nightly.yml) and from every PR check; never added to `tests-ok` +# or branch protection. This is the repository's only workflow that reads a secret: the +# live job runs on main only, the key is mapped step-scoped, and everything uploaded has +# passed the runner's evidence sanitizer (no `token=`, no key material, no cookies) and +# is an explicit allow-list — never a raw bench home. + +on: + schedule: + - cron: "30 16 * * *" # 00:30 Asia/Shanghai daily; clear of compat's 16:00Z + workflow_dispatch: + +concurrency: + group: live-e2e + cancel-in-progress: false + +permissions: + contents: read + +jobs: + window: + name: window check + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + run: ${{ steps.gate.outputs.run }} + steps: + - name: Check out main with full history + if: github.event_name == 'schedule' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - name: Decide whether main has new commits + id: gate + run: | + run=false + reason="" + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + run=true + reason="manual dispatch always runs" + elif [ -n "$(git log --since='24 hours ago' --pretty=oneline)" ]; then + run=true + reason="main had commits in the last 24 hours" + else + reason="main had no commits in the window; no tokens spent" + fi + echo "run=$run" >> "$GITHUB_OUTPUT" + { + echo "### Live E2E run decision" + echo "" + echo "- event: \`${{ github.event_name }}\`" + echo "- run: \`${run}\`: ${reason}" + } >> "$GITHUB_STEP_SUMMARY" + + live: + name: live · ubuntu · Node 24 · real model + needs: window + if: needs.window.outputs.run == 'true' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + WEB_PORT: '3196' + TAP_PORT: '8898' + BENCH: live-e2e + MODEL: deepseek-official/deepseek-flash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 24 + cache: npm + cache-dependency-path: core/package-lock.json + - name: Preflight (the key must exist; never a silent skip) + env: + DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} + run: | + [ -n "$DEEPSEEK_API_KEY" ] || { echo "::error::DEEPSEEK_API_KEY is not configured as a repository secret"; exit 1; } + - name: npm ci (core) + working-directory: core + run: npm ci --no-fund --no-audit + timeout-minutes: 5 + - name: Install the pinned DSH runtime + working-directory: core + run: | + DSH=$(node -p "require('./dev/harness/package.json').dependencies['@deepseek-ai/dsh']") + npm run harness -- --no-save "@deepseek-ai/dsh@$DSH" + echo "DSH=$DSH" >> "$GITHUB_ENV" + timeout-minutes: 5 + - name: Clone DeepSeek Harness at the pinned tag + run: | + git clone --depth 1 --branch "dsh-v$DSH" https://github.com/deepseek-ai/deepseek-harness.git deepseek-harness + corepack enable + pnpm install --dir deepseek-harness + timeout-minutes: 10 + - name: Start the usage tap + run: | + mkdir -p ".dsh-test-$BENCH" + setsid node core/dev/llm-tap.mjs "$TAP_PORT" https://api.deepseek.com ".dsh-test-$BENCH/tap.jsonl" \ + ".dsh-test-$BENCH/tap.out" 2>&1 & echo $! >".dsh-test-$BENCH/tap.pid" + i=0 + until (exec 3<>"/dev/tcp/127.0.0.1/$TAP_PORT") 2>/dev/null; do + i=$((i + 1)) + [ $i -ge 30 ] && { echo "::error::the usage tap did not start"; cat ".dsh-test-$BENCH/tap.out"; exit 1; } + sleep 1 + done + - name: Start the real-model bench + env: + DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} + run: | + DEEPSEEK_BASE_URL="http://127.0.0.1:$TAP_PORT/anthropic" node dev/live-e2e/bench.mjs "$BENCH" "$WEB_PORT" reset + timeout-minutes: 8 + - name: Run the four checks + id: run + env: + DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} + DEEPSEEK_BASE_URL: http://127.0.0.1:8898/anthropic + LIVE_E2E_TAP_UPSTREAM: https://api.deepseek.com + run: | + node dev/live-e2e/run.mjs ".dsh-test-$BENCH" "$WEB_PORT" \ + --model "$MODEL" --harness deepseek-harness \ + --tap ".dsh-test-$BENCH/tap.jsonl" --evidence evidence --tz Asia/Shanghai + - name: Stop the bench and the tap + if: always() + run: | + node dev/live-e2e/bench.mjs "$BENCH" "$WEB_PORT" stop || true + if [ -f ".dsh-test-$BENCH/tap.pid" ]; then + kill -TERM "-$(cat ".dsh-test-$BENCH/tap.pid")" 2>/dev/null || true + fi + - name: Sanitized web log (diagnostics only, never uploaded raw) + if: failure() + run: tail -80 ".dsh-test-$BENCH/web.log" | grep -v 'token=' || true + # Fail closed twice: the runner writes evidence/sanitized.txt only after a clean + # scan, and the upload requires that stamp. A bundle the sanitizer flagged (or one + # from a run that never reached the scan) never becomes a public artifact. + - name: Evidence gate + id: evidence-gate + if: always() && steps.run.outcome != 'skipped' + run: | + if [ ! -f evidence/sanitized.txt ]; then + echo "::error::no sanitizer-cleared evidence bundle (the run failed before the scan, or the scan found something); nothing is uploaded" + exit 1 + fi + # Only the stamped allow-list is uploaded: the report, the screenshots, the + # transcript — never the bench home, the tap log or any raw log. + - name: Upload evidence (allow-list, sanitizer-stamped) + if: always() && steps.evidence-gate.outcome == 'success' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: live-e2e-evidence + path: evidence/ + if-no-files-found: ignore + retention-days: 14 diff --git a/dev/live-e2e/README.md b/dev/live-e2e/README.md new file mode 100644 index 000000000..2f35394de --- /dev/null +++ b/dev/live-e2e/README.md @@ -0,0 +1,56 @@ +# Live E2E (#204) + +Real-model, browser-driven end-to-end checks: upload an orders CSV through the web UI, +let the Main Bot write `summary.json` / `summary-east.json` with the real DeepSeek +endpoint, then verify the files, their survival across a reload, and the sidebar +previews — by content, never by a filename echoed in a reply. The scenarios and +assertions are app-independent ([adapter contract](../../docs/interop/live-e2e-adapter.md)); +only `adapter.mjs` knows FlyBot. Nothing here ships in a package or runs in +`npm run verify`; the keyless fake-adapter tests do (`npm run test:live-e2e`). + +## Layout + +| File | Responsibility | +| --- | --- | +| `fixtures/orders.csv` | The fixed 5-row fixture the expected totals derive from. | +| `scenarios.mjs` | The four checks, task texts, pure assertions (parsed JSON, SHA-256 immutability). | +| `adapter.mjs` | The FlyBot web adapter: playwright-core from the harness checkout, UI task steps, read-only observation. | +| `bench.mjs` | A dsh web bench on this checkout with the real endpoint; refuses missing/mock keys, never touches `~/.dsh`. | +| `run.mjs` | The runner: guards, budgets, classification, report, evidence sanitizer. | +| `report.mjs` | Outcome contract, guard logic, usage/classification, report assembly (pure, tested keyless). | +| `sanitize.mjs` | The evidence gate: `token=` URLs, cookie material, the key value. | +| `test/` | Keyless fake-adapter coverage of every assertion path. | + +## Run it locally + +Node 22.19+, pnpm, a [deepseek-harness](https://github.com/deepseek-ai/deepseek-harness) +checkout with `pnpm install` run in it (`../deepseek-harness`, or set `DSH_HARNESS`), and +a real `DEEPSEEK_API_KEY` in the environment. Chrome comes from `CHROME_BIN`, else +`google-chrome`/`chromium`(-`browser`) on PATH, else playwright's own download. + +```sh +export DEEPSEEK_API_KEY=... # real key; the runner fails closed without it +node dev/live-e2e/bench.mjs live-e2e 3196 reset +node dev/live-e2e/run.mjs .dsh-test-live-e2e 3196 --model deepseek-official/deepseek-flash +node dev/live-e2e/bench.mjs live-e2e 3196 stop +``` + +The report lands in `evidence/report.json` (screenshots, transcript beside it). To +record per-call token usage, start `core/dev/llm-tap.mjs https://api.deepseek.com +` first, run the bench with +`DEEPSEEK_BASE_URL=http://127.0.0.1:/anthropic`, and pass `--tap `; +the loopback is accepted only with `LIVE_E2E_TAP_UPSTREAM=https://api.deepseek.com`. +A run costs real tokens (the pilot used ~116K); budgets default to 180 s per check and +250K tokens per run (`LIVE_E2E_CHECK_MS`, `LIVE_E2E_QUIET_MS`, `LIVE_E2E_MAX_TOKENS`), +with zero retries. + +## In CI + +`.github/workflows/live-e2e.yml` runs the same flow nightly at 00:30 Asia/Shanghai (and +on dispatch) on `main` only, with a window gate so a quiet `main` spends no tokens: it +clones the harness at the tag pinned in `core/dev/harness/package.json`, starts the tap, +resets the bench, runs the four checks and uploads the sanitizer-cleared allow-list. +The four decisions behind this setup are recorded in +[#204](https://github.com/ThinkFlowLab/issues/204). "Stable" means five consecutive +scheduled runs green at one recorded configuration; until then the suite is evidence, +not a gate. diff --git a/dev/live-e2e/adapter.mjs b/dev/live-e2e/adapter.mjs new file mode 100644 index 000000000..185822556 --- /dev/null +++ b/dev/live-e2e/adapter.mjs @@ -0,0 +1,163 @@ +// The FlyBot web adapter for the #204 scenarios: the only file in this suite that knows +// FlyBot's UI (docs/interop/live-e2e-adapter.md is the contract it implements; Orbit +// implements its own). Tasks go through the browser — the composer input for uploads, +// typing for messages, clicks for previews — while observation uses the app's own state +// API from inside the page and the run's storage on disk. Picking the Main Bot's model +// before the first task is environment setup, not a task step, and goes through the same +// in-page API like dev/memory-e2e.mjs does. +// +// playwright-core comes from the deepseek-harness checkout through the existing helper in +// core/dev/desktop-linux/lib.mjs (this repo installs no browser tooling). Chrome: +// CHROME_BIN, else google-chrome / chromium / chromium-browser on PATH, else playwright's +// own download. The login URL in web.log is used once and never printed. +import { execFileSync } from 'node:child_process' +import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs' +import { join } from 'node:path' +import { playwright, waitTeam } from '../../core/dev/desktop-linux/lib.mjs' + +export class AdapterError extends Error { + constructor(message) { + super(message) + this.name = 'AdapterError' + } +} + +const findChrome = () => { + if (process.env.CHROME_BIN) return process.env.CHROME_BIN + for (const name of ['google-chrome', 'chromium', 'chromium-browser']) { + try { return execFileSync('sh', ['-c', `command -v ${name}`], { encoding: 'utf8' }).trim() } catch {} + } + return undefined +} + +const sleep = ms => new Promise(done => setTimeout(done, ms)) + +// The run's storage lookup: the newest file with the name under the bench home, outside +// the profile, caches and screenshots. Uploads and artifacts both land there. +function findInHome(home, name) { + let best + const skip = new Set(['profiles', 'node_modules', 'cache', 'logs', 'shots']) + const walk = (dir, depth) => { + if (depth > 8) return + let entries + try { entries = readdirSync(dir, { withFileTypes: true }) } catch { return } + for (const entry of entries) { + if (entry.name.startsWith('.') || skip.has(entry.name)) continue + const path = join(dir, entry.name) + if (entry.isDirectory()) { walk(path, depth + 1); continue } + if (entry.name !== name) continue + const mtime = statSync(path).mtimeMs + if (!best || mtime > best.mtime) best = { path, mtime } + } + } + walk(home, 0) + return best?.path +} + +export async function openFlyBotWeb({ harness, home, port, model, outDir, quietMs, checkMs }) { + if (!existsSync(join(home, 'web.log'))) throw new AdapterError(`no web.log in ${home}; is the bench running?`) + const login = readFileSync(join(home, 'web.log'), 'utf8').match(new RegExp(`http://127\\.0\\.0\\.1:${port}/\\?token=[^\\s]+`))?.[0] + if (!login) throw new AdapterError(`no login URL in ${home}/web.log`) + + const { chromium } = playwright(harness) + const browser = await chromium.launch({ executablePath: findChrome(), headless: true }).catch(error => { throw new AdapterError(`Chrome did not launch (${error.message}); set CHROME_BIN`) }) + const page = await browser.newPage({ viewport: { width: 1400, height: 860 } }) + await page.goto(login) + // dsh shows a preview notice once per profile; clear it before any click. + const notice = page.getByRole('dialog').getByRole('button', { name: 'Continue' }) + await page.addLocatorHandler(notice, () => notice.click()) + await waitTeam(page) + + const api = async (endpoint, payload = {}) => { + const result = await page.evaluate(([e, p]) => fetch('api/bot', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ endpoint: e, payload: p }) }).then(r => r.json()), [endpoint, payload]) + if (!result.ok) throw new AdapterError(`${endpoint}: ${result.error?.message ?? JSON.stringify(result)}`) + return result.value + } + + // Setup: put the Main Bot on the asked model. A fresh team holds its greeting until + // then; the model card flow stays covered by the mock suites. + const view = await api('state') + const chief = view.bots.find(bot => bot.id === (view.mainBotId ?? view.mainBotIds?.[0])) + if (!chief) throw new AdapterError(`no Main Bot in a fresh team (${view.bots?.length ?? 0} bots)`) + const models = (await api('models')).models ?? view.models ?? [] + const ref = models.find(candidate => candidate.ref === model || candidate.ref.endsWith(`/${model}`))?.ref + ?? models.find(candidate => candidate.ref?.includes(model))?.ref + if (!ref) throw new AdapterError(`model ${model} is not offered; available: ${models.map(candidate => candidate.ref).join(', ') || 'none'}`) + if (chief.model !== ref) await api('update-bot', { id: chief.id, model: ref }) + await page.locator(`.bt-tile[data-bot-id="${chief.id}"]`).first().click() + await page.waitForTimeout(800) + const sent = [] + + const screenshot = async name => { await page.screenshot({ path: join(outDir, `${String(screenshot.index = (screenshot.index ?? 0) + 1).padStart(2, '0')}-${name}.png`) }) } + + // Quiet: no Bot busy and no memory review running, for quietMs, within checkMs. + async function settle(timeoutMs = checkMs) { + const deadline = Date.now() + timeoutMs + let since = Date.now() + for (;;) { + await sleep(800) + const current = await api('state') + const busy = Object.keys(current.activity ?? {}).length > 0 || (current.memoryReviews ?? 0) > 0 + if (busy) since = Date.now() + else if (Date.now() - since >= quietMs) return + if (Date.now() >= deadline) throw new AdapterError(`the team did not settle within ${timeoutMs} ms`) + } + } + + return { + modelId: ref, + page, + async uploadFile(name, bytes) { + const input = page.locator('[data-composer-card] input[type=file]').last() + await input.setInputFiles({ name, mimeType: 'text/csv', buffer: bytes }) + await page.waitForTimeout(600) + }, + async sendTask(text) { + sent.push(text) + const box = page.locator('[contenteditable="true"], textarea').last() + await box.click() + await page.keyboard.type(text) + await page.keyboard.press('Enter') + }, + settle, + async deliveredFileNames() { + return page.evaluate(() => { + const rows = [...document.querySelectorAll('[data-presented-files-row]')].map(node => node.textContent) + const chips = [...document.querySelectorAll('code')].map(node => node.textContent ?? '').filter(text => /\.json$/i.test(text.trim())) + return [...rows, ...chips.map(text => text.trim())] + }) + }, + async readFile(name) { + const path = findInHome(home, name) + return path === undefined ? undefined : readFileSync(path) + }, + async openPreview(name) { + const target = page.locator(`[data-presented-files-row]`).filter({ hasText: name }).last() + const chip = page.locator('code').filter({ hasText: name }).last() + const clickable = (await target.count()) > 0 ? target : chip + if ((await clickable.count()) === 0) throw new AdapterError(`nothing in the chat opens ${name}`) + await clickable.click() + await page.waitForTimeout(1500) + const text = await page.evaluate(() => document.querySelector('[data-slot="sidebar.right"]')?.innerText + ?? [...document.querySelectorAll('aside')].map(node => node.innerText).sort((a, b) => b.length - a.length)[0] + ?? '') + if (!text.trim()) throw new AdapterError(`clicking ${name} opened no preview`) + return text + }, + async reload() { + await page.reload({ waitUntil: 'load' }) + await waitTeam(page) + await page.locator(`.bt-tile[data-bot-id="${chief.id}"]`).first().click() + await page.waitForTimeout(1200) + }, + // Who said what is app-specific to render; for the scenarios it is enough that the + // task texts are still visible and that some reply beside them is. + async conversationLines() { + const text = await page.evaluate(() => document.body.innerText) + const lines = text.split('\n').map(line => line.trim()).filter(line => line.length > 0) + return lines.map(line => ({ who: sent.some(task => task.slice(0, 24) && line.includes(task.slice(0, 24))) ? 'user' : 'app', text: line })) + }, + screenshot, + async close() { await browser.close().catch(() => {}) }, + } +} diff --git a/dev/live-e2e/bench.mjs b/dev/live-e2e/bench.mjs new file mode 100644 index 000000000..96541d731 --- /dev/null +++ b/dev/live-e2e/bench.mjs @@ -0,0 +1,151 @@ +// Real-model test bench for the live E2E suite (#204): a dsh web whose profile links +// this checkout, talking to the real DeepSeek endpoint with DEEPSEEK_API_KEY from the +// caller's environment. The mock bench stays core/dev/bench.sh; this one never starts a +// mock and refuses to run without a real key, so the two suites cannot mix. +// +// node dev/live-e2e/bench.mjs [start|stop|reset|status] +// +// Data lives in /.dsh-test- (never under ~/.dsh; `reset` wipes the Bot team +// and Sessions before starting). Environment: DEEPSEEK_API_KEY (required; fails loudly +// when missing or the mock's), DEEPSEEK_BASE_URL (optional; defaults to the official +// endpoint — point it at dev/llm-tap.mjs to record per-call usage), DSH_HARNESS (a +// deepseek-harness checkout with its dependencies installed). +import { spawn, spawnSync } from 'node:child_process' +import { existsSync, mkdirSync, openSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { homedir } from 'node:os' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const ROOT = dirname(dirname(dirname(fileURLToPath(import.meta.url)))) +const [NAME, PORT_TEXT, ACTION = 'start'] = process.argv.slice(2) +const die = (message) => { console.error(message); process.exit(1) } +if (!NAME || !/^[a-z0-9-]+$/.test(NAME) || !PORT_TEXT) die('usage: node dev/live-e2e/bench.mjs [start|stop|reset|status]') +const WEB_PORT = Number(PORT_TEXT) +if (!Number.isInteger(WEB_PORT) || WEB_PORT <= 0) die(`not a port: ${PORT_TEXT}`) +if (WEB_PORT === 3080) die('port 3080 belongs to the real service') + +const HOME_DIR = join(ROOT, `.dsh-test-${NAME}`) +if (HOME_DIR === homedir() || HOME_DIR.startsWith(`${homedir()}/.dsh`)) die(`refusing to use ${HOME_DIR}: the bench never touches the user's own DSH data`) + +const apiKey = process.env.DEEPSEEK_API_KEY ?? '' +if (!apiKey || apiKey === 'mock-key' || apiKey.length < 16) die('DEEPSEEK_API_KEY is missing, too short, or the mock\'s; a real key is required (never printed)') + +const harness = process.env.DSH_HARNESS ?? join(ROOT, 'deepseek-harness') +if (!existsSync(join(harness, 'package.json'))) die(`no deepseek-harness checkout at ${harness}; set DSH_HARNESS (see CONTRIBUTING.md)`) + +const PLUGIN = join(ROOT, 'core') +const webPidFile = join(HOME_DIR, 'web.pid') +const logFile = join(HOME_DIR, 'web.log') +const sleep = ms => new Promise(done => setTimeout(done, ms)) +const alive = (pid) => { try { process.kill(-pid, 0); return true } catch { return false } } + +async function stopWeb() { + if (!existsSync(webPidFile)) return + const pid = Number(readFileSync(webPidFile, 'utf8').trim()) + try { process.kill(-pid, 'SIGTERM') } catch {} + for (let i = 0; i < 50 && alive(pid); i += 1) await sleep(200) + try { process.kill(-pid, 'SIGKILL') } catch {} + rmSync(webPidFile, { force: true }) +} + +function ensureProfile() { + const profile = join(HOME_DIR, 'profiles/web') + const manifest = join(profile, 'package.json') + // The install marker is the linked package itself: a crash between writing the + // manifest and finishing the install must not look like a done install. + if (existsSync(manifest) && readFileSync(manifest, 'utf8').includes(`"link:${PLUGIN}"`) && existsSync(join(profile, 'node_modules/ds-bot'))) return + mkdirSync(profile, { recursive: true }) + // Pin the harness checkout's own package manager, so corepack runs the version the + // harness expects instead of a cached default. + let packageManager = '' + try { packageManager = JSON.parse(readFileSync(join(harness, 'package.json'), 'utf8')).packageManager ?? '' } catch {} + const manifestContent = { + name: 'dsh-profile-web', + private: true, + ...(packageManager ? { packageManager } : {}), + dependencies: { 'ds-bot': `link:${PLUGIN}` }, + dsh: { profile: { bundles: ['@deepseek-ai/dsh-base', '@deepseek-ai/dsh-web-app', 'ds-bot'] } }, + } + writeFileSync(manifest, `${JSON.stringify(manifestContent, null, 2)}\n`) + writeFileSync(join(profile, 'pnpm-workspace.yaml'), 'packages:\n - .\n\nnodeLinker: hoisted\nautoInstallPeers: false\n') + const result = spawnSync('pnpm', ['install'], { cwd: profile, stdio: 'inherit', env: { ...process.env, DSH_HOME: HOME_DIR } }) + if (result.status !== 0) die(`pnpm install failed in ${profile}`) +} + +async function start() { + mkdirSync(HOME_DIR, { recursive: true }) + ensureProfile() + // The real catalog supplies its own context window: no models overlay. + writeFileSync(join(HOME_DIR, 'bench.patch.yml'), '[]\n') + if (existsSync(webPidFile) && alive(Number(readFileSync(webPidFile, 'utf8')))) return status() + rmSync(logFile, { force: true }) + // The login token URL lands in web.log, which the evidence sanitizer strips before + // anything is uploaded. Detached, the child is its own process group: the pid file + // names the group, and a later stop reaches the whole tree. + const out = openSync(logFile, 'a') + const child = spawn('pnpm', ['-s', 'dsh', '--profile', 'web', '--patch', 'apps/web/tests/pin-browse-picker.overlay.yml', '--patch', join(HOME_DIR, 'bench.patch.yml'), '--no-open', '--port', String(WEB_PORT)], { + cwd: harness, + detached: true, + stdio: ['ignore', out, out], + env: { + ...process.env, + DSH_HOME: HOME_DIR, + DEEPSEEK_API_KEY: apiKey, + ...(process.env.DEEPSEEK_BASE_URL ? { DEEPSEEK_BASE_URL: process.env.DEEPSEEK_BASE_URL } : {}), + }, + }) + child.unref() + writeFileSync(webPidFile, `${child.pid}\n`) + for (let i = 0; i < 180; i += 1) { + await sleep(1000) + if (existsSync(logFile) && readFileSync(logFile, 'utf8').includes('dsh web: http')) return status() + if (!alive(child.pid)) { + await stopWeb() + die(`dsh web exited early; see ${logFile} (tokens redacted):\n${tailSanitized()}`) + } + } + await stopWeb() + die(`dsh web did not start; see ${logFile} (tokens redacted):\n${tailSanitized()}`) +} + +function tailSanitized() { + if (!existsSync(logFile)) return '' + return readFileSync(logFile, 'utf8').split('\n').slice(-20).map(line => line.replace(/\?token=\S+/g, '?token=…')).join('\n') +} + +function status() { + const running = existsSync(webPidFile) && alive(Number(readFileSync(webPidFile, 'utf8'))) + console.log(`web: ${running ? `running (pid ${readFileSync(webPidFile, 'utf8').trim()})` : 'stopped'}`) + console.log(`home: ${HOME_DIR} web: http://127.0.0.1:${WEB_PORT} model: real (key from the environment)`) +} + +switch (ACTION) { + case 'start': + await start() + break + case 'stop': + await stopWeb() + status() + break + case 'reset': { + await stopWeb() + rmSync(join(HOME_DIR, 'cookies.txt'), { force: true }) + rmSync(join(HOME_DIR, 'bot'), { recursive: true, force: true }) + const sessions = join(HOME_DIR, 'sessions') + if (existsSync(sessions)) { + for (const dir of readdirSync(sessions)) { + if (!dir.includes('workspace')) continue + for (const entry of readdirSync(join(sessions, dir))) { + if (entry.startsWith('session-')) rmSync(join(sessions, dir, entry), { recursive: true, force: true }) + } + } + } + await start() + break + } + case 'status': + status() + break + default: + die(`unknown action ${ACTION}`) +} diff --git a/dev/live-e2e/fixtures/orders.csv b/dev/live-e2e/fixtures/orders.csv new file mode 100644 index 000000000..74fd5c71c --- /dev/null +++ b/dev/live-e2e/fixtures/orders.csv @@ -0,0 +1,6 @@ +order_id,region,status,amount +A001,华东,paid,100 +A002,华北,paid,200 +A003,华东,pending,300 +A004,华东,paid,40 +A005,华北,paid,60 diff --git a/dev/live-e2e/report.mjs b/dev/live-e2e/report.mjs new file mode 100644 index 000000000..000e45773 --- /dev/null +++ b/dev/live-e2e/report.mjs @@ -0,0 +1,135 @@ +// The outcome contract and the report assembly for the #204 live runner. Pure code, +// covered keyless by test/report.test.mjs. +// +// One run ends in exactly one outcome: +// pass (exit 0) every check passed within its budgets +// assertion_failed (exit 1) the app did the task wrong (content, immutability, preview) +// provider_failed (exit 2) the model endpoint rejected or failed a call (401/429/5xx) +// infra_failed (exit 3) bench, browser, secret or guard failures; never reported as a pass +// budget_exceeded (exit 4) a check or the token cap ran out; never retried silently +export const OUTCOMES = { pass: 0, assertion_failed: 1, provider_failed: 2, infra_failed: 3, budget_exceeded: 4 } + +export const DEFAULT_BUDGETS = { checkMs: 180_000, quietMs: 6_000, maxTokens: 250_000 } + +export function budgetsFrom(env) { + const num = (value, fallback) => { + const parsed = Number(value) + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback + } + return { + checkMs: num(env.LIVE_E2E_CHECK_MS, DEFAULT_BUDGETS.checkMs), + quietMs: num(env.LIVE_E2E_QUIET_MS, DEFAULT_BUDGETS.quietMs), + maxTokens: num(env.LIVE_E2E_MAX_TOKENS, DEFAULT_BUDGETS.maxTokens), + } +} + +// Fail-closed checks on the run's inputs. A loopback base URL is the mock bench (or the +// usage tap, which names the real upstream in LIVE_E2E_TAP_UPSTREAM); anything else +// without a key, or with the mock's key, is refused rather than run. +export function guardRun({ apiKey, baseUrl, tapUpstream, home, userHome }) { + const problems = [] + if (!apiKey || apiKey === 'mock-key' || apiKey.length < 16) problems.push('DEEPSEEK_API_KEY is missing, too short, or the mock key') + let host = 'api.deepseek.com' + let origin = 'default official endpoint' + if (baseUrl) { + try { + host = new URL(baseUrl).host + origin = baseUrl + } catch { problems.push(`DEEPSEEK_BASE_URL is not a URL: ${baseUrl}`) } + } + const loopback = /^(localhost|127\.|0\.0\.0\.0|\[::1\]|::1)/.test(host) + if (loopback) { + if (tapUpstream) { + try { + const upstream = new URL(tapUpstream).host + if (/^(localhost|127\.|0\.0\.0\.0|\[::1\]|::1)/.test(upstream)) problems.push(`the usage tap's upstream is also loopback: ${tapUpstream}`) + } catch { problems.push(`LIVE_E2E_TAP_UPSTREAM is not a URL: ${tapUpstream}`) } + } else { + problems.push(`DEEPSEEK_BASE_URL is loopback (${host}); the live suite never runs against the mock`) + } + } + if (home && userHome && (home === userHome || home.startsWith(`${userHome}/.dsh`))) { + problems.push(`the bench home ${home} is inside the user's own DSH data`) + } + return { problems, endpoint: { host, origin, viaTap: loopback } } +} + +// The provider's health since a mark: llm-tap writes one line per call, and a failed +// call keeps `status`/`error` instead of `text`. +export function providerFailures(tapLines) { + return tapLines.filter(line => line && line.status !== undefined && !String(line.status).startsWith('2') && !String(line.status).startsWith('3')) + .map(line => `${line.status ?? 'error'} ${line.error ?? ''}`.trim()) +} + +export function usageOf(tapLines) { + const usage = { calls: tapLines.length, prompt: 0, cached: 0, output: 0 } + for (const line of tapLines) { + const tokens = line.tokens + if (!tokens) continue + usage.prompt += tokens.prompt ?? 0 + usage.cached += tokens.cached ?? 0 + usage.output += tokens.output ?? 0 + } + return usage +} + +export function classify({ checkOutcomes, providerFailures: provider, tokenUsage, budgets }) { + if (tokenUsage.prompt + tokenUsage.output > budgets.maxTokens) return 'budget_exceeded' + if (provider.length > 0) return 'provider_failed' + if (checkOutcomes.includes('assertion_failed')) return 'assertion_failed' + if (checkOutcomes.includes('budget_exceeded')) return 'budget_exceeded' + if (checkOutcomes.includes('infra_failed')) return 'infra_failed' + return 'pass' +} + +// Absolute timestamps with an explicit offset, plus the zone the run was asked to report +// in (Asia/Shanghai in CI and the pilot); durations are monotonic milliseconds the runner +// measured with performance.now(). +export function buildReport({ runId, revision, config, endpoint, budgets, checks, usage, providerFailures = [], startedAt, endedAt, tz }) { + const outcome = classify({ checkOutcomes: checks.map(check => check.outcome), providerFailures, tokenUsage: usage, budgets }) + const offset = date => { + const minutes = -date.getTimezoneOffset() + const sign = minutes < 0 ? '-' : '+' + const abs = Math.abs(minutes) + return `${sign}${String(Math.floor(abs / 60)).padStart(2, '0')}:${String(abs % 60).padStart(2, '0')}` + } + const human = (date) => { + try { + return new Intl.DateTimeFormat('en-GB', { timeZone: tz, dateStyle: 'medium', timeStyle: 'medium' }).format(date) + } catch { return date.toISOString() } + } + return { + schema: 'flybot-live-e2e/1', + runId, + revision, + config, + endpoint, + budgets, + startedAt: { iso: startedAt.toISOString(), offset: offset(startedAt) }, + endedAt: { iso: endedAt.toISOString(), offset: offset(endedAt) }, + humanTimes: { zone: tz, startedAt: human(startedAt), endedAt: human(endedAt) }, + checks, + usage, + outcome, + exitCode: OUTCOMES[outcome], + } +} + +export function summaryLines(report) { + const lines = [ + `run ${report.runId} (${report.revision})`, + `model ${report.config.model} via ${report.endpoint.viaTap ? `tap ${report.endpoint.origin} -> ${report.endpoint.host}` : report.endpoint.origin}`, + `started ${report.startedAt.iso} (UTC${report.startedAt.offset}) / ${report.humanTimes.startedAt} ${report.humanTimes.zone}`, + `ended ${report.endedAt.iso} (UTC${report.endedAt.offset}) / ${report.humanTimes.endedAt} ${report.humanTimes.zone}`, + `usage ${report.usage.calls} calls, ${report.usage.prompt} prompt tokens (${report.usage.cached} cached), ${report.usage.output} output; cap ${report.budgets.maxTokens}`, + ] + for (const check of report.checks) { + lines.push(`${check.outcome === 'pass' ? 'ok ' : 'FAIL'} ${check.id} (${(check.durationMs ?? 0).toFixed(0)} ms) ${check.title}`) + for (const problem of check.problems ?? []) lines.push(` ${problem}`) + } + for (const problem of report.guardProblems ?? []) lines.push(`guard ${problem}`) + if (report.stoppedBecause) lines.push(`stopped ${report.stoppedBecause}`) + for (const finding of report.sanitizerFindings ?? []) lines.push(`unsanitized ${finding}`) + lines.push(`outcome ${report.outcome} (exit ${report.exitCode})`) + return lines +} diff --git a/dev/live-e2e/run.mjs b/dev/live-e2e/run.mjs new file mode 100644 index 000000000..89607bbc4 --- /dev/null +++ b/dev/live-e2e/run.mjs @@ -0,0 +1,147 @@ +// The live runner for #204: one run, one fresh bench, four checks, no retries. Tasks go +// through the FlyBot web adapter; usage comes from an llm-tap log when one is given. +// Fails closed on every guard (missing/mock key, loopback endpoint, the user's own DSH +// home) and never reports a pass it did not see. Evidence is written under --evidence +// and scanned by sanitize.mjs; a finding there is an infra failure, and the bundle is +// not fit to upload. +// +// node dev/live-e2e/run.mjs [--model deepseek-official/deepseek-flash] +// [--harness ] [--tap ] +// [--evidence ] [--tz Asia/Shanghai] +// +// Exit codes: 0 pass, 1 assertion_failed, 2 provider_failed, 3 infra_failed, 4 budget_exceeded. +import { execFileSync } from 'node:child_process' +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { homedir } from 'node:os' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { parseArgs } from 'node:util' +import { randomUUID } from 'node:crypto' +import { runScenarios } from './scenarios.mjs' +import { budgetsFrom, buildReport, guardRun, OUTCOMES, providerFailures, summaryLines, usageOf } from './report.mjs' +import { scanTree } from './sanitize.mjs' +import { openFlyBotWeb } from './adapter.mjs' + +const ROOT = dirname(dirname(dirname(fileURLToPath(import.meta.url)))) +const { values: args, positionals } = parseArgs({ + allowPositionals: true, + options: { + model: { type: 'string', default: 'deepseek-official/deepseek-flash' }, + harness: { type: 'string' }, + tap: { type: 'string' }, + evidence: { type: 'string' }, + tz: { type: 'string', default: process.env.TZ || 'Asia/Shanghai' }, + }, +}) +const [HOME, PORT] = positionals +if (!HOME || !PORT) { + console.error('usage: node dev/live-e2e/run.mjs [--model ref] [--harness dir] [--tap file] [--evidence dir] [--tz zone]') + process.exit(OUTCOMES.infra_failed) +} + +const budgets = budgetsFrom(process.env) +const evidence = args.evidence ?? join(HOME, 'evidence') +const startedAt = new Date() +const runId = randomUUID() +const revision = execFileSync('git', ['-C', ROOT, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim() +const dshVersion = JSON.parse(readFileSync(join(ROOT, 'core/dev/harness/package.json'), 'utf8')).dependencies['@deepseek-ai/dsh'] +const tapLines = () => (args.tap && existsSync(args.tap) ? readFileSync(args.tap, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line)) : []) + +const writeReport = (report) => { + mkdirSync(evidence, { recursive: true }) + writeFileSync(join(evidence, 'report.json'), `${JSON.stringify(report, null, 2)}\n`) +} +const conclude = (report) => { + writeReport(report) + console.log(summaryLines(report).join('\n')) + process.exit(report.exitCode) +} +const stoppedEarly = (outcome, problems) => { + const report = buildReport({ + runId, revision, + config: { node: process.version, dsh: dshVersion, model: args.model }, + endpoint: { host: 'not reached', origin: 'not reached', viaTap: false }, + budgets, checks: [], usage: usageOf(tapLines()), providerFailures: providerFailures(tapLines()), + startedAt, endedAt: new Date(), tz: args.tz, + }) + report.outcome = outcome + report.exitCode = OUTCOMES[outcome] + report.guardProblems = problems + return report +} + +// Guards first: never open a browser — or write anything — against a bench that should +// not exist, so a guard failure prints and exits without touching the passed home. +const guard = guardRun({ apiKey: process.env.DEEPSEEK_API_KEY, baseUrl: process.env.DEEPSEEK_BASE_URL, tapUpstream: process.env.LIVE_E2E_TAP_UPSTREAM, home: HOME, userHome: homedir() }) +if (guard.problems.length > 0) { + console.error(guard.problems.join('\n')) + console.error(`outcome infra_failed (exit ${OUTCOMES.infra_failed})`) + process.exit(OUTCOMES.infra_failed) +} +const harness = args.harness ?? process.env.DSH_HARNESS ?? join(ROOT, 'deepseek-harness') +if (!existsSync(harness)) conclude(stoppedEarly('infra_failed', [`no deepseek-harness checkout at ${harness}; pass --harness or set DSH_HARNESS`])) +mkdirSync(evidence, { recursive: true }) + +let adapter +try { + adapter = await openFlyBotWeb({ harness, home: HOME, port: Number(PORT), model: args.model, outDir: evidence, quietMs: budgets.quietMs, checkMs: budgets.checkMs }) +} catch (error) { + const provider = providerFailures(tapLines()) + conclude(stoppedEarly(provider.length > 0 ? 'provider_failed' : 'infra_failed', [error.message])) +} + +let checks = [] +let override +try { + const { results, stopped } = await runScenarios(adapter, { + checkMs: budgets.checkMs, + betweenChecks: () => { + const usage = usageOf(tapLines()) + return usage.prompt + usage.output > budgets.maxTokens + ? { stop: true, reason: `the token cap was reached (${usage.prompt + usage.output} > ${budgets.maxTokens})` } + : {} + }, + }) + checks = results + if (stopped) override = { outcome: 'budget_exceeded', problem: stopped } +} catch (error) { + // Adapter failures (bench, browser, timeouts) are infra unless the provider explains + // them; assertion mismatches are already inside the checks. + const provider = providerFailures(tapLines()) + override = { outcome: provider.length > 0 ? 'provider_failed' : 'infra_failed', problem: error.message } + checks.push({ id: 'run', title: 'the run itself', outcome: override.outcome, problems: [error.message] }) + await adapter.screenshot('failure').catch(() => {}) +} + +const report = buildReport({ + runId, revision, + config: { node: process.version, dsh: dshVersion, model: adapter.modelId ?? args.model }, + endpoint: guard.endpoint, + budgets, checks, usage: usageOf(tapLines()), + providerFailures: providerFailures(tapLines()), + startedAt, endedAt: new Date(), tz: args.tz, +}) +if (override) { + report.outcome = override.outcome + report.exitCode = OUTCOMES[override.outcome] + report.stoppedBecause = override.problem +} + +const transcript = await adapter.conversationLines().catch(() => []) +writeFileSync(join(evidence, 'transcript.txt'), transcript.map(line => `${line.who}: ${line.text}`).join('\n')) +await adapter.close().catch(() => {}) + +// Evidence gate: a finding means the bundle is not fit to upload; that is infra, not a +// pass. The stamp is written last, only after a clean scan and the finished report, and +// the workflow's upload step requires it — so a contaminated bundle never becomes an +// artifact (it stays on the bench for debugging). +const findings = scanTree(evidence, process.env.DEEPSEEK_API_KEY) +if (findings.length > 0) { + report.outcome = 'infra_failed' + report.exitCode = OUTCOMES.infra_failed + report.sanitizerFindings = findings +} +writeReport(report) +if (findings.length === 0) writeFileSync(join(evidence, 'sanitized.txt'), `run ${runId} sanitized at ${new Date().toISOString()}\n`) +console.log(summaryLines(report).join('\n')) +process.exit(report.exitCode) diff --git a/dev/live-e2e/sanitize.mjs b/dev/live-e2e/sanitize.mjs new file mode 100644 index 000000000..bb2348329 --- /dev/null +++ b/dev/live-e2e/sanitize.mjs @@ -0,0 +1,37 @@ +// The evidence gate: everything uploaded from a live run passes scanTree first. CI +// logs and artifacts of a public repository are world-readable, so this fails closed — +// a finding means the evidence is not uploaded and the run is infra_failed, never a pass. +// It looks for the web app's `?token=` login URLs, cookie material, and the secret's own +// value wherever it appears. Findings name the file and the kind of hit only: the matched +// text itself is what the scan exists to keep out, so it never enters a report. +import { readFileSync, readdirSync, statSync } from 'node:fs' +import { join } from 'node:path' + +const TOKEN_URL = /\?token=[^"'\s]+/ +const COOKIE = /\b(set-cookie|x-flybot-member-session)\b.{0,80}/i + +export function scanText(text, secret) { + const findings = [] + if (TOKEN_URL.test(text)) findings.push('login token URL') + if (COOKIE.test(text)) findings.push('cookie material') + if (secret && secret.length >= 8 && text.includes(secret)) findings.push('the API key value appears verbatim') + return findings +} + +export function scanTree(root, secret) { + const findings = [] + const walk = (dir) => { + for (const entry of readdirSync(dir)) { + if (entry.startsWith('.') || entry === 'node_modules' || entry === 'sanitized.txt') continue + const path = join(dir, entry) + const info = statSync(path) + if (info.isDirectory()) { walk(path); continue } + // Binary evidence (PNGs) is skipped: only text can hide a secret. + if (/\.(png|jpg|jpeg|webp|gif)$/i.test(entry)) continue + const text = readFileSync(path, 'utf8').replaceAll('\u0000', '') + for (const finding of scanText(text, secret)) findings.push(`${path}: ${finding}`) + } + } + walk(root) + return findings +} diff --git a/dev/live-e2e/scenarios.mjs b/dev/live-e2e/scenarios.mjs new file mode 100644 index 000000000..8e06d078e --- /dev/null +++ b/dev/live-e2e/scenarios.mjs @@ -0,0 +1,174 @@ +// The four #204 checks as app-independent code: everything here talks to an adapter +// (docs/interop/live-e2e-adapter.md) and never to FlyBot, DSH or a browser, so the same +// scenarios can run against any app that implements the contract, and the logic is +// covered keyless by test/scenarios.test.mjs with a fake adapter. The runner (run.mjs) +// owns time, budgets, classification and the report; scenarios only sequence and assert. +// +// Tasks go through the UI (uploadFile/sendTask/openPreview/reload); observation is the +// adapter's read-only side (readFile/deliveredFileNames/conversationLines), which for +// FlyBot reads the run's storage and the app's own state API. A filename echoed in the +// task reply is never evidence of delivery: assertions read the delivered bytes and the +// preview the UI shows. +import { createHash } from 'node:crypto' + +export const sha256 = bytes => createHash('sha256').update(bytes).digest('hex') + +// The fixture is fixed on purpose: the expected totals (4 / 400 and 2 / 140) depend on +// it, and freshness comes from a new run id and home per run, not from new data. +export const FIXTURE = 'orders.csv' + +// What the checks ask for, in the app's own task language. The wording may change; the +// assertions below must not. +export const TASKS = { + summarize: '把 orders.csv 里所有已支付（status 为 paid）的订单整理成 summary.json：包含每笔订单的完整信息（order_id、region、status、amount）、总笔数 count 和总金额 total。', + refine: '重新整理：只要华东地区已支付的订单，另存为 summary-east.json（同样包含订单明细、count 和 total）。不要改动 orders.csv 和 summary.json。', +} + +export function parseOrdersCsv(text) { + const lines = String(text).split(/\r?\n/).filter(Boolean) + const head = lines[0]?.split(',').map(name => name.trim()) + if (!head || !head.includes('order_id') || !head.includes('amount')) throw new Error(`unexpected CSV header: ${lines[0]}`) + return lines.slice(1).map(line => { + const cells = line.split(',').map(cell => cell.trim()) + const row = {} + head.forEach((name, index) => { row[name] = cells[index] }) + row.amount = Number(row.amount) + return row + }) +} + +// The expected answers, derived from the fixture rather than copied, so a fixture edit +// that breaks the stated totals fails its own unit test instead of the nightly. +export function expectations(rows) { + const paid = rows.filter(row => row.status === 'paid') + const east = paid.filter(row => row.region === '华东') + const shape = list => ({ + ids: list.map(row => row.order_id).sort(), + count: list.length, + total: list.reduce((sum, row) => sum + row.amount, 0), + rows: [...list].sort((a, b) => a.order_id.localeCompare(b.order_id)), + }) + return { paid: shape(paid), east: shape(east) } +} + +export class AssertionError extends Error {} + +const fail = message => { throw new AssertionError(message) } + +// Delivered JSON is compared parsed, never by bytes: indentation, key order and \u +// escaping differ between writers. `count` and `total` must be real numbers. +export function parseSummary(name, text) { + let json + try { json = JSON.parse(String(text)) } catch (error) { fail(`${name} is not JSON: ${error.message}`) } + if (typeof json.count !== 'number' || !Number.isFinite(json.count)) fail(`${name}.count is not a number: ${JSON.stringify(json.count)}`) + if (typeof json.total !== 'number' || !Number.isFinite(json.total)) fail(`${name}.total is not a number: ${JSON.stringify(json.total)}`) + if (!Array.isArray(json.orders) || json.orders.length === 0) fail(`${name}.orders is not a non-empty list`) + return json +} + +export function assertSummaryMatches(name, json, expected) { + const problems = [] + const ids = json.orders.map(row => String(row.order_id)).sort() + if (ids.join(',') !== expected.ids.join(',')) problems.push(`${name} ids ${ids.join(',')} != ${expected.ids.join(',')}`) + if (json.count !== expected.count) problems.push(`${name}.count ${json.count} != ${expected.count}`) + if (json.total !== expected.total) problems.push(`${name}.total ${json.total} != ${expected.total}`) + for (const row of json.orders) { + const want = expected.rows.find(candidate => candidate.order_id === String(row.order_id)) + if (!want) continue + if (Number(row.amount) !== want.amount) problems.push(`${name} ${row.order_id} amount ${row.amount} != ${want.amount}`) + if (String(row.region) !== want.region) problems.push(`${name} ${row.order_id} region ${row.region} != ${want.region}`) + if (String(row.status) !== want.status) problems.push(`${name} ${row.order_id} status ${row.status} != ${want.status}`) + } + if (problems.length > 0) fail(problems.join('; ')) +} + +function assertSameJson(shown, actual, name) { + let left + let right + try { + left = JSON.parse(String(shown)) + right = JSON.parse(String(actual)) + } catch (error) { fail(`${name} preview or file is not JSON: ${error.message}`) } + if (JSON.stringify(left) !== JSON.stringify(right)) fail(`${name}: the preview does not show the file's JSON`) +} + +const read = async (adapter, name) => { + const bytes = await adapter.readFile(name) + if (bytes === undefined) fail(`${name} was not found in the run's storage`) + return bytes +} + +// Runs the four checks against one adapter, in one conversation. Each check records its +// own monotonic duration; `betweenChecks(entry)` runs after each and may stop the run +// ({ stop: true, reason }) — the runner uses it for the token budget. Classification of +// the run as a whole stays with the runner (report.mjs). +export async function runScenarios(adapter, { checkMs, betweenChecks }) { + const csv = await read(adapter, FIXTURE) + const csvSha = sha256(csv) + const expected = expectations(parseOrdersCsv(csv.toString('utf8'))) + const checks = [] + let summarySha + + const check = (id, title) => { + const entry = { id, title, problems: [] } + checks.push(entry) + return entry + } + let stopped + const step = async (id, title, run) => { + const entry = check(id, title) + const started = performance.now() + try { await run() } catch (error) { entry.problems.push(error.message) } + entry.durationMs = performance.now() - started + const verdict = betweenChecks ? await betweenChecks(entry) : undefined + if (verdict?.stop) { stopped = verdict.reason ?? 'stopped between checks'; return false } + return true + } + + if (!await step('upload-and-summarize', 'upload the CSV; summary.json lists the paid orders', async () => { + await adapter.uploadFile(FIXTURE, csv) + await adapter.sendTask(TASKS.summarize) + await adapter.settle(checkMs) + if (!(await adapter.deliveredFileNames()).includes('summary.json')) fail('the UI does not list summary.json as delivered') + const summary = await read(adapter, 'summary.json') + assertSummaryMatches('summary.json', parseSummary('summary.json', summary.toString('utf8')), expected.paid) + summarySha = sha256(summary) + })) return finish() + + if (!await step('refine-and-preserve', 'summary-east.json narrows to 华东 paid; earlier files keep their bytes', async () => { + await adapter.sendTask(TASKS.refine) + await adapter.settle(checkMs) + if (!(await adapter.deliveredFileNames()).includes('summary-east.json')) fail('the UI does not list summary-east.json as delivered') + const east = await read(adapter, 'summary-east.json') + assertSummaryMatches('summary-east.json', parseSummary('summary-east.json', east.toString('utf8')), expected.east) + if (sha256(await read(adapter, 'summary.json')) !== summarySha) fail('summary.json changed during the refinement') + if (sha256(await read(adapter, FIXTURE)) !== csvSha) fail('the uploaded orders.csv changed during the run') + })) return finish() + + if (!await step('survive-reload', 'a reload keeps the conversation and both artifact cards', async () => { + await adapter.reload() + const visible = await adapter.conversationLines() + for (const text of [TASKS.summarize, TASKS.refine]) { + if (!visible.some(line => text.slice(0, 24) && line.text.includes(text.slice(0, 24)))) fail(`after reload the task is gone from the conversation: ${text.slice(0, 24)}…`) + } + if (visible.filter(line => line.who !== 'user').length === 0) fail('after reload no reply is visible') + const names = await adapter.deliveredFileNames() + for (const name of ['summary.json', 'summary-east.json']) { + if (!names.includes(name)) fail(`after reload the UI no longer lists ${name} as delivered`) + } + })) return finish() + + await step('open-previews', 'the sidebar previews show the delivered files\' JSON', async () => { + for (const name of ['summary.json', 'summary-east.json']) { + const shown = await adapter.openPreview(name) + const actual = await read(adapter, name) + assertSameJson(shown, actual.toString('utf8'), name) + } + }) + + function finish() { + const results = checks.map(({ problems, ...entry }) => ({ ...entry, outcome: problems.length === 0 ? 'pass' : 'assertion_failed', problems })) + return { results, stopped } + } + return finish() +} diff --git a/dev/live-e2e/test/fake-adapter.mjs b/dev/live-e2e/test/fake-adapter.mjs new file mode 100644 index 000000000..0b139ecd1 --- /dev/null +++ b/dev/live-e2e/test/fake-adapter.mjs @@ -0,0 +1,63 @@ +// A fake adapter for the keyless tests: an in-memory app that can be broken on purpose. +// It implements the whole contract of docs/interop/live-e2e-adapter.md so the scenario +// logic runs exactly as it does against FlyBot's web UI — minus the browser, the model +// and any network. +import { readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +export const fixtureCsv = () => readFileSync(join(dirname(fileURLToPath(import.meta.url)), '..', 'fixtures', 'orders.csv')) + +const paidJson = rows => { + const orders = rows.filter(row => row.status === 'paid') + return `${JSON.stringify({ orders, count: orders.length, total: orders.reduce((sum, row) => sum + row.amount, 0) }, null, 2)}\n` +} + +export class FakeAdapter { + constructor({ breakIds, mutateSummary, stalePreview, neverSettle } = {}) { + this.files = new Map([['orders.csv', fixtureCsv()]]) + this.conversation = [] + this.delivered = [] + this.breakIds = breakIds + this.mutateSummary = mutateSummary + this.stalePreview = stalePreview + this.neverSettle = neverSettle + } + + async uploadFile(name, bytes) { this.files.set(name, bytes) } + + async sendTask(text) { + this.conversation.push({ who: 'user', text }) + this.conversation.push({ who: 'app', text: '好的，我来处理。' }) + if (this.neverSettle) return + const rows = this.files.get('orders.csv').toString('utf8').split('\n').slice(1).filter(Boolean) + .map(line => { const [order_id, region, status, amount] = line.split(',').map(cell => cell.trim()); return { order_id, region, status, amount: Number(amount) } }) + if (!this.files.has('summary.json')) { + const paid = rows.filter(row => row.status === 'paid') + if (this.breakIds) paid[0].order_id = 'A099' + this.files.set('summary.json', Buffer.from(paidJson(paid))) + this.delivered.push('summary.json') + this.conversation.push({ who: 'app', text: '已生成 summary.json。' }) + } else if (!this.files.has('summary-east.json')) { + if (this.mutateSummary) this.files.set('summary.json', Buffer.from(`${paidJson(rows.filter(row => row.status === 'paid'))}// touched\n`)) + const east = rows.filter(row => row.status === 'paid' && row.region === '华东') + this.files.set('summary-east.json', Buffer.from(paidJson(east))) + this.delivered.push('summary-east.json') + this.conversation.push({ who: 'app', text: '已生成 summary-east.json。' }) + } + } + + async settle() { if (this.neverSettle) throw new Error('the team did not settle within 1 ms') } + + async deliveredFileNames() { return [...this.delivered] } + + async readFile(name) { return this.files.get(name) } + + async openPreview(name) { return this.stalePreview && name === 'summary.json' ? '{"orders": [], "count": 0, "total": 0}' : this.files.get(name)?.toString('utf8') ?? '' } + + async reload() { this.reloaded = true } + + async conversationLines() { return this.conversation.map(line => ({ ...line })) } + + async screenshot() { this.shots = (this.shots ?? 0) + 1 } +} diff --git a/dev/live-e2e/test/scenarios.test.mjs b/dev/live-e2e/test/scenarios.test.mjs new file mode 100644 index 000000000..5df25f8fd --- /dev/null +++ b/dev/live-e2e/test/scenarios.test.mjs @@ -0,0 +1,151 @@ +// Keyless coverage of the #204 scenario logic (node --test, no network, no browser, +// no key). Each case breaks the fake app in one specific way and the matching assertion +// must name it. +import { strict as assert } from 'node:assert' +import test from 'node:test' +import { FakeAdapter, fixtureCsv } from './fake-adapter.mjs' +import { expectations, parseOrdersCsv, parseSummary, runScenarios, sha256, TASKS } from '../scenarios.mjs' +import { buildReport, budgetsFrom, classify, guardRun, OUTCOMES, providerFailures, usageOf } from '../report.mjs' +import { scanText } from '../sanitize.mjs' + +test('the fixture derives the totals the issue states', () => { + const expected = expectations(parseOrdersCsv(fixtureCsv().toString('utf8'))) + assert.deepEqual(expected.paid.ids, ['A001', 'A002', 'A004', 'A005']) + assert.equal(expected.paid.count, 4) + assert.equal(expected.paid.total, 400) + assert.deepEqual(expected.east.ids, ['A001', 'A004']) + assert.equal(expected.east.count, 2) + assert.equal(expected.east.total, 140) +}) + +test('a correct app passes all four checks', async () => { + const { results, stopped } = await runScenarios(new FakeAdapter(), { checkMs: 1000 }) + assert.equal(stopped, undefined) + assert.deepEqual(results.map(result => result.outcome), ['pass', 'pass', 'pass', 'pass']) + for (const result of results) assert.ok(result.durationMs >= 0) +}) + +test('wrong ids fail the first check as an assertion, not as a crash', async () => { + const { results } = await runScenarios(new FakeAdapter({ breakIds: true }), { checkMs: 1000 }) + assert.equal(results[0].outcome, 'assertion_failed') + assert.match(results[0].problems.join('; '), /A099/) + // The corrupted summary still exists, so the run continues and the later checks say so. + assert.equal(results[1].outcome, 'assertion_failed') +}) + +test('touching summary.json during the refinement fails the immutability assertion', async () => { + const { results } = await runScenarios(new FakeAdapter({ mutateSummary: true }), { checkMs: 1000 }) + assert.equal(results[0].outcome, 'pass') + assert.equal(results[1].outcome, 'assertion_failed') + assert.match(results[1].problems.join('; '), /summary\.json changed/) +}) + +test('a stale preview fails the fourth check, not the file read', async () => { + const { results } = await runScenarios(new FakeAdapter({ stalePreview: true }), { checkMs: 1000 }) + assert.equal(results[3].outcome, 'assertion_failed') + assert.match(results[3].problems.join('; '), /preview does not show/) +}) + +test('a bench that never settles surfaces as an adapter failure for the runner to classify', async () => { + const { results } = await runScenarios(new FakeAdapter({ neverSettle: true }), { checkMs: 1000 }) + assert.equal(results[0].outcome, 'assertion_failed') + assert.match(results[0].problems.join('; '), /did not settle/) +}) + +test('the token budget stops the run between checks', async () => { + let calls = 0 + const { results, stopped } = await runScenarios(new FakeAdapter(), { + checkMs: 1000, + betweenChecks: () => { calls += 1; return calls === 1 ? { stop: true, reason: 'cap' } : {} }, + }) + assert.equal(stopped, 'cap') + assert.equal(results.length, 1) + assert.equal(results[0].outcome, 'pass') +}) + +test('summaries must be JSON with numeric count and total', () => { + assert.throws(() => parseSummary('x', 'not json'), /is not JSON/) + assert.throws(() => parseSummary('x', '{"orders": [{}], "count": "4", "total": 400}'), /count is not a number/) + assert.throws(() => parseSummary('x', '{"orders": [{}], "count": 4, "total": null}'), /total is not a number/) + const parsed = parseSummary('x', '{"orders": [{"order_id": "A001"}], "count": 1, "total": 100}') + assert.equal(parsed.count, 1) +}) + +test('the guards fail closed', () => { + assert.ok(guardRun({ apiKey: '', baseUrl: undefined }).problems.length > 0) + assert.ok(guardRun({ apiKey: 'mock-key', baseUrl: undefined }).problems.length > 0) + assert.ok(guardRun({ apiKey: 'short', baseUrl: undefined }).problems.length > 0) + const loopback = guardRun({ apiKey: 'a'.repeat(32), baseUrl: 'http://127.0.0.1:8898/anthropic' }) + assert.equal(loopback.problems.length, 1) + assert.match(loopback.problems[0], /loopback/) + const tap = guardRun({ apiKey: 'a'.repeat(32), baseUrl: 'http://127.0.0.1:8898/anthropic', tapUpstream: 'https://api.deepseek.com' }) + assert.deepEqual(tap.problems, []) + assert.equal(tap.endpoint.viaTap, true) + const home = guardRun({ apiKey: 'a'.repeat(32), baseUrl: undefined, home: '/Users/me/.dsh', userHome: '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/Users/me' }) + assert.match(home.problems.join('; '), /user's own DSH data/) + assert.deepEqual(guardRun({ apiKey: 'a'.repeat(32), baseUrl: undefined, home: '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/repo/.dsh-test-x', userHome: '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/Users/me' }).problems, []) +}) + +test('classification follows the outcome order', () => { + const budgets = budgetsFrom({}) + assert.deepEqual(budgets, { checkMs: 180_000, quietMs: 6_000, maxTokens: 250_000 }) + assert.equal(budgetsFrom({ LIVE_E2E_CHECK_MS: '5', LIVE_E2E_MAX_TOKENS: 'x' }).checkMs, 5) + assert.equal(classify({ checkOutcomes: ['pass'], providerFailures: [], tokenUsage: { prompt: 0, output: 0 }, budgets }), 'pass') + assert.equal(classify({ checkOutcomes: ['pass'], providerFailures: ['401 bad key'], tokenUsage: { prompt: 0, output: 0 }, budgets }), 'provider_failed') + assert.equal(classify({ checkOutcomes: ['assertion_failed'], providerFailures: [], tokenUsage: { prompt: 0, output: 0 }, budgets }), 'assertion_failed') + assert.equal(classify({ checkOutcomes: ['pass'], providerFailures: [], tokenUsage: { prompt: 250_001, output: 0 }, budgets }), 'budget_exceeded') +}) + +test('usage and provider failures come from the tap lines', () => { + const usage = usageOf([{ tokens: { prompt: 100, cached: 40, output: 10 } }, { tokens: { prompt: 5, cached: 0, output: 5 } }, {}]) + assert.deepEqual(usage, { calls: 3, prompt: 105, cached: 40, output: 15 }) + assert.deepEqual(providerFailures([{ status: 200, text: 'ok' }, { status: 429, error: 'slow down' }]), ['429 slow down']) + assert.deepEqual(providerFailures([{ text: 'fine' }]), []) +}) + +test('the report carries offsets, human times and exit codes', () => { + const report = buildReport({ + runId: 'id', revision: 'deadbeef', + config: { node: 'v24', dsh: '0.2.0-rc.2', model: 'deepseek-official/deepseek-flash' }, + endpoint: { host: 'api.deepseek.com', origin: 'default', viaTap: false }, + budgets: budgetsFrom({}), + checks: [{ id: 'x', title: 't', outcome: 'pass', durationMs: 12.5, problems: [] }], + usage: { calls: 1, prompt: 1, cached: 0, output: 1 }, providerFailures: [], + startedAt: new Date('2026-10-18T00:30:00+08:00'), endedAt: new Date('2026-10-18T00:31:00+08:00'), tz: 'Asia/Shanghai', + }) + assert.equal(report.outcome, 'pass') + assert.equal(report.exitCode, 0) + assert.equal(report.schema, 'flybot-live-e2e/1') + assert.equal(report.startedAt.iso, '2026-10-17T16:30:00.000Z') + assert.match(report.startedAt.offset, /^[+-]\d\d:\d\d$/) + assert.equal(report.humanTimes.zone, 'Asia/Shanghai') + assert.ok(!Number.isNaN(Date.parse(report.startedAt.iso))) +}) + +test('the sanitizer catches login URLs, cookies and the key itself — and never quotes the hit', () => { + assert.deepEqual(scanText('plain text', 'secret'), []) + assert.deepEqual(scanText('see http://127.0.0.1:3196/?token=abc123 now', undefined), ['login token URL']) + assert.deepEqual(scanText('x-flybot-member-session: 1', undefined), ['cookie material']) + assert.deepEqual(scanText('the key is sk12345678', 'sk12345678'), ['the API key value appears verbatim']) + assert.deepEqual(scanText('the key is not here', 'sk12345678'), []) + // The finding names the kind only: the matched text is what the scan exists to keep + // out of reports and artifacts. + const mixed = scanText(`set-cookie: a=1 and http://x/?token=zzz and sk12345678`, 'sk12345678').join('|') + assert.ok(!mixed.includes('zzz') && !mixed.includes('abc123') && !mixed.includes('sk12345678'), mixed) + assert.match(mixed, /login token URL/) +}) + +test('outcome names map to distinct exit codes', () => { + const codes = Object.values(OUTCOMES) + assert.deepEqual([...new Set(codes)].sort(), [0, 1, 2, 3, 4]) +}) + +test('task wording stays intact for the reload check', () => { + assert.ok(TASKS.summarize.length > 24 && TASKS.refine.length > 24) + assert.ok(TASKS.summarize.includes('summary.json') && TASKS.refine.includes('summary-east.json')) +}) + +test('sha256 is stable for the immutability chain', () => { + assert.equal(sha256(Buffer.from('abc')), sha256(Buffer.from('abc'))) + assert.notEqual(sha256(Buffer.from('abc')), sha256(Buffer.from('abd'))) +}) diff --git a/docs/interop/live-e2e-adapter.md b/docs/interop/live-e2e-adapter.md new file mode 100644 index 000000000..27faf098f --- /dev/null +++ b/docs/interop/live-e2e-adapter.md @@ -0,0 +1,62 @@ +# Live E2E adapter contract (#204) + +**Status: implemented for FlyBot; one adapter exists.** The scenarios, fixtures and +assertions in [dev/live-e2e/](../../dev/live-e2e/) are app-independent: they talk to +any implementation of the interface below and to nothing else — no browser, no DSH, no +FlyBot import. The FlyBot web adapter (`dev/live-e2e/adapter.mjs`) is the first +implementation; running the same scenarios on Orbit needs its own adapter +([#209](https://github.com/ThinkFlowLab/FlyBot/issues/209), *not exercised*). + +## Relation to the OpenCode pilot contract + +[opencode-pilot.md](opencode-pilot.md) specifies the offline, envelope-level contract +between two agent runtimes (typed JSON envelopes, approvals, journals), simulated by +`dev/interop/` without a model or network. This document specifies the other axis: the +UI-level contract for driving a real app through a real user path — upload, task, +delivery, preview, reload. Both are development-only contracts that ship in no package; +one verifies interop semantics offline, the other verifies the app end to end online. + +## The interface + +An adapter is one conversation with one app instance on a fresh, isolated home. Task +steps must go through the app's real entry surface (the UI); observation may use the +app's own read-only APIs and storage. Setting up the environment before the first task +(for FlyBot: picking the Main Bot's model) is not a task step and may use the app's API. + +| Operation | Meaning | +| --- | --- | +| `uploadFile(name, bytes)` | Attach a file to the conversation the way a user does. | +| `sendTask(text)` | Send a task message and return; settling is explicit. | +| `settle(budgetMs)` | Bounded wait until the app is idle after a task; throws on timeout. | +| `deliveredFileNames()` | Names the UI itself lists as delivered (cards/chips), for presence checks only. | +| `readFile(name)` | The file's bytes from the run's storage, or `undefined`. Evidence of delivery. | +| `openPreview(name)` | Open the artifact's preview in the UI and return the displayed text. | +| `reload()` | Reload the app view (survival check). | +| `conversationLines()` | Visible lines as `{ who: 'user' \| 'app', text }`. | +| `screenshot(name)` | Evidence capture; adapters that cannot, no-op. | + +Rules an adapter must hold: + +- Fresh run id, home and workspace per run; the fixture data is fixed by the suite. +- Nothing a task needs may be done through the observation side — a filename echoed in + a reply is never evidence of delivery (`readFile`/`openPreview` are). +- Every wait is bounded by the caller's budget; no internal retries. + +## Assertions (owned by the suite, not the adapter) + +- Delivered JSON is compared **parsed**; `count` and `total` must be numbers. Only the + input CSV is compared byte-for-byte (SHA-256, before and after each later task; + `summary.json` is pinned by SHA-256 once delivered). +- Expected values are derived from the fixture CSV (paid = A001/A002/A004/A005 → 4/400; + paid ∧ 华东 = A001/A004 → 2/140), so a fixture change that breaks the stated totals + fails the keyless tests, not the nightly. + +## Outcome contract + +`pass | assertion_failed | provider_failed | infra_failed | budget_exceeded`, exit codes +0–4, one structured report per run (`flybot-live-e2e/1`): run id, revision, runtime and +model config, resolved endpoint (non-loopback, key not the mock's — fail closed +otherwise), absolute ISO-8601 timestamps with offset plus human times in the run's zone +(Asia/Shanghai), monotonic per-check durations, token/cache usage, and the sanitizer +verdict over the uploaded evidence. Budgets default to 180 s per check, 250K tokens per +run; a budget stop is reported as `budget_exceeded`, never retried. diff --git a/docs/subsystems/README.md b/docs/subsystems/README.md index 5e0f01e70..1c794e580 100644 --- a/docs/subsystems/README.md +++ b/docs/subsystems/README.md @@ -13,6 +13,7 @@ Detail pages for each FlyBot subsystem. Each page lists `owned_paths`, what the | install | [install.md](install.md) | `core/scripts/`, arch-doc tests, `plugins/README.md` | | team | [team.md](team.md) | `docs/subsystems/team.md` — **planned contract only** (no code paths; multi-human is not shipped) | | interop | [interop.md](interop.md) | `dev/interop/`, `scripts/lint-interop.mjs` — offline development harness only | +| live-e2e | [live-e2e.md](live-e2e.md) | `dev/live-e2e/`, `scripts/lint-live-e2e.mjs` — real-model browser suite (#204) | The `team` page is a planning contract: it defines the proposed human ↔ Chief ↔ specialist authority, routing and approval rules **before** schema work. Team mode stays disabled until its authorization gates pass. diff --git a/docs/subsystems/live-e2e.md b/docs/subsystems/live-e2e.md new file mode 100644 index 000000000..3ba23171f --- /dev/null +++ b/docs/subsystems/live-e2e.md @@ -0,0 +1,35 @@ +--- +subsystem: live-e2e +owned_paths: + - dev/live-e2e/ + - scripts/lint-live-e2e.mjs +--- + +# live-e2e (real-model browser suite) + +The [#204](https://github.com/ThinkFlowLab/FlyBot/issues/204) live end-to-end suite: +app-independent scenarios with one adapter for the FlyBot web UI, run against the real +DeepSeek endpoint. Commands, budgets and the evidence rules are in +[the suite README](../../dev/live-e2e/README.md); the adapter contract is in +[docs/interop/live-e2e-adapter.md](../interop/live-e2e-adapter.md). + +## Owns + +- `dev/live-e2e/`: the fixture and scenario/assertion modules, the FlyBot web adapter, + the real-model bench launcher, the runner (guards, budgets, report, evidence + sanitizer) and the keyless fake-adapter tests. +- `scripts/lint-live-e2e.mjs`: lint using core's installed harness tooling. + +## Invariants + +- Scenario, fixture and assertion code never imports the adapter, a browser or DSH; + it is covered keyless (`npm run test:live-e2e`, part of `verify`) through a fake + adapter. The live command itself is never part of `verify`. +- The live run uses a real key from the environment and a fresh isolated bench home + (`.dsh-test-*`, never under `~/.dsh`). It fails closed on a missing/mock key, a + loopback endpoint (except the usage tap with a real upstream), or the user's own + DSH home. No retries hide regressions. +- The mock suites stay separate: this bench never starts the mock model, and the mock + benches never gain a real key. +- Evidence is an explicit allow-list that passed the sanitizer (no `token=` URLs, no + key material, no cookies); a sanitizer finding is an infra failure, never a pass. diff --git a/package.json b/package.json index f2b3277b8..dfb89bfc6 100644 --- a/package.json +++ b/package.json @@ -6,9 +6,11 @@ "node": ">=22" }, "scripts": { - "verify": "npm --prefix core run lint && node scripts/lint-plugins.mjs && node core/scripts/build-client.mjs --check && node scripts/ds-bot-patches.mjs check && npm --prefix core test && npm --prefix plugins/social test && npm --prefix plugins/tree test && node --test \".github/scripts/*.test.mjs\" && npm run lint:interop && npm run test:interop", + "verify": "npm --prefix core run lint && node scripts/lint-plugins.mjs && node core/scripts/build-client.mjs --check && node scripts/ds-bot-patches.mjs check && npm --prefix core test && npm --prefix plugins/social test && npm --prefix plugins/tree test && node --test \".github/scripts/*.test.mjs\" && npm run lint:interop && npm run test:interop && npm run lint:live-e2e && npm run test:live-e2e", "lint:interop": "node scripts/lint-interop.mjs", "test:interop": "node --import ./dev/interop/no-network.mjs --test \"dev/interop/test/*.test.mjs\"", - "demo:interop": "node --import ./dev/interop/no-network.mjs dev/interop/demo.mjs" + "demo:interop": "node --import ./dev/interop/no-network.mjs dev/interop/demo.mjs", + "lint:live-e2e": "node scripts/lint-live-e2e.mjs", + "test:live-e2e": "node --test \"dev/live-e2e/test/*.test.mjs\"" } } diff --git a/scripts/lint-live-e2e.mjs b/scripts/lint-live-e2e.mjs new file mode 100644 index 000000000..650176929 --- /dev/null +++ b/scripts/lint-live-e2e.mjs @@ -0,0 +1,12 @@ +import { existsSync } from 'node:fs' +import { join } from 'node:path' +import { spawnSync } from 'node:child_process' +import { fileURLToPath } from 'node:url' + +const root = fileURLToPath(new URL('../', import.meta.url)) +const harness = process.env.DSH_HARNESS || join(root, 'core/dev/harness') +const bin = join(harness, 'node_modules/oxlint/bin/oxlint') +if (!existsSync(bin)) throw new Error('Run npm run harness in core/, or set DSH_HARNESS') +const result = spawnSync(process.execPath, [bin, '-c', 'core/.oxlintrc.json', 'dev/live-e2e', 'scripts/lint-live-e2e.mjs'], { cwd: root, stdio: 'inherit' }) +if (result.error) throw result.error +process.exitCode = result.status ?? 1