Add the real-model browser E2E suite (#204)

.github/workflows/live-e2e.yml

@@ -0,0 +1,158 @@
+name: Live E2E
+
+# The real-model browser suite of #204: four task checks driven through the FlyBot web
+# UI on a fresh isolated bench, against the real DeepSeek endpoint. Separate from the
+# compatibility matrix (nightly.yml) and from every PR check; never added to `tests-ok`
+# or branch protection. This is the repository's only workflow that reads a secret: the
+# live job runs on main only, the key is mapped step-scoped, and everything uploaded has
+# passed the runner's evidence sanitizer (no `token=`, no key material, no cookies) and
+# is an explicit allow-list — never a raw bench home.
+
+on:
+  schedule:
+    - cron: "30 16 * * *" # 00:30 Asia/Shanghai daily; clear of compat's 16:00Z
+  workflow_dispatch:
+
+concurrency:
+  group: live-e2e
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  window:
+    name: window check
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      run: ${{ steps.gate.outputs.run }}
+    steps:
+      - name: Check out main with full history
+        if: github.event_name == 'schedule'
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Decide whether main has new commits
+        id: gate
+        run: |
+          run=false
+          reason=""
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            run=true
+            reason="manual dispatch always runs"
+          elif [ -n "$(git log --since='24 hours ago' --pretty=oneline)" ]; then
+            run=true
+            reason="main had commits in the last 24 hours"
+          else
+            reason="main had no commits in the window; no tokens spent"
+          fi
+          echo "run=$run" >> "$GITHUB_OUTPUT"
+          {
+            echo "### Live E2E run decision"
+            echo ""
+            echo "- event: \`${{ github.event_name }}\`"
+            echo "- run: \`${run}\`: ${reason}"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+  live:
+    name: live · ubuntu · Node 24 · real model
+    needs: window
+    if: needs.window.outputs.run == 'true' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    env:
+      WEB_PORT: '3196'
+      TAP_PORT: '8898'
+      BENCH: live-e2e
+      MODEL: deepseek-official/deepseek-flash
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1  # v7.1.0
+        with:
+          node-version: 24
+          cache: npm
+          cache-dependency-path: core/package-lock.json
+      - name: Preflight (the key must exist; never a silent skip)
+        env:
+          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
+        run: |
+          [ -n "$DEEPSEEK_API_KEY" ] || { echo "::error::DEEPSEEK_API_KEY is not configured as a repository secret"; exit 1; }
+      - name: npm ci (core)
+        working-directory: core
+        run: npm ci --no-fund --no-audit
+        timeout-minutes: 5
+      - name: Install the pinned DSH runtime
+        working-directory: core
+        run: |
+          DSH=$(node -p "require('./dev/harness/package.json').dependencies['@deepseek-ai/dsh']")
+          npm run harness -- --no-save "@deepseek-ai/dsh@$DSH"
+          echo "DSH=$DSH" >> "$GITHUB_ENV"
+        timeout-minutes: 5
+      - name: Clone DeepSeek Harness at the pinned tag
+        run: |
+          git clone --depth 1 --branch "dsh-v$DSH" https://github.com/deepseek-ai/deepseek-harness.git deepseek-harness
+          corepack enable
+          pnpm install --dir deepseek-harness
+        timeout-minutes: 10
+      - name: Start the usage tap
+        run: |
+          mkdir -p ".dsh-test-$BENCH"
+          setsid node core/dev/llm-tap.mjs "$TAP_PORT" https://api.deepseek.com ".dsh-test-$BENCH/tap.jsonl" \
+            </dev/null >".dsh-test-$BENCH/tap.out" 2>&1 & echo $! >".dsh-test-$BENCH/tap.pid"
+          i=0
+          until (exec 3<>"/dev/tcp/127.0.0.1/$TAP_PORT") 2>/dev/null; do
+            i=$((i + 1))
+            [ $i -ge 30 ] && { echo "::error::the usage tap did not start"; cat ".dsh-test-$BENCH/tap.out"; exit 1; }
+            sleep 1
+          done
+      - name: Start the real-model bench
+        env:
+          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
+        run: |
+          DEEPSEEK_BASE_URL="http://127.0.0.1:$TAP_PORT/anthropic" node dev/live-e2e/bench.mjs "$BENCH" "$WEB_PORT" reset
+        timeout-minutes: 8
+      - name: Run the four checks
+        id: run
+        env:
+          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
+          DEEPSEEK_BASE_URL: http://127.0.0.1:8898/anthropic
+          LIVE_E2E_TAP_UPSTREAM: https://api.deepseek.com
+        run: |
+          node dev/live-e2e/run.mjs ".dsh-test-$BENCH" "$WEB_PORT" \
+            --model "$MODEL" --harness deepseek-harness \
+            --tap ".dsh-test-$BENCH/tap.jsonl" --evidence evidence --tz Asia/Shanghai
+      - name: Stop the bench and the tap
+        if: always()
+        run: |
+          node dev/live-e2e/bench.mjs "$BENCH" "$WEB_PORT" stop || true
+          if [ -f ".dsh-test-$BENCH/tap.pid" ]; then
+            kill -TERM "-$(cat ".dsh-test-$BENCH/tap.pid")" 2>/dev/null || true
+          fi
+      - name: Sanitized web log (diagnostics only, never uploaded raw)
+        if: failure()
+        run: tail -80 ".dsh-test-$BENCH/web.log" | grep -v 'token=' || true
+      # Fail closed twice: the runner writes evidence/sanitized.txt only after a clean
+      # scan, and the upload requires that stamp. A bundle the sanitizer flagged (or one
+      # from a run that never reached the scan) never becomes a public artifact.
+      - name: Evidence gate
+        id: evidence-gate
+        if: always() && steps.run.outcome != 'skipped'
+        run: |
+          if [ ! -f evidence/sanitized.txt ]; then
+            echo "::error::no sanitizer-cleared evidence bundle (the run failed before the scan, or the scan found something); nothing is uploaded"
+            exit 1
+          fi
+      # Only the stamped allow-list is uploaded: the report, the screenshots, the
+      # transcript — never the bench home, the tap log or any raw log.
+      - name: Upload evidence (allow-list, sanitizer-stamped)
+        if: always() && steps.evidence-gate.outcome == 'success'
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
+        with:
+          name: live-e2e-evidence
+          path: evidence/
+          if-no-files-found: ignore
+          retention-days: 14

dev/live-e2e/README.md

@@ -0,0 +1,56 @@
+# Live E2E (#204)
+
+Real-model, browser-driven end-to-end checks: upload an orders CSV through the web UI,
+let the Main Bot write `summary.json` / `summary-east.json` with the real DeepSeek
+endpoint, then verify the files, their survival across a reload, and the sidebar
+previews — by content, never by a filename echoed in a reply. The scenarios and
+assertions are app-independent ([adapter contract](../../docs/interop/live-e2e-adapter.md));
+only `adapter.mjs` knows FlyBot. Nothing here ships in a package or runs in
+`npm run verify`; the keyless fake-adapter tests do (`npm run test:live-e2e`).
+
+## Layout
+
+| File | Responsibility |
+| --- | --- |
+| `fixtures/orders.csv` | The fixed 5-row fixture the expected totals derive from. |
+| `scenarios.mjs` | The four checks, task texts, pure assertions (parsed JSON, SHA-256 immutability). |
+| `adapter.mjs` | The FlyBot web adapter: playwright-core from the harness checkout, UI task steps, read-only observation. |
+| `bench.mjs` | A dsh web bench on this checkout with the real endpoint; refuses missing/mock keys, never touches `~/.dsh`. |
+| `run.mjs` | The runner: guards, budgets, classification, report, evidence sanitizer. |
+| `report.mjs` | Outcome contract, guard logic, usage/classification, report assembly (pure, tested keyless). |
+| `sanitize.mjs` | The evidence gate: `token=` URLs, cookie material, the key value. |
+| `test/` | Keyless fake-adapter coverage of every assertion path. |
+
+## Run it locally
+
+Node 22.19+, pnpm, a [deepseek-harness](https://github.com/deepseek-ai/deepseek-harness)
+checkout with `pnpm install` run in it (`../deepseek-harness`, or set `DSH_HARNESS`), and
+a real `DEEPSEEK_API_KEY` in the environment. Chrome comes from `CHROME_BIN`, else
+`google-chrome`/`chromium`(-`browser`) on PATH, else playwright's own download.
+
+```sh
+export DEEPSEEK_API_KEY=...            # real key; the runner fails closed without it
+node dev/live-e2e/bench.mjs live-e2e 3196 reset
+node dev/live-e2e/run.mjs .dsh-test-live-e2e 3196 --model deepseek-official/deepseek-flash
+node dev/live-e2e/bench.mjs live-e2e 3196 stop
+```
+
+The report lands in `evidence/report.json` (screenshots, transcript beside it). To
+record per-call token usage, start `core/dev/llm-tap.mjs <port> https://api.deepseek.com
+<tap.jsonl>` first, run the bench with
+`DEEPSEEK_BASE_URL=http://127.0.0.1:<port>/anthropic`, and pass `--tap <tap.jsonl>`;
+the loopback is accepted only with `LIVE_E2E_TAP_UPSTREAM=https://api.deepseek.com`.
+A run costs real tokens (the pilot used ~116K); budgets default to 180 s per check and
+250K tokens per run (`LIVE_E2E_CHECK_MS`, `LIVE_E2E_QUIET_MS`, `LIVE_E2E_MAX_TOKENS`),
+with zero retries.
+
+## In CI
+
+`.github/workflows/live-e2e.yml` runs the same flow nightly at 00:30 Asia/Shanghai (and
+on dispatch) on `main` only, with a window gate so a quiet `main` spends no tokens: it
+clones the harness at the tag pinned in `core/dev/harness/package.json`, starts the tap,
+resets the bench, runs the four checks and uploads the sanitizer-cleared allow-list.
+The four decisions behind this setup are recorded in
+[#204](https://github.com/ThinkFlowLab/issues/204). "Stable" means five consecutive
+scheduled runs green at one recorded configuration; until then the suite is evidence,
+not a gate.

dev/live-e2e/adapter.mjs

@@ -0,0 +1,163 @@
+// The FlyBot web adapter for the #204 scenarios: the only file in this suite that knows
+// FlyBot's UI (docs/interop/live-e2e-adapter.md is the contract it implements; Orbit
+// implements its own). Tasks go through the browser — the composer input for uploads,
+// typing for messages, clicks for previews — while observation uses the app's own state
+// API from inside the page and the run's storage on disk. Picking the Main Bot's model
+// before the first task is environment setup, not a task step, and goes through the same
+// in-page API like dev/memory-e2e.mjs does.
+//
+// playwright-core comes from the deepseek-harness checkout through the existing helper in
+// core/dev/desktop-linux/lib.mjs (this repo installs no browser tooling). Chrome:
+// CHROME_BIN, else google-chrome / chromium / chromium-browser on PATH, else playwright's
+// own download. The login URL in web.log is used once and never printed.
+import { execFileSync } from 'node:child_process'
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'
+import { join } from 'node:path'
+import { playwright, waitTeam } from '../../core/dev/desktop-linux/lib.mjs'
+
+export class AdapterError extends Error {
+  constructor(message) {
+    super(message)
+    this.name = 'AdapterError'
+  }
+}
+
+const findChrome = () => {
+  if (process.env.CHROME_BIN) return process.env.CHROME_BIN
+  for (const name of ['google-chrome', 'chromium', 'chromium-browser']) {
+    try { return execFileSync('sh', ['-c', `command -v ${name}`], { encoding: 'utf8' }).trim() } catch {}
+  }
+  return undefined
+}
+
+const sleep = ms => new Promise(done => setTimeout(done, ms))
+
+// The run's storage lookup: the newest file with the name under the bench home, outside
+// the profile, caches and screenshots. Uploads and artifacts both land there.
+function findInHome(home, name) {
+  let best
+  const skip = new Set(['profiles', 'node_modules', 'cache', 'logs', 'shots'])
+  const walk = (dir, depth) => {
+    if (depth > 8) return
+    let entries
+    try { entries = readdirSync(dir, { withFileTypes: true }) } catch { return }
+    for (const entry of entries) {
+      if (entry.name.startsWith('.') || skip.has(entry.name)) continue
+      const path = join(dir, entry.name)
+      if (entry.isDirectory()) { walk(path, depth + 1); continue }
+      if (entry.name !== name) continue
+      const mtime = statSync(path).mtimeMs
+      if (!best || mtime > best.mtime) best = { path, mtime }
+    }
+  }
+  walk(home, 0)
+  return best?.path
+}
+
+export async function openFlyBotWeb({ harness, home, port, model, outDir, quietMs, checkMs }) {
+  if (!existsSync(join(home, 'web.log'))) throw new AdapterError(`no web.log in ${home}; is the bench running?`)
+  const login = readFileSync(join(home, 'web.log'), 'utf8').match(new RegExp(`http://127\\.0\\.0\\.1:${port}/\\?token=[^\\s]+`))?.[0]
+  if (!login) throw new AdapterError(`no login URL in ${home}/web.log`)
+
+  const { chromium } = playwright(harness)
+  const browser = await chromium.launch({ executablePath: findChrome(), headless: true }).catch(error => { throw new AdapterError(`Chrome did not launch (${error.message}); set CHROME_BIN`) })
+  const page = await browser.newPage({ viewport: { width: 1400, height: 860 } })
+  await page.goto(login)
+  // dsh shows a preview notice once per profile; clear it before any click.
+  const notice = page.getByRole('dialog').getByRole('button', { name: 'Continue' })
+  await page.addLocatorHandler(notice, () => notice.click())
+  await waitTeam(page)
+
+  const api = async (endpoint, payload = {}) => {
+    const result = await page.evaluate(([e, p]) => fetch('api/bot', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ endpoint: e, payload: p }) }).then(r => r.json()), [endpoint, payload])
+    if (!result.ok) throw new AdapterError(`${endpoint}: ${result.error?.message ?? JSON.stringify(result)}`)
+    return result.value
+  }
+
+  // Setup: put the Main Bot on the asked model. A fresh team holds its greeting until
+  // then; the model card flow stays covered by the mock suites.
+  const view = await api('state')
+  const chief = view.bots.find(bot => bot.id === (view.mainBotId ?? view.mainBotIds?.[0]))
+  if (!chief) throw new AdapterError(`no Main Bot in a fresh team (${view.bots?.length ?? 0} bots)`)
+  const models = (await api('models')).models ?? view.models ?? []
+  const ref = models.find(candidate => candidate.ref === model || candidate.ref.endsWith(`/${model}`))?.ref
+    ?? models.find(candidate => candidate.ref?.includes(model))?.ref
+  if (!ref) throw new AdapterError(`model ${model} is not offered; available: ${models.map(candidate => candidate.ref).join(', ') || 'none'}`)
+  if (chief.model !== ref) await api('update-bot', { id: chief.id, model: ref })
+  await page.locator(`.bt-tile[data-bot-id="${chief.id}"]`).first().click()
+  await page.waitForTimeout(800)
+  const sent = []
+
+  const screenshot = async name => { await page.screenshot({ path: join(outDir, `${String(screenshot.index = (screenshot.index ?? 0) + 1).padStart(2, '0')}-${name}.png`) }) }
+
+  // Quiet: no Bot busy and no memory review running, for quietMs, within checkMs.
+  async function settle(timeoutMs = checkMs) {
+    const deadline = Date.now() + timeoutMs
+    let since = Date.now()
+    for (;;) {
+      await sleep(800)
+      const current = await api('state')
+      const busy = Object.keys(current.activity ?? {}).length > 0 || (current.memoryReviews ?? 0) > 0
+      if (busy) since = Date.now()
+      else if (Date.now() - since >= quietMs) return
+      if (Date.now() >= deadline) throw new AdapterError(`the team did not settle within ${timeoutMs} ms`)
+    }
+  }
+
+  return {
+    modelId: ref,
+    page,
+    async uploadFile(name, bytes) {
+      const input = page.locator('[data-composer-card] input[type=file]').last()
+      await input.setInputFiles({ name, mimeType: 'text/csv', buffer: bytes })
+      await page.waitForTimeout(600)
+    },
+    async sendTask(text) {
+      sent.push(text)
+      const box = page.locator('[contenteditable="true"], textarea').last()
+      await box.click()
+      await page.keyboard.type(text)
+      await page.keyboard.press('Enter')
+    },
+    settle,
+    async deliveredFileNames() {
+      return page.evaluate(() => {
+        const rows = [...document.querySelectorAll('[data-presented-files-row]')].map(node => node.textContent)
+        const chips = [...document.querySelectorAll('code')].map(node => node.textContent ?? '').filter(text => /\.json$/i.test(text.trim()))
+        return [...rows, ...chips.map(text => text.trim())]
+      })
+    },
+    async readFile(name) {
+      const path = findInHome(home, name)
+      return path === undefined ? undefined : readFileSync(path)
+    },
+    async openPreview(name) {
+      const target = page.locator(`[data-presented-files-row]`).filter({ hasText: name }).last()
+      const chip = page.locator('code').filter({ hasText: name }).last()
+      const clickable = (await target.count()) > 0 ? target : chip
+      if ((await clickable.count()) === 0) throw new AdapterError(`nothing in the chat opens ${name}`)
+      await clickable.click()
+      await page.waitForTimeout(1500)
+      const text = await page.evaluate(() => document.querySelector('[data-slot="sidebar.right"]')?.innerText
+        ?? [...document.querySelectorAll('aside')].map(node => node.innerText).sort((a, b) => b.length - a.length)[0]
+        ?? '')
+      if (!text.trim()) throw new AdapterError(`clicking ${name} opened no preview`)
+      return text
+    },
+    async reload() {
+      await page.reload({ waitUntil: 'load' })
+      await waitTeam(page)
+      await page.locator(`.bt-tile[data-bot-id="${chief.id}"]`).first().click()
+      await page.waitForTimeout(1200)
+    },
+    // Who said what is app-specific to render; for the scenarios it is enough that the
+    // task texts are still visible and that some reply beside them is.
+    async conversationLines() {
+      const text = await page.evaluate(() => document.body.innerText)
+      const lines = text.split('\n').map(line => line.trim()).filter(line => line.length > 0)
+      return lines.map(line => ({ who: sent.some(task => task.slice(0, 24) && line.includes(task.slice(0, 24))) ? 'user' : 'app', text: line }))
+    },
+    screenshot,
+    async close() { await browser.close().catch(() => {}) },
+  }
+}