diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 611d4cf44f75..25f0cb25f7ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,8 +14,11 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - check: - name: Check + # Lint, typecheck, and the desktop build each get a runner so they run side by + # side instead of one after another. The `Check` job at the end of this file + # passes only when every other job does. + lint: + name: Lint runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 10 steps: @@ -52,15 +55,73 @@ jobs: - name: Check run: vp check + typecheck: + name: Typecheck + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + sparse-checkout: | + /* + !/.repos/ + sparse-checkout-cone-mode: false + + - name: Setup Vite+ + uses: voidzero-dev/setup-vp@v1 + with: + node-version-file: package.json + cache: true + run-install: true + + - name: Ensure Electron runtime is installed + run: vp run --filter @t3tools/desktop ensure:electron + - name: Typecheck run: vpr typecheck + build: + name: Build + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + sparse-checkout: | + /* + !/.repos/ + sparse-checkout-cone-mode: false + - uses: ./.github/actions/setup-apt-mirrors - - name: Install browser secret helper build libraries + # Runs in the background while Vite+ installs; the step before the build waits for it. + - name: Start installing browser secret helper build libraries run: | sudo sed -i 's|http://|https://|g' /etc/apt/blacksmith-ubuntu-mirrors.txt /etc/apt/sources.list.d/ubuntu.sources - sudo apt-get update && sudo apt-get install -y libsecret-1-dev pkg-config + ( + set +e + sudo apt-get update && sudo apt-get install -y libsecret-1-dev pkg-config + echo "$?" > "$RUNNER_TEMP/apt-status" + ) > "$RUNNER_TEMP/apt.log" 2>&1 < /dev/null & + + - name: Setup Vite+ + uses: voidzero-dev/setup-vp@v1 + with: + node-version-file: package.json + cache: true + run-install: true + + - name: Ensure Electron runtime is installed + run: vp run --filter @t3tools/desktop ensure:electron + + - name: Finish installing browser secret helper build libraries + timeout-minutes: 5 + run: | + until test -s "$RUNNER_TEMP/apt-status"; do sleep 1; done + cat "$RUNNER_TEMP/apt.log" + exit "$(cat "$RUNNER_TEMP/apt-status")" - name: Build desktop pipeline run: vp run build:desktop @@ -68,11 +129,12 @@ jobs: - name: Verify preload bundle output run: node apps/desktop/scripts/verify-preload-bundle.mjs - # Everything except `t3` (apps/server). `--parallel` drops the package - # dependency ordering that `vp run` applies by default: these `test` tasks - # declare no `dependsOn` and resolve workspace deps from source, so ordering - # only bought us idle runners between dependency layers. The concurrency - # limit stays at the default 4 so peak load per runner is unchanged. + # Everything except `t3` (apps/server) and the web app, which has its own job + # below. `--parallel` drops the package dependency ordering that `vp run` + # applies by default: these `test` tasks declare no `dependsOn` and resolve + # workspace deps from source, so ordering only bought us idle runners between + # dependency layers. The concurrency limit stays at the default 4 so peak load + # per runner is unchanged. test: name: Test runs-on: blacksmith-8vcpu-ubuntu-2404 @@ -86,6 +148,19 @@ jobs: !/.repos/ sparse-checkout-cone-mode: false + - uses: ./.github/actions/setup-apt-mirrors + + # The desktop tests compile the browser secret helper. The install runs in + # the background while Vite+ installs; the step before the tests waits for it. + - name: Start installing browser secret helper build libraries + run: | + sudo sed -i 's|http://|https://|g' /etc/apt/blacksmith-ubuntu-mirrors.txt /etc/apt/sources.list.d/ubuntu.sources + ( + set +e + sudo apt-get update && sudo apt-get install -y libsecret-1-dev pkg-config + echo "$?" > "$RUNNER_TEMP/apt-status" + ) > "$RUNNER_TEMP/apt.log" 2>&1 < /dev/null & + - name: Setup Vite+ uses: voidzero-dev/setup-vp@v1 with: @@ -96,34 +171,61 @@ jobs: - name: Ensure Electron runtime is installed run: vp run --filter @t3tools/desktop ensure:electron - - uses: ./.github/actions/setup-apt-mirrors - - - name: Install browser secret helper build libraries - run: | - sudo sed -i 's|http://|https://|g' /etc/apt/blacksmith-ubuntu-mirrors.txt /etc/apt/sources.list.d/ubuntu.sources - sudo apt-get update && sudo apt-get install -y libsecret-1-dev pkg-config - - name: Test preview artifact validation run: python3 -B .github/scripts/stage-preview-bundle.test.py - name: Test nightly release checks run: node --test .github/scripts/check-nightly-release.test.cjs + - name: Finish installing browser secret helper build libraries + timeout-minutes: 5 + run: | + until test -s "$RUNNER_TEMP/apt-status"; do sleep 1; done + cat "$RUNNER_TEMP/apt.log" + exit "$(cat "$RUNNER_TEMP/apt-status")" + + - name: Test + run: vp run --parallel --concurrency-limit 4 --filter '!t3' --filter '!@t3tools/monorepo' --filter '!@t3tools/web' test + + # The web suite is the largest outside the server. In the shared Test job it + # started last and ran alone for about 40s after everything else finished. + test_web: + name: Test Web + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + sparse-checkout: | + /* + !/.repos/ + sparse-checkout-cone-mode: false + + - name: Setup Vite+ + uses: voidzero-dev/setup-vp@v1 + with: + node-version-file: package.json + cache: true + run-install: true + - name: Test - run: vp run --parallel --concurrency-limit 4 --filter '!t3' --filter '!@t3tools/monorepo' test + run: vp run --filter @t3tools/web test - # apps/server sets `fileParallelism: false`, so its 239 files run strictly + # apps/server sets `fileParallelism: false`, so its test files run strictly # one at a time. Sharding spreads them over separate runners instead of # separate workers, so no two server test files ever share a machine and the - # isolation that flag buys is preserved exactly. + # isolation that flag buys is preserved exactly. One file at a time keeps + # about 1.5 cores busy, so 4 vCPU runners lose nothing against 8. The shards + # split files by recorded duration (apps/server/src/testUtils/weightedShardSequencer.ts). test_server: name: Test Server ${{ matrix.shard }} - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 strategy: fail-fast: false matrix: - shard: [1, 2, 3] + shard: [1, 2, 3, 4, 5, 6] steps: - name: Checkout uses: actions/checkout@v6 @@ -347,3 +449,37 @@ jobs: - name: Exercise release-only workflow steps run: node scripts/release-smoke.ts + + # Branch protection requires this job by its name, `Check`. It fails when any + # other job fails, is cancelled, or is skipped without a reason, so splitting + # or resharding jobs needs no ruleset change. Add every new job to `needs`. + # `always()` rather than `!cancelled()`: a skipped job reports success to + # branch protection, so a cancelled run must still produce a failing check. + check: + name: Check + if: ${{ always() }} + needs: + [ + lint, + typecheck, + build, + test, + test_web, + test_server, + rust, + mobile_native_changes, + mobile_native_static_analysis, + release_smoke, + ] + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + steps: + - name: Require every job to pass + env: + RESULTS: ${{ toJSON(needs) }} + run: | + echo "$RESULTS" | jq -r 'to_entries[] | "\(.key): \(.value.result)"' + # Only the macOS lint skips on purpose, when no native mobile code changed. + echo "$RESULTS" | jq -e 'all(to_entries[]; + .value.result == "success" + or (.key == "mobile_native_static_analysis" and .value.result == "skipped"))' > /dev/null diff --git a/apps/server/scripts/update-test-shard-weights.ts b/apps/server/scripts/update-test-shard-weights.ts new file mode 100644 index 000000000000..91d87fbef05b --- /dev/null +++ b/apps/server/scripts/update-test-shard-weights.ts @@ -0,0 +1,54 @@ +// @effect-diagnostics nodeBuiltinImport:off - a one-off maintenance script with no +// Effect runtime; it only spawns vitest and reads and writes two files. +// Run with: node apps/server/scripts/update-test-shard-weights.ts +// Runs the whole server suite once and records how long each test file takes, +// so CI can split the suite into shards of equal duration. See +// src/testUtils/weightedShardSequencer.ts. Rerun it when a shard in CI runs +// much longer than the others. +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import * as NodeURL from "node:url"; + +// Files faster than this are all treated alike, which keeps the weights file short. +const MIN_RECORDED_SECONDS = 0.5; + +interface VitestJsonReport { + readonly testResults: ReadonlyArray<{ + readonly name: string; + readonly startTime: number; + readonly endTime: number; + }>; +} + +const serverDir = NodeURL.fileURLToPath(new URL("..", import.meta.url)); +const weightsPath = NodePath.join(serverDir, "src/testUtils/shardWeights.json"); +const reportDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-server-test-report-")); +const reportPath = NodePath.join(reportDir, "report.json"); + +// `node --run` puts the package's own `vp` on PATH, on every platform. Failing +// tests still report their duration, so the exit code is not checked. +NodeChildProcess.spawnSync( + process.execPath, + ["--run", "test", "--", "--reporter=json", `--outputFile=${reportPath}`], + { cwd: serverDir, stdio: "inherit" }, +); + +const report: VitestJsonReport = JSON.parse(NodeFS.readFileSync(reportPath, "utf8")); +NodeFS.rmSync(reportDir, { recursive: true, force: true }); + +const weights = Object.fromEntries( + report.testResults + .map( + (file) => + [ + NodePath.relative(serverDir, file.name).replaceAll("\\", "/"), + Math.round((file.endTime - file.startTime) / 100) / 10, + ] as const, + ) + .filter(([, seconds]) => seconds >= MIN_RECORDED_SECONDS) + .toSorted(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)), +); + +NodeFS.writeFileSync(weightsPath, `${JSON.stringify(weights, null, 2)}\n`); diff --git a/apps/server/src/testUtils/shardWeights.json b/apps/server/src/testUtils/shardWeights.json new file mode 100644 index 000000000000..443acb03658d --- /dev/null +++ b/apps/server/src/testUtils/shardWeights.json @@ -0,0 +1,40 @@ +{ + "integration/orchestrationEngine.integration.test.ts": 1.9, + "integration/providerService.integration.test.ts": 0.5, + "src/bin.test.ts": 0.8, + "src/checkpointing/CheckpointStore.test.ts": 0.6, + "src/device/sshDeviceScript.test.ts": 2.9, + "src/git/GitManager.test.ts": 18.7, + "src/orchestration/Layers/CheckpointReactor.test.ts": 7.1, + "src/orchestration/Layers/OrchestrationEngine.test.ts": 1.4, + "src/orchestration/Layers/ProjectionPipeline.test.ts": 0.5, + "src/orchestration/Layers/ProviderCommandReactor.test.ts": 3.8, + "src/orchestration/Layers/ProviderRuntimeIngestion.test.ts": 2.4, + "src/persistence/Layers/OrchestrationEventStore.test.ts": 2.2, + "src/process/externalLauncher.test.ts": 2.2, + "src/project/AgentSessionScanner.test.ts": 4.3, + "src/provider/CodexChatGptAuth.test.ts": 5.9, + "src/provider/Drivers/AntigravityDriver.test.ts": 2, + "src/provider/Layers/AntigravityAdapter.test.ts": 0.5, + "src/provider/Layers/ClaudeAdapter.test.ts": 0.7, + "src/provider/Layers/CodexCollabRuntime.integration.test.ts": 3.4, + "src/provider/Layers/CursorAdapter.test.ts": 8, + "src/provider/Layers/CursorProvider.test.ts": 1.4, + "src/provider/Layers/GrokAdapter.test.ts": 9, + "src/provider/Layers/GrokProvider.test.ts": 0.7, + "src/provider/Layers/ProviderService.test.ts": 0.5, + "src/provider/acp/AcpJsonRpcConnection.test.ts": 6.2, + "src/provider/acp/XAiAcpExtension.test.ts": 0.6, + "src/provider/opencodeRuntime.environment.test.ts": 1.1, + "src/pullRequest/PullRequestService.test.ts": 3.1, + "src/server.test.ts": 10.2, + "src/serverSettings.test.ts": 0.5, + "src/serviceLauncher.test.ts": 6.2, + "src/terminal/Manager.test.ts": 1.6, + "src/textGeneration/CursorTextGeneration.test.ts": 0.8, + "src/textGeneration/GrokTextGeneration.test.ts": 1.2, + "src/usage/usageTranscriptStreaming.test.ts": 0.8, + "src/vcs/GitVcsDriver.test.ts": 3.7, + "src/vcs/GitVcsDriverCore.test.ts": 6.5, + "src/workspace/WorkspaceEntries.test.ts": 1.9 +} diff --git a/apps/server/src/testUtils/weightedShardSequencer.test.ts b/apps/server/src/testUtils/weightedShardSequencer.test.ts new file mode 100644 index 000000000000..4d898a1bb9d9 --- /dev/null +++ b/apps/server/src/testUtils/weightedShardSequencer.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vite-plus/test"; +import type { TestSpecification, Vitest } from "vite-plus/test/node"; + +import shardWeights from "./shardWeights.json" with { type: "json" }; +import { WeightedShardSequencer } from "./weightedShardSequencer.ts"; + +const root = "/repo/apps/server"; +const recordedSeconds: Readonly> = shardWeights; +const recorded = Object.keys(recordedSeconds); +const files = [...recorded, ...Array.from({ length: 300 }, (_, i) => `src/fast${i}.test.ts`)]; +const specs = files.map((file) => ({ moduleId: `${root}/${file}` }) as TestSpecification); + +const shardModuleIds = (count: number) => + Promise.all( + Array.from({ length: count }, async (_, i) => { + const ctx = { config: { root, shard: { index: i + 1, count } } } as unknown as Vitest; + const shard = await new WeightedShardSequencer(ctx).shard(specs); + return shard.map((spec) => spec.moduleId); + }), + ); + +describe("WeightedShardSequencer", () => { + it("runs every file in exactly one shard", async () => { + const shards = await shardModuleIds(6); + + expect(shards.flat().toSorted()).toEqual(specs.map((spec) => spec.moduleId).toSorted()); + }); + + it("puts each of the slowest files in a different shard", async () => { + const slowest = new Set( + recorded + .toSorted((a, b) => (recordedSeconds[b] ?? 0) - (recordedSeconds[a] ?? 0)) + .slice(0, 6) + .map((file) => `${root}/${file}`), + ); + const shards = await shardModuleIds(6); + + for (const shard of shards) { + expect(shard.filter((moduleId) => slowest.has(moduleId))).toHaveLength(1); + } + }); +}); diff --git a/apps/server/src/testUtils/weightedShardSequencer.ts b/apps/server/src/testUtils/weightedShardSequencer.ts new file mode 100644 index 000000000000..acee4a9acef6 --- /dev/null +++ b/apps/server/src/testUtils/weightedShardSequencer.ts @@ -0,0 +1,43 @@ +// @effect-diagnostics nodeBuiltinImport:off - vitest loads this sequencer from the config, +// outside any Effect runtime, and only needs path.relative. +import * as NodePath from "node:path"; + +import { BaseSequencer, type TestSpecification } from "vite-plus/test/node"; + +import shardWeights from "./shardWeights.json" with { type: "json" }; + +// What a file costs beyond its recorded test time: CI spends about 0.2s per +// server test file on imports and setup, which the recorded times leave out. +const FILE_OVERHEAD_SECONDS = 0.25; + +const recordedSeconds: Readonly> = shardWeights; + +/** + * Splits server test files across `--shard` runs by recorded duration. Vitest's + * default split hashes file paths into equal-count shards, which can put the + * slowest files on one runner. Here the longest file goes to the lightest shard + * first. Every shard derives the same split from the same file list, so each + * file still runs in exactly one shard. Files without a recorded time count as + * fast files. Record fresh times with `node scripts/update-test-shard-weights.ts`. + */ +export class WeightedShardSequencer extends BaseSequencer { + override async shard(files: TestSpecification[]) { + const { index, count } = this.ctx.config.shard ?? { index: 1, count: 1 }; + const weighted = files + .map((spec) => { + const key = NodePath.relative(this.ctx.config.root, spec.moduleId).replaceAll("\\", "/"); + return { spec, key, seconds: (recordedSeconds[key] ?? 0) + FILE_OVERHEAD_SECONDS }; + }) + .toSorted((a, b) => b.seconds - a.seconds || (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)); + + const loads = Array.from({ length: count }, () => 0); + const picked: TestSpecification[] = []; + for (const file of weighted) { + const least = Math.min(...loads); + const lightest = loads.indexOf(least); + loads[lightest] = least + file.seconds; + if (lightest === index - 1) picked.push(file.spec); + } + return picked; + } +} diff --git a/apps/server/vite.config.ts b/apps/server/vite.config.ts index 14b73d2f0223..bcb0208d9c86 100644 --- a/apps/server/vite.config.ts +++ b/apps/server/vite.config.ts @@ -4,6 +4,7 @@ import { defineConfig, mergeConfig } from "vite-plus"; import baseConfig from "../../vite.config.ts"; import { loadRepoEnv } from "../../scripts/lib/public-config.ts"; import packageJson from "./package.json" with { type: "json" }; +import { WeightedShardSequencer } from "./src/testUtils/weightedShardSequencer.ts"; // The bundle used to inline only workspace packages, leaving every third-party // runtime dep external. External deps must exist on the real filesystem (the WSL @@ -133,6 +134,8 @@ export default mergeConfig( // The server suite exercises sqlite, git, temp worktrees, and orchestration // runtimes heavily. Running files in parallel introduces load-sensitive flakes. fileParallelism: false, + // CI runs the suite as `--shard` runs of equal recorded duration. + sequence: { sequencer: WeightedShardSequencer }, // Appended to the root setup, which mergeConfig concatenates. setupFiles: ["./src/testUtils/gitConfig.setup.ts"], // Server integration tests exercise sqlite, git, and orchestration together. diff --git a/knip.jsonc b/knip.jsonc index 2c7b76acfae0..1bdd744eafdc 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -25,6 +25,7 @@ "scripts/cli.ts", "scripts/evaluate-thread-titles.ts", "scripts/measure-pr-preview.ts", + "scripts/update-test-shard-weights.ts", "src/provider/testFixtures/*.mjs", ], // Keep the transitive Effect runtime pinned for standalone npm installs.