diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8890de34e721..387330713d05 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -259,13 +259,76 @@ jobs: - name: Typecheck run: vp run typecheck + # Keep tests on their own runners, using the same package split and server + # shards as CI, so the release check job does not spend its budget on tests. + test: + name: Release tests + needs: [preflight] + if: ${{ !failure() && !cancelled() && needs.preflight.result == 'success' }} + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + ref: ${{ needs.preflight.outputs.ref }} + sparse-checkout: | + /* + !/.repos/ + sparse-checkout-cone-mode: false + + - name: Setup Vite+ + uses: voidzero-dev/setup-vp@v1 + with: + node-version-file: package.json + cache: true + run-install: true + + - name: Ensure Electron runtime is installed + run: vp run --filter @t3tools/desktop ensure:electron + - uses: ./.github/actions/setup-apt-mirrors - name: Install browser secret helper build libraries - run: sudo apt-get update && sudo apt-get install -y libsecret-1-dev pkg-config + run: | + sudo sed -i 's|http://|https://|g' /etc/apt/blacksmith-ubuntu-mirrors.txt /etc/apt/sources.list.d/ubuntu.sources + sudo apt-get update && sudo apt-get install -y libsecret-1-dev pkg-config build-essential + + - name: Test + run: vp run --parallel --concurrency-limit 4 --filter '!t3' --filter '!@t3tools/monorepo' test + + test_server: + name: Release server tests ${{ matrix.shard }} + needs: [preflight] + if: ${{ !failure() && !cancelled() && needs.preflight.result == 'success' }} + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + ref: ${{ needs.preflight.outputs.ref }} + sparse-checkout: | + /* + !/.repos/ + sparse-checkout-cone-mode: false + + - name: Setup Vite+ + uses: voidzero-dev/setup-vp@v1 + with: + node-version-file: package.json + cache: true + run-install: true + # No Electron setup here: `t3` (apps/server) has no Electron dependency + # and none of its tests touch the runtime. Only the non-server `test` + # job, which covers @t3tools/desktop, needs the download. - name: Test - run: vp run test + run: vp run --filter t3 test --shard ${{ matrix.shard }}/${{ strategy.job-total }} relay_public_config: name: Resolve T3 Connect public config @@ -630,13 +693,15 @@ jobs: preflight, relay_public_config, quality, + test, + test_server, desktop_mac_arm64, desktop_linux_x64, desktop_linux_arm64, desktop_win_x64, desktop_win_arm64, ] - if: ${{ !failure() && !cancelled() && needs.preflight.result == 'success' && needs.relay_public_config.result == 'success' && needs.quality.result == 'success' && needs.desktop_mac_arm64.result == 'success' && needs.desktop_linux_x64.result == 'success' && needs.desktop_linux_arm64.result == 'success' && needs.desktop_win_x64.result == 'success' && needs.desktop_win_arm64.result == 'success' }} + if: ${{ !failure() && !cancelled() && needs.preflight.result == 'success' && needs.relay_public_config.result == 'success' && needs.quality.result == 'success' && needs.test.result == 'success' && needs.test_server.result == 'success' && needs.desktop_mac_arm64.result == 'success' && needs.desktop_linux_x64.result == 'success' && needs.desktop_linux_arm64.result == 'success' && needs.desktop_win_x64.result == 'success' && needs.desktop_win_arm64.result == 'success' }} runs-on: ubuntu-24.04 # blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 15 permissions: diff --git a/apps/desktop/src/electron/ElectronProtocol.test.ts b/apps/desktop/src/electron/ElectronProtocol.test.ts index 508a5c296898..0c2cb2bb7d17 100644 --- a/apps/desktop/src/electron/ElectronProtocol.test.ts +++ b/apps/desktop/src/electron/ElectronProtocol.test.ts @@ -105,7 +105,7 @@ describe("ElectronProtocol", () => { ); assert.include( response.headers.get("content-security-policy") ?? "", - "connect-src 'self' http: https: ws: wss:", + "connect-src 'self' blob: http: https: ws: wss:", ); assert.include( response.headers.get("content-security-policy") ?? "", @@ -255,7 +255,14 @@ describe("ElectronProtocol", () => { "https://clerk.t3.codes", "https://challenges.cloudflare.com", ]); - assert.deepEqual(directives["connect-src"], ["'self'", "http:", "https:", "ws:", "wss:"]); + assert.deepEqual(directives["connect-src"], [ + "'self'", + "blob:", + "http:", + "https:", + "ws:", + "wss:", + ]); assert.deepEqual(directives["img-src"], [ "'self'", "t3code:", diff --git a/apps/desktop/src/electron/ElectronProtocol.ts b/apps/desktop/src/electron/ElectronProtocol.ts index 96cd462bd156..c3cecf88b2e4 100644 --- a/apps/desktop/src/electron/ElectronProtocol.ts +++ b/apps/desktop/src/electron/ElectronProtocol.ts @@ -84,7 +84,8 @@ export function makeDesktopContentSecurityPolicy(input: DesktopProtocolRegistrat // the build-configured Clerk, relay, and OTLP endpoints. Those environment // origins are not known when this response policy is created, so restrict // connections by the network schemes the client supports instead of by host. - const connectSources = ["'self'", "http:", "https:", "ws:", "wss:"]; + // GLTFLoader fetches embedded textures through blob URLs after parsing the model. + const connectSources = ["'self'", "blob:", "http:", "https:", "ws:", "wss:"]; return [ "default-src 'self'", diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index f49b6dbd968e..e8a688c189a3 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -11,6 +11,7 @@ export const MENU_ACTION_CHANNEL = "desktop:menu-action"; export const PASTE_AS_TEXT_CHANNEL = "desktop:paste-as-text"; export const SNAP_SHOT_EVENT_CHANNEL = "desktop:snap-shot-event"; export const QUIT_SHORTCUT_CHANNEL = "desktop:quit-shortcut"; +export const TRACKPAD_SCROLL_END_CHANNEL = "desktop:trackpad-scroll-end"; export const GET_WINDOW_FULLSCREEN_STATE_CHANNEL = "desktop:get-window-fullscreen-state"; export const WINDOW_FULLSCREEN_STATE_CHANNEL = "desktop:window-fullscreen-state"; export const DESKTOP_APP_ACTIVATION_READY_CHANNEL = "desktop:app-activation-ready"; diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 7256f610fbeb..53d8cca7b088 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -79,6 +79,11 @@ contextBridge.exposeInMainWorld("desktopBridge", { ipcRenderer.on(IpcChannels.SET_NOTIFICATION_BADGE_CHANNEL, handler); return () => ipcRenderer.removeListener(IpcChannels.SET_NOTIFICATION_BADGE_CHANNEL, handler); }, + onTrackpadScrollEnd: (listener) => { + const handler = () => listener(); + ipcRenderer.on(IpcChannels.TRACKPAD_SCROLL_END_CHANNEL, handler); + return () => ipcRenderer.removeListener(IpcChannels.TRACKPAD_SCROLL_END_CHANNEL, handler); + }, getSystemLocale: () => { const result = ipcRenderer.sendSync(IpcChannels.GET_SYSTEM_LOCALE_CHANNEL); return typeof result === "string" ? result : null; diff --git a/apps/desktop/src/snapShot/MacModifierPairShortcutProcess.ts b/apps/desktop/src/snapShot/MacModifierPairShortcutProcess.ts index 1c638f2444de..ad7b2e03c495 100644 --- a/apps/desktop/src/snapShot/MacModifierPairShortcutProcess.ts +++ b/apps/desktop/src/snapShot/MacModifierPairShortcutProcess.ts @@ -11,7 +11,11 @@ const MAC_MODIFIER_PAIR_DEVICE_MASKS: Record { }), ); + it.effect("forwards native trackpad release to the renderer", () => + Effect.gen(function* () { + const fakeWindow = makeFakeBrowserWindow(); + const send = vi.spyOn(fakeWindow.window.webContents, "send"); + const createCount = yield* Ref.make(0); + const mainWindow = yield* Ref.make>(Option.none()); + const layer = makeTestLayer({ window: fakeWindow.window, createCount, mainWindow }); + + yield* Effect.gen(function* () { + const desktopWindow = yield* DesktopWindow.DesktopWindow; + yield* desktopWindow.handleBackendReady(new URL("http://127.0.0.1:3773")); + const onInput = fakeWindow.webContentsListeners.get("input-event"); + if (!onInput) return yield* Effect.die("input-event listener was not registered"); + onInput({}, { type: "gestureScrollUpdate" }); + assert.notInclude( + send.mock.calls.map(([channel]) => channel), + TRACKPAD_SCROLL_END_CHANNEL, + ); + onInput({}, { type: "gestureScrollEnd" }); + assert.isTrue(send.mock.calls.some(([channel]) => channel === TRACKPAD_SCROLL_END_CHANNEL)); + }).pipe(Effect.provide(layer)); + }), + ); + // Chromium hands the main window's zoom level down to embedded preview // guests, so every app zoom has to put the preview browser back at its own // zoom or zooming the UI drags the previewed page with it. diff --git a/apps/desktop/src/window/DesktopWindow.ts b/apps/desktop/src/window/DesktopWindow.ts index b3964cb929c7..754de3caa727 100644 --- a/apps/desktop/src/window/DesktopWindow.ts +++ b/apps/desktop/src/window/DesktopWindow.ts @@ -22,6 +22,7 @@ import { MENU_ACTION_CHANNEL, QUIT_SHORTCUT_CHANNEL, SNAP_SHOT_EVENT_CHANNEL, + TRACKPAD_SCROLL_END_CHANNEL, WINDOW_FULLSCREEN_STATE_CHANNEL, } from "../ipc/channels.ts"; import * as PreviewManager from "../preview/Manager.ts"; @@ -661,6 +662,9 @@ export const make = Effect.gen(function* () { event.preventDefault(); } }); + window.webContents.on("input-event", (_event, input) => { + if (input.type === "gestureScrollEnd") window.webContents.send(TRACKPAD_SCROLL_END_CHANNEL); + }); window.on("page-title-updated", (event) => { event.preventDefault(); diff --git a/apps/server/src/mcp/PreviewAutomationBroker.test.ts b/apps/server/src/mcp/PreviewAutomationBroker.test.ts index ef552f9b04d0..7a8ed739d734 100644 --- a/apps/server/src/mcp/PreviewAutomationBroker.test.ts +++ b/apps/server/src/mcp/PreviewAutomationBroker.test.ts @@ -685,6 +685,7 @@ it.effect("pins a provider session to its initial host despite later focus chang environmentId: scope.environmentId, connectionId: "connection-stale", focused: true, + liveTabs: [{ threadId: scope.threadId, tabId: PreviewTabId.make("stale-tab") }], }); expect(yield* broker.invoke({ scope, operation: "status", input: {} })).toBe( "second", @@ -725,6 +726,130 @@ it.effect("pins a provider session to its initial host despite later focus chang ), ); +it.effect("prefers the live tab owner for new sessions without moving existing leases", () => + Effect.scoped( + Effect.gen(function* () { + const broker = yield* makeBroker; + const connections = new Map(); + for (const clientId of ["owner", "other"]) { + const requests = requestsFrom( + yield* broker.connect(makeHost({ clientId })), + (connectionId) => connections.set(clientId, connectionId), + ); + yield* Stream.runForEach(requests, (request) => + broker.respond({ + clientId, + connectionId: request.connectionId, + requestId: request.requestId, + ok: true, + result: clientId, + }), + ).pipe(Effect.forkScoped); + } + yield* Effect.yieldNow; + yield* broker.focusHost({ + clientId: "owner", + environmentId: scope.environmentId, + connectionId: connections.get("owner")!, + focused: false, + liveTabs: [ + { threadId: scope.threadId, tabId: PreviewTabId.make("signed-in"), visible: true }, + ], + }); + yield* broker.focusHost({ + clientId: "other", + environmentId: scope.environmentId, + connectionId: connections.get("other")!, + focused: true, + liveTabs: [ + { threadId: scope.threadId, tabId: PreviewTabId.make("signed-in"), visible: false }, + { + threadId: ThreadId.make("another-thread"), + tabId: PreviewTabId.make("different-tab"), + visible: true, + }, + ], + }); + expect(yield* broker.invoke({ scope, operation: "evaluate", input: {} })).toBe( + "owner", + ); + expect( + yield* broker.invoke({ + scope: { ...scope, providerSessionId: "explicit-owner" }, + tabId: PreviewTabId.make("signed-in"), + operation: "snapshot", + input: {}, + }), + ).toBe("owner"); + expect( + yield* broker.invoke({ + scope: { ...scope, providerSessionId: "other-tab" }, + tabId: PreviewTabId.make("different-tab"), + operation: "evaluate", + input: {}, + }), + ).toBe("other"); + + yield* broker.focusHost({ + clientId: "owner", + environmentId: scope.environmentId, + connectionId: connections.get("owner")!, + focused: false, + liveTabs: [], + }); + expect(yield* broker.invoke({ scope, operation: "evaluate", input: {} })).toBe( + "owner", + ); + expect( + yield* broker.invoke({ + scope: { ...scope, providerSessionId: "after-tab-closed" }, + operation: "evaluate", + input: {}, + }), + ).toBe("other"); + }), + ), +); + +it.effect("prefers a focused host over unrelated extra capabilities for a new session", () => + Effect.scoped( + Effect.gen(function* () { + const broker = yield* makeBroker; + let focusedConnectionId = ""; + for (const [clientId, supportedOperations] of [ + ["focused", ["status"]], + ["background", ["status", "resize"]], + ] as const) { + const requests = requestsFrom( + yield* broker.connect(makeHost({ clientId, supportedOperations })), + (connectionId) => { + if (clientId === "focused") focusedConnectionId = connectionId; + }, + ); + yield* Stream.runForEach(requests, (request) => + broker.respond({ + clientId, + connectionId: request.connectionId, + requestId: request.requestId, + ok: true, + result: clientId, + }), + ).pipe(Effect.forkScoped); + } + yield* Effect.yieldNow; + yield* broker.focusHost({ + clientId: "focused", + environmentId: scope.environmentId, + connectionId: focusedConnectionId, + focused: true, + }); + expect(yield* broker.invoke({ scope, operation: "status", input: {} })).toBe( + "focused", + ); + }), + ), +); + it.effect("does not route new operations to legacy hosts that did not advertise support", () => Effect.scoped( Effect.gen(function* () { @@ -921,6 +1046,7 @@ it.effect("fails over a pinned provider session only after its host disconnects" environmentId: scope.environmentId, connectionId: firstConnectionId, focused: true, + liveTabs: [{ threadId: scope.threadId, tabId: firstTabId }], }); expect(yield* broker.invoke({ scope, operation: "open", input: {} })).toEqual({ host: "first", diff --git a/apps/server/src/mcp/PreviewAutomationBroker.ts b/apps/server/src/mcp/PreviewAutomationBroker.ts index 527f71f5915d..65e3064f49a5 100644 --- a/apps/server/src/mcp/PreviewAutomationBroker.ts +++ b/apps/server/src/mcp/PreviewAutomationBroker.ts @@ -73,6 +73,7 @@ interface ClientConnection { readonly environmentId: PreviewAutomationHost["environmentId"]; readonly supportedOperations: ReadonlySet; readonly focused: boolean; + readonly liveTabs: NonNullable; readonly focusOrder: number; readonly queue: Queue.Queue; } @@ -376,6 +377,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { environmentId: host.environmentId, supportedOperations: new Set(host.supportedOperations ?? PREVIEW_AUTOMATION_V1_OPERATIONS), focused: false, + liveTabs: [], focusOrder: 0, queue, }; @@ -432,6 +434,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { clients.set(host.clientId, { ...currentHost, focused: host.focused, + liveTabs: host.liveTabs ?? currentHost.liveTabs, focusOrder: host.focused ? focusSequence : currentHost.focusOrder, }); return { ...current, clients, focusSequence }; @@ -492,6 +495,13 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { // operation is not silently moved to a newer client: the caller gets a // capability failure and can deliberately start a fresh provider // session. A dead lease is pruned above and may fail over. + const ownsTargetTab = (host: ClientConnection, visibleOnly = false) => + host.liveTabs.some( + (tab) => + tab.threadId === input.scope.threadId && + (!visibleOnly || tab.visible === true) && + (input.tabId === undefined || tab.tabId === input.tabId), + ); const connection = hasLiveAssignment && supportsOperation(assignedConnection, input.operation) ? assignedConnection @@ -505,7 +515,8 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { ) .sort( (left, right) => - right.supportedOperations.size - left.supportedOperations.size || + Number(ownsTargetTab(right, true)) - Number(ownsTargetTab(left, true)) || + Number(ownsTargetTab(right)) - Number(ownsTargetTab(left)) || Number(right.focused) - Number(left.focused) || right.focusOrder - left.focusOrder, )[0]; diff --git a/apps/server/src/provider/Layers/ProviderRegistry.test.ts b/apps/server/src/provider/Layers/ProviderRegistry.test.ts index 354484e6d395..9ddf54deb3af 100644 --- a/apps/server/src/provider/Layers/ProviderRegistry.test.ts +++ b/apps/server/src/provider/Layers/ProviderRegistry.test.ts @@ -38,6 +38,7 @@ import { checkClaudeProviderStatus } from "./ClaudeProvider.ts"; import * as BackgroundPolicy from "../../background/BackgroundPolicy.ts"; import { AntigravityInstallation } from "../AntigravityInstallation.ts"; import * as ModelManifest from "../ModelManifest.ts"; +import { applyProviderCompatibility } from "../providerCompatibility.ts"; import * as CodexResetCredit from "./codexResetCredit.ts"; import * as OpenCodeRuntime from "../opencodeRuntime.ts"; import * as ProviderEventLoggers from "./ProviderEventLoggers.ts"; @@ -76,6 +77,12 @@ process.env.T3CODE_CURSOR_ENABLED = "1"; const encoder = new TextEncoder(); const TEST_EPOCH = DateTime.makeUnsafe("1970-01-01T00:00:00.000Z"); +const withBundledCompatibility = (snapshot: ServerProvider) => + applyProviderCompatibility( + snapshot, + undefined, + ModelManifest.BUNDLED_MODEL_MANIFEST.compatibility, + ); // Provider metadata checks use a bundled manifest and stubbed HTTP. const TestHttpClientLive = Layer.succeed( @@ -1754,7 +1761,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, ServerSettingsModule.layerTest(), Te ); assert.deepStrictEqual( recoveredProviders.find((provider) => provider.instanceId === codexInstanceId), - codexProvider, + withBundledCompatibility(codexProvider), ); yield* Ref.set(catalogSnapshot, changedCatalogProvider); @@ -1766,7 +1773,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, ServerSettingsModule.layerTest(), Te ); assert.deepStrictEqual( changedProviders.find((provider) => provider.instanceId === codexInstanceId), - codexProvider, + withBundledCompatibility(codexProvider), ); }).pipe(Effect.provide(runtimeServices)); @@ -1880,10 +1887,13 @@ it.layer(Layer.mergeAll(NodeServices.layer, ServerSettingsModule.layerTest(), Te yield* Fiber.join(persisted); const cachedProvider = yield* readProviderStatusCache(filePath); - assert.deepStrictEqual(cachedProvider, { - ...refreshedProvider, - models: [...initialProvider.models], - }); + assert.deepStrictEqual( + cachedProvider, + withBundledCompatibility({ + ...refreshedProvider, + models: [...initialProvider.models], + }), + ); }).pipe(Effect.provide(runtimeServices)); }), ); @@ -2095,10 +2105,14 @@ it.layer(Layer.mergeAll(NodeServices.layer, ServerSettingsModule.layerTest(), Te yield* Effect.gen(function* () { const registry = yield* ProviderRegistry.ProviderRegistry; - assert.deepStrictEqual(yield* registry.getProviders, [cachedProvider]); - assert.deepStrictEqual(yield* registry.refresh(codexDriver), [cachedProvider]); + assert.deepStrictEqual(yield* registry.getProviders, [ + withBundledCompatibility(cachedProvider), + ]); + assert.deepStrictEqual(yield* registry.refresh(codexDriver), [ + withBundledCompatibility(cachedProvider), + ]); assert.deepStrictEqual(yield* registry.refreshInstance(codexInstanceId), [ - cachedProvider, + withBundledCompatibility(cachedProvider), ]); }).pipe(Effect.provide(runtimeServices)); }), @@ -2206,7 +2220,9 @@ it.layer(Layer.mergeAll(NodeServices.layer, ServerSettingsModule.layerTest(), Te yield* Effect.gen(function* () { const registry = yield* ProviderRegistry.ProviderRegistry; - assert.deepStrictEqual(yield* registry.getProviders, [codexProvider]); + assert.deepStrictEqual(yield* registry.getProviders, [ + withBundledCompatibility(codexProvider), + ]); yield* Ref.set(failNextList, true); yield* PubSub.publish(changes, undefined); diff --git a/apps/server/src/provider/model-manifest.json b/apps/server/src/provider/model-manifest.json index 32a103be9d16..69e51bcb019a 100644 --- a/apps/server/src/provider/model-manifest.json +++ b/apps/server/src/provider/model-manifest.json @@ -1,12 +1,58 @@ { "version": 1, - "updatedAt": "2026-09-22T21:40:00Z", + "updatedAt": "2026-09-24T00:11:29Z", "compatibility": [ + { + "driver": "codex", + "t3CodeRange": ">=0.0.42", + "recommendedRange": ">=0.129.0", + "ranges": [ + { "range": ">=0.129.0", "status": "supported" }, + { "range": "<0.129.0", "status": "broken" } + ] + }, + { + "driver": "claudeAgent", + "t3CodeRange": ">=0.0.42", + "recommendedRange": ">=2.1.280", + "ranges": [ + { "range": ">=2.1.280", "status": "supported" }, + { "range": ">=2.1.111 <2.1.280", "status": "graceful" }, + { "range": "<2.1.111", "status": "unsupported" } + ] + }, + { + "driver": "cursor", + "t3CodeRange": ">=0.0.42", + "recommendedRange": ">=2026.05.09", + "ranges": [ + { "range": ">=2026.05.09", "status": "supported" }, + { "range": "<2026.05.09", "status": "unknown" } + ] + }, + { + "driver": "grok", + "t3CodeRange": ">=0.0.42", + "recommendedRange": ">=1.0.13", + "ranges": [ + { "range": ">=1.0.13", "status": "supported" }, + { "range": "<1.0.13", "status": "unknown" } + ] + }, { "driver": "opencode", "t3CodeRange": ">=0.0.42", "recommendedRange": ">=1.14.19", - "ranges": [{ "range": "<1.14.19", "status": "broken" }] + "ranges": [ + { "range": ">=1.14.19", "status": "supported" }, + { "range": "<1.14.19", "status": "broken" } + ] + }, + { + "driver": "antigravity", + "t3CodeRange": ">=0.0.42", + "recommendedRange": "=1.1.1", + "ranges": [{ "range": "=1.1.1", "status": "supported" }] } ], "currentModels": { diff --git a/apps/server/src/provider/providerCompatibility.test.ts b/apps/server/src/provider/providerCompatibility.test.ts index e6b4e1a608b9..3a43356d2caf 100644 --- a/apps/server/src/provider/providerCompatibility.test.ts +++ b/apps/server/src/provider/providerCompatibility.test.ts @@ -15,6 +15,7 @@ import { ProviderRegistry } from "./Services/ProviderRegistry.ts"; import { ProviderInstanceRegistry } from "./Services/ProviderInstanceRegistry.ts"; import type { ProviderInstance } from "./ProviderDriver.ts"; import { makeManualOnlyProviderMaintenanceCapabilities } from "./providerMaintenance.ts"; +import { BUILT_IN_DRIVERS } from "./builtInDrivers.ts"; import * as Schema from "effect/Schema"; import { applyProviderCompatibility, @@ -51,6 +52,68 @@ const provider: ServerProvider = { }; describe("provider compatibility", () => { + it("bundles a compatibility policy for every built-in harness", () => { + for (const builtIn of BUILT_IN_DRIVERS) { + assert.isDefined( + resolveProviderCompatibility( + ModelManifest.BUNDLED_MODEL_MANIFEST.compatibility, + builtIn.driverKind, + null, + ), + `Missing bundled compatibility policy for ${builtIn.driverKind}`, + ); + } + }); + + it("compares Cursor build dates without treating semver prereleases as stable", () => { + const cursor = ProviderDriverKind.make("cursor"); + const cursorPolicy: ProviderCompatibilityPolicy = { + driver: cursor, + t3CodeRange: policy.t3CodeRange, + ranges: [ + { range: "<2026.05.09", status: "unsupported" }, + { range: ">=2026.05.09", status: "supported" }, + ], + }; + for (const [version, expected] of [ + ["2026.05.08-a1b2c3d", "unsupported"], + ["2026.05.09-a1b2c3d", "supported"], + ["2026.09.22-f2b0fcd", "supported"], + ["2026.05.09", "supported"], + ["2026.05.09-beta.1", "unknown"], + ] as const) { + assert.strictEqual( + resolveProviderCompatibility([cursorPolicy], cursor, version)?.status, + expected, + ); + } + assert.strictEqual( + resolveProviderCompatibility([policy], driver, "2.0.0-a1b2c3d")?.status, + "unknown", + ); + }); + + it("recognizes Antigravity semver release tags while keeping dated candidates unknown", () => { + const antigravity = ProviderDriverKind.make("antigravity"); + const taggedPolicy = { + ...policy, + driver: antigravity, + ranges: [{ range: "=2.0.0", status: "supported" as const }], + }; + for (const [version, expected] of [ + ["agy_acp_server_2.0.0", "supported"], + ["2.0.0", "supported"], + ["agy_acp_server_2.0.1", "unknown"], + ["agy_acp_server_2.0.0-beta.1", "unknown"], + ["agy_acp_server_20260818_01_RC01", "unknown"], + ] as const) { + assert.strictEqual( + resolveProviderCompatibility([taggedPolicy], antigravity, version)?.status, + expected, + ); + } + }); + it("classifies boundaries and treats unlisted versions and release tags as unknown", () => { for (const [version, expected] of [ ["0.9.9", "broken"], diff --git a/apps/server/src/provider/providerCompatibility.ts b/apps/server/src/provider/providerCompatibility.ts index 11aedf8dca0d..b4ce66f87603 100644 --- a/apps/server/src/provider/providerCompatibility.ts +++ b/apps/server/src/provider/providerCompatibility.ts @@ -10,7 +10,7 @@ import * as Schema from "effect/Schema"; import packageJson from "../../package.json" with { type: "json" }; // Deliberately uses the shared CLI gate syntax: comparator groups joined by ||. -// Release tags and prereleases remain unknown instead of matching stable ranges. +// Prereleases and unrecognized release tags remain unknown. const StableVersion = TrimmedNonEmptyString.pipe( Schema.check(Schema.makeFilter((value) => /^\d+\.\d+\.\d+$/.test(value))), ); @@ -66,7 +66,15 @@ export function resolveProviderCompatibility( (entry) => entry.driver === driver && satisfiesSemverRange(t3CodeVersion, entry.t3CodeRange), ); if (!policy) return undefined; - const stable = version?.replace(/^v/, ""); + const unprefixed = version?.replace(/^v/, ""); + // Cursor appends a build hash to its date; Google's ACP runtime uses a release prefix. + // Strip only these driver-specific forms, keeping semver prereleases unknown. + const stable = + driver === "cursor" + ? unprefixed?.replace(/^(\d{4}\.\d{2}\.\d{2})-[a-f0-9]+$/, "$1") + : driver === "antigravity" + ? unprefixed?.replace(/^agy_acp_server_(\d+\.\d+\.\d+)$/, "$1") + : unprefixed; const status = stable && /^\d+\.\d+\.\d+$/.test(stable) ? (policy.ranges.find((entry) => satisfiesSemverRange(stable, entry.range))?.status ?? diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index 2da7c9183365..998c301cb1fb 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -211,7 +211,13 @@ function makeRegistry( const makeTestRunner = ( registry: ProviderRegistryShape, - manifest = ModelManifest.BUNDLED_MODEL_MANIFEST, + // Generic updater fixtures use synthetic versions. Keep their compatibility + // unknown so real harness minimums do not bypass the command under test. + manifest: ModelManifest.ModelManifestData = { + version: 1, + currentModels: {}, + compatibility: [{ driver: CODEX_DRIVER, t3CodeRange: ">=0.0.42", ranges: [] }], + }, ) => Effect.service(ProviderMaintenanceRunner.ProviderMaintenanceRunner).pipe( Effect.provide( diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index 1e7cdc3f00e6..32cf68a61e18 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -20,6 +20,7 @@ import { CommandId, ProviderInstanceId } from "@t3tools/contracts"; import { RelayClientTracer } from "@t3tools/shared/relayTracing"; import { RELAY_ACTIVITY_PUBLISH_TYP, verifyRelayJwt } from "@t3tools/shared/relayJwt"; import { describe, expect, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -325,6 +326,9 @@ describe.sequential("signRelayAgentActivityPublishProof", () => { const projectId = "project-1" as ProjectId; const activeThreadId = "thread-active" as ThreadId; const idleThreadId = "thread-idle" as ThreadId; + const oldCompletedId = "thread-old-completed" as ThreadId; + const newCompletedId = "thread-new-completed" as ThreadId; + const freshMessageId = "thread-fresh-message" as ThreadId; const baseThread = { projectId, @@ -351,6 +355,7 @@ describe.sequential("signRelayAgentActivityPublishProof", () => { expect( AgentAwarenessRelay.resolveAgentAwarenessRelayActiveThreadIds({ environmentId, + startedAt: Date.parse(now), projects: [ { id: projectId, @@ -374,6 +379,44 @@ describe.sequential("signRelayAgentActivityPublishProof", () => { ...baseThread, id: idleThreadId, }, + { + ...baseThread, + id: oldCompletedId, + latestTurn: { + turnId: "turn-old" as TurnId, + state: "completed", + requestedAt: "2026-05-24T00:00:00.000Z", + startedAt: "2026-05-24T00:00:00.000Z", + completedAt: "2026-05-24T00:01:00.000Z", + assistantMessageId: null, + }, + }, + { + ...baseThread, + id: newCompletedId, + latestTurn: { + turnId: "turn-new" as TurnId, + state: "completed", + requestedAt: "2026-05-25T00:00:01.000Z", + startedAt: "2026-05-25T00:00:01.000Z", + completedAt: "2026-05-25T00:00:02.000Z", + assistantMessageId: null, + }, + }, + { + ...baseThread, + id: freshMessageId, + latestUserMessageAt: "2026-05-25T00:00:01.000Z", + session: { + threadId: freshMessageId, + status: "ready", + providerName: "Codex", + runtimeMode: "full-access", + activeTurnId: null, + lastError: null, + updatedAt: "2026-05-25T00:00:02.000Z", + }, + }, { ...baseThread, id: "thread-missing-project" as ThreadId, @@ -389,7 +432,7 @@ describe.sequential("signRelayAgentActivityPublishProof", () => { }, ], }), - ).toEqual([activeThreadId]); + ).toEqual([activeThreadId, newCompletedId]); }); it.effect("signs the activity publish JWT and rejects tampering", () => @@ -797,4 +840,138 @@ describe.sequential("signRelayAgentActivityPublishProof", () => { }), ), ); + + it.effect("does not alert for historical completions after startup", () => + Effect.scoped( + Effect.gen(function* () { + const secrets = makeMemorySecretStore(); + const now = yield* DateTime.now; + const old = DateTime.formatIso(DateTime.add(now, { days: -7 })); + const threadId = "thread-old" as ThreadId; + const projectId = "project-1" as ProjectId; + const environmentId = "env-1" as EnvironmentId; + const project = { + id: projectId, + title: "T3 Code", + workspaceRoot: "/workspace", + repositoryIdentity: null, + defaultModelSelection: null, + scripts: [], + createdAt: old, + updatedAt: old, + } satisfies OrchestrationProjectShell; + const completedTurn = { + turnId: "turn-1" as TurnId, + state: "completed", + requestedAt: old, + startedAt: old, + completedAt: old, + assistantMessageId: null, + } as const; + const completedThread = { + id: threadId, + projectId, + title: "Old task", + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + pullRequests: [], + latestTurn: completedTurn, + createdAt: old, + updatedAt: old, + archivedAt: null, + settledOverride: null, + settledAt: null, + session: null, + latestUserMessageAt: old, + hasPendingApprovals: false, + hasPendingUserInput: false, + hasActionableProposedPlan: false, + } satisfies OrchestrationThreadShell; + let currentThread: OrchestrationThreadShell | null = completedThread; + let publishes = 0; + const originalFetch = globalThis.fetch; + globalThis.fetch = (() => { + publishes += 1; + return Promise.resolve(Response.json({ ok: true, deliveries: [] })); + }) as unknown as typeof fetch; + yield* Effect.addFinalizer(() => + Effect.sync(() => { + globalThis.fetch = originalFetch; + }), + ); + yield* secrets.setString(RELAY_URL_SECRET, "https://relay.example.test"); + yield* secrets.setString(RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "relay-credential"); + yield* secrets.setString(PUBLISH_AGENT_ACTIVITY_SECRET, "true"); + + const layer = Layer.mergeAll( + Layer.succeed(ServerSecretStore.ServerSecretStore, secrets.store), + Layer.succeed(ServerEnvironment.ServerEnvironment, { + getEnvironmentId: Effect.succeed(environmentId), + getDescriptor: Effect.die("unused descriptor"), + }), + Layer.succeed(OrchestrationEngineService, {} as OrchestrationEngineShape), + Layer.succeed(ProjectionSnapshotQuery, { + getThreadShellById: () => Effect.sync(() => Option.fromNullishOr(currentThread)), + getProjectShellById: () => Effect.succeed(Option.some(project)), + } as unknown as ProjectionSnapshotQueryShape), + ); + + yield* Effect.gen(function* () { + const relay = yield* AgentAwarenessRelay.AgentAwarenessRelay; + yield* relay.publishThread(threadId); + expect(publishes).toBe(0); + + currentThread = { + ...completedThread, + latestTurn: null, + latestUserMessageAt: DateTime.formatIso(DateTime.add(now, { seconds: 1 })), + session: { + threadId, + status: "ready", + providerName: "Codex", + runtimeMode: "full-access", + activeTurnId: null, + lastError: null, + updatedAt: DateTime.formatIso(DateTime.add(now, { seconds: 2 })), + }, + }; + expect( + AgentAwarenessRelay.resolveAgentAwarenessRelayPublishSnapshot({ + environmentId, + threadId, + thread: Option.some(currentThread), + project: Option.some(project), + }).state?.phase, + ).toBe("completed"); + yield* relay.publishThread(threadId); + expect(publishes).toBe(0); + + currentThread = { + ...completedThread, + session: { + threadId, + status: "error", + providerName: "Codex", + runtimeMode: "full-access", + activeTurnId: null, + lastError: "old failure", + updatedAt: old, + }, + }; + yield* relay.publishThread(threadId); + expect(publishes).toBe(0); + }).pipe( + Effect.provide( + AgentAwarenessRelay.layer.pipe( + Layer.provide(layer), + Layer.provideMerge(NodeServices.layer), + ), + ), + ); + }), + ), + ); }); diff --git a/apps/server/src/relay/AgentAwarenessRelay.ts b/apps/server/src/relay/AgentAwarenessRelay.ts index 052a67959ad1..29d2abecebb7 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.ts @@ -267,8 +267,13 @@ export function resolveAgentAwarenessRelayPublishSnapshot(input: { }; } +function terminalWorkSinceStart(thread: OrchestrationThreadShell, startedAt: number): boolean { + return Date.parse(thread.latestTurn?.completedAt ?? "") > startedAt; +} + export function resolveAgentAwarenessRelayActiveThreadIds(input: { readonly environmentId: EnvironmentId; + readonly startedAt: number; readonly projects: ReadonlyArray>; readonly threads: ReadonlyArray; }): ReadonlyArray { @@ -279,12 +284,16 @@ export function resolveAgentAwarenessRelayActiveThreadIds(input: { if (!project) { return false; } + const state = projectThreadAwareness({ + environmentId: input.environmentId, + project, + thread, + }); return ( - projectThreadAwareness({ - environmentId: input.environmentId, - project, - thread, - }) !== null + state !== null && + (state.phase !== "completed" && state.phase !== "failed" + ? true + : terminalWorkSinceStart(thread, input.startedAt)) ); }) .map((thread) => thread.id); @@ -298,6 +307,7 @@ export const make = Effect.gen(function* () { const orchestrationEngine = yield* OrchestrationEngine.OrchestrationEngineService; const crypto = yield* Crypto.Crypto; const cloudLinkKeyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(secrets); + const startedAt = (yield* DateTime.now).epochMilliseconds; const activeSnapshotPublishedRef = yield* Ref.make(false); const publishedStateByThreadRef = yield* Ref.make(new Map()); @@ -417,6 +427,14 @@ export const make = Effect.gen(function* () { }); const publishIdentity = agentAwarenessPublishIdentity(snapshot.state); const publishedStateByThread = yield* Ref.get(publishedStateByThreadRef); + if ( + (snapshot.state?.phase === "completed" || snapshot.state?.phase === "failed") && + !publishedStateByThread.has(threadId) + ) { + // Startup has no publish history. Only work from this server process may + // produce an initial terminal alert; historical threads remain quiet. + if (Option.isNone(thread) || !terminalWorkSinceStart(thread.value, startedAt)) return; + } if (publishedStateByThread.get(threadId) === publishIdentity) { // The projection is back at (or never left) the last published state, so // any pending deferred confirmation is moot. Leaving the deadline in @@ -495,7 +513,11 @@ export const make = Effect.gen(function* () { }); yield* Ref.update(publishedStateByThreadRef, (publishedStates) => { const nextPublishedStates = new Map(publishedStates); - nextPublishedStates.set(threadId, publishIdentity); + if (snapshot.state === null) { + nextPublishedStates.delete(threadId); + } else { + nextPublishedStates.set(threadId, publishIdentity); + } return nextPublishedStates; }); }); @@ -529,6 +551,7 @@ export const make = Effect.gen(function* () { const snapshot = yield* snapshotQuery.getShellSnapshot(); const activeThreadIds = resolveAgentAwarenessRelayActiveThreadIds({ environmentId, + startedAt, projects: snapshot.projects, threads: snapshot.threads, }); diff --git a/apps/web/src/components/BranchToolbar.tsx b/apps/web/src/components/BranchToolbar.tsx index 8e84ec5d1ed9..912ee901eec8 100644 --- a/apps/web/src/components/BranchToolbar.tsx +++ b/apps/web/src/components/BranchToolbar.tsx @@ -5,7 +5,6 @@ import { FolderGit2Icon, FolderGitIcon, FolderIcon, - HistoryIcon, ScaleIcon, } from "lucide-react"; import { @@ -41,6 +40,7 @@ import { } from "./BranchToolbarBranchSelector"; import { BranchToolbarEnvironmentSelector } from "./BranchToolbarEnvironmentSelector"; import { BranchToolbarEnvModeSelector } from "./BranchToolbarEnvModeSelector"; +import { PreviousWorktreeItemContent } from "./PreviousWorktreeItemContent"; import { ComposerControl } from "./chat/ComposerControl"; import { Menu, @@ -105,6 +105,7 @@ interface MobileRunContextSelectorProps { activeWorktreePath: string | null; onEnvModeChange: (mode: EnvMode) => void; previousWorktreeLabel: string | null; + previousWorktreeBranch: string | null; onUsePreviousWorktree: () => void; } @@ -123,6 +124,7 @@ const MobileRunContextSelector = memo(function MobileRunContextSelector({ activeWorktreePath, onEnvModeChange, previousWorktreeLabel, + previousWorktreeBranch, onUsePreviousWorktree, }: MobileRunContextSelectorProps) { const composerFloatingLayerProps = useComposerMenuProps(); @@ -217,7 +219,12 @@ const MobileRunContextSelector = memo(function MobileRunContextSelector({ {triggerContent} - + {showEnvironmentPicker && availableEnvironments && onEnvironmentChange ? ( <> @@ -295,10 +302,7 @@ const MobileRunContextSelector = memo(function MobileRunContextSelector({ {previousWorktreeLabel ? ( - - - - + ) : null} @@ -630,6 +634,7 @@ export const BranchToolbar = memo(function BranchToolbar({ activeWorktreePath={activeWorktreePath} onEnvModeChange={onEnvModeChange} previousWorktreeLabel={previousWorktreeLabel} + previousWorktreeBranch={previousWorktreeSeed?.branch ?? null} onUsePreviousWorktree={onUsePreviousWorktree} /> @@ -669,6 +674,7 @@ export const BranchToolbar = memo(function BranchToolbar({ activeWorktreePath={activeWorktreePath} onEnvModeChange={onEnvModeChange} previousWorktreeLabel={previousWorktreeLabel} + previousWorktreeBranch={previousWorktreeSeed?.branch ?? null} onUsePreviousWorktree={onUsePreviousWorktree} /> ) : null} diff --git a/apps/web/src/components/BranchToolbarEnvModeSelector.tsx b/apps/web/src/components/BranchToolbarEnvModeSelector.tsx index 1b8b81cfcd74..cb1faef9a50d 100644 --- a/apps/web/src/components/BranchToolbarEnvModeSelector.tsx +++ b/apps/web/src/components/BranchToolbarEnvModeSelector.tsx @@ -1,4 +1,4 @@ -import { FolderGit2Icon, FolderGitIcon, FolderIcon, HistoryIcon } from "lucide-react"; +import { FolderGit2Icon, FolderGitIcon, FolderIcon } from "lucide-react"; import { memo, useMemo } from "react"; import { @@ -8,6 +8,7 @@ import { type EnvMode, } from "./BranchToolbar.logic"; import { useComposerMenuProps } from "./chat/composerEventScope"; +import { PreviousWorktreeItemContent } from "./PreviousWorktreeItemContent"; import { Select, SelectGroup, @@ -28,6 +29,7 @@ interface BranchToolbarEnvModeSelectorProps { activeWorktreePath: string | null; onEnvModeChange: (mode: EnvMode) => void; previousWorktreeLabel?: string | null; + previousWorktreeBranch?: string | null; onUsePreviousWorktree?: () => void; } @@ -38,6 +40,7 @@ export const BranchToolbarEnvModeSelector = memo(function BranchToolbarEnvModeSe activeWorktreePath, onEnvModeChange, previousWorktreeLabel, + previousWorktreeBranch = null, onUsePreviousWorktree, }: BranchToolbarEnvModeSelectorProps) { const composerFloatingLayerProps = useComposerMenuProps(); @@ -142,7 +145,11 @@ export const BranchToolbarEnvModeSelector = memo(function BranchToolbarEnvModeSe : resolveCurrentWorkspaceLabel(activeWorktreePath)} - + Workspace @@ -163,10 +170,7 @@ export const BranchToolbarEnvModeSelector = memo(function BranchToolbarEnvModeSe {showPreviousWorktree && previousWorktreeLabel ? ( - - - {previousWorktreeLabel} - + ) : null} diff --git a/apps/web/src/components/ComposerPromptEditorTiptap.tsx b/apps/web/src/components/ComposerPromptEditorTiptap.tsx index 8af2d18b310e..50305ffdbef0 100644 --- a/apps/web/src/components/ComposerPromptEditorTiptap.tsx +++ b/apps/web/src/components/ComposerPromptEditorTiptap.tsx @@ -718,7 +718,7 @@ function ComposerPromptEditorTiptapInner(props: ComposerPromptEditorProps) { const editorAttributes = useMemo( () => ({ class: cn( - "composer-tiptap block max-h-50 min-h-17.5 w-full overflow-y-auto whitespace-pre-wrap wrap-break-word bg-transparent leading-relaxed text-foreground focus:outline-none", + "composer-tiptap -m-1 block max-h-52 min-h-19.5 overflow-y-auto p-1 whitespace-pre-wrap wrap-break-word bg-transparent leading-relaxed text-foreground focus:outline-none", className, ), "data-testid": "composer-editor", @@ -1242,7 +1242,7 @@ function ComposerPromptEditorTiptapInner(props: ComposerPromptEditorProps) {
diff --git a/apps/web/src/components/PreviousWorktreeItemContent.tsx b/apps/web/src/components/PreviousWorktreeItemContent.tsx new file mode 100644 index 000000000000..cd25c8c497e2 --- /dev/null +++ b/apps/web/src/components/PreviousWorktreeItemContent.tsx @@ -0,0 +1,19 @@ +import { HistoryIcon } from "lucide-react"; + +import { MiddleTruncate } from "./ui/middle-truncate"; + +export function PreviousWorktreeItemContent({ branch }: { branch: string | null }) { + return ( + + + + Previous worktree + {branch ? ( + + + + ) : null} + + + ); +} diff --git a/apps/web/src/components/RightPanelTabs.tsx b/apps/web/src/components/RightPanelTabs.tsx index d88b1309f0ad..33a1c9b05c3a 100644 --- a/apps/web/src/components/RightPanelTabs.tsx +++ b/apps/web/src/components/RightPanelTabs.tsx @@ -63,6 +63,7 @@ import { ScrollArea } from "~/components/ui/scroll-area"; import { PanelTabCloseButton } from "~/components/ui/panel-tab-close-button"; import { faviconUrlForOrigin } from "~/lib/favicon"; import { useTheme } from "~/hooks/useTheme"; +import { useDeviceState } from "~/state/device"; import { newestPullRequestSummary, pullRequestEnvironment, @@ -1244,7 +1245,17 @@ export function RightPanelTabs(props: RightPanelTabsProps) { } /> - {title} + + {surface.kind === "device" ? ( + + ) : ( + title + )} + )}
@@ -1421,3 +1432,27 @@ export function RightPanelTabs(props: RightPanelTabsProps) { ); } + +function DeviceTabTooltip(props: { + surface: Extract; + environmentId: EnvironmentId | null; + title: string; +}) { + const target = props.surface.target; + const { state } = useDeviceState(target ? props.environmentId : null); + const device = target + ? state.devices.find((entry) => entry.hostId === target.hostId && entry.id === target.deviceId) + : undefined; + const host = target ? state.hosts.find((entry) => entry.id === target.hostId) : undefined; + return ( +
+ {props.title} + {target ? ( + + {host?.label ?? "Device host"} ยท{" "} + {device?.version ?? (target.platform === "ios" ? "iOS" : "Android")} + + ) : null} +
+ ); +} diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index b377439882d5..15440c577ef3 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -6859,10 +6859,10 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) skills={selectedProviderSkills} containerClassName={cn(isComposerResting && "min-w-0 flex-1")} className={cn( - showMobilePendingAnswerActions && "max-sm:pb-11", + showMobilePendingAnswerActions && "max-sm:pb-12", isComposerResting && - "max-h-8 min-h-8 overflow-hidden whitespace-pre! leading-8", - isComposerApprovalState && "min-h-8", + "my-0 max-h-8 min-h-8 overflow-hidden py-0 whitespace-pre! leading-8", + isComposerApprovalState && "min-h-10", )} placeholderClassName={cn( isComposerResting && diff --git a/apps/web/src/components/device/DeviceControlsRail.tsx b/apps/web/src/components/device/DeviceControlsRail.tsx new file mode 100644 index 000000000000..443f111f01cc --- /dev/null +++ b/apps/web/src/components/device/DeviceControlsRail.tsx @@ -0,0 +1,308 @@ +import type { DevicePlatform } from "@t3tools/contracts"; +import { + Camera, + ChevronLeft, + Home, + Keyboard, + Box, + Maximize, + Moon, + MoreHorizontal, + PictureInPicture2, + Power, + RotateCcw, + SlidersHorizontal, + Smartphone, + Square, + Sun, + Type, + X, +} from "lucide-react"; +import type { ReactNode } from "react"; +import { Button } from "~/components/ui/button"; +import { + Menu, + MenuItem, + MenuPopup, + MenuSeparator, + MenuTrigger, + MenuRadioGroup, + MenuRadioItem, + MenuRadioItemIndicator, +} from "~/components/ui/menu"; +import { Tooltip, TooltipPopup, TooltipTrigger } from "~/components/ui/tooltip"; +import type { DeviceStreamHandle, DeviceViewControls } from "./DeviceStreamView"; +import type { DeviceControls } from "./useDeviceControls"; + +/** Stable floating controls for both presentations, regardless of panel width. */ +export function DeviceControlsRail(props: { + platform: DevicePlatform; + handle: DeviceStreamHandle | null; + view: DeviceViewControls; + controls: DeviceControls; + screenshotPending: boolean; + onScreenshot: () => void; + toolsOpen: boolean; + onTools: () => void; + onFloat: () => void; + onClose: () => void; + onPowerOff: () => void; +}) { + const { view, handle, controls } = props; + const popupSide = "left"; + const settings = controls.detail?.settings; + const inputDisabled = !handle?.inputConnected; + const nextAppearance = settings?.appearance === "dark" ? "light" : "dark"; + return ( + + ); +} + +function RailDivider() { + return
; +} + +function RailButton(props: { + tooltipSide: "left" | "bottom"; + label: string; + description?: string | undefined; + disabled?: boolean; + pressed?: boolean; + onClick: () => void; + children: ReactNode; +}) { + const button = ( + + ); + return ( + + {props.disabled && props.description ? ( + + } + > + {button} + + ) : ( + + )} + {props.description ?? props.label} + + ); +} diff --git a/apps/web/src/components/device/DevicePanel.tsx b/apps/web/src/components/device/DevicePanel.tsx index f46a9fbf8ed7..c6fdaad6f78d 100644 --- a/apps/web/src/components/device/DevicePanel.tsx +++ b/apps/web/src/components/device/DevicePanel.tsx @@ -5,17 +5,7 @@ import type { DeviceSummary, ScopedThreadRef, } from "@t3tools/contracts"; -import { - ChevronLeft, - Home, - PictureInPicture2, - Power, - RotateCcw, - SlidersHorizontal, - Smartphone, - Square, - X, -} from "lucide-react"; +import { Smartphone, X } from "lucide-react"; import { useEffect, useMemo, useState } from "react"; import { usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore"; @@ -25,16 +15,13 @@ import { DiscoveryList, DiscoveryListRow } from "~/components/ui/discovery-list" import { Dialog } from "~/components/ui/dialog"; import { WizardPopup } from "~/components/ui/wizard"; import { Spinner } from "~/components/ui/spinner"; -import { Toggle } from "~/components/ui/toggle"; -import { Tooltip, TooltipPopup, TooltipTrigger } from "~/components/ui/tooltip"; import { cn } from "~/lib/utils"; -import { deviceEnvironment, useDeviceHubAccess, useDeviceState } from "~/state/device"; +import { deviceEnvironment, useDeviceState } from "~/state/device"; import { formatEnvironmentQueryError } from "~/state/query"; import { useAtomCommand } from "~/state/use-atom-command"; -import { DeviceStreamView, type DeviceStreamHandle } from "./DeviceStreamView"; import { DeviceLoadingView } from "./DeviceLoadingView"; import { DeviceSetup } from "./DeviceSetup"; -import { DeviceToolsPanel } from "./DeviceToolsPanel"; +import { DeviceWorkspace } from "./DeviceWorkspace"; import { PreviewPanelShell, type PreviewPanelMode } from "../preview/PreviewPanelShell"; const platformLabel = (platform: DevicePlatform) => @@ -59,10 +46,6 @@ export function DevicePanel(props: { const [operationError, setOperationError] = useState(null); const [pendingDevice, setPendingDevice] = useState(null); const pendingDeviceKey = pendingDevice ? deviceKey(pendingDevice) : null; - const [handle, setHandle] = useState(null); - const [toolsOpen, setToolsOpen] = useState(false); - const [axOverlay, setAxOverlay] = useState(false); - const access = useDeviceHubAccess(environmentId); const hostDisabled = state.hostStatus === "disabled"; @@ -184,69 +167,7 @@ export function DevicePanel(props: { return ( -
- - {props.surface.target - ? `${state.hosts.find((host) => host.id === props.surface.target?.hostId)?.label ?? "Device host"} ยท ${activeDevice?.version ?? props.surface.target.platform}` - : (pendingDevice?.name ?? "Choose a device")} - - {activeDevice ? ( - <> - handle?.pressButton("home")} - disabled={!handle?.inputConnected} - > - - - {activeDevice.platform === "android" ? ( - <> - handle?.pressButton("back")} - disabled={!handle?.inputConnected} - > - - - handle?.pressButton("recents")} - disabled={!handle?.inputConnected} - > - - - - ) : ( - handle?.rotate()} - disabled={!handle?.inputConnected} - > - - - )} - setToolsOpen(Boolean(pressed))} - > - - - - - - closeActive(true)}> - - - closeActive(false)}> - - - - ) : null} -
- {hostReady && state.hostStatusDetail ? ( + {hostReady && !activeDevice && state.hostStatusDetail ? (
{activeDevice && activeSession ? ( - <> -
- host.id === activeDevice.hostId)?.label ?? "Device host"} ยท ${activeDevice.version}`} - deviceId={activeDevice.id} - hostId={activeDevice.hostId} - visible={props.visible} - axOverlay={axOverlay} - onHandle={setHandle} - /> -
- {toolsOpen ? ( - setToolsOpen(false)} - className="absolute inset-y-0 right-0 z-10 w-full max-w-72 border-l shadow-lg @[560px]:static @[560px]:w-72 @[560px]:shrink-0 @[560px]:shadow-none" - /> - ) : null} - + host.id === activeDevice.hostId)?.label ?? "Device host" + } + hostDiagnostics={state.hostStatusDetail} + visible={props.visible} + onFloat={floatActive} + onClose={() => closeActive(false)} + onPowerOff={() => closeActive(true)} + /> ) : pendingDevice || hostBusy || !loaded ? ( void; - readonly disabled?: boolean; - readonly children: React.ReactNode; -}) { - return ( - - - } - > - {props.children} - - {props.label} - - ); -} - function groupDevices(state: DeviceServiceState) { const groups: Array<{ platform: DevicePlatform; devices: DeviceSummary[] }> = []; for (const platform of ["ios", "android"] as const) { diff --git a/apps/web/src/components/device/DevicePhoneViewport.tsx b/apps/web/src/components/device/DevicePhoneViewport.tsx new file mode 100644 index 000000000000..66398c02b52b --- /dev/null +++ b/apps/web/src/components/device/DevicePhoneViewport.tsx @@ -0,0 +1,151 @@ +import { useEffect, useRef, type RefObject } from "react"; +import type { PhoneViewer } from "@t3tools/client-runtime/device/phone-viewer"; +import type { + DeviceAccessorySource, + DeviceModelSource, +} from "@t3tools/client-runtime/device/model"; +import type { DeviceShapeProfile } from "@t3tools/client-runtime/device/shape-profile"; +import { createPhoneInteraction } from "@t3tools/client-runtime/device/phone-interaction"; +import type { DeviceScreenSize, DeviceStreamClient } from "@t3tools/client-runtime/device/stream"; +import { bindPhoneTrackpad } from "./phoneTrackpad"; + +const loadPhoneViewer = () => import("@t3tools/client-runtime/device/phone-viewer"); + +/** Web shell for the framework-independent viewer. The decoded screen and input connection remain owned by DeviceStreamView. */ +export function DevicePhoneViewport(props: { + readonly profile: DeviceShapeProfile; + readonly model: DeviceModelSource | null; + readonly accessory: DeviceAccessorySource | null; + readonly source: RefObject; + readonly onFrameListener: (listener: (() => void) | null) => void; + readonly onResetReady: (reset: (() => void) | null) => void; + readonly onInputCancel: (cancel: (() => void) | null) => void; + readonly client: RefObject; + readonly screen: DeviceScreenSize | null; + readonly onUnavailable: () => void; +}) { + const hostRef = useRef(null); + const canvasRef = useRef(null); + const viewerRef = useRef(null); + const interactionRef = useRef | null>(null); + const screenRef = useRef(props.screen); + const profileRef = useRef(props.profile); + const modelRef = useRef(props.model); + const accessoryRef = useRef(props.accessory); + const { source, onFrameListener, client, onInputCancel, onResetReady, onUnavailable } = props; + + useEffect(() => { + screenRef.current = props.screen; + profileRef.current = props.profile; + modelRef.current = props.model; + accessoryRef.current = props.accessory; + interactionRef.current?.end(); + viewerRef.current?.setModel(props.model); + viewerRef.current?.setAccessory(props.accessory); + viewerRef.current?.setScreen(props.screen, props.profile); + }, [props.screen, props.profile, props.model, props.accessory]); + + useEffect(() => { + const host = hostRef.current; + const canvas = canvasRef.current; + const decoded = source.current; + if (!host || !canvas || !decoded) return; + let disposed = false; + let trackpad: ReturnType | null = null; + let stopTrackpadEnd: (() => void) | undefined; + const resize = () => { + const { width, height } = host.getBoundingClientRect(); + viewerRef.current?.resize(width, height, window.devicePixelRatio); + }; + const observer = new ResizeObserver(resize); + observer.observe(host); + const blur = () => { + interactionRef.current?.end(); + trackpad?.cancel(); + }; + onInputCancel(blur); + window.addEventListener("blur", blur); + void loadPhoneViewer() + .then(({ createPhoneViewer }) => { + if (disposed) return; + const viewer = createPhoneViewer({ + canvas, + source: decoded, + onUnavailable, + onModelError: (cause) => console.warn("Device 3D asset could not load", cause), + profile: profileRef.current, + model: modelRef.current, + accessory: accessoryRef.current, + }); + viewerRef.current = viewer; + onResetReady(viewer.resetPose); + viewer.setScreen(screenRef.current); + onFrameListener(viewer.frameUpdated); + interactionRef.current = createPhoneInteraction({ + screenPoint: (point, captured) => viewer.screenPoint(point.x, point.y, captured), + touch: (phase, point) => client.current?.sendTouch(phase, point.x, point.y), + orbit: viewer.orbit, + zoomBy: () => {}, + onInteractionActive: viewer.setInteractionActive, + }); + trackpad = bindPhoneTrackpad(canvas, interactionRef.current); + stopTrackpadEnd = window.desktopBridge?.onTrackpadScrollEnd?.(() => trackpad?.endOrbit()); + resize(); + viewer.frameUpdated(); + }) + .catch(() => { + if (!disposed) onUnavailable(); + }); + return () => { + disposed = true; + stopTrackpadEnd?.(); + trackpad?.dispose(); + interactionRef.current?.end(); + interactionRef.current = null; + onInputCancel(null); + onResetReady(null); + onFrameListener(null); + observer.disconnect(); + window.removeEventListener("blur", blur); + viewerRef.current?.dispose(); + viewerRef.current = null; + }; + }, [onInputCancel, onResetReady, client, onFrameListener, onUnavailable, source]); + + const point = (event: React.PointerEvent) => { + const rect = event.currentTarget.getBoundingClientRect(); + return { + x: (event.clientX - rect.left) / rect.width, + y: (event.clientY - rect.top) / rect.height, + }; + }; + return ( +
+
+
+ { + if (event.button !== 0) return; + if (!interactionRef.current?.begin(event.pointerId, point(event), event.altKey)) return; + event.preventDefault(); + event.currentTarget.setPointerCapture(event.pointerId); + (event.currentTarget.closest('[role="application"]') as HTMLElement | null)?.focus(); + }} + onPointerMove={(event) => interactionRef.current?.move(event.pointerId, point(event))} + onPointerUp={(event) => { + interactionRef.current?.move(event.pointerId, point(event)); + interactionRef.current?.end(event.pointerId); + }} + onPointerCancel={(event) => interactionRef.current?.end(event.pointerId)} + onLostPointerCapture={(event) => interactionRef.current?.end(event.pointerId)} + /> +
+
+ ); +} diff --git a/apps/web/src/components/device/DeviceStreamView.tsx b/apps/web/src/components/device/DeviceStreamView.tsx index 4d4744a6268f..3e2cd54e3c26 100644 --- a/apps/web/src/components/device/DeviceStreamView.tsx +++ b/apps/web/src/components/device/DeviceStreamView.tsx @@ -1,9 +1,14 @@ import type { DevicePlatform, EnvironmentId } from "@t3tools/contracts"; -import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState, type ReactNode } from "react"; import { cn } from "~/lib/utils"; import { Button } from "~/components/ui/button"; import { refreshDeviceHubAccess, useDeviceHubAccess } from "~/state/device"; +import { createCanvasFrameSink } from "@t3tools/client-runtime/device/frame"; +import { resolveDeviceShape } from "@t3tools/client-runtime/device/shape-profile"; +import { deviceKeyboard, deviceModel } from "./deviceModels"; +import { fitDeviceFrame } from "./deviceFrameLayout"; +import { DevicePhoneViewport } from "./DevicePhoneViewport"; import { DeviceLoadingView } from "./DeviceLoadingView"; import { type DeviceAxElement, fetchDeviceAxTree } from "./deviceHubApi"; import { @@ -15,6 +20,17 @@ import { } from "@t3tools/client-runtime/device/stream"; const AX_POLL_INTERVAL_MS = 2_000; +const CONTROLS_RAIL_WIDTH = 56; + +export interface DeviceViewControls { + readonly phone: boolean; + readonly streaming: boolean; + readonly phoneUnavailableReason: string | null; + readonly showPhone: () => void; + readonly showFlat: () => void; + readonly resetView: () => void; + readonly keyboard: { readonly attached: boolean; readonly toggle: () => void } | null; +} export interface DeviceStreamHandle { readonly pressButton: (button: DeviceHardwareButton) => void; @@ -37,11 +53,31 @@ export function DeviceStreamView(props: { readonly deviceDescription?: string; readonly visible: boolean; readonly hostId: string; + /** The full panel opts into the phone spike; compact viewers retain their flat presentation. */ + readonly allowPhoneView?: boolean; + readonly renderControls?: (view: DeviceViewControls) => ReactNode; /** Draw accessibility element frames over the screen. */ readonly axOverlay?: boolean; readonly onHandle?: (handle: DeviceStreamHandle | null) => void; readonly onScreen?: (screen: DeviceScreenSize | null) => void; }) { + const [presentation, setPresentation] = useState<"phone" | "flat">("phone"); + const [keyboardAttached, setKeyboardAttached] = useState(false); + const [phoneUnavailable, setPhoneUnavailable] = useState(false); + const onPhoneUnavailable = useCallback(() => setPhoneUnavailable(true), []); + const cancelPhoneInputRef = useRef<(() => void) | null>(null); + const cancelPhoneInput = useCallback(() => cancelPhoneInputRef.current?.(), []); + const resetViewRef = useRef<(() => void) | null>(null); + const onResetReady = useCallback((reset: (() => void) | null) => { + resetViewRef.current = reset; + }, []); + const frameListenerRef = useRef<(() => void) | null>(null); + const onFrameListener = useCallback((listener: (() => void) | null) => { + frameListenerRef.current = listener; + }, []); + const onInputCancel = useCallback((cancel: (() => void) | null) => { + cancelPhoneInputRef.current = cancel; + }, []); const access = useDeviceHubAccess(props.environmentId, props.hostId); const canvasRef = useRef(null); const clientRef = useRef(null); @@ -67,7 +103,7 @@ export function DeviceStreamView(props: { } const client = createDeviceStreamClient( { platform: props.platform, deviceId: props.deviceId, access }, - canvas, + createCanvasFrameSink(canvas, () => frameListenerRef.current?.()), { onStatus: (next, nextDetail) => { setStatus(next); @@ -101,6 +137,7 @@ export function DeviceStreamView(props: { client.start(); onHandle?.({ pressButton: client.pressButton, rotate: client.rotate, inputConnected: false }); return () => { + cancelPhoneInput(); client.stop(); clientRef.current = null; onHandle?.(null); @@ -109,6 +146,7 @@ export function DeviceStreamView(props: { }; }, [ access, + cancelPhoneInput, onHandle, onScreen, props.deviceId, @@ -131,6 +169,16 @@ export function DeviceStreamView(props: { return w / h; }, [props.platform, screen]); + const showPhone = + props.allowPhoneView && + status === "streaming" && + props.visible && + presentation === "phone" && + !phoneUnavailable && + !mjpegUrl && + !props.axOverlay; + const controlsInset = props.renderControls && !showPhone ? CONTROLS_RAIL_WIDTH : 0; + // The frame is the largest box at `aspect` that fits the container, so a // narrow panel shows a shorter phone rather than a squeezed one. CSS // `aspect-ratio` alone cannot do this: with the height pinned to 100% the @@ -154,12 +202,8 @@ export function DeviceStreamView(props: { return () => observer.disconnect(); }, []); const frame = useMemo(() => { - if (host.width === 0 || host.height === 0) return { width: 0, height: 0 }; - const byHeight = { width: host.height * aspect, height: host.height }; - return byHeight.width <= host.width - ? byHeight - : { width: host.width, height: host.width / aspect }; - }, [aspect, host]); + return fitDeviceFrame(aspect, host.width, host.height, controlsInset); + }, [aspect, controlsInset, host]); // serve-sim streams the raw framebuffer; rotate the display for a device // that reports landscape while its frames stay portrait. @@ -231,123 +275,230 @@ export function DeviceStreamView(props: { return { x: Math.min(1, Math.max(0, x)), y: Math.min(1, Math.max(0, y)) }; }; + const phoneUnavailableReason = phoneUnavailable + ? "3D is unavailable on this browser" + : mjpegUrl + ? "3D requires the H.264 stream" + : props.axOverlay + ? "Turn off accessibility frames to use 3D" + : null; + + const keyboardSource = deviceKeyboard(props.platform, props.deviceName ?? ""); + const resetView = useCallback(() => { + const orientation = keyboardAttached ? "landscape_right" : "portrait"; + if (screen?.orientation !== orientation) clientRef.current?.setOrientation(orientation); + resetViewRef.current?.(); + }, [keyboardAttached, screen?.orientation]); + const profile = resolveDeviceShape({ + platform: props.platform, + name: props.deviceName ?? "", + portraitAspect: Math.min(aspect, 1 / aspect), + }); + return (
{ - if (event.target !== event.currentTarget) return; - if (event.metaKey && !["r", "R"].includes(event.key)) return; - event.preventDefault(); - clientRef.current?.sendKey(event.nativeEvent, "down"); - }} - onKeyUp={(event) => { - if (event.target !== event.currentTarget) return; - clientRef.current?.sendKey(event.nativeEvent, "up"); - }} + className={cn( + "relative flex size-full min-h-0 min-w-0", + props.allowPhoneView ? "bg-background" : "bg-black/90", + )} > + {props.renderControls ? ( + { + cancelPhoneInput(); + if (!keyboardAttached && screen?.orientation !== "landscape_right") + clientRef.current?.setOrientation("landscape_right"); + setKeyboardAttached(!keyboardAttached); + }, + } + : null, + showPhone: () => setPresentation("phone"), + showFlat: () => setPresentation("flat"), + }} + onResetView={resetView} + /> + ) : null}
{ - event.currentTarget.setPointerCapture(event.pointerId); - (event.currentTarget.parentElement as HTMLElement | null)?.focus(); - pointerActive.current = true; - const { x, y } = normalizedPoint(event); - clientRef.current?.sendTouch("begin", x, y); + ref={hostRef} + className={cn( + "relative flex min-h-0 min-w-0 flex-1 items-center justify-center overflow-hidden outline-none", + controlsInset && "pr-14", + )} + tabIndex={0} + role="application" + aria-label={`${props.platform === "ios" ? "iOS Simulator" : "Android Emulator"} screen`} + onKeyDown={(event) => { + if (event.target !== event.currentTarget) return; + if (event.metaKey && !["r", "R"].includes(event.key)) return; + event.preventDefault(); + clientRef.current?.sendKey(event.nativeEvent, "down"); }} - onPointerMove={(event) => { - if (!pointerActive.current) return; - const { x, y } = normalizedPoint(event); - clientRef.current?.sendTouch("move", x, y); - }} - onPointerUp={(event) => { - if (!pointerActive.current) return; - pointerActive.current = false; - const { x, y } = normalizedPoint(event); - clientRef.current?.sendTouch("end", x, y); - }} - onPointerCancel={(event) => { - if (!pointerActive.current) return; - pointerActive.current = false; - const { x, y } = normalizedPoint(event); - clientRef.current?.sendTouch("end", x, y); + onKeyUp={(event) => { + if (event.target !== event.currentTarget) return; + clientRef.current?.sendKey(event.nativeEvent, "up"); }} > - - {props.visible && access && mjpegUrl ? ( - { + event.currentTarget.setPointerCapture(event.pointerId); + (event.currentTarget.parentElement as HTMLElement | null)?.focus(); + pointerActive.current = true; + const { x, y } = normalizedPoint(event); + clientRef.current?.sendTouch("begin", x, y); + }} + onPointerMove={(event) => { + if (!pointerActive.current) return; + const { x, y } = normalizedPoint(event); + clientRef.current?.sendTouch("move", x, y); + }} + onPointerUp={(event) => { + if (!pointerActive.current) return; + pointerActive.current = false; + const { x, y } = normalizedPoint(event); + clientRef.current?.sendTouch("end", x, y); + }} + onPointerCancel={(event) => { + if (!pointerActive.current) return; + pointerActive.current = false; + const { x, y } = normalizedPoint(event); + clientRef.current?.sendTouch("end", x, y); + }} + > + + {props.visible && access && mjpegUrl ? ( + + ) : null} + {axElements.length > 0 ? ( +
+ {axElements.map((element) => ( +
+ {element.label ? ( + + {element.label} + + ) : null} +
+ ))} +
+ ) : null} +
+ {showPhone ? ( + + ) : null} + {props.allowPhoneView && !props.renderControls && status === "streaming" ? ( +
+ + +
) : null} - {axElements.length > 0 ? ( -
- {axElements.map((element) => ( -
- {element.label ? ( - - {element.label} - - ) : null} -
- ))} + {status === "streaming" && !inputState.connected ? ( +
+ + Input disconnected{inputState.detail ? ` (${inputState.detail})` : ""}, reconnectingโ€ฆ + +
+ ) : null} + {status !== "streaming" ? ( +
+ + {status === "error" ? ( + + ) : null} +
) : null}
- {status === "streaming" && !inputState.connected ? ( -
- - Input disconnected{inputState.detail ? ` (${inputState.detail})` : ""}, reconnectingโ€ฆ - -
- ) : null} - {status !== "streaming" ? ( -
- - {status === "error" ? ( - - ) : null} - -
- ) : null}
); } + +function DeviceControlsSlot(props: { + renderControls: (view: DeviceViewControls) => ReactNode; + view: Omit; + onResetView: () => void; +}) { + return props.renderControls({ ...props.view, resetView: props.onResetView }); +} diff --git a/apps/web/src/components/device/DeviceToolsPanel.tsx b/apps/web/src/components/device/DeviceToolsPanel.tsx index 39b8123cd03b..3d7bd2ae7f2c 100644 --- a/apps/web/src/components/device/DeviceToolsPanel.tsx +++ b/apps/web/src/components/device/DeviceToolsPanel.tsx @@ -1,14 +1,7 @@ import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess"; -import type { - DeviceActionInput, - DeviceDetail, - DevicePermission, - DeviceSummary, - DeviceTextSize, - EnvironmentId, -} from "@t3tools/contracts"; +import type { DevicePermission, DeviceSummary, DeviceTextSize } from "@t3tools/contracts"; import { ChevronDown, X } from "lucide-react"; -import { useCallback, useEffect, useMemo, useState } from "react"; +import { useEffect, useState } from "react"; import { Button } from "~/components/ui/button"; import { Collapsible, CollapsiblePanel, CollapsibleTrigger } from "~/components/ui/collapsible"; @@ -24,21 +17,8 @@ import { Spinner } from "~/components/ui/spinner"; import { Switch } from "~/components/ui/switch"; import { Toggle, ToggleGroup } from "~/components/ui/toggle-group"; import { cn } from "~/lib/utils"; -import { deviceEnvironment } from "~/state/device"; -import { formatEnvironmentQueryError } from "~/state/query"; -import { useAtomCommand } from "~/state/use-atom-command"; -import { - type DeviceEventLogEntry, - type DeviceForegroundInfo, - subscribeDeviceEventLog, - subscribeDeviceForeground, -} from "./deviceHubApi"; - -type ActionBody = DeviceActionInput extends infer A - ? A extends { readonly type: string } - ? Omit - : never - : never; +import type { DeviceControls } from "./useDeviceControls"; +import { type DeviceEventLogEntry, subscribeDeviceEventLog } from "./deviceHubApi"; const TEXT_SIZES: ReadonlyArray<{ value: DeviceTextSize; label: string }> = [ { value: "small", label: "Small" }, @@ -102,7 +82,8 @@ const LOCATION_PRESETS = [ * local state so the controls never show a value the device did not confirm. */ export function DeviceToolsPanel(props: { - readonly environmentId: EnvironmentId; + readonly controls: DeviceControls; + readonly hostDiagnostics: string | undefined; readonly device: DeviceSummary; readonly access: DeviceHubAccess | null; readonly axOverlay: boolean; @@ -110,62 +91,10 @@ export function DeviceToolsPanel(props: { readonly onClose: () => void; readonly className?: string; }) { - const { environmentId, device } = props; - const readDetail = useAtomCommand(deviceEnvironment.detail, { reportFailure: false }); - const runAction = useAtomCommand(deviceEnvironment.action, { reportFailure: false }); - const [detail, setDetail] = useState(null); - const [pending, setPending] = useState(false); - const [error, setError] = useState(null); - const [foreground, setForeground] = useState(undefined); - const isIos = device.platform === "ios"; - - const target = useMemo( - () => ({ hostId: device.hostId, deviceId: device.id }), - [device.hostId, device.id], - ); - - // The panel is keyed by device, so a mount is always a fresh device. - useEffect(() => { - let cancelled = false; - void readDetail({ environmentId, input: target }).then((result) => { - if (cancelled) return; - if (result._tag === "Success") setDetail(result.value); - else setError(formatEnvironmentQueryError(result.cause)); - }); - return () => { - cancelled = true; - }; - }, [environmentId, readDetail, target]); - - useEffect(() => { - if (!props.access) return; - return subscribeDeviceForeground( - { access: props.access, platform: device.platform, deviceId: device.id }, - setForeground, - ); - }, [device.id, device.platform, props.access]); - - const act = useCallback( - async (body: ActionBody) => { - setPending(true); - setError(null); - try { - const result = await runAction({ - environmentId, - input: { ...target, ...body } as DeviceActionInput, - }); - if (result._tag === "Success") setDetail(result.value); - else setError(formatEnvironmentQueryError(result.cause)); - } finally { - setPending(false); - } - }, - [environmentId, runAction, target], - ); - + const { device, controls } = props; + const { detail, pending, error, foregroundApp, disabled, act } = controls; const settings = detail?.settings; - const foregroundApp = foreground === undefined ? (detail?.foregroundApp ?? null) : foreground; - const disabled = pending || detail === null; + const isIos = device.platform === "ios"; return (
) : null} + {props.hostDiagnostics ? ( +
+

+ {props.hostDiagnostics} +

+
+ ) : null} +
{foregroundApp?.id ?? "โ€”"} @@ -398,9 +335,7 @@ export function DeviceToolsPanel(props: { function Section(props: { readonly title: string; readonly children: React.ReactNode }) { return (
-

- {props.title} -

+

{props.title}

{props.children}
); @@ -698,7 +633,7 @@ function EventLogSection(props: { return ( - + Event log void; + onClose: () => void; + onPowerOff: () => void; +}) { + const [handle, setHandle] = useState(null); + const [toolsOpen, setToolsOpen] = useState(false); + const [axOverlay, setAxOverlay] = useState(false); + const [screenshotPending, setScreenshotPending] = useState(false); + const [screenshotError, setScreenshotError] = useState(null); + const captureRef = useRef(null); + const downloadRef = useRef(null); + const access = useDeviceHubAccess(props.environmentId, props.device.hostId); + const controls = useDeviceControls({ ...props, access }); + useEffect(() => { + if (!access || !props.visible) captureRef.current?.abort(); + return () => { + captureRef.current?.abort(); + if (downloadRef.current) URL.revokeObjectURL(downloadRef.current); + downloadRef.current = null; + }; + }, [access, props.visible]); + const saveScreenshot = () => { + if (!access || captureRef.current || !props.visible) return; + const controller = new AbortController(); + captureRef.current = controller; + setScreenshotPending(true); + setScreenshotError(null); + return captureDeviceScreenshot( + { access, platform: props.device.platform, deviceId: props.device.id }, + controller.signal, + ) + .then((image) => { + if (controller.signal.aborted) return; + if (downloadRef.current) URL.revokeObjectURL(downloadRef.current); + const url = URL.createObjectURL(image); + downloadRef.current = url; + const link = document.createElement("a"); + link.href = url; + link.download = `${props.device.name.replace(/[^a-z0-9-]/gi, "-")}-${Date.now()}.png`; + document.body.append(link); + link.click(); + link.remove(); + }) + .catch((cause: unknown) => { + if (!controller.signal.aborted) { + if (cause instanceof DeviceScreenshotError && cause.status === 401) + refreshDeviceHubAccess(props.environmentId); + setScreenshotError( + cause instanceof Error ? cause.message : "Screenshot capture failed. Try again.", + ); + } + }) + .finally(() => { + if (captureRef.current === controller) { + captureRef.current = null; + setScreenshotPending(false); + } + }); + }; + return ( + <> +
+ {screenshotError || (controls.error && !toolsOpen) ? ( +

+ {screenshotError ?? controls.error} +

+ ) : null} +
+ ( + void saveScreenshot()} + toolsOpen={toolsOpen} + onTools={() => setToolsOpen(!toolsOpen)} + onFloat={props.onFloat} + onClose={props.onClose} + onPowerOff={props.onPowerOff} + /> + )} + /> +
+
+ {toolsOpen ? ( + setToolsOpen(false)} + className="absolute inset-y-0 right-0 z-10 w-full max-w-72 border-l shadow-lg @[700px]:static @[700px]:w-72 @[700px]:shrink-0 @[700px]:shadow-none" + /> + ) : null} + + ); +} diff --git a/apps/web/src/components/device/deviceFrameLayout.test.ts b/apps/web/src/components/device/deviceFrameLayout.test.ts new file mode 100644 index 000000000000..c36c8141f89e --- /dev/null +++ b/apps/web/src/components/device/deviceFrameLayout.test.ts @@ -0,0 +1,9 @@ +import { expect, it } from "vite-plus/test"; +import { fitDeviceFrame } from "./deviceFrameLayout"; + +it("fits a flat device beside the fixed controls rail", () => { + const frame = fitDeviceFrame(9 / 19.5, 480, 1_000, 56); + expect(frame.width + 56).toBeLessThanOrEqual(480); + expect(frame.height).toBeLessThanOrEqual(1_000); + expect(frame.width / frame.height).toBeCloseTo(9 / 19.5); +}); diff --git a/apps/web/src/components/device/deviceFrameLayout.ts b/apps/web/src/components/device/deviceFrameLayout.ts new file mode 100644 index 000000000000..1842c3b2784e --- /dev/null +++ b/apps/web/src/components/device/deviceFrameLayout.ts @@ -0,0 +1,8 @@ +export function fitDeviceFrame(aspect: number, width: number, height: number, rightInset = 0) { + const availableWidth = Math.max(0, width - rightInset); + if (availableWidth === 0 || height === 0) return { width: 0, height: 0 }; + const byHeight = { width: height * aspect, height }; + return byHeight.width <= availableWidth + ? byHeight + : { width: availableWidth, height: availableWidth / aspect }; +} diff --git a/apps/web/src/components/device/deviceModels.ts b/apps/web/src/components/device/deviceModels.ts new file mode 100644 index 000000000000..682e0e6de4d5 --- /dev/null +++ b/apps/web/src/components/device/deviceModels.ts @@ -0,0 +1,35 @@ +import { + resolveDeviceModelId, + type DeviceAccessorySource, + type DeviceModelSource, +} from "@t3tools/client-runtime/device/model"; +import type { DevicePlatform } from "@t3tools/contracts"; +import iphone18Pro from "./models/iphone-18-pro.glb?url"; +import iphone18ProMax from "./models/iphone-18-pro-max.glb?url"; +import magicKeyboard from "./models/ipad-pro-13-m5-magic-keyboard.glb?url"; +import ipadPro13M5 from "./models/ipad-pro-13-m5.glb?url"; + +// Bundled URLs follow the client origin in local, desktop, hosted and remote sessions. +const models: Record = { + "iphone-18-pro": { id: "iphone-18-pro", url: iphone18Pro }, + "iphone-18-pro-max": { id: "iphone-18-pro-max", url: iphone18ProMax }, + "ipad-pro-13-m5": { id: "ipad-pro-13-m5", url: ipadPro13M5 }, +}; + +export function deviceModel(platform: DevicePlatform, name: string): DeviceModelSource | null { + const id = resolveDeviceModelId(platform, name); + return id ? models[id] : null; +} + +const keyboard: DeviceAccessorySource = { + id: "ipad-pro-13-m5-magic-keyboard", + modelId: "ipad-pro-13-m5", + url: magicKeyboard, +}; + +export function deviceKeyboard( + platform: DevicePlatform, + name: string, +): DeviceAccessorySource | null { + return resolveDeviceModelId(platform, name) === keyboard.modelId ? keyboard : null; +} diff --git a/apps/web/src/components/device/models/ipad-pro-13-m5-magic-keyboard.glb b/apps/web/src/components/device/models/ipad-pro-13-m5-magic-keyboard.glb new file mode 100644 index 000000000000..fd46333b4270 Binary files /dev/null and b/apps/web/src/components/device/models/ipad-pro-13-m5-magic-keyboard.glb differ diff --git a/apps/web/src/components/device/models/ipad-pro-13-m5.glb b/apps/web/src/components/device/models/ipad-pro-13-m5.glb new file mode 100644 index 000000000000..0611ac749607 Binary files /dev/null and b/apps/web/src/components/device/models/ipad-pro-13-m5.glb differ diff --git a/apps/web/src/components/device/models/iphone-18-pro-max.glb b/apps/web/src/components/device/models/iphone-18-pro-max.glb new file mode 100644 index 000000000000..52438e10f8d6 Binary files /dev/null and b/apps/web/src/components/device/models/iphone-18-pro-max.glb differ diff --git a/apps/web/src/components/device/models/iphone-18-pro.glb b/apps/web/src/components/device/models/iphone-18-pro.glb new file mode 100644 index 000000000000..2dae3e9beaa6 Binary files /dev/null and b/apps/web/src/components/device/models/iphone-18-pro.glb differ diff --git a/apps/web/src/components/device/models/sources.json b/apps/web/src/components/device/models/sources.json new file mode 100644 index 000000000000..513a56fc291d --- /dev/null +++ b/apps/web/src/components/device/models/sources.json @@ -0,0 +1,49 @@ +{ + "author": "Apple Inc.", + "assetLicense": "No open-source or redistribution license has been established.", + "conversion": { + "blender": "4.5.9", + "gltfTransform": "4.2.1", + "changes": [ + "Extracted device bodies and a separate Magic Keyboard accessory; excluded unrelated parts and presentation duplicates.", + "Normalized portrait axes, centered the screen and uniformly scaled its height to 2.2 scene units.", + "Replaced the baked marketing screen with a named placeholder for the live framebuffer.", + "Resized textures to at most 1024 pixels and encoded them as WebP; retained body geometry." + ] + }, + "models": [ + { + "id": "iphone-18-pro", + "file": "iphone-18-pro.glb", + "sourceUrl": "https://www.apple.com/105/media/us/iphone-18-pro/2026/591df885-5ee2-4173-86e6-401021249f7c/ar/iphone-18-pro-e-sim.usdz", + "sourceSha256": "0f2dac354fbc6505e7917983de460f2cc68e253f96aabf52e9f0468a4803f317", + "bodyNode": "UBGArkKGrAMRRnj", + "screenNode": "EHjrGDHNHzgNjtE.001" + }, + { + "id": "iphone-18-pro-max", + "file": "iphone-18-pro-max.glb", + "sourceUrl": "https://www.apple.com/105/media/us/iphone-18-pro/2026/591df885-5ee2-4173-86e6-401021249f7c/ar/iphone-18-pro-e-sim.usdz", + "sourceSha256": "0f2dac354fbc6505e7917983de460f2cc68e253f96aabf52e9f0468a4803f317", + "bodyNode": "wUOhcMgiBmCgGaw", + "screenNode": "EHjrGDHNHzgNjtE" + }, + { + "id": "ipad-pro-13-m5", + "file": "ipad-pro-13-m5.glb", + "sourceUrl": "https://www.apple.com/105/media/us/ipad-pro/2025/adee90db-c01e-430d-b726-fe64c0063f08/ar/ipad-pro-space-black.usdz", + "sourceSha256": "48098daf7c49117c81a4e4e1808899b93fad081faeba6c8b65124946b83b5ac6", + "bodyNode": "zRrSLDpdYmKeRJQ", + "screenNode": "lsDiIbtoSGSmWWZ" + }, + { + "id": "ipad-pro-13-m5-magic-keyboard", + "file": "ipad-pro-13-m5-magic-keyboard.glb", + "sourceUrl": "https://www.apple.com/105/media/us/ipad-pro/2025/adee90db-c01e-430d-b726-fe64c0063f08/ar/ipad-pro-space-black.usdz", + "sourceSha256": "48098daf7c49117c81a4e4e1808899b93fad081faeba6c8b65124946b83b5ac6", + "bodyNode": "zRrSLDpdYmKeRJQ", + "screenNode": "lsDiIbtoSGSmWWZ", + "accessoryNode": "PoBqSMmyhhcJsBX" + } + ] +} diff --git a/apps/web/src/components/device/phoneTrackpad.test.ts b/apps/web/src/components/device/phoneTrackpad.test.ts new file mode 100644 index 000000000000..b8fb3f334188 --- /dev/null +++ b/apps/web/src/components/device/phoneTrackpad.test.ts @@ -0,0 +1,105 @@ +import { afterEach, expect, it, vi } from "vite-plus/test"; +import { bindPhoneTrackpad } from "./phoneTrackpad"; + +afterEach(() => vi.useRealTimers()); + +class Canvas extends EventTarget { + getBoundingClientRect() { + return { width: 400, height: 800 } as DOMRect; + } +} +const wheel = (ctrlKey = false) => + Object.assign(new Event("wheel", { cancelable: true }), { + deltaX: 16, + deltaY: -20, + deltaMode: 0, + ctrlKey, + }); +const gesture = (type: string, scale: number) => + Object.assign(new Event(type, { cancelable: true }), { scale }); + +it("consumes scrolling and page zoom only on the bound canvas and releases listeners on detach", () => { + const canvas = new Canvas(); + const navigate = vi.fn(() => true); + const listeners = vi.spyOn(canvas, "addEventListener"); + const binding = bindPhoneTrackpad(canvas, { navigate, endWheel: vi.fn() }); + const swipe = wheel(); + canvas.dispatchEvent(swipe); + const pinch = wheel(true); + canvas.dispatchEvent(pinch); + expect(swipe.defaultPrevented).toBe(true); + expect(pinch.defaultPrevented).toBe(true); + expect(navigate.mock.calls).toEqual([[{ type: "orbit", x: -0.04, y: 0.025 }]]); + expect( + listeners.mock.calls.every( + ([, , options]) => typeof options === "object" && options.passive === false, + ), + ).toBe(true); + binding.dispose(); + const detached = wheel(true); + canvas.dispatchEvent(detached); + expect(detached.defaultPrevented).toBe(false); + expect(navigate).toHaveBeenCalledOnce(); +}); + +it("consumes Safari pinch without zooming the model and resumes orbit after cancellation", () => { + const canvas = new Canvas(); + const navigate = vi.fn(() => true); + const binding = bindPhoneTrackpad(canvas, { navigate, endWheel: vi.fn() }); + canvas.dispatchEvent(gesture("gesturestart", 1)); + canvas.dispatchEvent(gesture("gesturechange", 1.2)); + canvas.dispatchEvent(wheel(true)); + expect(navigate).not.toHaveBeenCalled(); + binding.cancel(); + canvas.dispatchEvent(wheel()); + expect(navigate).toHaveBeenCalledOnce(); + binding.dispose(); +}); + +it("normalizes Chrome and Safari pinch, expires wheel sequences and cancels on detach", () => { + vi.useFakeTimers(); + const canvas = new Canvas(); + const pinch = { begin: vi.fn(() => true), move: vi.fn(), end: vi.fn() }; + const navigate = vi.fn(() => true); + const binding = bindPhoneTrackpad(canvas, { navigate, endWheel: vi.fn() }, pinch); + canvas.dispatchEvent(wheel(true)); + canvas.dispatchEvent(wheel(true)); + expect(pinch.begin).toHaveBeenCalledOnce(); + expect(pinch.move.mock.calls).toEqual([[0.2], [0.2]]); + vi.advanceTimersByTime(180); + expect(pinch.end).toHaveBeenCalledOnce(); + canvas.dispatchEvent(gesture("gesturestart", 1)); + canvas.dispatchEvent(gesture("gesturechange", 1.2)); + expect(pinch.move).toHaveBeenLastCalledWith(Math.log(1.2)); + canvas.dispatchEvent(wheel(true)); + expect(pinch.move).toHaveBeenCalledTimes(3); + binding.dispose(); + const count = pinch.move.mock.calls.length; + vi.advanceTimersByTime(500); + canvas.dispatchEvent(wheel(true)); + expect(pinch.move).toHaveBeenCalledTimes(count); + expect(navigate).not.toHaveBeenCalled(); +}); + +it("keeps a paused wheel orbit active until native release or browser fallback", () => { + vi.useFakeTimers(); + const canvas = new Canvas(); + const navigate = vi.fn(() => true); + const endWheel = vi.fn(); + const binding = bindPhoneTrackpad(canvas, { navigate, endWheel }); + canvas.dispatchEvent(wheel()); + vi.advanceTimersByTime(500); + expect(endWheel).not.toHaveBeenCalled(); + binding.endOrbit(); + expect(endWheel).toHaveBeenCalledOnce(); + canvas.dispatchEvent(wheel()); + vi.advanceTimersByTime(1199); + expect(endWheel).toHaveBeenCalledOnce(); + vi.advanceTimersByTime(1); + expect(endWheel).toHaveBeenCalledTimes(2); + canvas.dispatchEvent(wheel()); + canvas.dispatchEvent(Object.assign(wheel(), { momentum: true })); + expect(endWheel).toHaveBeenCalledTimes(3); + expect(navigate).toHaveBeenCalledTimes(3); + binding.dispose(); +}); diff --git a/apps/web/src/components/device/phoneTrackpad.ts b/apps/web/src/components/device/phoneTrackpad.ts new file mode 100644 index 000000000000..2851013b61df --- /dev/null +++ b/apps/web/src/components/device/phoneTrackpad.ts @@ -0,0 +1,137 @@ +// @effect-diagnostics globalTimers:off - Canvas-local wheel gesture expiry. +import { + phoneWheelNavigation, + type createPhoneInteraction, +} from "@t3tools/client-runtime/device/phone-interaction"; + +/** Canvas-local, non-passive listeners consume browser zoom. Safari reports cumulative pinch scale instead of Ctrl-wheel. */ +export function bindPhoneTrackpad( + canvas: Pick< + HTMLCanvasElement, + "addEventListener" | "removeEventListener" | "getBoundingClientRect" + >, + interaction: Pick, "navigate" | "endWheel">, + pinch?: { + begin: (x: number, y: number) => boolean; + move: (logScale: number) => void; + end: () => void; + }, +) { + let scale: number | null = null; + let wheelActive = false; + let timer: ReturnType | null = null; + let orbitActive = false; + let orbitTimer: ReturnType | null = null; + const endOrbit = () => { + if (orbitTimer) clearTimeout(orbitTimer); + orbitTimer = null; + if (!orbitActive) return; + orbitActive = false; + interaction.endWheel(); + }; + const finish = () => { + if (timer) clearTimeout(timer); + timer = null; + wheelActive = false; + scale = null; + pinch?.end(); + }; + const begin = (event: Event) => { + const rect = canvas.getBoundingClientRect(); + const x = "clientX" in event && typeof event.clientX === "number" ? event.clientX : NaN; + const y = "clientY" in event && typeof event.clientY === "number" ? event.clientY : NaN; + pinch?.begin((x - rect.left) / rect.width, (y - rect.top) / rect.height); + }; + const consume = (event: Event) => { + event.preventDefault(); + event.stopPropagation(); + }; + const wheel = (event: WheelEvent) => { + consume(event); + if (scale !== null) return; + if (event.ctrlKey) { + endOrbit(); + if (!wheelActive) { + begin(event); + wheelActive = true; + } + const unit = + event.deltaMode === 1 + ? 16 + : event.deltaMode === 2 + ? canvas.getBoundingClientRect().height + : 1; + pinch?.move((-event.deltaY * unit) / 100); + if (timer) clearTimeout(timer); + timer = setTimeout(finish, 180); + return; + } + if (wheelActive) finish(); + if ("momentum" in event && event.momentum === true) { + endOrbit(); + return; + } + const rect = canvas.getBoundingClientRect(); + const navigation = phoneWheelNavigation({ + width: rect.width, + height: rect.height, + deltaX: event.deltaX, + deltaY: event.deltaY, + deltaMode: event.deltaMode, + ctrlKey: event.ctrlKey, + }); + if (navigation && interaction.navigate(navigation)) { + orbitActive = true; + if (orbitTimer) clearTimeout(orbitTimer); + // Browsers without a release signal still return to a useful view. + orbitTimer = setTimeout(endOrbit, 1200); + } + }; + const gestureScale = (event: Event) => { + if ( + !("scale" in event) || + typeof event.scale !== "number" || + !Number.isFinite(event.scale) || + event.scale <= 0 + ) + return null; + return event.scale; + }; + const start = (event: Event) => { + consume(event); + endOrbit(); + finish(); + begin(event); + scale = gestureScale(event) ?? 1; + }; + const change = (event: Event) => { + consume(event); + const next = gestureScale(event); + if (scale === null || next === null) return; + pinch?.move(Math.log(next / scale)); + scale = next; + }; + const end = (event: Event) => { + consume(event); + finish(); + }; + canvas.addEventListener("wheel", wheel, { passive: false }); + canvas.addEventListener("gesturestart", start, { passive: false }); + canvas.addEventListener("gesturechange", change, { passive: false }); + canvas.addEventListener("gestureend", end, { passive: false }); + return { + endOrbit, + cancel() { + endOrbit(); + finish(); + }, + dispose() { + canvas.removeEventListener("wheel", wheel); + canvas.removeEventListener("gesturestart", start); + canvas.removeEventListener("gesturechange", change); + canvas.removeEventListener("gestureend", end); + endOrbit(); + finish(); + }, + }; +} diff --git a/apps/web/src/components/device/useDeviceControls.test.tsx b/apps/web/src/components/device/useDeviceControls.test.tsx new file mode 100644 index 000000000000..cadc0c7852ee --- /dev/null +++ b/apps/web/src/components/device/useDeviceControls.test.tsx @@ -0,0 +1,232 @@ +import { EnvironmentId, type DeviceDetail, type DeviceSummary } from "@t3tools/contracts"; +import type { DeviceHubAccess } from "@t3tools/client-runtime/device/hub-access"; +import { act, useEffect } from "react"; +import { create, type ReactTestRenderer } from "react-test-renderer"; +import { afterEach, beforeEach, expect, it, vi } from "vite-plus/test"; +import { useDeviceControls, type DeviceControls } from "./useDeviceControls"; + +type Result = { _tag: "Success"; value: DeviceDetail } | { _tag: "Failure"; cause: unknown }; +const { read, action, subscribe } = vi.hoisted(() => ({ + read: vi.fn<() => Promise>(), + action: vi.fn<() => Promise>(), + subscribe: + vi.fn<(_target: unknown, onChange: (app: { id: string } | null) => void) => () => void>(), +})); +vi.mock("~/state/device", () => ({ deviceEnvironment: { detail: "detail", action: "action" } })); +vi.mock("~/state/use-atom-command", () => ({ + useAtomCommand: (command: string) => (command === "detail" ? read : action), +})); +vi.mock("~/state/query", () => ({ formatEnvironmentQueryError: () => "Device action failed" })); +vi.mock("./deviceHubApi", () => ({ subscribeDeviceForeground: subscribe })); + +const device: DeviceSummary = { + hostId: "remote", + id: "phone", + name: "Phone", + platform: "ios", + version: "iOS", + booted: true, + physical: false, +}; +const snapshot = (appearance: "light" | "dark"): Result => ({ + _tag: "Success", + value: { + hostId: "remote", + deviceId: "phone", + settings: { appearance }, + foregroundApp: null, + readAt: "2026-09-20T00:00:00Z", + }, +}); +function deferred() { + let resolve!: (value: Result) => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} +let renderer: ReactTestRenderer | undefined; +let controls: DeviceControls; +function Probe({ visible, access = null }: { visible: boolean; access?: DeviceHubAccess | null }) { + const next = useDeviceControls({ + environmentId: EnvironmentId.make("environment"), + device, + access, + visible, + }); + useEffect(() => { + controls = next; + }, [next]); + return null; +} +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + read.mockReset(); + action.mockReset(); + subscribe.mockReset(); + subscribe.mockReturnValue(() => {}); + read.mockResolvedValue(snapshot("light")); +}); +afterEach(async () => { + await act(async () => renderer?.unmount()); + vi.unstubAllGlobals(); +}); +async function mount() { + await act(async () => { + renderer = create(); + }); +} + +it("serializes rapid actions and retains confirmed settings while pending or after failure", async () => { + await mount(); + const response = deferred(); + action.mockReturnValueOnce(response.promise); + let pending!: Promise; + await act(async () => { + pending = controls.act({ type: "setAppearance", value: "dark" }); + await controls.act({ type: "setAppearance", value: "light" }); + }); + expect(action).toHaveBeenCalledOnce(); + expect(controls.disabled).toBe(true); + expect(controls.detail?.settings.appearance).toBe("light"); + await act(async () => { + response.resolve(snapshot("dark")); + await pending; + }); + expect(controls.detail?.settings.appearance).toBe("dark"); + action.mockResolvedValueOnce({ _tag: "Failure", cause: "offline" }); + await act(async () => controls.act({ type: "setAppearance", value: "light" })); + expect(controls.detail?.settings.appearance).toBe("dark"); + expect(controls.error).toBe("Device action failed"); + expect(controls.disabled).toBe(false); +}); + +it("serializes host actions across hide/reopen and discards detached results", async () => { + await mount(); + const response = deferred(); + action.mockReturnValueOnce(response.promise); + let pending!: Promise; + await act(async () => { + pending = controls.act({ type: "setAppearance", value: "dark" }); + }); + await act(async () => renderer!.update()); + expect(controls.disabled).toBe(true); + await act(async () => renderer!.update()); + await act(async () => controls.act({ type: "setAppearance", value: "light" })); + expect(action).toHaveBeenCalledOnce(); + expect(controls.disabled).toBe(true); + const confirmed = deferred(); + read.mockReturnValueOnce(confirmed.promise); + await act(async () => { + response.resolve(snapshot("dark")); + await response.promise; + }); + expect(controls.detail?.settings.appearance).toBe("light"); + expect(controls.disabled).toBe(true); + await act(async () => { + confirmed.resolve(snapshot("dark")); + await pending; + }); + expect(controls.detail?.settings.appearance).toBe("dark"); + expect(controls.disabled).toBe(false); + action.mockResolvedValueOnce(snapshot("dark")); + await act(async () => controls.act({ type: "setAppearance", value: "dark" })); + expect(action).toHaveBeenCalledTimes(2); + expect(controls.detail?.settings.appearance).toBe("dark"); +}); + +it("keeps actions disabled when settings cannot be confirmed after a detached command", async () => { + await mount(); + const response = deferred(); + action.mockReturnValueOnce(response.promise); + let pending!: Promise; + await act(async () => { + pending = controls.act({ type: "setAppearance", value: "dark" }); + }); + await act(async () => renderer!.update()); + await act(async () => renderer!.update()); + read.mockResolvedValueOnce({ _tag: "Failure", cause: "offline" }); + await act(async () => { + response.resolve(snapshot("dark")); + await pending; + }); + expect(controls.detail).toBeNull(); + expect(controls.disabled).toBe(true); + expect(controls.error).toBe("Device action failed"); + await act(async () => controls.act({ type: "setAppearance", value: "light" })); + expect(action).toHaveBeenCalledOnce(); + await act(async () => renderer!.update()); + await act(async () => renderer!.update()); + expect(controls.disabled).toBe(false); + expect(controls.error).toBeNull(); +}); + +it("disables stale settings after a failed reopen read and recovers on a successful read", async () => { + await mount(); + await act(async () => renderer!.update()); + read.mockResolvedValueOnce({ _tag: "Failure", cause: "offline" }); + await act(async () => renderer!.update()); + expect(controls.detail).toBeNull(); + expect(controls.disabled).toBe(true); + expect(controls.error).toBe("Device action failed"); + await act(async () => controls.act({ type: "setAppearance", value: "dark" })); + expect(action).not.toHaveBeenCalled(); + await act(async () => renderer!.update()); + await act(async () => renderer!.update()); + expect(controls.detail?.settings.appearance).toBe("light"); + expect(controls.disabled).toBe(false); + expect(controls.error).toBeNull(); +}); + +it("does not let an older refresh overwrite a newer confirmed action", async () => { + await mount(); + await act(async () => renderer!.update()); + const refresh = deferred(); + read.mockReturnValueOnce(refresh.promise); + await act(async () => renderer!.update()); + action.mockResolvedValueOnce(snapshot("dark")); + await act(async () => controls.act({ type: "setAppearance", value: "dark" })); + await act(async () => { + refresh.resolve(snapshot("light")); + await refresh.promise; + }); + expect(controls.detail?.settings.appearance).toBe("dark"); +}); + +it.each(["hidden", "access renewed"] as const)( + "uses the detail snapshot until a new foreground event after the subscription is %s", + async (restart) => { + const access: DeviceHubAccess = { + httpBase: "https://remote.test/api/device-hub", + wsBase: "wss://remote.test/api/device-hub", + query: {}, + credentials: false, + }; + const detail = snapshot("light"); + if (detail._tag !== "Success") throw new Error("Expected a successful snapshot"); + read.mockResolvedValue({ + ...detail, + value: { ...detail.value, foregroundApp: { id: "snapshot.app" } }, + }); + await act(async () => { + renderer = create(); + }); + const oldEvent = subscribe.mock.calls[0]![1]; + await act(async () => oldEvent({ id: "previous.live.app" })); + expect(controls.foregroundApp?.id).toBe("previous.live.app"); + if (restart === "hidden") { + await act(async () => renderer!.update()); + await act(async () => renderer!.update()); + } else { + await act(async () => + renderer!.update(), + ); + } + expect(controls.foregroundApp?.id).toBe("snapshot.app"); + await act(async () => oldEvent({ id: "late.previous.app" })); + expect(controls.foregroundApp?.id).toBe("snapshot.app"); + const newEvent = subscribe.mock.calls[1]![1]; + await act(async () => newEvent({ id: "current.live.app" })); + expect(controls.foregroundApp?.id).toBe("current.live.app"); + }, +); diff --git a/apps/web/src/components/device/useDeviceControls.ts b/apps/web/src/components/device/useDeviceControls.ts new file mode 100644 index 000000000000..1ac8d2862984 --- /dev/null +++ b/apps/web/src/components/device/useDeviceControls.ts @@ -0,0 +1,130 @@ +import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess"; +import type { + DeviceActionInput, + DeviceDetail, + DeviceSummary, + EnvironmentId, +} from "@t3tools/contracts"; +import { useEffect, useMemo, useRef, useState } from "react"; +import { deviceEnvironment } from "~/state/device"; +import { formatEnvironmentQueryError } from "~/state/query"; +import { useAtomCommand } from "~/state/use-atom-command"; +import { subscribeDeviceForeground, type DeviceForegroundInfo } from "./deviceHubApi"; + +type ActionBody = DeviceActionInput extends infer A + ? A extends { readonly type: string } + ? Omit + : never + : never; + +/** One confirmed settings snapshot and serialized actions for the rail and drawer. Mount keyed by device. */ +export function useDeviceControls(options: { + environmentId: EnvironmentId; + device: DeviceSummary; + access: DeviceHubAccess | null; + visible: boolean; +}) { + const { environmentId, device, access, visible } = options; + const readDetail = useAtomCommand(deviceEnvironment.detail, { reportFailure: false }); + const runAction = useAtomCommand(deviceEnvironment.action, { reportFailure: false }); + const [detail, setDetail] = useState(null); + const [pending, setPending] = useState(false); + const [error, setError] = useState(null); + const [foreground, setForeground] = useState(); + // Hiding invalidates UI results, but does not cancel host commands. Keep them serialized + // until settlement rather than allowing a reopened panel to race the previous command. + const busy = useRef(false); + const generation = useRef(0); + const visibleRead = useRef[0] | null>(null); + const target = useMemo( + () => ({ hostId: device.hostId, deviceId: device.id }), + [device.hostId, device.id], + ); + + useEffect(() => { + const revision = ++generation.current; + if (!visible) return; + const request = { environmentId, input: target }; + visibleRead.current = request; + void readDetail(request).then((result) => { + if (generation.current !== revision) return; + if (result._tag === "Success") { + setDetail(result.value); + setError(null); + } else { + setDetail(null); + setError(formatEnvironmentQueryError(result.cause)); + } + }); + return () => { + visibleRead.current = null; + generation.current++; + }; + }, [environmentId, readDetail, target, visible]); + + useEffect(() => { + if (!access || !visible) return; + let active = true; + const unsubscribe = subscribeDeviceForeground( + { access, platform: device.platform, deviceId: device.id }, + (app) => { + if (active) setForeground(app); + }, + ); + return () => { + active = false; + unsubscribe(); + setForeground(undefined); + }; + }, [access, device.id, device.platform, visible]); + + const available = detail !== null && visible; + const act = async (body: ActionBody) => { + if (busy.current || !available) return; + busy.current = true; + // A settings read started before this action must not overwrite its confirmed result. + const revision = ++generation.current; + setPending(true); + setError(null); + return runAction({ + environmentId, + input: { ...target, ...body } as DeviceActionInput, + }) + .then((result) => { + if (generation.current !== revision) { + const request = visibleRead.current; + if (!request) return; + // Reopening can read settings before the host command finishes. Confirm them again + // after it settles, keeping actions serialized through this refresh. + const refreshRevision = ++generation.current; + return readDetail(request).then((refreshed) => { + if (generation.current !== refreshRevision) return; + if (refreshed._tag === "Success") { + setDetail(refreshed.value); + setError(null); + } else { + setDetail(null); + setError(formatEnvironmentQueryError(refreshed.cause)); + } + }); + } + if (result._tag === "Success") setDetail(result.value); + else setError(formatEnvironmentQueryError(result.cause)); + }) + .finally(() => { + busy.current = false; + setPending(false); + }); + }; + + return { + detail, + pending, + error, + act, + disabled: pending || !detail || !visible, + foregroundApp: foreground === undefined ? (detail?.foregroundApp ?? null) : foreground, + }; +} + +export type DeviceControls = ReturnType; diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.test.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.test.tsx index 75ed20cc4fe7..542b84907b6e 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.test.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.test.tsx @@ -8,13 +8,22 @@ import { type PreviewOpenInput, type PreviewSessionSnapshot, } from "@t3tools/contracts"; +import type { AtomCommandResult } from "@t3tools/client-runtime/state/runtime"; +import * as Cause from "effect/Cause"; import { AsyncResult, Atom } from "effect/unstable/reactivity"; import { act } from "react"; import { create, type ReactTestRenderer } from "react-test-renderer"; import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; +import { useBrowserSurfaceStore } from "~/browser/browserSurfaceStore"; +import { previewRuntimeTabId } from "~/browser/previewRuntimeTabId"; import { __resetClientSettingsPersistenceForTests } from "~/hooks/useSettings"; -import { readThreadPreviewState, resetPreviewStateForTests } from "~/previewStateStore"; +import { + applyPreviewServerSnapshot, + readThreadPreviewState, + resetPreviewStateForTests, + applyPreviewDesktopState, +} from "~/previewStateStore"; import { appAtomRegistry, AppAtomRegistryProvider } from "~/rpc/atomRegistry"; import { PreviewAutomationHosts } from "./PreviewAutomationHosts"; @@ -31,7 +40,7 @@ const mocks = vi.hoisted(() => ({ vi.fn< (target: { environmentId: EnvironmentId; input: PreviewAutomationResponse }) => Promise >(), - focus: vi.fn(async () => undefined), + focus: vi.fn<() => Promise>>(), })); vi.mock("~/localApi", () => ({ @@ -110,12 +119,21 @@ beforeEach(async () => { vi.clearAllMocks(); mocks.getClientSettings.mockReset().mockResolvedValue(savedSettings); mocks.respond.mockReset(); + mocks.focus.mockReset().mockResolvedValue(AsyncResult.success(undefined)); __resetClientSettingsPersistenceForTests(); resetPreviewStateForTests(); + useBrowserSurfaceStore.setState({ byTabId: {} }); appAtomRegistry.set(requestsAtom, AsyncResult.initial(false)); vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); vi.stubGlobal("window", { addEventListener: vi.fn(), removeEventListener: vi.fn() }); - vi.stubGlobal("document", { hasFocus: () => false, querySelectorAll: () => [] }); + vi.stubGlobal( + "document", + Object.assign(new EventTarget(), { + hasFocus: () => false, + visibilityState: "visible", + querySelectorAll: () => [], + }), + ); await act(() => { renderer = create( @@ -161,7 +179,11 @@ describe("PreviewAutomationHosts open", () => { input: { threadId, viewport, profileId: "work" }, }); expect(mocks.getClientSettings).toHaveBeenCalledOnce(); - await expect(response.promise).resolves.toMatchObject({ requestId: "open-request", ok: true }); + await expect(response.promise).resolves.toMatchObject({ + requestId: "open-request", + ok: true, + result: { available: false, tabId: snapshot.tabId }, + }); expect(readThreadPreviewState(threadRef).snapshot).toEqual(snapshot); expect(mocks.setClientSettings).not.toHaveBeenCalled(); }); @@ -188,3 +210,154 @@ describe("PreviewAutomationHosts open", () => { expect(mocks.setClientSettings).not.toHaveBeenCalled(); }); }); + +describe("PreviewAutomationHosts ownership", () => { + it("reports only local live tabs and removes ownership when their web contents close", async () => { + await act(() => { + appAtomRegistry.set( + requestsAtom, + AsyncResult.success({ type: "connected", connectionId: "automation-connection" }), + ); + applyPreviewServerSnapshot(threadRef, snapshot); + }); + expect(mocks.focus).toHaveBeenLastCalledWith( + expect.objectContaining({ input: expect.objectContaining({ liveTabs: [] }) }), + ); + const overlay = { + hasWebContents: true, + canGoBack: false, + canGoForward: false, + loading: false, + zoomFactor: 1, + pictureInPicture: false, + colorScheme: "system" as const, + audioMuted: false, + audible: false, + controller: "none" as const, + favicon: null, + }; + await act(() => applyPreviewDesktopState(threadRef, snapshot.tabId, overlay)); + expect(mocks.focus).toHaveBeenLastCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ + liveTabs: [{ threadId, tabId: snapshot.tabId, visible: false }], + }), + }), + ); + const reportCount = mocks.focus.mock.calls.length; + await act(() => + applyPreviewDesktopState(threadRef, snapshot.tabId, { ...overlay, loading: true }), + ); + expect(mocks.focus).toHaveBeenCalledTimes(reportCount); + const runtimeTabId = previewRuntimeTabId(threadRef, null, snapshot.tabId); + const owner = Symbol(); + await act(() => { + useBrowserSurfaceStore.getState().claim(runtimeTabId, owner, false); + useBrowserSurfaceStore + .getState() + .present(runtimeTabId, owner, { x: 0, y: 0, width: 800, height: 600 }, true, 0, 1); + }); + expect(mocks.focus).toHaveBeenLastCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ + liveTabs: [{ threadId, tabId: snapshot.tabId, visible: true }], + }), + }), + ); + for (const visibilityState of ["hidden", "visible"]) { + await act(() => { + Object.assign(document, { visibilityState }); + document.dispatchEvent(new Event("visibilitychange")); + }); + expect(mocks.focus).toHaveBeenLastCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ + focused: false, + liveTabs: [{ threadId, tabId: snapshot.tabId, visible: visibilityState === "visible" }], + }), + }), + ); + } + await act(() => { + appAtomRegistry.set( + requestsAtom, + AsyncResult.success({ type: "connected", connectionId: "reconnected" }), + ); + }); + expect(mocks.focus).toHaveBeenLastCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ + connectionId: "reconnected", + liveTabs: [{ threadId, tabId: snapshot.tabId, visible: true }], + }), + }), + ); + await act(() => + applyPreviewDesktopState(threadRef, snapshot.tabId, { ...overlay, hasWebContents: false }), + ); + expect(mocks.focus).toHaveBeenLastCalledWith( + expect.objectContaining({ input: expect.objectContaining({ liveTabs: [] }) }), + ); + }); + + it.each([false, true])( + "retries failed reports without clearing newer connection reports (reconnect: %s)", + async (reconnect) => { + const report = deferred>>(); + mocks.focus.mockReturnValueOnce(report.promise); + await act(() => { + appAtomRegistry.set( + requestsAtom, + AsyncResult.success({ type: "connected", connectionId: "first" }), + ); + }); + if (reconnect) { + await act(() => { + appAtomRegistry.set( + requestsAtom, + AsyncResult.success({ type: "connected", connectionId: "second" }), + ); + }); + } + await act(async () => { + report.resolve(AsyncResult.failure(Cause.fail(new Error("Focus report failed")))); + await report.promise; + }); + expect(mocks.focus).toHaveBeenCalledTimes(reconnect ? 2 : 1); + await act(() => { + document.dispatchEvent(new Event("visibilitychange")); + }); + expect(mocks.focus).toHaveBeenCalledTimes(2); + expect(mocks.focus).toHaveBeenLastCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ connectionId: reconnect ? "second" : "first" }), + }), + ); + }, + ); + + it("does not claim an available runtime from a server snapshot alone", async () => { + const response = deferred(); + mocks.respond.mockImplementationOnce(async ({ input }) => response.resolve(input)); + await act(async () => { + applyPreviewServerSnapshot(threadRef, snapshot); + appAtomRegistry.set( + requestsAtom, + AsyncResult.success({ + ...requestEvent, + request: { + ...requestEvent.request, + operation: "status", + tabId: snapshot.tabId, + input: {}, + }, + }), + ); + await response.promise; + }); + await expect(response.promise).resolves.toMatchObject({ + ok: true, + result: { available: false, tabId: snapshot.tabId }, + }); + }); +}); diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index a793e8a2c8c0..173a027e9df2 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -1,6 +1,7 @@ "use client"; import { RegistryContext, useAtomSet, useAtomValue } from "@effect/atom-react"; +import { parseScopedThreadKey } from "@t3tools/client-runtime/environment"; import { squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; import { FILL_PREVIEW_VIEWPORT, @@ -22,12 +23,14 @@ import { import { resolvePreviewViewport } from "@t3tools/shared/previewViewport"; import { useCallback, useContext, useEffect, useMemo, useRef, useState } from "react"; import { Atom } from "effect/unstable/reactivity"; +import { useShallow } from "zustand/react/shallow"; import { applyPreviewServerSnapshot, readThreadPreviewState, reconcilePreviewServerSessions, updatePreviewServerSnapshot, + useActivePreviewSessions, } from "~/previewStateStore"; import { browserMiniPlayerSource, @@ -250,7 +253,7 @@ const currentStatus = async ( } const navStatus = snapshot?.navStatus; return { - available: Boolean(previewBridge?.automation), + available: false, visible, tabId, url: navStatus && navStatus._tag !== "Idle" ? navStatus.url : null, @@ -292,6 +295,30 @@ export function PreviewAutomationHosts() { function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) { const { environmentId } = props; + const previewSessions = useActivePreviewSessions(); + const visibleRuntimeTabIds = useBrowserSurfaceStore( + useShallow((state) => + Object.keys(state.byTabId).filter((tabId) => state.byTabId[tabId]?.visible), + ), + ); + const liveTabs = useMemo( + () => + Object.entries(previewSessions).flatMap(([key, state]) => { + const ref = parseScopedThreadKey(key); + if (ref?.environmentId !== environmentId) return []; + return Object.values(state.sessions) + .filter((tab) => state.desktopByTabId[tab.tabId]?.hasWebContents) + .map((tab) => ({ + threadId: ref.threadId, + tabId: tab.tabId, + visible: visibleRuntimeTabIds.includes( + previewRuntimeTabId(ref, state.serverEpoch, tab.tabId), + ), + })); + }), + [environmentId, previewSessions, visibleRuntimeTabIds], + ); + const lastFocusReportRef = useRef(null); const registry = useContext(RegistryContext); const [automationClientId] = useState(createPreviewAutomationClientId); const initialAutomationHost = useMemo( @@ -804,24 +831,35 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) useEffect(() => { const report = () => { if (!automationConnectionId) return; - void focusAutomationHost({ + const input = { + clientId: automationClientId, environmentId, - input: { - clientId: automationClientId, - environmentId, - connectionId: automationConnectionId, - focused: document.hasFocus(), - }, + connectionId: automationConnectionId, + focused: document.hasFocus() && document.visibilityState === "visible", + liveTabs: liveTabs.map((tab) => ({ + ...tab, + visible: tab.visible && document.visibilityState === "visible", + })), + }; + const reportKey = JSON.stringify(input); + if (lastFocusReportRef.current === reportKey) return; + lastFocusReportRef.current = reportKey; + void focusAutomationHost({ environmentId, input }).then((result) => { + if (result._tag === "Failure" && lastFocusReportRef.current === reportKey) { + lastFocusReportRef.current = null; + } }); }; report(); window.addEventListener("focus", report); window.addEventListener("blur", report); + document.addEventListener("visibilitychange", report); return () => { window.removeEventListener("focus", report); window.removeEventListener("blur", report); + document.removeEventListener("visibilitychange", report); }; - }, [automationClientId, automationConnectionId, environmentId, focusAutomationHost]); + }, [automationClientId, automationConnectionId, environmentId, focusAutomationHost, liveTabs]); return null; } diff --git a/apps/web/src/components/settings/SettingsFontPreviews.tsx b/apps/web/src/components/settings/SettingsFontPreviews.tsx index 2d64538d5c46..6f392598b9e7 100644 --- a/apps/web/src/components/settings/SettingsFontPreviews.tsx +++ b/apps/web/src/components/settings/SettingsFontPreviews.tsx @@ -46,7 +46,7 @@ export function PromptFontPreview() { skills={EMPTY_SKILLS} disabled={false} placeholder={DISCONNECTED_COMPOSER_PLACEHOLDER} - className="max-h-40 min-h-12" + className="max-h-42 min-h-14" onChange={onChange} onPaste={noop} /> diff --git a/apps/web/src/components/ui/switch.test.tsx b/apps/web/src/components/ui/switch.test.tsx new file mode 100644 index 000000000000..b63fa79e3849 --- /dev/null +++ b/apps/web/src/components/ui/switch.test.tsx @@ -0,0 +1,20 @@ +import { renderToStaticMarkup } from "react-dom/server"; +import { describe, expect, it } from "vite-plus/test"; + +import { Switch } from "./switch"; + +function renderSwitchRoot(props: Partial[0]>) { + const html = renderToStaticMarkup(); + return html.match(/]*role="switch"[^>]*>/)?.[0] ?? ""; +} + +describe("Switch accessibility", () => { + it("exposes the checked state to assistive tech", () => { + expect(renderSwitchRoot({ checked: true })).toContain('aria-checked="true"'); + expect(renderSwitchRoot({ checked: false })).toContain('aria-checked="false"'); + }); + + it("exposes the mixed state", () => { + expect(renderSwitchRoot({ checked: false, mixed: true })).toContain('aria-checked="mixed"'); + }); +}); diff --git a/apps/web/src/components/ui/switch.tsx b/apps/web/src/components/ui/switch.tsx index 33114c5512be..e6bf20aa64c7 100644 --- a/apps/web/src/components/ui/switch.tsx +++ b/apps/web/src/components/ui/switch.tsx @@ -27,7 +27,10 @@ function Switch({ data-size={size} data-slot="switch" data-mixed={mixed ? "" : undefined} - aria-checked={mixed ? "mixed" : undefined} + // Base UI copies every key we pass, even `undefined`, over its own + // aria-checked. Only pass the attribute when mixed so the real state + // survives for screen readers. + {...(mixed ? { "aria-checked": "mixed" as const } : {})} {...props} > + new Box3(new Vector3(x - width / 2, -1, -0.2), new Vector3(x + width / 2, 1, 0.2)); +it("fits both axes through a moving pivot, freezes for contact, and stops once framing settles", () => { + const framing = createDeviceFraming(); + framing.setBounds(box(0, 1), Math.PI / 8, 0.7, 0); + const initialDistance = framing.distance(); + const next = box(1.5, 3); + framing.setBounds(next, Math.PI / 8, 0.7, 10); + framing.advance(50); + expect(framing.center.x).toBeGreaterThan(0); + expect(framing.center.x).toBeLessThan(1.5); + for (const x of [next.min.x, next.max.x]) + expect( + Math.abs(x - framing.center.x) / + ((framing.distance() - next.max.z) * Math.tan(Math.PI / 8) * 0.7), + ).toBeLessThan(1); + framing.hold(true, 50); + const center = framing.center.clone(), + distance = framing.distance(); + framing.advance(10000); + expect(framing.center.equals(center)).toBe(true); + expect(framing.distance()).toBe(distance); + expect(framing.needsFrame()).toBe(false); + framing.hold(false, 10000); + framing.advance(10016); + expect(framing.center.x).toBeLessThan(1.5); + framing.advance(12000); + expect(framing.center.x).toBe(1.5); + expect(framing.distance()).toBeGreaterThan(initialDistance); + expect(framing.needsFrame()).toBe(false); +}); + +it("uses elapsed time independently of frame partitions and fits a changed viewport immediately", () => { + const run = (step: number) => { + const framing = createDeviceFraming(); + framing.setBounds(box(0, 1), Math.PI / 8, 1, 0); + framing.setBounds(box(-1, 2), Math.PI / 8, 1, 0); + for (let time = step; time < 400; time += step) framing.advance(time); + framing.advance(400); + return framing; + }; + const fast = run(8), + slow = run(33); + expect(fast.center.distanceTo(slow.center)).toBeLessThan(1e-8); + expect(fast.distance()).toBeCloseTo(slow.distance(), 8); + fast.setBounds(box(-1, 2), Math.PI / 8, 0.4, 400, true); + expect(fast.distance()).toBeGreaterThan(slow.distance()); + expect(fast.needsFrame()).toBe(false); +}); diff --git a/packages/client-runtime/src/device/deviceFraming.ts b/packages/client-runtime/src/device/deviceFraming.ts new file mode 100644 index 000000000000..1d8dddb10eb3 --- /dev/null +++ b/packages/client-runtime/src/device/deviceFraming.ts @@ -0,0 +1,82 @@ +import { Box3, Vector3 } from "three"; + +/** Fit the current assembly, with independent framing dynamics so a leaf pivot does not drag it off-screen. */ +export function createDeviceFraming() { + const center = new Vector3(); + const target = new Vector3(); + const velocity = new Vector3(); + const bounds = new Box3(); + let distance = 1; + let targetDistance = 1; + let distanceVelocity = 0; + let tanX = 1; + let tanY = 1; + let at = 0; + let initialized = false; + let moving = false; + let held = false; + const advance = (now: number, immediate = false) => { + if (held) return false; + const dt = Math.max(0, (now - at) / 1000); + at = now; + if (!moving) return false; + const omega = 14; + const decay = Math.exp(-omega * dt); + const error = center.clone().sub(target); + const coefficient = velocity.clone().addScaledVector(error, omega); + center.copy(error).addScaledVector(coefficient, dt).multiplyScalar(decay).add(target); + velocity.addScaledVector(coefficient, -omega * dt).multiplyScalar(decay); + const distanceError = distance - targetDistance; + const distanceCoefficient = distanceVelocity + omega * distanceError; + distance = targetDistance + (distanceError + distanceCoefficient * dt) * decay; + distanceVelocity = (distanceVelocity - omega * distanceCoefficient * dt) * decay; + moving = + center.distanceTo(target) > 0.0005 || + velocity.length() > 0.005 || + Math.abs(distance - targetDistance) > 0.0005 || + Math.abs(distanceVelocity) > 0.005; + if (immediate || !moving) { + center.copy(target); + distance = targetDistance; + velocity.set(0, 0, 0); + distanceVelocity = 0; + moving = false; + } + return true; + }; + return { + center, + setBounds(next: Box3, vertical: number, aspect: number, now: number, immediate = false) { + if (next.isEmpty()) return; + advance(now); + bounds.copy(next); + tanY = Math.tan(vertical); + tanX = tanY * aspect; + bounds.getCenter(target); + target.z = 0; + const size = bounds.getSize(new Vector3()); + targetDistance = Math.max(1, Math.max(size.x / tanX, size.y / tanY) * 0.565 + bounds.max.z); + moving = true; + advance(now, immediate || !initialized); + initialized = true; + }, + advance, + distance() { + // Spring lag must never clip the fitted assembly. + const clearance = Math.max( + Math.abs(bounds.min.x - center.x) / tanX, + Math.abs(bounds.max.x - center.x) / tanX, + Math.abs(bounds.min.y - center.y) / tanY, + Math.abs(bounds.max.y - center.y) / tanY, + ); + return Math.max(distance, clearance * 1.02 + bounds.max.z, 1); + }, + hold(active: boolean, now: number) { + held = active; + at = now; + }, + needsFrame() { + return moving && !held; + }, + }; +} diff --git a/packages/client-runtime/src/device/deviceMotion.test.ts b/packages/client-runtime/src/device/deviceMotion.test.ts new file mode 100644 index 000000000000..fc777133c351 --- /dev/null +++ b/packages/client-runtime/src/device/deviceMotion.test.ts @@ -0,0 +1,175 @@ +import { expect, it, vi } from "vite-plus/test"; +import { Quaternion, Vector3 } from "three"; +import { createDeviceMotion, rotationVector } from "./deviceMotion.ts"; +import { nearestDeviceView } from "./deviceViewSnap.ts"; + +const snaps = ["portrait", "landscape_left", "portrait_upside_down", "landscape_right"].map( + (orientation, index) => ({ + orientation, + rotation: new Quaternion().setFromAxisAngle(new Vector3(0, 0, 1), (-index * Math.PI) / 2), + yawLimit: Math.PI / 3, + }), +); +const rotation = (x: number, y: number, z = 0) => + new Quaternion().setFromAxisAngle(new Vector3(x, y, z).normalize(), Math.hypot(x, y, z)); + +it("selects the nearest quarter turn, retains a nearby yaw, and treats quaternion signs identically", () => { + const side = new Quaternion().setFromAxisAngle(new Vector3(0, 0, 1), -Math.PI / 2); + const released = side.clone().premultiply(rotation(0, 0.4)); + const nearest = nearestDeviceView(released, snaps)!; + expect(nearest.orientation).toBe("landscape_left"); + expect(nearest.rotation.angleTo(released)).toBeLessThan(1e-6); + const negative = released.clone().set(-released.x, -released.y, -released.z, -released.w); + expect(nearestDeviceView(negative, snaps)!.rotation.angleTo(nearest.rotation)).toBeLessThan(1e-6); + expect( + nearestDeviceView(rotation(0, 3), snaps)!.rotation.angleTo(new Quaternion()), + ).toBeLessThanOrEqual(Math.PI / 3 + 1e-6); +}); + +it("springs toward a cumulative drag without teleporting and keeps the same target across event partitions", () => { + const run = (parts: number) => { + const choose = vi.fn((q: Quaternion) => q); + const motion = createDeviceMotion({ choose }); + motion.dragActive(true, 0); + for (let part = 0; part < parts; part++) motion.orbit(160 / parts, 80 / parts, 0); + expect(motion.rotation.angleTo(new Quaternion())).toBeLessThan(1e-6); + for (let time = 8; time <= 400; time += 8) motion.advance(time); + expect(motion.rotation.angleTo(new Quaternion())).toBeGreaterThan(0.5); + motion.dragActive(false, 400); + for (let time = 408; time <= 2400; time += 8) motion.advance(time); + expect(choose).toHaveBeenCalledOnce(); + expect(motion.needsFrame()).toBe(false); + return motion.rotation; + }; + expect(run(1).angleTo(run(20))).toBeLessThan(1e-6); +}); + +it("selects the predicted nearest view once and preserves velocity when interrupted", () => { + const choose = vi.fn((q: Quaternion) => nearestDeviceView(q, snaps)!.rotation); + const motion = createDeviceMotion({ choose }); + motion.dragActive(true, 0); + motion.orbit(0, 240, 0); + motion.orbit(0, 20, 20); + motion.advance(40); + motion.dragActive(false, 40); + const before = motion.rotation.clone(); + motion.advance(80); + expect(motion.rotation.angleTo(before)).toBeGreaterThan(0.01); + const interrupted = motion.rotation.clone(); + motion.dragActive(true, 80); + expect(motion.rotation.angleTo(interrupted)).toBeLessThan(1e-6); + motion.orbit(30, -30, 80); + motion.advance(96); + motion.dragActive(false, 100); + for (let time = 116; time <= 2500; time += 16) motion.advance(time); + expect(choose).toHaveBeenCalledTimes(2); + expect(motion.needsFrame()).toBe(false); +}); + +it("lets a hard flick coast through multiple turns before settling on a visible view", () => { + const choose = vi.fn(() => new Quaternion()); + const motion = createDeviceMotion({ choose }); + motion.dragActive(true, 0); + motion.orbit(0, 80, 16); + motion.advance(16); + motion.orbit(0, 80, 32); + motion.advance(32); + motion.dragActive(false, 32); + + let previous = motion.rotation.clone(); + let travel = 0; + let lateSpeed = 0; + for (let time = 40; time <= 8500; time += 8) { + motion.advance(time); + const distance = motion.rotation.angleTo(previous); + travel += distance; + if (time >= 2500) lateSpeed = Math.max(lateSpeed, distance / 0.008); + previous = motion.rotation.clone(); + } + expect(travel).toBeGreaterThan(4 * Math.PI); + expect(lateSpeed).toBeLessThan(2.5); + expect(choose).toHaveBeenCalledOnce(); + expect(motion.rotation.angleTo(new Quaternion())).toBeLessThan(1e-6); + expect(motion.needsFrame()).toBe(false); +}); + +it("release uses elapsed time equally at different frame rates and a captured contact freezes projection", () => { + const run = (step: number) => { + const motion = createDeviceMotion({ choose: () => new Quaternion() }); + motion.dragActive(true, 0); + motion.orbit(160, 80, 0); + motion.advance(50); + motion.dragActive(false, 50); + for (let time = 50; time < 450; time += step) motion.advance(time); + motion.advance(450); + return motion; + }; + const fast = run(8), + slow = run(33), + throttled = run(1000); + expect(fast.rotation.angleTo(slow.rotation)).toBeLessThan(1e-6); + expect(fast.rotation.angleTo(throttled.rotation)).toBeLessThan(1e-6); + fast.hold(true, 450); + const contact = fast.rotation.clone(); + fast.orbit(100, 100, 500); + fast.advance(10000); + expect(fast.rotation.angleTo(contact)).toBeLessThan(1e-6); + fast.hold(false, 10000); + fast.advance(12000); + expect(fast.needsFrame()).toBe(false); +}); + +it("holds a trackpad orbit through a pause and snaps only when the gesture ends", () => { + const choose = vi.fn(() => new Quaternion()); + const motion = createDeviceMotion({ choose }); + motion.orbit(NaN, 0, 0); + expect(motion.needsFrame()).toBe(false); + motion.orbit(200, 100, 0); + motion.advance(100); + expect(choose).not.toHaveBeenCalled(); + expect(motion.rotation.angleTo(new Quaternion())).toBeGreaterThan(0); + motion.advance(140, true); + expect(choose).not.toHaveBeenCalled(); + const held = motion.rotation.clone(); + expect(held.angleTo(new Quaternion())).toBeGreaterThan(0.5); + expect(motion.needsFrame()).toBe(false); + motion.orbit(80, 0, 200); + motion.advance(340, true); + expect(motion.rotation.angleTo(held)).toBeGreaterThan(0.1); + expect(choose).not.toHaveBeenCalled(); + expect(motion.needsFrame()).toBe(false); + motion.dragActive(false, 340); + motion.advance(2500); + expect(choose).toHaveBeenCalledOnce(); + expect(motion.rotation.angleTo(new Quaternion())).toBeLessThan(1e-6); + expect(motion.needsFrame()).toBe(false); + expect(rotationVector(rotation(0, Math.PI)).length()).toBeCloseTo(Math.PI); +}); + +it("keeps release angular speed bounded even across a back-facing half turn", () => { + const motion = createDeviceMotion({ choose: () => rotation(0, Math.PI) }); + motion.setPose(rotation(0, Math.PI), 0); + let previous = motion.rotation.clone(); + for (let time = 8; time <= 1600; time += 8) { + motion.advance(time); + expect(motion.rotation.angleTo(previous)).toBeLessThanOrEqual(9 * 0.008 + 1e-5); + previous = motion.rotation.clone(); + } + expect(motion.rotation.angleTo(rotation(0, Math.PI))).toBeLessThan(1e-6); + expect(motion.needsFrame()).toBe(false); +}); + +it("a click without dragging resumes the interrupted resting view instead of stranding the device", () => { + const choose = vi.fn(() => new Quaternion()); + const motion = createDeviceMotion({ choose }); + const rest = rotation(0, 0.8); + motion.setPose(rest, 0); + motion.advance(50); + motion.dragActive(true, 50); + motion.advance(60); + motion.dragActive(false, 60); + motion.advance(2100); + expect(motion.rotation.angleTo(rest)).toBeLessThan(1e-6); + expect(choose).not.toHaveBeenCalled(); + expect(motion.needsFrame()).toBe(false); +}); diff --git a/packages/client-runtime/src/device/deviceMotion.ts b/packages/client-runtime/src/device/deviceMotion.ts new file mode 100644 index 000000000000..d2383c65665a --- /dev/null +++ b/packages/client-runtime/src/device/deviceMotion.ts @@ -0,0 +1,279 @@ +import { Quaternion, Vector3 } from "three"; + +/** Shortest rotation vector. q and -q represent the same orientation. */ +export function rotationVector(rotation: Quaternion) { + const q = rotation.clone().normalize(); + if (q.w < 0) q.set(-q.x, -q.y, -q.z, -q.w); + const length = Math.hypot(q.x, q.y, q.z); + return length < 1e-8 + ? new Vector3() + : new Vector3(q.x, q.y, q.z).multiplyScalar((2 * Math.atan2(length, q.w)) / length); +} +function fromVector(vector: Vector3) { + const angle = vector.length(); + return angle < 1e-8 + ? new Quaternion() + : new Quaternion().setFromAxisAngle(vector.clone().divideScalar(angle), angle); +} + +// Recovered from Bitrig 0.25's rotation dynamics: point gain, drag force limit, +// velocity limits, release prediction/blend, and the 0.5 / 0.78 release spring. +const gain = 0.006; +const response = 0.5; +const damping = 0.78; +const frequency = (2 * Math.PI) / response; +const decay = frequency * damping; +const stepSeconds = 1 / 120; +// Fast releases combine exponential friction with a critically damped correction. +const flickThreshold = 4.5; +const flickLimit = 22; +const spinDecay = 1.1; +const settleFrequency = 2; + +/** A moving spring target follows cumulative camera-space gestures; release latches the nearest useful view. */ +export function createDeviceMotion(options: { choose: (rotation: Quaternion) => Quaternion }) { + const rotation = new Quaternion(); + const target = new Quaternion(); + const velocity = new Vector3(); + const gestureVelocity = new Vector3(); + let spring: { at: number; rotation: Quaternion; velocity: Vector3; steps: number } | null = null; + let spin: { at: number; rotation: Quaternion; velocity: Vector3; correction: Vector3 } | null = + null; + let drag: { + start: Quaternion; + rest: Quaternion; + x: number; + y: number; + error: Vector3; + at: number; + } | null = null; + let held = false; + let interruptedDrag = false; + let lastInput = -Infinity; + const beginSpring = (next: Quaternion, now: number) => { + target.copy(next).normalize(); + spring = { + at: now, + rotation: rotation.clone(), + velocity: velocity.clone(), + steps: 0, + }; + }; + const release = (now: number) => { + if (!drag) return; + // A pause before lifting a pointer should not resurrect an old flick. + if (now - lastInput > 100) gestureVelocity.set(0, 0, 0); + const prediction = target + .clone() + .premultiply(fromVector(gestureVelocity.clone().multiplyScalar(0.085))); + velocity.multiplyScalar(0.2).addScaledVector(gestureVelocity, 0.8).clampLength(0, flickLimit); + const moved = drag.x !== 0 || drag.y !== 0; + const rest = drag.rest; + drag = null; + if (moved && velocity.length() >= flickThreshold) { + // Choose where the free spin would finish, then correct toward that view + // throughout the coast rather than handing off to a spring at low speed. + const projected = rotation + .clone() + .premultiply(fromVector(velocity.clone().multiplyScalar(1 / spinDecay))); + target.copy(options.choose(projected)).normalize(); + spin = { + at: now, + rotation: rotation.clone(), + velocity: velocity.clone(), + correction: rotationVector(target.clone().multiply(projected.invert())), + }; + return; + } + if (moved) beginSpring(options.choose(prediction), now); + else beginSpring(rest, now); + }; + const advance = (now: number, reduced = false) => { + if (held || !Number.isFinite(now)) return false; + if (spin) { + const seconds = Math.max(0, (now - spin.at) / 1000); + const coast = Math.exp(-spinDecay * seconds); + const settle = Math.exp(-settleFrequency * seconds); + const correction = fromVector( + spin.correction.clone().multiplyScalar(1 - (1 + settleFrequency * seconds) * settle), + ); + rotation + .copy(spin.rotation) + .premultiply(fromVector(spin.velocity.clone().multiplyScalar((1 - coast) / spinDecay))) + .premultiply(correction) + .normalize(); + velocity + .copy(spin.velocity) + .multiplyScalar(coast) + .applyQuaternion(correction) + .addScaledVector(spin.correction, settleFrequency ** 2 * seconds * settle); + if (reduced || (rotation.angleTo(target) < 0.01 && velocity.length() < 0.08)) { + spin = null; + rotation.copy(target); + velocity.set(0, 0, 0); + } + return true; + } + if (drag) { + // Follow the target during the gesture. Retain the logarithm's winding so + // a long drag cannot reverse its spring force when crossing a half turn. + const seconds = Math.min(0.05, Math.max(0, (now - drag.at) / 1000)); + drag.at = now; + const steps = Math.ceil(seconds * 120); + const dt = steps ? seconds / steps : 0; + for (let step = 0; step < steps; step++) { + const error = rotationVector(target.clone().multiply(rotation.clone().invert())); + if (error.lengthSq() > 1e-10) { + const axis = error.clone().normalize(); + const turns = Math.round((drag.error.dot(axis) - error.length()) / (2 * Math.PI)); + error.addScaledVector(axis, turns * 2 * Math.PI); + } + drag.error.copy(error); + const omega = (2 * Math.PI) / 0.68; + const acceleration = error + .clampLength(0, 1.2) + .multiplyScalar(omega * omega) + .addScaledVector(velocity, -2 * 1.12 * omega); + velocity.addScaledVector(acceleration, dt).clampLength(0, 9); + rotation.premultiply(fromVector(velocity.clone().multiplyScalar(dt))).normalize(); + } + if (reduced) { + rotation.copy(target); + velocity.set(0, 0, 0); + } + return steps > 0 || reduced; + } + if (!spring) return false; + const seconds = Math.max(0, (now - spring.at) / 1000); + const steps = Math.min(240, Math.floor(seconds / stepSeconds)); + const integrate = (q: Quaternion, speed: Vector3, dt: number) => { + const error = rotationVector(target.clone().multiply(q.clone().invert())); + const acceleration = error + .multiplyScalar(frequency * frequency) + .addScaledVector(speed, -2 * decay); + speed.addScaledVector(acceleration, dt).clampLength(0, 9); + q.premultiply(fromVector(speed.clone().multiplyScalar(dt))).normalize(); + }; + // Fixed ticks make release independent of display refresh rate. The final + // partial tick interpolates the next tick without committing it. This also + // keeps displayed angular speed bounded between tick boundaries. + for (; spring.steps < steps; spring.steps++) + integrate(spring.rotation, spring.velocity, stepSeconds); + rotation.copy(spring.rotation); + velocity.copy(spring.velocity); + const nextRotation = spring.rotation.clone(); + const nextVelocity = spring.velocity.clone(); + integrate(nextRotation, nextVelocity, stepSeconds); + const fraction = Math.min(1, (seconds - steps * stepSeconds) / stepSeconds); + rotation.slerp(nextRotation, fraction); + velocity.lerp(nextVelocity, fraction); + if ( + reduced || + steps === 240 || + (rotation.angleTo(target) < 0.0005 && velocity.length() < 0.005) + ) { + rotation.copy(target); + velocity.set(0, 0, 0); + spring = null; + } + return true; + }; + const beginDrag = (now: number) => { + if (held || drag) return; + advance(now); + drag = { + start: rotation.clone(), + rest: target.clone(), + x: 0, + y: 0, + error: new Vector3(), + at: now, + }; + target.copy(rotation); + gestureVelocity.set(0, 0, 0); + lastInput = -Infinity; + spring = null; + spin = null; + }; + return { + rotation, + setPose(next: Quaternion, now: number, immediate = false) { + advance(now); + drag = null; + spin = null; + beginSpring(next, now); + if (immediate) { + rotation.copy(target); + velocity.set(0, 0, 0); + spring = null; + } + }, + dragActive(active: boolean, now: number) { + if (active) { + // A pointer takes over at the displayed pose, even if a wheel gesture + // was paused with its target still ahead of the spring. + if (drag || spin) { + advance(now); + drag = null; + spin = null; + } + beginDrag(now); + } else { + advance(now); + release(now); + } + }, + /** Deltas are CSS pixels, independent of panel size and event partitioning. */ + orbit(x: number, y: number, now: number) { + if (held || ![x, y, now].every(Number.isFinite) || (!x && !y)) return; + advance(now); + beginDrag(now); + if (!drag) return; + const previous = target.clone(); + drag.x += x; + drag.y += y; + target + .copy(fromVector(new Vector3(drag.y * gain, drag.x * gain, 0))) + .multiply(drag.start) + .normalize(); + const seconds = (now - lastInput) / 1000; + gestureVelocity.copy(rotationVector(target.clone().multiply(previous.invert()))); + if (seconds > 0 && seconds < 0.1) + gestureVelocity.divideScalar(seconds).clampLength(0, flickLimit); + else gestureVelocity.set(0, 0, 0); + lastInput = now; + }, + /** Captured device touches and hinge edits freeze projection, independently of orbit dragging. */ + hold(active: boolean, now: number) { + if (held === active) return; + if (active) { + interruptedDrag = drag !== null || spin !== null; + drag = null; + spin = null; + spring = null; + velocity.set(0, 0, 0); + } + held = active; + if (!active) { + if (interruptedDrag) beginSpring(options.choose(rotation.clone()), now); + else if (rotation.angleTo(target) > 0.0005) beginSpring(target, now); + interruptedDrag = false; + } + }, + advance, + needsFrame() { + return ( + !held && + (spring !== null || + spin !== null || + (drag !== null && (rotation.angleTo(target) > 0.0005 || velocity.length() > 0.005))) + ); + }, + reset(next: Quaternion, now: number) { + advance(now); + drag = null; + spin = null; + beginSpring(next, now); + }, + }; +} diff --git a/packages/client-runtime/src/device/deviceViewSnap.ts b/packages/client-runtime/src/device/deviceViewSnap.ts new file mode 100644 index 000000000000..20402a80a35e --- /dev/null +++ b/packages/client-runtime/src/device/deviceViewSnap.ts @@ -0,0 +1,26 @@ +import { Quaternion, Vector3 } from "three"; +const y = new Vector3(0, 1, 0); + +/** Each useful view permits a small yaw. Pick the closest member of each family, then the closest family. */ +export function nearestDeviceView( + rotation: Quaternion, + snaps: readonly T[], +) { + let closest: T | null = null; + let distance = Infinity; + for (const snap of snaps) { + const relative = rotation.clone().multiply(snap.rotation.clone().invert()); + const turn = 2 * Math.atan2(relative.y, relative.w); + const yaw = Math.max( + -snap.yawLimit, + Math.min(snap.yawLimit, Math.atan2(Math.sin(turn), Math.cos(turn))), + ); + const candidate = snap.rotation.clone().premultiply(new Quaternion().setFromAxisAngle(y, yaw)); + const nextDistance = candidate.angleTo(rotation); + if (nextDistance < distance - 1e-8) { + distance = nextDistance; + closest = { ...snap, rotation: candidate }; + } + } + return closest; +} diff --git a/packages/client-runtime/src/device/frame.test.ts b/packages/client-runtime/src/device/frame.test.ts new file mode 100644 index 000000000000..b4ace96cd8da --- /dev/null +++ b/packages/client-runtime/src/device/frame.test.ts @@ -0,0 +1,32 @@ +import { expect, it, vi } from "vite-plus/test"; +import { createCanvasFrameSink } from "./frame.ts"; + +it("retains a borrowed frame at its native size before notifying the viewer", () => { + const drawImage = vi.fn(); + const canvas = { + width: 300, + height: 150, + getContext: () => ({ drawImage }), + } as unknown as HTMLCanvasElement; + const source = {} as CanvasImageSource; + const notify = vi.fn(() => { + expect(canvas.width).toBe(1170); + expect(canvas.height).toBe(2532); + expect(drawImage).toHaveBeenLastCalledWith(source, 0, 0, 1170, 2532); + }); + createCanvasFrameSink(canvas, notify).present(source, 1170, 2532); + expect(notify).toHaveBeenCalledOnce(); +}); + +it("reports when a canvas cannot present a frame", () => { + const canvas = { + width: 300, + height: 150, + getContext: () => null, + } as unknown as HTMLCanvasElement; + const notify = vi.fn(); + expect(createCanvasFrameSink(canvas, notify).present({} as CanvasImageSource, 1170, 2532)).toBe( + false, + ); + expect(notify).not.toHaveBeenCalled(); +}); diff --git a/packages/client-runtime/src/device/frame.ts b/packages/client-runtime/src/device/frame.ts new file mode 100644 index 000000000000..bb875b0b4a24 --- /dev/null +++ b/packages/client-runtime/src/device/frame.ts @@ -0,0 +1,24 @@ +/** A synchronous, borrowed frame. The producer releases its source after present returns. */ +export interface DeviceFrameSink { + readonly present: (source: CanvasImageSource, width: number, height: number) => boolean; +} + +/** Retains the latest frame in a canvas; consumers can invalidate textures after each draw. */ +export function createCanvasFrameSink( + canvas: HTMLCanvasElement, + onFrame?: (width: number, height: number) => void, +): DeviceFrameSink { + return { + present(source, width, height) { + const context = canvas.getContext("2d"); + if (!context) return false; + if (canvas.width !== width || canvas.height !== height) { + canvas.width = width; + canvas.height = height; + } + context.drawImage(source, 0, 0, width, height); + onFrame?.(width, height); + return true; + }, + }; +} diff --git a/packages/client-runtime/src/device/model.test.ts b/packages/client-runtime/src/device/model.test.ts new file mode 100644 index 000000000000..94db7942176b --- /dev/null +++ b/packages/client-runtime/src/device/model.test.ts @@ -0,0 +1,125 @@ +import { expect, it, vi } from "vite-plus/test"; +import { createDeviceModelSlot, resolveDeviceModelId, type DeviceModelSource } from "./model.ts"; + +const pro: DeviceModelSource = { id: "iphone-18-pro", url: "/pro.glb" }; +const max: DeviceModelSource = { id: "iphone-18-pro-max", url: "/max.glb" }; + +function fixture() { + const pending: { + signal: AbortSignal; + resolve: (model: { dispose: () => void }) => void; + reject: (error: Error) => void; + }[] = []; + const install = vi.fn(); + const load = vi.fn( + (_source: DeviceModelSource, signal: AbortSignal) => + new Promise<{ dispose: () => void }>((resolve, reject) => + pending.push({ signal, resolve, reject }), + ), + ); + const slot = createDeviceModelSlot({ load, install }); + return { slot, pending, install, load }; +} + +it("matches only exact supported hardware, including iPad size and generation", () => { + expect(resolveDeviceModelId("ios", "iPhone 18 Pro")).toBe(pro.id); + expect(resolveDeviceModelId("ios", "iPhone 18 Pro Max")).toBe(max.id); + expect(resolveDeviceModelId("ios", "iPad Pro 13-inch (M5)")).toBe("ipad-pro-13-m5"); + for (const name of [ + "iPhone 17 Pro", + "iPhone 18", + "iPad Pro 11-inch (M5)", + "iPad Pro 13-inch (M4)", + "Pixel_10_Pro", + ]) { + expect(resolveDeviceModelId("ios", name)).toBeNull(); + } + expect(resolveDeviceModelId("android", "iPhone 18 Pro")).toBeNull(); +}); + +it("cancels superseded downloads and releases a late decoded model without installing it", async () => { + const { slot, pending, install, load } = fixture(); + slot.set(pro); + slot.set({ ...pro }); + expect(load).toHaveBeenCalledTimes(1); + slot.set(max); + expect(pending[0]!.signal.aborted).toBe(true); + const latest = { dispose: vi.fn() }; + const stale = { dispose: vi.fn() }; + pending[1]!.resolve(latest); + await Promise.resolve(); + pending[0]!.resolve(stale); + await Promise.resolve(); + expect(install).toHaveBeenCalledExactlyOnceWith(latest); + expect(stale.dispose).toHaveBeenCalledOnce(); + slot.set(null); + expect(install).toHaveBeenLastCalledWith(null); + expect(latest.dispose).toHaveBeenCalledOnce(); + slot.dispose(); + expect(latest.dispose).toHaveBeenCalledOnce(); +}); + +it("keeps the fallback on request failure and disposes results that finish after teardown", async () => { + const { slot, pending, install } = fixture(); + slot.set(pro); + pending[0]!.reject(new Error("offline")); + await Promise.resolve(); + expect(install).not.toHaveBeenCalled(); + slot.set(max); + slot.dispose(); + expect(pending[1]!.signal.aborted).toBe(true); + const late = { dispose: vi.fn() }; + pending[1]!.resolve(late); + await Promise.resolve(); + expect(late.dispose).toHaveBeenCalledOnce(); + expect(install).not.toHaveBeenCalled(); +}); + +it("returns the current scene to the fallback before releasing its imported resources", async () => { + const { slot, pending, install } = fixture(); + slot.set(pro); + const model = { dispose: vi.fn(() => expect(install).toHaveBeenLastCalledWith(null)) }; + pending[0]!.resolve(model); + await Promise.resolve(); + slot.dispose(); + slot.dispose(); + expect(model.dispose).toHaveBeenCalledOnce(); +}); + +it("retries a failed source while ignoring rejection of a superseded request", async () => { + const { slot, pending, install, load } = fixture(); + slot.set(pro); + pending[0]!.reject(new Error("offline")); + await Promise.resolve(); + slot.set({ ...pro }); + expect(load).toHaveBeenCalledTimes(2); + slot.set(max); + pending[1]!.reject(new Error("cancelled")); + await Promise.resolve(); + slot.set({ ...max }); + expect(load).toHaveBeenCalledTimes(3); + const recovered = { dispose: vi.fn() }; + pending[2]!.resolve(recovered); + await Promise.resolve(); + expect(install).toHaveBeenCalledExactlyOnceWith(recovered); + slot.dispose(); +}); + +it("releases an invalid model and restores the fallback if scene preparation rejects it", async () => { + const { slot, pending, install } = fixture(); + install.mockImplementationOnce(() => { + throw new Error("invalid display"); + }); + slot.set(pro); + const model = { dispose: vi.fn() }; + pending[0]!.resolve(model); + await Promise.resolve(); + expect(model.dispose).toHaveBeenCalledOnce(); + expect(install).toHaveBeenLastCalledWith(null); + slot.set({ ...pro }); + const recovered = { dispose: vi.fn() }; + pending[1]!.resolve(recovered); + await Promise.resolve(); + expect(install).toHaveBeenLastCalledWith(recovered); + slot.dispose(); +}); diff --git a/packages/client-runtime/src/device/model.ts b/packages/client-runtime/src/device/model.ts new file mode 100644 index 000000000000..1f6524d4277c --- /dev/null +++ b/packages/client-runtime/src/device/model.ts @@ -0,0 +1,87 @@ +import type { DevicePlatform } from "@t3tools/contracts"; + +export type DeviceModelId = "iphone-18-pro" | "iphone-18-pro-max" | "ipad-pro-13-m5"; + +export interface DeviceModelSource { + readonly id: DeviceModelId; + readonly url: string; +} + +export interface DeviceAccessorySource { + readonly id: "ipad-pro-13-m5-magic-keyboard"; + readonly modelId: "ipad-pro-13-m5"; + readonly url: string; +} + +export type DeviceAssetSource = DeviceModelSource | DeviceAccessorySource; + +/** Match actual hardware, never stretch an available model to impersonate another device. */ +export function resolveDeviceModelId(platform: DevicePlatform, name: string): DeviceModelId | null { + if (platform !== "ios") return null; + if (/^iPhone 18 Pro Max$/i.test(name)) return "iphone-18-pro-max"; + if (/^iPhone 18 Pro$/i.test(name)) return "iphone-18-pro"; + if (/^iPad Pro 13-inch \(M5\)$/i.test(name)) return "ipad-pro-13-m5"; + return null; +} + +/** Each request owns its result. A cancelled or superseded parse must still release its resources. */ +export function createDeviceModelSlot< + T extends { dispose: () => void }, + Source extends DeviceAssetSource = DeviceModelSource, +>(options: { + load: (source: Source, signal: AbortSignal) => Promise; + install: (model: T | null) => void; + onError?: ((cause: unknown) => void) | undefined; +}) { + let request: { source: Source; controller: AbortController } | null = null; + let current: T | null = null; + let disposed = false; + const clear = () => { + if (current) options.install(null); + current?.dispose(); + current = null; + }; + return { + set(source: Source | null) { + if (disposed) return; + if (source?.id === request?.source.id && source?.url === request?.source.url) return; + request?.controller.abort(); + request = null; + clear(); + if (!source) return; + const next = { source, controller: new AbortController() }; + request = next; + void options.load(source, next.controller.signal).then( + (model) => { + if (disposed || request !== next) { + model.dispose(); + return; + } + try { + options.install(model); + current = model; + } catch (cause) { + options.install(null); + model.dispose(); + request = null; + options.onError?.(cause); + } + }, + (cause: unknown) => { + if (!disposed && request === next) { + request = null; + options.onError?.(cause); + } + // The procedural body stays installed on download, decoding or validation failure. + }, + ); + }, + dispose() { + if (disposed) return; + disposed = true; + request?.controller.abort(); + request = null; + clear(); + }, + }; +} diff --git a/packages/client-runtime/src/device/modelScene.test.ts b/packages/client-runtime/src/device/modelScene.test.ts new file mode 100644 index 000000000000..f58820d0afe1 --- /dev/null +++ b/packages/client-runtime/src/device/modelScene.test.ts @@ -0,0 +1,121 @@ +import { + BoxGeometry, + Group, + Mesh, + MeshBasicMaterial, + PerspectiveCamera, + PlaneGeometry, + Texture, + Vector3, +} from "three"; +import { expect, it, vi } from "vite-plus/test"; +import { createImportedPhoneScene, disposeDeviceModel } from "./modelScene.ts"; +import { phoneDisplayLayout } from "./phoneScene.ts"; + +function asset() { + const root = new Group(); + const body = new Mesh(new BoxGeometry(1.15, 2.3, 0.1), new MeshBasicMaterial()); + body.position.z = -0.02; + const screen = new Mesh(new PlaneGeometry(1, 2.2), new MeshBasicMaterial()); + screen.geometry.translate(0, 0, 0.043); + screen.name = "device-screen"; + root.add(body, screen); + return { root, screen, body }; +} + +it.each(["portrait", "landscape_left", "landscape_right", "portrait_upside_down"] as const)( + "retains the imported geometry while mapping screen input and framebuffer UVs in %s", + (orientation) => { + const { root, screen } = asset(); + const texture = new Texture(); + const original = screen.material; + // Optimization strips UVs from the placeholder material; the viewer generates framebuffer UVs. + screen.geometry.deleteAttribute("uv"); + const phone = createImportedPhoneScene(root, texture, phoneDisplayLayout(null, 1206, 2622)); + const layout = phoneDisplayLayout({ width: 1206, height: 2622, orientation }, 1206, 2622); + phone.setDisplay(texture, layout); + phone.root.rotation.set(0.2, -0.3, 0, "YXZ"); + phone.orientation.rotation.z = layout.rotation; + phone.root.updateMatrixWorld(true); + const camera = new PerspectiveCamera(32, 1, 0.1, 30); + camera.position.z = 6; + camera.updateMatrixWorld(true); + const point = phone.orientation.localToWorld(new Vector3(-0.25, -0.55, 0.043)).project(camera); + const hit = phone.screenPoint((point.x + 1) / 2, (1 - point.y) / 2, camera); + const expected = + orientation === "landscape_left" + ? [0.25, 0.25] + : orientation === "landscape_right" + ? [0.75, 0.75] + : orientation === "portrait_upside_down" + ? [0.75, 0.25] + : [0.25, 0.75]; + expect(hit?.x).toBeCloseTo(expected[0]!); + expect(hit?.y).toBeCloseTo(expected[1]!); + const geometry = screen.geometry; + const rotated = phoneDisplayLayout( + { width: 2622, height: 1206, orientation: "landscape_left" }, + 2622, + 1206, + ); + phone.setDisplay(texture, rotated); + expect(screen.geometry).toBe(geometry); + const pos = geometry.getAttribute("position"); + const uv = geometry.getAttribute("uv"); + for (let i = 0; i < pos.count; i++) { + expect(uv.getX(i)).toBeCloseTo(pos.getY(i) / 2.2 + 0.5); + expect(uv.getY(i)).toBeCloseTo(0.5 - pos.getX(i)); + } + phone.dispose(); + expect(screen.material).toBe(original); + disposeDeviceModel(root); + texture.dispose(); + }, +); + +it("rejects touches from the rear and preserves borrowed textures when releasing imported resources", () => { + const { root, screen, body } = asset(); + const texture = new Texture(); + const borrowed = vi.spyOn(texture, "dispose"); + const ownedTexture = new Texture(); + const owned = vi.spyOn(ownedTexture, "dispose"); + body.material.map = ownedTexture; + const geometry = vi.spyOn(body.geometry, "dispose"); + const phone = createImportedPhoneScene(root, texture, phoneDisplayLayout(null, 1206, 2622)); + const camera = new PerspectiveCamera(32, 1, 0.1, 30); + camera.position.z = 6; + phone.root.rotation.y = Math.PI; + expect(phone.screenPoint(0.5, 0.5, camera)).toBeNull(); + phone.dispose(); + expect(borrowed).not.toHaveBeenCalled(); + expect(geometry).not.toHaveBeenCalled(); + expect(screen.material.map).toBeNull(); + disposeDeviceModel(root); + expect(owned).toHaveBeenCalledOnce(); + expect(geometry).toHaveBeenCalledOnce(); + expect(borrowed).not.toHaveBeenCalled(); + texture.dispose(); +}); + +it("rejects missing and incorrectly normalized displays before changing their materials", () => { + const { root, screen } = asset(); + const original = screen.material; + const texture = new Texture(); + screen.geometry.scale(1, 2, 1); + expect(() => + createImportedPhoneScene(root, texture, phoneDisplayLayout(null, 1206, 2622)), + ).toThrow("not normalized"); + expect(screen.material).toBe(original); + screen.geometry.scale(1, 0.5, 1); + root.rotation.y = Math.PI; + expect(() => + createImportedPhoneScene(root, texture, phoneDisplayLayout(null, 1206, 2622)), + ).toThrow("not normalized"); + expect(screen.material).toBe(original); + screen.name = "wrong-screen"; + expect(() => + createImportedPhoneScene(root, texture, phoneDisplayLayout(null, 1206, 2622)), + ).toThrow("one device-screen"); + disposeDeviceModel(root); + texture.dispose(); +}); diff --git a/packages/client-runtime/src/device/modelScene.ts b/packages/client-runtime/src/device/modelScene.ts new file mode 100644 index 000000000000..6e1ce6eb82b3 --- /dev/null +++ b/packages/client-runtime/src/device/modelScene.ts @@ -0,0 +1,118 @@ +// @effect-diagnostics globalFetch:off - This browser-only renderer owns abortable asset requests without an Effect runtime, like the live stream. +import { Box3, Group, Mesh, MeshBasicMaterial, Texture, type Material, type Object3D } from "three"; +import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; +import type { DeviceAssetSource } from "./model.ts"; +import { createDisplayProjection, updateDisplayUv, type PhoneDisplayLayout } from "./phoneScene.ts"; + +/** GLB resources belong to one viewer; the live framebuffer texture belongs to its stream presentation. */ +export function disposeDeviceModel(root: Object3D) { + const geometries = new Set(); + const materials = new Set(); + const textures = new Set(); + root.traverse((object) => { + if (!(object instanceof Mesh)) return; + geometries.add(object.geometry); + for (const material of Array.isArray(object.material) ? object.material : [object.material]) { + materials.add(material); + for (const value of Object.values(material)) { + if (value instanceof Texture) textures.add(value); + } + } + }); + for (const geometry of geometries) geometry.dispose(); + for (const material of materials) material.dispose(); + const images = new Set(); + for (const texture of textures) { + texture.dispose(); + if (typeof ImageBitmap !== "undefined" && texture.image instanceof ImageBitmap) + images.add(texture.image); + } + for (const image of images) image.close(); +} + +/** Assets are converted offline: portrait, front +Z, centered display, height 2.2 and planar display UVs. */ +export function createImportedPhoneScene( + asset: Group, + texture: Texture, + initial: PhoneDisplayLayout, +) { + const screens: Mesh[] = []; + asset.traverse((object) => { + if (object instanceof Mesh && object.name === "device-screen") screens.push(object); + }); + const display = screens[0]; + if (!display || screens.length !== 1) + throw new Error("Device model must have one device-screen mesh"); + asset.updateMatrixWorld(true); + const screenBounds = new Box3().setFromObject(display); + const screenWidth = screenBounds.max.x - screenBounds.min.x; + const screenHeight = screenBounds.max.y - screenBounds.min.y; + if ( + display.matrixWorld.elements.some( + (value, index) => + !Number.isFinite(value) || Math.abs(value - (index % 5 === 0 ? 1 : 0)) > 0.001, + ) || + !Number.isFinite(screenWidth) || + !Number.isFinite(screenHeight) || + !Number.isFinite(screenBounds.min.z) || + !Number.isFinite(screenBounds.max.z) || + screenWidth <= 0 || + Math.abs(screenHeight - 2.2) > 0.001 || + Math.abs(screenBounds.min.x + screenBounds.max.x) > 0.001 || + Math.abs(screenBounds.min.y + screenBounds.max.y) > 0.001 || + screenBounds.max.z - screenBounds.min.z > 0.001 + ) + throw new Error("Device model display is not normalized"); + updateDisplayUv(display.geometry, screenWidth, screenHeight, initial); + const originalMaterial = display.material; + const screenMaterial = new MeshBasicMaterial({ map: texture, toneMapped: false }); + display.material = screenMaterial; + const root = new Group(); + const orientation = new Group(); + orientation.add(asset); + root.add(orientation); + const bounds = new Box3().setFromObject(asset); + let layout = initial; + const projection = createDisplayProjection( + display, + orientation, + screenWidth, + screenHeight, + () => layout, + ); + return { + root, + orientation, + width: bounds.max.x - bounds.min.x, + height: bounds.max.y - bounds.min.y, + setDisplay(nextTexture: Texture, nextLayout: PhoneDisplayLayout) { + layout = nextLayout; + screenMaterial.map = nextTexture; + updateDisplayUv(display.geometry, screenWidth, screenHeight, layout); + }, + screenPoint: projection, + dispose() { + // The model slot owns imported resources. This scene owns only its replacement screen material. + display.material = originalMaterial; + screenMaterial.map = null; + screenMaterial.dispose(); + orientation.remove(asset); + }, + }; +} + +export async function loadDeviceModel(source: DeviceAssetSource, signal: AbortSignal) { + const response = await fetch(source.url, { signal }); + if (!response.ok) throw new Error(`Device model request failed: ${response.status}`); + const data = await response.arrayBuffer(); + signal.throwIfAborted(); + const gltf = await new GLTFLoader().parseAsync( + data, + new URL(".", new URL(source.url, globalThis.location.href)).href, + ); + if (signal.aborted) { + disposeDeviceModel(gltf.scene); + signal.throwIfAborted(); + } + return { asset: gltf.scene, dispose: () => disposeDeviceModel(gltf.scene) }; +} diff --git a/packages/client-runtime/src/device/phoneInteraction.test.ts b/packages/client-runtime/src/device/phoneInteraction.test.ts new file mode 100644 index 000000000000..ddd304f66d25 --- /dev/null +++ b/packages/client-runtime/src/device/phoneInteraction.test.ts @@ -0,0 +1,105 @@ +import { expect, it, vi } from "vite-plus/test"; +import { createPhoneInteraction, phoneWheelNavigation } from "./phoneInteraction.ts"; + +it("keeps a captured screen gesture separate from orbit and ends it once on cancellation", () => { + const touch = vi.fn(); + const orbit = vi.fn(); + const screenPoint = vi + .fn() + .mockReturnValueOnce({ x: 0.2, y: 0.3 }) + .mockReturnValueOnce({ x: 1, y: 0.6 }); + const input = createPhoneInteraction({ screenPoint, touch, orbit, zoomBy: vi.fn() }); + expect(input.begin(1, { x: 0.4, y: 0.3 })).toBe(true); + expect(input.begin(2, { x: 0.2, y: 0.2 })).toBe(false); + input.move(2, { x: 0.8, y: 0.8 }); + input.move(1, { x: 1.3, y: 0.6 }); + input.end(2); + input.end(); + input.end(1); + expect(touch.mock.calls).toEqual([ + ["begin", { x: 0.2, y: 0.3 }], + ["move", { x: 1, y: 0.6 }], + ["end", { x: 1, y: 0.6 }], + ]); + expect(orbit).not.toHaveBeenCalled(); + expect(screenPoint.mock.calls[1]?.[1]).toBe(true); +}); + +it("orbits without sending touches when dragged outside the screen or with the orbit modifier", () => { + const touch = vi.fn(); + const orbit = vi.fn(); + const input = createPhoneInteraction({ screenPoint: () => null, touch, orbit, zoomBy: vi.fn() }); + input.begin(1, { x: 0.1, y: 0.2 }); + input.move(1, { x: 0.4, y: 0.5 }); + input.end(); + input.begin(2, { x: 0.5, y: 0.5 }, true); + input.move(2, { x: 0.6, y: 0.5 }); + input.end(); + expect(orbit).toHaveBeenCalledTimes(2); + expect(orbit.mock.calls[0]?.[0]).toBeCloseTo(0.3); + expect(touch).not.toHaveBeenCalled(); +}); + +it("blocks navigation during either captured gesture, then resumes without sending device touches", () => { + const touch = vi.fn(); + const orbit = vi.fn(); + const zoomBy = vi.fn(); + const input = createPhoneInteraction({ + screenPoint: () => ({ x: 0.5, y: 0.5 }), + touch, + orbit, + zoomBy, + }); + input.begin(1, { x: 0.5, y: 0.5 }); + expect(input.navigate({ type: "zoom", delta: 0.2 })).toBe(false); + expect(input.navigate({ type: "orbit", x: 0.1, y: 0.2 })).toBe(false); + input.end(); + input.begin(2, { x: 0.5, y: 0.5 }, true); + expect(input.navigate({ type: "zoom", delta: 0.2 })).toBe(false); + input.end(); + expect(input.navigate({ type: "zoom", delta: 0.2 })).toBe(true); + expect(input.navigate({ type: "orbit", x: 0.1, y: 0.2 })).toBe(true); + expect(zoomBy.mock.calls).toEqual([[0.2]]); + expect(orbit.mock.calls).toEqual([[0.1, 0.2]]); + expect(touch.mock.calls.map(([phase]) => phase)).toEqual(["begin", "end"]); +}); + +it("ends wheel orbit without ending a captured screen touch", () => { + const onInteractionActive = vi.fn(); + const input = createPhoneInteraction({ + screenPoint: () => ({ x: 0.5, y: 0.5 }), + touch: vi.fn(), + orbit: vi.fn(), + zoomBy: vi.fn(), + onInteractionActive, + }); + input.endWheel(); + expect(onInteractionActive).toHaveBeenCalledWith(false, "orbit"); + input.begin(1, { x: 0.5, y: 0.5 }); + onInteractionActive.mockClear(); + input.endWheel(); + expect(onInteractionActive).not.toHaveBeenCalled(); + input.end(1); + expect(onInteractionActive).toHaveBeenCalledWith(false, "touch"); +}); + +it("normalizes wheel units, distinguishes pinch, and bounds jumps from coarse scroll wheels", () => { + const wheel = { width: 400, height: 800, deltaX: 16, deltaY: 32, deltaMode: 0, ctrlKey: false }; + expect(phoneWheelNavigation(wheel)).toEqual({ type: "orbit", x: -0.04, y: -0.04 }); + expect(phoneWheelNavigation({ ...wheel, deltaX: 1, deltaY: 2, deltaMode: 1 })).toEqual( + phoneWheelNavigation(wheel), + ); + expect(phoneWheelNavigation({ ...wheel, deltaX: 1, deltaY: 1, deltaMode: 2 })).toEqual({ + type: "orbit", + x: -0.25, + y: -0.25, + }); + expect(phoneWheelNavigation({ ...wheel, ctrlKey: true })).toEqual({ type: "zoom", delta: -0.32 }); + expect(phoneWheelNavigation({ ...wheel, deltaY: -10000, ctrlKey: true })).toEqual({ + type: "zoom", + delta: 1, + }); + expect(phoneWheelNavigation({ ...wheel, width: 0 })).toBeNull(); + expect(phoneWheelNavigation({ ...wheel, deltaY: NaN })).toBeNull(); + expect(phoneWheelNavigation({ ...wheel, deltaMode: 9 })).toBeNull(); +}); diff --git a/packages/client-runtime/src/device/phoneInteraction.ts b/packages/client-runtime/src/device/phoneInteraction.ts new file mode 100644 index 000000000000..dd701f1a8b57 --- /dev/null +++ b/packages/client-runtime/src/device/phoneInteraction.ts @@ -0,0 +1,83 @@ +interface Point { + readonly x: number; + readonly y: number; +} + +export type PhoneNavigation = + | { readonly type: "orbit"; readonly x: number; readonly y: number } + | { readonly type: "zoom"; readonly delta: number }; + +/** Browser wheel units vary by device. Navigation uses viewport fractions and logarithmic zoom. */ +export function phoneWheelNavigation(input: { + readonly deltaX: number; + readonly deltaY: number; + readonly deltaMode: number; + readonly ctrlKey: boolean; + readonly width: number; + readonly height: number; +}): PhoneNavigation | null { + const { deltaX, deltaY, deltaMode, ctrlKey, width, height } = input; + if (![deltaX, deltaY, width, height].every(Number.isFinite) || width <= 0 || height <= 0) + return null; + if (deltaMode !== 0 && deltaMode !== 1 && deltaMode !== 2) return null; + const unit = deltaMode === 1 ? 16 : deltaMode === 2 ? height : 1; + const clamp = (value: number, limit: number) => Math.min(limit, Math.max(-limit, value)); + if (ctrlKey) return { type: "zoom", delta: clamp(-deltaY * unit * 0.01, 1) }; + return { + type: "orbit", + x: clamp((-deltaX * unit) / width, 0.25), + y: clamp((-deltaY * unit) / height, 0.25), + }; +} + +/** A single pointer owns either a device gesture or an orbit until released or cancelled. */ +export function createPhoneInteraction(options: { + readonly screenPoint: (point: Point, captured: boolean) => Point | null; + readonly touch: (phase: "begin" | "move" | "end", point: Point) => void; + readonly orbit: (deltaX: number, deltaY: number) => void; + readonly zoomBy: (logDelta: number) => void; + readonly onInteractionActive?: (active: boolean, mode: "touch" | "orbit") => void; +}) { + let active: { id: number; mode: "touch" | "orbit"; last: Point; screen: Point | null } | null = + null; + return { + navigate(gesture: PhoneNavigation) { + // Moving the camera during a captured device touch would change its projected coordinates. + if (active) return false; + if (gesture.type === "zoom") options.zoomBy(gesture.delta); + else options.orbit(gesture.x, gesture.y); + return true; + }, + endWheel() { + if (!active) options.onInteractionActive?.(false, "orbit"); + }, + begin(id: number, point: Point, forceOrbit = false) { + if (active) return false; + const screen = forceOrbit ? null : options.screenPoint(point, false); + active = { id, mode: screen ? "touch" : "orbit", last: point, screen }; + options.onInteractionActive?.(true, active.mode); + if (screen) options.touch("begin", screen); + return true; + }, + move(id: number, point: Point) { + if (active?.id !== id) return; + if (active.mode === "touch") { + const screen = options.screenPoint(point, true); + if (screen) { + active.screen = screen; + options.touch("move", screen); + } + } else { + options.orbit(point.x - active.last.x, point.y - active.last.y); + } + active.last = point; + }, + end(id?: number) { + if (!active || (id !== undefined && active.id !== id)) return; + const previous = active; + active = null; + if (previous.mode === "touch" && previous.screen) options.touch("end", previous.screen); + options.onInteractionActive?.(false, previous.mode); + }, + }; +} diff --git a/packages/client-runtime/src/device/phoneScene.test.ts b/packages/client-runtime/src/device/phoneScene.test.ts new file mode 100644 index 000000000000..188b8891803f --- /dev/null +++ b/packages/client-runtime/src/device/phoneScene.test.ts @@ -0,0 +1,150 @@ +import { Box3, PerspectiveCamera, Texture, Vector3 } from "three"; +import { describe, expect, it } from "vite-plus/test"; +import { createPhoneScene, phoneDisplayLayout } from "./phoneScene.ts"; +import { IOS_TABLET_SHAPE, IOS_PHONE_SHAPE, ANDROID_PHONE_SHAPE } from "./shapeProfile.ts"; + +const orientations = [ + "portrait", + "landscape_left", + "landscape_right", + "portrait_upside_down", +] as const; + +describe("3D phone input", () => { + it.each([IOS_TABLET_SHAPE, IOS_PHONE_SHAPE, ANDROID_PHONE_SHAPE])( + "preserves input projection for the $id silhouette in every orientation", + (profile) => { + for (const orientation of orientations) { + const texture = new Texture(); + const aspect = profile.id === "ios-tablet" ? 0.75 : 0.45; + const layout = phoneDisplayLayout( + { width: 1200 * aspect, height: 1200, orientation }, + 1200 * aspect, + 1200, + ); + const phone = createPhoneScene(texture, layout, profile); + const camera = new PerspectiveCamera(32, 1, 0.1, 30); + camera.position.z = 6; + camera.updateMatrixWorld(true); + phone.root.rotation.set(0.2, -0.3, 0, "YXZ"); + phone.orientation.rotation.z = layout.rotation; + phone.root.updateMatrixWorld(true); + const point = new Vector3((-2.2 * aspect) / 4, -2.2 / 4, 0.043); + phone.orientation.localToWorld(point); + point.project(camera); + const hit = phone.screenPoint((point.x + 1) / 2, (1 - point.y) / 2, camera); + const expected = + orientation === "landscape_left" + ? { x: 0.25, y: 0.25 } + : orientation === "landscape_right" + ? { x: 0.75, y: 0.75 } + : orientation === "portrait_upside_down" + ? { x: 0.75, y: 0.25 } + : { x: 0.25, y: 0.75 }; + expect(hit?.x).toBeCloseTo(expected.x); + expect(hit?.y).toBeCloseTo(expected.y); + phone.dispose(); + texture.dispose(); + } + }, + ); + it.each([0.6, 2.2])("keeps device coordinates accurate at %s camera zoom", (zoom) => { + const texture = new Texture(); + const layout = phoneDisplayLayout(null, 900, 1950); + const phone = createPhoneScene(texture, layout); + const camera = new PerspectiveCamera(32, 0.65, 0.1, 30); + camera.position.z = 6 / zoom; + camera.updateMatrixWorld(true); + phone.root.rotation.set(0.2, -0.3, 0, "YXZ"); + phone.root.updateMatrixWorld(true); + const point = new Vector3((-2.2 * layout.aspect) / 4, -2.2 / 4, 0.043); + phone.orientation.localToWorld(point); + point.project(camera); + const hit = phone.screenPoint((point.x + 1) / 2, (1 - point.y) / 2, camera); + expect(hit?.x).toBeCloseTo(0.25); + expect(hit?.y).toBeCloseTo(0.75); + phone.dispose(); + texture.dispose(); + }); + + it.each(orientations)("maps perspective hits to displayed coordinates in %s", (orientation) => { + const texture = new Texture(); + const layout = phoneDisplayLayout({ width: 1170, height: 2532, orientation }, 1170, 2532); + const phone = createPhoneScene(texture, layout); + const camera = new PerspectiveCamera(32, 0.65, 0.1, 30); + camera.position.z = 6; + camera.updateMatrixWorld(true); + phone.root.rotation.set(0.3, -0.45, 0, "YXZ"); + phone.orientation.rotation.z = layout.rotation; + phone.root.updateMatrixWorld(true); + // A point one quarter across and three quarters down the portrait display. + const point = new Vector3((-2.2 * layout.aspect) / 4, -2.2 / 4, 0.043); + phone.orientation.localToWorld(point); + point.project(camera); + const hit = phone.screenPoint((point.x + 1) / 2, (1 - point.y) / 2, camera); + const expected = + orientation === "landscape_left" + ? { x: 0.25, y: 0.25 } + : orientation === "landscape_right" + ? { x: 0.75, y: 0.75 } + : orientation === "portrait_upside_down" + ? { x: 0.75, y: 0.25 } + : { x: 0.25, y: 0.75 }; + expect(hit?.x).toBeCloseTo(expected.x); + expect(hit?.y).toBeCloseTo(expected.y); + phone.dispose(); + texture.dispose(); + }); + + it("rejects new gestures outside the display, but clamps captured drags at its edge", () => { + const texture = new Texture(); + const phone = createPhoneScene(texture, phoneDisplayLayout(null, 900, 1950)); + const camera = new PerspectiveCamera(32, 1, 0.1, 30); + camera.position.z = 5; + expect(phone.screenPoint(1, 0.5, camera)).toBeNull(); + expect(phone.screenPoint(1, 0.5, camera, true)).toEqual({ x: 1, y: 0.5 }); + phone.root.rotation.y = Math.PI; + expect(phone.screenPoint(0.5, 0.5, camera)).toBeNull(); + phone.dispose(); + texture.dispose(); + }); + + it.each(["landscape_left", "landscape_right"] as const)( + "unrotates an already-landscape framebuffer in %s", + (orientation) => { + const texture = new Texture(); + const layout = phoneDisplayLayout({ width: 1950, height: 900, orientation }, 1950, 900); + const phone = createPhoneScene(texture, layout); + phone.orientation.rotation.z = layout.rotation; + const camera = new PerspectiveCamera(32, 1, 0.1, 30); + camera.position.z = 5; + expect(phone.screenPoint(0.5, 0.5, camera)).toEqual({ x: 0.5, y: 0.5 }); + phone.dispose(); + texture.dispose(); + }, + ); +}); + +it.each([IOS_PHONE_SHAPE, IOS_TABLET_SHAPE, ANDROID_PHONE_SHAPE])( + "attaches the $id rear camera plate to the back and seats its lenses on the housing", + (profile) => { + const texture = new Texture(); + const phone = createPhoneScene(texture, phoneDisplayLayout(null, 900, 1950), profile); + phone.root.updateMatrixWorld(true); + const back = new Box3().setFromObject(phone.root.getObjectByName("device-back")!); + const plate = new Box3().setFromObject(phone.root.getObjectByName("camera-plate")!); + expect(plate.min.z).toBeLessThan(back.min.z); + expect(plate.max.z).toBeGreaterThanOrEqual(back.min.z); + const assembly = phone.root.getObjectByName("rear-camera")!; + const rings = assembly.children.filter((part) => part.name === "camera-ring"); + const lenses = assembly.children.filter((part) => part.name === "camera-lens"); + for (const [index, ring] of rings.entries()) { + const ringBounds = new Box3().setFromObject(ring); + expect(ringBounds.intersectsBox(plate)).toBe(true); + const lensBounds = new Box3().setFromObject(lenses[index]!); + expect(Math.abs(lensBounds.max.z - ringBounds.min.z)).toBeLessThan(0.001); + } + phone.dispose(); + texture.dispose(); + }, +); diff --git a/packages/client-runtime/src/device/phoneScene.ts b/packages/client-runtime/src/device/phoneScene.ts new file mode 100644 index 000000000000..824288031452 --- /dev/null +++ b/packages/client-runtime/src/device/phoneScene.ts @@ -0,0 +1,302 @@ +import { + BoxGeometry, + CircleGeometry, + CylinderGeometry, + ExtrudeGeometry, + Float32BufferAttribute, + Group, + Mesh, + MeshBasicMaterial, + MeshPhysicalMaterial, + MeshStandardMaterial, + Plane, + Raycaster, + Shape, + ShapeGeometry, + Vector2, + Vector3, + type Camera, + type BufferGeometry, + type Texture, +} from "three"; +import type { DeviceScreenSize } from "./stream.ts"; +import { IOS_PHONE_SHAPE, type DeviceShapeProfile } from "./shapeProfile.ts"; + +const SCREEN_HEIGHT = 2.2; + +function roundedPath(width: number, height: number, radius: number, path = new Shape()) { + const x = -width / 2; + const y = -height / 2; + path.moveTo(x + radius, y); + path.lineTo(x + width - radius, y); + path.quadraticCurveTo(x + width, y, x + width, y + radius); + path.lineTo(x + width, y + height - radius); + path.quadraticCurveTo(x + width, y + height, x + width - radius, y + height); + path.lineTo(x + radius, y + height); + path.quadraticCurveTo(x, y + height, x, y + height - radius); + path.lineTo(x, y + radius); + path.quadraticCurveTo(x, y, x + radius, y); + return path; +} + +/** Display orientation is independent of orbit. Texture coordinates stay in raw framebuffer space. */ +export function phoneDisplayLayout( + screen: DeviceScreenSize | null, + rawWidth: number, + rawHeight: number, +) { + const width = rawWidth || screen?.width || 900; + const height = rawHeight || screen?.height || 1950; + const landscape = + screen?.orientation === "landscape_left" || screen?.orientation === "landscape_right"; + const rotation = + screen?.orientation === "landscape_left" + ? -Math.PI / 2 + : screen?.orientation === "landscape_right" + ? Math.PI / 2 + : screen?.orientation === "portrait_upside_down" + ? Math.PI + : 0; + return { + aspect: Math.min(width, height) / Math.max(width, height), + rotation, + landscape, + rawLandscape: width > height, + }; +} + +/** An original procedural device body. It makes no claim to reproduce a particular hardware model. */ +export function createPhoneScene( + texture: Texture, + layout: ReturnType, + profile: DeviceShapeProfile = IOS_PHONE_SHAPE, +) { + const { aspect } = layout; + const root = new Group(); + const orientation = new Group(); + root.add(orientation); + const screenWidth = SCREEN_HEIGHT * aspect; + const width = screenWidth + profile.bezel * 2; + const height = SCREEN_HEIGHT + profile.bezel * 2; + const backZ = 0.01 - profile.depth; + const metal = new MeshStandardMaterial({ color: 0xb5bcc7, metalness: 0.88, roughness: 0.27 }); + const glass = new MeshPhysicalMaterial({ + color: 0x141820, + metalness: 0.15, + roughness: 0.2, + clearcoat: 1, + }); + const backMaterial = new MeshStandardMaterial({ + color: profile.backColor, + metalness: 0.45, + roughness: 0.32, + }); + const lensMaterial = new MeshPhysicalMaterial({ + color: 0x071326, + metalness: 0.6, + roughness: 0.12, + clearcoat: 1, + }); + const materials = [metal, glass, backMaterial, lensMaterial]; + + const body = new Mesh( + new ExtrudeGeometry(roundedPath(width, height, profile.bodyRadius), { + depth: profile.depth, + bevelEnabled: true, + bevelSize: 0.012, + bevelThickness: 0.012, + bevelSegments: 3, + steps: 1, + curveSegments: 12, + }), + metal, + ); + body.position.z = 0.025 - profile.depth; + orientation.add(body); + const face = new Mesh( + new ShapeGeometry(roundedPath(width - 0.014, height - 0.014, profile.bodyRadius - 0.01), 16), + glass, + ); + face.position.z = 0.04; + orientation.add(face); + const back = new Mesh( + new ShapeGeometry(roundedPath(width - 0.012, height - 0.012, profile.bodyRadius - 0.01), 16), + backMaterial, + ); + back.name = "device-back"; + back.rotation.y = Math.PI; + back.position.z = backZ; + orientation.add(back); + + const screenGeometry = new ShapeGeometry( + roundedPath(screenWidth, SCREEN_HEIGHT, profile.screenRadius), + 20, + ); + updateDisplayUv(screenGeometry, screenWidth, SCREEN_HEIGHT, layout); + const screenMaterial = new MeshBasicMaterial({ map: texture, toneMapped: false }); + const display = new Mesh(screenGeometry, screenMaterial); + display.position.z = 0.043; + orientation.add(display); + + for (const { edge, offset, length } of profile.buttons) { + const button = new Mesh( + new BoxGeometry( + edge === "top" ? length : 0.026, + edge === "top" ? 0.026 : length, + profile.depth * 0.65, + ), + metal, + ); + button.position.set( + edge === "top" ? offset : (edge === "left" ? -1 : 1) * (width / 2 + 0.015), + edge === "top" ? height / 2 + 0.015 : offset, + (0.025 + backZ) / 2, + ); + orientation.add(button); + } + // Rear components share a surface-relative coordinate system, with outward positive Z. + const rearCamera = new Group(); + rearCamera.name = "rear-camera"; + rearCamera.position.set( + width / 2 - profile.camera.insetX, + height / 2 - profile.camera.insetY, + back.position.z + 0.001, + ); + rearCamera.rotation.y = Math.PI; + orientation.add(rearCamera); + const plateFront = 0.025 + 0.007; + const ringDepth = 0.025; + const cameraPlate = new Mesh( + new ExtrudeGeometry( + roundedPath( + profile.camera.width, + profile.camera.height, + Math.min(profile.camera.width, profile.camera.height) / 4, + ), + { + depth: 0.025, + bevelEnabled: true, + bevelSize: 0.009, + bevelThickness: 0.007, + bevelSegments: 2, + }, + ), + backMaterial, + ); + cameraPlate.name = "camera-plate"; + rearCamera.add(cameraPlate); + for (const [x, y] of profile.camera.lenses) { + const radius = profile.camera.lensRadius; + const ring = new Mesh(new CylinderGeometry(radius + 0.014, radius + 0.014, 0.025, 32), metal); + ring.rotation.x = Math.PI / 2; + ring.name = "camera-ring"; + ring.position.set(x, y, plateFront + ringDepth / 2 - 0.003); + rearCamera.add(ring); + const lens = new Mesh(new CircleGeometry(radius, 32), lensMaterial); + lens.name = "camera-lens"; + lens.position.set(x, y, ring.position.z + ringDepth / 2 + 0.0005); + rearCamera.add(lens); + } + const flashMaterial = new MeshBasicMaterial({ color: 0xf2ead6 }); + if (profile.camera.flash) { + const flash = new Mesh(new CircleGeometry(0.022, 20), flashMaterial); + flash.position.set(profile.camera.flash[0], profile.camera.flash[1], plateFront + 0.0005); + rearCamera.add(flash); + } + + let activeLayout = layout; + const screenPoint = createDisplayProjection( + display, + orientation, + screenWidth, + SCREEN_HEIGHT, + () => activeLayout, + ); + return { + root, + orientation, + width, + height, + screenPoint, + setDisplay(nextTexture: Texture, nextLayout: PhoneDisplayLayout) { + activeLayout = nextLayout; + screenMaterial.map = nextTexture; + updateDisplayUv(screenGeometry, screenWidth, SCREEN_HEIGHT, activeLayout); + }, + dispose() { + root.traverse((object) => { + if (object instanceof Mesh) object.geometry.dispose(); + }); + for (const material of materials) material.dispose(); + screenMaterial.dispose(); + flashMaterial.dispose(); + }, + }; +} + +export type PhoneDisplayLayout = ReturnType; + +/** Canonical portrait geometry maps to raw framebuffer coordinates in every OS orientation. */ +export function updateDisplayUv( + geometry: BufferGeometry, + width: number, + height: number, + layout: PhoneDisplayLayout, +) { + const position = geometry.getAttribute("position"); + if (!geometry.hasAttribute("uv")) { + geometry.setAttribute( + "uv", + new Float32BufferAttribute(new Float32Array(position.count * 2), 2), + ); + } + const uv = geometry.getAttribute("uv"); + for (let i = 0; i < position.count; i++) { + const u = (position.getX(i) + width / 2) / width; + const v = (position.getY(i) + height / 2) / height; + uv.setXY( + i, + layout.rawLandscape ? (layout.rotation > 0 ? 1 - v : v) : u, + layout.rawLandscape ? (layout.rotation > 0 ? u : 1 - u) : v, + ); + } + uv.needsUpdate = true; +} + +/** New touches hit only the front display; captured drags project onto its plane and clamp. */ +export function createDisplayProjection( + display: Mesh, + orientation: Group, + width: number, + height: number, + getLayout: () => PhoneDisplayLayout, +) { + const raycaster = new Raycaster(); + const pointer = new Vector2(); + const local = new Vector3(); + display.geometry.computeBoundingBox(); + const z = display.position.z + (display.geometry.boundingBox?.max.z ?? 0); + const plane = new Plane(new Vector3(0, 0, 1), -z); + return (x: number, y: number, camera: Camera, captured = false) => { + orientation.updateWorldMatrix(true, true); + camera.updateMatrixWorld(true); + pointer.set(x * 2 - 1, 1 - y * 2); + raycaster.setFromCamera(pointer, camera); + if (!captured) { + const hit = raycaster.intersectObject(display, false)[0]; + if (!hit) return null; + local.copy(hit.point); + orientation.worldToLocal(local); + } else { + const ray = raycaster.ray.clone().applyMatrix4(orientation.matrixWorld.clone().invert()); + if (!ray.intersectPlane(plane, local)) return null; + } + const u = Math.min(1, Math.max(0, (local.x + width / 2) / width)); + const v = Math.min(1, Math.max(0, (local.y + height / 2) / height)); + const { rotation } = getLayout(); + if (rotation === -Math.PI / 2) return { x: v, y: u }; + if (rotation === Math.PI / 2) return { x: 1 - v, y: 1 - u }; + if (rotation === Math.PI) return { x: 1 - u, y: v }; + return { x: u, y: 1 - v }; + }; +} diff --git a/packages/client-runtime/src/device/phoneViewer.test.ts b/packages/client-runtime/src/device/phoneViewer.test.ts new file mode 100644 index 000000000000..ddb08bddf976 --- /dev/null +++ b/packages/client-runtime/src/device/phoneViewer.test.ts @@ -0,0 +1,350 @@ +import type { PerspectiveCamera, Scene, Object3D } from "three"; +import { afterEach, expect, it, vi } from "vite-plus/test"; + +const gpu = vi.hoisted(() => ({ + instances: [] as { + blank: boolean; + allocations: number; + disposed: boolean; + size: { width: number; height: number; pixelRatio: number }; + frames: { + scene: Scene; + phone: Object3D | undefined; + rotation: Quaternion | undefined; + yaw: number | undefined; + cameraZ: number; + }[]; + }[], +})); + +vi.mock("three", async () => { + const actual = await vi.importActual("three"); + return { + ...actual, + WebGLRenderer: class { + outputColorSpace = ""; + state: (typeof gpu.instances)[number] = { + blank: true, + allocations: 0, + disposed: false, + size: { width: 0, height: 0, pixelRatio: 1 }, + frames: [], + }; + constructor() { + gpu.instances.push(this.state); + } + setDrawingBufferSize(width: number, height: number, pixelRatio: number) { + this.state.size = { width, height, pixelRatio }; + this.state.allocations++; + this.state.blank = true; + } + setSize(width: number, height: number) { + this.setDrawingBufferSize(width, height, this.state.size.pixelRatio); + } + setPixelRatio(pixelRatio: number) { + this.setDrawingBufferSize(this.state.size.width, this.state.size.height, pixelRatio); + } + render(scene: Scene, camera: PerspectiveCamera) { + const phone = scene.children.find((child) => child.type === "Group"); + this.state.frames.push({ + scene, + phone, + rotation: phone?.quaternion.clone(), + yaw: phone?.rotation.y, + cameraZ: camera.position.z, + }); + this.state.blank = false; + } + dispose() { + this.state.disposed = true; + } + forceContextLoss() {} + }, + }; +}); + +const models = vi.hoisted(() => ({ + pending: [] as { + signal: AbortSignal; + resolve: (model: { asset: import("three").Group; dispose: () => void }) => void; + }[], +})); +vi.mock("./modelScene.ts", async () => { + const actual = await vi.importActual("./modelScene.ts"); + return { + ...actual, + loadDeviceModel: (_source: unknown, signal: AbortSignal) => + new Promise((resolve) => models.pending.push({ signal, resolve })), + }; +}); + +import { BoxGeometry, Group, Mesh, MeshBasicMaterial, PlaneGeometry, Quaternion } from "three"; +import { disposeDeviceModel } from "./modelScene.ts"; +import { createPhoneViewer } from "./phoneViewer.ts"; +import { IOS_TABLET_SHAPE } from "./shapeProfile.ts"; + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + gpu.instances.length = 0; + models.pending.length = 0; +}); + +function fixture() { + const pending = new Map(); + let id = 0; + let now = 0; + vi.spyOn(performance, "now").mockImplementation(() => now); + vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => { + pending.set(++id, callback); + return id; + }); + vi.stubGlobal("cancelAnimationFrame", (id: number) => pending.delete(id)); + const canvas = Object.assign(new EventTarget(), { width: 0, height: 0 }) as HTMLCanvasElement; + const source = { width: 1206, height: 2622 } as HTMLCanvasElement; + const onUnavailable = vi.fn(); + const onFramingAspect = vi.fn(); + const viewer = createPhoneViewer({ canvas, source, onUnavailable, onFramingAspect }); + const draw = (time = now) => { + now = time; + const callbacks = [...pending.values()]; + pending.clear(); + callbacks.forEach((callback) => callback(0)); + }; + viewer.resize(400, 700, 2); + draw(); + return { + viewer, + draw, + pending, + source, + onUnavailable, + onFramingAspect, + state: gpu.instances[0]!, + }; +} + +it("retains the drawn phone until resize and redraw commit together, without replacing scene or pose", () => { + const { viewer, draw, pending, source, state, onUnavailable } = fixture(); + viewer.orbit(0.08, 0.04); + draw(); + const previous = state.frames.at(-1)!; + const allocations = state.allocations; + viewer.resize(500, 700, 2); + viewer.resize(450, 700, 2); + viewer.frameUpdated(); + // ResizeObserver runs after rAF. A cleared buffer here would reach the browser's next paint. + expect(state.blank).toBe(false); + expect(state.allocations).toBe(allocations); + expect(pending.size).toBe(1); + draw(); + expect(state.size).toEqual({ width: 450, height: 700, pixelRatio: 2 }); + expect(state.blank).toBe(false); + expect(state.allocations).toBe(allocations + 1); + expect(state.frames.at(-1)?.scene).toBe(previous.scene); + expect(state.frames.at(-1)?.phone).toBe(previous.phone); + expect(state.frames.at(-1)?.yaw).toBe(previous.yaw); + viewer.resize(400, 700, 2); + draw(); + expect(state.frames.at(-1)?.cameraZ).toBeCloseTo(previous.cameraZ); + expect(source).toMatchObject({ width: 1206, height: 2622 }); + expect(gpu.instances).toHaveLength(1); + expect(onUnavailable).not.toHaveBeenCalled(); + viewer.dispose(); +}); + +it("ignores redundant and invalid sizes and combines a DPR change into one allocation and draw", () => { + const { viewer, draw, pending, state } = fixture(); + const allocations = state.allocations; + viewer.resize(400, 700, 2); + viewer.resize(400, 700, 5); + viewer.resize(0, 700, 2); + viewer.resize(NaN, 700, 2); + expect(pending.size).toBe(0); + viewer.resize(400, 700, 1); + expect(state.blank).toBe(false); + draw(); + expect(state.size.pixelRatio).toBe(1); + expect(state.allocations).toBe(allocations + 1); + viewer.resize(450, 700, 1); + viewer.dispose(); + draw(); + expect(state.allocations).toBe(allocations + 1); + expect(state.disposed).toBe(true); +}); + +it("changes device shape without replacing the renderer, decoded source or pose", () => { + const { viewer, draw, source, state } = fixture(); + viewer.orbit(0.1, 0.05); + draw(); + const previous = state.frames.at(-1)!; + const allocations = state.allocations; + viewer.setScreen(null, IOS_TABLET_SHAPE); + draw(); + expect(state.frames.at(-1)?.phone).not.toBe(previous.phone); + expect(state.frames.at(-1)?.yaw).toBe(previous.yaw); + expect(state.frames.at(-1)?.scene).toBe(previous.scene); + expect(state.allocations).toBe(allocations); + expect(source.width).toBe(1206); + expect(gpu.instances).toHaveLength(1); + const tablet = state.frames.at(-1)?.phone; + viewer.frameUpdated(); + draw(); + expect(state.frames.at(-1)?.phone).toBe(tablet); + viewer.dispose(); +}); + +it("retains the loaded model and pose through rotation and framebuffer resolution changes, then releases it once", async () => { + const { viewer, draw, source, state } = fixture(); + viewer.orbit(0.08, 0.04); + draw(); + const yaw = state.frames.at(-1)!.yaw; + viewer.setModel({ id: "iphone-18-pro", url: "/pro.glb" }); + const asset = new Group(); + const body = new Mesh(new BoxGeometry(1.15, 2.3, 0.1), new MeshBasicMaterial()); + body.position.z = -0.02; + const display = new Mesh(new PlaneGeometry(1, 2.2), new MeshBasicMaterial()); + display.geometry.translate(0, 0, 0.043); + display.name = "device-screen"; + asset.add(body, display); + const release = vi.spyOn(body.geometry, "dispose"); + const dispose = vi.fn(() => disposeDeviceModel(asset)); + models.pending[0]!.resolve({ asset, dispose }); + await Promise.resolve(); + draw(); + const loaded = state.frames.at(-1)!.phone; + expect(loaded?.getObjectByName("device-screen")).toBe(display); + expect(state.frames.at(-1)!.yaw).toBe(yaw); + source.width = 2622; + source.height = 1206; + viewer.setScreen({ width: 2622, height: 1206, orientation: "landscape_left" }); + viewer.frameUpdated(); + viewer.resize(700, 400, 1); + draw(); + expect(state.frames.at(-1)!.phone).toBe(loaded); + expect(state.frames.at(-1)!.yaw).toBe(yaw); + expect(display.material.map?.image).toBe(source); + expect(release).not.toHaveBeenCalled(); + expect(gpu.instances).toHaveLength(1); + viewer.dispose(); + viewer.dispose(); + expect(dispose).toHaveBeenCalledOnce(); + expect(release).toHaveBeenCalledOnce(); +}); + +it("attaches and detaches an accessory without replacing the device, stream texture, renderer or pose", async () => { + const { viewer, draw, source, state, onFramingAspect } = fixture(); + viewer.setModel({ id: "ipad-pro-13-m5", url: "/ipad.glb" }); + const asset = new Group(); + const body = new Mesh(new BoxGeometry(1.8, 2.3, 0.1), new MeshBasicMaterial()); + const display = new Mesh(new PlaneGeometry(1.7, 2.2), new MeshBasicMaterial()); + display.geometry.translate(0, 0, 0.043); + display.name = "device-screen"; + asset.add(body, display); + const releaseBody = vi.fn(() => disposeDeviceModel(asset)); + models.pending[0]!.resolve({ asset, dispose: releaseBody }); + await Promise.resolve(); + viewer.orbit(0.1, 0.05); + draw(); + const before = state.frames.at(-1)!; + const bareAspect = onFramingAspect.mock.lastCall![0]; + const keyboard = new Group(); + const deck = new Mesh(new BoxGeometry(2, 1, 1), new MeshBasicMaterial()); + deck.position.set(1, 0, 0.5); + keyboard.add(deck); + const releaseKeyboard = vi.fn(() => disposeDeviceModel(keyboard)); + viewer.setAccessory({ + id: "ipad-pro-13-m5-magic-keyboard", + modelId: "ipad-pro-13-m5", + url: "/keyboard.glb", + }); + models.pending[1]!.resolve({ asset: keyboard, dispose: releaseKeyboard }); + await Promise.resolve(); + draw(); + expect(state.frames.at(-1)!.phone).toBe(before.phone); + expect(keyboard.parent).toBe(asset.parent); + expect(state.frames.at(-1)!.yaw).toBe(before.yaw); + expect(display.material.map?.image).toBe(source); + expect(state.frames.at(-1)!.cameraZ).toBeGreaterThan(before.cameraZ); + expect(onFramingAspect.mock.lastCall![0]).toBeGreaterThan(bareAspect); + const calls = onFramingAspect.mock.calls.length; + viewer.orbit(0.1, 0.05); + viewer.frameUpdated(); + draw(); + expect(onFramingAspect).toHaveBeenCalledTimes(calls); + viewer.setAccessory(null); + draw(); + expect(keyboard.parent).toBeNull(); + expect(onFramingAspect.mock.lastCall![0]).toBeCloseTo(bareAspect); + expect(releaseKeyboard).toHaveBeenCalledOnce(); + expect(releaseBody).not.toHaveBeenCalled(); + expect(state.frames.at(-1)!.phone).toBe(before.phone); + expect(state.frames.at(-1)!.cameraZ).toBeCloseTo(before.cameraZ); + expect(gpu.instances).toHaveLength(1); + viewer.dispose(); + expect(releaseBody).toHaveBeenCalledOnce(); + expect(releaseKeyboard).toHaveBeenCalledOnce(); +}); + +it("rejects incompatible accessories and releases an accessory that finishes after detaching", async () => { + const { viewer } = fixture(); + const keyboard = { + id: "ipad-pro-13-m5-magic-keyboard", + modelId: "ipad-pro-13-m5", + url: "/keyboard.glb", + } as const; + viewer.setAccessory(keyboard); + expect(models.pending).toHaveLength(0); + viewer.setModel({ id: "ipad-pro-13-m5", url: "/ipad.glb" }); + viewer.setAccessory(keyboard); + viewer.setAccessory(null); + expect(models.pending[1]!.signal.aborted).toBe(true); + const dispose = vi.fn(); + const asset = new Group(); + models.pending[1]!.resolve({ asset, dispose }); + await Promise.resolve(); + expect(asset.parent).toBeNull(); + expect(dispose).toHaveBeenCalledOnce(); + viewer.dispose(); +}); + +it("springs an ordinary device back toward its screen, freezes captured input, and stops rendering at rest", () => { + const { viewer, draw, pending, state } = fixture(); + const initialYaw = state.frames.at(-1)!.yaw!; + viewer.setInteractionActive(true, "orbit"); + viewer.orbit(0.8, 0.4); + draw(80); + const dragged = state.frames.at(-1)!; + expect(Math.abs(dragged.yaw! - initialYaw)).toBeGreaterThan(0.05); + viewer.setInteractionActive(false, "orbit"); + draw(160); + viewer.setInteractionActive(true, "touch"); + draw(180); + const captured = state.frames.at(-1)!; + viewer.orbit(1, 1); + viewer.frameUpdated(); + draw(1000); + expect(state.frames.at(-1)!.yaw).toBe(captured.yaw); + expect(state.frames.at(-1)!.cameraZ).toBe(captured.cameraZ); + expect(pending.size).toBe(0); + viewer.setInteractionActive(false, "touch"); + for (let time = 1016; time <= 4000 && pending.size; time += 16) draw(time); + expect(pending.size).toBe(0); + expect(Math.abs(state.frames.at(-1)!.yaw!)).toBeLessThanOrEqual(Math.PI / 3 + 1e-6); + expect(state.frames.at(-1)!.phone).toBe(dragged.phone); + viewer.dispose(); +}); + +it("resets the device to a square front view after orbiting", () => { + const { viewer, draw, pending, state } = fixture(); + const front = state.frames.at(-1)!.rotation!; + expect(front.angleTo(new Quaternion())).toBeLessThan(1e-6); + viewer.orbit(0.8, 0.4); + draw(80); + expect(state.frames.at(-1)!.rotation!.angleTo(front)).toBeGreaterThan(0.05); + viewer.resetPose(); + for (let time = 96; time <= 3000 && pending.size; time += 16) draw(time); + expect(pending.size).toBe(0); + expect(state.frames.at(-1)!.rotation!.angleTo(new Quaternion())).toBeLessThan(1e-6); + viewer.dispose(); +}); diff --git a/packages/client-runtime/src/device/phoneViewer.ts b/packages/client-runtime/src/device/phoneViewer.ts new file mode 100644 index 000000000000..0794490606d1 --- /dev/null +++ b/packages/client-runtime/src/device/phoneViewer.ts @@ -0,0 +1,332 @@ +import { + AmbientLight, + Box3, + CanvasTexture, + DirectionalLight, + LinearFilter, + Matrix4, + PerspectiveCamera, + Quaternion, + Scene, + SRGBColorSpace, + Vector3, + WebGLRenderer, +} from "three"; +import { + createDeviceModelSlot, + type DeviceModelSource, + type DeviceAccessorySource, +} from "./model.ts"; +import { createImportedPhoneScene, loadDeviceModel } from "./modelScene.ts"; +import { createPhoneScene, phoneDisplayLayout } from "./phoneScene.ts"; +import { createRenderScheduler } from "./renderScheduler.ts"; +import { createDeviceMotion } from "./deviceMotion.ts"; +import { createDeviceFraming } from "./deviceFraming.ts"; +import { nearestDeviceView } from "./deviceViewSnap.ts"; +import type { DeviceScreenSize } from "./stream.ts"; +import { IOS_PHONE_SHAPE, type DeviceShapeProfile } from "./shapeProfile.ts"; + +export interface PhoneViewer { + readonly setModel: (source: DeviceModelSource | null) => void; + readonly setAccessory: (source: DeviceAccessorySource | null) => void; + readonly frameUpdated: () => void; + readonly setScreen: (screen: DeviceScreenSize | null, profile?: DeviceShapeProfile) => void; + readonly resize: (width: number, height: number, pixelRatio: number) => void; + readonly screenPoint: ( + x: number, + y: number, + captured?: boolean, + ) => { x: number; y: number } | null; + readonly orbit: (deltaX: number, deltaY: number) => void; + readonly setInteractionActive: (active: boolean, mode: "touch" | "orbit") => void; + readonly resetPose: () => void; + readonly dispose: () => void; +} + +/** Owns only presentation resources. The caller retains the decoded canvas and the stream connection. */ +export function createPhoneViewer(options: { + readonly canvas: HTMLCanvasElement; + readonly source: HTMLCanvasElement; + readonly onUnavailable: () => void; + readonly onModelError?: (cause: unknown) => void; + readonly onFramingAspect?: (aspect: number) => void; + readonly profile?: DeviceShapeProfile; + readonly model?: DeviceModelSource | null; + readonly accessory?: DeviceAccessorySource | null; +}): PhoneViewer { + const renderer = new WebGLRenderer({ + canvas: options.canvas, + alpha: true, + antialias: true, + powerPreference: "low-power", + }); + renderer.outputColorSpace = SRGBColorSpace; + const makeTexture = () => { + const next = new CanvasTexture(options.source); + next.colorSpace = SRGBColorSpace; + next.minFilter = LinearFilter; + next.magFilter = LinearFilter; + next.generateMipmaps = false; + return next; + }; + let texture = makeTexture(); + let textureWidth = options.source.width; + let textureHeight = options.source.height; + const scene = new Scene(); + const camera = new PerspectiveCamera(32, 1, 0.1, 30); + camera.position.z = 5.5; + const ambient = new AmbientLight(0xffffff, 2.4); + const key = new DirectionalLight(0xe4edff, 5); + key.position.set(-3, 4, 5); + const rim = new DirectionalLight(0xffffff, 4); + rim.position.set(3, 1, -3); + const fill = new DirectionalLight(0x9facd4, 2); + fill.position.set(-2, -2, -4); + scene.add(ambient, key, rim, fill); + + let screen: DeviceScreenSize | null = null; + let layout = phoneDisplayLayout(screen, options.source.width, options.source.height); + let profile = options.profile ?? IOS_PHONE_SHAPE; + let imported: Awaited> | null = null; + let modelSource = options.model ?? null; + let accessory: Awaited> | null = null; + let accessoryBounds: Box3 | null = null; + let phone = createPhoneScene(texture, layout, profile); + scene.add(phone.root); + let disposed = false; + const rest = new Quaternion(); + const motion = createDeviceMotion({ + choose: (rotation) => + nearestDeviceView(rotation, [{ rotation: new Quaternion(), yawLimit: Math.PI / 3 }])! + .rotation, + }); + motion.setPose(rest, performance.now(), true); + const framing = createDeviceFraming(); + const reducedMotion = () => + globalThis.matchMedia?.("(prefers-reduced-motion: reduce)").matches ?? false; + let viewport = { width: 0, height: 0, pixelRatio: 1 }; + let drawingBuffer = { width: 0, height: 0, pixelRatio: 0 }; + + let framingAspect: number | null = null; + const fit = (immediate = false) => { + if (!viewport.width || !viewport.height) return; + camera.aspect = viewport.width / viewport.height; + const bounds = new Box3( + new Vector3(-phone.width / 2, -phone.height / 2, 0), + new Vector3(phone.width / 2, phone.height / 2, 0), + ); + if (imported && accessoryBounds) bounds.union(accessoryBounds); + bounds.applyMatrix4(new Matrix4().makeRotationZ(layout.rotation)); + const size = bounds.getSize(new Vector3()); + const aspect = size.x / size.y; + if (aspect !== framingAspect) { + framingAspect = aspect; + options.onFramingAspect?.(aspect); + } + phone.root.updateMatrixWorld(true); + framing.setBounds( + new Box3().setFromObject(phone.root), + (camera.fov * Math.PI) / 360, + camera.aspect, + performance.now(), + immediate, + ); + applyCamera(); + }; + const applyCamera = () => { + camera.position.set(framing.center.x, framing.center.y, framing.distance()); + camera.lookAt(framing.center.x, framing.center.y, 0); + camera.updateProjectionMatrix(); + }; + const applyPose = () => { + phone.root.quaternion.copy(motion.rotation); + phone.orientation.rotation.z = layout.rotation; + }; + const scheduler = createRenderScheduler(() => { + if (disposed || !viewport.width || !viewport.height) return; + try { + if ( + drawingBuffer.width !== viewport.width || + drawingBuffer.height !== viewport.height || + drawingBuffer.pixelRatio !== viewport.pixelRatio + ) { + // Canvas allocation clears the previous image. Commit it with the redraw, + // rather than exposing an empty buffer between ResizeObserver and the next frame. + renderer.setDrawingBufferSize(viewport.width, viewport.height, viewport.pixelRatio); + drawingBuffer = viewport; + } + const now = performance.now(); + if (motion.advance(now, reducedMotion())) { + applyPose(); + fit(reducedMotion()); + } + framing.advance(now, reducedMotion()); + applyCamera(); + renderer.render(scene, camera); + if (motion.needsFrame() || framing.needsFrame()) scheduler.invalidate(); + } catch { + options.onUnavailable(); + } + }); + const updateLayout = (nextProfile = profile) => { + const next = phoneDisplayLayout(screen, options.source.width, options.source.height); + const resized = + textureWidth !== options.source.width || textureHeight !== options.source.height; + if ( + resized || + nextProfile !== profile || + next.aspect !== layout.aspect || + next.rawLandscape !== layout.rawLandscape || + next.rotation !== layout.rotation + ) { + // The model and renderer survive framebuffer rotation and native resolution changes. + if (resized) { + const previous = texture; + texture = makeTexture(); + textureWidth = options.source.width; + textureHeight = options.source.height; + phone.setDisplay(texture, next); + previous.dispose(); + } + if (!imported && (nextProfile !== profile || next.aspect !== layout.aspect)) { + scene.remove(phone.root); + phone.dispose(); + phone = createPhoneScene(texture, next, nextProfile); + scene.add(phone.root); + } else { + phone.setDisplay(texture, next); + } + layout = next; + profile = nextProfile; + applyPose(); + fit(true); + } + applyPose(); + }; + const modelSlot = createDeviceModelSlot({ + load: loadDeviceModel, + onError: options.onModelError, + install(model) { + // Validate and prepare the next scene before releasing the visible one. + const next = disposed + ? null + : model + ? createImportedPhoneScene(model.asset, texture, layout) + : createPhoneScene(texture, layout, profile); + scene.remove(phone.root); + phone.dispose(); + imported = model; + if (!next) return; + phone = next; + scene.add(phone.root); + if (accessory) { + if (imported) phone.orientation.add(accessory.asset); + else accessory.asset.removeFromParent(); + } + applyPose(); + fit(true); + scheduler.invalidate(); + }, + }); + const accessorySlot = createDeviceModelSlot({ + load: loadDeviceModel, + onError: options.onModelError, + install(model) { + const bounds = model ? new Box3().setFromObject(model.asset) : null; + if ( + bounds && + (bounds.isEmpty() || + ![...bounds.min.toArray(), ...bounds.max.toArray()].every(Number.isFinite)) + ) + throw new Error("Device accessory has invalid bounds"); + accessory?.asset.removeFromParent(); + accessory = model; + accessoryBounds = bounds; + if (disposed) return; + if (imported && accessory) phone.orientation.add(accessory.asset); + fit(true); + scheduler.invalidate(); + }, + }); + const setAccessory = (source: DeviceAccessorySource | null) => { + accessorySlot.set(source?.modelId === modelSource?.id ? source : null); + }; + const contextLost = (event: Event) => { + event.preventDefault(); + options.onUnavailable(); + }; + options.canvas.addEventListener("webglcontextlost", contextLost); + applyPose(); + modelSlot.set(options.model ?? null); + setAccessory(options.accessory ?? null); + return { + setModel(source) { + if (source?.id !== modelSource?.id || source?.url !== modelSource?.url) + accessorySlot.set(null); + modelSource = source; + modelSlot.set(source); + }, + setAccessory, + frameUpdated() { + if (disposed) return; + updateLayout(); + texture.needsUpdate = true; + scheduler.invalidate(); + }, + setScreen(next, nextProfile = profile) { + if (disposed) return; + screen = next; + updateLayout(nextProfile); + scheduler.invalidate(); + }, + resize(width, height, pixelRatio) { + if (disposed) return; + if (![width, height, pixelRatio].every(Number.isFinite) || width <= 0 || height <= 0) return; + const ratio = Math.min(2, Math.max(1, pixelRatio)); + if (viewport.width === width && viewport.height === height && viewport.pixelRatio === ratio) + return; + viewport = { width, height, pixelRatio: ratio }; + fit(true); + scheduler.invalidate(); + }, + screenPoint(x, y, captured = false) { + if (disposed) return null; + applyPose(); + return phone.screenPoint(x, y, camera, captured); + }, + orbit(deltaX, deltaY) { + if (disposed) return; + motion.orbit(deltaX * viewport.width, deltaY * viewport.height, performance.now()); + scheduler.invalidate(); + }, + setInteractionActive(active, mode) { + if (disposed) return; + const now = performance.now(); + if (mode === "orbit") motion.dragActive(active, now); + else { + motion.hold(active, now); + framing.hold(active, now); + } + scheduler.invalidate(); + }, + resetPose() { + if (disposed) return; + motion.reset(rest, performance.now()); + scheduler.invalidate(); + }, + dispose() { + if (disposed) return; + disposed = true; + scheduler.dispose(); + const hadImported = imported !== null; + accessorySlot.dispose(); + modelSlot.dispose(); + options.canvas.removeEventListener("webglcontextlost", contextLost); + scene.remove(phone.root); + if (!hadImported) phone.dispose(); + texture.dispose(); + renderer.dispose(); + renderer.forceContextLoss(); + }, + }; +} diff --git a/packages/client-runtime/src/device/renderScheduler.test.ts b/packages/client-runtime/src/device/renderScheduler.test.ts new file mode 100644 index 000000000000..9a3e98db7e42 --- /dev/null +++ b/packages/client-runtime/src/device/renderScheduler.test.ts @@ -0,0 +1,27 @@ +import { expect, it, vi } from "vite-plus/test"; +import { createRenderScheduler } from "./renderScheduler.ts"; + +it("coalesces frame and pointer invalidations, then stops scheduling when idle or disposed", () => { + const callbacks: FrameRequestCallback[] = []; + const request = vi.fn((callback: FrameRequestCallback) => { + callbacks.push(callback); + return callbacks.length; + }); + const cancel = vi.fn(); + const render = vi.fn(); + const scheduler = createRenderScheduler(render, request, cancel); + scheduler.invalidate(); + scheduler.invalidate(); + expect(request).toHaveBeenCalledTimes(1); + callbacks[0]?.(0); + expect(render).toHaveBeenCalledTimes(1); + expect(request).toHaveBeenCalledTimes(1); + scheduler.invalidate(); + scheduler.dispose(); + scheduler.dispose(); + scheduler.invalidate(); + callbacks[1]?.(0); + expect(cancel).toHaveBeenCalledExactlyOnceWith(2); + expect(render).toHaveBeenCalledTimes(1); + expect(request).toHaveBeenCalledTimes(2); +}); diff --git a/packages/client-runtime/src/device/renderScheduler.ts b/packages/client-runtime/src/device/renderScheduler.ts new file mode 100644 index 000000000000..203b15bfc054 --- /dev/null +++ b/packages/client-runtime/src/device/renderScheduler.ts @@ -0,0 +1,23 @@ +/** Coalesces invalidations into one render. No work is scheduled while the view is idle. */ +export function createRenderScheduler( + render: () => void, + request: (callback: FrameRequestCallback) => number = requestAnimationFrame, + cancel: (id: number) => void = cancelAnimationFrame, +) { + let pending: number | null = null; + let disposed = false; + return { + invalidate() { + if (disposed || pending !== null) return; + pending = request(() => { + pending = null; + if (!disposed) render(); + }); + }, + dispose() { + disposed = true; + if (pending !== null) cancel(pending); + pending = null; + }, + }; +} diff --git a/packages/client-runtime/src/device/screenshot.test.ts b/packages/client-runtime/src/device/screenshot.test.ts new file mode 100644 index 000000000000..b4530837bd36 --- /dev/null +++ b/packages/client-runtime/src/device/screenshot.test.ts @@ -0,0 +1,50 @@ +import { afterEach, expect, it, vi } from "vite-plus/test"; +import { captureDeviceScreenshot, DeviceScreenshotError } from "./screenshot.ts"; + +afterEach(() => vi.unstubAllGlobals()); +const target = { + platform: "android" as const, + deviceId: "phone with spaces", + access: { + httpBase: "https://remote.example/api/device-hub/hosts/remote", + wsBase: "wss://remote.example", + query: { ticket: "capture-ticket" }, + credentials: false, + }, +}; + +it("captures native bytes from the selected remote host with its media credentials", async () => { + const fetch = vi.fn( + async () => + new Response(new Uint8Array([137, 80, 78, 71]), { headers: { "Content-Type": "image/png" } }), + ); + vi.stubGlobal("fetch", fetch); + const image = await captureDeviceScreenshot(target, new AbortController().signal); + expect(image.type).toBe("image/png"); + expect(Array.from(new Uint8Array(await image.arrayBuffer()))).toEqual([137, 80, 78, 71]); + expect(fetch).toHaveBeenCalledWith( + "https://remote.example/api/device-hub/hosts/remote/vendor/serve-emu/api/screenshot?device=phone%20with%20spaces&ticket=capture-ticket", + expect.objectContaining({ method: "POST", credentials: "omit" }), + ); +}); + +it("preserves authorization failures and cancels an in-flight capture when its owner detaches", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("expired", { status: 401 })), + ); + await expect(captureDeviceScreenshot(target, new AbortController().signal)).rejects.toMatchObject( + new DeviceScreenshotError(401), + ); + vi.stubGlobal( + "fetch", + (_url: string, init: RequestInit) => + new Promise((_resolve, reject) => { + init.signal?.addEventListener("abort", () => reject(init.signal?.reason), { once: true }); + }), + ); + const controller = new AbortController(); + const capture = captureDeviceScreenshot(target, controller.signal); + controller.abort(new Error("detached")); + await expect(capture).rejects.toThrow("detached"); +}); diff --git a/packages/client-runtime/src/device/screenshot.ts b/packages/client-runtime/src/device/screenshot.ts new file mode 100644 index 000000000000..547fabfaa381 --- /dev/null +++ b/packages/client-runtime/src/device/screenshot.ts @@ -0,0 +1,30 @@ +// @effect-diagnostics globalFetch:off - This browser transport runs without an Effect runtime, like the live stream. +import { withDeviceHubQuery } from "./hubAccess.ts"; +import type { DeviceStreamTarget } from "./stream.ts"; + +export class DeviceScreenshotError extends Error { + readonly status: number; + + constructor(status: number) { + super(`Screenshot capture failed (${status}). Try again.`); + this.status = status; + } +} + +/** Captures native pixels through the same host and credentials as the live stream, in either presentation. */ +export async function captureDeviceScreenshot(target: DeviceStreamTarget, signal: AbortSignal) { + const vendor = target.platform === "ios" ? "serve-sim" : "serve-emu"; + const url = withDeviceHubQuery( + `${target.access.httpBase}/vendor/${vendor}/api/screenshot?device=${encodeURIComponent(target.deviceId)}`, + target.access, + ); + const response = await fetch(url, { + method: "POST", + credentials: target.access.credentials ? "include" : "omit", + signal: AbortSignal.any([signal, AbortSignal.timeout(20_000)]), + }); + if (!response.ok) throw new DeviceScreenshotError(response.status); + const image = await response.blob(); + if (!image.size) throw new Error("The device returned an empty screenshot. Try again."); + return image; +} diff --git a/packages/client-runtime/src/device/shapeProfile.test.ts b/packages/client-runtime/src/device/shapeProfile.test.ts new file mode 100644 index 000000000000..7f612769cccf --- /dev/null +++ b/packages/client-runtime/src/device/shapeProfile.test.ts @@ -0,0 +1,27 @@ +import { expect, it } from "vite-plus/test"; +import { resolveDeviceShape } from "./shapeProfile.ts"; + +it("chooses tablet and phone families without mistaking display rotation for device shape", () => { + expect( + resolveDeviceShape({ platform: "ios", name: "iPad Pro 11-inch (M5)", portraitAspect: 0.75 }).id, + ).toBe("ios-tablet"); + expect( + resolveDeviceShape({ platform: "ios", name: "iPhone 18 Pro", portraitAspect: 0.46 }).id, + ).toBe("ios-phone"); + expect( + resolveDeviceShape({ platform: "android", name: "Pixel 9", portraitAspect: 0.45 }).id, + ).toBe("android-phone"); + expect( + resolveDeviceShape({ platform: "android", name: "Pixel Tablet", portraitAspect: 0.625 }).id, + ).toBe("android-tablet"); +}); + +it("uses the screen shape for renamed devices and a phone fallback before metadata arrives", () => { + expect(resolveDeviceShape({ platform: "ios", name: "Julius", portraitAspect: 0.75 }).id).toBe( + "ios-tablet", + ); + expect(resolveDeviceShape({ platform: "android", portraitAspect: 0.45 }).id).toBe( + "android-phone", + ); + expect(resolveDeviceShape({ platform: "ios", portraitAspect: Number.NaN }).id).toBe("ios-phone"); +}); diff --git a/packages/client-runtime/src/device/shapeProfile.ts b/packages/client-runtime/src/device/shapeProfile.ts new file mode 100644 index 000000000000..5ea04778e3ab --- /dev/null +++ b/packages/client-runtime/src/device/shapeProfile.ts @@ -0,0 +1,124 @@ +import type { DevicePlatform } from "@t3tools/contracts"; + +export interface DeviceShapeProfile { + readonly id: "ios-phone" | "ios-tablet" | "android-phone" | "android-tablet"; + readonly bezel: number; + readonly bodyRadius: number; + readonly screenRadius: number; + readonly depth: number; + readonly backColor: number; + readonly buttons: ReadonlyArray<{ + readonly edge: "left" | "right" | "top"; + readonly offset: number; + readonly length: number; + }>; + readonly camera: { + readonly width: number; + readonly height: number; + readonly insetX: number; + readonly insetY: number; + readonly lensRadius: number; + readonly lenses: ReadonlyArray; + readonly flash: readonly [number, number] | null; + }; +} + +/** Original family silhouettes, rather than claims to reproduce individual hardware models. */ +export const IOS_PHONE_SHAPE: DeviceShapeProfile = { + id: "ios-phone", + bezel: 0.055, + bodyRadius: 0.15, + screenRadius: 0.105, + depth: 0.085, + backColor: 0x424b5d, + buttons: [ + { edge: "right", offset: 0.35, length: 0.3 }, + { edge: "left", offset: 0.48, length: 0.18 }, + { edge: "left", offset: 0.22, length: 0.18 }, + ], + camera: { + width: 0.39, + height: 0.44, + insetX: 0.25, + insetY: 0.29, + lensRadius: 0.068, + lenses: [ + [-0.08, 0.095], + [0.08, -0.095], + ], + flash: [0.085, 0.11], + }, +}; + +export const IOS_TABLET_SHAPE: DeviceShapeProfile = { + id: "ios-tablet", + bezel: 0.065, + bodyRadius: 0.105, + screenRadius: 0.045, + depth: 0.055, + backColor: 0x9ca5af, + buttons: [ + { edge: "top", offset: 0.5, length: 0.15 }, + { edge: "right", offset: 0.78, length: 0.13 }, + { edge: "right", offset: 0.58, length: 0.13 }, + ], + camera: { + width: 0.19, + height: 0.19, + insetX: 0.15, + insetY: 0.15, + lensRadius: 0.045, + lenses: [[0, 0]], + flash: null, + }, +}; + +export const ANDROID_PHONE_SHAPE: DeviceShapeProfile = { + id: "android-phone", + bezel: 0.035, + bodyRadius: 0.115, + screenRadius: 0.08, + depth: 0.085, + backColor: 0x344449, + buttons: [ + { edge: "right", offset: 0.2, length: 0.24 }, + { edge: "right", offset: 0.65, length: 0.32 }, + ], + camera: { + width: 0.24, + height: 0.47, + insetX: 0.18, + insetY: 0.29, + lensRadius: 0.056, + lenses: [ + [0, 0.11], + [0, -0.11], + ], + flash: [0.1, 0], + }, +}; + +const ANDROID_TABLET_SHAPE: DeviceShapeProfile = { + ...IOS_TABLET_SHAPE, + id: "android-tablet", + backColor: 0x697b80, +}; + +/** Names identify a family when available; wide unknown screens get a generic tablet shell. */ +export function resolveDeviceShape(options: { + platform: DevicePlatform; + name?: string; + portraitAspect: number; +}): DeviceShapeProfile { + const name = options.name ?? ""; + const namedTablet = /\b(ipad|tablet)\b/i.test(name); + const namedPhone = /\b(iphone|phone)\b/i.test(name); + const tablet = namedTablet || (!namedPhone && options.portraitAspect >= 0.6); + return options.platform === "ios" + ? tablet + ? IOS_TABLET_SHAPE + : IOS_PHONE_SHAPE + : tablet + ? ANDROID_TABLET_SHAPE + : ANDROID_PHONE_SHAPE; +} diff --git a/packages/client-runtime/src/device/stream.test.ts b/packages/client-runtime/src/device/stream.test.ts index b947f5c17a74..7ea5aca27abf 100644 --- a/packages/client-runtime/src/device/stream.test.ts +++ b/packages/client-runtime/src/device/stream.test.ts @@ -111,6 +111,7 @@ describe("native device stream transport", () => { expect(imageUrl.searchParams.get("hostId")).toBe("ssh-host"); expect(fetch).toHaveBeenCalledTimes(1); socket.onopen?.(); + client.setOrientation("landscape_right"); client.pressButton("home"); client.sendTouch("begin", 0.25, 0.75); const messages = socket.send.mock.calls.slice(1).map(([bytes]) => { @@ -121,6 +122,7 @@ describe("native device stream transport", () => { }; }); expect(messages).toEqual([ + { tag: 0x07, body: { orientation: "landscape_right" } }, { tag: 0x04, body: { button: "home" } }, { tag: 0x03, body: { type: "begin", x: 0.25, y: 0.75 } }, ]); @@ -140,6 +142,7 @@ describe("native device stream transport", () => { client.sendTouch("begin", 0.2, 0.8); client.sendTouch("move", 0.2, 0.4); client.sendTouch("end", 0.2, 0.1); + client.setOrientation("landscape_right"); client.pressButton("home"); client.pressButton("appSwitcher"); expect( diff --git a/packages/client-runtime/src/device/stream.ts b/packages/client-runtime/src/device/stream.ts index 9275f7be464b..cc849ea3e15e 100644 --- a/packages/client-runtime/src/device/stream.ts +++ b/packages/client-runtime/src/device/stream.ts @@ -19,6 +19,7 @@ * The decoder only runs while frames arrive and the viewer is attached; a * hidden panel calls `stop()` so an idle device costs nothing on the GPU. */ +import { createCanvasFrameSink, type DeviceFrameSink } from "./frame.ts"; import { type DeviceHubAccess, withDeviceHubQuery } from "./hubAccess.ts"; import type { DevicePlatform } from "@t3tools/contracts"; @@ -217,6 +218,7 @@ export interface DeviceStreamClient { readonly sendKey: (event: KeyboardEvent, phase: "down" | "up") => void; readonly pressButton: (button: DeviceHardwareButton) => void; readonly rotate: () => void; + readonly setOrientation: (orientation: DeviceScreenSize["orientation"]) => void; } const HID_USAGE_BY_CODE: Readonly> = { @@ -282,10 +284,11 @@ const IOS_ORIENTATIONS: ReadonlyArray = [ export function createDeviceStreamClient( target: DeviceStreamTarget, - canvas: HTMLCanvasElement, + output: HTMLCanvasElement | DeviceFrameSink, events: DeviceStreamEvents, ): DeviceStreamClient { const { access, platform, deviceId } = target; + const sink = "present" in output ? output : createCanvasFrameSink(output); const vendor = platform === "ios" ? "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/vendor/serve-sim" : "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/vendor/serve-emu"; const device = encodeURIComponent(deviceId); const httpUrl = (path: string) => @@ -402,17 +405,14 @@ export function createDeviceStreamClient( const paint = (source: CanvasImageSource, width: number, height: number) => { if (stopped) return; - if (canvas.width !== width || canvas.height !== height) { - canvas.width = width; - canvas.height = height; - if (platform === "android") { - screen = { width, height, orientation: width > height ? "landscape_left" : "portrait" }; - events.onScreen(screen); - } + if (platform === "android" && (screen?.width !== width || screen.height !== height)) { + screen = { width, height, orientation: width > height ? "landscape_left" : "portrait" }; + events.onScreen(screen); + } + if (!sink.present(source, width, height)) { + fail("Could not display the device stream. Reconnect to try again."); + return; } - const context = canvas.getContext("2d"); - if (!context) return fail("Could not display the device stream. Reconnect to try again."); - context.drawImage(source, 0, 0, width, height); frameReceived(); }; @@ -861,5 +861,8 @@ export function createDeviceStreamClient( IOS_ORIENTATIONS[(IOS_ORIENTATIONS.indexOf(current) + 1) % IOS_ORIENTATIONS.length]!; send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next })); }, + setOrientation: (orientation) => { + if (platform === "ios") send(taggedJson(IOS_MSG_ORIENTATION, { orientation })); + }, }; } diff --git a/packages/client-runtime/src/device/streamFrames.test.ts b/packages/client-runtime/src/device/streamFrames.test.ts new file mode 100644 index 000000000000..8021fe72647a --- /dev/null +++ b/packages/client-runtime/src/device/streamFrames.test.ts @@ -0,0 +1,97 @@ +import { afterEach, expect, it, vi } from "vite-plus/test"; +import { createDeviceStreamClient } from "./stream.ts"; + +afterEach(() => vi.unstubAllGlobals()); + +it("delivers borrowed decoded frames, reports presentation failure, and discards late output", async () => { + let decoderOutput: VideoFrameOutputCallback | undefined; + let resolveOutput!: () => void; + const outputReady = new Promise((resolve) => { + resolveOutput = resolve; + }); + class Decoder { + static isConfigSupported = async () => ({ supported: true }); + state = "unconfigured"; + constructor(options: VideoDecoderInit) { + decoderOutput = options.output; + resolveOutput(); + } + configure() { + this.state = "configured"; + } + close() { + this.state = "closed"; + } + } + vi.stubGlobal("VideoDecoder", Decoder); + vi.stubGlobal("EncodedVideoChunk", vi.fn()); + vi.stubGlobal( + "WebSocket", + class { + close() {} + }, + ); + const envelope = new Uint8Array([0, 0, 0, 5, 1, 1, 0x64, 0, 0x1f]); + vi.stubGlobal( + "fetch", + vi.fn( + async (url: string) => + new Response( + url.includes("avcc") + ? new ReadableStream({ + start(controller) { + controller.enqueue(envelope); + }, + }) + : "seed", + ), + ), + ); + const present = vi.fn(() => true); + const events = { + onStatus: vi.fn(), + onScreen: vi.fn(), + onUnauthorized: vi.fn(), + onMjpegFallback: vi.fn(), + onInputConnected: vi.fn(), + }; + const client = createDeviceStreamClient( + { + platform: "ios", + deviceId: "phone", + access: { + httpBase: "https://test", + wsBase: "wss://test", + credentials: false, + query: {}, + }, + }, + { present }, + events, + ); + client.start(); + await outputReady; + const frame = { + displayWidth: 1170, + displayHeight: 2532, + close: vi.fn(), + } as unknown as VideoFrame; + decoderOutput?.(frame); + expect(present).toHaveBeenCalledExactlyOnceWith(frame, 1170, 2532); + expect(frame.close).toHaveBeenCalledOnce(); + present.mockImplementationOnce(() => { + throw new Error("renderer failed"); + }); + expect(() => decoderOutput?.(frame)).toThrow("renderer failed"); + expect(frame.close).toHaveBeenCalledTimes(2); + present.mockReturnValueOnce(false); + decoderOutput?.(frame); + expect(events.onStatus).toHaveBeenLastCalledWith( + "error", + "Could not display the device stream. Reconnect to try again.", + ); + expect(frame.close).toHaveBeenCalledTimes(3); + decoderOutput?.(frame); + expect(present).toHaveBeenCalledTimes(3); + expect(frame.close).toHaveBeenCalledTimes(4); +}); diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index c67ecd27c0e1..ffa8661eb9c8 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -1125,6 +1125,7 @@ export interface DesktopBridge { getClientPlatform?: () => string; setNotificationBadge?: (badge: { count: number; image: string | null }) => Promise; onNotificationBadgeClear?: (listener: () => void) => () => void; + onTrackpadScrollEnd?: (listener: () => void) => () => void; /** * The OS locale as a BCP-47 tag, which the renderer cannot read for itself: * the packaged app ships only the `en-US` Chromium locale pak, so diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index 59387fdacfd9..431e83189d22 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -590,6 +590,15 @@ export const PreviewAutomationHostFocus = Schema.Struct({ ...PreviewAutomationHostIdentity.fields, connectionId: PreviewAutomationConnectionId, focused: Schema.Boolean, + liveTabs: Schema.optional( + Schema.Array( + Schema.Struct({ + threadId: ThreadId, + tabId: PreviewTabId, + visible: Schema.optional(Schema.Boolean), + }), + ), + ), }); export type PreviewAutomationHostFocus = typeof PreviewAutomationHostFocus.Type; diff --git a/packages/shared/src/delimitedPreview.test.ts b/packages/shared/src/delimitedPreview.test.ts index 303e9042855c..0188f9da5023 100644 --- a/packages/shared/src/delimitedPreview.test.ts +++ b/packages/shared/src/delimitedPreview.test.ts @@ -21,6 +21,35 @@ describe("delimited file previews", () => { ["", "three"], ]); }); + describe.each([",", "\t"] as const)("with delimiter %j", (delimiter) => { + it("preserves a final quoted empty record without requiring a line ending", () => { + for (const ending of ["", "\n", "\r\n"]) { + expect(parseDelimitedPreview(`name\r\nAlice\r\n""${ending}`, delimiter)).toEqual({ + rows: [["name"], ["Alice"], [""]], + truncated: false, + }); + } + expect(parseDelimitedPreview('""', delimiter)).toEqual({ + rows: [[""]], + truncated: false, + }); + }); + it("does not invent records for empty input or a trailing line ending", () => { + for (const prefix of ["", "\ufeff"]) { + expect(parseDelimitedPreview(prefix, delimiter).rows).toEqual([]); + expect(parseDelimitedPreview(`${prefix}name\r\n`, delimiter).rows).toEqual([["name"]]); + expect(parseDelimitedPreview(`${prefix}""`, delimiter).rows).toEqual([[""]]); + } + }); + it("keeps a quoted empty record at the row limit", () => { + const text = `${"name\n".repeat(99)}""`; + const preview = parseDelimitedPreview(text, delimiter); + expect(preview.rows).toHaveLength(100); + expect(preview.rows.at(-1)).toEqual([""]); + expect(preview.truncated).toBe(false); + expect(parseDelimitedPreview(`${text}\nextra`, delimiter).truncated).toBe(true); + }); + }); it("bounds rows, columns and cell length and reports partial content", () => { const table = parseDelimitedPreview( Array.from({ length: 101 }, () => diff --git a/packages/shared/src/delimitedPreview.ts b/packages/shared/src/delimitedPreview.ts index d4e24409a06a..1cfbd41a7812 100644 --- a/packages/shared/src/delimitedPreview.ts +++ b/packages/shared/src/delimitedPreview.ts @@ -16,12 +16,13 @@ export function parseDelimitedPreview(text: string, delimiter: "," | "\t") { let cell = ""; let quoted = false; let truncated = false; + let rowStart = text.charCodeAt(0) === 0xfeff ? 1 : 0; const endCell = () => { if (row.length < 30) row.push(cell); else truncated = true; cell = ""; }; - for (let index = text.charCodeAt(0) === 0xfeff ? 1 : 0; index < text.length; index++) { + for (let index = rowStart; index < text.length; index++) { const char = text[index]; if (char === '"') { if (quoted && text[index + 1] === '"') { @@ -41,12 +42,13 @@ export function parseDelimitedPreview(text: string, delimiter: "," | "\t") { rows.push(row); row = []; if (char === "\r" && text[index + 1] === "\n") index++; + rowStart = index + 1; if (rows.length === 100) return { rows, truncated: truncated || index < text.length - 1 }; } } else if (cell.length < 2000) cell += char; else truncated = true; } - if (cell.length || row.length) { + if (rowStart < text.length) { endCell(); rows.push(row); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5df6021f7131..3aba8fad0369 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -886,6 +886,9 @@ importers: remark-parse: specifier: ^11.0.0 version: 11.0.0 + three: + specifier: 0.180.0 + version: 0.180.0 unified: specifier: ^11.0.5 version: 11.0.5 @@ -893,6 +896,9 @@ importers: '@effect/vitest': specifier: 4.0.0-rc.115 version: 4.0.0-rc.115(patch_hash=172bd95ef73a13d322888984415d5d18d9d2aa8e13fa529eac235efac837e902)(effect@4.0.0-rc.115(patch_hash=0dfc4bb8ebd80fb3e06b91ef61346f5259517ab0f2437644fe95ae531084b1f5)) + '@types/three': + specifier: 0.180.0 + version: 0.180.0 micromark-util-types: specifier: ^2.0.2 version: 2.0.2 @@ -2042,6 +2048,9 @@ packages: '@types/react': optional: true + '@dimforge/rapier3d-compat@0.12.0': + resolution: {integrity: sha512-uekIGetywIgopfD97oDL5PfeezkFpNhwlzlaEYNOA0N6ghdsOvh/HYjSMek5Q2O1PYvRSDFcqFVJl4r4ZBwOow==} + '@distilled.cloud/aws@1.0.0-rc.12': resolution: {integrity: sha512-L4TTEwMKB1ymXbYUqvYeoy5UWgmJtqQSzry2qWtAvD2ccwVGy0F7cFViEIYQk7boF4ZIP6SfhAls4+y1fVIi/g==} peerDependencies: @@ -5246,6 +5255,9 @@ packages: '@tiptap/starter-kit@3.31.3': resolution: {integrity: sha512-WKof9RewdmGHvWJ1wn0/HVNG2mV+HOgVRyJkKekuM9fgr6BZAAH/xZsWE1eon+94JnQ+KtK2ThydXQM/qc6b2A==} + '@tweenjs/tween.js@23.1.3': + resolution: {integrity: sha512-vJmvvwFxYuGnF2axRtPYocag6Clbb5YS7kLL+SO/TeVFzHqDIWrNKYtcsPMibjDx9O+bu+psAy9NKfWklassUA==} + '@tybys/wasm-util@0.10.3': resolution: {integrity: sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==} @@ -5383,9 +5395,15 @@ packages: '@types/serve-static@2.2.0': resolution: {integrity: sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==} + '@types/stats.js@0.17.4': + resolution: {integrity: sha512-jIBvWWShCvlBqBNIZt0KAshWpvSjhkwkEu4ZUcASoAvhmrgAUI2t1dXrjSL4xXVLB4FznPrIsX3nKXFl/Dt4vA==} + '@types/statuses@2.0.6': resolution: {integrity: sha512-xMAgYwceFhRA2zY+XbEA7mxYbA093wdiW8Vu6gZPGWy9cmOyU9XesH1tNcEWsKFd5Vzrqx5T3D38PWx1FIIXkA==} + '@types/three@0.180.0': + resolution: {integrity: sha512-ykFtgCqNnY0IPvDro7h+9ZeLY+qjgUWv+qEvUt84grhenO60Hqd4hScHE7VTB9nOQ/3QM8lkbNE+4vKjEpUxKg==} + '@types/unist@2.0.11': resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} @@ -5398,6 +5416,9 @@ packages: '@types/webidl-conversions@7.0.3': resolution: {integrity: sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==} + '@types/webxr@0.5.24': + resolution: {integrity: sha512-h8fgEd/DpoS9CBrjEQXR+dIDraopAEfu4wYVNY2tEPwk60stPWhvZMf4Foo5FakuQ7HFZoa8WceaWFervK2Ovg==} + '@types/whatwg-url@11.0.5': resolution: {integrity: sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==} @@ -5730,6 +5751,9 @@ packages: '@vscode/l10n@0.0.18': resolution: {integrity: sha512-KYSIHVmslkaCDyw013pphY+d7x1qV8IZupYfeIfzNA+nsaWHbn5uPuQRvdRFsa9zFzGeudPuoGoZ1Op4jrJXIQ==} + '@webgpu/types@0.1.74': + resolution: {integrity: sha512-lgiI4hbuLcI9unnm2cL/tvCaQU45dp0xcLWJh5uB/9MBGvIA4F8XIk7nSiBeg+K4xWGYwgZKn80LmERI5CxoTA==} + '@xmldom/xmldom@0.8.15': resolution: {integrity: sha512-/5NV/vDALVFDXgLmfsy9TRCBlKwO2LNBFzpzvb9iIj+jR+eSc6DLYYvVOdivT/jm7MtU6TebYuRmzEOI7w40UA==} engines: {node: '>=10.0.0'} @@ -8492,6 +8516,9 @@ packages: merge-stream@2.0.0: resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} + meshoptimizer@0.22.0: + resolution: {integrity: sha512-IebiK79sqIy+E4EgOr+CAw+Ke8hAspXKzBd0JdgEmPHiAwmvEj2S4h1rfvo+o/BnfEYd/jAOg5IeeIjzlzSnDg==} + metro-babel-transformer@0.84.4: resolution: {integrity: sha512-rvCfz8snl9h20VcvpOHxZuHP1SlAkv4HXbzw7nyyVwu6Eqo5PRerbakQ9XmUCOsRy70spJ37O+G1TK8oMzo48g==} engines: {node: ^20.19.4 || ^22.13.0 || ^24.3.0 || >= 25.0.0} @@ -10253,6 +10280,9 @@ packages: engines: {node: '>=10'} hasBin: true + three@0.180.0: + resolution: {integrity: sha512-o+qycAMZrh+TsE01GqWUxUIKR1AL0S8pq7zDkYOQw8GqfX8b8VoCKYUoHbhiX5j+7hr8XsuHDVU6+gkQJQKg9w==} + throat@5.0.0: resolution: {integrity: sha512-fcwX4mndzpLQKBS1DVYhGAcYaYt7vsHNIvQV+WXMvnow5cgjPphq5CaayLaGsjRdSCKZFNGt7/GYAuXaNOiYCA==} @@ -12252,6 +12282,8 @@ snapshots: optionalDependencies: '@types/react': 19.2.16 + '@dimforge/rapier3d-compat@0.12.0': {} + '@distilled.cloud/aws@1.0.0-rc.12(effect@4.0.0-rc.115(patch_hash=0dfc4bb8ebd80fb3e06b91ef61346f5259517ab0f2437644fe95ae531084b1f5))': dependencies: '@aws-sdk/types': 3.974.5 @@ -15476,6 +15508,8 @@ snapshots: '@tiptap/extensions': 3.31.3(@tiptap/core@3.31.3(@tiptap/pm@3.31.3))(@tiptap/pm@3.31.3) '@tiptap/pm': 3.31.3 + '@tweenjs/tween.js@23.1.3': {} + '@tybys/wasm-util@0.10.3': dependencies: tslib: 2.8.1 @@ -15647,9 +15681,21 @@ snapshots: '@types/http-errors': 2.0.5 '@types/node': 24.12.4 + '@types/stats.js@0.17.4': {} + '@types/statuses@2.0.6': optional: true + '@types/three@0.180.0': + dependencies: + '@dimforge/rapier3d-compat': 0.12.0 + '@tweenjs/tween.js': 23.1.3 + '@types/stats.js': 0.17.4 + '@types/webxr': 0.5.24 + '@webgpu/types': 0.1.74 + fflate: 0.8.3 + meshoptimizer: 0.22.0 + '@types/unist@2.0.11': {} '@types/unist@3.0.3': {} @@ -15659,6 +15705,8 @@ snapshots: '@types/webidl-conversions@7.0.3': optional: true + '@types/webxr@0.5.24': {} + '@types/whatwg-url@11.0.5': dependencies: '@types/webidl-conversions': 7.0.3 @@ -15996,6 +16044,8 @@ snapshots: '@vscode/l10n@0.0.18': {} + '@webgpu/types@0.1.74': {} + '@xmldom/xmldom@0.8.15': {} '@xmldom/xmldom@0.9.12': {} @@ -19112,6 +19162,8 @@ snapshots: merge-stream@2.0.0: {} + meshoptimizer@0.22.0: {} + metro-babel-transformer@0.84.4: dependencies: '@babel/core': 7.29.7 @@ -21641,6 +21693,8 @@ snapshots: commander: 2.20.3 source-map-support: 0.5.21 + three@0.180.0: {} + throat@5.0.0: {} through@2.3.8: {} diff --git a/scripts/convert-device-model.py b/scripts/convert-device-model.py new file mode 100644 index 000000000000..cba2d9c98ed4 --- /dev/null +++ b/scripts/convert-device-model.py @@ -0,0 +1,108 @@ +# Run with Blender 4.5.9: +# blender --background --python scripts/convert-device-model.py -- +# Then optimize using @gltf-transform/cli 4.2.1 with --compress false --simplify false +# --join false --palette false --flatten false --texture-size 1024 --texture-compress webp. +import hashlib +import json +import sys +from pathlib import Path + +import bpy +import numpy as np +from mathutils import Matrix, Vector + +model_id, source, output = sys.argv[sys.argv.index("--") + 1 :] +manifest_path = ( + Path(__file__).resolve().parents[1] + / "apps/web/src/components/device/models/sources.json" +) +manifest = json.loads(manifest_path.read_text()) +model = next(m for m in manifest["models"] if m["id"] == model_id) +if hashlib.sha256(Path(source).read_bytes()).hexdigest() != model["sourceSha256"]: + raise ValueError("Source checksum differs from the recorded Apple asset") +body_name, screen_name = model["bodyNode"], model["screenNode"] +bpy.ops.wm.read_factory_settings(use_empty=True) +bpy.ops.wm.usd_import(filepath=source, import_materials=True) +body = bpy.data.objects[model.get("accessoryNode", body_name)] +screen = bpy.data.objects[screen_name] +objects = [o for o in body.children_recursive if o.type == "MESH"] +if body.type == "MESH": + objects.append(body) +uv = screen.data.uv_layers.active +points = [] +coords = [] +for loop in screen.data.loops: + points.append( + tuple(screen.matrix_world @ screen.data.vertices[loop.vertex_index].co) + ) + coords.append((*uv.data[loop.index].uv, 1)) +coef = np.linalg.lstsq(np.array(coords), np.array(points), rcond=None)[0] +right = Vector(coef[0]).normalized() +up = Vector(coef[1]).normalized() +normal = right.cross(up).normalized() +# The authored UVs establish the device's front and its physical portrait axes. +# No body proportions are changed. The runtime uses a standard planar framebuffer UV. +rot = Matrix((right, up, normal)).to_4x4() +world = [rot @ Vector(p) for p in points] +lo = Vector(tuple(min(p[i] for p in world) for i in range(3))) +hi = Vector(tuple(max(p[i] for p in world) for i in range(3))) +center = (lo + hi) / 2 +scale = 2.2 / (hi.y - lo.y) +transform = ( + Matrix.Translation(Vector((0, 0, 0.043))) + @ Matrix.Diagonal(Vector((scale, scale, scale, 1))) + @ Matrix.Translation(-center) + @ rot +) +for o in objects: + matrix = transform @ o.matrix_world + o.parent = None + o.matrix_world = Matrix.Identity(4) + o.data.transform(matrix) + # Blender is Z-up; glTF export converts to Y-up. Store canonical Y-up geometry as Blender Z-up. + o.data.transform(Matrix.Rotation(np.pi / 2, 4, "X")) + if o == screen: + o.name = "device-screen" + o.data.name = "device-screen" + for loop in o.data.loops: + p = o.data.vertices[loop.vertex_index].co + uv.data[loop.index].uv = ( + (p.x / ((hi.x - lo.x) * scale)) + 0.5, + p.z / 2.2 + 0.5, + ) + # This material is replaced by the viewer and must not retain the marketing screenshot. + mat = bpy.data.materials.new("device-screen") + mat.diffuse_color = (0, 0, 0, 1) + o.data.materials.clear() + o.data.materials.append(mat) +for o in list(bpy.data.objects): + if o not in objects: + bpy.data.objects.remove(o, do_unlink=True) +for image in bpy.data.images: + if image.size[0] > 1024 or image.size[1] > 1024: + k = 1024 / max(image.size) + image.scale(max(1, round(image.size[0] * k)), max(1, round(image.size[1] * k))) +for o in objects: + o.select_set(True) +bpy.ops.export_scene.gltf( + filepath=output, + export_format="GLB", + use_selection=True, + export_yup=True, + export_animations=False, + export_extras=False, + export_image_format="AUTO", + export_copyright="Apple Inc. Original device assets; see models/sources.json for provenance.", +) +print( + "MODEL_METRICS", + json.dumps( + { + "width": (hi.x - lo.x) * scale, + "height": 2.2, + "bodyMeshes": len(objects), + "screen": screen.name, + "source": source, + } + ), +)