From 5d4c202e3f5be0993efea751ca0f0bd4dedb64d4 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 20 Oct 2026 11:32:16 -0700 Subject: [PATCH 01/17] fix(web,mobile): Limits bars show the even-pace mark as a tick in a gap (#22408) Co-authored-by: Claude Opus 5.5 --- .../src/features/usage/UsageLimitsPooled.tsx | 73 ++- .../src/features/usage/UsageLimitsSection.tsx | 74 ++- .../usage/UsageLimits.browser.test.tsx | 89 +++ apps/web/src/components/usage/UsageLimits.tsx | 65 ++- .../usage/UsageLimitsPooled.browser.test.tsx | 17 + .../components/usage/UsageLimitsPooled.tsx | 38 +- .../test/fixtures/usage-limits-hub/README.md | 8 + .../test/fixtures/usage-limits-hub/hub.mjs | 91 +++ .../fixtures/usage-limits-hub/scenarios.json | 516 ++++++++++++++++++ knip.jsonc | 5 +- 10 files changed, 912 insertions(+), 64 deletions(-) create mode 100644 apps/web/src/components/usage/UsageLimits.browser.test.tsx create mode 100644 apps/web/test/fixtures/usage-limits-hub/README.md create mode 100644 apps/web/test/fixtures/usage-limits-hub/hub.mjs create mode 100644 apps/web/test/fixtures/usage-limits-hub/scenarios.json diff --git a/apps/mobile/src/features/usage/UsageLimitsPooled.tsx b/apps/mobile/src/features/usage/UsageLimitsPooled.tsx index 9013a3dfcc9fb..010c902364c4b 100644 --- a/apps/mobile/src/features/usage/UsageLimitsPooled.tsx +++ b/apps/mobile/src/features/usage/UsageLimitsPooled.tsx @@ -41,6 +41,53 @@ function accountName(account: LimitAccount) { return `${local[0] ?? ""}${domain[0] ?? ""}`.toUpperCase() || "Account"; } +// Same gap-and-tick as the Limits rows: two clipped copies of the segment leave a see-through gap. +function PaceGap({ + timeLeft, + children, +}: { + readonly timeLeft: number | null; + readonly children: ReactNode; +}) { + const [width, setWidth] = useState(0); + const tickX = + timeLeft === null || width < 6 + ? null + : Math.min(Math.max((width * timeLeft) / 100, 3), width - 3); + return ( + setWidth(event.nativeEvent.layout.width)} + > + {tickX === null + ? children + : ( + [ + [0, tickX - 3], + [tickX + 3, width], + ] as const + ).map(([start, end]) => ( + + + {children} + + + ))} + {tickX !== null ? ( + + ) : null} + + ); +} + function AccountSegment({ remaining, color, @@ -155,22 +202,18 @@ function PoolWindowCard({ accessibilityLabel={`Segment ${index + 1}, ${accountName(account)}, ${remainingPercent(window)}% left${timeLeft === null ? "" : `, ${timeLeft}% of the window left`}`} accessibilityHint="Show account details" onPress={() => openAccount(account)} - className="h-7 min-w-0 flex-1 rounded-md bg-subtle" + className="h-7 min-w-0 flex-1 rounded-md" > - - - - {timeLeft !== null ? ( - - ) : null} + + + + + + 0 ? { width: barWidth } : undefined} + > + 30 && props.color ? { backgroundColor: props.color } : null, + ]} + /> + + + ); return ( @@ -57,27 +83,35 @@ function WindowRow(props: { {remaining}% left - - - 30 && props.color ? { backgroundColor: props.color } : null, - ]} - /> - - - {timeLeft !== null ? ( + setBarWidth(event.nativeEvent.layout.width)} + > + {tickX === null ? ( + bar + ) : ( + + {/* Clip the same full-width bar so neither fill proportions nor corner radii change. */} + {( + [ + [0, tickX - 3], + [tickX + 3, barWidth], + ] as const + ).map(([start, end]) => ( + + {bar} + + ))} + + )} + {tickX !== null ? ( ) : null} diff --git a/apps/web/src/components/usage/UsageLimits.browser.test.tsx b/apps/web/src/components/usage/UsageLimits.browser.test.tsx new file mode 100644 index 0000000000000..c0786c675f247 --- /dev/null +++ b/apps/web/src/components/usage/UsageLimits.browser.test.tsx @@ -0,0 +1,89 @@ +import { render } from "../../../test/render"; +import { ProviderDriverKind, type ServerProviderUsageWindow } from "@t3tools/contracts"; +import { expect, it, vi } from "vite-plus/test"; +import { page } from "vite-plus/test/browser"; + +import "~/index.css"; + +vi.mock("../../state/session", () => ({ + useEnvironmentScope: () => true, + readEnvironmentScope: () => true, +})); +vi.mock("../../state/server", () => ({ serverEnvironment: { consumeResetCredit: null } })); +vi.mock("../../state/use-atom-command", () => ({ useAtomCommand: () => vi.fn() })); +vi.mock("../../hooks/useSettings", () => ({ usePrimarySettings: () => "24h" })); + +import { LimitWindows } from "./UsageLimits"; + +const codex = ProviderDriverKind.make("codex"); +const now = Date.parse("2026-09-03T12:00:00Z"); + +const sessionWindow = (minutesUntilReset: number | null): ServerProviderUsageWindow => ({ + id: "five_hour", + kind: "session", + label: "Session", + usedPercent: 40, + ...(minutesUntilReset === null + ? {} + : { + windowDurationMins: 300, + resetsAt: new Date(now + minutesUntilReset * 60_000).toISOString(), + }), +}); + +const sized = (width: string) => { + const container = document.createElement("div"); + container.style.width = width; + return container; +}; + +const bar = () => { + const element = page.getByRole("img", { name: /^Session:/ }).element(); + const mark = element.querySelector(":scope > span[aria-hidden]"); + return { + rect: element.getBoundingClientRect(), + mark: mark?.getBoundingClientRect(), + clip: getComputedStyle(element.querySelector(":scope > div")!).clipPath, + }; +}; + +it("places the even-pace tick in a gap at the share of the window left", async () => { + await using _screen = await render( + , + { container: sized("640px") }, + ); + const { rect, mark, clip } = bar(); + expect(mark!.left + mark!.width / 2 - rect.left).toBeCloseTo(rect.width * 0.3, 0); + expect(clip).toMatch(/^polygon/); +}); + +it("keeps the tick inside the bar at both ends of the window", async () => { + for (const minutesUntilReset of [0, 300]) { + await using _screen = await render( + , + { container: sized("640px") }, + ); + const { rect, mark } = bar(); + expect(mark!.left).toBeGreaterThanOrEqual(rect.left); + expect(mark!.right).toBeLessThanOrEqual(rect.right); + } +}); + +it("draws a continuous bar when the window timing is unknown", async () => { + await using _screen = await render( + , + { container: sized("640px") }, + ); + expect(bar()).toMatchObject({ mark: undefined, clip: "none" }); +}); + +it("draws neither gap nor tick on a bar narrower than 6px", async () => { + // 11rem + 1rem + 3.5px + 1rem + 7rem: the bar column gets 3.5px. + await using _screen = await render( + , + { container: sized("323.5px") }, + ); + const { rect, mark, clip } = bar(); + expect(rect.width).toBeCloseTo(3.5, 1); + expect({ mark, clip }).toEqual({ mark: undefined, clip: "none" }); +}); diff --git a/apps/web/src/components/usage/UsageLimits.tsx b/apps/web/src/components/usage/UsageLimits.tsx index 0ac9cf2362924..bac2682a0245e 100644 --- a/apps/web/src/components/usage/UsageLimits.tsx +++ b/apps/web/src/components/usage/UsageLimits.tsx @@ -18,9 +18,10 @@ import { remainingPercent, } from "@t3tools/shared/usageLimits"; import { GaugeIcon, TrendingDownIcon, TrendingUpIcon } from "lucide-react"; -import { Fragment, type ReactNode, useState } from "react"; +import { Fragment, type ReactNode, useLayoutEffect, useState } from "react"; import { usePrimarySettings } from "../../hooks/useSettings"; +import { observeResize } from "../../lib/observeResize"; import { environmentPresentations } from "../../state/presentation"; import { useEnvironmentScope, readEnvironmentScope } from "../../state/session"; import { serverEnvironment } from "../../state/server"; @@ -73,6 +74,43 @@ export function PaceIcon({ pace }: { readonly pace: LimitPace }) { ); } +// The tick sits in a transparent gap cut from the bar, clamped so it stays inside at 0% and 100%. +// The gap is a clip-path slot rather than a mask: a mask composites its layer and re-antialiases +// the bar's rounded ends, and the polygon reaches 1px past the box so only the slot edges clip. +function paceMarkStyles(timeLeft: number | null) { + if (timeLeft === null) return null; + const at = `clamp(3px, ${timeLeft}%, calc(100% - 3px))`; + const [left, right] = [`calc(${at} - 3px)`, `calc(${at} + 3px)`]; + const [near, far] = ["-1px", "calc(100% + 1px)"]; + return { + tick: { left: at }, + gap: { + clipPath: `polygon(${near} ${near}, ${left} ${near}, ${left} ${far}, ${right} ${far}, ${right} ${near}, ${far} ${near}, ${far} ${far}, ${near} ${far})`, + }, + // For a track painted on an element that also draws focus rings, which a clip would cut. + gappedTrack: { + backgroundColor: "transparent", + backgroundImage: `linear-gradient(to right, var(--color-muted) ${left}, transparent ${left}, transparent ${right}, var(--color-muted) ${right})`, + }, + }; +} + +/** + * Pace mark styles for the bar given the returned ref. Below 6px there is no room for the gap + * and tick, so the bar draws neither, as the mobile bars do. + */ +export function usePaceMark(timeLeft: number | null) { + const [bar, setBar] = useState(null); + const [roomy, setRoomy] = useState(false); + useLayoutEffect(() => { + if (!bar) return; + const measure = () => setRoomy(bar.getBoundingClientRect().width >= 6); + measure(); + return observeResize(bar, measure); + }, [bar]); + return [setBar, roomy ? paceMarkStyles(timeLeft) : null] as const; +} + function WindowBar({ color, window, @@ -85,6 +123,7 @@ function WindowBar({ const timestampFormat = usePrimarySettings((settings) => settings.timestampFormat); const remaining = remainingPercent(window); const timeLeft = paceRemainingPercent(window, now); + const [barRef, paceMark] = usePaceMark(timeLeft); const resetsIn = formatResetsIn(window, now); const resetsAt = window.resetsAt ? formatUpcomingTimestamp(window.resetsAt, timestampFormat, now) @@ -98,6 +137,7 @@ function WindowBar({ } > -
- {remaining > 0 ? ( -
- ) : null} - {timeLeft !== null ? ( + {/* Clip both layers so the gap exposes even a translucent composer surface. */} +
+
+ {remaining > 0 ? ( +
+ ) : null} +
+ {paceMark ? ( ) : null} diff --git a/apps/web/src/components/usage/UsageLimitsPooled.browser.test.tsx b/apps/web/src/components/usage/UsageLimitsPooled.browser.test.tsx index b5b6f03415102..b175bbd3b0976 100644 --- a/apps/web/src/components/usage/UsageLimitsPooled.browser.test.tsx +++ b/apps/web/src/components/usage/UsageLimitsPooled.browser.test.tsx @@ -177,6 +177,23 @@ it("omits an empty reset plate but keeps countdowns and banked credits visible", expect(plate()?.textContent).toBe("1"); }); +it("cuts the pace gap through the segment track and keeps the tick inside it", async () => { + setup(); + // Five hours before the reset is the start of the window, the right-hand end of the bar. + await using _screen = await render( + , + ); + const segment = page.getByRole("button", { name: /^Alpha:/ }).element(); + const tick = segment.querySelector(':scope > span[style*="left"]')!.getBoundingClientRect(); + const rect = segment.getBoundingClientRect(); + expect(getComputedStyle(segment).backgroundImage).toMatch(/^linear-gradient/); + expect(tick.left).toBeGreaterThanOrEqual(rect.left); + expect(tick.right).toBeLessThanOrEqual(rect.right); +}); + it("moves the one limit popover between windows on hover", async () => { setup(); await using screen = await render(initial()); diff --git a/apps/web/src/components/usage/UsageLimitsPooled.tsx b/apps/web/src/components/usage/UsageLimitsPooled.tsx index 8fc4b775b850c..76be2555ff8c5 100644 --- a/apps/web/src/components/usage/UsageLimitsPooled.tsx +++ b/apps/web/src/components/usage/UsageLimitsPooled.tsx @@ -42,6 +42,7 @@ import { PaceIcon, ResetCreditDialog, barColor, + usePaceMark, resetCreditsSummary, useResetCredit, } from "./UsageLimits"; @@ -342,6 +343,7 @@ function SegmentTriggers({ const payload: SegmentPayload = { account, window, reset, redeem }; const remaining = remainingPercent(window); const timeLeft = paceRemainingPercent(window, now); + const [segmentRef, paceMark] = usePaceMark(timeLeft); const resetsIn = formatResetsIn(window, now); const credits = account.limits.resetCredits?.availableCount ?? 0; return ( @@ -352,35 +354,39 @@ function SegmentTriggers({ openOnHover render={ +
+ {filters.map((filter, index) => ( + // Always stacked: the dialog stays narrow on wide windows, so a single line + // would cut off typical paths and values. The grid wrappers size each control + // without restyling it; a grid item stretches to its minmax(0, 1fr) track. +
+
+
+ update(index, { path: event.target.value })} + /> +
+ +
+
+ + {filter.op === "exists" ? null : ( +
+ update(index, { value: event.target.value })} + /> +
+ )} +
+
+ ))} +
+ ); +} + function Field({ label, hint, @@ -232,14 +335,23 @@ const DELIVERY_OUTCOME_LABELS: Record void; }) { const deliveriesQuery = useEnvironmentQuery( - serverEnvironment.scheduledTaskWebhookDeliveries({ environmentId, input: { id: task.id } }), + serverEnvironment.scheduledTaskWebhookDeliveries({ + environmentId, + input: { id: task.id, acceptAnyOutcome: true }, + }), ); const [selectedId, setSelectedId] = useState( null, @@ -736,7 +851,7 @@ function WebhookDeliveriesDialog({
- {DELIVERY_OUTCOME_LABELS[selected.outcome]} + {deliveryOutcomeLabel(selected.outcome)} {selected.method} · @@ -745,7 +860,17 @@ function WebhookDeliveriesDialog({ Signature verified ) : null}
- {selected.error ?

{selected.error}

: null} + {selected.error ? ( +

+ {selected.error} +

+ ) : null} {selected.renderedPrompt ?? "No run was started for this request."} @@ -782,12 +907,15 @@ function WebhookDeliveriesDialog({ onClick={() => setSelectedId(delivery.id)} > - {DELIVERY_OUTCOME_LABELS[delivery.outcome]} + {deliveryOutcomeLabel(delivery.outcome)} {delivery.method} + {delivery.outcome === "filtered" && delivery.error ? ( + {delivery.error} + ) : null} {delivery.missingFields.length > 0 ? ( {delivery.missingFields.length} empty placeholder @@ -1058,6 +1186,11 @@ function ScheduledTaskEditorForm({ reportFailure("Scheduled task is incomplete", "Add a title, prompt, project, and model."); return; } + const parsedFilters = parseWebhookFilterDrafts(draft.filters); + if (draft.scheduleMode === "webhook" && "error" in parsedFilters) { + reportFailure("Invalid filter", parsedFilters.error); + return; + } const schedule = scheduleFromDraft(draft); if (schedule === null) { reportFailure( @@ -1142,7 +1275,8 @@ function ScheduledTaskEditorForm({ -
+ {/* Fieldsets default to min-width: min-content, which lets one wide control widen the form past the dialog. */} +
{!connected ? (

Reconnect this environment before saving.

) : null} @@ -1351,7 +1485,8 @@ function ScheduledTaskEditorForm({ minute.

) : null} -
+ {/* Wraps the toggle below the label when both don't fit, as on phones. */} +
+ setDraft((current) => ({ ...current, filters }))} + />
diff --git a/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts b/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts index 9c67ec46e8e06..2b9264b9dd70e 100644 --- a/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts +++ b/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts @@ -218,6 +218,31 @@ describe("webhook scheduled tasks", () => { expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: invalid })).toBeNull(); } }); + + it("sends filters only when the form changed them", () => { + const filters = [ + { path: "headers.x-github-event", op: "equals", value: "pull_request" }, + { path: "body.action", op: "in", value: ["opened", "reopened"] }, + ] as const; + const draft = taskToDraft({ + ...webhookTask, + schedule: { type: "webhook", signature, filters }, + }); + expect(draft.filters.map((filter) => filter.op)).toEqual(["equals", "in"]); + // A title-only save leaves the key out, so the server keeps whatever is stored now. + expect(scheduleFromDraft({ ...draft, title: "Renamed" })).not.toHaveProperty("filters"); + expect(scheduleFromDraft({ ...draft, filters: draft.filters.slice(0, 1) })).toMatchObject({ + filters: filters.slice(0, 1), + }); + // Removing every row is a change, so it clears the stored filters. + expect(scheduleFromDraft({ ...draft, filters: [] })).toMatchObject({ filters: [] }); + expect( + scheduleFromDraft({ + ...draft, + filters: [{ id: "f1", path: "body.action", op: "in", value: " , " }], + }), + ).toBeNull(); + }); }); describe("scheduled task model defaults", () => { diff --git a/apps/web/src/components/settings/scheduledTasksSettings.logic.ts b/apps/web/src/components/settings/scheduledTasksSettings.logic.ts index 0767f99190c9c..472ba9b998a8a 100644 --- a/apps/web/src/components/settings/scheduledTasksSettings.logic.ts +++ b/apps/web/src/components/settings/scheduledTasksSettings.logic.ts @@ -9,7 +9,13 @@ import { type ProviderInteractionMode, type ServerSettings, } from "@t3tools/contracts"; -import { parseMaxDeliveryAge } from "@t3tools/client-runtime/scheduled-task-webhook"; +import { + parseMaxDeliveryAge, + parseWebhookFilterDrafts, + webhookFilterDrafts, + webhookFilterDraftsChanged, + type WebhookFilterDraft, +} from "@t3tools/client-runtime/scheduled-task-webhook"; import { resolveProjectSettings, @@ -77,6 +83,9 @@ export interface DraftState { /** Write-only: empty keeps the secret already stored on the server. */ readonly signatureSecret: string; readonly maxDeliveryAgeMinutes: string; + readonly filters: ReadonlyArray; + /** The filter rows the form opened with. */ + readonly savedFilters: ReadonlyArray; } export const WEBHOOK_SIGNATURE_DEFAULTS = { @@ -89,6 +98,8 @@ export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedul if (draft.scheduleMode === "webhook") { const maxDeliveryAgeMinutes = parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes); if (maxDeliveryAgeMinutes === undefined) return null; + const parsedFilters = parseWebhookFilterDrafts(draft.filters); + if (!("filters" in parsedFilters)) return null; const secret = draft.signatureSecret.trim(); return { type: "webhook", @@ -101,6 +112,10 @@ export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedul } : null, maxDeliveryAgeMinutes, + // Only edited filters are sent; the server keeps the stored ones otherwise. + ...(webhookFilterDraftsChanged(draft.savedFilters, draft.filters) + ? { filters: parsedFilters.filters } + : {}), }; } if (draft.scheduleMode === "interval") { @@ -117,6 +132,7 @@ export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedul export function taskToDraft(task: ScheduledTask): DraftState { const schedule = task.schedule; + const filters = webhookFilterDrafts(schedule.type === "webhook" ? schedule.filters : undefined); const weekdays = schedule.type === "fixed_time" && schedule.weekdays && schedule.weekdays.length > 0 ? new Set(schedule.weekdays) @@ -161,6 +177,8 @@ export function taskToDraft(task: ScheduledTask): DraftState { schedule.type === "webhook" && schedule.maxDeliveryAgeMinutes != null ? String(schedule.maxDeliveryAgeMinutes) : "", + filters, + savedFilters: filters, }; } diff --git a/docs/user/project-settings.md b/docs/user/project-settings.md index bb89de6717e85..cdae02679be6e 100644 --- a/docs/user/project-settings.md +++ b/docs/user/project-settings.md @@ -109,6 +109,15 @@ same secret in the repository's webhook settings with content type up on desktop or web; mobile keeps an existing signature check but can't turn one on. +To run only for some requests, add **Filters**. Each one tests a value named +like a placeholder, without the braces, such as `headers.x-github-event` equals +`pull_request` or `body.action` is one of `opened, reopened`. A task runs only +when every filter matches; other requests are answered normally, logged as +**Filtered** with the first filter that failed, and start nothing. Values +compare as text, and a value that is an object or a list only passes +**exists**. Separate **is one of** values with commas; a value can't contain +a comma or start or end with a space. + On desktop and web, pick **Deliveries** from a task's menu to see recent requests and the prompt each one produced. diff --git a/packages/client-runtime/src/scheduledTaskWebhook.test.ts b/packages/client-runtime/src/scheduledTaskWebhook.test.ts index c2de7d240bc7e..6e35d8446964e 100644 --- a/packages/client-runtime/src/scheduledTaskWebhook.test.ts +++ b/packages/client-runtime/src/scheduledTaskWebhook.test.ts @@ -1,6 +1,13 @@ import { describe, expect, it } from "vite-plus/test"; -import { parseMaxDeliveryAge } from "./scheduledTaskWebhook.ts"; +import { + parseMaxDeliveryAge, + emptyWebhookFilterDraft, + parseWebhookFilterDrafts, + removeWebhookFilterDraft, + webhookFilterDrafts, + webhookFilterDraftsChanged, +} from "./scheduledTaskWebhook.ts"; describe("parseMaxDeliveryAge", () => { it("treats blank as no limit and rejects values the server would not accept", () => { @@ -11,3 +18,57 @@ describe("parseMaxDeliveryAge", () => { } }); }); + +describe("webhook filter drafts", () => { + it("round-trips saved filters through the form rows", () => { + const filters = [ + { path: "headers.x-github-event", op: "equals", value: "pull_request" }, + { path: "body.action", op: "in", value: ["opened", "reopened"] }, + { path: "body.label.name", op: "in", value: ["needs review", "a-b"] }, + { path: "query.source", op: "exists" }, + ] as const; + const drafts = webhookFilterDrafts(filters); + expect(drafts[1]).toMatchObject({ path: "body.action", op: "in", value: "opened, reopened" }); + expect(parseWebhookFilterDrafts(drafts)).toEqual({ filters }); + }); + + it("names the first row the server would refuse", () => { + expect( + parseWebhookFilterDrafts([ + { id: "f1", path: "body.action", op: "equals", value: "opened" }, + { id: "f2", path: "body.action", op: "in", value: " , " }, + { id: "f3", path: "", op: "exists", value: "" }, + ]), + ).toEqual({ + error: "Filter 2 needs a path under body, headers or query, and at least one value.", + }); + expect(parseWebhookFilterDrafts([{ id: "f1", path: " ", op: "exists", value: "" }])).toEqual({ + error: "Filter 1 needs a path under body, headers or query.", + }); + const seventeen = Array.from({ length: 17 }, (_, index) => ({ + id: `f${index}`, + path: "body.action", + op: "exists" as const, + value: "", + })); + expect(parseWebhookFilterDrafts(seventeen)).toEqual({ error: "Use at most 16 filters." }); + }); + + it("notices added, removed and edited rows but not an untouched list", () => { + const saved = webhookFilterDrafts([{ path: "body.action", op: "equals", value: "opened" }]); + expect(webhookFilterDraftsChanged(saved, [...saved])).toBe(false); + expect(webhookFilterDraftsChanged(saved, [])).toBe(true); + expect(webhookFilterDraftsChanged(saved, [{ ...saved[0]!, value: "closed" }])).toBe(true); + expect(webhookFilterDraftsChanged([], [])).toBe(false); + }); + + it("removing the first row keeps the second row's id", () => { + const drafts = [ + ...webhookFilterDrafts([{ path: "body.action", op: "equals", value: "opened" }]), + emptyWebhookFilterDraft(), + ]; + expect(new Set(drafts.map((draft) => draft.id)).size).toBe(2); + const remaining = removeWebhookFilterDraft(drafts, drafts[0]!.id); + expect(remaining.map((draft) => draft.id)).toEqual([drafts[1]!.id]); + }); +}); diff --git a/packages/client-runtime/src/scheduledTaskWebhook.ts b/packages/client-runtime/src/scheduledTaskWebhook.ts index 0285917096593..2b19d282dceb1 100644 --- a/packages/client-runtime/src/scheduledTaskWebhook.ts +++ b/packages/client-runtime/src/scheduledTaskWebhook.ts @@ -1,4 +1,11 @@ -import { MAX_WEBHOOK_DELIVERY_AGE_MINUTES } from "@t3tools/contracts"; +import { + MAX_WEBHOOK_DELIVERY_AGE_MINUTES, + MAX_WEBHOOK_FILTERS, + ScheduledTaskWebhookFilter, + type ScheduledTaskWebhookFilterOp, +} from "@t3tools/contracts"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; export const DEFAULT_WEBHOOK_PROMPT = "Handle this webhook:\n{{body}}"; @@ -10,3 +17,118 @@ export function parseMaxDeliveryAge(value: string): number | null | undefined { ? minutes : undefined; } + +/** + * One editable filter row. `in` takes a comma-separated list, which is + * lossless because list values can't contain commas; `exists` ignores the value. + */ +export interface WebhookFilterDraft { + /** Stable row identity for the form; never sent to the server. */ + readonly id: string; + readonly path: string; + readonly op: ScheduledTaskWebhookFilterOp; + readonly value: string; +} + +export const WEBHOOK_FILTER_OPS: ReadonlyArray<{ + readonly op: ScheduledTaskWebhookFilterOp; + readonly label: string; +}> = [ + { op: "equals", label: "equals" }, + { op: "notEquals", label: "does not equal" }, + { op: "in", label: "is one of" }, + { op: "startsWith", label: "starts with" }, + { op: "exists", label: "exists" }, +]; + +let lastWebhookFilterDraftId = 0; +const nextWebhookFilterDraftId = () => `filter-${++lastWebhookFilterDraftId}`; + +export function emptyWebhookFilterDraft(): WebhookFilterDraft { + return { id: nextWebhookFilterDraftId(), path: "", op: "equals", value: "" }; +} + +/** By id, so a removed row's identity never passes to the row after it. */ +export function removeWebhookFilterDraft( + drafts: ReadonlyArray, + id: string, +): ReadonlyArray { + return drafts.filter((draft) => draft.id !== id); +} + +export function webhookFilterDrafts( + filters: ReadonlyArray | undefined, +): ReadonlyArray { + return (filters ?? []).map((filter) => ({ + id: nextWebhookFilterDraftId(), + path: filter.path, + op: filter.op, + value: + filter.op === "exists" ? "" : filter.op === "in" ? filter.value.join(", ") : filter.value, + })); +} + +const decodeFilter = Schema.decodeUnknownOption(ScheduledTaskWebhookFilter); + +/** + * Checks each row with the server's own schema, so a draft that parses here + * also saves. The error names the first row that would not. + */ +export function parseWebhookFilterDrafts( + drafts: ReadonlyArray, +): { readonly filters: ReadonlyArray } | { readonly error: string } { + if (drafts.length > MAX_WEBHOOK_FILTERS) { + return { error: `Use at most ${MAX_WEBHOOK_FILTERS} filters.` }; + } + const filters: ScheduledTaskWebhookFilter[] = []; + for (const [index, draft] of drafts.entries()) { + const path = draft.path.trim(); + const filter = decodeFilter( + draft.op === "exists" + ? { path, op: draft.op } + : draft.op === "in" + ? { + path, + op: draft.op, + value: draft.value + .split(",") + .map((item) => item.trim()) + .filter((item) => item !== ""), + } + : { path, op: draft.op, value: draft.value }, + ); + if (Option.isNone(filter)) { + return { + error: `Filter ${index + 1} needs a path under body, headers or query${ + draft.op === "exists" + ? "." + : draft.op === "in" + ? ", and at least one value." + : ", and a value." + }`, + }; + } + filters.push(filter.value); + } + return { filters }; +} + +/** + * Whether a form's filter rows differ from the ones it opened with. A save + * leaves `filters` out unless they did, so editing only the title can't + * overwrite filters another device changed in the meantime. + */ +export function webhookFilterDraftsChanged( + saved: ReadonlyArray, + current: ReadonlyArray, +): boolean { + return ( + saved.length !== current.length || + saved.some( + (filter, index) => + filter.path !== current[index]?.path || + filter.op !== current[index]?.op || + (filter.op !== "exists" && filter.value !== current[index]?.value), + ) + ); +} diff --git a/packages/contracts/src/scheduledTask.test.ts b/packages/contracts/src/scheduledTask.test.ts index c3373d08abf42..61af412b05558 100644 --- a/packages/contracts/src/scheduledTask.test.ts +++ b/packages/contracts/src/scheduledTask.test.ts @@ -3,6 +3,7 @@ import * as Schema from "effect/Schema"; import { MIN_SCHEDULED_TASK_INTERVAL_MS, + ScheduledTaskListWebhookDeliveriesResult, ScheduledTaskSchedule, ScheduledTaskUpsertSchedule, } from "./scheduledTask.ts"; @@ -50,3 +51,72 @@ describe("ScheduledTaskUpsertSchedule", () => { ).toThrow(); }); }); + +describe("webhook filters", () => { + const accepts = (filters: unknown) => { + try { + decodeUpsertSchedule({ type: "webhook", filters }); + return true; + } catch { + return false; + } + }; + + it("keeps valid filters", () => { + const filters = [ + { path: "headers.x-github-event", op: "equals", value: "pull_request" }, + { path: "body.action", op: "in", value: ["opened", "labeled"] }, + { path: "body.label.name", op: "startsWith", value: "agent:" }, + { path: "query.source", op: "exists" }, + ]; + expect(decodeUpsertSchedule({ type: "webhook", filters })).toEqual({ + type: "webhook", + filters, + }); + expect(decodeSchedule({ type: "webhook", signature: null, filters })).toMatchObject({ + filters, + }); + }); + + it("rejects empty or unknown paths, unknown ops, and values that do not fit the op", () => { + expect(accepts([{ path: "", op: "exists" }])).toBe(false); + expect(accepts([{ path: " ", op: "exists" }])).toBe(false); + expect(accepts([{ path: "request", op: "exists" }])).toBe(false); + expect(accepts([{ path: "header.x-github-event", op: "exists" }])).toBe(false); + expect(accepts([{ path: "body.action", op: "in", value: "opened" }])).toBe(false); + expect(accepts([{ path: "body.action", op: "in", value: [] }])).toBe(false); + for (const value of ["", " opened", "opened ", "a,b"]) { + expect(accepts([{ path: "body.action", op: "in", value: [value] }])).toBe(false); + } + expect(accepts([{ path: "body.label.name", op: "in", value: ["needs review"] }])).toBe(true); + expect(accepts([{ path: "body.action", op: "equals" }])).toBe(false); + expect(accepts([{ path: "body.action", op: "matches", value: "^opened$" }])).toBe(false); + }); + + it("allows at most 16 filters", () => { + const sixteen = Array.from({ length: 16 }, () => ({ path: "body.action", op: "exists" })); + expect(accepts(sixteen)).toBe(true); + expect(accepts([...sixteen, { path: "body.action", op: "exists" }])).toBe(false); + }); +}); + +describe("ScheduledTaskWebhookDeliverySummary", () => { + it("decodes an outcome this client does not know yet", () => { + const summary = { + id: "delivery:1", + taskId: "scheduled-task:1", + receivedAt: "2026-10-20T00:00:00.000Z", + method: "POST", + contentType: null, + bodyBytes: 0, + outcome: "some_future_outcome", + signatureVerified: false, + missingFields: [], + error: null, + }; + expect( + Schema.decodeUnknownSync(ScheduledTaskListWebhookDeliveriesResult)({ deliveries: [summary] }) + .deliveries[0]?.outcome, + ).toBe("some_future_outcome"); + }); +}); diff --git a/packages/contracts/src/scheduledTask.ts b/packages/contracts/src/scheduledTask.ts index 9a24e1b8abe3a..77825113992f0 100644 --- a/packages/contracts/src/scheduledTask.ts +++ b/packages/contracts/src/scheduledTask.ts @@ -83,12 +83,56 @@ const WebhookMaxDeliveryAgeMinutes = Schema.Int.check( "Skip requests the relay held longer than this many minutes while the environment was offline. Null runs every request.", }); +export const MAX_WEBHOOK_FILTERS = 16; + +const WebhookFilterPath = TrimmedNonEmptyString.check( + Schema.isPattern(/^(body|headers|query)(\.|$)/), +).annotate({ + description: + "Request value to test, in placeholder syntax without braces: headers.x-github-event, body.action, query.source.", +}); + +/** + * Forms edit an `in` list as comma-separated text, so a value that has no + * comma and no surrounding whitespace survives that round trip unchanged. + */ +const WebhookFilterListValue = Schema.String.check( + Schema.isPattern(/^[^,\s](?:[^,]*[^,\s])?$/), +).annotate({ description: "Non-empty, without commas or surrounding whitespace." }); + +export const ScheduledTaskWebhookFilter = Schema.Union([ + Schema.Struct({ + path: WebhookFilterPath, + op: Schema.Literals(["equals", "notEquals", "startsWith"]), + value: Schema.String, + }), + Schema.Struct({ + path: WebhookFilterPath, + op: Schema.Literal("in"), + value: Schema.Array(WebhookFilterListValue).check(Schema.isMinLength(1)), + }), + Schema.Struct({ path: WebhookFilterPath, op: Schema.Literal("exists") }), +]).annotate({ + description: + "Scalars compare as strings; objects and arrays only satisfy 'exists'. Every op but 'exists' fails when the path is missing.", +}); +export type ScheduledTaskWebhookFilter = typeof ScheduledTaskWebhookFilter.Type; +export type ScheduledTaskWebhookFilterOp = ScheduledTaskWebhookFilter["op"]; + +const ScheduledTaskWebhookFilters = Schema.Array(ScheduledTaskWebhookFilter) + .check(Schema.isMaxLength(MAX_WEBHOOK_FILTERS)) + .annotate({ + description: + "Run only when every filter matches the request; other requests are logged as filtered. Up to 16. Omit to keep the stored filters; pass [] to clear them.", + }); + const ScheduledTaskWebhookSchedule = Schema.Struct({ type: Schema.Literal("webhook").annotate({ description: "Run when the task's webhook URL receives a request.", }), signature: Schema.NullOr(ScheduledTaskWebhookSignature), maxDeliveryAgeMinutes: Schema.optional(Schema.NullOr(WebhookMaxDeliveryAgeMinutes)), + filters: Schema.optional(ScheduledTaskWebhookFilters), }).annotate({ description: "Run on each request to the task's webhook URL. The prompt may use {{body.path}}, {{headers.name}}, {{query.name}}, {{body}} and {{request}} placeholders.", @@ -115,6 +159,7 @@ const ScheduledTaskUpsertWebhookSchedule = Schema.Struct({ description: "Signature check; omit or null to accept requests by URL token only.", }), maxDeliveryAgeMinutes: Schema.optional(Schema.NullOr(WebhookMaxDeliveryAgeMinutes)), + filters: Schema.optional(ScheduledTaskWebhookFilters), }).annotate({ description: "Run on each request to the task's webhook URL. The prompt may use {{body.path}}, {{headers.name}}, {{query.name}}, {{body}} and {{request}} placeholders.", @@ -253,6 +298,7 @@ export const ScheduledTaskWebhookDeliveryOutcome = Schema.Literals([ "disabled", "rate_limited", "expired", + "filtered", ]); export type ScheduledTaskWebhookDeliveryOutcome = typeof ScheduledTaskWebhookDeliveryOutcome.Type; @@ -263,7 +309,8 @@ export const ScheduledTaskWebhookDeliverySummary = Schema.Struct({ method: TrimmedNonEmptyString, contentType: Schema.NullOr(Schema.String), bodyBytes: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), - outcome: ScheduledTaskWebhookDeliveryOutcome, + // Any string, so an outcome added later doesn't break a client's whole list. + outcome: Schema.Union([ScheduledTaskWebhookDeliveryOutcome, Schema.String]), signatureVerified: Schema.Boolean, missingFields: Schema.Array(Schema.String), error: Schema.NullOr(Schema.String), @@ -282,6 +329,12 @@ export type ScheduledTaskWebhookDelivery = typeof ScheduledTaskWebhookDelivery.T export const ScheduledTaskListWebhookDeliveriesInput = Schema.Struct({ id: ScheduledTaskId, + /** + * Sent by clients that decode any outcome. Shipped clients decode a closed + * list and fail on the whole response, so without it the server leaves out + * rows with outcomes they don't know. + */ + acceptAnyOutcome: Schema.optional(Schema.Boolean), }); export type ScheduledTaskListWebhookDeliveriesInput = typeof ScheduledTaskListWebhookDeliveriesInput.Type; From ec585729e0a48c318f4cfb0c232b816c029d0541 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 20 Oct 2026 13:30:02 -0700 Subject: [PATCH 06/17] fix(mobile): prevent update checker from crashing iOS at launch (#22488) --- .../src/features/updates/app-updates.test.ts | 37 ++++++++ .../src/features/updates/app-updates.ts | 85 +++++++++---------- 2 files changed, 77 insertions(+), 45 deletions(-) diff --git a/apps/mobile/src/features/updates/app-updates.test.ts b/apps/mobile/src/features/updates/app-updates.test.ts index e35f173a4729f..d980164b32d37 100644 --- a/apps/mobile/src/features/updates/app-updates.test.ts +++ b/apps/mobile/src/features/updates/app-updates.test.ts @@ -1,7 +1,9 @@ +import { AppState } from "react-native"; import { describe, expect, it, vi } from "vite-plus/test"; import { createAppUpdateDeferral, + createAppUpdateForegroundRecheck, createAppUpdateLaunchCheck, FOREGROUND_APP_UPDATE_RECHECK_AFTER_MS, registerHiddenUpdateTap, @@ -19,6 +21,13 @@ vi.mock("expo-updates", () => ({ reloadAsync: vi.fn(), })); +vi.mock("react-native", () => ({ + AppState: { + currentState: "active", + addEventListener: vi.fn(() => ({ remove: vi.fn() })), + }, +})); + function makeUpdateClient(overrides: Partial = {}): AppUpdateClient { return { isEnabled: true, @@ -640,3 +649,31 @@ describe("registerHiddenUpdateTap", () => { expect(count).toBe(0); }); }); + +describe("createAppUpdateForegroundRecheck", () => { + it("registers once and checks after returning from a long background stay", () => { + vi.stubGlobal("__DEV__", false); + const now = vi.spyOn(Date, "now"); + const listener = vi.mocked(AppState.addEventListener); + listener.mockClear(); + const client = makeUpdateClient(); + const start = createAppUpdateForegroundRecheck(client, createAppUpdateDeferral()); + + try { + start(); + start(); + expect(listener).toHaveBeenCalledOnce(); + const onChange = listener.mock.calls[0]![1]; + now.mockReturnValue(1000); + onChange("background"); + now.mockReturnValue(1000 + FOREGROUND_APP_UPDATE_RECHECK_AFTER_MS); + onChange("active"); + expect(client.checkForUpdateAsync).toHaveBeenCalledOnce(); + onChange("active"); + expect(client.checkForUpdateAsync).toHaveBeenCalledOnce(); + } finally { + now.mockRestore(); + vi.unstubAllGlobals(); + } + }); +}); diff --git a/apps/mobile/src/features/updates/app-updates.ts b/apps/mobile/src/features/updates/app-updates.ts index 2cfe3aa7aebf7..6390256108ecf 100644 --- a/apps/mobile/src/features/updates/app-updates.ts +++ b/apps/mobile/src/features/updates/app-updates.ts @@ -1,4 +1,6 @@ import * as Updates from "expo-updates"; +// Metro namespace imports evaluate unrelated React Native getters, including unavailable native modules. +import { AppState } from "react-native"; import { type AtomCommandResult, @@ -418,22 +420,19 @@ async function defaultFlushPendingWrites(): Promise { async function defaultIsSafeToRestartInBackground(): Promise { const { isForegroundHandoffActive } = await import("../../lib/foreground-handoff"); if (isForegroundHandoffActive()) return false; - const { AppState } = await import("react-native"); return AppState.currentState === "background"; } function defaultOnNextBackground(apply: () => void, includeCurrent: boolean): void { - void import("react-native").then(({ AppState }) => { - const subscription = AppState.addEventListener("change", (state) => { - if (state !== "background") return; - subscription.remove(); - apply(); - }); - if (includeCurrent && AppState.currentState === "background") { - subscription.remove(); - apply(); - } + const subscription = AppState.addEventListener("change", (state) => { + if (state !== "background") return; + subscription.remove(); + apply(); }); + if (includeCurrent && AppState.currentState === "background") { + subscription.remove(); + apply(); + } } export const DEFERRED_INSTALL_PROMPT_AFTER_MS = 30 * 60 * 1000; @@ -443,25 +442,23 @@ export const DEFERRED_INSTALL_PROMPT_AFTER_MS = 30 * 60 * 1000; * the timer running. */ function defaultOnForegroundStay(apply: () => void): void { - void import("react-native").then(({ AppState }) => { - let timer: ReturnType | undefined; - const arm = () => { - timer ??= setTimeout(() => { - subscription.remove(); - apply(); - }, DEFERRED_INSTALL_PROMPT_AFTER_MS); - }; - const disarm = () => { - if (timer === undefined) return; - clearTimeout(timer); - timer = undefined; - }; - const subscription = AppState.addEventListener("change", (state) => { - if (state === "active") arm(); - else if (state === "background") disarm(); - }); - if (AppState.currentState === "active") arm(); + let timer: ReturnType | undefined; + const arm = () => { + timer ??= setTimeout(() => { + subscription.remove(); + apply(); + }, DEFERRED_INSTALL_PROMPT_AFTER_MS); + }; + const disarm = () => { + if (timer === undefined) return; + clearTimeout(timer); + timer = undefined; + }; + const subscription = AppState.addEventListener("change", (state) => { + if (state === "active") arm(); + else if (state === "background") disarm(); }); + if (AppState.currentState === "active") arm(); } const defaultAppUpdateEnvironment: AppUpdateEnvironment = { @@ -528,22 +525,20 @@ export function createAppUpdateForegroundRecheck( return () => { if (started || !isAppUpdateCheckAvailable(client)) return; started = true; - void import("react-native").then(({ AppState }) => { - let backgroundedAtMs: number | null = null; - AppState.addEventListener("change", (state) => { - if (state === "background") { - backgroundedAtMs = Date.now(); - return; - } - if (state !== "active") return; - const shouldCheck = shouldRecheckAppUpdateOnForeground( - backgroundedAtMs, - Date.now(), - deferral.pendingInstall, - ); - backgroundedAtMs = null; - if (shouldCheck) void runAppUpdateCheck({ client, deferral }); - }); + let backgroundedAtMs: number | null = null; + AppState.addEventListener("change", (state) => { + if (state === "background") { + backgroundedAtMs = Date.now(); + return; + } + if (state !== "active") return; + const shouldCheck = shouldRecheckAppUpdateOnForeground( + backgroundedAtMs, + Date.now(), + deferral.pendingInstall, + ); + backgroundedAtMs = null; + if (shouldCheck) void runAppUpdateCheck({ client, deferral }); }); }; } From 2ebe280e5602f23fc2742297da3d3b6f2b1b8fca Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 20 Oct 2026 13:49:47 -0700 Subject: [PATCH 07/17] feat(scheduled-tasks): replay dedupe and coalescing for webhook deliveries (#22234) --- .../SettingsScheduledTasksRouteScreen.tsx | 42 + .../settings/scheduledTaskDraft.test.ts | 44 +- .../features/settings/scheduledTaskDraft.ts | 38 + .../src/mcp/toolkits/orchestrator/tools.ts | 4 +- apps/server/src/persistence/Migrations.ts | 2 + .../Migrations/055_OrchestrationV2.test.ts | 4 +- .../Migrations/067_WebhookDedupe.ts | 29 + .../reconcileV2PreviewMigration.test.ts | 2 + .../scheduledTasks/ScheduledTaskService.ts | 742 ++++-- .../ScheduledTaskService.webhook.test.ts | 2028 ++++++++++++++++- .../src/scheduledTasks/webhookDedupe.ts | 128 ++ .../src/scheduledTasks/webhookRoute.test.ts | 1 + .../server/src/scheduledTasks/webhookRoute.ts | 2 +- .../src/scheduledTasks/webhookTemplate.ts | 4 +- .../settings/ScheduledTasksSettings.tsx | 117 +- .../scheduledTasksSettings.logic.test.ts | 98 + .../settings/scheduledTasksSettings.logic.ts | 73 + docs/user/project-settings.md | 14 + packages/contracts/src/scheduledTask.test.ts | 51 +- packages/contracts/src/scheduledTask.ts | 26 + 20 files changed, 3123 insertions(+), 326 deletions(-) create mode 100644 apps/server/src/persistence/Migrations/067_WebhookDedupe.ts create mode 100644 apps/server/src/scheduledTasks/webhookDedupe.ts diff --git a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx index aadc124ba4a5d..36c58dc893d0d 100644 --- a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx @@ -716,6 +716,13 @@ function TaskForm({ Dialog.alert("Invalid filter", parsedFilters.error); return; } + if (draft.schedule.mode === "webhook" && schedule === null) { + Dialog.alert( + "Invalid webhook settings", + "Each dedupe key must contain a valid request placeholder. The coalesce window must be whole seconds from 10 to 3600, or blank.", + ); + return; + } if ( !draft.title.trim() || !draft.prompt.trim() || @@ -1060,6 +1067,41 @@ function TaskForm({ setDraft({ ...draft, schedule: { ...draft.schedule, filters } }) } /> + + setDraft({ ...draft, schedule: { ...draft.schedule, replayKey } }) + } + /> + + setDraft({ ...draft, schedule: { ...draft.schedule, coalesceKey } }) + } + /> + + setDraft({ ...draft, schedule: { ...draft.schedule, coalesceWindowSeconds } }) + } + /> + + Replay keys suppress repeats for 7 days. Coalescing runs the latest payload when the + window from the first delivery closes. Empty keys run normally. + ) : ( <> diff --git a/apps/mobile/src/features/settings/scheduledTaskDraft.test.ts b/apps/mobile/src/features/settings/scheduledTaskDraft.test.ts index bfe363cf06524..aa7d36156e50f 100644 --- a/apps/mobile/src/features/settings/scheduledTaskDraft.test.ts +++ b/apps/mobile/src/features/settings/scheduledTaskDraft.test.ts @@ -30,6 +30,37 @@ describe("scheduleDraftForTask", () => { expect(scheduleFromDraft(scheduleDraftForTask({ schedule }))).toEqual(schedule); }); + it("round-trips dedupe settings and rejects invalid windows", () => { + const dedupe = { + replayKey: "{{headers.delivery-id}}", + coalesceKey: "{{body.id}}", + coalesceWindowSeconds: 3600, + }; + const draft = scheduleDraftForTask({ schedule: { type: "webhook", signature: null, dedupe } }); + expect(scheduleFromDraft(draft)).toMatchObject({ dedupe }); + expect( + scheduleFromDraft({ + ...draft, + replayKey: " \n ", + coalesceKey: " \t ", + coalesceWindowSeconds: "", + }), + ).toMatchObject({ dedupe: {} }); + expect( + scheduleFromDraft({ + ...draft, + replayKey: ` ${dedupe.replayKey} `, + coalesceKey: `\t${dedupe.coalesceKey}\n`, + }), + ).toMatchObject({ dedupe }); + expect( + scheduleFromDraft({ ...draft, replayKey: "", coalesceKey: "", coalesceWindowSeconds: "" }), + ).toMatchObject({ dedupe: {} }); + for (const coalesceWindowSeconds of ["9", "3601", "1.5", "abc"]) { + expect(scheduleFromDraft({ ...draft, coalesceWindowSeconds })).toBeNull(); + } + }); + it("round-trips a webhook schedule without a signature", () => { const draft = scheduleDraftForTask({ schedule: { type: "webhook", signature: null } }); expect(draft.mode).toBe("webhook"); @@ -37,6 +68,7 @@ describe("scheduleDraftForTask", () => { type: "webhook", signature: null, maxDeliveryAgeMinutes: null, + dedupe: {}, }); }); @@ -50,6 +82,7 @@ describe("scheduleDraftForTask", () => { expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: "0" })).toBeNull(); expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: "" })).toMatchObject({ maxDeliveryAgeMinutes: null, + dedupe: {}, }); }); @@ -62,7 +95,7 @@ describe("scheduleDraftForTask", () => { const saved = scheduleFromDraft( scheduleDraftForTask({ schedule: { type: "webhook", signature } }), ); - expect(saved).toEqual({ type: "webhook", signature, maxDeliveryAgeMinutes: null }); + expect(saved).toEqual({ type: "webhook", signature, maxDeliveryAgeMinutes: null, dedupe: {} }); expect(saved?.type === "webhook" && saved.signature && "secret" in saved.signature).toBe(false); }); @@ -369,3 +402,12 @@ describe("scheduled task model defaults", () => { ).toBeNull(); }); }); + +it("rejects dedupe keys without valid request placeholders", () => { + expect( + scheduleFromDraft({ ...DEFAULT_SCHEDULE, mode: "webhook", replayKey: "literal" }), + ).toBeNull(); + expect( + scheduleFromDraft({ ...DEFAULT_SCHEDULE, mode: "webhook", coalesceKey: "{{body.bad" }), + ).toBeNull(); +}); diff --git a/apps/mobile/src/features/settings/scheduledTaskDraft.ts b/apps/mobile/src/features/settings/scheduledTaskDraft.ts index 599fb4780e882..3ef1a2f3faf6e 100644 --- a/apps/mobile/src/features/settings/scheduledTaskDraft.ts +++ b/apps/mobile/src/features/settings/scheduledTaskDraft.ts @@ -1,3 +1,4 @@ +import { isWebhookDedupeTemplate } from "@t3tools/contracts"; import type { ModelSelection, ServerConfig, @@ -56,6 +57,9 @@ export type ScheduleDraft = { readonly intervalMinutes: string; readonly signature: ScheduledTaskWebhookSignature | null; readonly maxDeliveryAgeMinutes: string; + readonly replayKey: string; + readonly coalesceKey: string; + readonly coalesceWindowSeconds: string; readonly filters: ReadonlyArray; /** The filter rows the form opened with. */ readonly savedFilters: ReadonlyArray; @@ -68,6 +72,9 @@ export const DEFAULT_SCHEDULE: ScheduleDraft = { intervalMinutes: "15", signature: null, maxDeliveryAgeMinutes: "", + replayKey: "", + coalesceKey: "", + coalesceWindowSeconds: "", filters: [], savedFilters: [], }; @@ -93,6 +100,12 @@ export function scheduleDraftForTask(task: Pick): Sch ...DEFAULT_SCHEDULE, mode: "webhook", signature: task.schedule.signature, + replayKey: task.schedule.dedupe?.replayKey ?? "", + coalesceKey: task.schedule.dedupe?.coalesceKey ?? "", + coalesceWindowSeconds: + task.schedule.dedupe?.coalesceWindowSeconds === undefined + ? "" + : String(task.schedule.dedupe.coalesceWindowSeconds), maxDeliveryAgeMinutes: task.schedule.maxDeliveryAgeMinutes == null ? "" @@ -107,6 +120,19 @@ export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSche if (draft.mode === "webhook") { const maxDeliveryAgeMinutes = parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes); if (maxDeliveryAgeMinutes === undefined) return null; + const replayKey = draft.replayKey.trim(); + const coalesceKey = draft.coalesceKey.trim(); + if ([replayKey, coalesceKey].some((key) => key && !isWebhookDedupeTemplate(key))) return null; + const coalesceWindowSeconds = draft.coalesceWindowSeconds?.trim() + ? Number(draft.coalesceWindowSeconds) + : undefined; + if ( + coalesceWindowSeconds !== undefined && + (!Number.isInteger(coalesceWindowSeconds) || + coalesceWindowSeconds < 10 || + coalesceWindowSeconds > 3600) + ) + return null; const parsedFilters = parseWebhookFilterDrafts(draft.filters); if (!("filters" in parsedFilters)) return null; // No secret is sent, so the server keeps the stored one. @@ -121,6 +147,15 @@ export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSche prefix: draft.signature.prefix, }, maxDeliveryAgeMinutes, + ...(replayKey || coalesceKey || coalesceWindowSeconds !== undefined + ? { + dedupe: { + ...(replayKey ? { replayKey } : {}), + ...(coalesceKey ? { coalesceKey } : {}), + ...(coalesceWindowSeconds === undefined ? {} : { coalesceWindowSeconds }), + }, + } + : { dedupe: {} }), // Only edited filters are sent; the server keeps the stored ones otherwise. ...(webhookFilterDraftsChanged(draft.savedFilters, draft.filters) ? { filters: parsedFilters.filters } @@ -179,6 +214,9 @@ function draftSignature(draft: ScheduledTaskDraft): string { [...draft.schedule.weekdays].sort((a, b) => a - b), draft.schedule.intervalMinutes, draft.schedule.maxDeliveryAgeMinutes, + draft.schedule.replayKey, + draft.schedule.coalesceKey, + draft.schedule.coalesceWindowSeconds, draft.schedule.filters.map(({ path, op, value }) => [path, op, value]), draft.workspace, draft.baseRef, diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index 2e74482f690a2..91187bc2a1311 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -116,7 +116,7 @@ const WorkflowStopTool = Tool.make("t3_workflow_stop", { export const ScheduleTaskTool = Tool.make("schedule_task", { description: - "Create persistent work in the app scheduler that runs even when no turn is active. Pass schedule as a STRUCTURED OBJECT, never JSON text. Timers: {type:'interval', everyMs:3600000} is hourly; {type:'fixed_time', timeOfDay:'09:00', weekdays:[1,2,3,4,5]} is weekday mornings; report the returned nextRunAt. Webhooks: {type:'webhook'} runs once per request to a generated URL. The run sees the request ONLY through prompt placeholders: {{body.path}} (e.g. {{body.action}}, {{body.release.tag_name}}), {{headers.name}}, {{query.name}}, {{body}}, or {{request}} (method, headers with credentials redacted, and body). Each value arrives framed as untrusted request data, and $skill, @file or /command mentions inside it do not act; without a signature, anyone with the URL can trigger a run. For a sender that signs requests, first call request_secret so the user enters the secret privately (never ask for it in chat or invent one), then set signature with the returned secretRef, e.g. GitHub: {type:'webhook', signature:{header:'x-hub-signature-256', encoding:'hex', prefix:'sha256=', secretRef}}. To run only for some requests, add filters, all of which must match, e.g. filters:[{path:'headers.x-github-event', op:'equals', value:'pull_request'}, {path:'body.action', op:'in', value:['opened','reopened']}]; ops are equals, notEquals, in, startsWith and exists. The result's webhookUrl is the public URL to give the user; if it is absent, this environment has no T3 Connect managed tunnel, so tell the user to enable T3 Connect remote access rather than sharing a path. Omit projectId for this thread's project. In this thread's project, runs post into THIS thread by default (bindToCurrentThread=true), which suits an orchestrator that sees every trigger, delegates work, and can dedupe against what is in flight; use false only when the user wants a fresh top-level thread per run. Elsewhere each run launches a fresh thread. Provider, model, and runtime settings inherit from this thread, or from the project default when there is no calling thread.", + "Create persistent work in the app scheduler that runs even when no turn is active. Pass schedule as a STRUCTURED OBJECT, never JSON text. Timers: {type:'interval', everyMs:3600000} is hourly; {type:'fixed_time', timeOfDay:'09:00', weekdays:[1,2,3,4,5]} is weekday mornings; report the returned nextRunAt. Webhook dedupe accepts replayKey and coalesceKey templates with coalesceWindowSeconds (10–3600, default 120); replay keys expire after 7 days and coalescing keeps the latest payload in a window measured from its first delivery. Webhooks: {type:'webhook'} runs once per request to a generated URL. The run sees the request ONLY through prompt placeholders: {{body.path}} (e.g. {{body.action}}, {{body.release.tag_name}}), {{headers.name}}, {{query.name}}, {{body}}, or {{request}} (method, headers with credentials redacted, and body). Each value arrives framed as untrusted request data, and $skill, @file or /command mentions inside it do not act; without a signature, anyone with the URL can trigger a run. For a sender that signs requests, first call request_secret so the user enters the secret privately (never ask for it in chat or invent one), then set signature with the returned secretRef, e.g. GitHub: {type:'webhook', signature:{header:'x-hub-signature-256', encoding:'hex', prefix:'sha256=', secretRef}}. To run only for some requests, add filters, all of which must match, e.g. filters:[{path:'headers.x-github-event', op:'equals', value:'pull_request'}, {path:'body.action', op:'in', value:['opened','reopened']}]; ops are equals, notEquals, in, startsWith and exists. The result's webhookUrl is the public URL to give the user; if it is absent, this environment has no T3 Connect managed tunnel, so tell the user to enable T3 Connect remote access rather than sharing a path. Omit projectId for this thread's project. In this thread's project, runs post into THIS thread by default (bindToCurrentThread=true), which suits an orchestrator that sees every trigger, delegates work, and can dedupe against what is in flight; use false only when the user wants a fresh top-level thread per run. Elsewhere each run launches a fresh thread. Provider, model, and runtime settings inherit from this thread, or from the project default when there is no calling thread.", parameters: OrchestratorMcpScheduleTaskInput, success: OrchestratorMcpScheduleTaskResult, failure: OrchestratorMcpFailure, @@ -143,7 +143,7 @@ const ListScheduledTasksTool = Tool.make("list_scheduled_tasks", { const UpdateScheduledTaskTool = Tool.make("update_scheduled_task", { description: - "Update an existing scheduled task by scheduledTaskId (from list_scheduled_tasks). Only the provided fields change; omit a field to leave it as-is. Use enabled=false to pause a task without deleting it. Set bindToCurrentThread to move the task between posting into this thread and launching a fresh thread per run. A webhook schedule without filters keeps the stored filters; pass filters to replace them, or filters:[] to clear them.", + "Update an existing scheduled task by scheduledTaskId (from list_scheduled_tasks). Webhook schedule dedupe accepts replayKey, coalesceKey, and coalesceWindowSeconds (10–3600, default 120) to suppress replays for 7 days and retain the latest payload in each fixed window. Only the provided fields change; omit a field to leave it as-is. Use enabled=false to pause a task without deleting it. Set bindToCurrentThread to move the task between posting into this thread and launching a fresh thread per run. A webhook schedule without filters keeps the stored filters; pass filters to replace them, or filters:[] to clear them.", parameters: OrchestratorMcpUpdateScheduledTaskInput, success: OrchestratorMcpScheduleTaskResult, failure: OrchestratorMcpFailure, diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 9a0dff477368a..8dfbfe712a7de 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -69,6 +69,7 @@ import Migration0063 from "./Migrations/063_DropUnusedV1Projections.ts"; import Migration0064 from "./Migrations/064_CanonicalizeThreadPullRequestLinks.ts"; import Migration0065 from "./Migrations/065_CodexUsageNativeTurnIndex.ts"; import Migration0066 from "./Migrations/066_RepairClientSettlementTimestamps.ts"; +import Migration0067 from "./Migrations/067_WebhookDedupe.ts"; export const migrationEntries = [ [1, "OrchestrationEvents", Migration0001], @@ -138,6 +139,7 @@ export const migrationEntries = [ [64, "CanonicalizeThreadPullRequestLinks", Migration0064], [65, "CodexUsageNativeTurnIndex", Migration0065], [66, "RepairClientSettlementTimestamps", Migration0066], + [67, "WebhookDedupe", Migration0067], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts b/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts index cd926c899d978..5bbf3cd9a7e62 100644 --- a/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts +++ b/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts @@ -15,7 +15,7 @@ layer("055_OrchestrationV2", (it) => { Effect.sync(() => { assert.deepStrictEqual( migrationEntries.map(([id]) => id), - Array.from({ length: 66 }, (_, index) => index + 1), + Array.from({ length: 67 }, (_, index) => index + 1), ); }), ); @@ -40,6 +40,7 @@ layer("055_OrchestrationV2", (it) => { [64, "CanonicalizeThreadPullRequestLinks"], [65, "CodexUsageNativeTurnIndex"], [66, "RepairClientSettlementTimestamps"], + [67, "WebhookDedupe"], ]); assert.deepStrictEqual(yield* runMigrations(), []); @@ -72,6 +73,7 @@ layer("055_OrchestrationV2", (it) => { { migration_id: 64, name: "CanonicalizeThreadPullRequestLinks" }, { migration_id: 65, name: "CodexUsageNativeTurnIndex" }, { migration_id: 66, name: "RepairClientSettlementTimestamps" }, + { migration_id: 67, name: "WebhookDedupe" }, ]); const tables = yield* sql<{ readonly name: string }>` diff --git a/apps/server/src/persistence/Migrations/067_WebhookDedupe.ts b/apps/server/src/persistence/Migrations/067_WebhookDedupe.ts new file mode 100644 index 0000000000000..b132e7c6dd146 --- /dev/null +++ b/apps/server/src/persistence/Migrations/067_WebhookDedupe.ts @@ -0,0 +1,29 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`ALTER TABLE scheduled_tasks ADD COLUMN last_run_webhook_delivery_id TEXT`; + yield* sql`CREATE TABLE scheduled_task_webhook_replay_keys ( + task_id TEXT NOT NULL REFERENCES scheduled_tasks(task_id) ON DELETE CASCADE, + replay_key TEXT NOT NULL, + delivery_id TEXT NOT NULL, + expires_at INTEGER NOT NULL, + PRIMARY KEY (task_id, replay_key) + )`; + yield* sql`CREATE INDEX idx_webhook_replay_expiry ON scheduled_task_webhook_replay_keys(expires_at)`; + yield* sql`CREATE INDEX idx_webhook_replay_delivery ON scheduled_task_webhook_replay_keys(delivery_id)`; + yield* sql`CREATE TABLE scheduled_task_webhook_pending ( + delivery_id TEXT PRIMARY KEY, + task_id TEXT NOT NULL REFERENCES scheduled_tasks(task_id) ON DELETE CASCADE, + coalesce_key TEXT NOT NULL, + due_at INTEGER NOT NULL, + state TEXT NOT NULL DEFAULT 'pending', + task_json TEXT NOT NULL, + request_json TEXT NOT NULL, + delivery_ids_json TEXT NOT NULL + )`; + yield* sql`CREATE INDEX idx_webhook_pending_due ON scheduled_task_webhook_pending(state, due_at)`; + yield* sql`CREATE UNIQUE INDEX idx_webhook_pending_key + ON scheduled_task_webhook_pending(task_id, coalesce_key) WHERE state = 'pending'`; +}); diff --git a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts index f1495d1e62a4f..990d450cde6b5 100644 --- a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts +++ b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts @@ -46,6 +46,7 @@ describe("V2 preview upgrade", () => { [64, "CanonicalizeThreadPullRequestLinks"], [65, "CodexUsageNativeTurnIndex"], [66, "RepairClientSettlementTimestamps"], + [67, "WebhookDedupe"], ]); assert.deepStrictEqual(yield* runMigrations(), []); assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports); @@ -135,6 +136,7 @@ describe("V2 preview upgrade", () => { [64, "CanonicalizeThreadPullRequestLinks"], [65, "CodexUsageNativeTurnIndex"], [66, "RepairClientSettlementTimestamps"], + [67, "WebhookDedupe"], ]); }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), ); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 89522077bae4d..ea681a30afe03 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -29,6 +29,7 @@ import * as Crypto from "effect/Crypto"; import * as Data from "effect/Data"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as Duration from "effect/Duration"; import * as Metric from "effect/Metric"; import * as Layer from "effect/Layer"; @@ -54,14 +55,30 @@ import { renderWebhookPrompt, type WebhookRequest, } from "./webhookTemplate.ts"; +import { + acceptWebhookDedupe, + acceptImmediateWebhookDedupe, + releaseWebhookReplay, + decodePendingTask, + decodePendingRequest, + dedupeError, +} from "./webhookDedupe.ts"; import { describeFailedWebhookFilter, findFailedWebhookFilter } from "./webhookFilter.ts"; import { constantTimeEquals, verifyWebhookSignature } from "./webhookVerification.ts"; export const WEBHOOK_ROUTE_PREFIX = "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/api/hooks"; +const isScheduledTaskError = Schema.is(ScheduledTaskError); +const PROMPT_TOO_LONG = "The filled-in prompt is too long."; const WEBHOOK_DELIVERY_RETENTION = 50; -/** Kept apart from the rest, so a stream of filtered requests can't evict real deliveries. */ +const WEBHOOK_DEDUPE_DELIVERY_RETENTION = 20; const WEBHOOK_FILTERED_DELIVERY_RETENTION = 20; -/** The outcomes clients decoded before outcomes became open-ended. */ +const WEBHOOK_DEDUPE_LOG_PER_MINUTE = 10; +const WEBHOOK_FILTERED_LOG_PER_MINUTE = 10; +const WEBHOOK_DELIVERY_LOG_BODY_LIMIT = 64 * 1024; +const WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT = 64 * 1024; +const WEBHOOK_MAX_QUEUED_PER_TASK = 20; +/** Accepted deliveries per task per minute, enforced here as well as on the relay because the tunnel hostname is public too. */ +const WEBHOOK_RATE_LIMIT_PER_MINUTE = 60; const LEGACY_DELIVERY_OUTCOMES: ReadonlySet = new Set([ "accepted", "dispatch_failed", @@ -70,16 +87,6 @@ const LEGACY_DELIVERY_OUTCOMES: ReadonlySet = new Set([ "rate_limited", "expired", ]); -/** - * Filtered requests take no run slot, so this bounds the log writes they - * cause instead; past it a filtered request is only counted in metrics. - */ -const WEBHOOK_FILTERED_LOG_PER_MINUTE = 10; -const WEBHOOK_DELIVERY_LOG_BODY_LIMIT = 64 * 1024; -const WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT = 64 * 1024; -const WEBHOOK_MAX_QUEUED_PER_TASK = 20; -/** Accepted deliveries per task per minute, enforced here as well as on the relay because the tunnel hostname is public too. */ -const WEBHOOK_RATE_LIMIT_PER_MINUTE = 60; interface WebhookOrigin { readonly relayHookBaseUrl: string | null; @@ -111,7 +118,7 @@ class WebhookDeliverySkipped extends Data.TaggedError("WebhookDeliverySkipped")< }> {} interface RateWindow { - readonly accepted: ReadonlyArray; + readonly accepted: ReadonlyArray<{ readonly at: number }>; readonly rejectedLogged: boolean; } @@ -126,6 +133,7 @@ export interface WebhookTriggerRequest extends WebhookRequest { export type WebhookDeliveryOutcome = | "accepted" | "duplicate" + | "coalesced" | "prompt_too_long" | "queue_full" | "rate_limited" @@ -140,7 +148,7 @@ export type WebhookTriggerResult = | { readonly _tag: "accepted"; readonly deliveryId: ScheduledTaskWebhookDeliveryId; - readonly outcome: "accepted" | "duplicate" | "prompt_too_long" | "filtered"; + readonly outcome: "accepted" | "duplicate" | "coalesced" | "prompt_too_long" | "filtered"; } | { readonly _tag: "not_found" } | { readonly _tag: "rejected_signature" } @@ -211,6 +219,7 @@ interface WebhookDeliveryRow { readonly signature_verified: number; readonly missing_fields_json: string; readonly rendered_prompt: string | null; + readonly pending_window: number; readonly error: string | null; } @@ -405,6 +414,13 @@ export const layer = Layer.effect( // finish late release their own count, never a recreated task's. const webhookQueued = yield* Ref.make>(new Map()); const webhookRateWindows = yield* Ref.make>(new Map()); + const webhookRejectedRateWindows = yield* Ref.make>( + new Map(), + ); + const webhookDedupeLogWindows = yield* Ref.make< + ReadonlyMap> + >(new Map()); + const pendingInFlight = yield* Ref.make>(new Set()); const webhookFilteredLogWindows = yield* Ref.make< ReadonlyMap> >(new Map()); @@ -627,11 +643,12 @@ export const layer = Layer.effect( ), ); - const markRunning = (id: ScheduledTaskId, startedAtIso: string) => + const markRunning = (id: ScheduledTaskId, startedAtIso: string, webhookDeliveryId?: string) => sql<{ task_id: string }>` UPDATE scheduled_tasks SET updated_at = ${startedAtIso}, last_run_at = ${startedAtIso}, + last_run_webhook_delivery_id = ${webhookDeliveryId ?? null}, last_run_status = 'running', last_run_error = NULL WHERE task_id = ${id} @@ -673,9 +690,10 @@ export const layer = Layer.effect( // markCompleted, write a full terminal record so runDueTasks neither skips // the task forever (it filters out 'running' rows) nor re-fires it // immediately: the dispatch may already have gone out, so next_run_at must - // advance and run_count must count the attempt. + // advance. Pending webhook attempts count only with their completion state; + // other attempts count here. const releaseStuckRun = Effect.fn("ScheduledTaskService.releaseStuckRun")( - function* (task: ScheduledTask, message: string) { + function* (task: ScheduledTask, message: string, pendingDeliveryId?: string) { const now = yield* localNow; const reread = yield* Effect.result(findTask(task.id)); if (Result.isSuccess(reread) && reread.success === null) return; @@ -686,12 +704,15 @@ export const layer = Layer.effect( last_run_error = ${message}, next_run_at = ${nextRunAt(source, now)}, updated_at = ${iso(now)}, - run_count = run_count + 1 + run_count = run_count + CASE WHEN EXISTS ( + SELECT 1 FROM scheduled_task_webhook_pending + WHERE delivery_id = ${pendingDeliveryId ?? null} AND state = 'running' + ) THEN 0 ELSE 1 END WHERE task_id = ${task.id} AND last_run_status = 'running' `; yield* notifyChanged; }, - (effect, task, _message) => + (effect, task, _message, _pendingDeliveryId) => effect.pipe( Effect.catch((cause) => Effect.logWarning("Could not release stuck schedule task run", { @@ -705,7 +726,11 @@ export const layer = Layer.effect( const runTask = Effect.fn("ScheduledTaskService.runTask")(function* ( task: ScheduledTask, trigger: "scheduled" | "manual" | "webhook", - webhook?: { readonly deliveryId: string; readonly prompt: string }, + webhook?: { + readonly deliveryId: string; + readonly prompt: string; + readonly dispatched: Ref.Ref; + }, ) { const reserved = yield* Ref.modify(activeRuns, (active) => { if (active.has(task.id)) return [false, active] as const; @@ -739,6 +764,12 @@ export const layer = Layer.effect( } return { task, threadId: null }; } + if ( + webhook !== undefined && + (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE delivery_id = ${webhook.deliveryId} AND state = 'completed'`) + .length > 0 + ) + return { task: { ...active, lastRunStatus: "succeeded" as const }, threadId: null }; const parsedNextRunAt = active.nextRunAt === null ? Option.none() : DateTime.make(active.nextRunAt); if ( @@ -764,7 +795,22 @@ export const layer = Layer.effect( if (reason !== null) return yield* new WebhookDeliverySkipped({ reason }); } - yield* markRunning(active.id, startedAtIso); + if (webhook === undefined) yield* markRunning(active.id, startedAtIso); + else + yield* sql + .withTransaction( + Effect.gen(function* () { + yield* markRunning(active.id, startedAtIso, webhook.deliveryId); + // Recovery must distinguish this delivery from other windows waiting on the same task. + yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'running' WHERE delivery_id = ${webhook.deliveryId}`; + }), + ) + .pipe( + Effect.mapError((cause) => + taskError("Could not start webhook run.", { taskId: task.id, cause }), + ), + ); + yield* notifyChanged; const fireKey = @@ -821,8 +867,9 @@ export const layer = Layer.effect( .pipe(Effect.as(ThreadId.make(active.threadId))), ); - const completedAt = yield* localNow; const runSucceeded = result._tag === "Success"; + if (webhook !== undefined && runSucceeded) yield* Ref.set(webhook.dispatched, true); + const completedAt = yield* localNow; const lastRunStatus = runSucceeded ? ("succeeded" as const) : ("failed" as const); const lastRunError = runSucceeded ? null : errorMessage(result.cause); const threadId = runSucceeded ? result.value : null; @@ -841,7 +888,7 @@ export const layer = Layer.effect( // startedAtIso in the guard ensures this writes only to the row this // run marked as running — a task deleted mid-run and recreated with // the same id (idempotent commandId replay) must not be stamped. - yield* markCompleted({ + const completion = markCompleted({ id: task.id, completedAtIso: completed.updatedAt, nextRunAtIso: completed.nextRunAt, @@ -849,11 +896,32 @@ export const layer = Layer.effect( error: lastRunError, startedAtIso, }); + if (webhook === undefined) yield* completion; + else + yield* sql + .withTransaction( + Effect.gen(function* () { + const pending = + yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE delivery_id = ${webhook.deliveryId}`; + if (pending.length === 0) yield* completion; + else { + const counted = + yield* sql`UPDATE scheduled_task_webhook_pending SET state = ${runSucceeded ? "completed" : "failed"} + WHERE delivery_id = ${webhook.deliveryId} AND state NOT IN ('completed', 'failed') RETURNING delivery_id`; + if (counted.length > 0) yield* completion; + } + }), + ) + .pipe( + Effect.mapError((cause) => + taskError("Could not record webhook completion.", { taskId: task.id, cause }), + ), + ); yield* notifyChanged; } return { task: completed, threadId }; }).pipe( - Effect.onError((cause) => releaseStuckRun(task, errorMessage(cause))), + Effect.onError((cause) => releaseStuckRun(task, errorMessage(cause), webhook?.deliveryId)), Effect.ensuring( Ref.update(activeRuns, (active) => { const next = new Set(active); @@ -930,7 +998,11 @@ export const layer = Layer.effect( last_run_error = 'Run was interrupted by a server restart.', next_run_at = ${nextRunAt(decoded.success, now)}, updated_at = ${iso(now)}, - run_count = run_count + 1 + run_count = run_count + CASE WHEN EXISTS ( + SELECT 1 FROM scheduled_task_webhook_pending + WHERE task_id IS ${row.task_id} AND state = 'running' + AND delivery_id = scheduled_tasks.last_run_webhook_delivery_id + ) THEN 0 ELSE 1 END WHERE task_id IS ${row.task_id} AND last_run_status = 'running' `; return; @@ -944,7 +1016,11 @@ export const layer = Layer.effect( SET last_run_status = 'failed', last_run_error = 'Run was interrupted by a server restart.', updated_at = ${iso(now)}, - run_count = run_count + 1 + run_count = run_count + CASE WHEN EXISTS ( + SELECT 1 FROM scheduled_task_webhook_pending + WHERE task_id IS ${row.task_id} AND state = 'running' + AND delivery_id = scheduled_tasks.last_run_webhook_delivery_id + ) THEN 0 ELSE 1 END WHERE task_id IS ${row.task_id} AND last_run_status = 'running' `; }), @@ -1017,6 +1093,12 @@ export const layer = Layer.effect( prefix: input.schedule.signature.prefix, }, maxDeliveryAgeMinutes: input.schedule.maxDeliveryAgeMinutes ?? null, + ...(input.schedule.dedupe === undefined + ? existingTask?.schedule.type === "webhook" && + existingTask.schedule.dedupe !== undefined + ? { dedupe: existingTask.schedule.dedupe } + : {} + : { dedupe: input.schedule.dedupe }), ...webhookFilters( // Omitted keeps the stored filters, so a client that predates // them can re-save a task without clearing them. @@ -1139,6 +1221,16 @@ export const layer = Layer.effect( next.delete(id); return next; }), + Ref.update(webhookRejectedRateWindows, (windows) => { + const next = new Map(windows); + next.delete(id); + return next; + }), + Ref.update(webhookDedupeLogWindows, (windows) => { + const next = new Map(windows); + next.delete(id); + return next; + }), Ref.update(webhookFilteredLogWindows, (windows) => { const next = new Map(windows); next.delete(id); @@ -1243,6 +1335,7 @@ export const layer = Layer.effect( bodyBytes: row.body_bytes, outcome: row.outcome as ScheduledTaskWebhookDeliveryOutcome, signatureVerified: row.signature_verified === 1, + pendingWindow: row.outcome === "accepted" && row.pending_window === 1, missingFields: Option.getOrElse(decodeMissingFieldsJson(row.missing_fields_json), () => []), error: row.error, }); @@ -1251,15 +1344,17 @@ export const layer = Layer.effect( input, ) => sql` - SELECT * FROM scheduled_task_webhook_deliveries - WHERE task_id = ${input.id} + SELECT d.*, EXISTS (SELECT 1 FROM scheduled_task_webhook_pending p + WHERE p.delivery_id = d.delivery_id AND p.state IN ('pending', 'dispatching')) AS pending_window + FROM scheduled_task_webhook_deliveries d WHERE task_id = ${input.id} ORDER BY received_at DESC, rowid DESC `.pipe( Effect.map((rows) => ({ - deliveries: (input.acceptAnyOutcome === true - ? rows - : rows.filter((row) => LEGACY_DELIVERY_OUTCOMES.has(row.outcome)) - ).map(decodeDeliverySummary), + deliveries: rows + .filter( + (row) => input.acceptAnyOutcome === true || LEGACY_DELIVERY_OUTCOMES.has(row.outcome), + ) + .map(decodeDeliverySummary), })), Effect.mapError((cause) => taskError("Could not list webhook deliveries.", { taskId: input.id, cause }), @@ -1270,15 +1365,19 @@ export const layer = Layer.effect( "ScheduledTaskService.getWebhookDelivery", )(function* (input) { const rows = yield* sql` - SELECT * FROM scheduled_task_webhook_deliveries - WHERE task_id = ${input.id} AND delivery_id = ${input.deliveryId} + SELECT d.*, EXISTS (SELECT 1 FROM scheduled_task_webhook_pending p + WHERE p.delivery_id = d.delivery_id AND p.state IN ('pending', 'dispatching')) AS pending_window + FROM scheduled_task_webhook_deliveries d WHERE task_id = ${input.id} AND delivery_id = ${input.deliveryId} `.pipe( Effect.mapError((cause) => taskError("Could not load webhook delivery.", { taskId: input.id, cause }), ), ); const row = rows[0]; - if (row === undefined) { + if ( + row === undefined || + (input.acceptAnyOutcome !== true && !LEGACY_DELIVERY_OUTCOMES.has(row.outcome)) + ) { return yield* taskError("Webhook delivery not found.", { taskId: input.id }); } return { @@ -1293,6 +1392,24 @@ export const layer = Layer.effect( }; }); + const deliveryRequestBody = (request: WebhookTriggerRequest) => { + const truncated = request.body.byteLength > WEBHOOK_DELIVERY_LOG_BODY_LIMIT; + return { + truncated, + body: truncated + ? new TextDecoder().decode(request.body.subarray(0, WEBHOOK_DELIVERY_LOG_BODY_LIMIT)) + : request.bodyText, + }; + }; + const replaceDeliveryRequest = (deliveryId: string, request: WebhookTriggerRequest) => { + const logged = deliveryRequestBody(request); + return sql`UPDATE scheduled_task_webhook_deliveries SET + method = ${request.method}, query = ${redactQuery(request.query)}, + headers_json = ${encodeHeadersJson(redactHeaders(request.headers))}, body = ${logged.body}, + body_bytes = ${request.body.byteLength}, body_truncated = ${logged.truncated ? 1 : 0} + WHERE delivery_id = ${deliveryId}`.pipe(Effect.asVoid, Effect.mapError(dedupeError)); + }; + const recordDelivery = (input: { readonly id: string; readonly taskId: ScheduledTaskId; @@ -1304,10 +1421,7 @@ export const layer = Layer.effect( readonly renderedPrompt: string | null; readonly error?: string; }) => { - const truncated = input.request.body.byteLength > WEBHOOK_DELIVERY_LOG_BODY_LIMIT; - const loggedBody = truncated - ? new TextDecoder().decode(input.request.body.subarray(0, WEBHOOK_DELIVERY_LOG_BODY_LIMIT)) - : input.request.bodyText; + const { truncated, body: loggedBody } = deliveryRequestBody(input.request); return sql .withTransaction( Effect.gen(function* () { @@ -1338,17 +1452,28 @@ export const layer = Layer.effect( rendered_prompt = excluded.rendered_prompt, error = excluded.error `; - const filtered = input.outcome === "filtered" ? 1 : 0; + // Each suppression bucket retains its own rows, never real run history. + const bucket = (outcome: string, error: string | undefined) => + outcome === "filtered" + ? "filtered" + : outcome === "duplicate" || + outcome === "coalesced" || + outcome === "rate_limited" || + error === PROMPT_TOO_LONG + ? "dedupe" + : "main"; + const retentionBucket = bucket(input.outcome, input.error); + const rowBucket = sql.literal(`CASE WHEN outcome = 'filtered' THEN 'filtered' + WHEN outcome IN ('duplicate', 'coalesced', 'rate_limited') OR error = 'The filled-in prompt is too long.' THEN 'dedupe' + ELSE 'main' END`); yield* sql` DELETE FROM scheduled_task_webhook_deliveries - WHERE task_id = ${input.taskId} - AND (outcome = 'filtered') = ${filtered} + WHERE task_id = ${input.taskId} AND ${rowBucket} = ${retentionBucket} AND delivery_id NOT IN ( SELECT delivery_id FROM scheduled_task_webhook_deliveries - WHERE task_id = ${input.taskId} AND (outcome = 'filtered') = ${filtered} - -- rowid breaks timestamp ties in arrival order; delivery ids are random. + WHERE task_id = ${input.taskId} AND ${rowBucket} = ${retentionBucket} ORDER BY received_at DESC, rowid DESC - LIMIT ${filtered ? WEBHOOK_FILTERED_DELIVERY_RETENTION : WEBHOOK_DELIVERY_RETENTION} + LIMIT ${retentionBucket === "main" ? WEBHOOK_DELIVERY_RETENTION : retentionBucket === "filtered" ? WEBHOOK_FILTERED_DELIVERY_RETENTION : WEBHOOK_DEDUPE_DELIVERY_RETENTION} ) `; }), @@ -1367,13 +1492,15 @@ export const layer = Layer.effect( WHERE delivery_id = ${deliveryId} `.pipe(Effect.ignore); - /** - * Sliding one-minute window counting every request with a valid token, - * including ones the signature check later rejects. - */ - const takeRateSlot = (id: ScheduledTaskId, nowMs: number) => + /** Sliding one-minute window for deliveries that can start work. */ + const takeRateSlot = ( + id: ScheduledTaskId, + nowMs: number, + rateWindows = webhookRateWindows, + reservation = { at: nowMs }, + ) => Ref.modify( - webhookRateWindows, + rateWindows, ( windows, ): readonly [ @@ -1381,7 +1508,7 @@ export const layer = Layer.effect( ReadonlyMap, ] => { const current = windows.get(id); - const recent = (current?.accepted ?? []).filter((at) => nowMs - at < 60_000); + const recent = (current?.accepted ?? []).filter((entry) => nowMs - entry.at < 60_000); if (recent.length >= WEBHOOK_RATE_LIMIT_PER_MINUTE) { const first = current?.rejectedLogged !== true; return [ @@ -1391,11 +1518,23 @@ export const layer = Layer.effect( } return [ "allowed", - new Map(windows).set(id, { accepted: [...recent, nowMs], rejectedLogged: false }), + new Map(windows).set(id, { accepted: [...recent, reservation], rejectedLogged: false }), ]; }, ); + const takeDedupeLogSlot = (id: ScheduledTaskId, nowMs: number) => + Ref.modify(webhookDedupeLogWindows, (windows) => { + const recent = (windows.get(id) ?? []).filter((at) => nowMs - at < 60_000); + return recent.length >= WEBHOOK_DEDUPE_LOG_PER_MINUTE + ? ([false, windows] as const) + : ([true, new Map(windows).set(id, [...recent, nowMs])] as const); + }); + const releaseReplay = (deliveryId: string) => + releaseWebhookReplay(deliveryId).pipe( + Effect.provideService(SqlClient.SqlClient, sql), + Effect.mapError(dedupeError), + ); /** Sliding one-minute window of filtered requests written to the delivery log. */ const takeFilteredLogSlot = (id: ScheduledTaskId, nowMs: number) => Ref.modify( @@ -1424,6 +1563,83 @@ export const layer = Layer.effect( const serviceScope = yield* Effect.scope; + const releaseFailedDelivery = Effect.fn("ScheduledTaskService.releaseFailedDelivery")( + function* (deliveryId: string) { + yield* releaseReplay(deliveryId); + yield* sql`DELETE FROM scheduled_task_webhook_relay_deliveries WHERE relay_delivery_id IN ( + SELECT substr(value, ${"delivery:relay:".length + 1}) FROM scheduled_task_webhook_pending, json_each(delivery_ids_json) + WHERE delivery_id = ${deliveryId} AND value LIKE 'delivery:relay:%' + )`.pipe(Effect.mapError(dedupeError)); + if (deliveryId.startsWith("delivery:relay:")) + yield* sql`DELETE FROM scheduled_task_webhook_relay_deliveries WHERE relay_delivery_id = ${deliveryId.slice("delivery:relay:".length)}`.pipe( + Effect.mapError(dedupeError), + ); + }, + (effect) => sql.withTransaction(effect).pipe(Effect.mapError(dedupeError)), + ); + + const runWebhookDelivery = Effect.fn("ScheduledTaskService.runWebhookDelivery")(function* ( + task: ScheduledTask, + deliveryId: string, + prompt: string, + ) { + const runOutcome = (outcome: "started" | "skipped" | "failed") => + Effect.all([ + Effect.annotateCurrentSpan({ "scheduled_task.webhook.run_outcome": outcome }), + Metrics.increment(Metrics.webhookRunsTotal, { outcome }), + ]); + const dispatched = yield* Ref.make(false); + return yield* runTask(task, "webhook", { deliveryId, prompt, dispatched }).pipe( + Effect.flatMap(({ task: completed }) => + completed.lastRunStatus === "failed" + ? runOutcome("failed").pipe( + Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")), + Effect.andThen(releaseFailedDelivery(deliveryId)), + ) + : runOutcome("started"), + ), + Effect.catchTags({ + WebhookDeliverySkipped: (skipped) => + runOutcome("skipped").pipe( + Effect.andThen(markDeliveryFailed(deliveryId, skipped.reason)), + Effect.andThen(releaseFailedDelivery(deliveryId)), + ), + }), + Effect.as(true), + // Dispatch success owns the replay key even if later bookkeeping fails. + Effect.catchCause((cause) => + Effect.gen(function* () { + const started = yield* Ref.get(dispatched); + yield* Effect.logWarning("Webhook dispatch failed", { taskId: task.id, cause }); + yield* runOutcome(started ? "started" : "failed"); + yield* markDeliveryFailed( + deliveryId, + started + ? "The run started, but completion could not be recorded." + : "The run failed to start.", + ); + if (!started) { + yield* sql + .withTransaction( + Effect.gen(function* () { + const counted = + yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'failed' + WHERE delivery_id = ${deliveryId} AND state = 'running' RETURNING delivery_id`; + if (counted.length > 0) + yield* sql`UPDATE scheduled_tasks SET run_count = run_count + 1 + WHERE task_id = ${task.id} AND created_at = ${task.createdAt}`; + }), + ) + .pipe(Effect.mapError(dedupeError)); + yield* releaseFailedDelivery(deliveryId); + } + // Pending windows retry the same command id to finish bookkeeping. + return !started; + }), + ), + ); + }); + // Delivery attributes belong to the enclosing triggerWebhook span. const observeDelivery = Effect.fnUntraced(function* ( request: WebhookTriggerRequest, @@ -1571,71 +1787,168 @@ export const layer = Layer.effect( ...(details.error === undefined ? {} : { error: details.error }), }); + // Reject floods before signature work or rejection-log writes; valid + // deliveries give this reservation back once validation succeeds. + const rejectionReservation = { at: DateTime.toEpochMillis(now) }; + const rejectionSlot = yield* takeRateSlot( + task.id, + DateTime.toEpochMillis(now), + webhookRejectedRateWindows, + rejectionReservation, + ); + if (rejectionSlot !== "allowed") { + if (rejectionSlot === "first_rejected") yield* log("rate_limited"); + yield* observe("rate_limited"); + return yield* releaseClaim({ _tag: "rate_limited", outcome: "rate_limited" } as const); + } + if (!task.enabled) { + yield* log("disabled"); + yield* observe("disabled"); + return { _tag: "disabled" as const }; + } const signature = schedule.signature; - const signatureVerified = - signature === null || - (row.webhook_secret !== null && + if (signature !== null) { + const verified = + row.webhook_secret !== null && verifyWebhookSignature({ signature, secret: row.webhook_secret, headers: request.headers, body: request.body, - })); + }); + if (!verified) { + yield* log("rejected_signature"); + yield* observe("rejected_signature"); + return { _tag: "rejected_signature" as const }; + } + } const maxAgeMinutes = schedule.maxDeliveryAgeMinutes ?? null; - const tooOld = + if ( maxAgeMinutes !== null && DateTime.toEpochMillis(now) - DateTime.toEpochMillis(receivedAt) > - maxAgeMinutes * 60_000; - - // Filters: checked after the signature and age, and before the rate - // limit, so requests the task ignores never take a run's slot. The - // request is accepted so the sender doesn't retry it, and its relay - // claim is released because nothing ran. - if (task.enabled && signatureVerified && !tooOld) { - const failedFilter = findFailedWebhookFilter(schedule.filters ?? [], request); - if (failedFilter !== null) { - if (yield* takeFilteredLogSlot(task.id, DateTime.toEpochMillis(now))) { - yield* log("filtered", { - signatureVerified: signature !== null, - error: describeFailedWebhookFilter(failedFilter), - }); - } - yield* observe("filtered"); - return yield* releaseClaim({ - _tag: "accepted" as const, - deliveryId, - outcome: "filtered" as const, - }); - } + maxAgeMinutes * 60_000 + ) { + yield* log("expired", { signatureVerified: signature !== null }); + yield* observe("expired"); + return { _tag: "expired" as const }; } - // Only the first rejected request in a window is logged, so a flood - // cannot write rows or push the real deliveries out of the log. - const slot = yield* takeRateSlot(task.id, DateTime.toEpochMillis(now)); - if (slot !== "allowed") { - if (slot === "first_rejected") yield* log("rate_limited"); - yield* observe("rate_limited"); + const refundRejectionSlot = Ref.update(webhookRejectedRateWindows, (windows) => { + const current = windows.get(task.id); + if (current === undefined) return windows; + return new Map(windows).set(task.id, { + ...current, + accepted: current.accepted.filter((entry) => entry !== rejectionReservation), + }); + }); + + const failedFilter = findFailedWebhookFilter(schedule.filters ?? [], request); + if (failedFilter !== null) { + if (yield* takeFilteredLogSlot(task.id, DateTime.toEpochMillis(now))) { + yield* log("filtered", { + signatureVerified: signature !== null, + error: describeFailedWebhookFilter(failedFilter), + }); + } + yield* refundRejectionSlot; + yield* observe("filtered"); return yield* releaseClaim({ - _tag: "rate_limited" as const, - outcome: "rate_limited" as const, + _tag: "accepted" as const, + deliveryId, + outcome: "filtered" as const, }); } - if (!task.enabled) { - yield* log("disabled"); - yield* observe("disabled"); - return { _tag: "disabled" as const }; + + const runWindow = (yield* Ref.get(webhookRateWindows)).get(task.id); + if ( + schedule.dedupe === undefined && + (runWindow?.accepted.filter((entry) => DateTime.toEpochMillis(now) - entry.at < 60_000) + .length ?? 0) >= WEBHOOK_RATE_LIMIT_PER_MINUTE + ) { + if ((yield* takeRateSlot(task.id, DateTime.toEpochMillis(now))) === "first_rejected") + yield* log("rate_limited"); + yield* observe("rate_limited"); + return yield* releaseClaim({ _tag: "rate_limited", outcome: "rate_limited" } as const); } - if (!signatureVerified) { - yield* log("rejected_signature"); - yield* observe("rejected_signature"); - return { _tag: "rejected_signature" as const }; + const reserve = takeRateSlot(task.id, DateTime.toEpochMillis(now)).pipe( + Effect.flatMap((slot) => (slot === "allowed" ? Effect.void : Effect.fail(slot))), + ); + const queueKey = `${task.id}\u0000${task.createdAt}`; + const checkCapacity = Effect.gen(function* () { + const [row] = yield* sql<{ + count: number; + }>`SELECT COUNT(*) AS count FROM scheduled_task_webhook_pending + WHERE task_id = ${task.id} AND state IN ('pending', 'dispatching', 'running')`; + if ( + (row?.count ?? 0) + ((yield* Ref.get(webhookQueued)).get(queueKey) ?? 0) >= + WEBHOOK_MAX_QUEUED_PER_TASK + ) + return yield* Effect.fail("queue_full" as const); + }).pipe( + Effect.mapError((cause) => (cause === "queue_full" ? cause : dedupeError(cause))), + ); + const admit = (effect: Effect.Effect) => + sql.withTransaction(effect).pipe( + Effect.catch((cause) => + cause === "first_rejected" || cause === "rejected" || cause === "queue_full" + ? Effect.succeed({ + outcome: + cause === "queue_full" + ? ("queue_full" as const) + : ("rate_limited" as const), + log: cause === "first_rejected", + }) + : Effect.fail(isScheduledTaskError(cause) ? cause : dedupeError(cause)), + ), + ); + const deduped = + schedule.dedupe === undefined + ? "run" + : yield* admit( + Effect.gen(function* () { + const outcome = yield* acceptWebhookDedupe( + task, + request, + deliveryId, + DateTime.toEpochMillis(now), + reserve, + takeDedupeLogSlot(task.id, DateTime.toEpochMillis(now)), + checkCapacity, + (ownerId) => replaceDeliveryRequest(ownerId, request), + ).pipe(Effect.provideService(SqlClient.SqlClient, sql)); + if ( + outcome === "pending" || + outcome === "coalesced" || + (outcome === "duplicate" && + (yield* takeDedupeLogSlot(task.id, DateTime.toEpochMillis(now)))) + ) + yield* log(outcome === "pending" ? "accepted" : outcome, { + signatureVerified: signature !== null, + }); + return outcome; + }), + ); + if (typeof deduped !== "string") { + yield* refundRejectionSlot; + if (deduped.log) yield* log("rate_limited"); + yield* observe(deduped.outcome); + return yield* releaseClaim({ _tag: "rate_limited", outcome: deduped.outcome } as const); } - if (tooOld) { - yield* log("expired", { signatureVerified: signature !== null }); - yield* observe("expired"); - return { _tag: "expired" as const }; + if (deduped !== "run") { + if (deduped !== "coalesced" && deduped !== "coalesced_unlogged") + yield* refundRejectionSlot; + const outcome = + deduped === "pending" + ? "accepted" + : deduped === "coalesced_unlogged" + ? "coalesced" + : deduped; + yield* observe(outcome); + if (deduped === "duplicate") + return yield* releaseClaim({ _tag: "accepted", deliveryId, outcome } as const); + claimAcquired = false; + return { _tag: "accepted", deliveryId, outcome } as const; } - const frameId = yield* crypto.randomBytes(6).pipe( Effect.map((bytes) => Buffer.from(bytes).toString("hex")), Effect.mapError((cause) => taskError("Could not generate frame id.", { cause })), @@ -1648,85 +1961,192 @@ export const layer = Layer.effect( signatureVerified: signature !== null, missing: rendered.missing, renderedPrompt: rendered.prompt, - error: "The filled-in prompt is too long.", + error: PROMPT_TOO_LONG, }); + yield* releaseReplay(deliveryId); yield* observe("prompt_too_long"); - return { _tag: "accepted" as const, deliveryId, outcome: "prompt_too_long" as const }; + return yield* releaseClaim({ + _tag: "accepted" as const, + deliveryId, + outcome: "prompt_too_long" as const, + }); } // Bound the deliveries one task holds, so steady traffic to a stuck // task cannot pile up parked fibers. - const queueKey = `${task.id}\u0000${task.createdAt}`; - const queued = yield* Ref.modify(webhookQueued, (counts) => { - const count = counts.get(queueKey) ?? 0; - return count >= WEBHOOK_MAX_QUEUED_PER_TASK - ? ([false, counts] as const) - : ([true, new Map(counts).set(queueKey, count + 1)] as const); + yield* refundRejectionSlot; + let queueReserved = false; + const release = Effect.suspend(() => { + if (!queueReserved) return Effect.void; + queueReserved = false; + return Ref.update(webhookQueued, (counts) => { + const next = new Map(counts); + const count = (next.get(queueKey) ?? 1) - 1; + if (count <= 0) next.delete(queueKey); + else next.set(queueKey, count); + return next; + }); }); - if (!queued) { - yield* observe("queue_full"); + // Serialize immediate reservations with persisted window admission. + const reserveImmediate = Effect.gen(function* () { + yield* checkCapacity; + yield* reserve; + yield* Ref.update(webhookQueued, (counts) => + new Map(counts).set(queueKey, (counts.get(queueKey) ?? 0) + 1), + ); + queueReserved = true; + }); + const admitted = yield* admit( + Effect.gen(function* () { + const outcome = yield* acceptImmediateWebhookDedupe( + task, + request, + deliveryId, + DateTime.toEpochMillis(now), + reserveImmediate, + ).pipe(Effect.provideService(SqlClient.SqlClient, sql)); + if (outcome === "duplicate") { + if (yield* takeDedupeLogSlot(task.id, DateTime.toEpochMillis(now))) + yield* log("duplicate", { signatureVerified: signature !== null }); + } else + yield* log("accepted", { + signatureVerified: signature !== null, + missing: rendered.missing, + renderedPrompt: rendered.prompt, + }); + return outcome; + }), + ).pipe(Effect.onError(() => release)); + if (typeof admitted !== "string") { + yield* release; + if (admitted.log) yield* log("rate_limited"); + yield* observe(admitted.outcome); return yield* releaseClaim({ - _tag: "rate_limited" as const, - outcome: "queue_full" as const, - }); + _tag: "rate_limited", + outcome: admitted.outcome, + } as const); + } + if (admitted === "duplicate") { + yield* release; + yield* observe("duplicate"); + return yield* releaseClaim({ + _tag: "accepted", + deliveryId, + outcome: "duplicate", + } as const); } - const release = Ref.update(webhookQueued, (counts) => { - const next = new Map(counts); - const count = (next.get(queueKey) ?? 1) - 1; - if (count <= 0) next.delete(queueKey); - else next.set(queueKey, count); - return next; - }); - yield* log("accepted", { - signatureVerified: signature !== null, - missing: rendered.missing, - renderedPrompt: rendered.prompt, - }).pipe(Effect.onError(() => release)); yield* observe("accepted"); const permit = yield* webhookPermit(task.id); - const runOutcome = (outcome: "started" | "skipped" | "failed") => - Effect.all([ - Effect.annotateCurrentSpan({ "scheduled_task.webhook.run_outcome": outcome }), - Metrics.increment(Metrics.webhookRunsTotal, { outcome }), - ]); - yield* runTask(task, "webhook", { deliveryId, prompt: rendered.prompt }).pipe( - Effect.flatMap(({ task: completed }) => - completed.lastRunStatus === "failed" - ? runOutcome("failed").pipe( - Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")), - ) - : runOutcome("started"), - ), - Effect.catchTags({ - WebhookDeliverySkipped: (skipped) => - runOutcome("skipped").pipe( - Effect.andThen(markDeliveryFailed(deliveryId, skipped.reason)), - ), - }), - // The log is readable over RPC, so it gets a fixed reason; the - // cause, which can carry request data, stays in the server log. - Effect.catchCause((cause) => - Effect.logWarning("Webhook dispatch failed", { taskId: task.id, cause }).pipe( - Effect.andThen(runOutcome("failed")), - Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")), - ), - ), + yield* runWebhookDelivery(task, deliveryId, rendered.prompt).pipe( permit.withPermits(1), - Effect.withSpan("ScheduledTaskService.runWebhookDelivery", { - root: true, - attributes: { - "scheduled_task.id": task.id, - "scheduled_task.webhook.delivery_id": deliveryId, - }, - }), Effect.ensuring(release), Effect.forkIn(serviceScope), ); // Dispatch owns the accepted delivery now; retries must keep its claim. claimAcquired = false; return { _tag: "accepted" as const, deliveryId, outcome: "accepted" as const }; - }).pipe(Effect.onError(() => releaseClaim(undefined))), + }).pipe( + Effect.onError(() => + Effect.all([ + releaseClaim(undefined), + (request.relayDeliveryId === undefined || claimAcquired + ? releaseReplay(deliveryId) + : Effect.void + ).pipe(Effect.ignore), + ]), + ), + ), + ); + }); + + const dispatchPending = Effect.fn("ScheduledTaskService.dispatchPendingWebhooks")(function* () { + const now = DateTime.toEpochMillis(yield* localNow); + yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'dispatching' WHERE state = 'pending' AND due_at <= ${now}`; + const tasks = yield* sql<{ task_id: ScheduledTaskId }>` + SELECT DISTINCT task_id FROM scheduled_task_webhook_pending WHERE state IN ('dispatching', 'running', 'completed', 'failed') + `; + yield* Effect.forEach( + tasks, + ({ task_id: taskId }) => + Effect.gen(function* () { + const reserved = yield* Ref.modify(pendingInFlight, (active) => + active.has(taskId) + ? ([false, active] as const) + : ([true, new Set(active).add(taskId)] as const), + ); + if (!reserved) return; + const permit = yield* webhookPermit(taskId); + // One worker per task waits on its permit, then drains persisted rows in order. + // Other tasks remain independent and ticks never reload an active task's payloads. + const dispatch = Effect.gen(function* () { + while (true) { + const [entry] = yield* sql<{ + delivery_id: string; + task_json: string; + request_json: string; + state: string; + }>` + SELECT delivery_id, task_json, request_json, state FROM scheduled_task_webhook_pending + WHERE task_id = ${taskId} AND state IN ('dispatching', 'running', 'completed', 'failed') ORDER BY due_at, rowid LIMIT 1 + `; + if (entry === undefined) return; + const remove = sql`DELETE FROM scheduled_task_webhook_pending WHERE delivery_id = ${entry.delivery_id}`; + if (entry.state === "completed" || entry.state === "failed") { + if (entry.state === "failed") { + yield* markDeliveryFailed(entry.delivery_id, "The run failed to start."); + yield* releaseFailedDelivery(entry.delivery_id); + } + yield* remove; + continue; + } + const prepared = yield* Effect.gen(function* () { + const task = yield* decodePendingTask(entry.task_json); + const request = yield* decodePendingRequest(entry.request_json); + const frameId = Buffer.from(yield* crypto.randomBytes(6)).toString("hex"); + return { task, rendered: renderWebhookPrompt(task.prompt, request, frameId) }; + }).pipe(Effect.exit); + if (Exit.isFailure(prepared)) { + if (Cause.hasInterruptsOnly(prepared.cause)) + return yield* Effect.failCause(prepared.cause); + yield* markDeliveryFailed( + entry.delivery_id, + "The pending webhook payload could not be prepared.", + ); + yield* releaseFailedDelivery(entry.delivery_id); + yield* remove; + continue; + } + const { task, rendered } = prepared.value; + if (rendered.prompt.trim().length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) { + yield* markDeliveryFailed(entry.delivery_id, PROMPT_TOO_LONG); + yield* releaseFailedDelivery(entry.delivery_id); + } else { + yield* sql`UPDATE scheduled_task_webhook_deliveries SET outcome = 'accepted', error = NULL, rendered_prompt = ${rendered.prompt.slice(0, WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT)}, missing_fields_json = ${encodeMissingFieldsJson(rendered.missing)} WHERE delivery_id = ${entry.delivery_id}`; + if (!(yield* runWebhookDelivery(task, entry.delivery_id, rendered.prompt))) + return; + } + yield* remove; + } + }); + yield* dispatch.pipe( + permit.withPermits(1), + Effect.catch((cause) => + Effect.logWarning("Pending webhook dispatch failed", { cause }), + ), + Effect.ensuring( + Ref.update(pendingInFlight, (active) => { + const next = new Set(active); + next.delete(taskId); + return next; + }), + ), + Effect.withSpan("ScheduledTaskService.dispatchPendingWebhook", { root: true }), + Effect.forkIn(serviceScope), + ); + }), + { discard: true }, ); }); + yield* scheduler.register("webhook-coalescing", dispatchPending()); return ScheduledTaskService.of({ list, diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index 2485edf7e0901..e8a4957e84c02 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -3,8 +3,13 @@ import * as NodeCrypto from "node:crypto"; import * as NodePlatformCrypto from "@effect/platform-node/NodeCrypto"; import { assert, it } from "@effect/vitest"; -import { ScheduledTaskUpsertInput, SecretRequestError } from "@t3tools/contracts"; +import { + ScheduledTaskUpsertInput, + ScheduledTaskWebhookDeliveryId, + SecretRequestError, +} from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; +import * as Crypto from "effect/Crypto"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; @@ -24,9 +29,94 @@ import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as Scheduler from "../scheduling/Scheduler.ts"; import * as ScheduledTaskService from "./ScheduledTaskService.ts"; +import { renderWebhookPrompt } from "./webhookTemplate.ts"; const decodeUpsertInput = Schema.decodeUnknownEffect(ScheduledTaskUpsertInput); +it.effect( + "preserves omitted dedupe settings, releases duplicate claims, and retains coalesced claims", + () => + withService(({ service, sql, launches }) => + Effect.gen(function* () { + const dedupe = { + replayKey: "{{headers.delivery-id}}", + coalesceKey: "{{body.group}}", + coalesceWindowSeconds: 10, + }; + const { task } = yield* service.upsert( + yield* webhookTaskInput({ schedule: { type: "webhook", dedupe } }), + ); + const updated = yield* service.upsert(yield* webhookTaskInput()); + assert.deepEqual( + updated.task.schedule.type === "webhook" && updated.task.schedule.dedupe, + dedupe, + ); + const request = (id: string, bodyText = "{}", relayDeliveryId?: string) => + requestFor(task, { + headers: { "delivery-id": id }, + bodyText, + body: new TextEncoder().encode(bodyText), + ...(relayDeliveryId === undefined ? {} : { relayDeliveryId }), + }); + yield* service.triggerWebhook(request("A")); + yield* Queue.take(launches); + yield* service.triggerWebhook(request("B1", '{"group":"B"}')); + for (let i = 0; i < 5; i++) { + const duplicate = yield* service.triggerWebhook(request("A", "{}", `duplicate-${i}`)); + assert.equal(duplicate._tag === "accepted" && duplicate.outcome, "duplicate"); + const coalesced = yield* service.triggerWebhook( + request(`B${i + 2}`, '{"group":"B"}', `coalesced-${i}`), + ); + assert.equal(coalesced._tag === "accepted" && coalesced.outcome, "coalesced"); + } + const claims = yield* sql<{ + relay_delivery_id: string; + }>`SELECT relay_delivery_id FROM scheduled_task_webhook_relay_deliveries ORDER BY relay_delivery_id`; + assert.deepEqual( + claims.map((claim) => claim.relay_delivery_id), + Array.from({ length: 5 }, (_, i) => `coalesced-${i}`), + ); + assert.isTrue( + (yield* service.listWebhookDeliveries({ id: task.id })).deliveries.every( + (delivery) => delivery.outcome !== "duplicate" && delivery.outcome !== "coalesced", + ), + ); + const fresh = yield* service.triggerWebhook(request("C", "{}", "duplicate-0")); + assert.equal(fresh._tag === "accepted" && fresh.outcome, "accepted"); + yield* Queue.take(launches); + const cleared = yield* service.upsert( + yield* webhookTaskInput({ schedule: { type: "webhook", dedupe: {} } }), + ); + assert.deepEqual( + cleared.task.schedule.type === "webhook" && cleared.task.schedule.dedupe, + {}, + ); + }), + ), +); + +const waitForSpan = (work: Effect.Effect, name: string) => + Effect.gen(function* () { + const done = yield* Deferred.make(); + const tracer = Tracer.make({ + span: (options) => { + const span = new Tracer.NativeSpan(options); + const end = span.end.bind(span); + span.end = (...args) => { + end(...args); + if (span.name === name) Deferred.doneUnsafe(done, Effect.void); + }; + return span; + }, + }); + const result = yield* work.pipe(Effect.withTracer(tracer)); + yield* Deferred.await(done); + return result; + }); + +const drainPendingDispatch = (workers: Map>) => + waitForSpan(workers.get("webhook-coalescing")!, "ScheduledTaskService.dispatchPendingWebhook"); + type LaunchInput = ThreadLaunchService.ThreadLaunchInput; const webhookTaskInput = (overrides: Record = {}) => @@ -70,20 +160,76 @@ const withService = ( readonly sql: SqlClient.SqlClient; readonly secretsByRef: Map; }) => Effect.Effect, - options: { readonly gate?: Deferred.Deferred; readonly relayHookBaseUrl?: string } = {}, + options: { + readonly onFrame?: () => void; + readonly frameGate?: { + readonly started: Deferred.Deferred; + readonly release: Deferred.Deferred; + }; + readonly completedLaunches?: Set; + readonly failLaunches?: { remaining: number }; + readonly gate?: Deferred.Deferred; + readonly gateForTask?: (input: LaunchInput) => Deferred.Deferred | undefined; + readonly relayHookBaseUrl?: string; + readonly sql?: SqlClient.SqlClient; + readonly workers?: Map>; + } = {}, ) => Effect.gen(function* () { const launches = yield* Queue.unbounded(); const secretsByRef = new Map(); const layerDependencies = Layer.mergeAll( - NodePlatformCrypto.layer, - Scheduler.layer, + options.frameGate === undefined && options.onFrame === undefined + ? NodePlatformCrypto.layer + : Layer.effect( + Crypto.Crypto, + Effect.gen(function* () { + const crypto = yield* Crypto.Crypto; + return { + ...crypto, + randomBytes: (size: number) => + Effect.suspend(() => { + if (size === 6) options.onFrame?.(); + return size === 6 && options.frameGate !== undefined + ? Deferred.succeed(options.frameGate.started, undefined).pipe( + Effect.andThen(Deferred.await(options.frameGate.release)), + Effect.andThen(crypto.randomBytes(size)), + ) + : crypto.randomBytes(size); + }), + }; + }), + ).pipe(Layer.provide(NodePlatformCrypto.layer)), + options.workers === undefined + ? Scheduler.layer + : Layer.succeed(Scheduler.Scheduler, { + register: (name: string, work: Effect.Effect) => + Effect.gen(function* () { + const context = yield* Effect.context(); + options.workers!.set(name, work.pipe(Effect.provideContext(context), Effect.orDie)); + }), + }), Layer.mock(ThreadLaunchService.ThreadLaunchService)({ launch: (input) => - Queue.offer(launches, input).pipe( - Effect.andThen(options.gate ? Deferred.await(options.gate) : Effect.void), - Effect.as({ threadId: "thread-1", resumed: false } as never), - ), + Effect.suspend(() => { + if (options.completedLaunches?.has(input.commandId)) + return Effect.succeed({ threadId: "thread-1", resumed: false } as never); + options.completedLaunches?.add(input.commandId); + return Queue.offer(launches, input).pipe( + Effect.andThen( + (options.gateForTask?.(input) ?? options.gate) + ? Deferred.await((options.gateForTask?.(input) ?? options.gate)!) + : Effect.void, + ), + Effect.andThen( + Effect.sync(() => { + if (options.failLaunches && options.failLaunches.remaining-- > 0) + throw new Error("injected start failure"); + }), + ), + Effect.as({ threadId: "thread-1", resumed: false } as never), + ); + }), }), Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ namedProjectsRoot: "/projects", @@ -108,8 +254,18 @@ const withService = ( const service = yield* ScheduledTaskService.ScheduledTaskService; const sql = yield* SqlClient.SqlClient; return yield* body({ service, launches, sql, secretsByRef }); - }).pipe(Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies)))); - }).pipe(Effect.provide(SqlitePersistence.layerMemory)); + }).pipe( + Effect.provide( + Layer.fresh(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies))), + ), + ); + }).pipe( + Effect.provide( + options.sql === undefined + ? SqlitePersistence.layerMemory + : Layer.succeed(SqlClient.SqlClient, options.sql), + ), + ); it.effect("dispatches exactly the rendered prompt and logs the delivery", () => withService(({ service, launches }) => @@ -132,7 +288,10 @@ it.effect("dispatches exactly the rendered prompt and logs the delivery", () => `scheduled-task:${task.id}:webhook:${result._tag === "accepted" ? result.deliveryId : ""}`, ); - const { deliveries } = yield* service.listWebhookDeliveries({ id: task.id }); + const { deliveries } = yield* service.listWebhookDeliveries({ + id: task.id, + acceptAnyOutcome: true, + }); assert.equal(deliveries.length, 1); assert.equal(deliveries[0]?.outcome, "accepted"); const { delivery } = yield* service.getWebhookDelivery({ @@ -188,7 +347,11 @@ it.effect("answers not found for a wrong token or unknown hook without logging", assert.equal(wrongToken._tag, "not_found"); const unknown = yield* service.triggerWebhook(requestFor(task, { hookId: "missing" })); assert.equal(unknown._tag, "not_found"); - assert.equal((yield* service.listWebhookDeliveries({ id: task.id })).deliveries.length, 0); + assert.equal( + (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries + .length, + 0, + ); }), ), ); @@ -253,9 +416,10 @@ it.effect("checks the configured signature and keeps the secret write-only", () ); assert.isTrue(resaved.task.webhook?.hasSecret); - const outcomes = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries.map( - (delivery) => [delivery.outcome, delivery.signatureVerified], - ); + const outcomes = (yield* service.listWebhookDeliveries({ + id: task.id, + acceptAnyOutcome: true, + })).deliveries.map((delivery) => [delivery.outcome, delivery.signatureVerified]); assert.deepEqual(outcomes.toSorted(), [ ["accepted", true], ["rejected_signature", false], @@ -300,23 +464,38 @@ it.effect("queues a burst of deliveries instead of dropping them", () => }), ); -it.effect("rate limits a hook past 60 deliveries a minute", () => - withService(({ service }) => - Effect.gen(function* () { - const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false })); - const results = yield* Effect.forEach(Array.from({ length: 61 }), () => - service.triggerWebhook(requestFor(task)), - ); - assert.equal(results.at(-2)?._tag, "disabled"); - assert.equal(results.at(-1)?._tag, "rate_limited"); - yield* Effect.forEach([1, 2, 3], () => service.triggerWebhook(requestFor(task))); - const outcomes = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries.map( - (delivery) => delivery.outcome, - ); - assert.equal(outcomes.filter((outcome) => outcome === "rate_limited").length, 1); - }), - ), -); +it.effect("rate limits a hook past 60 deliveries a minute", () => { + let frames = 0; + return withService( + ({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput()); + const results = yield* Effect.forEach(Array.from({ length: 61 }), () => + service + .triggerWebhook(requestFor(task)) + .pipe( + Effect.tap((result) => + result._tag === "accepted" ? Queue.take(launches) : Effect.void, + ), + ), + ); + assert.equal(results.at(-2)?._tag, "accepted"); + assert.equal(results.at(-1)?._tag, "rate_limited"); + yield* Effect.forEach([1, 2, 3], () => service.triggerWebhook(requestFor(task))); + const outcomes = (yield* service.listWebhookDeliveries({ + id: task.id, + acceptAnyOutcome: true, + })).deliveries.map((delivery) => delivery.outcome); + assert.equal(outcomes.filter((outcome) => outcome === "rate_limited").length, 1); + assert.equal(frames, 60); + }), + { + onFrame: () => { + frames++; + }, + }, + ); +}); it.effect("a save carrying a stale token cannot undo a rotation", () => withService(({ service }) => @@ -337,7 +516,10 @@ it.effect("keeps the newest 50 deliveries when they share a timestamp", () => const ids = yield* Effect.forEach(Array.from({ length: 55 }), (_, index) => service.triggerWebhook(requestFor(task, { query: `n=${index}` })).pipe(Effect.as(index)), ); - const { deliveries } = yield* service.listWebhookDeliveries({ id: task.id }); + const { deliveries } = yield* service.listWebhookDeliveries({ + id: task.id, + acceptAnyOutcome: true, + }); assert.equal(deliveries.length, 50); const first = yield* service.getWebhookDelivery({ id: task.id, @@ -368,7 +550,10 @@ it.effect("keeps credential headers and query values out of the delivery log", ( }, }), ); - const [summary] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries; + const [summary] = (yield* service.listWebhookDeliveries({ + id: task.id, + acceptAnyOutcome: true, + })).deliveries; const { delivery } = yield* service.getWebhookDelivery({ id: task.id, deliveryId: summary!.id, @@ -479,7 +664,10 @@ it.effect("caps deliveries waiting behind a stuck run", () => ); assert.equal(waiting.filter((result) => result._tag === "accepted").length, 19); assert.equal(waiting.at(-1)?._tag, "rate_limited"); - const logged = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries; + const logged = (yield* service.listWebhookDeliveries({ + id: task.id, + acceptAnyOutcome: true, + })).deliveries; assert.equal(logged.length, 20); yield* Deferred.succeed(gate, undefined); }), @@ -495,7 +683,10 @@ it.effect("logs a body's first 64 KiB by bytes, not characters", () => const text = "界".repeat(30_000); const body = new TextEncoder().encode(text); yield* service.triggerWebhook(requestFor(task, { body, bodyText: text })); - const [summary] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries; + const [summary] = (yield* service.listWebhookDeliveries({ + id: task.id, + acceptAnyOutcome: true, + })).deliveries; const { delivery } = yield* service.getWebhookDelivery({ id: task.id, deliveryId: summary!.id, @@ -559,7 +750,8 @@ it.effect("a held request already delivered directly runs only once", () => assert.equal(direct._tag, "accepted"); assert.deepEqual(replayed, { ...direct, outcome: "duplicate" } as typeof replayed); yield* Queue.take(launches); - const logged = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries; + const logged = (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })) + .deliveries; assert.equal(logged.length, 1); }), ), @@ -580,7 +772,9 @@ it.effect("relay retries recover failed acceptance without releasing another req const error = yield* service.triggerWebhook(request).pipe(Effect.flip); assert.equal(error.message, "Could not record webhook delivery."); assert.equal(yield* Queue.size(launches), 0); - assert.isEmpty((yield* service.listWebhookDeliveries({ id: task.id })).deliveries); + assert.isEmpty( + (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries, + ); yield* sql`DROP TRIGGER fail_webhook_delivery_log`; const retried = yield* Effect.all( @@ -606,7 +800,11 @@ it.effect("relay retries recover failed acceptance without releasing another req const replayed = yield* service.triggerWebhook(request); assert.equal(replayed._tag === "accepted" ? replayed.outcome : replayed._tag, "duplicate"); assert.equal(yield* Queue.size(launches), 0); - assert.equal((yield* service.listWebhookDeliveries({ id: task.id })).deliveries.length, 1); + assert.equal( + (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries + .length, + 1, + ); }), ), ); @@ -657,9 +855,9 @@ it.effect("a held request runs once even after the log has trimmed it", () => it.effect("a rate-limited held request can run on a later pass", () => withService(({ service, launches }) => Effect.gen(function* () { - const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false })); + const { task } = yield* service.upsert(yield* webhookTaskInput()); yield* Effect.forEach(Array.from({ length: 60 }), () => - service.triggerWebhook(requestFor(task)), + service.triggerWebhook(requestFor(task)).pipe(Effect.tap(() => Queue.take(launches))), ); const limited = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "later" })); assert.equal(limited._tag, "rate_limited"); @@ -678,7 +876,10 @@ it.effect("logs a held request at the time the relay received it", () => const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false })); const receivedAt = DateTime.formatIso(DateTime.subtract(yield* DateTime.now, { minutes: 5 })); yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "relay-2", receivedAt })); - const [delivery] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries; + const [delivery] = (yield* service.listWebhookDeliveries({ + id: task.id, + acceptAnyOutcome: true, + })).deliveries; assert.equal(delivery?.receivedAt, receivedAt); }), ), @@ -699,7 +900,10 @@ it.effect("a receive time in the future counts as now", () => ); assert.equal(result._tag, "accepted"); yield* Queue.take(launches); - const [delivery] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries; + const [delivery] = (yield* service.listWebhookDeliveries({ + id: task.id, + acceptAnyOutcome: true, + })).deliveries; assert.equal(delivery?.receivedAt, DateTime.formatIso(now)); }), ), @@ -723,9 +927,10 @@ it.effect("skips a held request older than the task's max age", () => requestFor(task, { relayDeliveryId: "fresh", receivedAt: fresh }), ); assert.equal(ok._tag, "accepted"); - const outcomes = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries.map( - (delivery) => delivery.outcome, - ); + const outcomes = (yield* service.listWebhookDeliveries({ + id: task.id, + acceptAnyOutcome: true, + })).deliveries.map((delivery) => delivery.outcome); assert.includeMembers(outcomes, ["expired", "accepted"]); }), ), @@ -753,7 +958,11 @@ it.effect("deleting a task removes its delivery log", () => const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false })); yield* service.triggerWebhook(requestFor(task)); yield* service.delete({ id: task.id }); - assert.equal((yield* service.listWebhookDeliveries({ id: task.id })).deliveries.length, 0); + assert.equal( + (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries + .length, + 0, + ); }), ), ); @@ -954,105 +1163,1602 @@ it.effect("counts each handled request by what happened to it", () => ), ); -it.effect("checks filters after the signature and max age, and runs only matching deliveries", () => +it.effect("dedupes replay keys per task, persists them, and expires them after seven days", () => + withService( + ({ service, launches, sql }) => + Effect.gen(function* () { + const schedule = { type: "webhook", dedupe: { replayKey: "{{headers.delivery-id}}" } }; + const { task } = yield* service.upsert(yield* webhookTaskInput({ schedule })); + const request = requestFor(task, { headers: { "delivery-id": "A" } }); + const first = yield* service.triggerWebhook(request); + assert.equal(first._tag === "accepted" && first.outcome, "accepted"); + yield* Queue.take(launches); + const replay = yield* service.triggerWebhook(request); + assert.equal(replay._tag === "accepted" && replay.outcome, "duplicate"); + assert.equal(yield* Queue.size(launches), 0); + const other = yield* service.upsert( + yield* webhookTaskInput({ id: "scheduled-task:other", schedule }), + ); + yield* service.triggerWebhook(requestFor(other.task, { headers: request.headers })); + yield* Queue.take(launches); + yield* withService( + ({ service: restarted }) => + Effect.gen(function* () { + const replay = yield* restarted.triggerWebhook(request); + assert.equal(replay._tag === "accepted" && replay.outcome, "duplicate"); + }), + { sql, workers: new Map() }, + ); + yield* TestClock.adjust("7 days"); + const expired = yield* service.triggerWebhook(request); + assert.equal(expired._tag === "accepted" && expired.outcome, "accepted"); + yield* Queue.take(launches); + assert.include( + (yield* service.listWebhookDeliveries({ + id: task.id, + acceptAnyOutcome: true, + })).deliveries.map((d) => d.outcome), + "duplicate", + ); + }), + { workers: new Map() }, + ), +); + +it.effect.each([1, 3])( + "coalesces %i deliveries into one run with the latest body, headers and query", + (count) => { + const workers = new Map>(); + return withService( + ({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + prompt: "{{body.value}} {{headers.version}} {{query.version}}", + schedule: { + type: "webhook", + dedupe: { coalesceKey: "{{body.key}}", coalesceWindowSeconds: 10 }, + }, + }), + ); + for (let value = 1; value <= count; value++) { + const bodyText = JSON.stringify({ key: "B", value }); + yield* service.triggerWebhook( + requestFor(task, { + bodyText, + body: new TextEncoder().encode(bodyText), + headers: { version: String(value) }, + query: `version=${value}`, + }), + ); + if (value < count) yield* TestClock.adjust("2 seconds"); + } + const owner = (yield* service.listWebhookDeliveries({ + id: task.id, + acceptAnyOutcome: true, + })).deliveries.find((delivery) => delivery.outcome === "accepted")!; + assert.isTrue(owner.pendingWindow); + const beforeDispatch = (yield* service.getWebhookDelivery({ + id: task.id, + deliveryId: owner.id, + acceptAnyOutcome: true, + })).delivery; + assert.isTrue(beforeDispatch.pendingWindow); + assert.equal(beforeDispatch.body, JSON.stringify({ key: "B", value: count })); + assert.equal(beforeDispatch.headers.version, String(count)); + assert.equal(beforeDispatch.query, `version=${count}`); + assert.equal( + beforeDispatch.bodyBytes, + new TextEncoder().encode(beforeDispatch.body).byteLength, + ); + assert.equal(yield* Queue.size(launches), 0); + yield* workers.get("webhook-coalescing")!; + assert.equal(yield* Queue.size(launches), 0); + yield* TestClock.adjust(count === 1 ? "10 seconds" : "6 seconds"); + yield* workers.get("webhook-coalescing")!; + const launched = yield* Queue.take(launches); + assert.equal( + (launched.initialMessage!.text.match(new RegExp(`>${count}<`, "g")) ?? []).length, + 3, + ); + yield* workers.get("webhook-coalescing")!; + assert.equal(yield* Queue.size(launches), 0); + const outcomes = (yield* service.listWebhookDeliveries({ + id: task.id, + acceptAnyOutcome: true, + })).deliveries.map((d) => d.outcome); + const afterDispatch = (yield* service.getWebhookDelivery({ + id: task.id, + deliveryId: owner.id, + acceptAnyOutcome: true, + })).delivery; + assert.isFalse(afterDispatch.pendingWindow); + assert.equal(afterDispatch.body, beforeDispatch.body); + assert.include(afterDispatch.renderedPrompt!, `>${count}<`); + assert.equal(outcomes.filter((o) => o === "accepted").length, 1); + assert.equal(outcomes.filter((o) => o === "coalesced").length, count - 1); + }), + { workers }, + ); + }, +); + +it.effect("recovers a persisted coalescing window after restarting the service", () => { + const workers = new Map>(); + return withService( + ({ service, sql }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { + type: "webhook", + dedupe: { coalesceKey: "{{headers.content-type}}", coalesceWindowSeconds: 10 }, + }, + }), + ); + yield* service.triggerWebhook(requestFor(task)); + yield* TestClock.adjust("4 seconds"); + const restartedWorkers = new Map>(); + yield* withService( + ({ launches }) => + Effect.gen(function* () { + yield* restartedWorkers.get("webhook-coalescing")!; + assert.equal(yield* Queue.size(launches), 0); + yield* TestClock.adjust("6 seconds"); + yield* restartedWorkers.get("webhook-coalescing")!; + yield* Queue.take(launches); + yield* restartedWorkers.get("webhook-coalescing")!; + assert.equal(yield* Queue.size(launches), 0); + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0); + }), + { sql, workers: restartedWorkers }, + ); + }), + { workers }, + ); +}); + +it.effect("empty dedupe key renders run normally and malformed templates fail at save time", () => withService(({ service, launches }) => Effect.gen(function* () { const { task } = yield* service.upsert( yield* webhookTaskInput({ schedule: { type: "webhook", - signature: { - header: "x-hub-signature-256", - encoding: "hex", - prefix: "sha256=", - secret: "s3cret", - }, - maxDeliveryAgeMinutes: 30, - filters: [ - { path: "headers.x-github-event", op: "equals", value: "pull_request" }, - { path: "body.action", op: "in", value: ["opened", "reopened"] }, - ], + dedupe: { replayKey: "{{headers.absent}}", coalesceKey: "{{body.absent}}" }, }, }), ); - const delivery = ( - event: string, - action: string, - options: { readonly signed?: boolean; readonly receivedAt?: string } = {}, - ) => { - const body = new TextEncoder().encode( - JSON.stringify({ action, pull_request: { url: "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/org/repo/pull/45" } }), + for (let i = 0; i < 2; i++) { + const result = yield* service.triggerWebhook(requestFor(task, { headers: { absent: "" } })); + assert.equal(result._tag === "accepted" && result.outcome, "accepted"); + yield* Queue.take(launches); + } + for (const replayKey of ["literal", "{{body.bad"]) { + const invalid = yield* webhookTaskInput({ + schedule: { type: "webhook", dedupe: { replayKey } }, + }).pipe(Effect.result); + assert.equal(invalid._tag, "Failure"); + } + for (const coalesceWindowSeconds of [9, 3601, 10.5]) { + const result = yield* webhookTaskInput({ + schedule: { type: "webhook", dedupe: { coalesceWindowSeconds } }, + }).pipe(Effect.result); + assert.equal(result._tag, "Failure"); + } + }), + ), +); + +it.effect("applies replay before coalescing and defaults the fixed window to 120 seconds", () => { + const workers = new Map>(); + return withService( + ({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + prompt: "{{body.value}}", + schedule: { + type: "webhook", + dedupe: { replayKey: "{{headers.delivery-id}}", coalesceKey: "{{body.group}}" }, + }, + }), ); - const signature = `sha256=${NodeCrypto.createHmac("sha256", "s3cret").update(body).digest("hex")}`; - return requestFor(task, { - headers: { - "content-type": "application/json", - "x-github-event": event, - ...(options.signed === false ? {} : { "x-hub-signature-256": signature }), - }, - body, - bodyText: new TextDecoder().decode(body), - ...(options.receivedAt === undefined - ? {} - : { relayDeliveryId: "old", receivedAt: options.receivedAt }), - }); - }; - const now = yield* DateTime.now; - const old = DateTime.formatIso(DateTime.subtract(now, { minutes: 31 })); + const request = (id: string, value: string) => { + const bodyText = JSON.stringify({ group: "B", value }); + return requestFor(task, { + bodyText, + body: new TextEncoder().encode(bodyText), + headers: { "delivery-id": id }, + }); + }; + yield* service.triggerWebhook(request("B1", "first")); + yield* service.triggerWebhook(request("B2", "latest")); + const replay = yield* service.triggerWebhook( + request("B2", "replay must not replace latest"), + ); + assert.equal(replay._tag === "accepted" && replay.outcome, "duplicate"); + yield* TestClock.adjust("119 seconds"); + yield* workers.get("webhook-coalescing")!; + assert.equal(yield* Queue.size(launches), 0); + yield* TestClock.adjust("1 second"); + yield* workers.get("webhook-coalescing")!; + assert.include((yield* Queue.take(launches)).initialMessage!.text, ">latest<"); + }), + { workers }, + ); +}); - // A request failing a filter is still refused first for its signature or age. - const unsigned = yield* service.triggerWebhook(delivery("push", "opened", { signed: false })); - assert.equal(unsigned._tag, "rejected_signature"); - const tooOld = yield* service.triggerWebhook(delivery("push", "opened", { receivedAt: old })); - assert.equal(tooOld._tag, "expired"); +it.effect("rolls back replay and pending state when recording a delivery fails", () => { + const workers = new Map>(); + return withService( + ({ service, sql, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { + type: "webhook", + dedupe: { + replayKey: "{{headers.content-type}}", + coalesceKey: "{{headers.content-type}}", + coalesceWindowSeconds: 10, + }, + }, + }), + ); + yield* sql`CREATE TRIGGER fail_dedupe_log BEFORE INSERT ON scheduled_task_webhook_deliveries BEGIN SELECT RAISE(FAIL, 'injected'); END`; + yield* service.triggerWebhook(requestFor(task)).pipe(Effect.flip); + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0); + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0); + yield* sql`DROP TRIGGER fail_dedupe_log`; + const retry = yield* service.triggerWebhook(requestFor(task)); + assert.equal(retry._tag === "accepted" && retry.outcome, "accepted"); + yield* TestClock.adjust("10 seconds"); + yield* workers.get("webhook-coalescing")!; + yield* Queue.take(launches); + }), + { workers }, + ); +}); - const wrongEvent = yield* service.triggerWebhook(delivery("push", "opened")); - const wrongAction = yield* service.triggerWebhook(delivery("pull_request", "closed")); - assert.deepInclude(wrongEvent, { _tag: "accepted", outcome: "filtered" }); - assert.deepInclude(wrongAction, { _tag: "accepted", outcome: "filtered" }); - assert.equal(yield* Queue.size(launches), 0); - const [listed] = (yield* service.list()).tasks; - assert.equal(listed?.runCount, 0); - assert.equal(listed?.lastRunStatus, "never"); +it.effect("starts a new fixed window at the deadline without replacing the expired payload", () => { + const workers = new Map>(); + return withService( + ({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + prompt: "{{body}}", + schedule: { + type: "webhook", + dedupe: { coalesceKey: "{{headers.content-type}}", coalesceWindowSeconds: 10 }, + }, + }), + ); + const request = (bodyText: string) => + requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }); + yield* service.triggerWebhook(request("first")); + yield* TestClock.adjust("10 seconds"); + yield* service.triggerWebhook(request("second")); + yield* workers.get("webhook-coalescing")!; + assert.include((yield* Queue.take(launches)).initialMessage!.text, ">first<"); + assert.equal(yield* Queue.size(launches), 0); + yield* TestClock.adjust("10 seconds"); + yield* workers.get("webhook-coalescing")!; + assert.include((yield* Queue.take(launches)).initialMessage!.text, ">second<"); + }), + { workers }, + ); +}); - const matched = yield* service.triggerWebhook(delivery("pull_request", "opened")); - assert.deepInclude(matched, { _tag: "accepted", outcome: "accepted" }); - assert.include((yield* Queue.take(launches)).initialMessage?.text ?? "", "pull/45"); +it.effect("pausing a pending task prevents dispatch and deletion clears its dedupe state", () => { + const workers = new Map>(); + return withService( + ({ service, launches, sql }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { + type: "webhook", + dedupe: { + replayKey: "{{headers.content-type}}", + coalesceKey: "{{headers.content-type}}", + coalesceWindowSeconds: 10, + }, + }, + }), + ); + yield* service.triggerWebhook(requestFor(task)); + yield* service.setEnabled({ id: task.id, enabled: false }); + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + assert.equal(yield* Queue.size(launches), 0); + assert.equal( + (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })) + .deliveries[0]?.outcome, + "dispatch_failed", + ); + yield* service.delete({ id: task.id }); + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0); + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0); + }), + { workers }, + ); +}); - const deliveries = (yield* service.listWebhookDeliveries({ - id: task.id, - acceptAnyOutcome: true, - })).deliveries; - const filtered = deliveries - .filter((entry) => entry.outcome === "filtered") - .map((entry) => [entry.signatureVerified, entry.error]); - assert.sameDeepMembers(filtered, [ - [true, "Filter 1 (headers.x-github-event) did not match."], - [true, "Filter 2 (body.action) did not match."], - ]); - }), - ), +it.effect( + "caps duplicate writes at ten per minute without evicting runs or consuming their rate slots", + () => + withService(({ service, launches, sql }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { type: "webhook", dedupe: { replayKey: "{{body}}" } }, + }), + ); + yield* service.triggerWebhook(requestFor(task)); + yield* Queue.take(launches); + const before = yield* deliveriesCounted("duplicate", "direct"); + for (let i = 0; i < 75; i++) { + const result = yield* service.triggerWebhook(requestFor(task)); + assert.equal(result._tag === "accepted" && result.outcome, "duplicate"); + } + assert.equal((yield* deliveriesCounted("duplicate", "direct")) - before, 75); + const rows = (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })) + .deliveries; + assert.equal(rows.filter((row) => row.outcome === "duplicate").length, 10); + assert.equal(rows.filter((row) => row.outcome === "accepted").length, 1); + const keys = yield* sql<{ + replay_key: string; + }>`SELECT replay_key FROM scheduled_task_webhook_replay_keys`; + assert.match(keys[0]!.replay_key, /^[a-f0-9]{64}$/); + for (let i = 0; i < 59; i++) { + const bodyText = JSON.stringify({ i }); + const result = yield* service.triggerWebhook( + requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }), + ); + assert.equal(result._tag, "accepted"); + yield* Queue.take(launches); + } + }), + ), ); -const pullRequestFilterInput = () => - webhookTaskInput({ - schedule: { - type: "webhook", - filters: [{ path: "headers.x-github-event", op: "equals", value: "pull_request" }], - }, - }); -const githubEvent = (event: string) => ({ - headers: { "content-type": "application/json", "x-github-event": event }, +it.effect("caps coalesced history while persisting the latest overflow payload", () => { + const workers = new Map>(); + return withService( + ({ service, sql, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + prompt: "{{body.value}}", + schedule: { + type: "webhook", + dedupe: { replayKey: "{{headers.delivery-id}}", coalesceKey: "{{body.group}}" }, + }, + }), + ); + const request = (i: number) => { + const bodyText = JSON.stringify({ group: "B", value: i }); + return requestFor(task, { + bodyText, + body: new TextEncoder().encode(bodyText), + headers: { "delivery-id": String(i) }, + }); + }; + for (let i = 0; i < 40; i++) + assert.equal((yield* service.triggerWebhook(request(i)))._tag, "accepted"); + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 40); + const rows = (yield* service.listWebhookDeliveries({ + id: task.id, + acceptAnyOutcome: true, + })).deliveries; + assert.equal(rows.filter((row) => row.outcome === "coalesced").length, 10); + assert.equal(rows.filter((row) => row.outcome === "accepted").length, 1); + const pending = yield* sql<{ + coalesce_key: string; + }>`SELECT coalesce_key FROM scheduled_task_webhook_pending`; + assert.match(pending[0]!.coalesce_key, /^[a-f0-9]{64}$/); + yield* TestClock.adjust("120 seconds"); + yield* drainPendingDispatch(workers); + assert.include((yield* Queue.take(launches)).initialMessage!.text, ">39<"); + assert.equal(yield* Queue.size(launches), 0); + }), + { workers }, + ); }); -it.effect("filtered deliveries take no run slot and log at most 10 rows a minute", () => +it.effect("a partially missing key is empty even with a literal prefix", () => withService(({ service, launches }) => Effect.gen(function* () { - const { task } = yield* service.upsert(yield* pullRequestFilterInput()); - const ignored = yield* Effect.forEach(Array.from({ length: 70 }), () => - service.triggerWebhook(requestFor(task, githubEvent("push"))), + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { + type: "webhook", + dedupe: { replayKey: "gh:{{headers.absent}}", coalesceKey: "pr:{{body.absent}}" }, + }, + }), ); - assert.isTrue( + for (let i = 0; i < 2; i++) { + const result = yield* service.triggerWebhook(requestFor(task, { headers: { absent: "" } })); + assert.equal(result._tag === "accepted" && result.outcome, "accepted"); + yield* Queue.take(launches); + } + }), + ), +); + +it.effect("queue-full deliveries release replay keys so a later retry runs", () => + Effect.gen(function* () { + const gate = yield* Deferred.make(); + yield* withService( + ({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { type: "webhook", dedupe: { replayKey: "{{headers.delivery-id}}" } }, + }), + ); + const request = (i: number) => + requestFor(task, { headers: { "delivery-id": String(i) } }); + for (let i = 0; i < 20; i++) yield* service.triggerWebhook(request(i)); + yield* Queue.take(launches); + const full = yield* service.triggerWebhook(request(20)); + assert.equal(full._tag === "rate_limited" && full.outcome, "queue_full"); + yield* Deferred.succeed(gate, undefined); + for (let i = 1; i < 20; i++) yield* Queue.take(launches); + const retry = yield* service.triggerWebhook(request(20)); + assert.equal(retry._tag === "accepted" && retry.outcome, "accepted"); + yield* Queue.take(launches); + }), + { gate }, + ); + }), +); + +it.effect.each(["task_json", "request_json"] as const)( + "removes corrupt pending %s instead of retrying forever", + (column) => { + const workers = new Map>(); + return withService( + ({ service, sql }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { + type: "webhook", + dedupe: { + replayKey: "{{body}}", + coalesceKey: "{{body}}", + coalesceWindowSeconds: 10, + }, + }, + }), + ); + yield* service.triggerWebhook(requestFor(task)); + yield* sql`UPDATE scheduled_task_webhook_pending SET ${sql(column)} = 'broken'`; + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0); + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0); + assert.equal( + (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!.outcome, + "dispatch_failed", + ); + yield* workers.get("webhook-coalescing")!; + }), + { workers }, + ); + }, +); + +it.effect( + "restarting after a coalesced launch before pending deletion does not launch or count it twice", + () => { + const workers = new Map>(); + return withService( + ({ service, sql, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { + type: "webhook", + dedupe: { coalesceKey: "{{body}}", coalesceWindowSeconds: 10 }, + }, + }), + ); + const accepted = yield* service.triggerWebhook(requestFor(task)); + yield* sql`CREATE TRIGGER fail_pending_delete BEFORE DELETE ON scheduled_task_webhook_pending BEGIN SELECT RAISE(FAIL,'crash after launch'); END`; + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + const launch = yield* Queue.take(launches); + assert.equal( + launch.commandId, + `scheduled-task:${task.id}:webhook:${accepted._tag === "accepted" && accepted.deliveryId}`, + ); + assert.equal((yield* service.list()).tasks[0]!.runCount, 1); + assert.equal( + (yield* sql`SELECT * FROM scheduled_task_webhook_pending WHERE state = 'completed'`) + .length, + 1, + ); + yield* sql`DROP TRIGGER fail_pending_delete`; + const restartedWorkers = new Map>(); + yield* withService( + ({ service: restarted, launches: secondLaunches }) => + Effect.gen(function* () { + yield* drainPendingDispatch(restartedWorkers); + assert.equal(yield* Queue.size(secondLaunches), 0); + assert.equal((yield* restarted.list()).tasks[0]!.runCount, 1); + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0); + }), + { sql, workers: restartedWorkers }, + ); + }), + { workers }, + ); + }, +); + +it.effect("many windows on a busy task share one worker without delaying another task", () => + Effect.gen(function* () { + const gate = yield* Deferred.make(); + const workers = new Map>(); + yield* withService( + ({ service, launches, sql }) => + Effect.gen(function* () { + const schedule = { + type: "webhook", + dedupe: { coalesceKey: "{{body.group}}", coalesceWindowSeconds: 10 }, + }; + const { task: a } = yield* service.upsert( + yield* webhookTaskInput({ id: "scheduled-task:a", title: "A", schedule }), + ); + const { task: b } = yield* service.upsert( + yield* webhookTaskInput({ id: "scheduled-task:b", title: "B", schedule }), + ); + yield* service.triggerWebhook(requestFor(a)); + assert.equal((yield* Queue.take(launches)).title, "A"); + const pendingRequest = (task: typeof a, group: string) => { + const bodyText = JSON.stringify({ group }); + return requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }); + }; + for (let i = 0; i < 19; i++) { + const result = yield* service.triggerWebhook(pendingRequest(a, String(i))); + assert.equal(result._tag === "accepted" && result.outcome, "accepted"); + } + const overflow = pendingRequest(a, "overflow"); + const full = yield* service.triggerWebhook({ ...overflow, relayDeliveryId: "overflow" }); + assert.equal(full._tag === "rate_limited" && full.outcome, "queue_full"); + assert.equal((yield* service.triggerWebhook(requestFor(a)))._tag, "rate_limited"); + yield* service.triggerWebhook(pendingRequest(b, "B")); + let dispatchWorkers = 0; + const drained = yield* Deferred.make(); + let finishedWorkers = 0; + const tracer = Tracer.make({ + span: (options) => { + if (options.name === "ScheduledTaskService.dispatchPendingWebhook") dispatchWorkers++; + const span = new Tracer.NativeSpan(options); + const end = span.end.bind(span); + span.end = (...args) => { + end(...args); + if ( + options.name === "ScheduledTaskService.dispatchPendingWebhook" && + ++finishedWorkers === 2 + ) + Deferred.doneUnsafe(drained, Effect.void); + }; + return span; + }, + }); + yield* TestClock.adjust("10 seconds"); + yield* workers.get("webhook-coalescing")!.pipe(Effect.withTracer(tracer)); + assert.equal((yield* Queue.take(launches)).title, "B"); + for (let i = 0; i < 10; i++) + yield* workers.get("webhook-coalescing")!.pipe(Effect.withTracer(tracer)); + assert.equal(dispatchWorkers, 2); + assert.equal( + (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE task_id = ${a.id}`) + .length, + 19, + ); + yield* Deferred.succeed(gate, undefined); + yield* Deferred.await(drained); + for (let i = 0; i < 19; i++) assert.equal((yield* Queue.take(launches)).title, "A"); + const retried = yield* service.triggerWebhook({ + ...overflow, + relayDeliveryId: "overflow", + }); + assert.equal(retried._tag === "accepted" && retried.outcome, "accepted"); + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + assert.equal((yield* Queue.take(launches)).title, "A"); + }), + { workers, gateForTask: (input) => (input.title === "A" ? gate : undefined) }, + ); + }), +); + +it.effect("guards bad-signature floods separately from valid run admission", () => + withService(({ service, sql }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { + type: "webhook", + signature: { header: "signature", encoding: "hex", prefix: "", secret: "secret" }, + }, + }), + ); + const results = yield* Effect.forEach(Array.from({ length: 75 }), () => + service.triggerWebhook(requestFor(task)), + ); + assert.equal(results.filter((result) => result._tag === "rejected_signature").length, 60); + assert.equal(results.filter((result) => result._tag === "rate_limited").length, 15); + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_deliveries`).length, 51); + }), + ), +); + +it.effect("start failures release both replay keys and relay claims for redelivery", () => + withService( + ({ service, sql, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { type: "webhook", dedupe: { replayKey: "{{body}}" } }, + }), + ); + const request = requestFor(task, { relayDeliveryId: "failed-start" }); + yield* waitForSpan( + service.triggerWebhook(request), + "ScheduledTaskService.runWebhookDelivery", + ); + yield* Queue.take(launches); + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0); + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_relay_deliveries`).length, 0); + const retry = yield* waitForSpan( + service.triggerWebhook(request), + "ScheduledTaskService.runWebhookDelivery", + ); + assert.equal(retry._tag === "accepted" && retry.outcome, "accepted"); + yield* Queue.take(launches); + assert.equal((yield* service.list()).tasks[0]!.lastRunStatus, "succeeded"); + }), + { failLaunches: { remaining: 1 } }, + ), +); + +it.effect("bounds oversized prompt floods without evicting run or filtered history", () => + withService(({ service, sql, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput({ prompt: "{{body}}" })); + const accepted = yield* service.triggerWebhook(requestFor(task)); + yield* Queue.take(launches); + assert.equal(accepted._tag, "accepted"); + // A sibling filter step owns these rows; suppression must preserve its bucket too. + yield* sql`INSERT INTO scheduled_task_webhook_deliveries + SELECT 'filtered-fixture', task_id, received_at, method, query, headers_json, body, + body_bytes, body_truncated, 'filtered', signature_verified, missing_fields_json, rendered_prompt, error + FROM scheduled_task_webhook_deliveries LIMIT 1`; + const bodyText = "x".repeat(200_000); + const request = requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }); + const outcomes = yield* Effect.forEach(Array.from({ length: 75 }), () => + service.triggerWebhook(request), + ); + assert.equal(outcomes.filter((result) => result._tag === "accepted").length, 60); + assert.equal(outcomes.filter((result) => result._tag === "rate_limited").length, 15); + const rows = yield* sql<{ + outcome: string; + }>`SELECT outcome FROM scheduled_task_webhook_deliveries`; + assert.equal(rows.filter((row) => row.outcome === "accepted").length, 1); + assert.equal(rows.filter((row) => row.outcome === "filtered").length, 1); + assert.equal(rows.length, 22); + assert.equal(yield* Queue.size(launches), 0); + yield* TestClock.adjust("61 seconds"); + assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "accepted"); + yield* Queue.take(launches); + }), + ), +); + +it.effect("single delivery lookup hides new outcomes unless the client opts in", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { + type: "webhook", + dedupe: { replayKey: "{{body}}", coalesceKey: "{{body.group}}" }, + }, + }), + ); + yield* service.triggerWebhook(requestFor(task)); + yield* Queue.take(launches); + yield* service.triggerWebhook(requestFor(task)); + for (const value of [1, 2]) { + const bodyText = JSON.stringify({ group: "B", value }); + yield* service.triggerWebhook( + requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }), + ); + } + const rows = (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })) + .deliveries; + for (const row of rows.filter( + (row) => row.outcome === "duplicate" || row.outcome === "coalesced", + )) { + const input = { id: task.id, deliveryId: row.id }; + const hidden = yield* service.getWebhookDelivery(input).pipe(Effect.result); + assert.equal(hidden._tag, "Failure"); + assert.equal( + (yield* service.getWebhookDelivery({ ...input, acceptAnyOutcome: true })).delivery + .outcome, + row.outcome, + ); + } + assert.equal( + rows.filter((row) => row.outcome === "duplicate" || row.outcome === "coalesced").length, + 2, + ); + }), + ), +); + +it.effect("main retention keeps fifty runs and preserves filtered and dedupe buckets", () => + withService(({ service, sql, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput()); + yield* service.triggerWebhook(requestFor(task)); + yield* Queue.take(launches); + for (const outcome of ["filtered", "duplicate"]) { + yield* sql`WITH RECURSIVE n(value) AS (SELECT 1 UNION ALL SELECT value + 1 FROM n WHERE value < 20) + INSERT INTO scheduled_task_webhook_deliveries + SELECT ${outcome} || n.value, d.task_id, d.received_at, d.method, d.query, d.headers_json, d.body, + d.body_bytes, d.body_truncated, ${outcome}, d.signature_verified, d.missing_fields_json, d.rendered_prompt, d.error + FROM n CROSS JOIN (SELECT * FROM scheduled_task_webhook_deliveries WHERE outcome = 'accepted' LIMIT 1) d`; + } + for (let i = 0; i < 59; i++) { + assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "accepted"); + yield* Queue.take(launches); + } + const counts = yield* sql<{ + outcome: string; + count: number; + }>`SELECT outcome, COUNT(*) AS count FROM scheduled_task_webhook_deliveries GROUP BY outcome ORDER BY outcome`; + assert.deepEqual(counts, [ + { outcome: "accepted", count: 50 }, + { outcome: "duplicate", count: 20 }, + { outcome: "filtered", count: 20 }, + ]); + }), + ), +); + +it.effect( + "a valid request refunds its own reservation without expiring an interleaved rejection", + () => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const release = yield* Deferred.make(); + yield* withService( + ({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { + type: "webhook", + signature: { header: "signature", encoding: "hex", prefix: "", secret: "secret" }, + }, + }), + ); + const signature = NodeCrypto.createHmac("sha256", "secret") + .update(pullRequestBody) + .digest("hex"); + const valid = yield* service + .triggerWebhook(requestFor(task, { headers: { signature } })) + .pipe(Effect.forkChild); + yield* Deferred.await(started); + yield* TestClock.adjust("30 seconds"); + assert.equal( + (yield* service.triggerWebhook(requestFor(task)))._tag, + "rejected_signature", + ); + yield* Deferred.succeed(release, undefined); + assert.equal((yield* Fiber.join(valid))._tag, "accepted"); + yield* Queue.take(launches); + // Only the valid reservation would have expired now; the rejection must still count. + yield* TestClock.adjust("31 seconds"); + for (let i = 0; i < 59; i++) + assert.equal( + (yield* service.triggerWebhook(requestFor(task)))._tag, + "rejected_signature", + ); + assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "rate_limited"); + }), + { frameGate: { started, release } }, + ); + }), +); + +it.effect("keeps replay keys when dispatch succeeds but completion bookkeeping fails", () => + withService(({ service, sql, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { + type: "webhook", + dedupe: { replayKey: "{{body}}" }, + }, + }), + ); + yield* sql`CREATE TRIGGER fail_completion BEFORE UPDATE ON scheduled_tasks + WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`; + yield* waitForSpan( + service.triggerWebhook(requestFor(task)), + "ScheduledTaskService.runWebhookDelivery", + ); + yield* Queue.take(launches); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 1); + yield* sql`DROP TRIGGER fail_completion`; + const retry = yield* service.triggerWebhook(requestFor(task)); + assert.equal(retry._tag === "accepted" && retry.outcome, "duplicate"); + assert.equal(yield* Queue.size(launches), 0); + assert.equal((yield* service.list()).tasks[0]!.runCount, 1); + }), + ), +); + +it.effect("restart before a coalesced completion commits counts the delivery once", () => + Effect.gen(function* () { + const gate = yield* Deferred.make(); + const completedLaunches = new Set(); + const workers = new Map>(); + yield* withService( + ({ service, sql, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { + type: "webhook", + dedupe: { + replayKey: "{{body}}", + coalesceKey: "{{body}}", + coalesceWindowSeconds: 10, + }, + }, + }), + ); + yield* service.triggerWebhook(requestFor(task)); + yield* TestClock.adjust("10 seconds"); + yield* workers.get("webhook-coalescing")!; + const launch = yield* Queue.take(launches); + assert.isTrue(completedLaunches.has(launch.commandId)); + assert.equal( + (yield* sql<{ state: string }>`SELECT state FROM scheduled_task_webhook_pending`)[0]! + .state, + "running", + ); + assert.equal( + (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE state = 'completed'`) + .length, + 0, + ); + assert.equal((yield* service.list()).tasks[0]!.lastRunStatus, "running"); + const restartedWorkers = new Map>(); + yield* withService( + ({ service: restarted, launches: recoveredLaunches }) => + Effect.gen(function* () { + assert.equal((yield* restarted.list()).tasks[0]!.runCount, 0); + yield* drainPendingDispatch(restartedWorkers); + assert.equal(yield* Queue.size(recoveredLaunches), 0); + assert.equal((yield* restarted.list()).tasks[0]!.runCount, 1); + assert.equal((yield* restarted.list()).tasks[0]!.lastRunStatus, "succeeded"); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0); + const retry = yield* restarted.triggerWebhook(requestFor(task)); + assert.equal(retry._tag === "accepted" && retry.outcome, "duplicate"); + }), + { sql, workers: restartedWorkers, completedLaunches }, + ); + }), + { workers, gate, completedLaunches }, + ); + }), +); + +it.effect.each([119_000, 120_000])( + "bounds coalesced prompt logs and checks trimmed length for %i body characters", + (length) => { + const workers = new Map>(); + return withService( + ({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + prompt: `{{body.absent}}${" ".repeat(10_000)}{{body.text}}`, + schedule: { + type: "webhook", + dedupe: { coalesceKey: "{{body.group}}", coalesceWindowSeconds: 10 }, + }, + }), + ); + const bodyText = JSON.stringify({ group: "B", text: "x".repeat(length) }); + const accepted = yield* service.triggerWebhook( + requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }), + ); + assert.equal(accepted._tag, "accepted"); + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + const delivery = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!; + if (length === 119_000) { + const launch = yield* Queue.take(launches); + assert.isAbove(launch.initialMessage!.text.length, 120_000); + assert.equal(delivery.outcome, "accepted"); + const detail = yield* service.getWebhookDelivery({ + id: task.id, + deliveryId: delivery.id, + }); + assert.equal(detail.delivery.renderedPrompt!.length, 64 * 1024); + } else { + assert.equal(yield* Queue.size(launches), 0); + assert.equal(delivery.outcome, "dispatch_failed"); + assert.equal(delivery.error, "The filled-in prompt is too long."); + } + }), + { workers }, + ); + }, +); + +it.effect("retries coalesced completion bookkeeping without launching or counting twice", () => { + const workers = new Map>(); + return withService( + ({ service, sql, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { + type: "webhook", + dedupe: { replayKey: "{{body}}", coalesceKey: "{{body}}", coalesceWindowSeconds: 10 }, + }, + }), + ); + yield* service.triggerWebhook(requestFor(task)); + yield* sql`CREATE TRIGGER fail_coalesced_completion BEFORE UPDATE ON scheduled_tasks + WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`; + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + yield* Queue.take(launches); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 1); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 1); + assert.equal( + (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE state = 'completed'`) + .length, + 0, + ); + assert.equal((yield* service.list()).tasks[0]!.runCount, 0); + yield* sql`DROP TRIGGER fail_coalesced_completion`; + yield* drainPendingDispatch(workers); + assert.equal(yield* Queue.size(launches), 0); + assert.equal((yield* service.list()).tasks[0]!.runCount, 1); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0); + const delivery = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!; + assert.equal(delivery.outcome, "accepted"); + assert.equal(delivery.error, null); + }), + { workers, completedLaunches: new Set() }, + ); +}); + +it.effect("propagates coalescing log failures with their task id and rolls back acceptance", () => { + const workers = new Map>(); + return withService( + ({ service, sql, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { + type: "webhook", + dedupe: { replayKey: "{{body}}", coalesceKey: "{{body}}", coalesceWindowSeconds: 10 }, + }, + }), + ); + const request = requestFor(task, { relayDeliveryId: "coalescing-log-failure" }); + yield* sql`CREATE TRIGGER fail_coalescing_log BEFORE INSERT ON scheduled_task_webhook_deliveries + BEGIN SELECT RAISE(FAIL, 'injected coalescing log failure'); END`; + const error = yield* service.triggerWebhook(request).pipe(Effect.flip); + assert.equal(error.message, "Could not record webhook delivery."); + assert.equal(error.taskId, task.id); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 0); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length, 0); + yield* sql`DROP TRIGGER fail_coalescing_log`; + assert.equal((yield* service.triggerWebhook(request))._tag, "accepted"); + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + yield* Queue.take(launches); + }), + { workers }, + ); +}); + +it.effect.each([1, 11])( + "keeps relay claims for %i coalesced updates after their window closes", + (count) => { + const workers = new Map>(); + return withService( + ({ service, sql, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + prompt: "{{body.value}}", + schedule: { + type: "webhook", + dedupe: { coalesceKey: "{{body.group}}", coalesceWindowSeconds: 10 }, + }, + }), + ); + const request = (value: number) => { + const bodyText = JSON.stringify({ group: "B", value }); + return requestFor(task, { + relayDeliveryId: `coalesced-relay-${value}`, + bodyText, + body: new TextEncoder().encode(bodyText), + }); + }; + yield* service.triggerWebhook(request(0)); + for (let i = 1; i <= count; i++) { + const result = yield* service.triggerWebhook(request(i)); + assert.equal(result._tag === "accepted" && result.outcome, "coalesced"); + } + assert.equal( + (yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length, + count + 1, + ); + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + assert.include((yield* Queue.take(launches)).initialMessage!.text, `>${count}<`); + const retry = yield* service.triggerWebhook(request(count)); + assert.equal(retry._tag === "accepted" && retry.outcome, "duplicate"); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0); + assert.equal((yield* service.list()).tasks[0]!.runCount, 1); + assert.equal(yield* Queue.size(launches), 0); + }), + { workers }, + ); + }, +); + +it.effect("failed windows release every member's replay key and relay claim", () => { + const workers = new Map>(); + return withService( + ({ service, sql, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + prompt: "{{body.value}}", + schedule: { + type: "webhook", + dedupe: { + replayKey: "{{headers.delivery-id}}", + coalesceKey: "{{body.group}}", + coalesceWindowSeconds: 10, + }, + }, + }), + ); + const request = (id: string) => { + const bodyText = JSON.stringify({ group: "same", value: id }); + return requestFor(task, { + bodyText, + body: new TextEncoder().encode(bodyText), + headers: { "delivery-id": id }, + relayDeliveryId: id, + }); + }; + yield* service.triggerWebhook(request("A")); + for (let i = 0; i < 12; i++) yield* service.triggerWebhook(request(`B${i}`)); + assert.equal( + (yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length, + 13, + ); + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + yield* Queue.take(launches); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 0); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length, 0); + const retry = yield* service.triggerWebhook(request("B11")); + assert.equal(retry._tag === "accepted" && retry.outcome, "accepted"); + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + assert.include((yield* Queue.take(launches)).initialMessage!.text, "B11"); + assert.equal((yield* service.list()).tasks[0]!.runCount, 2); + }), + { workers, failLaunches: { remaining: 1 } }, + ); +}); + +it.effect( + "rejects a twenty-first persisted window and accepts its relay retry after draining", + () => { + const workers = new Map>(); + return withService( + ({ service, sql, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + prompt: "{{body.group}}", + schedule: { + type: "webhook", + dedupe: { + replayKey: "{{body.group}}", + coalesceKey: "{{body.group}}", + coalesceWindowSeconds: 10, + }, + }, + }), + ); + const request = (group: number) => { + const bodyText = JSON.stringify({ group }); + return requestFor(task, { + bodyText, + body: new TextEncoder().encode(bodyText), + relayDeliveryId: `window-${group}`, + }); + }; + for (let i = 0; i < 20; i++) + assert.equal((yield* service.triggerWebhook(request(i)))._tag, "accepted"); + const overflow = yield* service.triggerWebhook(request(20)); + assert.equal(overflow._tag === "rate_limited" && overflow.outcome, "queue_full"); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 20); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 20); + assert.equal( + (yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length, + 20, + ); + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + for (let i = 0; i < 20; i++) yield* Queue.take(launches); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0); + const retry = yield* service.triggerWebhook(request(20)); + assert.equal(retry._tag === "accepted" && retry.outcome, "accepted"); + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + assert.include((yield* Queue.take(launches)).initialMessage!.text, "20"); + assert.equal((yield* service.list()).tasks[0]!.runCount, 21); + }), + { workers }, + ); + }, +); + +it.effect( + "recovery finishes failed-window claim cleanup without counting the attempt twice", + () => { + const workers = new Map>(); + return withService( + ({ service, sql, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { + type: "webhook", + dedupe: { + replayKey: "{{headers.delivery-id}}", + coalesceKey: "{{body}}", + coalesceWindowSeconds: 10, + }, + }, + }), + ); + const request = (id: string) => + requestFor(task, { headers: { "delivery-id": id }, relayDeliveryId: id }); + yield* service.triggerWebhook(request("A")); + yield* service.triggerWebhook(request("B")); + yield* sql`CREATE TRIGGER fail_claim_cleanup BEFORE DELETE ON scheduled_task_webhook_relay_deliveries + BEGIN SELECT RAISE(FAIL, 'cleanup unavailable'); END`; + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + yield* Queue.take(launches); + assert.equal((yield* service.list()).tasks[0]!.runCount, 1); + assert.equal( + (yield* sql<{ state: string }>`SELECT state FROM scheduled_task_webhook_pending`)[0]! + .state, + "failed", + ); + yield* sql`DROP TRIGGER fail_claim_cleanup`; + const restartedWorkers = new Map>(); + yield* withService( + ({ service: restarted, launches: restartedLaunches }) => + Effect.gen(function* () { + yield* drainPendingDispatch(restartedWorkers); + assert.equal(yield* Queue.size(restartedLaunches), 0); + assert.equal((yield* restarted.list()).tasks[0]!.runCount, 1); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0); + assert.equal( + (yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, + 0, + ); + assert.equal( + (yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length, + 0, + ); + const retry = yield* restarted.triggerWebhook(request("B")); + assert.equal(retry._tag === "accepted" && retry.outcome, "accepted"); + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(restartedWorkers); + yield* Queue.take(restartedLaunches); + assert.equal((yield* restarted.list()).tasks[0]!.runCount, 2); + }), + { sql, workers: restartedWorkers }, + ); + }), + { workers, failLaunches: { remaining: 1 } }, + ); + }, +); + +it.effect("a full run budget still accepts replay hits and the latest open-window payload", () => { + const workers = new Map>(); + return withService( + ({ service, launches, sql }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + prompt: "{{body.value}}", + schedule: { + type: "webhook", + dedupe: { + replayKey: "{{headers.delivery-id}}", + coalesceKey: "{{body.group}}", + coalesceWindowSeconds: 10, + }, + }, + }), + ); + const request = (id: string, value: string, group?: string) => { + const bodyText = JSON.stringify({ value, ...(group === undefined ? {} : { group }) }); + return requestFor(task, { + bodyText, + body: new TextEncoder().encode(bodyText), + headers: { "delivery-id": id }, + }); + }; + for (let i = 0; i < 59; i++) { + const result = yield* waitForSpan( + service.triggerWebhook(request(String(i), String(i))), + "ScheduledTaskService.runWebhookDelivery", + ); + assert.equal(result._tag === "accepted" && result.outcome, "accepted"); + yield* Queue.take(launches); + } + const opened = yield* service.triggerWebhook(request("B1", "old", "B")); + assert.equal(opened._tag === "accepted" && opened.outcome, "accepted"); + const duplicate = yield* service.triggerWebhook(request("0", "duplicate")); + assert.equal(duplicate._tag === "accepted" && duplicate.outcome, "duplicate"); + const update = yield* service.triggerWebhook(request("B2", "latest", "B")); + assert.equal(update._tag === "accepted" && update.outcome, "coalesced"); + assert.equal( + (yield* service.triggerWebhook(request("C", "new window", "C")))._tag, + "rate_limited", + ); + assert.equal( + (yield* service.triggerWebhook(request("D", "new immediate")))._tag, + "rate_limited", + ); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 1); + assert.equal(yield* Queue.size(launches), 0); + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + assert.include((yield* Queue.take(launches)).initialMessage!.text, "latest"); + assert.equal(yield* Queue.size(launches), 0); + assert.equal((yield* service.list()).tasks[0]!.runCount, 60); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0); + }), + { workers }, + ); +}); + +it.effect("dispatches an overdue window exactly once after its deadline passed while stopped", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const task = yield* withService( + ({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { + type: "webhook", + dedupe: { coalesceKey: "{{body}}", coalesceWindowSeconds: 10 }, + }, + }), + ); + yield* service.triggerWebhook(requestFor(task)); + assert.equal(yield* Queue.size(launches), 0); + return task; + }), + { sql, workers: new Map() }, + ); + // The first service scope is closed before the deadline advances. + yield* TestClock.adjust("30 seconds"); + const restartedWorkers = new Map>(); + yield* withService( + ({ service, launches }) => + Effect.gen(function* () { + yield* drainPendingDispatch(restartedWorkers); + yield* Queue.take(launches); + for (let i = 0; i < 3; i++) yield* restartedWorkers.get("webhook-coalescing")!; + assert.equal(yield* Queue.size(launches), 0); + assert.equal((yield* service.list()).tasks[0]!.runCount, 1); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0); + }), + { sql, workers: restartedWorkers }, + ); + const secondRestartWorkers = new Map>(); + yield* withService( + ({ service, launches }) => + Effect.gen(function* () { + yield* secondRestartWorkers.get("webhook-coalescing")!; + assert.equal(yield* Queue.size(launches), 0); + assert.equal( + (yield* service.list()).tasks.find((entry) => entry.id === task.id)!.runCount, + 1, + ); + }), + { sql, workers: secondRestartWorkers }, + ); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("removes a window with a render defect and dispatches the task's next window", () => { + const workers = new Map>(); + return withService( + ({ service, sql, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + prompt: "{{body.deep}}", + schedule: { + type: "webhook", + dedupe: { + replayKey: "{{headers.delivery-id}}", + coalesceKey: "{{headers.group}}", + coalesceWindowSeconds: 10, + }, + }, + }), + ); + const request = (id: string, bodyText: string) => + requestFor(task, { + bodyText, + body: new TextEncoder().encode(bodyText), + headers: { "delivery-id": id, group: id }, + relayDeliveryId: id, + }); + const bad = request( + "bad-render", + '{"deep":' + "[".repeat(20000) + "0" + "]".repeat(20000) + "}", + ); + assert.throws(() => renderWebhookPrompt(task.prompt, bad, "frame"), RangeError); + const accepted = yield* service.triggerWebhook(bad); + assert.equal(accepted._tag, "accepted"); + yield* service.triggerWebhook(request("good-render", '{"deep":"next window"}')); + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + assert.include((yield* Queue.take(launches)).initialMessage!.text, "next window"); + assert.equal(yield* Queue.size(launches), 0); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0); + assert.equal( + (yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys WHERE delivery_id = 'delivery:relay:bad-render'`) + .length, + 0, + ); + assert.equal( + (yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries WHERE relay_delivery_id = 'bad-render'`) + .length, + 0, + ); + const failed = (yield* service.getWebhookDelivery({ + id: task.id, + deliveryId: ScheduledTaskWebhookDeliveryId.make("delivery:relay:bad-render"), + acceptAnyOutcome: true, + })).delivery; + assert.equal(failed.outcome, "dispatch_failed"); + assert.equal(failed.error, "The pending webhook payload could not be prepared."); + yield* workers.get("webhook-coalescing")!; + assert.equal((yield* service.list()).tasks[0]!.runCount, 1); + }), + { workers }, + ); +}); + +it.effect.each([false, true])( + "counts an interrupted immediate run beside an unrelated running window (corrupt task: %s)", + (corrupt) => { + const workers = new Map>(); + const completedLaunches = new Set(); + return Effect.gen(function* () { + const gate = yield* Deferred.make(); + yield* withService( + ({ service, sql, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + prompt: "{{body.value}}", + schedule: { + type: "webhook", + dedupe: { coalesceKey: "{{headers.group}}", coalesceWindowSeconds: 10 }, + }, + }), + ); + const window = yield* service.triggerWebhook( + requestFor(task, { headers: { group: "old" }, bodyText: '{"value":"old"}' }), + ); + yield* sql`CREATE TRIGGER fail_old_completion BEFORE UPDATE ON scheduled_tasks + WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`; + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + yield* Queue.take(launches); + yield* sql`DROP TRIGGER fail_old_completion`; + assert.equal( + (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE state = 'running'`) + .length, + 1, + ); + const immediate = yield* service.triggerWebhook( + requestFor(task, { bodyText: '{"value":"immediate"}' }), + ); + yield* Queue.take(launches); + const [before] = yield* sql<{ + last_run_webhook_delivery_id: string; + model_selection_json: string; + run_count: number; + }>`SELECT last_run_webhook_delivery_id, model_selection_json, run_count FROM scheduled_tasks WHERE task_id = ${task.id}`; + assert.equal( + before!.last_run_webhook_delivery_id, + immediate._tag === "accepted" ? immediate.deliveryId : "", + ); + assert.notEqual( + before!.last_run_webhook_delivery_id, + window._tag === "accepted" ? window.deliveryId : "", + ); + assert.equal(before!.run_count, 0); + if (corrupt) + yield* sql`UPDATE scheduled_tasks SET model_selection_json = 'broken' WHERE task_id = ${task.id}`; + const restartedWorkers = new Map>(); + yield* withService( + ({ service: restarted, launches: recoveredLaunches }) => + Effect.gen(function* () { + const [recovered] = yield* sql<{ + run_count: number; + last_run_status: string; + }>`SELECT run_count, last_run_status FROM scheduled_tasks WHERE task_id = ${task.id}`; + assert.equal(recovered!.run_count, 1); + assert.equal(recovered!.last_run_status, "failed"); + if (corrupt) + yield* sql`UPDATE scheduled_tasks SET model_selection_json = ${before!.model_selection_json} WHERE task_id = ${task.id}`; + yield* drainPendingDispatch(restartedWorkers); + assert.equal(yield* Queue.size(recoveredLaunches), 0); + assert.equal((yield* restarted.list()).tasks[0]!.runCount, 2); + assert.equal( + (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, + 0, + ); + yield* restartedWorkers.get("webhook-coalescing")!; + assert.equal((yield* restarted.list()).tasks[0]!.runCount, 2); + }), + { sql, workers: restartedWorkers, completedLaunches }, + ); + }), + { + workers, + completedLaunches, + gateForTask: (input) => + input.initialMessage?.text.includes(">immediate<") ? gate : undefined, + }, + ); + }); + }, +); + +it.effect("checks filters after the signature and max age, and runs only matching deliveries", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { + type: "webhook", + signature: { + header: "x-hub-signature-256", + encoding: "hex", + prefix: "sha256=", + secret: "s3cret", + }, + maxDeliveryAgeMinutes: 30, + filters: [ + { path: "headers.x-github-event", op: "equals", value: "pull_request" }, + { path: "body.action", op: "in", value: ["opened", "reopened"] }, + ], + }, + }), + ); + const delivery = ( + event: string, + action: string, + options: { readonly signed?: boolean; readonly receivedAt?: string } = {}, + ) => { + const body = new TextEncoder().encode( + JSON.stringify({ action, pull_request: { url: "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/org/repo/pull/45" } }), + ); + const signature = `sha256=${NodeCrypto.createHmac("sha256", "s3cret").update(body).digest("hex")}`; + return requestFor(task, { + headers: { + "content-type": "application/json", + "x-github-event": event, + ...(options.signed === false ? {} : { "x-hub-signature-256": signature }), + }, + body, + bodyText: new TextDecoder().decode(body), + ...(options.receivedAt === undefined + ? {} + : { relayDeliveryId: "old", receivedAt: options.receivedAt }), + }); + }; + const now = yield* DateTime.now; + const old = DateTime.formatIso(DateTime.subtract(now, { minutes: 31 })); + + // A request failing a filter is still refused first for its signature or age. + const unsigned = yield* service.triggerWebhook(delivery("push", "opened", { signed: false })); + assert.equal(unsigned._tag, "rejected_signature"); + const tooOld = yield* service.triggerWebhook(delivery("push", "opened", { receivedAt: old })); + assert.equal(tooOld._tag, "expired"); + + const wrongEvent = yield* service.triggerWebhook(delivery("push", "opened")); + const wrongAction = yield* service.triggerWebhook(delivery("pull_request", "closed")); + assert.deepInclude(wrongEvent, { _tag: "accepted", outcome: "filtered" }); + assert.deepInclude(wrongAction, { _tag: "accepted", outcome: "filtered" }); + assert.equal(yield* Queue.size(launches), 0); + const [listed] = (yield* service.list()).tasks; + assert.equal(listed?.runCount, 0); + assert.equal(listed?.lastRunStatus, "never"); + + const matched = yield* service.triggerWebhook(delivery("pull_request", "opened")); + assert.deepInclude(matched, { _tag: "accepted", outcome: "accepted" }); + assert.include((yield* Queue.take(launches)).initialMessage?.text ?? "", "pull/45"); + + const deliveries = (yield* service.listWebhookDeliveries({ + id: task.id, + acceptAnyOutcome: true, + })).deliveries; + const filtered = deliveries + .filter((entry) => entry.outcome === "filtered") + .map((entry) => [entry.signatureVerified, entry.error]); + assert.sameDeepMembers(filtered, [ + [true, "Filter 1 (headers.x-github-event) did not match."], + [true, "Filter 2 (body.action) did not match."], + ]); + }), + ), +); + +const pullRequestFilterInput = () => + webhookTaskInput({ + schedule: { + type: "webhook", + filters: [{ path: "headers.x-github-event", op: "equals", value: "pull_request" }], + }, + }); +const githubEvent = (event: string) => ({ + headers: { "content-type": "application/json", "x-github-event": event }, +}); + +it.effect("filtered deliveries take no run slot and log at most 10 rows a minute", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* pullRequestFilterInput()); + const ignored = yield* Effect.forEach(Array.from({ length: 70 }), () => + service.triggerWebhook(requestFor(task, githubEvent("push"))), + ); + assert.isTrue( ignored.every((result) => result._tag === "accepted" && result.outcome === "filtered"), ); const matched = yield* service.triggerWebhook(requestFor(task, githubEvent("pull_request"))); @@ -1157,3 +2863,75 @@ it.effect("leaves filtered rows out of the list for clients that only decode old }), ), ); + +it.effect("filters run before replay and coalescing without claiming ignored deliveries", () => { + const workers = new Map>(); + return withService( + ({ service, launches, sql }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + prompt: "{{body.value}}", + schedule: { + type: "webhook", + filters: [{ path: "body.run", op: "equals", value: "true" }], + dedupe: { + replayKey: "{{headers.delivery-id}}", + coalesceKey: "{{body.group}}", + coalesceWindowSeconds: 10, + }, + }, + }), + ); + const request = (id: string, run: boolean, value: string) => { + const bodyText = JSON.stringify({ run, group: "B", value }); + return requestFor(task, { + headers: { "delivery-id": id }, + bodyText, + body: new TextEncoder().encode(bodyText), + relayDeliveryId: id, + }); + }; + const ignored = yield* service.triggerWebhook(request("A", false, "ignored")); + assert.deepInclude(ignored, { _tag: "accepted", outcome: "filtered" }); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 0); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length, 0); + const filteredId = ignored._tag === "accepted" ? ignored.deliveryId : assert.fail(); + const oldClient = yield* service + .getWebhookDelivery({ id: task.id, deliveryId: filteredId }) + .pipe(Effect.result); + assert.equal(oldClient._tag, "Failure"); + const currentClient = yield* service.getWebhookDelivery({ + id: task.id, + deliveryId: filteredId, + acceptAnyOutcome: true, + }); + assert.equal(currentClient.delivery.outcome, "filtered"); + assert.deepInclude(yield* service.triggerWebhook(request("A", true, "first")), { + _tag: "accepted", + outcome: "accepted", + }); + assert.deepInclude( + yield* service.triggerWebhook({ + ...request("A", false, "ignored replay"), + relayDeliveryId: "ignored-replay", + }), + { _tag: "accepted", outcome: "filtered" }, + ); + assert.deepInclude(yield* service.triggerWebhook(request("B", false, "ignored update")), { + _tag: "accepted", + outcome: "filtered", + }); + assert.deepInclude(yield* service.triggerWebhook(request("B", true, "latest")), { + _tag: "accepted", + outcome: "coalesced", + }); + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + assert.include((yield* Queue.take(launches)).initialMessage!.text, "latest"); + assert.equal(yield* Queue.size(launches), 0); + }), + { workers }, + ); +}); diff --git a/apps/server/src/scheduledTasks/webhookDedupe.ts b/apps/server/src/scheduledTasks/webhookDedupe.ts new file mode 100644 index 0000000000000..21d40b25ac5ed --- /dev/null +++ b/apps/server/src/scheduledTasks/webhookDedupe.ts @@ -0,0 +1,128 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- Keys need a deterministic digest, not random bytes. +import * as NodeCrypto from "node:crypto"; +import { ScheduledTask, ScheduledTaskError } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/sql/SqlClient"; +import { renderWebhookPrompt, type WebhookRequest } from "./webhookTemplate.ts"; + +function key(template: string | undefined, request: WebhookRequest): string { + if (template === undefined) return ""; + const rendered = renderWebhookPrompt(template, request, "", true); + return rendered.missing.length > 0 || rendered.prompt === "" + ? "" + : NodeCrypto.createHash("sha256").update(rendered.prompt).digest("hex"); +} + +export const releaseWebhookReplay = Effect.fn("ScheduledTaskService.releaseWebhookReplay")( + function* (deliveryId: string) { + const sql = yield* SqlClient.SqlClient; + yield* sql`DELETE FROM scheduled_task_webhook_replay_keys WHERE delivery_id = ${deliveryId}`; + }, +); + +const hasReplayKey = Effect.fn(function* (taskId: string, replayKey: string, now: number) { + const sql = yield* SqlClient.SqlClient; + return ( + replayKey !== "" && + (yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys + WHERE task_id = ${taskId} AND replay_key = ${replayKey} AND expires_at > ${now}`).length > 0 + ); +}); +const claimReplayKey = Effect.fn(function* ( + taskId: string, + replayKey: string, + deliveryId: string, + now: number, +) { + if (replayKey === "") return; + const sql = yield* SqlClient.SqlClient; + yield* sql`DELETE FROM scheduled_task_webhook_replay_keys WHERE expires_at <= ${now}`; + yield* sql`INSERT INTO scheduled_task_webhook_replay_keys (task_id, replay_key, delivery_id, expires_at) + VALUES (${taskId}, ${replayKey}, ${deliveryId}, ${now + 7 * 24 * 60 * 60 * 1000})`; +}); + +export const acceptWebhookDedupe = Effect.fn("ScheduledTaskService.acceptWebhookDedupe")(function* ( + task: ScheduledTask, + request: WebhookRequest, + deliveryId: string, + now: number, + reserve: Effect.Effect, + reserveCoalescedLog: Effect.Effect, + checkCapacity: Effect.Effect, + replaceDeliveryRequest: (ownerId: string) => Effect.Effect, +) { + const sql = yield* SqlClient.SqlClient; + const dedupe = task.schedule.type === "webhook" ? task.schedule.dedupe : undefined; + if (dedupe === undefined) return "run" as const; + const replayKey = key(dedupe.replayKey, request); + if (yield* hasReplayKey(task.id, replayKey, now)) return "duplicate" as const; + const coalesceKey = key(dedupe.coalesceKey, request); + if (coalesceKey === "") return "run" as const; + const pending = yield* sql<{ + delivery_id: string; + }>`SELECT delivery_id FROM scheduled_task_webhook_pending + WHERE task_id = ${task.id} AND coalesce_key = ${coalesceKey} AND state = 'pending' AND due_at > ${now}`; + const logCoalesced = pending.length > 0 ? yield* reserveCoalescedLog : true; + if (pending.length === 0) { + yield* checkCapacity; + yield* reserve; + } + // Coalesced requests retain their validation slot, bounding writes without dropping accepted payloads. + yield* claimReplayKey(task.id, replayKey, pending[0]?.delivery_id ?? deliveryId, now); + const requestJson = yield* encodePendingRequest({ + method: request.method, + path: request.path, + query: request.query, + headers: request.headers, + bodyText: request.bodyText, + }); + if (pending.length > 0) { + yield* sql`UPDATE scheduled_task_webhook_pending SET request_json = ${requestJson}, + delivery_ids_json = json_insert(delivery_ids_json, '$[#]', ${deliveryId}) WHERE delivery_id = ${pending[0]!.delivery_id}`; + yield* replaceDeliveryRequest(pending[0]!.delivery_id); + return logCoalesced ? ("coalesced" as const) : ("coalesced_unlogged" as const); + } + // Expired windows are immutable, even if the scheduler has not dispatched them yet. + yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'dispatching' + WHERE task_id = ${task.id} AND coalesce_key = ${coalesceKey} AND state = 'pending' AND due_at <= ${now}`; + yield* sql`INSERT INTO scheduled_task_webhook_pending (delivery_id, task_id, coalesce_key, due_at, task_json, request_json, delivery_ids_json) + VALUES (${deliveryId}, ${task.id}, ${coalesceKey}, ${now + (dedupe.coalesceWindowSeconds ?? 120) * 1000}, ${yield* encodePendingTask(task)}, ${requestJson}, json_array(${deliveryId}))`; + return "pending" as const; +}); + +export const acceptImmediateWebhookDedupe = Effect.fn( + "ScheduledTaskService.acceptImmediateWebhookDedupe", +)(function* ( + task: ScheduledTask, + request: WebhookRequest, + deliveryId: string, + now: number, + reserve: Effect.Effect, +) { + const replayKey = key( + task.schedule.type === "webhook" ? task.schedule.dedupe?.replayKey : undefined, + request, + ); + if (yield* hasReplayKey(task.id, replayKey, now)) return "duplicate" as const; + yield* reserve; + yield* claimReplayKey(task.id, replayKey, deliveryId, now); + return "run" as const; +}); + +const PendingTaskJson = Schema.fromJsonString(ScheduledTask); +const encodePendingTask = Schema.encodeEffect(PendingTaskJson); +export const decodePendingTask = Schema.decodeUnknownEffect(PendingTaskJson); +const PendingRequestJson = Schema.fromJsonString( + Schema.Struct({ + method: Schema.String, + path: Schema.String, + query: Schema.String, + headers: Schema.Record(Schema.String, Schema.String), + bodyText: Schema.String, + }), +); +const encodePendingRequest = Schema.encodeEffect(PendingRequestJson); +export const decodePendingRequest = Schema.decodeUnknownEffect(PendingRequestJson); +export const dedupeError = (cause: unknown) => + new ScheduledTaskError({ message: "Could not persist webhook dedupe state.", cause }); diff --git a/apps/server/src/scheduledTasks/webhookRoute.test.ts b/apps/server/src/scheduledTasks/webhookRoute.test.ts index 218281053704c..332e6778ba1b3 100644 --- a/apps/server/src/scheduledTasks/webhookRoute.test.ts +++ b/apps/server/src/scheduledTasks/webhookRoute.test.ts @@ -236,6 +236,7 @@ describe("webhook route", () => { const response = await handler(post("/api/hooks/id/tok", "{}")); expect(response.status).toBe(status); expect(response.headers.get("x-t3-hook-outcome")).toBe(outcome); + if (status === 429) expect(await response.json()).toEqual({ error: outcome }); } finally { await dispose(); } diff --git a/apps/server/src/scheduledTasks/webhookRoute.ts b/apps/server/src/scheduledTasks/webhookRoute.ts index ce67fa157d334..2566349d273b6 100644 --- a/apps/server/src/scheduledTasks/webhookRoute.ts +++ b/apps/server/src/scheduledTasks/webhookRoute.ts @@ -118,7 +118,7 @@ export const layer = HttpApiBuilder.group( case "disabled": return json(409, { error: "hook_disabled" }, "disabled"); case "rate_limited": - return json(429, { error: "rate_limited" }, result.outcome); + return json(429, { error: result.outcome }, result.outcome); case "expired": return json(410, { error: "delivery_too_old" }, "expired"); case "error": diff --git a/apps/server/src/scheduledTasks/webhookTemplate.ts b/apps/server/src/scheduledTasks/webhookTemplate.ts index 5f3ef760ad8b4..063d41a7ec462 100644 --- a/apps/server/src/scheduledTasks/webhookTemplate.ts +++ b/apps/server/src/scheduledTasks/webhookTemplate.ts @@ -150,6 +150,7 @@ export function renderWebhookPrompt( template: string, request: WebhookRequest, frameId: string, + raw = false, ): RenderedWebhookPrompt { const missing: string[] = []; const resolve = webhookExpressionResolver(request); @@ -160,12 +161,13 @@ export function renderWebhookPrompt( const expression = rawExpression.trim(); const value = resolve(expression); const text = value === undefined || value === null ? undefined : stringify(value); - if (text === undefined) { + if (text === undefined || (raw && text === "")) { if (!missing.includes(expression)) missing.push(expression); return ""; } // An empty value carries no request data, so it needs no frame or note. if (text === "") return ""; + if (raw) return text; framed = true; return `<${tag}>${text.replace(MENTION, "$&\u2060")}`; }); diff --git a/apps/web/src/components/settings/ScheduledTasksSettings.tsx b/apps/web/src/components/settings/ScheduledTasksSettings.tsx index 5f54581669085..82d2092f61fbe 100644 --- a/apps/web/src/components/settings/ScheduledTasksSettings.tsx +++ b/apps/web/src/components/settings/ScheduledTasksSettings.tsx @@ -22,7 +22,6 @@ import type { ScheduledTaskId, ScheduledTaskSchedule, ScheduledTaskUpsertInput, - ScheduledTaskWebhookDeliveryOutcome, ScheduledTaskWebhookDeliverySummary, ThreadId, } from "@t3tools/contracts"; @@ -81,6 +80,8 @@ import { SettingsGroup } from "./SettingsGroup"; import { settingsScopeEnvironmentLabel, settingsScopeEnvironmentLabels } from "./settingsScopeAxis"; import { WEBHOOK_SIGNATURE_DEFAULTS, + webhookDraftErrors, + webhookDeliveryLabel, matchesScheduledTaskScope, scheduleFromDraft, scheduledTaskDefaultModel, @@ -152,6 +153,9 @@ const EMPTY_DRAFT: DraftState = { ...WEBHOOK_SIGNATURE_DEFAULTS, signatureSecret: "", maxDeliveryAgeMinutes: "", + replayKey: "", + coalesceKey: "", + coalesceWindowSeconds: "", filters: [], savedFilters: [], }; @@ -328,26 +332,15 @@ export function ScheduledRelativeLabel(props: { value: string | null }) { return relativeLabel(props.value, Date.parse(`${nowMinute}:00.000Z`)); } -const DELIVERY_OUTCOME_LABELS: Record = { - accepted: "Ran", - dispatch_failed: "Run failed", - rejected_signature: "Bad signature", - disabled: "Task paused", - rate_limited: "Rate limited", - expired: "Too old", - filtered: "Filtered", -}; - -// A newer server can report an outcome this client has no label for. -function deliveryOutcomeLabel(outcome: string): string { - return Object.hasOwn(DELIVERY_OUTCOME_LABELS, outcome) - ? DELIVERY_OUTCOME_LABELS[outcome as ScheduledTaskWebhookDeliveryOutcome] - : outcome.replaceAll("_", " "); -} - function deliveryOutcomeVariant(outcome: string) { if (outcome === "accepted") return "success"; - if (outcome === "disabled" || outcome === "rate_limited" || outcome === "expired") { + if ( + outcome === "duplicate" || + outcome === "coalesced" || + outcome === "disabled" || + outcome === "rate_limited" || + outcome === "expired" + ) { return "warning"; } if (outcome === "dispatch_failed" || outcome === "rejected_signature") return "error"; @@ -822,7 +815,7 @@ function WebhookDeliveriesDialog({ ? null : serverEnvironment.scheduledTaskWebhookDelivery({ environmentId, - input: { id: task.id, deliveryId: selectedId }, + input: { id: task.id, deliveryId: selectedId, acceptAnyOutcome: true }, }), ); const deliveries = deliveriesQuery.data?.deliveries ?? null; @@ -850,8 +843,12 @@ function WebhookDeliveriesDialog({ ) : selected ? (
- - {deliveryOutcomeLabel(selected.outcome)} + + {webhookDeliveryLabel(selected)} {selected.method} · @@ -871,8 +868,13 @@ function WebhookDeliveriesDialog({ {selected.error}

) : null} - - {selected.renderedPrompt ?? "No run was started for this request."} + + {selected.renderedPrompt ?? + (selected.pendingWindow + ? "Run scheduled when the window closes; no run started yet." + : "No run was started for this request.")} {selected.missingFields.length > 0 ? (

@@ -906,8 +908,12 @@ function WebhookDeliveriesDialog({ className="flex w-full flex-wrap items-center gap-2 py-2 text-left text-sm hover:bg-accent/40" onClick={() => setSelectedId(delivery.id)} > - - {deliveryOutcomeLabel(delivery.outcome)} + + {webhookDeliveryLabel(delivery)} {delivery.method} @@ -1193,10 +1199,7 @@ function ScheduledTaskEditorForm({ } const schedule = scheduleFromDraft(draft); if (schedule === null) { - reportFailure( - "Invalid age limit", - `Enter whole minutes from 1 to ${MAX_WEBHOOK_DELIVERY_AGE_MINUTES}, or leave it blank.`, - ); + for (const error of webhookDraftErrors(draft)) reportFailure(error.title, error.description); return; } if ( @@ -1543,6 +1546,60 @@ function ScheduledTaskEditorForm({ filters={draft.filters} onChange={(filters) => setDraft((current) => ({ ...current, filters }))} /> + + + setDraft((current) => ({ ...current, replayKey: event.target.value })) + } + /> + + + + setDraft((current) => ({ ...current, coalesceKey: event.target.value })) + } + /> + + + + setDraft((current) => ({ + ...current, + coalesceWindowSeconds: event.target.value, + })) + } + /> + +

+ Replay keys suppress repeat deliveries for 7 days. Coalescing runs the latest + payload once when the window from the first delivery closes. Empty keys run + normally. +

diff --git a/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts b/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts index 2b9264b9dd70e..4be149ac9d718 100644 --- a/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts +++ b/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts @@ -20,6 +20,8 @@ import { scheduledTaskDefaultModel, matchesScheduledTaskScope, scheduleFromDraft, + webhookDraftErrors, + webhookDeliveryLabel, taskToDraft, } from "./scheduledTasksSettings.logic"; @@ -175,6 +177,43 @@ describe("webhook scheduled tasks", () => { webhook: { path: "/api/hooks/legacy-task/token", url: null, hasSecret: true }, }; + it("round-trips dedupe settings, clears them, and rejects invalid windows", () => { + const dedupe = { + replayKey: "{{headers.delivery-id}}", + coalesceKey: "{{body.id}}", + coalesceWindowSeconds: 10, + }; + const draft = taskToDraft({ + ...webhookTask, + schedule: { type: "webhook", signature: null, dedupe }, + }); + expect(scheduleFromDraft(draft)).toMatchObject({ dedupe }); + expect( + scheduleFromDraft({ + ...draft, + replayKey: " \n ", + coalesceKey: " \t ", + coalesceWindowSeconds: "", + }), + ).toMatchObject({ dedupe: {} }); + expect( + scheduleFromDraft({ + ...draft, + replayKey: ` ${dedupe.replayKey} `, + coalesceKey: `\t${dedupe.coalesceKey}\n`, + }), + ).toMatchObject({ dedupe }); + expect( + scheduleFromDraft({ ...draft, replayKey: "", coalesceKey: "", coalesceWindowSeconds: "" }), + ).toMatchObject({ dedupe: {} }); + expect(scheduleFromDraft({ ...draft, coalesceWindowSeconds: "" })).toMatchObject({ + dedupe: { replayKey: dedupe.replayKey, coalesceKey: dedupe.coalesceKey }, + }); + for (const coalesceWindowSeconds of ["9", "3601", "1.5", "abc"]) { + expect(scheduleFromDraft({ ...draft, coalesceWindowSeconds })).toBeNull(); + } + }); + it("keeps a stored secret when the secret field is left blank", () => { const draft = taskToDraft(webhookTask); expect(draft.scheduleMode).toBe("webhook"); @@ -183,11 +222,13 @@ describe("webhook scheduled tasks", () => { type: "webhook", signature, maxDeliveryAgeMinutes: null, + dedupe: {}, }); expect(scheduleFromDraft({ ...draft, signatureSecret: " new " })).toEqual({ type: "webhook", signature: { ...signature, secret: "new" }, maxDeliveryAgeMinutes: null, + dedupe: {}, }); }); @@ -199,6 +240,7 @@ describe("webhook scheduled tasks", () => { type: "webhook", signature: null, maxDeliveryAgeMinutes: null, + dedupe: {}, }); }); @@ -212,6 +254,7 @@ describe("webhook scheduled tasks", () => { for (const blank of ["", " "]) { expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: blank })).toMatchObject({ maxDeliveryAgeMinutes: null, + dedupe: {}, }); } for (const invalid of ["0", "-5", "1.5", "abc", "1441"]) { @@ -339,3 +382,58 @@ describe("scheduled task model defaults", () => { ).toBeNull(); }); }); + +it("rejects dedupe templates without a request placeholder", () => { + const draft = taskToDraft({ ...legacyTask, schedule: { type: "webhook", signature: null } }); + expect(scheduleFromDraft({ ...draft, replayKey: "literal" })).toBeNull(); + expect(scheduleFromDraft({ ...draft, coalesceKey: "{{body.bad" })).toBeNull(); +}); + +describe("webhook delivery labels and validation messages", () => { + it("preserves Ran for legacy and dispatched deliveries and distinguishes pending windows", () => { + expect(webhookDeliveryLabel({ outcome: "accepted" })).toBe("Ran"); + expect(webhookDeliveryLabel({ outcome: "accepted", pendingWindow: false })).toBe("Ran"); + expect(webhookDeliveryLabel({ outcome: "accepted", pendingWindow: true })).toBe( + "Opened window", + ); + }); + + it("keeps the existing age-limit title and body", () => { + const draft = taskToDraft({ ...legacyTask, schedule: { type: "webhook", signature: null } }); + expect( + webhookDraftErrors({ ...draft, maxDeliveryAgeMinutes: "0", replayKey: "literal" }), + ).toEqual([ + { + title: "Invalid age limit", + description: "Enter whole minutes from 1 to 1440, or leave it blank.", + }, + ]); + }); + + it.each([ + [{ replayKey: "literal" }, "Replay key must contain a valid request placeholder."], + [{ coalesceKey: "{{body.bad" }, "Coalesce key must contain a valid request placeholder."], + [{ coalesceWindowSeconds: "9" }, "Coalesce window must be whole seconds from 10 to 3600."], + ])("names only the invalid webhook field for %o", (invalid, description) => { + const draft = taskToDraft({ ...legacyTask, schedule: { type: "webhook", signature: null } }); + expect(webhookDraftErrors({ ...draft, ...invalid })).toEqual([ + { title: "Invalid webhook settings", description }, + ]); + }); + + it("reports separate short errors when several dedupe fields fail", () => { + const draft = taskToDraft({ ...legacyTask, schedule: { type: "webhook", signature: null } }); + expect( + webhookDraftErrors({ + ...draft, + replayKey: "bad", + coalesceKey: "bad", + coalesceWindowSeconds: "0", + }).map((error) => error.description), + ).toEqual([ + "Replay key must contain a valid request placeholder.", + "Coalesce key must contain a valid request placeholder.", + "Coalesce window must be whole seconds from 10 to 3600.", + ]); + }); +}); diff --git a/apps/web/src/components/settings/scheduledTasksSettings.logic.ts b/apps/web/src/components/settings/scheduledTasksSettings.logic.ts index 472ba9b998a8a..43030010c5aec 100644 --- a/apps/web/src/components/settings/scheduledTasksSettings.logic.ts +++ b/apps/web/src/components/settings/scheduledTasksSettings.logic.ts @@ -1,5 +1,7 @@ +import { isWebhookDedupeTemplate } from "@t3tools/contracts"; import { EnvironmentId, + MAX_WEBHOOK_DELIVERY_AGE_MINUTES, type ProjectId, ScheduledTaskId, type ScheduledTask, @@ -83,6 +85,9 @@ export interface DraftState { /** Write-only: empty keeps the secret already stored on the server. */ readonly signatureSecret: string; readonly maxDeliveryAgeMinutes: string; + readonly replayKey: string; + readonly coalesceKey: string; + readonly coalesceWindowSeconds: string; readonly filters: ReadonlyArray; /** The filter rows the form opened with. */ readonly savedFilters: ReadonlyArray; @@ -94,9 +99,62 @@ export const WEBHOOK_SIGNATURE_DEFAULTS = { signaturePrefix: "sha256=", } as const; +export function webhookDraftErrors(draft: DraftState) { + if (draft.scheduleMode !== "webhook") return []; + if (parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes) === undefined) + return [ + { + title: "Invalid age limit", + description: `Enter whole minutes from 1 to ${MAX_WEBHOOK_DELIVERY_AGE_MINUTES}, or leave it blank.`, + }, + ]; + const errors: Array<{ title: string; description: string }> = []; + for (const [label, value] of [ + ["Replay key", draft.replayKey], + ["Coalesce key", draft.coalesceKey], + ] as const) { + if (value.trim() && !isWebhookDedupeTemplate(value.trim())) + errors.push({ + title: "Invalid webhook settings", + description: `${label} must contain a valid request placeholder.`, + }); + } + const window = draft.coalesceWindowSeconds.trim(); + if (window && (!Number.isInteger(Number(window)) || Number(window) < 10 || Number(window) > 3600)) + errors.push({ + title: "Invalid webhook settings", + description: "Coalesce window must be whole seconds from 10 to 3600.", + }); + return errors; +} + +export function webhookDeliveryLabel(delivery: { + readonly outcome: string; + readonly pendingWindow?: boolean | undefined; +}) { + if (delivery.outcome === "accepted") return delivery.pendingWindow ? "Opened window" : "Ran"; + const labels: Record = { + dispatch_failed: "Run failed", + duplicate: "Duplicate", + coalesced: "Coalesced", + filtered: "Filtered", + rejected_signature: "Bad signature", + disabled: "Task paused", + rate_limited: "Rate limited", + expired: "Too old", + }; + return labels[delivery.outcome] ?? delivery.outcome.replaceAll("_", " "); +} + export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedule | null { if (draft.scheduleMode === "webhook") { + if (webhookDraftErrors(draft).length > 0) return null; const maxDeliveryAgeMinutes = parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes); + const replayKey = draft.replayKey.trim(); + const coalesceKey = draft.coalesceKey.trim(); + const coalesceWindowSeconds = draft.coalesceWindowSeconds.trim() + ? Number(draft.coalesceWindowSeconds) + : undefined; if (maxDeliveryAgeMinutes === undefined) return null; const parsedFilters = parseWebhookFilterDrafts(draft.filters); if (!("filters" in parsedFilters)) return null; @@ -112,6 +170,15 @@ export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedul } : null, maxDeliveryAgeMinutes, + ...(replayKey || coalesceKey || coalesceWindowSeconds !== undefined + ? { + dedupe: { + ...(replayKey ? { replayKey } : {}), + ...(coalesceKey ? { coalesceKey } : {}), + ...(coalesceWindowSeconds === undefined ? {} : { coalesceWindowSeconds }), + }, + } + : { dedupe: {} }), // Only edited filters are sent; the server keeps the stored ones otherwise. ...(webhookFilterDraftsChanged(draft.savedFilters, draft.filters) ? { filters: parsedFilters.filters } @@ -173,6 +240,12 @@ export function taskToDraft(task: ScheduledTask): DraftState { } : { signatureEnabled: false, ...WEBHOOK_SIGNATURE_DEFAULTS }), signatureSecret: "", + replayKey: schedule.type === "webhook" ? (schedule.dedupe?.replayKey ?? "") : "", + coalesceKey: schedule.type === "webhook" ? (schedule.dedupe?.coalesceKey ?? "") : "", + coalesceWindowSeconds: + schedule.type === "webhook" && schedule.dedupe?.coalesceWindowSeconds !== undefined + ? String(schedule.dedupe.coalesceWindowSeconds) + : "", maxDeliveryAgeMinutes: schedule.type === "webhook" && schedule.maxDeliveryAgeMinutes != null ? String(schedule.maxDeliveryAgeMinutes) diff --git a/docs/user/project-settings.md b/docs/user/project-settings.md index cdae02679be6e..db994a7fe8de7 100644 --- a/docs/user/project-settings.md +++ b/docs/user/project-settings.md @@ -118,6 +118,20 @@ compare as text, and a value that is an object or a list only passes **exists**. Separate **is one of** values with commas; a value can't contain a comma or start or end with a space. +To suppress repeated deliveries, set a **Replay key** template, such as +`{{headers.delivery-id}}`. The same key is ignored for seven days per task. +To combine a burst, set a **Coalesce key**, such as `{{body.id}}`, and a window +of 10–3600 seconds (120 by default). One run uses the latest body, headers, +and query when the window from the first delivery closes; later requests +do not extend it. Pending windows survive a server restart. Keys must contain +request placeholders; if any placeholder is missing, or the render is empty, +that key's rule is skipped. Clear the templates to turn dedupe off. + +Duplicate and coalesced log entries share a limit of 10 per task per minute. +Additional deliveries are accepted without a log entry, and coalesced deliveries +always replace the persisted payload. Coalesced updates count toward a separate +60-per-minute validation limit; exceeding it receives HTTP 429. + On desktop and web, pick **Deliveries** from a task's menu to see recent requests and the prompt each one produced. diff --git a/packages/contracts/src/scheduledTask.test.ts b/packages/contracts/src/scheduledTask.test.ts index 61af412b05558..4e781d3c62003 100644 --- a/packages/contracts/src/scheduledTask.test.ts +++ b/packages/contracts/src/scheduledTask.test.ts @@ -2,14 +2,49 @@ import { describe, expect, it } from "vite-plus/test"; import * as Schema from "effect/Schema"; import { + isWebhookDedupeTemplate, MIN_SCHEDULED_TASK_INTERVAL_MS, ScheduledTaskListWebhookDeliveriesResult, ScheduledTaskSchedule, ScheduledTaskUpsertSchedule, + ScheduledTaskWebhookDeliverySummary, } from "./scheduledTask.ts"; const decodeSchedule = Schema.decodeUnknownSync(ScheduledTaskSchedule); const decodeUpsertSchedule = Schema.decodeUnknownSync(ScheduledTaskUpsertSchedule); +const decodeDeliverySummary = Schema.decodeSync(ScheduledTaskWebhookDeliverySummary); +const decodeDeliveryList = Schema.decodeUnknownSync(ScheduledTaskListWebhookDeliveriesResult); + +it("rejects a long unterminated dotted dedupe key without excessive backtracking", () => { + const template = "{{body." + "a.".repeat(5000) + "x"; + const startedAt = performance.now(); + const valid = isWebhookDedupeTemplate(template); + const elapsedMs = performance.now() - startedAt; + expect(valid).toBe(false); + expect(elapsedMs).toBeLessThan(50); + for (const field of ["replayKey", "coalesceKey"]) { + expect(() => + decodeUpsertSchedule({ type: "webhook", dedupe: { [field]: template } }), + ).toThrow(); + } +}); + +it.each(["duplicate", "coalesced", "future_outcome"])("decodes delivery outcome %s", (outcome) => { + expect( + decodeDeliverySummary({ + id: "delivery:test", + taskId: "scheduled-task:test", + receivedAt: "2026-10-20T00:00:00.000Z", + method: "POST", + contentType: null, + bodyBytes: 0, + outcome, + signatureVerified: false, + missingFields: [], + error: null, + }).outcome, + ).toBe(outcome); +}); describe("ScheduledTaskSchedule", () => { it("keeps legacy sub-minute persisted schedules readable", () => { @@ -52,6 +87,15 @@ describe("ScheduledTaskUpsertSchedule", () => { }); }); +it.each(["literal", " ", "\t\n", "{{body.bad", "{{unknown.field}}"])( + "rejects invalid dedupe template %s", + (replayKey) => { + expect(() => decodeUpsertSchedule({ type: "webhook", dedupe: { replayKey } })).toThrow(); + expect(() => + decodeUpsertSchedule({ type: "webhook", dedupe: { coalesceKey: replayKey } }), + ).toThrow(); + }, +); describe("webhook filters", () => { const accepts = (filters: unknown) => { try { @@ -114,9 +158,8 @@ describe("ScheduledTaskWebhookDeliverySummary", () => { missingFields: [], error: null, }; - expect( - Schema.decodeUnknownSync(ScheduledTaskListWebhookDeliveriesResult)({ deliveries: [summary] }) - .deliveries[0]?.outcome, - ).toBe("some_future_outcome"); + expect(decodeDeliveryList({ deliveries: [summary] }).deliveries[0]?.outcome).toBe( + "some_future_outcome", + ); }); }); diff --git a/packages/contracts/src/scheduledTask.ts b/packages/contracts/src/scheduledTask.ts index 77825113992f0..67e2314a8b010 100644 --- a/packages/contracts/src/scheduledTask.ts +++ b/packages/contracts/src/scheduledTask.ts @@ -83,6 +83,26 @@ const WebhookMaxDeliveryAgeMinutes = Schema.Int.check( "Skip requests the relay held longer than this many minutes while the environment was offline. Null runs every request.", }); +export function isWebhookDedupeTemplate(template: string): boolean { + const placeholders = /\{\{\s*(?:(?:body|headers|query)(?:\.[^{}.\s]+)*|request)\s*\}\}/g; + const matches = [...template.matchAll(placeholders)]; + const remainder = template.replace(placeholders, ""); + return matches.length > 0 && !remainder.includes("{{") && !remainder.includes("}}"); +} + +const WebhookDedupeTemplate = Schema.String.check(Schema.makeFilter(isWebhookDedupeTemplate)); + +const ScheduledTaskWebhookDedupe = Schema.Struct({ + replayKey: Schema.optional(WebhookDedupeTemplate), + coalesceKey: Schema.optional(WebhookDedupeTemplate), + coalesceWindowSeconds: Schema.optional( + Schema.Int.check(Schema.isBetween({ minimum: 10, maximum: 3600 })), + ), +}).annotate({ + description: + "Replay key template (7 days), coalesce key template, and fixed window in seconds (default 120).", +}); + export const MAX_WEBHOOK_FILTERS = 16; const WebhookFilterPath = TrimmedNonEmptyString.check( @@ -132,6 +152,7 @@ const ScheduledTaskWebhookSchedule = Schema.Struct({ }), signature: Schema.NullOr(ScheduledTaskWebhookSignature), maxDeliveryAgeMinutes: Schema.optional(Schema.NullOr(WebhookMaxDeliveryAgeMinutes)), + dedupe: Schema.optional(ScheduledTaskWebhookDedupe), filters: Schema.optional(ScheduledTaskWebhookFilters), }).annotate({ description: @@ -159,6 +180,7 @@ const ScheduledTaskUpsertWebhookSchedule = Schema.Struct({ description: "Signature check; omit or null to accept requests by URL token only.", }), maxDeliveryAgeMinutes: Schema.optional(Schema.NullOr(WebhookMaxDeliveryAgeMinutes)), + dedupe: Schema.optional(ScheduledTaskWebhookDedupe), filters: Schema.optional(ScheduledTaskWebhookFilters), }).annotate({ description: @@ -294,6 +316,8 @@ export type ScheduledTaskWebhookDeliveryId = typeof ScheduledTaskWebhookDelivery export const ScheduledTaskWebhookDeliveryOutcome = Schema.Literals([ "accepted", "dispatch_failed", + "duplicate", + "coalesced", "rejected_signature", "disabled", "rate_limited", @@ -312,6 +336,7 @@ export const ScheduledTaskWebhookDeliverySummary = Schema.Struct({ // Any string, so an outcome added later doesn't break a client's whole list. outcome: Schema.Union([ScheduledTaskWebhookDeliveryOutcome, Schema.String]), signatureVerified: Schema.Boolean, + pendingWindow: Schema.optional(Schema.Boolean), missingFields: Schema.Array(Schema.String), error: Schema.NullOr(Schema.String), }); @@ -348,6 +373,7 @@ export type ScheduledTaskListWebhookDeliveriesResult = export const ScheduledTaskGetWebhookDeliveryInput = Schema.Struct({ id: ScheduledTaskId, deliveryId: ScheduledTaskWebhookDeliveryId, + acceptAnyOutcome: Schema.optional(Schema.Boolean), }); export type ScheduledTaskGetWebhookDeliveryInput = typeof ScheduledTaskGetWebhookDeliveryInput.Type; From 14b47ee1b83aa302cb8c5cc511b56da16f2bcbc2 Mon Sep 17 00:00:00 2001 From: Jacob Hands Date: Tue, 20 Oct 2026 16:51:21 -0400 Subject: [PATCH 08/17] ci(release): Make Windows arm64 CI job 50% faster (#22428) --- .github/workflows/release-desktop.yml | 402 +++++++++++++-------- .github/workflows/release.yml | 5 +- docs/operations/release.md | 6 +- knip.jsonc | 11 +- patches/app-builder-lib@26.16.1.patch | 15 + pnpm-lock.yaml | 12 +- pnpm-workspace.yaml | 1 + scripts/build-cli-archive.ts | 21 +- scripts/build-desktop-artifact.test.ts | 70 +++- scripts/build-desktop-artifact.ts | 53 ++- scripts/build-npm-launcher.ts | 2 +- scripts/lib/windows-jsign.ts | 65 ++++ scripts/lib/windows-tar.ts | 4 + scripts/package.json | 1 + scripts/smoke-cli-archive.ts | 2 +- scripts/verify-windows-desktop-artifact.ts | 48 +++ 16 files changed, 537 insertions(+), 181 deletions(-) create mode 100644 patches/app-builder-lib@26.16.1.patch create mode 100644 scripts/lib/windows-jsign.ts create mode 100644 scripts/lib/windows-tar.ts create mode 100644 scripts/verify-windows-desktop-artifact.ts diff --git a/.github/workflows/release-desktop.yml b/.github/workflows/release-desktop.yml index 08552d1cd53a1..36738a03cf9d3 100644 --- a/.github/workflows/release-desktop.yml +++ b/.github/workflows/release-desktop.yml @@ -64,6 +64,13 @@ on: required: false default: false type: boolean + # A runner of the target's own platform and architecture, for a `runner` + # that cannot execute the target. The build job skips everything that + # runs the packaged binaries, and a second job runs it there. + verify_runner: + required: false + default: "" + type: string version: required: true type: string @@ -129,39 +136,6 @@ jobs: git sparse-checkout set --no-cone '/*' '!/.repos/' git -c checkout.workers=0 checkout --detach FETCH_HEAD - # The runner's PowerShell 7.6/.NET 10 intermittently crashes during signing: - # https://github.com/dotnet/runtime/issues/131636 - # Put the .NET 8 release first for both workflow shells and electron-builder. - - name: Use PowerShell 7.4 for Windows ARM builds - if: inputs.platform == 'win' && inputs.arch == 'arm64' - shell: powershell - run: | - $ErrorActionPreference = "Stop" - $ProgressPreference = "SilentlyContinue" - $version = "7.4.20" - $archive = Join-Path $env:RUNNER_TEMP "PowerShell-$version-win-arm64.zip" - $installDir = Join-Path $env:RUNNER_TEMP "powershell-$version-arm64" - Invoke-WebRequest -UseBasicParsing ` - -Uri "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/PowerShell/PowerShell/releases/download/v$version/PowerShell-$version-win-arm64.zip" ` - -OutFile $archive - $expectedHash = "67eff749d5dae357501c95a3eec1dcd93f669824e3ca2b1d2adcdecdebf04f35" - if ((Get-FileHash $archive -Algorithm SHA256).Hash -ne $expectedHash) { - throw "PowerShell archive checksum mismatch" - } - Expand-Archive -Path $archive -DestinationPath $installDir - $installDir | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - & "$installDir\pwsh.exe" -NoProfile -NonInteractive -Command '$PSVersionTable; [System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture' - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - - name: Verify Windows ARM PowerShell selection - if: inputs.platform == 'win' && inputs.arch == 'arm64' - shell: pwsh - run: | - $expectedPath = Join-Path $env:RUNNER_TEMP "powershell-7.4.20-arm64\pwsh.exe" - if ((Get-Command pwsh.exe).Source -ne $expectedPath -or $PSVersionTable.PSVersion -ne [version]"7.4.20") { - throw "Windows ARM builds must use the pinned PowerShell runtime" - } - # Windows installs straight from the registry. There, linking the packages # takes the time, not downloading them, so a ~700 MB package cache did not # pay for its own restore. Across recent nightlies, restore plus install @@ -172,7 +146,7 @@ jobs: uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1 with: node-version-file: package.json - cache: ${{ inputs.platform != 'win' }} + cache: ${{ runner.os != 'Windows' }} run-install: false # pnpm checks the lockfile and policy before reusing this result. A missing @@ -245,7 +219,7 @@ jobs: # monitor, the desktop build's preflight skips its MSVC check, and node-pty # ships prebuilt Windows binaries, so the 30-60s install is skipped too. - name: Install Spectre-mitigated MSVC libs - if: inputs.platform == 'win' && steps.resource_monitor_cache.outputs.cache-hit != 'true' + if: runner.os == 'Windows' && steps.resource_monitor_cache.outputs.cache-hit != 'true' shell: pwsh run: | $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" @@ -261,27 +235,21 @@ jobs: exit $code } - - uses: ./.github/actions/setup-apt-mirrors - if: inputs.platform == 'linux' + - name: Install cargo-xwin + if: runner.os != 'Windows' && inputs.platform == 'win' && steps.resource_monitor_cache.outputs.cache-hit != 'true' + uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2 + with: + tool: cargo-xwin - - name: Install Linux desktop build libraries - if: inputs.platform == 'linux' + # Built here rather than inside the desktop build so the CLI archive, + # which bundles the same binary, can be built and uploaded first. + - name: Build resource monitor + if: steps.resource_monitor_cache.outputs.cache-hit != 'true' shell: bash - run: | - sudo apt-get update - sudo apt-get install -y libsecret-1-dev pkg-config - if ! command -v magick >/dev/null 2>&1 && ! command -v convert >/dev/null 2>&1; then - sudo apt-get install -y imagemagick - fi - - if command -v magick >/dev/null 2>&1; then - magick -version - else - convert -version - fi + run: cargo ${{ runner.os != 'Windows' && inputs.platform == 'win' && 'xwin build' || 'build' }} --locked --release --manifest-path native/resource-monitor/Cargo.toml --target "${{ inputs.rust_target }}" - name: Prepare Azure Trusted Signing - if: inputs.platform == 'win' + if: inputs.platform == 'win' && runner.os == 'Windows' shell: pwsh env: AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} @@ -342,6 +310,153 @@ jobs: Select-Object -Unique "PSModulePath=$($modulePathEntries -join ';')" >> $env:GITHUB_ENV + # Off Windows, scripts/lib/windows-jsign.ts signs through jsign instead of + # the TrustedSigning PowerShell module. + - name: Install jsign + if: inputs.platform == 'win' && runner.os != 'Windows' + shell: bash + run: | + set -euo pipefail + dir="$RUNNER_TEMP/jsign" + mkdir -p "$dir" + curl -fsSL --retry 3 -o "$dir/jsign.jar" https://github.com/ebourg/jsign/releases/download/7.6/jsign-7.6.jar + echo "6cf58aa080e7ff82656f320cff8d4a3bff30408df8766806dcba3740dd839fa5 $dir/jsign.jar" | sha256sum -c - + printf '#!/bin/sh\nexec java -jar "%s" "$@"\n' "$dir/jsign.jar" > "$dir/jsign" + chmod +x "$dir/jsign" + echo "T3CODE_JSIGN=$dir/jsign" >> "$GITHUB_ENV" + + # The desktop build applies the same branding to apps/server/dist/client + # (resolveDesktopWebAssetBrand in scripts/build-desktop-artifact.ts); the + # CLI archive below ships that client and is now built first. + - name: Apply web client branding + if: inputs.cli_archive + shell: bash + run: | + brand=production + if [[ "${{ inputs.version }}" =~ -nightly\.[0-9]{8}\.[0-9]+$ ]]; then brand=nightly; fi + node scripts/apply-web-brand-assets.ts "$brand" apps/server/dist/client + + # The single-executable is built with a Node that supports --build-sea + # (25.7+); the repo itself stays on the engines.node version. A native + # build injects into the runner's own Node: tsdown's cross-target download + # path runs `tar` on a drive-letter path on Windows, which GNU tar reads as + # a remote host. Off Windows, a Windows target downloads its node.exe. + - name: Build CLI single-executable + if: inputs.cli_archive + shell: bash + env: + # The exact version, not a major: vp downloads it from nodejs.org/dist on + # the runner, and only exact versions have a dist directory. Keep in + # step with SEA_NODE_VERSION in apps/server/vite.config.ts. + VP_NODE_VERSION: "26.8.2" + # The executable bundles the server again, so it needs the same stamp + # as release.yml's JS bundle. + T3CODE_TELEMETRY_CHANNEL: ${{ github.repository == 'pingdotgg/t3code' && inputs.release_channel || 'dev' }} + run: node apps/server/scripts/cli.ts build-exe --verbose ${{ runner.os != 'Windows' && inputs.platform == 'win' && format('--target win-{0}', inputs.arch) || '' }} + + - name: Import macOS signing certificate for the CLI archive + if: inputs.cli_archive && inputs.platform == 'mac' + shell: bash + env: + CSC_LINK: ${{ secrets.CSC_LINK }} + CSC_KEY_PASSWORD: ${{ secrets.CSC_KEY_PASSWORD }} + run: | + set -euo pipefail + if [[ -z "$CSC_LINK" || -z "$CSC_KEY_PASSWORD" ]]; then + echo "macOS CLI signing disabled (missing CSC_LINK); the archive is signed ad hoc." + exit 0 + fi + keychain="$RUNNER_TEMP/t3-cli-signing.keychain-db" + keychain_password="$(openssl rand -hex 16)" + cert_path="$RUNNER_TEMP/t3-cli-signing.p12" + printf '%s' "$CSC_LINK" | base64 --decode > "$cert_path" + security create-keychain -p "$keychain_password" "$keychain" + security set-keychain-settings -lut 21600 "$keychain" + security unlock-keychain -p "$keychain_password" "$keychain" + security import "$cert_path" -k "$keychain" -P "$CSC_KEY_PASSWORD" -T /usr/bin/codesign + security set-key-partition-list -S apple-tool:,apple: -s -k "$keychain_password" "$keychain" >/dev/null + security list-keychains -d user -s "$keychain" $(security list-keychains -d user | tr -d '"') + identity="$(security find-identity -v -p codesigning "$keychain" | sed -n 's/.*"\(Developer ID Application: [^"]*\)".*/\1/p' | head -n 1)" + if [[ -z "$identity" ]]; then + echo "No Developer ID Application identity found in CSC_LINK." >&2 + exit 1 + fi + echo "::add-mask::$keychain_password" + echo "T3CODE_CLI_MAC_SIGN_IDENTITY=$identity" >> "$GITHUB_ENV" + echo "macOS CLI signing enabled." + + - name: Stage resource monitor for the CLI archive + if: inputs.cli_archive + shell: bash + run: | + set -euo pipefail + binary_name="t3-resource-monitor" + if [[ "${{ inputs.platform }}" == "win" ]]; then + binary_name="${binary_name}.exe" + fi + target_dir="$RUNNER_TEMP/cli-resource-monitor/${{ inputs.resource_key }}" + mkdir -p "$target_dir" + cp "native/resource-monitor/target/${{ inputs.rust_target }}/release/${binary_name}" "$target_dir/$binary_name" + + - name: Build CLI archive + if: inputs.cli_archive + shell: bash + env: + APPLE_API_KEY: ${{ secrets.APPLE_API_KEY }} + APPLE_API_KEY_ID: ${{ secrets.APPLE_API_KEY_ID }} + APPLE_API_ISSUER: ${{ secrets.APPLE_API_ISSUER }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + AZURE_TRUSTED_SIGNING_ENDPOINT: ${{ secrets.AZURE_TRUSTED_SIGNING_ENDPOINT }} + AZURE_TRUSTED_SIGNING_ACCOUNT_NAME: ${{ secrets.AZURE_TRUSTED_SIGNING_ACCOUNT_NAME }} + AZURE_TRUSTED_SIGNING_CERTIFICATE_PROFILE_NAME: ${{ secrets.AZURE_TRUSTED_SIGNING_CERTIFICATE_PROFILE_NAME }} + run: | + set -euo pipefail + if [[ "${{ inputs.platform }}" == "mac" && -n "${APPLE_API_KEY:-}" ]]; then + key_path="$RUNNER_TEMP/AuthKey_cli_${APPLE_API_KEY_ID}.p8" + printf '%s' "$APPLE_API_KEY" > "$key_path" + export APPLE_API_KEY="$key_path" + fi + node scripts/build-cli-archive.ts \ + --platform "${{ inputs.platform }}" \ + --arch "${{ inputs.arch }}" \ + --version "${{ inputs.version }}" \ + --resource-monitor-dir "$RUNNER_TEMP/cli-resource-monitor" \ + --output-dir release-cli + + - name: Smoke-test CLI archive + if: inputs.cli_archive && inputs.verify_runner == '' + shell: bash + run: node scripts/smoke-cli-archive.ts --archive release-cli/* --expect-version "${{ inputs.version }}" + + - name: Upload CLI archive + if: inputs.cli_archive + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: cli-${{ inputs.platform }}-${{ inputs.arch }} + path: release-cli/* + if-no-files-found: error + + - uses: ./.github/actions/setup-apt-mirrors + if: inputs.platform == 'linux' + + - name: Install Linux desktop build libraries + if: inputs.platform == 'linux' + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y libsecret-1-dev pkg-config + if ! command -v magick >/dev/null 2>&1 && ! command -v convert >/dev/null 2>&1; then + sudo apt-get install -y imagemagick + fi + + if command -v magick >/dev/null 2>&1; then + magick -version + else + convert -version + fi + # The WSL backend runs the Linux CLI archive inside the distro, so the # Windows desktop embeds the same-arch archive the release attaches. # This job starts alongside that Linux job instead of after it, and the @@ -407,7 +522,7 @@ jobs: shell: bash env: pnpm_config_cache_dir: ${{ runner.temp }}/pnpm-metadata - T3CODE_DESKTOP_REUSE_RESOURCE_MONITOR: ${{ steps.resource_monitor_cache.outputs.cache-hit == 'true' }} + T3CODE_DESKTOP_REUSE_RESOURCE_MONITOR: true T3CODE_DESKTOP_REUSE_LINUX_CAPTURE_HELPERS: ${{ steps.capture_helper_cache.outputs.cache-hit == 'true' }} CSC_LINK: ${{ secrets.CSC_LINK }} CSC_KEY_PASSWORD: ${{ secrets.CSC_KEY_PASSWORD }} @@ -433,6 +548,9 @@ jobs: --skip-build --verbose ) + if [[ -n "${{ inputs.verify_runner }}" ]]; then + args+=(--keep-stage) + fi has_all() { for value in "$@"; do @@ -488,107 +606,35 @@ jobs: vp run dist:desktop:artifact "${args[@]}" - # The single-executable is built with a Node that supports --build-sea - # (25.7+); the repo itself stays on the engines.node version. It always - # injects into the runner's own Node: tsdown's cross-target download path - # runs `tar` on a drive-letter path on Windows, which GNU tar reads as a - # remote host, and a cross-built macOS binary cannot be smoke-tested. - - name: Build CLI single-executable - if: inputs.cli_archive - shell: bash - env: - # The exact version, not a major: vp downloads it from nodejs.org/dist on - # the runner, and only exact versions have a dist directory. Keep in - # step with SEA_NODE_VERSION in apps/server/vite.config.ts. - VP_NODE_VERSION: "26.8.2" - # The executable bundles the server again, so it needs the same stamp - # as release.yml's JS bundle. - T3CODE_TELEMETRY_CHANNEL: ${{ github.repository == 'pingdotgg/t3code' && inputs.release_channel || 'dev' }} - run: node apps/server/scripts/cli.ts build-exe --verbose - - - name: Import macOS signing certificate for the CLI archive - if: inputs.cli_archive && inputs.platform == 'mac' - shell: bash - env: - CSC_LINK: ${{ secrets.CSC_LINK }} - CSC_KEY_PASSWORD: ${{ secrets.CSC_KEY_PASSWORD }} - run: | - set -euo pipefail - if [[ -z "$CSC_LINK" || -z "$CSC_KEY_PASSWORD" ]]; then - echo "macOS CLI signing disabled (missing CSC_LINK); the archive is signed ad hoc." - exit 0 - fi - keychain="$RUNNER_TEMP/t3-cli-signing.keychain-db" - keychain_password="$(openssl rand -hex 16)" - cert_path="$RUNNER_TEMP/t3-cli-signing.p12" - printf '%s' "$CSC_LINK" | base64 --decode > "$cert_path" - security create-keychain -p "$keychain_password" "$keychain" - security set-keychain-settings -lut 21600 "$keychain" - security unlock-keychain -p "$keychain_password" "$keychain" - security import "$cert_path" -k "$keychain" -P "$CSC_KEY_PASSWORD" -T /usr/bin/codesign - security set-key-partition-list -S apple-tool:,apple: -s -k "$keychain_password" "$keychain" >/dev/null - security list-keychains -d user -s "$keychain" $(security list-keychains -d user | tr -d '"') - identity="$(security find-identity -v -p codesigning "$keychain" | sed -n 's/.*"\(Developer ID Application: [^"]*\)".*/\1/p' | head -n 1)" - if [[ -z "$identity" ]]; then - echo "No Developer ID Application identity found in CSC_LINK." >&2 - exit 1 - fi - echo "::add-mask::$keychain_password" - echo "T3CODE_CLI_MAC_SIGN_IDENTITY=$identity" >> "$GITHUB_ENV" - echo "macOS CLI signing enabled." - - - name: Stage resource monitor for the CLI archive - if: inputs.cli_archive + # The verify job runs what this host could not: the packaged natives and + # the CLI. Its scripts are bundled so that job needs no install. + - name: Stage verification inputs + if: inputs.verify_runner != '' shell: bash run: | set -euo pipefail - binary_name="t3-resource-monitor" - if [[ "${{ inputs.platform }}" == "win" ]]; then - binary_name="${binary_name}.exe" - fi - target_dir="$RUNNER_TEMP/cli-resource-monitor/${{ inputs.resource_key }}" - mkdir -p "$target_dir" - cp "native/resource-monitor/target/${{ inputs.rust_target }}/release/${binary_name}" "$target_dir/$binary_name" - - - name: Build CLI archive - if: inputs.cli_archive - shell: bash - env: - APPLE_API_KEY: ${{ secrets.APPLE_API_KEY }} - APPLE_API_KEY_ID: ${{ secrets.APPLE_API_KEY_ID }} - APPLE_API_ISSUER: ${{ secrets.APPLE_API_ISSUER }} - AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} - AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} - AZURE_TRUSTED_SIGNING_ENDPOINT: ${{ secrets.AZURE_TRUSTED_SIGNING_ENDPOINT }} - AZURE_TRUSTED_SIGNING_ACCOUNT_NAME: ${{ secrets.AZURE_TRUSTED_SIGNING_ACCOUNT_NAME }} - AZURE_TRUSTED_SIGNING_CERTIFICATE_PROFILE_NAME: ${{ secrets.AZURE_TRUSTED_SIGNING_CERTIFICATE_PROFILE_NAME }} - run: | - set -euo pipefail - if [[ "${{ inputs.platform }}" == "mac" && -n "${APPLE_API_KEY:-}" ]]; then - key_path="$RUNNER_TEMP/AuthKey_cli_${APPLE_API_KEY_ID}.p8" - printf '%s' "$APPLE_API_KEY" > "$key_path" - export APPLE_API_KEY="$key_path" - fi - node scripts/build-cli-archive.ts \ - --platform "${{ inputs.platform }}" \ - --arch "${{ inputs.arch }}" \ - --version "${{ inputs.version }}" \ - --resource-monitor-dir "$RUNNER_TEMP/cli-resource-monitor" \ - --output-dir release-cli - - - name: Smoke-test CLI archive - if: inputs.cli_archive - shell: bash - run: node scripts/smoke-cli-archive.ts --archive release-cli/* --expect-version "${{ inputs.version }}" - - - name: Upload CLI archive - if: inputs.cli_archive + mkdir -p verify + unpacked=$(ls -d "${TMPDIR:-/tmp}"/t3code-desktop-${{ inputs.platform }}-stage-*/app/dist/${{ inputs.platform }}-${{ inputs.arch }}-unpacked) + tar -C "$(dirname "$unpacked")" -cf verify/unpacked.tar "$(basename "$unpacked")" + esbuild=scripts/node_modules/.bin/esbuild + banner="import{createRequire as __cr}from'node:module';const require=__cr(import.meta.url);" + # build-desktop-artifact.ts's own import.meta.main guard must not run + # its CLI inside the verify bundle. + "$esbuild" scripts/verify-windows-desktop-artifact.ts --bundle --platform=node --format=esm \ + --target=node24 --define:import.meta.main=false --banner:js="$banner" \ + --outfile=verify/verify-windows-desktop-artifact.mjs + "$esbuild" scripts/smoke-cli-archive.ts --bundle --platform=node --format=esm \ + --target=node24 --banner:js="$banner" --outfile=verify/smoke-cli-archive.mjs + + - name: Upload verification inputs + if: inputs.verify_runner != '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: cli-${{ inputs.platform }}-${{ inputs.arch }} - path: release-cli/* + name: verify-${{ inputs.platform }}-${{ inputs.arch }} + path: verify/* if-no-files-found: error + retention-days: 1 + compression-level: 1 - name: Collect release assets shell: bash @@ -661,3 +707,63 @@ jobs: name: resource-monitor-${{ inputs.resource_key }} path: resource-monitor-publish/${{ inputs.resource_key }}/* if-no-files-found: error + + verify: + name: Verify + needs: [build] + if: inputs.verify_runner != '' + runs-on: ${{ inputs.verify_runner }} + timeout-minutes: 15 + steps: + - name: Download verification inputs + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: verify-${{ inputs.platform }}-${{ inputs.arch }} + path: verify + + - name: Download CLI archive + if: inputs.cli_archive + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: cli-${{ inputs.platform }}-${{ inputs.arch }} + path: release-cli + + - name: Verify packaged natives + shell: bash + run: | + set -euo pipefail + tar -C verify -xf verify/unpacked.tar + node verify/verify-windows-desktop-artifact.mjs \ + --app-dir "$(cygpath -w "$PWD/verify/${{ inputs.platform }}-${{ inputs.arch }}-unpacked")" \ + --arch "${{ inputs.arch }}" --build-version "${{ inputs.version }}" --verbose + + - name: Smoke-test CLI archive + if: inputs.cli_archive + shell: bash + run: node verify/smoke-cli-archive.mjs --archive release-cli/* --expect-version "${{ inputs.version }}" + + # cargo-xwin links this binary on Linux, and nothing else runs it before users do. + - name: Smoke-test resource monitor + shell: pwsh + run: | + $startInfo = [System.Diagnostics.ProcessStartInfo]::new( + "verify\${{ inputs.platform }}-${{ inputs.arch }}-unpacked\resources\resource-monitor\t3-resource-monitor.exe") + $startInfo.UseShellExecute = $false + $startInfo.RedirectStandardInput = $true + $startInfo.RedirectStandardOutput = $true + $monitor = [System.Diagnostics.Process]::Start($startInfo) + $hello = $monitor.StandardOutput.ReadLine() | ConvertFrom-Json + if ($hello.type -ne "hello" -or $hello.arch -ne "${{ inputs.arch == 'arm64' && 'aarch64' || 'x86_64' }}") { + throw "Unexpected hello: $($hello | ConvertTo-Json -Compress)" + } + $monitor.StandardInput.WriteLine((@{ version = $hello.version; type = "configure"; rootPid = $PID; sampleIntervalMs = 1000 } | ConvertTo-Json -Compress)) + $monitor.StandardInput.WriteLine((@{ version = $hello.version; type = "sampleNow"; requestId = "verify" } | ConvertTo-Json -Compress)) + $monitor.StandardInput.Close() + $output = $monitor.StandardOutput.ReadToEnd() + $monitor.WaitForExit() + $sample = $output -split "`n" | Where-Object { $_.Trim() } | ForEach-Object { $_ | ConvertFrom-Json } | + Where-Object { $_.requestId -eq "verify" } + if ($monitor.ExitCode -ne 0 -or $sample.type -ne "snapshot" -or -not ($sample.processes | Where-Object { $_.pid -eq $PID })) { + throw "Resource monitor exited $($monitor.ExitCode) with: $output" + } + "Resource monitor sampled $($sample.scannedProcessCount) processes." diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 21e9e7ba99c56..df8a719f9e791 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -659,6 +659,8 @@ jobs: resource_key: win32-x64 cli_archive: true + # Windows arm64 runners have 4 vCPUs, and the packaging is mostly CPU-bound, + # so the build runs on Linux and only the native checks run on arm64. desktop_win_arm64: name: Desktop Windows arm64 needs: [preflight, relay_public_config, build_bundle] @@ -674,7 +676,8 @@ jobs: clerk_cli_oauth_client_id: ${{ needs.relay_public_config.outputs.clerk_cli_oauth_client_id }} relay_url: ${{ needs.relay_public_config.outputs.relay_url }} label: Windows arm64 - runner: windows-11-arm + runner: blacksmith-16vcpu-ubuntu-2404 + verify_runner: windows-11-arm platform: win target: nsis arch: arm64 diff --git a/docs/operations/release.md b/docs/operations/release.md index c96551bc00c06..c6e151434ca34 100644 --- a/docs/operations/release.md +++ b/docs/operations/release.md @@ -31,7 +31,7 @@ the version. - Runs lint, typecheck, and tests alongside artifact builds. Publishing waits for every check. - Reads the shared production T3 Connect relay URL and Clerk client configuration before packaging clients. - Builds the platform-independent JS (server bundle, web client, Electron main) once in the `build_bundle` job and hands it to every platform job as the `js-bundle` artifact; the platform jobs only package it, so no runner rebuilds it. -- Builds six desktop artifacts in parallel for both channels, each as its own job (`desktop__`, one call of `release-desktop.yml`) on hardware of its own architecture, gated only on the bundle. The Windows jobs embed the same-arch Linux CLI archive as the WSL runtime and wait for that artifact partway through, not for the whole Linux job: +- Builds six desktop artifacts in parallel for both channels, each as its own job (`desktop__`, one call of `release-desktop.yml`), gated only on the bundle. Every target builds on hardware of its own architecture except Windows arm64: its runners have 4 vCPUs, so it builds and signs (with jsign) on a Linux runner, and a `windows-11-arm` job runs the packaged natives and the CLI archive (`verify_runner`). The Windows jobs embed the same-arch Linux CLI archive as the WSL runtime and wait for that artifact partway through, not for the whole Linux job: - macOS `arm64` DMG - macOS `x64` DMG - Linux `x64` and `arm64` AppImage and `.deb`, from one electron-builder run. The `.deb` updates in the app through electron-updater, which installs it with `dpkg`. @@ -42,11 +42,11 @@ the version. - Nightly runs are always GitHub prereleases and never marked latest. - Automatically generated release notes are pinned to the previous tag in the same channel, so stable compares to the previous stable tag and nightly compares to the previous nightly tag. - Includes Electron auto-update metadata (for example `latest*.yml`, `nightly*.yml`, and `*.blockmap`) in release assets. -- Builds a self-contained CLI archive per platform (`t3---.tar.gz`, `.zip` on Windows) in the same job as that target's desktop artifact and attaches them to the GitHub Release with a `SHA256SUMS` file, on every channel, for five targets: macOS arm64, Linux x64 and arm64, Windows x64 and arm64. Every archive is built, signed, and smoke-tested on hardware of its own architecture. There is no macOS x64 archive: Node single-executables are unsupported on x64 macOS (the SEA docs list macOS as arm64 only) and the binary segfaults on start; the x64 desktop app is Electron and unaffected. +- Builds a self-contained CLI archive per platform (`t3---.tar.gz`, `.zip` on Windows) in the same job as that target's desktop artifact and attaches them to the GitHub Release with a `SHA256SUMS` file, on every channel, for five targets: macOS arm64, Linux x64 and arm64, Windows x64 and arm64. Every archive is smoke-tested on hardware of its own architecture, and every one but Windows arm64 is also built and signed there. There is no macOS x64 archive: Node single-executables are unsupported on x64 macOS (the SEA docs list macOS as arm64 only) and the binary segfaults on start; the x64 desktop app is Electron and unaffected. - The archive holds the server as a Node single-executable (`scripts/build-cli-archive.ts`), so unpacking it needs neither Node, npm, nor a compiler. It is the only form in which T3 Code manages a runtime: the desktop's SSH environments, the boot service, `t3 update`, and the install scripts all download and verify this archive against `SHA256SUMS`. The `t3` npm package exists for people who run `npx t3` or `npm install -g t3` themselves, and it downloads this same archive too; nothing in the product installs from npm. The `curl | sh` installers are `scripts/install.sh` and `scripts/install.ps1`; the marketing site copies them into its `public/` at build time (`apps/marketing/scripts/stage-install-scripts.mjs`) and serves them at `t3.codes/install.sh` and `/install.ps1`. - The executable is built with a Node that supports `--build-sea` (`VP_NODE_VERSION=26.8.2`, kept in step with `SEA_NODE_VERSION` in `apps/server/vite.config.ts`), while the repo stays on `engines.node`. - macOS archives are signed with the Developer ID certificate and notarized when the Apple secrets are present (ad hoc otherwise, which still runs from `curl`/`tar` installs). Windows executables use the same Azure Trusted Signing setup as the installer. Every native addon in the macOS archive is signed too, since the hardened runtime refuses unsigned libraries. - - Each archive is extracted and executed on its build runner (`scripts/smoke-cli-archive.ts`) before it is uploaded. + - Each archive is extracted and executed (`scripts/smoke-cli-archive.ts`) on its build runner, or on the `verify_runner` for Windows arm64. - After the GitHub Release is published, publishes the `t3` npm package with OIDC trusted publishing from the same workflow file. It is only a launcher (`scripts/npm-launcher/t3.mjs`): on first run it downloads its exact version's CLI archive from the GitHub Release, verifies it against `SHA256SUMS`, and unpacks it into `/runtime/versions/`, the layout the install scripts and `t3 update` share. The executables are not on npm. They used to be, as `@t3code/t3--` optional dependencies, but npm's publish-time scanning held those 80-250 MB packages back for up to an hour after the launcher was live, and every install in that window got a launcher with no executable. Those packages stay on npm at their old versions and get no new ones. Because the launcher reads from the release, `publish_cli` runs after `release`, never before. - stable releases publish npm dist-tag `latest` - nightly releases publish npm dist-tag `nightly` diff --git a/knip.jsonc b/knip.jsonc index fd2587116159b..0815e45e46262 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -18,8 +18,15 @@ }, "scripts": { // Knip loads its preprocessor through a CLI option; native verification and - // worktree setup (from t3.json) run directly. - "entry": ["knip-schemas.ts", "mobile-native-client.ts", "setup-worktree.ts"], + // worktree setup (from t3.json) run directly. release-desktop.yml bundles the + // Windows verify script with esbuild through a path knip does not trace. + "entry": [ + "knip-schemas.ts", + "mobile-native-client.ts", + "setup-worktree.ts", + "verify-windows-desktop-artifact.ts", + ], + "ignoreDependencies": ["esbuild"], // The `t3` npm package's files: copied into the package, never imported. "ignore": ["npm-launcher/**"], }, diff --git a/patches/app-builder-lib@26.16.1.patch b/patches/app-builder-lib@26.16.1.patch new file mode 100644 index 0000000000000..72655d9f0a62d --- /dev/null +++ b/patches/app-builder-lib@26.16.1.patch @@ -0,0 +1,15 @@ +diff --git a/out/targets/nsis/NsisTarget.js b/out/targets/nsis/NsisTarget.js +index 8a3462b185b078a260483635e6d774e2ece6e269..105778c145ab63a26444179215c643a7f22148a0 100644 +--- a/out/targets/nsis/NsisTarget.js ++++ b/out/targets/nsis/NsisTarget.js +@@ -366,7 +366,9 @@ class NsisTarget extends core_1.Target { + await this.executeMakensis(defines, commands, sharedHeader + (await this.computeFinalScript(script, false, archs))); + // http://forums.winamp.com/showthread.php?p=3078545 + // TODO: remove workaround when wine is fully upgraded to 11 +- if ((0, macosVersion_1.isMacOsCatalina)()) { ++ // T3 Code builds Windows installers on Linux without wine; read the ++ // uninstaller out of the stub the way macOS hosts already do. ++ if ((0, macosVersion_1.isMacOsCatalina)() || process.platform === "linux") { + try { + await nsisUtil_1.UninstallerReader.exec(installerPath, uninstallerPath); + } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d1455a1b206b4..d71e2422a3b63 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -179,6 +179,7 @@ patchedDependencies: '@react-native-menu/menu@2.0.0': a4866ecfcd44f318043de839d20110c17e14909004900f002ffe12050376f2ba '@react-navigation/native-stack@7.21.0': caae6552051bce91a85f567f3a08f9c8d25ce4c53f65a9391a67f016bb1d798e alchemy@2.0.0-beta.80: bea1b6c0c0b23157fd5f439a1d7feff430f28ac1621ba135d8a13eeec60ccacd + app-builder-lib@26.16.1: a28c2dfcfc2b90d6c6247e199e6537391db9ecd941854c650910d607442512d3 dbus-next@0.10.2: cfff57561b0ee59b5addb3b2e6c6f20906e967507a530ab67e8db8108e520ba4 expo-audio@58.0.9: 2c2f8c73a23d48804c8b537b8cd73c25eb331aa47d82eaa481c6766cb81e5add expo-modules-core@58.0.22: f0b0fd100550ed76e5c25b4d13eb7c9ff62825f8f197c53edf7bf447c50c9fb7 @@ -1962,6 +1963,9 @@ importers: '@types/pngjs': specifier: 6.0.5 version: 6.0.5 + esbuild: + specifier: 0.28.2 + version: 0.28.2 typescript: specifier: 'catalog:' version: 7.0.2 @@ -18181,7 +18185,7 @@ snapshots: anynum@1.0.1: optional: true - app-builder-lib@26.16.1(dmg-builder@26.16.1)(electron-builder-squirrel-windows@26.16.1): + app-builder-lib@26.16.1(patch_hash=a28c2dfcfc2b90d6c6247e199e6537391db9ecd941854c650910d607442512d3)(dmg-builder@26.16.1)(electron-builder-squirrel-windows@26.16.1): dependencies: '@electron/asar': 3.4.1 '@electron/fuses': 1.8.0 @@ -19234,7 +19238,7 @@ snapshots: dmg-builder@26.16.1(electron-builder-squirrel-windows@26.16.1): dependencies: - app-builder-lib: 26.16.1(dmg-builder@26.16.1)(electron-builder-squirrel-windows@26.16.1) + app-builder-lib: 26.16.1(patch_hash=a28c2dfcfc2b90d6c6247e199e6537391db9ecd941854c650910d607442512d3)(dmg-builder@26.16.1)(electron-builder-squirrel-windows@26.16.1) builder-util: 26.16.0 fs-extra: 10.1.0 js-yaml: 4.3.2 @@ -19326,7 +19330,7 @@ snapshots: electron-builder-squirrel-windows@26.16.1(dmg-builder@26.16.1): dependencies: - app-builder-lib: 26.16.1(dmg-builder@26.16.1)(electron-builder-squirrel-windows@26.16.1) + app-builder-lib: 26.16.1(patch_hash=a28c2dfcfc2b90d6c6247e199e6537391db9ecd941854c650910d607442512d3)(dmg-builder@26.16.1)(electron-builder-squirrel-windows@26.16.1) builder-util: 26.16.0 electron-winstaller: 5.4.0 transitivePeerDependencies: @@ -19335,7 +19339,7 @@ snapshots: electron-builder@26.16.1(electron-builder-squirrel-windows@26.16.1): dependencies: - app-builder-lib: 26.16.1(dmg-builder@26.16.1)(electron-builder-squirrel-windows@26.16.1) + app-builder-lib: 26.16.1(patch_hash=a28c2dfcfc2b90d6c6247e199e6537391db9ecd941854c650910d607442512d3)(dmg-builder@26.16.1)(electron-builder-squirrel-windows@26.16.1) builder-util: 26.16.0 builder-util-runtime: 9.7.0 chalk: 4.1.2 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index b7f755edc3b4d..6e0d596dd7434 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -357,6 +357,7 @@ patchedDependencies: # Android: Maven-resolved gradle plugins (expo-widgets) have no sourceDir; without this # skip, the fingerprint silently drops every Expo native module and OTAs reach stale binaries. "@expo/fingerprint@0.21.5": patches/@expo__fingerprint@0.21.5.patch + app-builder-lib@26.16.1: patches/app-builder-lib@26.16.1.patch peerDependencyRules: allowAny: diff --git a/scripts/build-cli-archive.ts b/scripts/build-cli-archive.ts index c56106871385d..1367c27c180d5 100644 --- a/scripts/build-cli-archive.ts +++ b/scripts/build-cli-archive.ts @@ -30,6 +30,8 @@ import { STAGE_INSTALL_ARGS, } from "./build-desktop-artifact.ts"; import { selectCliRuntimeExternalDependencies } from "./lib/cli-external-packages.ts"; +import { signWithJsign } from "./lib/windows-jsign.ts"; +import { windowsSystemTar } from "./lib/windows-tar.ts"; import { resolveCatalogDependencies } from "./lib/resolve-catalog.ts"; const BuildPlatform = Schema.Literals(["mac", "linux", "win"]); @@ -95,11 +97,6 @@ export function cliArchiveFileName(version: string, platform: BuildPlatform, arc return `${cliArchiveStem(version, platform, arch)}.${platform === "win" ? "zip" : "tar.gz"}`; } -export function windowsSystemTar(): string { - const systemRoot = process.env.SystemRoot ?? process.env.windir ?? "C:\\Windows"; - return `${systemRoot}\\System32\\tar.exe`; -} - const runCommand = Effect.fn("runCommand")(function* ( command: ChildProcess.Command, label: string, @@ -411,6 +408,11 @@ const signWindowsExecutable = Effect.fn("signWindowsExecutable")(function* ( return; } yield* stripStaleAuthenticodeEntry(executablePath); + if ((yield* HostProcess.Platform) !== "win32") { + yield* signWithJsign(executablePath); + yield* Effect.log("[cli-archive] Signed t3.exe (Azure Trusted Signing via jsign)."); + return; + } const quote = (value: string) => `'${value.replaceAll("'", "''")}'`; const script = [ "$ErrorActionPreference = 'Stop';", @@ -507,7 +509,14 @@ const buildCliArchive = Effect.fn("buildCliArchive")(function* (input: { cliArchiveFileName(input.version, input.platform, input.arch), ); yield* fs.remove(archivePath, { force: true }); - if (input.platform === "win") { + if (input.platform === "win" && hostPlatform !== "win32") { + yield* runCommand( + ChildProcess.make("zip", ["-q", "-r", "-X", path.resolve(archivePath), stem], { + cwd: stageRoot, + }), + "zip", + ); + } else if (input.platform === "win") { // Windows ships bsdtar, which writes zip natively. Name it by path: under // the Git Bash shell CI uses, a bare `tar` is GNU tar, which neither // writes zip nor accepts a drive-letter path. diff --git a/scripts/build-desktop-artifact.test.ts b/scripts/build-desktop-artifact.test.ts index 77cf9ba3a26e9..7bf7fbe173e0b 100644 --- a/scripts/build-desktop-artifact.test.ts +++ b/scripts/build-desktop-artifact.test.ts @@ -83,6 +83,7 @@ import { LinuxBrowserSecretHostError, stageBrowserSecret, validateWindowsPackagedPayload, + verifyPackagedBundleIsSelfContained, WindowsPrimaryNativeProbeError, WindowsDesktopBuildPrerequisitesMissingError, WindowsPackagedPayloadValidationError, @@ -2063,21 +2064,44 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => { it.effect("rejects a sidecar whose extracted server bundle cannot resolve", () => Effect.scoped( Effect.gen(function* () { + const path = yield* Path.Path; const fixture = yield* makeWindowsPayloadFixture({ copyUnpackedNatives: true, serverEntrySource: 'import "t3code-deliberately-missing-package";\n', }); - const error = yield* validateWindowsPackagedPayload({ - stageDistDir: fixture.stageDistDir, - appExecutableName: fixture.appExecutableName, - targetArch: "x64", - appVersion: WINDOWS_PAYLOAD_FIXTURE_VERSION, + const error = yield* verifyPackagedBundleIsSelfContained({ + asarPath: path.join(fixture.packagedAppDir, "resources", WINDOWS_SERVER_ASAR_RESOURCE), + verbose: false, }).pipe(Effect.flip); assert.instanceOf(error, BundleNotSelfContainedError); assert.include(error.output, "t3code-deliberately-missing-package"); }), - ).pipe(Effect.provideService(HostProcess.Platform, "linux")), + ), + ); + + it.effect("leaves the bundle self-check of a cross-built payload to the verify job", () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* makeWindowsPayloadFixture({ + copyUnpackedNatives: true, + serverEntrySource: 'import "t3code-deliberately-missing-package";\n', + }); + yield* validateWindowsPackagedPayload({ + stageDistDir: fixture.stageDistDir, + appExecutableName: fixture.appExecutableName, + targetArch: "arm64", + appVersion: WINDOWS_PAYLOAD_FIXTURE_VERSION, + }); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + Layer.succeed(HostProcess.Platform, "linux"), + Layer.succeed(HostProcess.Architecture, "x64"), + ), + ), + ), ); it.effect("preserves both Linux icon resize failures with structural context", () => { @@ -2438,6 +2462,40 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => { }).pipe(Effect.provide(ConfigProvider.layer(ConfigProvider.fromEnv({ env: {} })))), ); + it.effect("signs Windows builds with signtool on Windows and the jsign hook elsewhere", () => + Effect.gen(function* () { + const signedWinConfig = (hostPlatform: string) => + createBuildConfig("win", "nsis", "1.2.3", true, false, undefined, undefined).pipe( + Effect.map((config) => config.win as Record), + Effect.provideService(HostProcess.Platform, hostPlatform), + ); + + const onWindows = yield* signedWinConfig("win32"); + assert.equal((onWindows.azureSignOptions as Record).publisherName, "T3"); + assert.notProperty(onWindows, "signtoolOptions"); + + const onLinux = yield* signedWinConfig("linux"); + assert.notProperty(onLinux, "azureSignOptions"); + const signtoolOptions = onLinux.signtoolOptions as Record; + assert.match(signtoolOptions.sign as string, /scripts[\\/]lib[\\/]windows-jsign\.ts$/); + assert.deepStrictEqual(signtoolOptions.signingHashAlgorithms, ["sha256"]); + assert.deepStrictEqual(signtoolOptions.publisherName, ["T3"]); + }).pipe( + Effect.provide( + ConfigProvider.layer( + ConfigProvider.fromEnv({ + env: { + AZURE_TRUSTED_SIGNING_PUBLISHER_NAME: "T3", + AZURE_TRUSTED_SIGNING_ENDPOINT: "https://eus.codesigning.azure.net/", + AZURE_TRUSTED_SIGNING_CERTIFICATE_PROFILE_NAME: "profile", + AZURE_TRUSTED_SIGNING_ACCOUNT_NAME: "account", + }, + }), + ), + ), + ), + ); + it("stages the resource monitor as an external executable resource", () => { assert.deepStrictEqual(DESKTOP_EXTRA_RESOURCES, [ { diff --git a/scripts/build-desktop-artifact.ts b/scripts/build-desktop-artifact.ts index 66f05377c323a..d2cff47263455 100644 --- a/scripts/build-desktop-artifact.ts +++ b/scripts/build-desktop-artifact.ts @@ -2247,7 +2247,7 @@ export const copyDirectoryPreservingSymlinks = Effect.fn("copyDirectoryPreservin }, ); -const verifyPackagedBundleIsSelfContained = Effect.fn("verifyPackagedBundleIsSelfContained")( +export const verifyPackagedBundleIsSelfContained = Effect.fn("verifyPackagedBundleIsSelfContained")( function* (input: { readonly asarPath: string; readonly verbose: boolean }) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -3017,8 +3017,16 @@ export const createBuildConfig = Effect.fn("createBuildConfig")(function* ( // packaged executable with Electron's stock icon. signAndEditExecutable: true, }; - if (signed) { + if (signed && (yield* HostProcess.Platform) === "win32") { winConfig.azureSignOptions = yield* AzureTrustedSigningOptionsConfig; + } else if (signed) { + const path = yield* Path.Path; + const { publisherName } = yield* AzureTrustedSigningOptionsConfig; + winConfig.signtoolOptions = { + sign: yield* path.fromFileUrl(new URL("./lib/windows-jsign.ts", import.meta.url)), + signingHashAlgorithms: ["sha256"], + publisherName: [publisherName], + }; } buildConfig.win = winConfig; } @@ -3345,6 +3353,32 @@ export const verifyWindowsPrimaryFffNativeLoad = Effect.fn( ); }); +/** + * The fff probe needs a host of the target's platform and architecture; the + * bundle self-check needs a Windows host. A cross build defers both to + * scripts/verify-windows-desktop-artifact.ts. + */ +export const verifyWindowsPackagedRuntime = Effect.fn( + "desktopArtifact.verifyWindowsPackagedRuntime", +)(function* (input: { + readonly packagedAppDir: string; + readonly appExecutableName: string; + readonly targetArch: typeof BuildArch.Type; + readonly verbose: boolean; +}) { + const path = yield* Path.Path; + const asarPath = path.join(input.packagedAppDir, "resources", WINDOWS_SERVER_ASAR_RESOURCE); + yield* verifyWindowsPrimaryFffNativeLoad({ ...input, asarPath }); + const hostPlatform = yield* HostProcess.Platform; + if (hostPlatform !== "win32") { + yield* Effect.log( + `[desktop-artifact] Skipped the bundle self-check on a ${hostPlatform} host.`, + ); + return; + } + yield* verifyPackagedBundleIsSelfContained({ asarPath, verbose: input.verbose }); +}); + export const validateWindowsPackagedPayload = Effect.fn( "desktopArtifact.validateWindowsPackagedPayload", )(function* (input: { @@ -3558,19 +3592,13 @@ export const validateWindowsPackagedPayload = Effect.fn( }); } - yield* verifyWindowsPrimaryFffNativeLoad({ + yield* verifyWindowsPackagedRuntime({ packagedAppDir, - asarPath, appExecutableName: input.appExecutableName, targetArch: input.targetArch, verbose: input.verbose ?? false, }); - yield* verifyPackagedBundleIsSelfContained({ - asarPath, - verbose: input.verbose ?? false, - }); - yield* Effect.log( `[desktop-artifact] Validated Windows payload (${String(fileCount)} files, ${String(unpackedFiles.length)} sidecar natives).`, ); @@ -3993,6 +4021,13 @@ const buildDesktopArtifactUnlocked = Effect.fn("buildDesktopArtifactUnlocked")(f // takes seconds on a many-core runner instead of about two minutes. buildEnv.XZ_DEFAULTS = "-T0"; } + if (options.platform === "win") { + // Windows signs only through Azure Trusted Signing. Off Windows, + // electron-builder's signtool manager would read CSC_LINK (the macOS + // certificate) as a Windows certificate. + delete buildEnv.CSC_LINK; + delete buildEnv.CSC_KEY_PASSWORD; + } if (!options.signed) { buildEnv.CSC_IDENTITY_AUTO_DISCOVERY = "false"; delete buildEnv.CSC_LINK; diff --git a/scripts/build-npm-launcher.ts b/scripts/build-npm-launcher.ts index e12186e90abbf..0f226a943f5e3 100644 --- a/scripts/build-npm-launcher.ts +++ b/scripts/build-npm-launcher.ts @@ -26,7 +26,7 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; import { fromJsonStringPretty } from "@t3tools/shared/schemaJson"; import serverPackageJson from "../apps/server/package.json" with { type: "json" }; -import { windowsSystemTar } from "./build-cli-archive.ts"; +import { windowsSystemTar } from "./lib/windows-tar.ts"; export const NPM_LAUNCHER_PACKAGE_NAME = "t3"; diff --git a/scripts/lib/windows-jsign.ts b/scripts/lib/windows-jsign.ts new file mode 100644 index 0000000000000..9b3d1fcfcfe60 --- /dev/null +++ b/scripts/lib/windows-jsign.ts @@ -0,0 +1,65 @@ +// Azure Trusted Signing from a non-Windows host. The Windows-only path is +// electron-builder's azureSignOptions (PowerShell + signtool.exe); this one is +// electron-builder's `win.signtoolOptions.sign` hook and the CLI archive's +// signer. jsign reads AZURE_TENANT_ID, AZURE_CLIENT_ID and AZURE_CLIENT_SECRET +// itself when no --storepass is given. + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Config from "effect/Config"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + +export class JsignFailedError extends Schema.TaggedError()("JsignFailedError", { + file: Schema.String, + exitCode: Schema.Number, +}) { + override get message() { + return `jsign exited with code ${String(this.exitCode)} signing ${this.file}.`; + } +} + +export const signWithJsign = Effect.fn("signWithJsign")(function* (file: string) { + const { jsign, endpoint, account, profile } = yield* Config.all({ + jsign: Config.String("T3CODE_JSIGN").pipe(Config.withDefault("jsign")), + endpoint: Config.String("AZURE_TRUSTED_SIGNING_ENDPOINT"), + account: Config.String("AZURE_TRUSTED_SIGNING_ACCOUNT_NAME"), + profile: Config.String("AZURE_TRUSTED_SIGNING_CERTIFICATE_PROFILE_NAME"), + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn( + ChildProcess.make( + jsign, + [ + "--storetype", + "TRUSTEDSIGNING", + "--keystore", + new URL(endpoint).host, + "--alias", + `${account}/${profile}`, + "--alg", + "SHA-256", + "--tsaurl", + "http://timestamp.acs.microsoft.com", + "--tsmode", + "RFC3161", + // jsign appends a nested signature by default; signtool replaces. + "--replace", + file, + ], + { stdout: "inherit", stderr: "inherit" }, + ), + ); + const exitCode = Number(yield* child.exitCode); + if (exitCode !== 0) { + return yield* new JsignFailedError({ file, exitCode }); + } +}); + +export default function sign(configuration: { readonly path: string }): Promise { + return signWithJsign(configuration.path).pipe( + Effect.scoped, + Effect.provide(NodeServices.layer), + Effect.runPromise, + ); +} diff --git a/scripts/lib/windows-tar.ts b/scripts/lib/windows-tar.ts new file mode 100644 index 0000000000000..8d2417f90d022 --- /dev/null +++ b/scripts/lib/windows-tar.ts @@ -0,0 +1,4 @@ +export function windowsSystemTar(): string { + const systemRoot = process.env.SystemRoot ?? process.env.windir ?? "C:\\Windows"; + return `${systemRoot}\\System32\\tar.exe`; +} diff --git a/scripts/package.json b/scripts/package.json index 23a3c194cecd2..29029cef9d3e3 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -21,6 +21,7 @@ "@effect/vitest": "catalog:", "@types/plist": "catalog:", "@types/pngjs": "6.0.5", + "esbuild": "0.28.2", "typescript": "catalog:", "typescript-legacy": "npm:typescript@~6.0.3", "vite-plus": "catalog:" diff --git a/scripts/smoke-cli-archive.ts b/scripts/smoke-cli-archive.ts index dd545b9671751..7c994153469bc 100644 --- a/scripts/smoke-cli-archive.ts +++ b/scripts/smoke-cli-archive.ts @@ -16,7 +16,7 @@ import { ChildProcess, ChildProcessSpawner } from "effect/process"; import * as NetService from "@t3tools/shared/Net"; import * as HostProcess from "@t3tools/shared/HostProcess"; -import { windowsSystemTar } from "./build-cli-archive.ts"; +import { windowsSystemTar } from "./lib/windows-tar.ts"; import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http"; export class CliArchiveSmokeError extends Schema.TaggedError()( diff --git a/scripts/verify-windows-desktop-artifact.ts b/scripts/verify-windows-desktop-artifact.ts new file mode 100644 index 0000000000000..2867a9389ed67 --- /dev/null +++ b/scripts/verify-windows-desktop-artifact.ts @@ -0,0 +1,48 @@ +#!/usr/bin/env node + +// Runs the native-load probes a cross-built Windows desktop artifact skipped, +// on a host of the target's architecture. + +import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Logger from "effect/Logger"; +import { Command, Flag } from "effect/cli"; + +import { + resolveDesktopProductName, + verifyWindowsPackagedRuntime, +} from "./build-desktop-artifact.ts"; + +const verifyWindowsDesktopArtifactCli = Command.make("verify-windows-desktop-artifact", { + appDir: Flag.String("app-dir").pipe( + Flag.withDescription("The electron-builder `win--unpacked` directory."), + ), + arch: Flag.Literals("arch", ["arm64", "x64"] as const), + buildVersion: Flag.String("build-version").pipe( + Flag.withDescription("The artifact's build version."), + ), + verbose: Flag.Boolean("verbose").pipe(Flag.withDefault(false)), +}).pipe( + Command.withDescription("Verify a Windows desktop artifact's packaged natives load."), + Command.withHandler(({ appDir, arch, buildVersion, verbose }) => + Effect.gen(function* () { + yield* verifyWindowsPackagedRuntime({ + packagedAppDir: appDir, + appExecutableName: `${resolveDesktopProductName(buildVersion)}.exe`, + targetArch: arch, + verbose, + }); + yield* Effect.log(`[verify-windows-desktop-artifact] ${appDir} passed.`); + }), + ), +); + +// No import.meta.main guard: CI bundles this file into one module together with +// build-desktop-artifact.ts, whose own guard is defined false in that bundle. +Command.run(verifyWindowsDesktopArtifactCli, { version: "0.0.0" }).pipe( + Effect.scoped, + Effect.provide(Layer.mergeAll(Logger.layer([Logger.consolePretty()]), NodeServices.layer)), + NodeRuntime.runMain, +); From 4c48118da2fa45c5cbd10a87b8fc6db003cad907 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 20 Oct 2026 14:00:24 -0700 Subject: [PATCH 09/17] ci(release): Windows arm64 verify no longer runs the crashing ARM PowerShell (#22494) Co-authored-by: Claude Opus 5.5 --- .github/workflows/release-desktop.yml | 60 +++++++++++++++++---------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/.github/workflows/release-desktop.yml b/.github/workflows/release-desktop.yml index 36738a03cf9d3..cbd61e0604ef4 100644 --- a/.github/workflows/release-desktop.yml +++ b/.github/workflows/release-desktop.yml @@ -239,7 +239,7 @@ jobs: if: runner.os != 'Windows' && inputs.platform == 'win' && steps.resource_monitor_cache.outputs.cache-hit != 'true' uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2 with: - tool: cargo-xwin + tool: cargo-xwin@0.23.1 # Built here rather than inside the desktop build so the CLI archive, # which bundles the same binary, can be built and uploaded first. @@ -743,27 +743,45 @@ jobs: run: node verify/smoke-cli-archive.mjs --archive release-cli/* --expect-version "${{ inputs.version }}" # cargo-xwin links this binary on Linux, and nothing else runs it before users do. + # Node rather than pwsh: PowerShell on Windows ARM crashes intermittently + # (https://github.com/dotnet/runtime/issues/131636), which is why this + # target stopped building on Windows. - name: Smoke-test resource monitor - shell: pwsh + shell: bash + env: + MONITOR: verify/${{ inputs.platform }}-${{ inputs.arch }}-unpacked/resources/resource-monitor/t3-resource-monitor.exe + EXPECTED_ARCH: ${{ inputs.arch == 'arm64' && 'aarch64' || 'x86_64' }} run: | - $startInfo = [System.Diagnostics.ProcessStartInfo]::new( - "verify\${{ inputs.platform }}-${{ inputs.arch }}-unpacked\resources\resource-monitor\t3-resource-monitor.exe") - $startInfo.UseShellExecute = $false - $startInfo.RedirectStandardInput = $true - $startInfo.RedirectStandardOutput = $true - $monitor = [System.Diagnostics.Process]::Start($startInfo) - $hello = $monitor.StandardOutput.ReadLine() | ConvertFrom-Json - if ($hello.type -ne "hello" -or $hello.arch -ne "${{ inputs.arch == 'arm64' && 'aarch64' || 'x86_64' }}") { - throw "Unexpected hello: $($hello | ConvertTo-Json -Compress)" + node --input-type=module <<'EOF' + import { spawn } from "node:child_process"; + const monitor = spawn(process.env.MONITOR, { stdio: ["pipe", "pipe", "inherit"] }); + monitor.stdout.setEncoding("utf8"); + let output = ""; + const exited = new Promise((resolve) => monitor.on("close", resolve)); + const hello = await new Promise((resolve, reject) => { + monitor.on("error", reject); + monitor.on("close", () => reject(new Error(`Resource monitor exited before hello: ${output}`))); + monitor.stdout.on("data", (chunk) => { + output += chunk; + const end = output.indexOf("\n"); + if (end !== -1) resolve(JSON.parse(output.slice(0, end))); + }); + }); + if (hello.type !== "hello" || hello.arch !== process.env.EXPECTED_ARCH) { + throw new Error(`Unexpected hello: ${JSON.stringify(hello)}`); } - $monitor.StandardInput.WriteLine((@{ version = $hello.version; type = "configure"; rootPid = $PID; sampleIntervalMs = 1000 } | ConvertTo-Json -Compress)) - $monitor.StandardInput.WriteLine((@{ version = $hello.version; type = "sampleNow"; requestId = "verify" } | ConvertTo-Json -Compress)) - $monitor.StandardInput.Close() - $output = $monitor.StandardOutput.ReadToEnd() - $monitor.WaitForExit() - $sample = $output -split "`n" | Where-Object { $_.Trim() } | ForEach-Object { $_ | ConvertFrom-Json } | - Where-Object { $_.requestId -eq "verify" } - if ($monitor.ExitCode -ne 0 -or $sample.type -ne "snapshot" -or -not ($sample.processes | Where-Object { $_.pid -eq $PID })) { - throw "Resource monitor exited $($monitor.ExitCode) with: $output" + const send = (message) => monitor.stdin.write(`${JSON.stringify({ version: hello.version, ...message })}\n`); + send({ type: "configure", rootPid: process.pid, sampleIntervalMs: 1000 }); + send({ type: "sampleNow", requestId: "verify" }); + monitor.stdin.end(); + const exitCode = await exited; + const sample = output + .split("\n") + .filter((line) => line.trim()) + .map((line) => JSON.parse(line)) + .find((message) => message.requestId === "verify"); + if (exitCode !== 0 || sample?.type !== "snapshot" || !sample.processes.some((p) => p.pid === process.pid)) { + throw new Error(`Resource monitor exited ${exitCode} with: ${output}`); } - "Resource monitor sampled $($sample.scannedProcessCount) processes." + console.log(`Resource monitor sampled ${sample.scannedProcessCount} processes.`); + EOF From ca140c435f38612656dd655fc42e2f78efbfb4c9 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 20 Oct 2026 14:09:42 -0700 Subject: [PATCH 10/17] refactor(source-control): GitCafe creates repositories and pull requests over REST (#22495) Co-authored-by: Claude Opus 5.5 --- .../GitCafeSourceControlProvider.test.ts | 170 ++++++++++++------ .../server/GitCafeSourceControlProvider.ts | 122 ++++++------- 2 files changed, 170 insertions(+), 122 deletions(-) diff --git a/packages/source-control-gitcafe/src/server/GitCafeSourceControlProvider.test.ts b/packages/source-control-gitcafe/src/server/GitCafeSourceControlProvider.test.ts index 20f8ab3941f99..266d63079e5f6 100644 --- a/packages/source-control-gitcafe/src/server/GitCafeSourceControlProvider.test.ts +++ b/packages/source-control-gitcafe/src/server/GitCafeSourceControlProvider.test.ts @@ -2,6 +2,7 @@ import { SourceControlProviderKind } from "@t3tools/contracts"; import { assert, describe, it } from "@effect/vitest"; import { GitCommandError } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; @@ -18,6 +19,7 @@ import * as GitCafeCredentials from "./GitCafeCredentials.ts"; import * as GitCafeSourceControlProvider from "./GitCafeSourceControlProvider.ts"; const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); +const decodeJson = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown)); const authAnswer = (input: { stdout?: unknown; stderr?: unknown; exitCode?: number }) => ({ stdout: input.stdout === undefined ? "" : encodeJson(input.stdout), @@ -29,6 +31,36 @@ const failure = (code: string, status: number | null) => ({ error: { code, status, message: `Failure ${code}` }, }); +/** Most calls touch no file; one that does fails the test. */ +const NoFiles = FileSystem.layerNoop({}); + +/** A provider on git.cafe whose checkout has no remote and which may not run `cafe`. */ +const layerFor = ( + client: HttpClient.HttpClient, + options: { readonly files?: Layer.Layer } = {}, +) => + Layer.merge(GitCafeApi.layer, options.files ?? NoFiles).pipe( + Layer.provideMerge(GitCafeCredentials.layer), + Layer.provideMerge( + TestSourceControlHost.layer({ + git: { + resolvePrimaryRemoteName: () => + Effect.fail( + new GitCommandError({ + operation: "resolvePrimaryRemoteName", + command: "git remote", + cwd: "/repo", + detail: "No git remote is configured.", + }), + ), + }, + }), + ), + Layer.provide(Layer.succeed(HostProcess.Environment, { CAFE_TOKEN: "env-token" })), + Layer.provide(Layer.succeed(HostProcess.WorkingDirectory, "/server")), + Layer.provide(Layer.succeed(HttpClient.HttpClient, client)), + ); + describe("GitCafeSourceControlProvider", () => { it("reads a signed-in account and keeps transient failures apart from sign-out", () => { const { parseAuth } = GitCafeSourceControlProvider; @@ -94,7 +126,7 @@ describe("GitCafeSourceControlProvider", () => { ]); }).pipe( Effect.provide( - GitCafeApi.layer.pipe( + Layer.merge(GitCafeApi.layer, NoFiles).pipe( Layer.provideMerge(GitCafeCredentials.layer), Layer.provideMerge(TestSourceControlHost.layer()), Layer.provide( @@ -140,7 +172,7 @@ describe("GitCafeSourceControlProvider", () => { assert.deepStrictEqual(urls, ["https://git.cafe/api/repos/team/project"]); }).pipe( Effect.provide( - GitCafeApi.layer.pipe( + Layer.merge(GitCafeApi.layer, NoFiles).pipe( Layer.provideMerge(GitCafeCredentials.layer), Layer.provideMerge( TestSourceControlHost.layer({ @@ -222,7 +254,7 @@ describe("GitCafeSourceControlProvider", () => { assert.deepStrictEqual(remotes, [expected]); }).pipe( Effect.provide( - GitCafeApi.layer.pipe( + Layer.merge(GitCafeApi.layer, NoFiles).pipe( Layer.provideMerge(GitCafeCredentials.layer), Layer.provideMerge( TestSourceControlHost.layer({ @@ -246,22 +278,19 @@ describe("GitCafeSourceControlProvider", () => { ); }); - it.effect("waits for a created repository's admission before handing out its URLs", () => { - const urls: Array = []; - const states = ["running", "complete"]; + it.effect("creates a repository over REST and waits for its admission", () => { + const sent: Array<{ method: string; url: string; body: unknown }> = []; const client = HttpClient.make((request) => { - urls.push(request.url); + const body = + request.body._tag === "Uint8Array" + ? decodeJson(new TextDecoder().decode(request.body.body)) + : undefined; + sent.push({ method: request.method, url: request.url, body }); + const state = request.method === "POST" ? "running" : "complete"; return Effect.succeed( HttpClientResponse.fromWeb( request, - new Response( - encodeJson({ - repoId: "repo_1", - state: states.shift(), - owner: "team", - name: "project", - }), - ), + new Response(encodeJson({ repoId: "repo_1", state, owner: "team", name: "project" })), ), ); }); @@ -276,51 +305,76 @@ describe("GitCafeSourceControlProvider", () => { url: "https://git.cafe/team/project", sshUrl: "ssh@git.cafe:team/project.git", }); - assert.deepStrictEqual(urls, [ - "https://git.cafe/api/orgs/team/admissions/repo_1", - "https://git.cafe/api/orgs/team/admissions/repo_1", - ]); - }).pipe( - Effect.provide( - GitCafeApi.layer.pipe( - Layer.provideMerge(GitCafeCredentials.layer), - Layer.provideMerge( - TestSourceControlHost.layer({ - process: { - run: () => - Effect.succeed( - TestSourceControlHost.processOutput( - encodeJson({ - schemaVersion: 1, - data: { - resource: { - repoId: "repo_1", - state: "running", - owner: "team", - name: "project", - }, - }, - }), - ), - ), - }, - git: { - resolvePrimaryRemoteName: () => - Effect.fail( - new GitCommandError({ - operation: "resolvePrimaryRemoteName", - command: "git remote", - cwd: "/repo", - detail: "No git remote is configured.", - }), - ), - }, - }), + assert.deepStrictEqual( + sent.map((request) => `${request.method} ${request.url}`), + [ + "POST https://git.cafe/api/orgs/team/repos", + "GET https://git.cafe/api/orgs/team/admissions/repo_1", + ], + ); + const { requestKey, ...body } = sent[0]?.body as { requestKey: string }; + assert.match(requestKey, /^[0-9a-f-]{36}$/u); + assert.deepStrictEqual(body, { + name: "project", + visibility: "private", + defaultBranch: "main", + }); + }).pipe(Effect.provide(layerFor(client))); + }); + + it.effect("opens a pull request from a fork over REST with the body file's text", () => { + const sent: Array<{ method: string; url: string; body: unknown }> = []; + const client = HttpClient.make((request) => { + const body = + request.body._tag === "Uint8Array" + ? decodeJson(new TextDecoder().decode(request.body.body)) + : undefined; + sent.push({ method: request.method, url: request.url, body }); + return Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response( + encodeJson( + request.method === "POST" + ? { number: 3 } + : { id: "repo_fork", name: "project", defaultBranch: "main" }, + ), ), - Layer.provide(Layer.succeed(HostProcess.Environment, { CAFE_TOKEN: "env-token" })), - Layer.provide(Layer.succeed(HostProcess.WorkingDirectory, "/server")), - Layer.provide(Layer.succeed(HttpClient.HttpClient, client)), ), + ); + }); + return Effect.gen(function* () { + const provider = yield* GitCafeSourceControlProvider.make; + yield* provider.createChangeRequest({ + cwd: "/repo", + target: { repository: "team/project", refName: "main" }, + baseRefName: "main", + headSelector: "fork:feature", + title: "Add a thing", + bodyFile: "/tmp/body.md", + }); + assert.deepStrictEqual( + sent.map((request) => `${request.method} ${request.url}`), + [ + "GET https://git.cafe/api/repos/fork/project", + "POST https://git.cafe/api/repos/team/project/pulls", + ], + ); + assert.deepStrictEqual(sent[1]?.body, { + repo: "repo_fork", + source: "feature", + target: "main", + title: "Add a thing", + description: "Body", + }); + }).pipe( + Effect.provide( + layerFor(client, { + files: FileSystem.layerNoop({ + readFileString: (path) => + path === "/tmp/body.md" ? Effect.succeed("Body") : Effect.die(path), + }), + }), ), ); }); diff --git a/packages/source-control-gitcafe/src/server/GitCafeSourceControlProvider.ts b/packages/source-control-gitcafe/src/server/GitCafeSourceControlProvider.ts index 57304f2d4ad82..fe624532f74b3 100644 --- a/packages/source-control-gitcafe/src/server/GitCafeSourceControlProvider.ts +++ b/packages/source-control-gitcafe/src/server/GitCafeSourceControlProvider.ts @@ -1,5 +1,6 @@ import { SourceControlProviderKind } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Option from "effect/Option"; import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; @@ -16,6 +17,7 @@ import { type SourceControlCliDiscoverySpec, type SourceControlManagedCliDiscoverySpec, } from "@t3tools/source-control-core/server/discovery"; +import { randomUuidV4 } from "@t3tools/provider-core/server/randomUuid"; import * as SourceControlHost from "@t3tools/source-control-core/server/SourceControlHost"; import * as SourceControlProvider from "@t3tools/source-control-core/server/SourceControlProvider"; @@ -146,10 +148,8 @@ const RepositoryAdmission = Schema.Struct({ owner: TrimmedNonEmptyString, name: TrimmedNonEmptyString, }); -const CreatedRepository = Schema.Struct({ - data: Schema.Struct({ resource: RepositoryAdmission }), -}); -const decodeCreatedRepository = Schema.decodeEffect(Schema.fromJsonString(CreatedRepository)); +const RepositoryId = Schema.Struct({ id: TrimmedNonEmptyString }); +const CreatedPull = Schema.Struct({ number: PositiveInt }); const encodeBranches = Schema.encodeSync(Schema.fromJsonString(Schema.Array(Schema.String))); interface Target { @@ -171,7 +171,7 @@ const pullUrlPattern = /^https:\/\/([^/]+)\/([^/]+\/[^/]+)\/pulls\/(\d+)\/?(?:[? export const make = Effect.gen(function* () { const api = yield* GitCafeApi.GitCafeApi; const host = yield* SourceControlHost.SourceControlHost; - const credentials = yield* GitCafeCredentials.GitCafeCredentials; + const fileSystem = yield* FileSystem.FileSystem; const error = (operation: string, cwd: string, detail: string, cause?: unknown) => new SourceControlProviderError({ @@ -197,33 +197,6 @@ export const make = Effect.gen(function* () { ), ), ); - const cafe = (operation: string, cwd: string, target: Target, args: ReadonlyArray) => - credentials - .cliEnv(target.host) - .pipe( - Effect.flatMap((env) => - host.process.run({ - operation: `GitCafeSourceControlProvider.${operation}`, - command: "cafe", - cwd, - args: [...cliArgs(target.host), ...args], - env: { CAFE_OUTPUT: "json", ...env }, - timeoutMs: 30_000, - }), - ), - ) - .pipe( - Effect.mapError((cause) => - error( - operation, - cwd, - (cause._tag === "VcsProcessExitError" && cliFailure(cause.detail)?.message) || - "GitCafe CLI command failed.", - cause, - ), - ), - ); - const resolveTarget = Effect.fn("GitCafeSourceControlProvider.resolveTarget")(function* ( operation: string, input: { @@ -371,21 +344,42 @@ export const make = Effect.gen(function* () { ? `${source.owner}/${target.repository.split("/")[1]}` : target.repository); const branch = SourceControlProvider.sourceBranch(input); - yield* cafe("createChangeRequest", input.cwd, target, [ - "pr", - "create", - "--json", - "--repo", - target.repository, - "--head", - sourceRepository === target.repository ? branch : `${sourceRepository}:${branch}`, - "--base", - input.target?.refName ?? input.baseRefName, - "--title", - input.title, - "--body-file", - input.bodyFile, - ]); + // GitCafe names the head by its repository's id, so the head repository is looked up first. + const head = yield* read( + "createChangeRequest", + input.cwd, + { host: target.host, path: `/repos/${sourceRepository}` }, + RepositoryId, + ); + const description = yield* fileSystem + .readFileString(input.bodyFile) + .pipe( + Effect.mapError((cause) => + error( + "createChangeRequest", + input.cwd, + "Could not read the pull request description.", + cause, + ), + ), + ); + yield* read( + "createChangeRequest", + input.cwd, + { + host: target.host, + method: "POST", + path: `/repos/${target.repository}/pulls`, + body: { + repo: head.id, + source: branch, + target: input.target?.refName ?? input.baseRefName, + title: input.title, + ...(description.trim().length === 0 ? {} : { description }), + }, + }, + CreatedPull, + ); }, ), getRepositoryCloneUrls: Effect.fn("GitCafeSourceControlProvider.getRepositoryCloneUrls")( @@ -403,25 +397,25 @@ export const make = Effect.gen(function* () { createRepository: Effect.fn("GitCafeSourceControlProvider.createRepository")(function* (input) { const target = yield* resolveTarget("createRepository", input); const [owner = "", name = ""] = target.repository.split("/"); - const output = yield* cafe("createRepository", input.cwd, target, [ - "repo", - "create", - name, - "--org", - owner, - "--visibility", - input.visibility, - "--json", - ]); - const created = yield* decodeCreatedRepository(output.stdout).pipe( - Effect.mapError((cause) => - error("createRepository", input.cwd, "GitCafe returned an unreadable response.", cause), - ), + const pending = yield* read( + "createRepository", + input.cwd, + { + host: target.host, + method: "POST", + path: `/orgs/${owner}/repos`, + body: { + name, + requestKey: yield* randomUuidV4, + visibility: input.visibility, + defaultBranch: "main", + }, + }, + RepositoryAdmission, ); - // GitCafe admits a repository asynchronously: `cafe repo create` answers while admission is - // still running, so wait for it to settle the way `cafe repo create --clone` does. + // GitCafe admits a repository asynchronously and answers while admission is still running, + // so wait for it to settle the way `cafe repo create --clone` does. const settled = (state: string) => state === "complete" || state === "blocked"; - const pending = created.data.resource; const resource = settled(pending.state) ? pending : yield* read( From 5bcc90e1116e396a86cb1802fd26361457c9548a Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 20 Oct 2026 14:47:19 -0700 Subject: [PATCH 11/17] test(web): fix release browser test failures (#22508) --- .../chat/ComposerBannerStack.browser.test.tsx | 14 ++++++++++++-- .../diffs/StyledDiffCodeView.browser.test.tsx | 3 ++- .../PullRequestDetailPanel.browser.test.tsx | 3 ++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/apps/web/src/components/chat/ComposerBannerStack.browser.test.tsx b/apps/web/src/components/chat/ComposerBannerStack.browser.test.tsx index 01a1ed13fdcb0..757f3db20c029 100644 --- a/apps/web/src/components/chat/ComposerBannerStack.browser.test.tsx +++ b/apps/web/src/components/chat/ComposerBannerStack.browser.test.tsx @@ -77,14 +77,19 @@ it("keeps only the usage limit attached while an active goal waits in the collap const limit = page.getByText("Usage limit reached", { exact: true }); const goal = page.getByText("Goal active", { exact: true }); - expect(visibleBanners()).toHaveLength(1); + // A retained pointer can expand the stack as it mounts; start outside the hover target. + await userEvent.hover(limit); + await expect.poll(visibleBanners).toHaveLength(1); await expect.element(limit).toBeVisible(); await expect.element(goal).not.toBeVisible(); expect(container.querySelector('[data-slot="composer-banner-attachment"]')?.textContent).toBe( "Usage limit reached", ); - await page.getByRole("button", { name: "Show other notices" }).click(); + const peek = page.getByRole("button", { name: "Show other notices" }); + // Hover opens the stack and hides the peek before a pointer click can finish. + await act(() => peek.element().focus()); + await userEvent.keyboard("{Enter}"); await expect.element(goal).toBeVisible(); await expect.element(limit).toBeVisible(); await expect.poll(visibleBanners).toHaveLength(2); @@ -93,6 +98,11 @@ it("keeps only the usage limit attached while an active goal waits in the collap await expect.element(goal).not.toBeVisible(); await expect.element(limit).toBeVisible(); expect(visibleBanners()).toHaveLength(1); + + await userEvent.hover(peek); + await expect.element(goal).toBeVisible(); + await expect.element(limit).toBeVisible(); + await expect.poll(visibleBanners).toHaveLength(2); }); it("reveals the full running command only while its title is clipped", async () => { diff --git a/apps/web/src/components/diffs/StyledDiffCodeView.browser.test.tsx b/apps/web/src/components/diffs/StyledDiffCodeView.browser.test.tsx index 8fbd98a358601..71dd1e89ee492 100644 --- a/apps/web/src/components/diffs/StyledDiffCodeView.browser.test.tsx +++ b/apps/web/src/components/diffs/StyledDiffCodeView.browser.test.tsx @@ -200,7 +200,8 @@ describe("code-view worker lifecycle", () => { }; } - it("starts on demand, keeps quick reopens warm, and expires the last closed pool", async () => { + // Two cold starts can each use Pierre's 10-second initialization budget. + it("starts workers on demand and expires idle pools", { timeout: 30_000 }, async () => { setup(); await using workers = await workerEnvironment(); const view = await workers.render(renderViews(0)); diff --git a/apps/web/src/components/pullRequest/PullRequestDetailPanel.browser.test.tsx b/apps/web/src/components/pullRequest/PullRequestDetailPanel.browser.test.tsx index 323297b7309c1..24e2a1955ee2e 100644 --- a/apps/web/src/components/pullRequest/PullRequestDetailPanel.browser.test.tsx +++ b/apps/web/src/components/pullRequest/PullRequestDetailPanel.browser.test.tsx @@ -398,6 +398,7 @@ describe.each([ ); }); + // The first Code-tab action starts real diff workers as well as driving the panel. it.each([ "Resolve conflicts", "Ask a question", @@ -405,7 +406,7 @@ describe.each([ "Fix findings in this thread", "Fix check", "Add to agent", - ])("%s writes to the correct composer", async (action) => { + ])("%s writes to the correct composer", { timeout: 30_000 }, async (action) => { setup(); // Cover both a host without inline comments and an account that cannot write them. currentDetail = From c8667fd5f4cd635b82b70d9e082c98c06eb78dcc Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 20 Oct 2026 15:07:43 -0700 Subject: [PATCH 12/17] fix(mobile): connecting an environment no longer crashes the header (#22521) Co-authored-by: Claude Opus 5.5 --- .../src/native/NativeHeaderToolbar.test.tsx | 51 ++++++++++++++++++- apps/mobile/src/native/StackHeader.tsx | 32 +++++++++++- 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/apps/mobile/src/native/NativeHeaderToolbar.test.tsx b/apps/mobile/src/native/NativeHeaderToolbar.test.tsx index 41d8777f6dfb6..9dba4f4a4f882 100644 --- a/apps/mobile/src/native/NativeHeaderToolbar.test.tsx +++ b/apps/mobile/src/native/NativeHeaderToolbar.test.tsx @@ -1,5 +1,5 @@ import type { NativeStackHeaderItem } from "@react-navigation/native-stack"; -import { act } from "react"; +import { act, createContext, type ReactElement } from "react"; import { createRoot, type Root } from "react-dom/client"; import { afterEach, expect, it, vi } from "vite-plus/test"; @@ -56,6 +56,43 @@ function renderDocument(path: string, placement: Placement) { act(() => root!.render()); } +// Like React Navigation's NavigationStateContext, whose default value throws when +// read. React restores a context's default outside render, so layout effects see it. +const ContainerStateContext = createContext({ + get getIsInitial(): never { + throw new Error("Couldn't find a navigation context."); + }, +}); +const containerState = {}; + +function ConnectingToolbar(props: { readonly label: string }) { + return ( + + + + + + ); +} + +function renderConnecting(label: string) { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + root ??= createRoot(document.createElement("div")); + act(() => + root!.render( + + + , + ), + ); +} + +function renderedCustomLabel() { + const item = harness.renderedItems[0]; + if (item?.type !== "custom") throw new Error("Expected the custom header element"); + return (item.element as ReactElement<{ "aria-label": string }>).props["aria-label"]; +} + function openRenderedDocument() { const menu = harness.renderedItems[0]; if (menu?.type !== "menu") throw new Error("Expected the native file menu"); @@ -98,3 +135,15 @@ it.each(["left", "right", "bottom"] as const)( expect(harness.openedDocuments).toEqual(["first/guide.pdf"]); }, ); + +it("applies a custom header element rendered inside a navigation container", () => { + renderConnecting("Connecting"); + expect(renderedCustomLabel()).toBe("Connecting"); + + const applied = harness.setOptionsCalls; + renderConnecting("Connecting"); + expect(harness.setOptionsCalls).toBe(applied); + + renderConnecting("Reconnecting"); + expect(renderedCustomLabel()).toBe("Reconnecting"); +}); diff --git a/apps/mobile/src/native/StackHeader.tsx b/apps/mobile/src/native/StackHeader.tsx index 8240fbc74df44..698a345035cca 100644 --- a/apps/mobile/src/native/StackHeader.tsx +++ b/apps/mobile/src/native/StackHeader.tsx @@ -4,7 +4,7 @@ import type { NativeStackNavigationOptions, NativeStackNavigationProp, } from "@react-navigation/native-stack"; -import { use, useEffect, useLayoutEffect, useMemo, useRef } from "react"; +import { isValidElement, use, useEffect, useLayoutEffect, useMemo, useRef } from "react"; import { NativeWorkspaceTabActivityContext } from "./v5-workspace-context"; import type { ColorValue } from "react-native"; @@ -55,6 +55,29 @@ function normalizeScreenOptions( return normalized as NativeStackNavigationOptions; } +// React tags elements, component types and contexts with `$$typeof`; their other +// fields are React internals. In development an element's `_owner` is a fiber that +// reaches the whole tree, including contexts that hold their default value outside +// render, and React Navigation's NavigationStateContext default throws when read. +function isReactObject(value: object): boolean { + return "$$typeof" in value; +} + +const identityIds = new WeakMap(); +let nextIdentityId = 0; + +function identitySignature(value: unknown): string { + if (value === null || (typeof value !== "object" && typeof value !== "function")) { + return String(value); + } + let id = identityIds.get(value); + if (id === undefined) { + id = nextIdentityId++; + identityIds.set(value, id); + } + return `identity:${id}`; +} + function optionsSignature(value: unknown, seen = new WeakSet()): string { if (value === null) return "null"; switch (typeof value) { @@ -75,6 +98,11 @@ function optionsSignature(value: unknown, seen = new WeakSet()): string return `bigint:${String(value)}`; case "object": { const object = value as object; + if (isValidElement(object)) { + // Elements are recreated on every render, so compare what they render. + return `element:${identitySignature(object.type)}:${String(object.key)}:${optionsSignature(object.props, seen)}`; + } + if (isReactObject(object)) return identitySignature(object); if (seen.has(object)) return "[circular]"; seen.add(object); if (Array.isArray(value)) { @@ -121,7 +149,7 @@ function stabilizeOptionFunctions( ); } if (value !== null && typeof value === "object") { - if (seen.has(value) || "current" in value) return value; + if (seen.has(value) || "current" in value || isReactObject(value)) return value; seen.add(value); return Object.fromEntries( Object.entries(value as Record).map(([key, entry]) => [ From 4af32312db6eebad26b158e65a0f2771abecd543 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 20 Oct 2026 15:07:44 -0700 Subject: [PATCH 13/17] fix(mobile): folded Back clears the detail route before the next selection (#22526) Co-authored-by: Claude Opus 5.5 --- .../createWorkspaceStackNavigator.ios.tsx | 54 +++++++-- .../native/workspace-stack-projection.test.ts | 107 ++++++++++++++++++ .../src/native/workspace-stack-projection.ts | 47 ++++++++ 3 files changed, 196 insertions(+), 12 deletions(-) diff --git a/apps/mobile/src/features/layout/createWorkspaceStackNavigator.ios.tsx b/apps/mobile/src/features/layout/createWorkspaceStackNavigator.ios.tsx index 2b23ad865f042..c4c0946fffd02 100644 --- a/apps/mobile/src/features/layout/createWorkspaceStackNavigator.ios.tsx +++ b/apps/mobile/src/features/layout/createWorkspaceStackNavigator.ios.tsx @@ -44,9 +44,11 @@ import { import { V5StackHeader } from "../../native/V5StackHeader.ios"; import type { AppNativeStackNavigationOptions } from "../../native/StackHeader"; import { + collapsedSplitTransition, nativeWorkspacePopAction, projectWorkspaceStack, partitionStackPresentations, + type CollapsedSplitColumn, } from "../../native/workspace-stack-projection"; import { NativePrimaryColumnContext, @@ -65,6 +67,7 @@ function ColumnScreen(props: { readonly selectedPullRequestKey?: string | null; readonly sidebarFilter?: NativeStackHeaderItemMenu; readonly onDismiss: (routeKey: string) => void; + readonly onAppearance: (type: "willAppear" | "didAppear" | "willDisappear", key: string) => void; readonly navigation: ViewProps["navigation"]; }) { const { descriptor } = props; @@ -85,15 +88,18 @@ function ColumnScreen(props: { onDismiss={(event) => { if (event.nativeEvent.isNativeDismiss) props.onDismiss(key); }} - onWillAppear={() => - props.navigation.emit({ type: "transitionStart", target: key, data: { closing: false } }) - } - onDidAppear={() => - props.navigation.emit({ type: "transitionEnd", target: key, data: { closing: false } }) - } - onWillDisappear={() => - props.navigation.emit({ type: "transitionStart", target: key, data: { closing: true } }) - } + onWillAppear={() => { + props.onAppearance("willAppear", key); + props.navigation.emit({ type: "transitionStart", target: key, data: { closing: false } }); + }} + onDidAppear={() => { + props.onAppearance("didAppear", key); + props.navigation.emit({ type: "transitionEnd", target: key, data: { closing: false } }); + }} + onWillDisappear={() => { + props.onAppearance("willDisappear", key); + props.navigation.emit({ type: "transitionStart", target: key, data: { closing: true } }); + }} onDidDisappear={() => props.navigation.emit({ type: "transitionEnd", target: key, data: { closing: true } }) } @@ -130,7 +136,8 @@ function WorkspaceColumns( const hostRef = useRef(null); const activeDetailKey = props.detail.at(-1)?.key; const compactColumn = activeDetailKey ? "secondary" : "primary"; - const shownCompactColumn = useRef(compactColumn); + const collapsed = useRef(!layout.usesSplitView); + const collapsedColumn = useRef({ visible: compactColumn, leaving: null }); const threadParams = props.detail.findLast((route) => route.name === "Thread")?.params; const pullRequestParams = props.detail.findLast( (route) => @@ -169,10 +176,24 @@ function WorkspaceColumns( }, [props.navigation], ); + const followCollapsedSplit = ( + type: "willAppear" | "didAppear" | "willDisappear", + key: string, + ) => { + if (!collapsed.current || layout.usesSplitView) return; + const transition = collapsedSplitTransition( + collapsedColumn.current, + { type, key }, + props.navigation.getState(), + props.primary.key, + ); + collapsedColumn.current = transition.column; + if (transition.pop) props.navigation.dispatch(transition.pop); + }; useEffect(() => { - if (layout.usesSplitView || shownCompactColumn.current === compactColumn) return; - shownCompactColumn.current = compactColumn; + if (layout.usesSplitView || collapsedColumn.current.visible === compactColumn) return; + collapsedColumn.current = { ...collapsedColumn.current, visible: compactColumn }; hostRef.current?.show(compactColumn); }, [compactColumn, layout.usesSplitView]); @@ -185,6 +206,13 @@ function WorkspaceColumns( testID="adaptive-workspace-layout" preferredSplitBehavior="tile" topColumnForCollapsing={compactColumn} + onCollapse={() => { + collapsed.current = true; + collapsedColumn.current = { visible: compactColumn, leaving: null }; + }} + onExpand={() => { + collapsed.current = false; + }} preferredDisplayMode={ panes.primarySidebarVisible || (!layout.usesSplitView && !activeDetailKey) ? "oneBesideSecondary" @@ -211,6 +239,7 @@ function WorkspaceColumns( selectedThreadKey={selectedThreadKey} selectedPullRequestKey={selectedPullRequestKey} onDismiss={handleNativeDismiss} + onAppearance={followCollapsedSplit} navigation={props.navigation} /> @@ -252,6 +281,7 @@ function WorkspaceColumns( key={route.key} descriptor={descriptor} onDismiss={handleNativeDismiss} + onAppearance={followCollapsedSplit} navigation={props.navigation} sidebarFilter={sidebarFilter} /> diff --git a/apps/mobile/src/native/workspace-stack-projection.test.ts b/apps/mobile/src/native/workspace-stack-projection.test.ts index e813c48faa6b8..99901652473d2 100644 --- a/apps/mobile/src/native/workspace-stack-projection.test.ts +++ b/apps/mobile/src/native/workspace-stack-projection.test.ts @@ -3,12 +3,14 @@ import type { ParamListBase, StackNavigationState } from "@react-navigation/nati import { describe, expect, it } from "vite-plus/test"; import { + collapsedSplitTransition, nativeStackPopAction, nativeWorkspacePopAction, nativeWorkspacePopCount, projectWorkspaceStack, partitionStackPresentations, reconcileStackScreens, + type CollapsedSplitColumn, } from "./workspace-stack-projection"; const requireNavigation = NodeModule.createRequire( @@ -185,6 +187,111 @@ describe("native card pop handoff", () => { }); }); +describe("collapsed split Back", () => { + const routerOptions = { + routeNames: history([home]).routeNames, + routeParamList: {}, + routeGetIdList: {}, + }; + type Appearance = "willAppear" | "didAppear" | "willDisappear"; + // UIKit's appearance callbacks for one collapsed transition, against the router at each step. + function replay( + steps: ReadonlyArray]>, + ) { + let column: CollapsedSplitColumn = { visible: "secondary", leaving: null }; + let pop: ReturnType = null; + for (const [type, key, state] of steps) { + const transition = collapsedSplitTransition(column, { type, key }, state, home.key); + column = transition.column; + pop ??= transition.pop; + } + return { visible: column.visible, pop }; + } + + it("returns the router to the list so the next selection opens a new detail", () => { + const state = history([home, thread]); + const back = replay([ + ["willDisappear", thread.key, state], + ["willAppear", home.key, state], + ["didAppear", home.key, state], + ]); + expect(back.visible).toBe("primary"); + const router = StackRouter({}); + const listed = router.getStateForAction(state, back.pop!, routerOptions); + expect(listed?.routes).toEqual([home]); + if (!listed || listed.stale !== false) throw new Error("Expected a rehydrated router history"); + const select = { + type: "NAVIGATE", + payload: { name: "Thread", params: { threadId: "next" } }, + } as const; + expect(router.getStateForAction(listed, select, routerOptions)?.routes[1]?.key).not.toBe( + thread.key, + ); + // Left on the hidden detail, the selection would only rename it and never show the column. + expect(router.getStateForAction(state, select, routerOptions)?.routes[1]?.key).toBe(thread.key); + }); + + it("clears every detail while keeping a presentation opened during the Back animation", () => { + const leaving = history([home, thread, files]); + const presented = history([home, thread, files, settings]); + const { pop } = replay([ + ["willDisappear", files.key, leaving], + ["didAppear", home.key, presented], + ]); + expect(StackRouter({}).getStateForAction(presented, pop!, routerOptions)?.routes).toEqual([ + home, + settings, + ]); + }); + + it("keeps the detail when it appears again before the list finishes appearing", () => { + const state = history([home, thread]); + const cancelledSwipe = replay([ + ["willDisappear", thread.key, state], + ["willAppear", home.key, state], + ["willAppear", thread.key, state], + ["didAppear", thread.key, state], + ]); + expect(cancelledSwipe).toEqual({ visible: "secondary", pop: null }); + expect( + replay([ + ["willDisappear", thread.key, state], + ["willAppear", thread.key, state], + ["didAppear", home.key, state], + ]).pop, + ).toBeNull(); + }); + + it("does not mistake JS navigation or a cold link for native Back", () => { + const pushed = history([home, thread, files]); + expect( + replay([ + ["willDisappear", thread.key, pushed], + ["didAppear", files.key, pushed], + ]), + ).toEqual({ visible: "secondary", pop: null }); + const presented = history([home, thread, settings]); + expect( + replay([ + ["willDisappear", thread.key, presented], + ["didAppear", home.key, presented], + ]).pop, + ).toBeNull(); + const popped = history([home]); + expect( + replay([ + ["willDisappear", thread.key, popped], + ["didAppear", home.key, popped], + ]).pop, + ).toBeNull(); + // A linked thread whose list appears first still needs its detail column shown. + expect(replay([["didAppear", home.key, history([home, thread])]])).toEqual({ + visible: "primary", + pop: null, + }); + }); +}); + describe("v5 stack handoff", () => { it("releases a completed native pop when the router acknowledges it", () => { const completed = new Set([files.key]); diff --git a/apps/mobile/src/native/workspace-stack-projection.ts b/apps/mobile/src/native/workspace-stack-projection.ts index 2d6afae419eef..71ebec0ed9c2f 100644 --- a/apps/mobile/src/native/workspace-stack-projection.ts +++ b/apps/mobile/src/native/workspace-stack-projection.ts @@ -70,6 +70,53 @@ export function nativeStackPopAction( }; } +/** The column UIKit shows while a split is collapsed, and the active detail it began removing. */ +export interface CollapsedSplitColumn { + readonly visible: "primary" | "secondary"; + readonly leaving: string | null; +} + +/** + * A collapsed split nests the detail column's navigation controller in the list column's. Back + * from the detail root pops that controller whole, so no screen leaves its own stack or reports a + * dismissal: the list appearing after the active detail began leaving is the only sign of it. + */ +export function collapsedSplitTransition( + column: CollapsedSplitColumn, + event: { readonly type: "willAppear" | "didAppear" | "willDisappear"; readonly key: string }, + state: Pick, + primaryKey: string, +): { + readonly column: CollapsedSplitColumn; + readonly pop: ReturnType; +} { + switch (event.type) { + case "willDisappear": + // JS pushes, pops, and presentations have already moved the router past this screen. + return { + column: + state.index > 0 && state.routes[state.index]?.key === event.key + ? { ...column, leaving: event.key } + : column, + pop: null, + }; + case "willAppear": + // A cancelled swipe, or an expansion that keeps the detail on screen. + return { + column: column.leaving === event.key ? { ...column, leaving: null } : column, + pop: null, + }; + case "didAppear": + return { + column: { visible: event.key === primaryKey ? "primary" : "secondary", leaving: null }, + pop: + event.key === primaryKey && column.leaving + ? nativeStackPopAction(state, column.leaving, primaryKey) + : null, + }; + } +} + export function partitionStackPresentations( routes: readonly T[], isModal: (route: T) => boolean, From fe27e8e3dc12dcbb108e9a28fd9b8cb65f34ae92 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 20 Oct 2026 15:08:35 -0700 Subject: [PATCH 14/17] ci(release): release tests stop timing out and finish in about 5 minutes (#22524) Co-authored-by: Claude Opus 5.5 --- .github/workflows/ci.yml | 1 + .github/workflows/release.yml | 59 ++++++++++++++++++++++++++++++----- 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45b3339732610..d29d970c75ff1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -252,6 +252,7 @@ jobs: # The web suite is the largest outside the server. In the shared Test job it # started last and ran alone for about 40s after everything else finished. + # release.yml mirrors this test, test_web, and test_server split. test_web: name: Test Web needs: changes diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index df8a719f9e791..5d387de16b836 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -256,7 +256,7 @@ jobs: run: vp run typecheck # Keep tests on their own runners, using the same package split and server - # shards as CI, so the release check job does not spend its budget on tests. + # shards as CI (test, test_web, test_server in ci.yml). Keep the two in sync. test: name: Release tests needs: [preflight] @@ -286,23 +286,67 @@ jobs: sudo sed -i 's|http://|https://|g' /etc/apt/blacksmith-ubuntu-mirrors.txt /etc/apt/sources.list.d/ubuntu.sources sudo apt-get update && sudo apt-get install -y libsecret-1-dev pkg-config build-essential - # The web package's browser project runs React tests in headless Chromium. + - name: Test + run: vp run --parallel --concurrency-limit 4 --filter '!t3' --filter '!@t3tools/monorepo' --filter '!@t3tools/web' test + + # Web browser tests time out when they share a runner with the other package + # suites: Chromium gets too little CPU and diff workers miss their 10s start budget. + test_web: + name: Release web tests + needs: [preflight] + if: ${{ !failure() && !cancelled() && needs.preflight.result == 'success' }} + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + ref: ${{ needs.preflight.outputs.ref }} + sparse-checkout: | + /* + !/.repos/ + sparse-checkout-cone-mode: false + + - name: Setup dependencies + uses: ./.github/actions/setup-ci-deps + + - name: Resolve Playwright version + id: playwright_version + run: echo "version=$(node -p "require('./apps/web/node_modules/playwright/package.json').version")" >> "$GITHUB_OUTPUT" + + - name: Cache Chromium + id: chromium_cache + uses: actions/cache@v6 + with: + path: ~/.cache/ms-playwright + key: playwright-chromium-shell-${{ runner.os }}-${{ runner.arch }}-${{ steps.playwright_version.outputs.version }} + - name: Install Chromium for web browser tests - run: node apps/web/node_modules/playwright/cli.js install --with-deps --only-shell chromium + if: steps.chromium_cache.outputs.cache-hit != 'true' + run: node apps/web/node_modules/playwright/cli.js install --only-shell chromium + + - name: Install missing Chromium system dependencies + run: | + browser=$(find "$HOME/.cache/ms-playwright" -name chrome-headless-shell -type f -print -quit) + test -n "$browser" + if ldd "$browser" | grep '=> not found'; then + node apps/web/node_modules/playwright/cli.js install-deps chromium + fi - name: Test - run: vp run --parallel --concurrency-limit 4 --filter '!t3' --filter '!@t3tools/monorepo' test + run: vp run --filter @t3tools/web test + # Server test files run one at a time, so more small runners beat fewer big ones. test_server: name: Release server tests ${{ matrix.shard }} needs: [preflight] if: ${{ !failure() && !cancelled() && needs.preflight.result == 'success' }} - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 strategy: fail-fast: false matrix: - shard: [1, 2, 3] + shard: [1, 2, 3, 4, 5, 6, 7, 8] steps: - name: Checkout uses: actions/checkout@v6 @@ -693,6 +737,7 @@ jobs: relay_public_config, quality, test, + test_web, test_server, desktop_mac_arm64, desktop_mac_x64, @@ -701,7 +746,7 @@ jobs: desktop_win_x64, desktop_win_arm64, ] - if: ${{ !failure() && !cancelled() && needs.preflight.result == 'success' && needs.relay_public_config.result == 'success' && needs.quality.result == 'success' && needs.test.result == 'success' && needs.test_server.result == 'success' && needs.desktop_mac_arm64.result == 'success' && needs.desktop_mac_x64.result == 'success' && needs.desktop_linux_x64.result == 'success' && needs.desktop_linux_arm64.result == 'success' && needs.desktop_win_x64.result == 'success' && needs.desktop_win_arm64.result == 'success' }} + if: ${{ !failure() && !cancelled() && needs.preflight.result == 'success' && needs.relay_public_config.result == 'success' && needs.quality.result == 'success' && needs.test.result == 'success' && needs.test_web.result == 'success' && needs.test_server.result == 'success' && needs.desktop_mac_arm64.result == 'success' && needs.desktop_mac_x64.result == 'success' && needs.desktop_linux_x64.result == 'success' && needs.desktop_linux_arm64.result == 'success' && needs.desktop_win_x64.result == 'success' && needs.desktop_win_arm64.result == 'success' }} runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 30 permissions: From 5d27ecd79c33a642efcdf259e89e3eba5a560247 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 20 Oct 2026 15:28:08 -0700 Subject: [PATCH 15/17] test(server): Claude steer test no longer hangs for 120s under CI load (#22533) Co-authored-by: Claude Opus 5.5 --- .../Adapters/ClaudeAdapterV2.test.ts | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 138528263fbac..9c681f605ae41 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -17500,6 +17500,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { const FINISHED_TOOL_USE_ID = "toolu-steer-finished"; const calls: Array = []; let beforeAnswer: Effect.Effect = Effect.void; + const lateAnswer = yield* Deferred.make(); const harness = yield* makeWakeHarnessWithOptions({ backgroundTasks: (toolUseId) => Effect.sync(() => calls.push(toolUseId)).pipe( @@ -17517,7 +17518,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { : backgrounding === "never answers" ? Effect.never : backgrounding === "answers after the steer went out" - ? Effect.sleep("6 seconds").pipe(Effect.as(true)) + ? Deferred.await(lateAnswer).pipe(Effect.as(true)) : Effect.succeed(true), ), ), @@ -17670,16 +17671,29 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); } + // The steer arms its 5-second backgrounding timeout after the request + // starts on another fiber, so one TestClock.adjust can land before that + // timer exists and leave the steer waiting forever. Tick until it ends. + const awaitSteerTimeout = (fiber: Fiber.Fiber) => + Effect.gen(function* () { + for (let tick = 0; tick < 100 && fiber.pollUnsafe() === undefined; tick++) { + yield* TestClock.adjust("1 second"); + yield* Effect.yieldNow; + } + if (fiber.pollUnsafe() === undefined) { + return yield* Effect.die("Timed out waiting for the steer."); + } + return yield* Fiber.await(fiber); + }); const steering = yield* Effect.forkScoped(steer); - if ( + const timesOut = backgrounding === "never answers" || - backgrounding === "answers after the steer went out" - ) { + backgrounding === "answers after the steer went out"; + if (timesOut) { yield* awaitUntil(() => calls.length === 1, "background request"); assert.equal(harness.offeredMessages.length, offered); - yield* TestClock.adjust("5 seconds"); } - const steered = yield* Fiber.await(steering); + const steered = yield* timesOut ? awaitSteerTimeout(steering) : Fiber.await(steering); assert.deepEqual(calls, [FOREGROUND_TOOL_USE_ID]); if (backgrounding === "answers after the turn ended") { @@ -17693,17 +17707,13 @@ describe("ClaudeAdapterV2 background wake turns", () => { if (backgrounding === "never answers") { for (let repeat = 1; repeat <= 2; repeat++) { const again = yield* Effect.forkScoped(steer); - for (let tick = 0; tick < 100 && again.pollUnsafe() === undefined; tick++) { - yield* TestClock.adjust("1 second"); - yield* Effect.yieldNow; - } - assert.isTrue(Exit.isSuccess(yield* Fiber.await(again))); + assert.isTrue(Exit.isSuccess(yield* awaitSteerTimeout(again))); assert.equal(harness.offeredMessages.length, offered + 1 + repeat); } assert.deepEqual(calls, [FOREGROUND_TOOL_USE_ID]); } if (backgrounding === "answers after the steer went out") { - yield* TestClock.adjust("1 second"); + yield* Deferred.succeed(lateAnswer, undefined); } else if ( backgrounding !== "succeeds" && backgrounding !== "answers after the subagent ran again" From f9dd25914dbd5f061f2685965bcbe2db8cfb912e Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 20 Oct 2026 16:08:57 -0700 Subject: [PATCH 16/17] test: remove the timing races behind recurring CI and release failures (#22523) Co-authored-by: Claude Opus 5.5 --- .../transferBudgetV2.integration.test.ts | 12 ++- apps/server/scripts/acp-mock-agent.ts | 17 ++-- .../Adapters/ClaudeAdapterV2.test.ts | 37 ++++----- .../Adapters/ClaudeAdapterV2.ts | 15 +++- .../chat/ComposerBannerStack.browser.test.tsx | 4 +- .../chat/MessagesTimeline.browser.test.tsx | 9 +++ .../diffs/StyledDiffCodeView.browser.test.tsx | 78 ++++++++++++------- .../PullRequestDetailPanel.browser.test.tsx | 11 ++- ...rojectFaviconPickerDialog.browser.test.tsx | 7 +- .../ProjectSettingsPanel.browser.test.tsx | 35 +++++---- apps/web/test/render.tsx | 13 +++- apps/web/vite.config.ts | 7 ++ .../src/server/OpenCodeServerLedger.test.ts | 18 ++++- 13 files changed, 176 insertions(+), 87 deletions(-) diff --git a/apps/server/integration/transferBudgetV2.integration.test.ts b/apps/server/integration/transferBudgetV2.integration.test.ts index 0ec8b1c37b291..b4dd358c17b1a 100644 --- a/apps/server/integration/transferBudgetV2.integration.test.ts +++ b/apps/server/integration/transferBudgetV2.integration.test.ts @@ -1,5 +1,6 @@ // @effect-diagnostics nodeBuiltinImport:off - Measures real local HTTP/WebSocket sockets. import * as NodeHttp from "node:http"; +import * as NodeUtil from "node:util"; import { assert, it } from "@effect/vitest"; import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"; import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform"; @@ -299,9 +300,16 @@ it.live( const events = turnEvents(provider, TRANSFER_HISTORY_TURN_COUNT, true); const stored = yield* sink.write({ events }); const last = stored.at(-1)!; - const terminal = (item: StreamItem) => + const threadTerminal = (item: StreamItem) => "sequence" in item && item.sequence >= last.sequence; - for (const queue of [tq, sq, ttq, ssq]) yield* collectUntil(queue, terminal); + // Shell items carry the shell read when their batch is projected, and unchanged + // shells are skipped, so a turn split across batches may never yield an item at + // the last sequence. Wait for the final shell instead. + const finalShell = yield* projection.getThreadShell(THREAD_ID); + const shellTerminal = (item: StreamItem) => + item.kind === "thread.updated" && NodeUtil.isDeepStrictEqual(item.thread, finalShell); + for (const queue of [tq, ttq]) yield* collectUntil(queue, threadTerminal); + for (const queue of [sq, ssq]) yield* collectUntil(queue, shellTerminal); const measuredTurnWebSocket = difference(thread.recorder.totals(), threadBefore); const measuredTurnShellWebSocket = difference(shell.recorder.totals(), shellBefore); const measuredTurnSecondClientWebSocket = difference( diff --git a/apps/server/scripts/acp-mock-agent.ts b/apps/server/scripts/acp-mock-agent.ts index c2d29e98047e8..ae06aba0355dd 100644 --- a/apps/server/scripts/acp-mock-agent.ts +++ b/apps/server/scripts/acp-mock-agent.ts @@ -1383,6 +1383,7 @@ const program = Effect.gen(function* () { (emitRunningCommandThenHangOnFirstPrompt && promptCount === 1) ) { const toolCallId = "tool-call-running-1"; + let runningCommandPublished: Promise = Promise.resolve(); if (runningCommandPidPath !== undefined) { const command = runningCommandIgnoresTerm ? 'trap "" TERM; bash -c \'trap "" TERM; while :; do sleep 1; done\' & child=$!; printf "%s %s\n" "$$" "$child" > "$1"; wait "$child"' @@ -1390,19 +1391,24 @@ const program = Effect.gen(function* () { if (runningCommandSeparateSession) { const launcher = [ 'const { spawn } = require("node:child_process");', - "const child = spawn(process.argv[1], process.argv.slice(2), { stdio: 'ignore' });", + "const child = spawn(process.argv[1], process.argv.slice(2), { stdio: ['ignore', 'inherit', 'ignore'] });", "child.once('exit', (code, signal) => process.exitCode = code ?? (signal ? 1 : 0));", ].join(" "); const detachedCommand = runningCommandIgnoresTerm - ? 'trap "" TERM; bash -c \'trap "" TERM; while :; do sleep 1; done\' & child=$!; printf "%s %s %s\n" "$PPID" "$$" "$child" > "$1"; wait "$child"' - : 'sleep 120 & child=$!; printf "%s %s %s\n" "$PPID" "$$" "$child" > "$1"; wait "$child"'; + ? 'trap "" TERM; bash -c \'trap "" TERM; while :; do sleep 1; done\' & child=$!; printf "%s %s %s\n" "$PPID" "$$" "$child" > "$1"; echo published; wait "$child"' + : 'sleep 120 & child=$!; printf "%s %s %s\n" "$PPID" "$$" "$child" > "$1"; echo published; wait "$child"'; // Nested bash publishes "$PPID $$ $child" once it starts. Do not // write the launcher PID alone here: that races with bash and can - // clobber the triple that interrupt tests wait for. + // clobber the triple that interrupt tests wait for. It then prints a + // line, so an agent that exits after launch only exits once the + // triple exists instead of racing a slow launcher start. const detachedLauncher = NodeChildProcess.spawn( process.execPath, ["-e", launcher, "bash", "-c", detachedCommand, "bash", runningCommandPidPath], - { detached: true, stdio: "ignore" }, + { detached: true, stdio: ["ignore", "pipe", "ignore"] }, + ); + runningCommandPublished = new Promise((resolve) => + detachedLauncher.stdout.once("data", () => resolve()), ); detachedLauncher.unref(); } else { @@ -1460,6 +1466,7 @@ const program = Effect.gen(function* () { }, }); if (exitAfterRunningCommandLaunch) { + yield* Effect.promise(() => runningCommandPublished); yield* Effect.sleep("100 millis"); return yield* Effect.sync(() => process.exit(0)); } diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 9c681f605ae41..acdec8cf3d035 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -17499,11 +17499,13 @@ describe("ClaudeAdapterV2 background wake turns", () => { const NESTED_TOOL_USE_ID = "toolu-steer-nested"; const FINISHED_TOOL_USE_ID = "toolu-steer-finished"; const calls: Array = []; + const backgroundRequested = yield* Deferred.make(); let beforeAnswer: Effect.Effect = Effect.void; const lateAnswer = yield* Deferred.make(); const harness = yield* makeWakeHarnessWithOptions({ backgroundTasks: (toolUseId) => Effect.sync(() => calls.push(toolUseId)).pipe( + Effect.andThen(Deferred.succeed(backgroundRequested, undefined)), Effect.andThen( backgrounding === "answers after the turn ended" || backgrounding === "answers after the subagent ran again" @@ -17671,29 +17673,19 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); } - // The steer arms its 5-second backgrounding timeout after the request - // starts on another fiber, so one TestClock.adjust can land before that - // timer exists and leave the steer waiting forever. Tick until it ends. - const awaitSteerTimeout = (fiber: Fiber.Fiber) => - Effect.gen(function* () { - for (let tick = 0; tick < 100 && fiber.pollUnsafe() === undefined; tick++) { - yield* TestClock.adjust("1 second"); - yield* Effect.yieldNow; - } - if (fiber.pollUnsafe() === undefined) { - return yield* Effect.die("Timed out waiting for the steer."); - } - return yield* Fiber.await(fiber); - }); - const steering = yield* Effect.forkScoped(steer); - const timesOut = + // Started at once so the steer reads its backgrounding deadline before + // this test moves the clock. Its message is built with real file I/O, + // so a scheduled start could read the clock after the moves below. + const steering = yield* Effect.forkScoped(steer, { startImmediately: true }); + if ( backgrounding === "never answers" || - backgrounding === "answers after the steer went out"; - if (timesOut) { - yield* awaitUntil(() => calls.length === 1, "background request"); + backgrounding === "answers after the steer went out" + ) { + yield* Deferred.await(backgroundRequested); assert.equal(harness.offeredMessages.length, offered); + yield* TestClock.adjust("5 seconds"); } - const steered = yield* timesOut ? awaitSteerTimeout(steering) : Fiber.await(steering); + const steered = yield* Fiber.await(steering); assert.deepEqual(calls, [FOREGROUND_TOOL_USE_ID]); if (backgrounding === "answers after the turn ended") { @@ -17706,8 +17698,9 @@ describe("ClaudeAdapterV2 background wake turns", () => { assert.equal(harness.offeredMessages.at(-1)?.priority, "now"); if (backgrounding === "never answers") { for (let repeat = 1; repeat <= 2; repeat++) { - const again = yield* Effect.forkScoped(steer); - assert.isTrue(Exit.isSuccess(yield* awaitSteerTimeout(again))); + const again = yield* Effect.forkScoped(steer, { startImmediately: true }); + yield* TestClock.adjust("5 seconds"); + assert.isTrue(Exit.isSuccess(yield* Fiber.await(again))); assert.equal(harness.offeredMessages.length, offered + 1 + repeat); } assert.deepEqual(calls, [FOREGROUND_TOOL_USE_ID]); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 53c71b9eed71a..f1795704844ce 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -97,6 +97,7 @@ import { } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as Crypto from "effect/Crypto"; import * as DateTime from "effect/DateTime"; @@ -304,6 +305,9 @@ const CLAUDE_WAKE_TURN_START_DEADLINE = "2 minutes"; // reply can land after the progress frame that settles it. Its transcript // stays the source of its answer this long after it settles. const WORKFLOW_MEMBER_TRANSCRIPT_FLUSH_MS = 500; +// A steer waits at most this long, from its request, for its foreground +// subagents to move to the background before it is sent. +const BACKGROUND_MOVE_TIMEOUT_MS = 5_000; /** * When the CLI exits it stops the MCP servers it started, but not their children: a plugin's @@ -9684,6 +9688,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( // its result arrives as a task notification like any background agent. const backgroundForegroundSubagents = Effect.fnUntraced(function* ( live: ClaudeLiveQueryContext, + deadline: number, ) { const foreground: Array<{ readonly taskId: string; @@ -9739,11 +9744,17 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( backgroundMovesByToolUseId.set(toolUseId, move); moves.push(move); } - yield* Fiber.joinAll(moves).pipe(Effect.timeoutOption("5 seconds")); + const remaining = deadline - (yield* Clock.currentTimeMillis); + yield* Fiber.joinAll(moves).pipe(Effect.timeoutOption(Math.max(0, remaining))); }); const steerTurn = Effect.fn("ClaudeAdapterV2.steerTurn")( function* (turnInput: ProviderAdapter.ProviderAdapterV2SteerInput) { + // Measured from the request, not from when the moves are forked: + // building the message and the moves can take clock time a timer + // started later would not count, and the steer would wait past it. + const backgroundDeadline = + (yield* Clock.currentTimeMillis) + BACKGROUND_MOVE_TIMEOUT_MS; const existing = yield* Ref.get(queryContext); if (existing === null) { return yield* new ProviderAdapter.ProviderAdapterProtocolError({ @@ -9769,7 +9780,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( fileSystem, skills: yield* userInvocableSkills(currentTurn.input.runtimePolicy.cwd), }); - yield* backgroundForegroundSubagents(existing); + yield* backgroundForegroundSubagents(existing, backgroundDeadline); // The turn can end while the moves are answered. A steer offered // then would start a new turn the orchestrator never asked for, so // the steer fails and the orchestrator starts the follow-up turn. diff --git a/apps/web/src/components/chat/ComposerBannerStack.browser.test.tsx b/apps/web/src/components/chat/ComposerBannerStack.browser.test.tsx index 757f3db20c029..c6551d4fcb7d1 100644 --- a/apps/web/src/components/chat/ComposerBannerStack.browser.test.tsx +++ b/apps/web/src/components/chat/ComposerBannerStack.browser.test.tsx @@ -77,9 +77,7 @@ it("keeps only the usage limit attached while an active goal waits in the collap const limit = page.getByText("Usage limit reached", { exact: true }); const goal = page.getByText("Goal active", { exact: true }); - // A retained pointer can expand the stack as it mounts; start outside the hover target. - await userEvent.hover(limit); - await expect.poll(visibleBanners).toHaveLength(1); + expect(visibleBanners()).toHaveLength(1); await expect.element(limit).toBeVisible(); await expect.element(goal).not.toBeVisible(); expect(container.querySelector('[data-slot="composer-banner-attachment"]')?.textContent).toBe( diff --git a/apps/web/src/components/chat/MessagesTimeline.browser.test.tsx b/apps/web/src/components/chat/MessagesTimeline.browser.test.tsx index 665eb5e16f690..3452282be49c8 100644 --- a/apps/web/src/components/chat/MessagesTimeline.browser.test.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.browser.test.tsx @@ -43,6 +43,15 @@ vi.mock(import("~/state/query"), async (original) => ({ })) as never, })); +// The first Mermaid render loads the library and its flowchart chunks, which took over +// 10 seconds on a busy release runner. Pay that while the file loads, not inside a test. +{ + const { default: mermaid } = await import("mermaid"); + mermaid.initialize({ startOnLoad: false }); + await mermaid.render("mermaid-warmup", "flowchart LR\n A --> B"); + document.getElementById("dmermaid-warmup")?.remove(); +} + const ENVIRONMENT_ID = EnvironmentId.make("environment-local"); const CREATED_AT = "2026-03-17T19:12:28.000Z"; diff --git a/apps/web/src/components/diffs/StyledDiffCodeView.browser.test.tsx b/apps/web/src/components/diffs/StyledDiffCodeView.browser.test.tsx index 71dd1e89ee492..62fd79f6706c1 100644 --- a/apps/web/src/components/diffs/StyledDiffCodeView.browser.test.tsx +++ b/apps/web/src/components/diffs/StyledDiffCodeView.browser.test.tsx @@ -1,4 +1,4 @@ -import { act, render, fakeTimers } from "../../../test/render"; +import { act, render, fakeTimers, type Screen } from "../../../test/render"; import { CodeView, FileDiff, @@ -146,9 +146,16 @@ function renderViews(count: number) { ); } -const codeFile = (name: string) => - document.querySelector(`[data-code-file="${name}"]`)?.textContent ?? ""; -const loadingViews = () => document.querySelectorAll('[role="status"]').length; +// Pierre rejects a pool whose workers take 10s to start. Real workers share the runner with the +// rest of the suite, so outside the test of that limit the test's own timeout is the only deadline. +const UNLIMITED_WORKER_STARTUP_MS = 120_000; + +// Queries stay inside the test's own container: a timed-out test keeps running and leaves its +// views mounted while the next test starts. +const codeFile = (container: HTMLElement, name: string) => + container.querySelector(`[data-code-file="${name}"]`)?.textContent ?? ""; +const loadingViews = (container: HTMLElement) => + container.querySelectorAll('[role="status"]').length; describe("code-view worker lifecycle", () => { function setup() { @@ -165,7 +172,7 @@ describe("code-view worker lifecycle", () => { // Worker startup runs on real time, so only the idle expiry checks use this clock. // Faking startup too would let advancing the idle clock time out a worker startup. const idleClock = () => fakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); - async function workerEnvironment() { + async function workerEnvironment({ startupLimit = UNLIMITED_WORKER_STARTUP_MS } = {}) { await using stack = new AsyncDisposableStack(); // Pins the pool at two workers whatever machine runs the suite. stack.use(vi.spyOn(navigator, "hardwareConcurrency", "get").mockReturnValue(2)); @@ -175,6 +182,9 @@ describe("code-view worker lifecycle", () => { this: WorkerPoolManager, ...args ) { + // The provider builds the pool without this option, so it is set before startup reads it. + const options: { workerInitializationTimeout?: number } = Reflect.get(this, "options"); + options.workerInitializationTimeout = startupLimit; const pending = initialize.apply(this, args); testState.pools.set(this, pending); return pending; @@ -200,12 +210,13 @@ describe("code-view worker lifecycle", () => { }; } - // Two cold starts can each use Pierre's 10-second initialization budget. + // Two cold starts of real workers can outlast the default timeout on a loaded runner. it("starts workers on demand and expires idle pools", { timeout: 30_000 }, async () => { setup(); await using workers = await workerEnvironment(); const view = await workers.render(renderViews(0)); - const button = document.querySelector("button")!; + const { container } = view; + const button = container.querySelector("button")!; expect(testState.workers).toHaveLength(0); expect(testState.pools.size).toBe(0); await act(async () => button.click()); @@ -214,7 +225,9 @@ describe("code-view worker lifecycle", () => { await act(async () => testState.pools.get(pool)); expect(pool.isInitialized()).toBe(true); expect(testState.workers).toHaveLength(2); - await vi.waitFor(() => expect(codeFile("notes.txt")).toContain("Plain text is ready.")); + await vi.waitFor(() => + expect(codeFile(container, "notes.txt")).toContain("Plain text is ready."), + ); expect(testState.requests.filter((request) => request.type === "file")).toHaveLength(0); await view.rerender(renderViews(2)); @@ -223,7 +236,7 @@ describe("code-view worker lifecycle", () => { expect(testState.workers).toHaveLength(2); expect(new Set(testState.renderPools)).toEqual(new Set([pool])); expect(pool.getFileResultCache(files[1]!)).toBeDefined(); - await vi.waitFor(() => expect(codeFile("answer.ts")).toContain("answer")); + await vi.waitFor(() => expect(codeFile(container, "answer.ts")).toContain("answer")); { using _idleClock = idleClock(); @@ -238,7 +251,7 @@ describe("code-view worker lifecycle", () => { await view.rerender(renderViews(1)); expect(testState.pools.size).toBe(1); expect(testState.workers).toHaveLength(2); - expect(loadingViews()).toBe(0); + expect(loadingViews(container)).toBe(0); expect(pool.getFileResultCache(files[1]!)).toBeDefined(); await vi.advanceTimersByTimeAsync(30_000); expect(pool.getStats().totalWorkers).toBe(2); @@ -254,16 +267,18 @@ describe("code-view worker lifecycle", () => { await act(async () => testState.pools.get(reopenedPool)); expect(reopenedPool.isInitialized()).toBe(true); expect(testState.workers).toHaveLength(4); - expect(document.querySelector("button")).toBe(button); + expect(container.querySelector("button")).toBe(button); expect(button.textContent).toBe("Edited draft"); - await vi.waitFor(() => expect(codeFile("notes.txt")).toContain("Plain text is ready.")); + await vi.waitFor(() => + expect(codeFile(container, "notes.txt")).toContain("Plain text is ready."), + ); }); it("renders through Pierre's main-thread fallback when worker creation fails", async () => { setup(); await using workers = await workerEnvironment(); testState.failWorkers = true; - await workers.render(renderViews(1)); + const { container } = await workers.render(renderViews(1)); const pool = [...testState.pools.keys()][0]!; await act(async () => { await expect(testState.pools.get(pool)).rejects.toMatchObject({ _tag: "DiffWorkerError" }); @@ -275,13 +290,16 @@ describe("code-view worker lifecycle", () => { }); expect(pool.isWorkingPool()).toBe(false); expect(testState.workers).toHaveLength(0); - await vi.waitFor(() => expect(codeFile("notes.txt")).toContain("Plain text is ready.")); + await vi.waitFor(() => + expect(codeFile(container, "notes.txt")).toContain("Plain text is ready."), + ); }); it("keeps an initializing pool until its last view closes and ignores late responses", async () => { setup(); using _idleClock = idleClock(); - await using workers = await workerEnvironment(); + // Runs Pierre's own startup limit on the fake clock. + await using workers = await workerEnvironment({ startupLimit: 10_000 }); testState.holdInitialization = true; const held = new Promise((resolve) => { testState.onInitializationHeld = resolve; @@ -292,14 +310,14 @@ describe("code-view worker lifecycle", () => { await held; expect(pool.isInitialized()).toBe(false); expect(testState.pools.size).toBe(1); - expect(loadingViews()).toBe(2); + expect(loadingViews(view.container)).toBe(2); expect(testState.renderPools).toHaveLength(0); await view.rerender(renderViews(1)); expect(pool.getStats().totalWorkers).toBe(2); expect(testState.terminations).toBe(0); await view.rerender(renderViews(0)); - // Pierre gives up on a held startup after 10 seconds; the idle timer then tears the pool down. + // Pierre gives up on the held startup after 10 seconds; the idle timer then tears the pool down. const timedOut = expect(pending).rejects.toThrow("worker initialization timed out"); await vi.advanceTimersByTimeAsync(30_000); await timedOut; @@ -310,7 +328,7 @@ describe("code-view worker lifecycle", () => { expect(testState.workers).toHaveLength(2); expect(testState.terminations).toBe(2); expect(testState.renderPools).toHaveLength(0); - expect(document.querySelectorAll("pre")).toHaveLength(0); + expect(view.container.querySelectorAll("pre")).toHaveLength(0); }); }); @@ -346,7 +364,11 @@ describe("collapsed diff content", () => { testState.failWorkers = false; testState.holdInitialization = false; const pool = new WorkerPoolManager( - { workerFactory: () => new DiffsWorker(), poolSize: 1 }, + { + workerFactory: () => new DiffsWorker(), + poolSize: 1, + workerInitializationTimeout: UNLIMITED_WORKER_STARTUP_MS, + }, { theme: "pierre-dark", langs: ["typescript"], preferredHighlighter: "shiki-wasm" }, ); stack.defer(() => pool.terminate()); @@ -569,7 +591,7 @@ function RevealViews({ } describe("code-view file reveals", () => { - const reveal = () => act(async () => document.querySelector("button")!.click()); + const reveal = (view: Screen) => act(async () => view.container.querySelector("button")!.click()); function setup() { revealCalls.length = 0; @@ -580,7 +602,7 @@ describe("code-view file reveals", () => { await using view = await render( , ); - await reveal(); + await reveal(view); expect(revealCalls.at(-1)?.fileKey).toBe("tree-file"); revealCalls.length = 0; @@ -597,7 +619,7 @@ describe("code-view file reveals", () => { await using view = await render( , ); - await reveal(); + await reveal(view); expect(revealCalls).toEqual([]); await view.rerender(); @@ -612,7 +634,7 @@ describe("code-view file reveals", () => { revealCalls.length = 0; // A click that lands in the same update that hides the viewer waits for it to return. await act(async () => { - document.querySelector("button")!.click(); + view.container.querySelector("button")!.click(); await view.rerender(); }); expect(revealCalls).toEqual([]); @@ -628,7 +650,7 @@ describe("code-view file reveals", () => { await using view = await render( , ); - await reveal(); + await reveal(view); await view.rerender( , ); @@ -636,7 +658,7 @@ describe("code-view file reveals", () => { revealCalls.length = 0; await view.rerender(); - await reveal(); + await reveal(view); await view.rerender(); await view.rerender(); expect(revealCalls).toEqual([]); @@ -644,9 +666,9 @@ describe("code-view file reveals", () => { it("reveals after a collapsed file expands and honors another click on the same file", async () => { setup(); - await using _view = await render(); - await reveal(); - await reveal(); + await using view = await render(); + await reveal(view); + await reveal(view); expect(revealCalls).toEqual([ { scope: "diff", fileKey: "tree-file", mounted: true, collapsed: false }, { scope: "diff", fileKey: "tree-file", mounted: true, collapsed: false }, diff --git a/apps/web/src/components/pullRequest/PullRequestDetailPanel.browser.test.tsx b/apps/web/src/components/pullRequest/PullRequestDetailPanel.browser.test.tsx index 24e2a1955ee2e..27eae0e297669 100644 --- a/apps/web/src/components/pullRequest/PullRequestDetailPanel.browser.test.tsx +++ b/apps/web/src/components/pullRequest/PullRequestDetailPanel.browser.test.tsx @@ -153,6 +153,14 @@ vi.mock(import("./PullRequestSummaryTab"), async (original) => { }) as never, }; }); +// The Code tab waits for the diff worker pool, and a cold pool starts real Shiki workers that +// a loaded runner can keep past Pierre's 10s startup limit. These tests are about the panel, +// so the pool falls back to main-thread highlighting at once. +vi.mock("@pierre/diffs/worker/worker.js?worker", () => ({ + default: function UnavailableDiffsWorker() { + throw new Error("Diff workers are disabled in panel tests"); + }, +})); vi.mock(import("../ui/toast"), async (original) => { const actual = await original(); return { @@ -398,7 +406,6 @@ describe.each([ ); }); - // The first Code-tab action starts real diff workers as well as driving the panel. it.each([ "Resolve conflicts", "Ask a question", @@ -406,7 +413,7 @@ describe.each([ "Fix findings in this thread", "Fix check", "Add to agent", - ])("%s writes to the correct composer", { timeout: 30_000 }, async (action) => { + ])("%s writes to the correct composer", async (action) => { setup(); // Cover both a host without inline comments and an account that cannot write them. currentDetail = diff --git a/apps/web/src/components/settings/ProjectFaviconPickerDialog.browser.test.tsx b/apps/web/src/components/settings/ProjectFaviconPickerDialog.browser.test.tsx index 6d0de80caa482..39aa4a19cc17b 100644 --- a/apps/web/src/components/settings/ProjectFaviconPickerDialog.browser.test.tsx +++ b/apps/web/src/components/settings/ProjectFaviconPickerDialog.browser.test.tsx @@ -129,9 +129,14 @@ const imageUrl = `data:image/svg+xml,${encodeURIComponent( '', )}`; +// The registry drops an unsubscribed atom on its next task. A row calls createUrl while it +// renders but subscribes only after React commits, so on a busy runner a URL published +// in between would be dropped. A server keeps its answer; keepAlive models that. function addFiles(paths: string[]) { state.entries = paths.map((path) => ({ path, kind: "file" })); - for (const path of paths) state.assets.set(path, Atom.make(AsyncResult.initial())); + for (const path of paths) { + state.assets.set(path, Atom.make(AsyncResult.initial()).pipe(Atom.keepAlive)); + } } function fileRow(path: string) { diff --git a/apps/web/src/components/settings/ProjectSettingsPanel.browser.test.tsx b/apps/web/src/components/settings/ProjectSettingsPanel.browser.test.tsx index 5f7137ca5b33e..8d39c065f579a 100644 --- a/apps/web/src/components/settings/ProjectSettingsPanel.browser.test.tsx +++ b/apps/web/src/components/settings/ProjectSettingsPanel.browser.test.tsx @@ -429,35 +429,36 @@ describe("project settings permissions", () => { }); describe("project settings scope", () => { - it("offers project checkout environments while keeping unselected and unknown projects recoverable", async () => { + const environmentMenu = () => page.getByRole("button", { name: /^Environment scope:/ }); + const primaryOption = () => page.getByRole("menuitemradio", { name: /^primary/ }); + const remoteOption = () => page.getByRole("menuitemradio", { name: /^remote/ }); + + it("offers every environment while no project is selected", async () => { setup(); state.projects = state.projects.filter((project) => project.environmentId === primaryId); - const key = projectKey(); - await using screen = await mountSettings({}); - const environmentMenu = () => page.getByRole("button", { name: /^Environment scope:/ }); - const primaryOption = () => page.getByRole("menuitemradio", { name: /^primary/ }); - const remoteOption = () => page.getByRole("menuitemradio", { name: /^remote/ }); + await using _screen = await mountSettings({}); await userEvent.click(environmentMenu()); await expect.element(primaryOption()).toBeVisible(); await expect.element(remoteOption()).toBeVisible(); - await userEvent.keyboard("{Escape}"); + }); + + it("offers only the selected project's checkout environments", async () => { + setup(); + state.projects = state.projects.filter((project) => project.environmentId === primaryId); + await using _screen = await mountSettings({ project: projectKey() }); - await act(async () => - screen.router.navigate({ to: "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/settings/projects", search: () => ({ project: key }) }), - ); await expect.element(page.getByRole("textbox", { name: "Project name" })).toBeVisible(); await userEvent.click(environmentMenu()); await expect.element(primaryOption()).toBeVisible(); await expect.element(remoteOption()).not.toBeInTheDocument(); - await userEvent.keyboard("{Escape}"); + }); + + it("keeps an unknown project recoverable from any environment", async () => { + setup(); + state.projects = state.projects.filter((project) => project.environmentId === primaryId); + await using screen = await mountSettings({ project: "removed-project" }); - await act(async () => - screen.router.navigate({ - to: "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/settings/projects", - search: () => ({ project: "removed-project" }), - }), - ); await userEvent.click(environmentMenu()); await expect.element(primaryOption()).toBeVisible(); await expect.element(remoteOption()).toBeVisible(); diff --git a/apps/web/test/render.tsx b/apps/web/test/render.tsx index 0ad52c0ec2288..05ce93e024b3f 100644 --- a/apps/web/test/render.tsx +++ b/apps/web/test/render.tsx @@ -8,12 +8,19 @@ import { import { act as reactAct, createContext, use, type ReactNode } from "react"; import { createRoot } from "react-dom/client"; import { vi } from "vite-plus/test"; -import { page, utils, type Locator, type LocatorSelectors } from "vite-plus/test/browser"; +import { commands, page, utils, type Locator, type LocatorSelectors } from "vite-plus/test/browser"; declare global { var IS_REACT_ACT_ENVIRONMENT: boolean | undefined; } +declare module "vite-plus/test/browser" { + interface BrowserCommands { + /** Defined in vite.config.ts. Moves Playwright's mouse off the page. */ + resetPointer: () => Promise; + } +} + // React warns about every update outside act() while this flag is on, and a real // browser drives updates from userEvent, timers and network that act() cannot wrap. // So it is on only while act() runs, and nested calls restore the outer value. @@ -97,6 +104,10 @@ export async function render( await unmount(); } finally { container.remove(); + // The real mouse outlives the test, and whatever the next test mounts under it + // starts hovered. Parking it here keeps render() itself free of awaits before + // the first mount, which tests rely on. + await commands.resetPointer(); } }, }; diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index 6e550f6814679..e58708100d6b2 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -108,6 +108,13 @@ const browserTestProject = { provider: playwright(), instances: [{ browser: "chromium" }], screenshotFailures: false, + commands: { + // Playwright's mouse outlives each test, so whatever mounts under its last + // position starts hovered. test/render.tsx parks it off the page after each test. + resetPointer: async ({ page }) => { + await page.mouse.move(-1, -1); + }, + }, }, }, } satisfies TestProjectInlineConfiguration; diff --git a/packages/provider-opencode/src/server/OpenCodeServerLedger.test.ts b/packages/provider-opencode/src/server/OpenCodeServerLedger.test.ts index 040e4ead2b59b..90c72cf491301 100644 --- a/packages/provider-opencode/src/server/OpenCodeServerLedger.test.ts +++ b/packages/provider-opencode/src/server/OpenCodeServerLedger.test.ts @@ -28,9 +28,14 @@ const groupExists = (pgid: number) => { } }; +// A group of zombies can be reaped between a check and the kill, so ESRCH just means it is gone. const killGroup = (pgid: number) => Effect.sync(() => { - if (groupExists(pgid)) process.kill(-pgid, "SIGKILL"); + try { + process.kill(-pgid, "SIGKILL"); + } catch (cause) { + if (!(cause instanceof Error && "code" in cause && cause.code === "ESRCH")) throw cause; + } }); const spawnGroup = (args: ReadonlyArray, script = "sleep 600 & wait") => @@ -38,13 +43,15 @@ const spawnGroup = (args: ReadonlyArray, script = "sleep 600 & wait") => Effect.gen(function* () { const child = NodeChildProcess.spawn("/bin/sh", ["-c", script, ...args], { detached: true, - stdio: "ignore", + stdio: ["ignore", "pipe", "ignore"], }); const exited = yield* Deferred.make(); + const printed = yield* Deferred.make(); child.once("exit", (_code, signal) => Deferred.doneUnsafe(exited, Effect.succeed(signal))); + child.stdout.once("data", () => Deferred.doneUnsafe(printed, Effect.void)); const pid = child.pid; if (pid === undefined) return yield* Effect.die("spawn failed"); - return { pid, exited: Deferred.await(exited) }; + return { pid, exited: Deferred.await(exited), printed: Deferred.await(printed) }; }), ({ pid }) => killGroup(pid), ); @@ -96,11 +103,14 @@ describe.each(observedPlatforms)("OpenCodeServerLedger observing as %s", (platfo const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-opencode-ledger-" }); + // The inner shell prints once it has exec'd. Until then it still carries the wrapper's + // command line, and recording that would make the reaper skip the group as recycled. const wrapper = yield* spawnGroup( SERVE_ARGS, - '/bin/sh -c "sleep 600 & wait" "$0" "$@" & exit 0', + '/bin/sh -c "echo ready; sleep 600 & wait" "$0" "$@" & exit 0', ); yield* wrapper.exited; + yield* wrapper.printed; expect(groupExists(wrapper.pid)).toBe(true); yield* recordFromDeadServer(stateDir, wrapper); From fcb20e49aa0c1c5fc736034687c42e20cd2637b3 Mon Sep 17 00:00:00 2001 From: maria Date: Tue, 20 Oct 2026 23:36:45 +0000 Subject: [PATCH 17/17] feat(web): render Mermaid diagrams with pele (#21472) Co-authored-by: maria-rcks <254055478+maria-rcks@users.noreply.github.com> --- apps/web/package.json | 2 +- apps/web/src/components/ChatMarkdown.tsx | 2 +- .../chat/MermaidDiagram.browser.test.tsx | 66 -- .../src/components/chat/MermaidDiagram.tsx | 379 +++++---- apps/web/src/index.css | 14 + pnpm-lock.yaml | 789 +----------------- pnpm-workspace.yaml | 1 + 7 files changed, 233 insertions(+), 1020 deletions(-) delete mode 100644 apps/web/src/components/chat/MermaidDiagram.browser.test.tsx diff --git a/apps/web/package.json b/apps/web/package.json index dbb889a1bb8ef..17f40f695472b 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -65,8 +65,8 @@ "katex": "^0.16.47", "lucide": "^0.564.0", "lucide-react": "^0.564.0", - "mermaid": "^11.17.2", "morphicons": "^1.7.1", + "pele": "^1.8.0", "react": "catalog:", "react-dom": "catalog:", "react-markdown": "^10.1.0", diff --git a/apps/web/src/components/ChatMarkdown.tsx b/apps/web/src/components/ChatMarkdown.tsx index baf23d2b4cec5..78468b0c85808 100644 --- a/apps/web/src/components/ChatMarkdown.tsx +++ b/apps/web/src/components/ChatMarkdown.tsx @@ -1107,7 +1107,7 @@ function MarkdownMermaidCodeBlock({ } > - + {searching ? (