diff --git a/apps/sim/background/cleanup-stale-executions.test.ts b/apps/sim/background/cleanup-stale-executions.test.ts index d06ed9b9d23..9d529fe6b70 100644 --- a/apps/sim/background/cleanup-stale-executions.test.ts +++ b/apps/sim/background/cleanup-stale-executions.test.ts @@ -80,6 +80,11 @@ function collectSqlParams(fragment: unknown): unknown[] { }) } +/** A swept log as the sweep's update returns it: recent, so it is not marked for the rollup. */ +function sweptRun(id: string) { + return { id, workspaceId: 'workspace-1', startedAt: new Date() } +} + describe('stale execution cleanup deadline grace', () => { beforeEach(() => { resetDbChainMock() @@ -324,9 +329,9 @@ describe('stale execution cleanup deadline grace', () => { })) for (let batch = 0; batch < 10; batch++) { - const workflowBatch = Array.from({ length: 100 }, (_, index) => ({ - id: `workflow-state-${batch}-${index}`, - })) + const workflowBatch = Array.from({ length: 100 }, (_, index) => + sweptRun(`workflow-state-${batch}-${index}`) + ) queueTableRows(workflowExecutionLogs, workflowBatch) dbChainMockFns.returning.mockResolvedValueOnce(workflowBatch) } @@ -396,12 +401,10 @@ describe('stale execution cleanup deadline grace', () => { }) it('preserves committed workflow cleanup counts when a later batch fails', async () => { - const firstBatch = Array.from({ length: 100 }, (_, index) => ({ - id: `execution-${index}`, - })) - const failedBatch = Array.from({ length: 37 }, (_, index) => ({ - id: `failed-execution-${index}`, - })) + const firstBatch = Array.from({ length: 100 }, (_, index) => sweptRun(`execution-${index}`)) + const failedBatch = Array.from({ length: 37 }, (_, index) => + sweptRun(`failed-execution-${index}`) + ) queueTableRows(workflowExecutionLogs, firstBatch) queueTableRows(workflowExecutionLogs, failedBatch) queueTableRows(asyncJobs, [{ id: 'async-job-1' }]) @@ -426,11 +429,11 @@ describe('stale execution cleanup deadline grace', () => { }) it('continues draining when an atomic race updates fewer rows than were selected', async () => { - const firstCandidates = Array.from({ length: 100 }, (_, index) => ({ - id: `execution-${index}`, - })) + const firstCandidates = Array.from({ length: 100 }, (_, index) => + sweptRun(`execution-${index}`) + ) const firstUpdated = firstCandidates.slice(0, 99) - const secondBatch = [{ id: 'execution-100' }] + const secondBatch = [sweptRun('execution-100')] queueTableRows(workflowExecutionLogs, firstCandidates) queueTableRows(workflowExecutionLogs, secondBatch) dbChainMockFns.returning.mockResolvedValueOnce(firstUpdated).mockResolvedValueOnce(secondBatch) diff --git a/apps/sim/background/cleanup-stale-executions.ts b/apps/sim/background/cleanup-stale-executions.ts index 1d2eb7ffb0f..0cc969ca71c 100644 --- a/apps/sim/background/cleanup-stale-executions.ts +++ b/apps/sim/background/cleanup-stale-executions.ts @@ -13,6 +13,7 @@ import { toError } from '@sim/utils/errors' import { task } from '@trigger.dev/sdk' import { and, eq, exists, gt, inArray, isNull, lt, ne, or, sql } from 'drizzle-orm' import { alias } from 'drizzle-orm/pg-core' +import { markFinishedRunsDirty } from '@/lib/billing/activity-rollup/marks' import { JOB_PENDING_RETENTION_HOURS, JOB_RETENTION_HOURS, @@ -287,7 +288,13 @@ export async function runCleanupStaleExecutions() { ) ) ) - .returning({ id: workflowExecutionLogs.id }) + .returning({ + id: workflowExecutionLogs.id, + workspaceId: workflowExecutionLogs.workspaceId, + startedAt: workflowExecutionLogs.startedAt, + }) + // Every batch reuses the sweep's one end stamp, so a late batch can commit behind the scan. + await markFinishedRunsDirty(tx, updatedExecutions) return { candidates, updatedExecutions } }) diff --git a/apps/sim/lib/billing/activity-rollup/marks.ts b/apps/sim/lib/billing/activity-rollup/marks.ts index 51d491b0c29..3b2948bb722 100644 --- a/apps/sim/lib/billing/activity-rollup/marks.ts +++ b/apps/sim/lib/billing/activity-rollup/marks.ts @@ -8,7 +8,13 @@ import { inflightBackoffMs, recomputeCell, } from '@/lib/billing/activity-rollup/cells' -import { floorHour, msInterval, utcTimestamp } from '@/lib/billing/activity-rollup/source' +import { + ACTIVITY_ROLLUP_SETTLE_MS, + floorHour, + HOUR_MS, + msInterval, + utcTimestamp, +} from '@/lib/billing/activity-rollup/source' import { type ActivityRollupState, MAINTENANCE_GAP_MS, @@ -21,7 +27,8 @@ const logger = createLogger('ActivityRollupMarks') /** * How far behind now the late-end scan stays: an end is stamped by the app before its write * commits, so the scan only consumes ends old enough that every write stamped with them is - * visible. Ends younger than this are read by a later pass. + * visible. Ends younger than this are read by a later pass. A writer whose stamp can trail its + * commit by longer marks the runs it finishes itself (see {@link markFinishedRunsDirty}). */ export const LATE_END_SCAN_MARGIN_MS = 5 * 60 * 1000 @@ -39,7 +46,8 @@ interface ChangedRun { /** * Marks the rollup cells of `runs` for recomputing, in the caller's transaction so the marks * commit exactly when the change does. For changes the rollup pass cannot see on its own: - * deleted runs, and finished runs reopened without an end. + * deleted runs, finished runs reopened without an end, and runs whose end committed too late for + * the late-end scan (see {@link markFinishedRunsDirty}). * * Runs in a savepoint and never throws: a mark that fails (the table missing mid-deploy) is logged * and the caller's change goes ahead, so retention and resumes behave exactly as without the @@ -70,6 +78,8 @@ export async function markWorkflowActivityDirty( SELECT 1 FROM ${activityRollupState} WHERE maintained_at > now() - ${msInterval(MAINTENANCE_GAP_MS)} ) + -- Writers of several marks lock them in key order, so two can never deadlock. + ORDER BY 1, 2 ON CONFLICT (workspace_id, bucket_start) DO UPDATE SET due_at = excluded.due_at `) }) @@ -81,6 +91,22 @@ export async function markWorkflowActivityDirty( } } +/** Runs younger than this lie in hours not computed yet: the settle lag less an hour of clock margin. */ +const FINISHED_RUN_MARK_AGE_MS = ACTIVITY_ROLLUP_SETTLE_MS - HOUR_MS + +/** + * Marks the cells of runs whose final status is being written, in the transaction that writes it, + * when their hours may already be computed. Their ends were stamped before the write, so it can + * commit after the late-end scan passed the end, or after the hour was computed without a late end. + */ +export async function markFinishedRunsDirty(tx: DbTransaction, runs: ChangedRun[]): Promise { + const settled = Date.now() - FINISHED_RUN_MARK_AGE_MS + await markWorkflowActivityDirty( + tx, + runs.filter((run) => run.startedAt.getTime() <= settled) + ) +} + /** * Marks the cells of runs that ended at least two hours after they started since the last scan, * and moves the scan forward. Their hours were computed before they ended (HITL runs resumed @@ -123,6 +149,7 @@ export async function captureLateEnds( FROM ${workflowExecutionLogs} l, bound WHERE l.ended_at >= l.started_at + interval '2 hours' AND l.ended_at > bound.cursor AND l.ended_at <= bound.boundary + ORDER BY 1, 2 ON CONFLICT (workspace_id, bucket_start) DO NOTHING RETURNING 1 ), moved AS ( diff --git a/apps/sim/lib/billing/activity-rollup/pass.ts b/apps/sim/lib/billing/activity-rollup/pass.ts index 6d8a8908ef8..72e175da95a 100644 --- a/apps/sim/lib/billing/activity-rollup/pass.ts +++ b/apps/sim/lib/billing/activity-rollup/pass.ts @@ -31,6 +31,7 @@ import { processDirtyMarks, } from '@/lib/billing/activity-rollup/marks' import { + ACTIVITY_ROLLUP_SETTLE_MS, floorHour, HOUR_MS, type InstantRange, @@ -56,14 +57,6 @@ import type { DbTransaction } from '@/lib/db/types' const logger = createLogger('ActivityRollupPass') -/** - * How long after an hour ends before it is computed. A run that ends within two hours of its start - * (the predicate of `workflow_execution_logs_late_end_idx`) has ended, and its write committed, by - * then; one that ends later is found by its late end. The hour of margin covers commit and replica - * lag, so this must stay at least an hour above that threshold. - */ -const ACTIVITY_ROLLUP_SETTLE_MS = 3 * HOUR_MS - /** The lease a heavy-hour retry adds on top of its own budget, for the statements around it. */ const LEASE_MARGIN_MS = 60 * 1000 @@ -479,6 +472,7 @@ export function markStaleHours(workspaceId: string, range: InstantRange): Promis SELECT 1 FROM ${activityRollupInflight} q WHERE q.workspace_id = ${workspaceId} AND q.bucket_start = d.hour ) + ORDER BY d.hour ON CONFLICT (workspace_id, bucket_start) DO NOTHING `) return marked.count diff --git a/apps/sim/lib/billing/activity-rollup/source.ts b/apps/sim/lib/billing/activity-rollup/source.ts index 316e9a9c1cb..c1d86851581 100644 --- a/apps/sim/lib/billing/activity-rollup/source.ts +++ b/apps/sim/lib/billing/activity-rollup/source.ts @@ -16,6 +16,14 @@ export interface InstantRange { export const HOUR_MS = 60 * 60 * 1000 +/** + * How long after an hour ends before it is computed. A run that ends within two hours of its start + * (the predicate of `workflow_execution_logs_late_end_idx`) has ended by then; one that ends later + * is found by its late end, or marked when its end commits. The hour of margin covers commit and + * replica lag, so this must stay at least an hour above that threshold. + */ +export const ACTIVITY_ROLLUP_SETTLE_MS = 3 * HOUR_MS + /** An instant as the UTC wall clock `started_at` stores, whatever the session's time zone. */ export function utcTimestamp(instant: Date): SQL { return sql`(${instant.toISOString()}::timestamptz AT TIME ZONE 'UTC')` diff --git a/apps/sim/lib/billing/core/organization-activity-rollup.integration.ts b/apps/sim/lib/billing/core/organization-activity-rollup.integration.ts index 62a9a11340d..31365a977fb 100644 --- a/apps/sim/lib/billing/core/organization-activity-rollup.integration.ts +++ b/apps/sim/lib/billing/core/organization-activity-rollup.integration.ts @@ -1,6 +1,7 @@ import * as schema from '@sim/db/schema' import { readTestDatabaseUrl } from '@sim/db/testing/test-infrastructure' import { generateId } from '@sim/utils/id' +import { eq, sql } from 'drizzle-orm' import { drizzle } from 'drizzle-orm/postgres-js' import postgres from 'postgres' import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' @@ -48,6 +49,7 @@ vi.mock('@/lib/billing/core/usage-segment-cache', () => ({ }, })) +import { markFinishedRunsDirty } from '@/lib/billing/activity-rollup/marks' import { type ActivityRollupPassOptions, markStaleHours, @@ -634,6 +636,28 @@ describe('organization activity rollup', () => { await expectEquivalent() }, 120_000) + it('marks a late-committing completion so its computed hour reads live', async () => { + const run = await oneLog( + `status = 'running' AND started_at < (now() AT TIME ZONE 'UTC') - interval '3 days'` + ) + await database.transaction(async (tx) => { + const [finished] = await tx + .update(schema.workflowExecutionLogs) + .set({ + status: 'completed', + totalDurationMs: 1800, + endedAt: sql`${schema.workflowExecutionLogs.startedAt} + interval '30 minutes'`, + }) + .where(eq(schema.workflowExecutionLogs.id, run.id)) + .returning() + if (!finished) throw new Error('No run to finish') + await markFinishedRunsDirty(tx, [finished]) + }) + await expectEquivalent() + await runPass() + await expectEquivalent() + }, 120_000) + it('runs one pass at a time and rewrites nothing when run again', async () => { const results = await Promise.all([runPass(), runPass()]) expect(results.map((result) => result.status).sort()).toEqual(['leased', 'ran']) @@ -711,13 +735,16 @@ describe('organization activity rollup', () => { FOR EACH ROW WHEN (NEW.bucket_start = '${failing.toISOString().replace('T', ' ').slice(0, 19)}') EXECUTE FUNCTION fail_rollup_write(); `) - await expect(runPass()).rejects.toThrow() - const crashed = await readRollupState() - expect(crashed?.coverageStart?.getTime()).toBeGreaterThan(failing.getTime()) - expect(crashed?.leaseOwner).toBeNull() - await connection.unsafe( - `DROP TRIGGER fail_rollup_write ON workflow_activity_hourly; DROP FUNCTION fail_rollup_write()` - ) + try { + await expect(runPass()).rejects.toThrow() + const crashed = await readRollupState() + expect(crashed?.coverageStart?.getTime()).toBeGreaterThan(failing.getTime()) + expect(crashed?.leaseOwner).toBeNull() + } finally { + await connection.unsafe( + `DROP TRIGGER fail_rollup_write ON workflow_activity_hourly; DROP FUNCTION fail_rollup_write()` + ) + } expect((await runPass()).backfillComplete).toBe(true) await expectEquivalent() }, 300_000) diff --git a/apps/sim/lib/logs/execution/logger.ts b/apps/sim/lib/logs/execution/logger.ts index e1bdc8e0e10..dce29e84919 100644 --- a/apps/sim/lib/logs/execution/logger.ts +++ b/apps/sim/lib/logs/execution/logger.ts @@ -17,6 +17,7 @@ import { import { generateId } from '@sim/utils/id' import { filterUndefined, isRecordLike } from '@sim/utils/object' import { and, eq, inArray, sql } from 'drizzle-orm' +import { markFinishedRunsDirty } from '@/lib/billing/activity-rollup/marks' import { checkUsageStatus as checkResolvedUsageStatus } from '@/lib/billing/calculations/usage-monitor' import { type BillingAttributionSnapshot, toBillingContext } from '@/lib/billing/core/attribution' import { readSoftGateUsageCost } from '@/lib/billing/core/reporting-usage-cache' @@ -1374,6 +1375,7 @@ export class ExecutionLogger { }, completedExecutionLargeValueKeys ) + await markFinishedRunsDirty(tx, [log]) return { updatedLog: log, completionPersisted: true } }) diff --git a/packages/db/migrations/0462_workflow_activity_rollup.sql b/packages/db/migrations/0462_workflow_activity_rollup.sql index ae23d5e26e9..8209ab967de 100644 --- a/packages/db/migrations/0462_workflow_activity_rollup.sql +++ b/packages/db/migrations/0462_workflow_activity_rollup.sql @@ -4,6 +4,8 @@ -- activity-rollup-maintenance and organization-activity-rollup-reads flags are turned on, so the -- deployed app is unaffected. The tables are new, so their indexes build on empty tables inside -- this migration's transaction. +-- 0461 ends the batch transaction with a concurrent index build; keep these tables and their journal row atomic. +BEGIN;--> statement-breakpoint CREATE TABLE "activity_rollup_dirty" ( "workspace_id" text NOT NULL, "bucket_start" timestamp NOT NULL, diff --git a/packages/db/migrations/0464_activity_rollup_maintenance_state.sql b/packages/db/migrations/0464_activity_rollup_maintenance_state.sql index 99b16849483..72bea87fd22 100644 --- a/packages/db/migrations/0464_activity_rollup_maintenance_state.sql +++ b/packages/db/migrations/0464_activity_rollup_maintenance_state.sql @@ -2,6 +2,8 @@ -- when a pass last completed (which gates writers' marks and decides a rebuild after a gap). A new -- table, and a nullable column on the one-row activity_rollup_state, which no deployed code -- writes: metadata-only and invisible to the running app. +-- 0463 ends the batch transaction with a concurrent index build; keep this table and its journal row atomic. +BEGIN;--> statement-breakpoint CREATE TABLE "activity_rollup_blocked" ( "bucket_start" timestamp PRIMARY KEY NOT NULL, "attempts" integer NOT NULL,