Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 16 additions & 13 deletions apps/sim/background/cleanup-stale-executions.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,11 @@ function collectSqlParams(fragment: unknown): unknown[] {
})
}

/** A swept log as the sweep's update returns it: recent, so it is not marked for the rollup. */
function sweptRun(id: string) {
return { id, workspaceId: 'workspace-1', startedAt: new Date() }
}

describe('stale execution cleanup deadline grace', () => {
beforeEach(() => {
resetDbChainMock()
Expand Down Expand Up @@ -324,9 +329,9 @@ describe('stale execution cleanup deadline grace', () => {
}))

for (let batch = 0; batch < 10; batch++) {
const workflowBatch = Array.from({ length: 100 }, (_, index) => ({
id: `workflow-state-${batch}-${index}`,
}))
const workflowBatch = Array.from({ length: 100 }, (_, index) =>
sweptRun(`workflow-state-${batch}-${index}`)
)
queueTableRows(workflowExecutionLogs, workflowBatch)
dbChainMockFns.returning.mockResolvedValueOnce(workflowBatch)
}
Expand Down Expand Up @@ -396,12 +401,10 @@ describe('stale execution cleanup deadline grace', () => {
})

it('preserves committed workflow cleanup counts when a later batch fails', async () => {
const firstBatch = Array.from({ length: 100 }, (_, index) => ({
id: `execution-${index}`,
}))
const failedBatch = Array.from({ length: 37 }, (_, index) => ({
id: `failed-execution-${index}`,
}))
const firstBatch = Array.from({ length: 100 }, (_, index) => sweptRun(`execution-${index}`))
const failedBatch = Array.from({ length: 37 }, (_, index) =>
sweptRun(`failed-execution-${index}`)
)
queueTableRows(workflowExecutionLogs, firstBatch)
queueTableRows(workflowExecutionLogs, failedBatch)
queueTableRows(asyncJobs, [{ id: 'async-job-1' }])
Expand All @@ -426,11 +429,11 @@ describe('stale execution cleanup deadline grace', () => {
})

it('continues draining when an atomic race updates fewer rows than were selected', async () => {
const firstCandidates = Array.from({ length: 100 }, (_, index) => ({
id: `execution-${index}`,
}))
const firstCandidates = Array.from({ length: 100 }, (_, index) =>
sweptRun(`execution-${index}`)
)
const firstUpdated = firstCandidates.slice(0, 99)
const secondBatch = [{ id: 'execution-100' }]
const secondBatch = [sweptRun('execution-100')]
queueTableRows(workflowExecutionLogs, firstCandidates)
queueTableRows(workflowExecutionLogs, secondBatch)
dbChainMockFns.returning.mockResolvedValueOnce(firstUpdated).mockResolvedValueOnce(secondBatch)
Expand Down
9 changes: 8 additions & 1 deletion apps/sim/background/cleanup-stale-executions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import { toError } from '@sim/utils/errors'
import { task } from '@trigger.dev/sdk'
import { and, eq, exists, gt, inArray, isNull, lt, ne, or, sql } from 'drizzle-orm'
import { alias } from 'drizzle-orm/pg-core'
import { markFinishedRunsDirty } from '@/lib/billing/activity-rollup/marks'
import {
JOB_PENDING_RETENTION_HOURS,
JOB_RETENTION_HOURS,
Expand Down Expand Up @@ -287,7 +288,13 @@ export async function runCleanupStaleExecutions() {
)
)
)
.returning({ id: workflowExecutionLogs.id })
.returning({
id: workflowExecutionLogs.id,
workspaceId: workflowExecutionLogs.workspaceId,
startedAt: workflowExecutionLogs.startedAt,
})
// Every batch reuses the sweep's one end stamp, so a late batch can commit behind the scan.
await markFinishedRunsDirty(tx, updatedExecutions)

return { candidates, updatedExecutions }
})
Expand Down
33 changes: 30 additions & 3 deletions apps/sim/lib/billing/activity-rollup/marks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,13 @@ import {
inflightBackoffMs,
recomputeCell,
} from '@/lib/billing/activity-rollup/cells'
import { floorHour, msInterval, utcTimestamp } from '@/lib/billing/activity-rollup/source'
import {
ACTIVITY_ROLLUP_SETTLE_MS,
floorHour,
HOUR_MS,
msInterval,
utcTimestamp,
} from '@/lib/billing/activity-rollup/source'
import {
type ActivityRollupState,
MAINTENANCE_GAP_MS,
Expand All @@ -21,7 +27,8 @@ const logger = createLogger('ActivityRollupMarks')
/**
* How far behind now the late-end scan stays: an end is stamped by the app before its write
* commits, so the scan only consumes ends old enough that every write stamped with them is
* visible. Ends younger than this are read by a later pass.
* visible. Ends younger than this are read by a later pass. A writer whose stamp can trail its
* commit by longer marks the runs it finishes itself (see {@link markFinishedRunsDirty}).
*/
export const LATE_END_SCAN_MARGIN_MS = 5 * 60 * 1000

Expand All @@ -39,7 +46,8 @@ interface ChangedRun {
/**
* Marks the rollup cells of `runs` for recomputing, in the caller's transaction so the marks
* commit exactly when the change does. For changes the rollup pass cannot see on its own:
* deleted runs, and finished runs reopened without an end.
* deleted runs, finished runs reopened without an end, and runs whose end committed too late for
* the late-end scan (see {@link markFinishedRunsDirty}).
*
* Runs in a savepoint and never throws: a mark that fails (the table missing mid-deploy) is logged
* and the caller's change goes ahead, so retention and resumes behave exactly as without the
Expand Down Expand Up @@ -70,6 +78,8 @@ export async function markWorkflowActivityDirty(
SELECT 1 FROM ${activityRollupState}
WHERE maintained_at > now() - ${msInterval(MAINTENANCE_GAP_MS)}
)
-- Writers of several marks lock them in key order, so two can never deadlock.
ORDER BY 1, 2
ON CONFLICT (workspace_id, bucket_start) DO UPDATE SET due_at = excluded.due_at
`)
})
Expand All @@ -81,6 +91,22 @@ export async function markWorkflowActivityDirty(
}
}

/** Runs younger than this lie in hours not computed yet: the settle lag less an hour of clock margin. */
const FINISHED_RUN_MARK_AGE_MS = ACTIVITY_ROLLUP_SETTLE_MS - HOUR_MS

/**
* Marks the cells of runs whose final status is being written, in the transaction that writes it,
* when their hours may already be computed. Their ends were stamped before the write, so it can
* commit after the late-end scan passed the end, or after the hour was computed without a late end.
*/
export async function markFinishedRunsDirty(tx: DbTransaction, runs: ChangedRun[]): Promise<void> {
const settled = Date.now() - FINISHED_RUN_MARK_AGE_MS
await markWorkflowActivityDirty(
tx,
runs.filter((run) => run.startedAt.getTime() <= settled)
)
}

/**
* Marks the cells of runs that ended at least two hours after they started since the last scan,
* and moves the scan forward. Their hours were computed before they ended (HITL runs resumed
Expand Down Expand Up @@ -123,6 +149,7 @@ export async function captureLateEnds(
FROM ${workflowExecutionLogs} l, bound
WHERE l.ended_at >= l.started_at + interval '2 hours'
AND l.ended_at > bound.cursor AND l.ended_at <= bound.boundary
ORDER BY 1, 2
ON CONFLICT (workspace_id, bucket_start) DO NOTHING
RETURNING 1
), moved AS (
Expand Down
10 changes: 2 additions & 8 deletions apps/sim/lib/billing/activity-rollup/pass.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import {
processDirtyMarks,
} from '@/lib/billing/activity-rollup/marks'
import {
ACTIVITY_ROLLUP_SETTLE_MS,
floorHour,
HOUR_MS,
type InstantRange,
Expand All @@ -56,14 +57,6 @@ import type { DbTransaction } from '@/lib/db/types'

const logger = createLogger('ActivityRollupPass')

/**
* How long after an hour ends before it is computed. A run that ends within two hours of its start
* (the predicate of `workflow_execution_logs_late_end_idx`) has ended, and its write committed, by
* then; one that ends later is found by its late end. The hour of margin covers commit and replica
* lag, so this must stay at least an hour above that threshold.
*/
const ACTIVITY_ROLLUP_SETTLE_MS = 3 * HOUR_MS

/** The lease a heavy-hour retry adds on top of its own budget, for the statements around it. */
const LEASE_MARGIN_MS = 60 * 1000

Expand Down Expand Up @@ -479,6 +472,7 @@ export function markStaleHours(workspaceId: string, range: InstantRange): Promis
SELECT 1 FROM ${activityRollupInflight} q
WHERE q.workspace_id = ${workspaceId} AND q.bucket_start = d.hour
)
ORDER BY d.hour
ON CONFLICT (workspace_id, bucket_start) DO NOTHING
`)
return marked.count
Expand Down
8 changes: 8 additions & 0 deletions apps/sim/lib/billing/activity-rollup/source.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,14 @@ export interface InstantRange {

export const HOUR_MS = 60 * 60 * 1000

/**
* How long after an hour ends before it is computed. A run that ends within two hours of its start
* (the predicate of `workflow_execution_logs_late_end_idx`) has ended by then; one that ends later
* is found by its late end, or marked when its end commits. The hour of margin covers commit and
* replica lag, so this must stay at least an hour above that threshold.
*/
export const ACTIVITY_ROLLUP_SETTLE_MS = 3 * HOUR_MS

/** An instant as the UTC wall clock `started_at` stores, whatever the session's time zone. */
export function utcTimestamp(instant: Date): SQL {
return sql`(${instant.toISOString()}::timestamptz AT TIME ZONE 'UTC')`
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import * as schema from '@sim/db/schema'
import { readTestDatabaseUrl } from '@sim/db/testing/test-infrastructure'
import { generateId } from '@sim/utils/id'
import { eq, sql } from 'drizzle-orm'
import { drizzle } from 'drizzle-orm/postgres-js'
import postgres from 'postgres'
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
Expand Down Expand Up @@ -48,6 +49,7 @@ vi.mock('@/lib/billing/core/usage-segment-cache', () => ({
},
}))

import { markFinishedRunsDirty } from '@/lib/billing/activity-rollup/marks'
import {
type ActivityRollupPassOptions,
markStaleHours,
Expand Down Expand Up @@ -634,6 +636,28 @@ describe('organization activity rollup', () => {
await expectEquivalent()
}, 120_000)

it('marks a late-committing completion so its computed hour reads live', async () => {
const run = await oneLog(
`status = 'running' AND started_at < (now() AT TIME ZONE 'UTC') - interval '3 days'`
)
await database.transaction(async (tx) => {
const [finished] = await tx
.update(schema.workflowExecutionLogs)
.set({
status: 'completed',
totalDurationMs: 1800,
endedAt: sql`${schema.workflowExecutionLogs.startedAt} + interval '30 minutes'`,
})
.where(eq(schema.workflowExecutionLogs.id, run.id))
.returning()
if (!finished) throw new Error('No run to finish')
await markFinishedRunsDirty(tx, [finished])
})
await expectEquivalent()
await runPass()
await expectEquivalent()
}, 120_000)

it('runs one pass at a time and rewrites nothing when run again', async () => {
const results = await Promise.all([runPass(), runPass()])
expect(results.map((result) => result.status).sort()).toEqual(['leased', 'ran'])
Expand Down Expand Up @@ -711,13 +735,16 @@ describe('organization activity rollup', () => {
FOR EACH ROW WHEN (NEW.bucket_start = '${failing.toISOString().replace('T', ' ').slice(0, 19)}')
EXECUTE FUNCTION fail_rollup_write();
`)
await expect(runPass()).rejects.toThrow()
const crashed = await readRollupState()
expect(crashed?.coverageStart?.getTime()).toBeGreaterThan(failing.getTime())
expect(crashed?.leaseOwner).toBeNull()
await connection.unsafe(
`DROP TRIGGER fail_rollup_write ON workflow_activity_hourly; DROP FUNCTION fail_rollup_write()`
)
try {
await expect(runPass()).rejects.toThrow()
const crashed = await readRollupState()
expect(crashed?.coverageStart?.getTime()).toBeGreaterThan(failing.getTime())
expect(crashed?.leaseOwner).toBeNull()
} finally {
await connection.unsafe(
`DROP TRIGGER fail_rollup_write ON workflow_activity_hourly; DROP FUNCTION fail_rollup_write()`
)
}
expect((await runPass()).backfillComplete).toBe(true)
await expectEquivalent()
}, 300_000)
Expand Down
2 changes: 2 additions & 0 deletions apps/sim/lib/logs/execution/logger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import {
import { generateId } from '@sim/utils/id'
import { filterUndefined, isRecordLike } from '@sim/utils/object'
import { and, eq, inArray, sql } from 'drizzle-orm'
import { markFinishedRunsDirty } from '@/lib/billing/activity-rollup/marks'
import { checkUsageStatus as checkResolvedUsageStatus } from '@/lib/billing/calculations/usage-monitor'
import { type BillingAttributionSnapshot, toBillingContext } from '@/lib/billing/core/attribution'
import { readSoftGateUsageCost } from '@/lib/billing/core/reporting-usage-cache'
Expand Down Expand Up @@ -1374,6 +1375,7 @@ export class ExecutionLogger {
},
completedExecutionLargeValueKeys
)
await markFinishedRunsDirty(tx, [log])

return { updatedLog: log, completionPersisted: true }
})
Expand Down
2 changes: 2 additions & 0 deletions packages/db/migrations/0462_workflow_activity_rollup.sql
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
-- activity-rollup-maintenance and organization-activity-rollup-reads flags are turned on, so the
-- deployed app is unaffected. The tables are new, so their indexes build on empty tables inside
-- this migration's transaction.
-- 0461 ends the batch transaction with a concurrent index build; keep these tables and their journal row atomic.
BEGIN;--> statement-breakpoint
CREATE TABLE "activity_rollup_dirty" (
"workspace_id" text NOT NULL,
"bucket_start" timestamp NOT NULL,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
-- when a pass last completed (which gates writers' marks and decides a rebuild after a gap). A new
-- table, and a nullable column on the one-row activity_rollup_state, which no deployed code
-- writes: metadata-only and invisible to the running app.
-- 0463 ends the batch transaction with a concurrent index build; keep this table and its journal row atomic.
BEGIN;--> statement-breakpoint
CREATE TABLE "activity_rollup_blocked" (
"bucket_start" timestamp PRIMARY KEY NOT NULL,
"attempts" integer NOT NULL,
Expand Down
Loading