Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 28 additions & 15 deletions apps/sim/lib/billing/activity-rollup/cells.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ import { describe, expect, it, vi } from 'vitest'
import {
type ActivityCell,
aggregateHour,
HeavyHourError,
type HourReaders,
PassDeadlineError,
} from '@/lib/billing/activity-rollup/cells'
import type { InstantRange } from '@/lib/billing/activity-rollup/source'

Expand Down Expand Up @@ -76,52 +76,65 @@ describe('aggregateHour', () => {
}
)

it('reads a slice that still times out a page of workspaces at a time', async () => {
it('pages the rest of the hour once after a slice times out, not slice by slice', async () => {
const span = vi.fn(async (range: InstantRange) => {
if (isWholeHour(range) || quarter(range) === 3) throw timeout()
if (isWholeHour(range) || quarter(range) === 1) throw timeout()
return [cell('a', 1, 5)]
})
const workspacePage = vi.fn(async (_range: InstantRange, after: string | null) =>
after === null
? { cells: [cell('a', 2, 7)], last: 'ws-500' }
: { cells: [cell('b', 1, 1)], last: null }
? { cells: [cell('a', 2, 7)], last: 'ws-500', done: false }
: { cells: [cell('b', 1, 1)], last: null, done: true }
)

const cells = await aggregateHour(hour, Number.POSITIVE_INFINITY, {
readers: readers({ span, workspacePage }),
})

expect(workspacePage.mock.calls.map(([range, after]) => [quarter(range), after])).toEqual([
[3, null],
[3, 'ws-500'],
expect(
span.mock.calls.map(([range]) => (isWholeHour(range) ? 'hour' : quarter(range)))
).toEqual(['hour', 0, 1])
expect(
workspacePage.mock.calls.map(([range, after]) => [
range.from.toISOString(),
range.to.toISOString(),
after,
])
).toEqual([
['2026-05-01T10:15:00.000Z', '2026-05-01T11:00:00.000Z', null],
['2026-05-01T10:15:00.000Z', '2026-05-01T11:00:00.000Z', 'ws-500'],
])
expect(cells).toEqual([cell('a', 5, 22), cell('b', 1, 1)])
expect(cells).toEqual([cell('a', 3, 12), cell('b', 1, 1)])
})

it('gives the hour up as too heavy rather than page workspaces past the pass budget', async () => {
it('gives up before paging once the pass budget has run out, without calling the hour heavy', async () => {
const span = vi.fn(async () => {
throw timeout()
})
const workspacePage = vi.fn(async () => ({ cells: [], last: null }))
const workspacePage = vi.fn(async () => ({ cells: [], last: null, done: true }))

await expect(
aggregateHour(hour, Date.now() - 1, { readers: readers({ span, workspacePage }) })
).rejects.toBeInstanceOf(HeavyHourError)
).rejects.toBeInstanceOf(PassDeadlineError)
expect(workspacePage).not.toHaveBeenCalled()
})

it('retries an hour known to be too heavy a page of workspaces at a time from the start', async () => {
const span = vi.fn(async () => [cell('a', 9, 9)])
const workspacePage = vi.fn(async () => ({ cells: [cell('a', 1, 2)], last: null }))
const workspacePage = vi.fn(async (_range: InstantRange) => ({
cells: [cell('a', 1, 2)],
last: null,
done: true,
}))

const cells = await aggregateHour(hour, Number.POSITIVE_INFINITY, {
readers: readers({ span, workspacePage }),
byWorkspace: true,
})

expect(span).not.toHaveBeenCalled()
expect(workspacePage).toHaveBeenCalledTimes(4)
expect(cells).toEqual([cell('a', 4, 8)])
expect(workspacePage.mock.calls.map(([range]) => isWholeHour(range))).toEqual([true])
expect(cells).toEqual([cell('a', 1, 2)])
})

it('fails without slicing when the read fails for any other reason', async () => {
Expand Down
160 changes: 104 additions & 56 deletions apps/sim/lib/billing/activity-rollup/cells.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
import { db, dbReplica } from '@sim/db'
import { activityRollupInflight, workflowActivityHourly, workspace } from '@sim/db/schema'
import {
activityRollupInflight,
workflowActivityHourly,
workflowExecutionLogs,
} from '@sim/db/schema'
import { createLogger } from '@sim/logger'
import { getPostgresCancellationReason } from '@sim/utils/errors'
import { chunkArray } from '@sim/utils/helpers'
import { sql } from 'drizzle-orm'
import {
HOUR_MS,
Expand Down Expand Up @@ -108,17 +113,23 @@ function sumCells(parts: ActivityCell[][]): ActivityCell[] {
return [...byKey.values()]
}

/** Workspaces read per statement once a slice is too heavy to read at once. */
const WORKSPACE_PAGE = 500
/** Distinct workspaces with runs a page walks, and how many of them one statement aggregates. */
const WORKSPACE_WALK = 1000
const WORKSPACE_CHUNK = 100

/** Reads of an hour's runs, from the whole span down to one page of workspaces at a time. */
export interface HourReaders {
span: (range: InstantRange) => Promise<ActivityCell[]>
/** The cells of up to a page of workspaces after `after` (by id), and the last one read. */
/**
* The cells of the next page of workspaces with runs in `range`, after `after` (by id), the
* workspace the next page starts after, and whether every workspace with runs was read. A page
* stops early, before an aggregate, once `deadline` passes.
*/
workspacePage: (
range: InstantRange,
after: string | null
) => Promise<{ cells: ActivityCell[]; last: string | null }>
after: string | null,
deadline: number
) => Promise<{ cells: ActivityCell[]; last: string | null; done: boolean }>
}

/** An hour that could not be read within the statement bounds, even a page of workspaces at a time. */
Expand All @@ -129,6 +140,14 @@ export class HeavyHourError extends Error {
}
}

/** The pass ran out of budget while paging an hour: not a sign the hour is too heavy. */
export class PassDeadlineError extends Error {
constructor(hour: Date) {
super(`Activity rollup pass budget ran out while paging hour ${hour.toISOString()}`)
this.name = 'PassDeadlineError'
}
}

/** Whether a read gave up on its own statement bound, or the replica cancelled it for recovery. */
function isBoundedCancellation(error: unknown): boolean {
const reason = getPostgresCancellationReason(error)
Expand All @@ -146,81 +165,112 @@ function onReplica<T>(read: (tx: DbTransaction) => Promise<T>): Promise<T> {
)
}

/**
* The next workspaces that have any run, after `after`, by a bounded loose index scan on the
* activity index (one descent per workspace, never a scan of runs), each with whether it has runs
* in `range` (one more descent: a LATERAL `LIMIT 1`, which the planner cannot turn into a hashed
* scan of the whole range).
*/
async function walkWorkspaces(
tx: DbTransaction,
range: InstantRange,
after: string | null
): Promise<{ id: string; active: number }[]> {
return tx.execute<{ id: string; active: number }>(sql`
WITH RECURSIVE walk AS (
(SELECT l.workspace_id AS id, 1 AS n FROM ${workflowExecutionLogs} l
${after === null ? sql`` : sql`WHERE l.workspace_id > ${after}`}
ORDER BY l.workspace_id LIMIT 1)
UNION ALL
SELECT (
SELECT l.workspace_id FROM ${workflowExecutionLogs} l
WHERE l.workspace_id > walk.id ORDER BY l.workspace_id LIMIT 1
), walk.n + 1
FROM walk WHERE walk.id IS NOT NULL AND walk.n < ${WORKSPACE_WALK}
)
SELECT walk.id, (hit.found IS NOT NULL)::int AS active
FROM walk
LEFT JOIN LATERAL (
SELECT 1 AS found FROM ${workflowExecutionLogs} l
WHERE l.workspace_id = walk.id
AND l.started_at >= ${utcTimestamp(range.from)} AND l.started_at < ${utcTimestamp(range.to)}
LIMIT 1
) hit ON true
WHERE walk.id IS NOT NULL
`)
}

const replicaReaders: HourReaders = {
span: (range) => onReplica((tx) => aggregateRange(tx, range)),
workspacePage: (range, after) =>
workspacePage: (range, after, deadline) =>
onReplica(async (tx) => {
const page = await tx.execute<{ id: string }>(sql`
SELECT id FROM ${workspace} ${after === null ? sql`` : sql`WHERE id > ${after}`}
ORDER BY id LIMIT ${WORKSPACE_PAGE}
`)
const ids = page.map(({ id }) => id)
if (ids.length === 0) return { cells: [], last: null }
const cells = await aggregateRange(tx, range, ids)
return { cells, last: ids.length < WORKSPACE_PAGE ? null : (ids.at(-1) ?? null) }
const walked = await walkWorkspaces(tx, range, after)
const active = walked.filter((row) => Number(row.active) === 1).map((row) => row.id)
/** Chunks hold different workspaces, so their cells never share a key. */
const cells: ActivityCell[] = []
let last = after
for (const chunk of chunkArray(active, WORKSPACE_CHUNK)) {
if (Date.now() >= deadline) return { cells, last, done: false }
cells.push(...(await aggregateRange(tx, range, chunk)))
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved
last = chunk.at(-1) ?? last
}
const done = walked.length < WORKSPACE_WALK
return { cells, last: done ? null : (walked.at(-1)?.id ?? null), done }
}),
}

/**
* Every cell of one UTC hour, read on the replica, at the coarsest grain that fits the statement
* bounds. A span that times out (or that the replica cancels for recovery) is read again in
* quarter-hour slices, and a slice that still does is read a page of workspaces at a time, each on
* the workspace activity index. All of these count exactly the same runs, so their cells are
* summed. Throws {@link HeavyHourError} when even a page of workspaces cannot be read, or the
* pass's `deadline` passes while paging.
* quarter-hour slices; once a slice still does, the rest of the hour is read once, a page of
* workspaces with runs at a time, each on the workspace activity index. All of these count exactly
* the same runs, so their cells are summed. Throws {@link HeavyHourError} when even a page cannot
* be read, and {@link PassDeadlineError} when `deadline` passes while paging.
*/
export async function aggregateHour(
hour: Date,
deadline: number,
{
readers = replicaReaders,
byWorkspace: startByWorkspace = false,
byWorkspace = false,
}: {
readers?: HourReaders
/** Skips straight to workspace pages: a retry of an hour already known to be too heavy. */
/** Skips straight to workspace pages: a retry of an hour a previous pass had to page. */
byWorkspace?: boolean
} = {}
): Promise<ActivityCell[]> {
if (!startByWorkspace) {
try {
return await readers.span({ from: hour, to: new Date(hour.getTime() + HOUR_MS) })
} catch (error) {
if (!isBoundedCancellation(error)) throw error
logger.warn('Activity rollup hour read in slices', {
hour: hour.toISOString(),
reason: getPostgresCancellationReason(error),
})
}
const hourEnd = new Date(hour.getTime() + HOUR_MS)
const wholeHour = { from: hour, to: hourEnd }
if (byWorkspace) return readByWorkspace(hour, wholeHour, deadline, readers)
try {
return await readers.span(wholeHour)
} catch (error) {
if (!isBoundedCancellation(error)) throw error
logger.warn('Activity rollup hour read in slices', {
hour: hour.toISOString(),
reason: getPostgresCancellationReason(error),
})
}
const sliceMs = HOUR_MS / SLICES_PER_HOUR
const parts: ActivityCell[][] = []
// Once one slice needs workspace pages, later slices skip the span read bound to fail.
let byWorkspace = startByWorkspace
for (let slice = 0; slice < SLICES_PER_HOUR; slice++) {
const from = new Date(hour.getTime() + slice * sliceMs)
const range = { from, to: new Date(from.getTime() + sliceMs) }
if (!byWorkspace) {
try {
parts.push(await readers.span(range))
continue
} catch (error) {
if (!isBoundedCancellation(error)) throw error
logger.warn('Activity rollup slice read by workspace', {
from: from.toISOString(),
reason: getPostgresCancellationReason(error),
})
byWorkspace = true
}
try {
parts.push(await readers.span({ from, to: new Date(from.getTime() + sliceMs) }))
} catch (error) {
if (!isBoundedCancellation(error)) throw error
logger.warn('Activity rollup hour read by workspace', {
from: from.toISOString(),
reason: getPostgresCancellationReason(error),
})
parts.push(await readByWorkspace(hour, { from, to: hourEnd }, deadline, readers))
break
}
parts.push(await readByWorkspace(hour, range, deadline, readers))
}
return sumCells(parts)
}

/**
* Reads a slice a page of workspaces at a time. Gives the hour up as too heavy when a page still
* times out, or when the pass's budget runs out first, so a heavy hour never outlives the lease.
*/
/** Reads `range` a page of workspaces with runs at a time. */
async function readByWorkspace(
hour: Date,
range: InstantRange,
Expand All @@ -230,13 +280,11 @@ async function readByWorkspace(
const parts: ActivityCell[][] = []
let after: string | null = null
for (;;) {
if (Date.now() >= deadline) {
throw new HeavyHourError(hour, { cause: new Error('Pass budget ran out') })
}
if (Date.now() >= deadline) throw new PassDeadlineError(hour)
try {
const page = await readers.workspacePage(range, after)
const page = await readers.workspacePage(range, after, deadline)
parts.push(page.cells)
if (page.last === null) return sumCells(parts)
if (page.done) return sumCells(parts)
after = page.last
} catch (error) {
throw isBoundedCancellation(error) ? new HeavyHourError(hour, { cause: error }) : error
Expand Down
Loading
Loading