Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion apps/api/src/routes/requests/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1134,7 +1134,9 @@ apiRoute(ctx.app, ctx.registry, {
const data = resources.map((r) => ({ ...r, id: r._id }));

// Enrich `processing` runs with the latest liveness heartbeat from
// Redis (single MGET; heartbeats live there, not Mongo).
// Redis. The store's mget is cluster-safe (a pipeline of single-key GETs,
// not a cross-slot MGET) so this works on clustered Redis too — see
// clusterSafeMget / issue #1064. Heartbeats live in Redis, not Mongo.
const processingRunIds = data
.filter((r) => r.run?.status === "processing" && r.run._id)
.map((r) => r.run!._id!);
Expand Down
43 changes: 43 additions & 0 deletions apps/scheduler/src/stuck-run-reaper.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,49 @@ describe("StuckRunReaper", () => {
expect(collection.findOneAndUpdate).toHaveBeenCalledTimes(1);
});

it("aborts the reap when a single-key confirmation read finds a fresh beat (cross-slot mget under-reported)", async () => {
// Reproduces issue #1064: on a clustered Redis the multi-key mget fails
// cross-slot and is swallowed into an empty map, so a healthy run looks
// heartbeat-less across both sweeps — but a single-key get (what the run
// detail view uses) still returns the fresh beat. The reaper must not fail
// the run.
const docs = [
{ _id: "req-1", run: { _id: "run-1", startedAt: OLD(), worker: { instanceId: "w-1" } } },
];
const collection = mockCollection(docs);
const store = {
ping: vi.fn().mockResolvedValue(true),
mget: vi.fn().mockResolvedValue(new Map()), // cross-slot failure → empty
get: vi.fn().mockResolvedValue(new Date()), // single-key read works → fresh
delete: vi.fn(),
} as any;
const reaper = makeReaper(collection, store);

await reaper.sweep(); // strike 1 (looks missing via mget)
await reaper.sweep(); // strike 2 → reapOne, but confirmation get() aborts
expect(store.get).toHaveBeenCalledWith("run-1");
expect(collection.findOneAndUpdate).not.toHaveBeenCalled();
expect(store.delete).not.toHaveBeenCalled();
});

it("still reaps when the confirmation read also reports a stale/missing beat", async () => {
const docs = [
{ _id: "req-1", run: { _id: "run-1", startedAt: OLD(), worker: { instanceId: "w-1" } } },
];
const collection = mockCollection(docs);
const store = {
ping: vi.fn().mockResolvedValue(true),
mget: vi.fn().mockResolvedValue(new Map()),
get: vi.fn().mockResolvedValue(null), // genuinely dead — no beat
delete: vi.fn(),
} as any;
const reaper = makeReaper(collection, store);

await reaper.sweep();
await reaper.sweep();
expect(collection.findOneAndUpdate).toHaveBeenCalledTimes(1);
});

it("does NOT reap a run with a fresh heartbeat", async () => {
const docs = [
{ _id: "req-1", run: { _id: "run-1", startedAt: OLD(), worker: { instanceId: "w-1" } } },
Expand Down
50 changes: 48 additions & 2 deletions apps/scheduler/src/stuck-run-reaper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ import type { HeartbeatStore, RequestDocument } from "shared";
* is reaped. A transient Redis read blip (which makes the whole fleet look
* stale for one sweep) is absorbed because nothing was stale in the prior
* sweep; only genuinely-dead runs persist across both.
* - Confirmation re-read: immediately before failing a run, its heartbeat is
* re-read with a single-key GET. A fresh beat here aborts the reap
* regardless of why the batch read missed it (issue #1064).
* - Per-sweep circuit-breaker: if more runs would be reaped than
* `maxPerSweep`, the sweep is skipped and logged loudly — a correlated
* spike is almost certainly systemic (CosmosDB throttling, network) rather
Expand Down Expand Up @@ -170,12 +173,41 @@ export class StuckRunReaper {
return;
}

// Zero readable beats for a non-empty candidate set points at a systemic
// heartbeat-read/Redis-misconfig problem, not N simultaneous worker deaths
// (genuine deaths still expose present-but-stale beats inside the TTL).
// Surface it loudly; the per-candidate confirmation read in reapOne() is
// the actual false-positive guard (issue #1064).
if (heartbeats.size === 0 && candidates.length > 0) {
console.warn(
`[StuckRunReaper] Read 0 live heartbeats for ${candidates.length} processing candidate(s) ` +
`while Redis is reachable — likely a heartbeat read failure or Redis misconfiguration, ` +
`not worker deaths. Each candidate is re-verified with a direct read before any reap. ` +
`Verify the scheduler's REDIS_HOST/PORT/PASSWORD/TLS match the workers and API.`,
);
}

const now = Date.now();
let missingCount = 0;
let staleCount = 0;
const currentStale = candidates.filter((c) => {
const hb = heartbeats.get(c.runId);
if (!hb) return true; // missing & startedAt already old (filtered in findCandidates)
return now - hb.getTime() > this.staleThresholdMs;
if (!hb) {
missingCount++;
return true; // missing & startedAt already old (filtered in findCandidates)
}
if (now - hb.getTime() > this.staleThresholdMs) {
staleCount++;
return true;
}
return false;
});
if (currentStale.length > 0) {
console.log(
`[StuckRunReaper] Sweep: ${candidates.length} candidate(s), ${heartbeats.size} live beat(s), ` +
`${staleCount} stale, ${missingCount} missing; ${this.previouslyStale.size} stale in prior sweep.`,
);
}
const currentStaleIds = new Set(currentStale.map((c) => c.runId));

// Two-strikes: only reap runs that were ALSO stale in the previous sweep.
Expand Down Expand Up @@ -262,6 +294,20 @@ export class StuckRunReaper {
}

private async reapOne(c: Candidate, now: number): Promise<void> {
// Confirmation re-read (issue #1064): a single-key GET is always
// shard-routable, so a fresh beat here means the worker is alive — abort the
// reap regardless of why the batch read under-reported it.
const confirmBeat = await this.heartbeatStore.get(c.runId);
if (confirmBeat && now - confirmBeat.getTime() <= this.staleThresholdMs) {
console.warn(
`[StuckRunReaper] Aborting reap of request=${c._id} (runId=${c.runId}) — confirmation read found a ` +
`fresh heartbeat (${Math.round((now - confirmBeat.getTime()) / 1000)}s ago). The batch heartbeat ` +
`read under-reported liveness (likely a cross-slot MGET failure on clustered Redis); not reaping.`,
);
this.previouslyStale.delete(c.runId);
return;
}

const ageDesc = c.startedAt
? `${Math.round((now - c.startedAt.getTime()) / 1000)}s`
: "unknown";
Expand Down
11 changes: 7 additions & 4 deletions docs/architecture/queue-scheduler.md
Original file line number Diff line number Diff line change
Expand Up @@ -144,9 +144,11 @@ Azure Storage Queues guarantee at-least-once delivery, so a worker may receive a
|-------|-------|------|---------|
| Mongo (`run.*`) | `worker` | `{ instanceId, podName? }` | Identity of the worker process currently processing the run. `instanceId` is a per-process UUID; `podName` is the K8s pod name (`HOSTNAME`) when running in a pod. Stamped atomically at the `queued → processing` pickup. Persisted for forensics; never updated by heartbeat ticks. |
| Mongo (`run.*`) | `startedAt` | `Date` | Wall-clock time of the pickup. Used as the missing-heartbeat fallback (see below). |
| Redis | `run-heartbeat:<runId>` | `Date` (ISO string, TTL ≈ 5×visibility) | Wall-clock time the owning worker last beat. Refreshed every 15s by the per-run heartbeat callback wired into [`startVisibilityHeartbeat`](../../packages/shared/src/queue/visibility-heartbeat.ts). |
| Redis | `run-heartbeat:<runId>` | `Date` (ISO string, TTL ≈ 5×visibility) | Wall-clock time the owning worker last beat. Refreshed every 15s by the per-run liveness loop in [`startVisibilityHeartbeat`](../../packages/shared/src/queue/visibility-heartbeat.ts). This loop runs on a **dedicated `setInterval`, fully decoupled from the queue-visibility extension** — so a transient/slow `queueClient.updateMessage` (e.g. Azure Storage under concurrent load) can never starve the liveness write and get a healthy, busy worker reaped (issue #1064). |

**Why Redis for the heartbeat?** The previous design wrote `run.lastHeartbeatAt` to Mongo on every beat. With CosmosDB-compatible Mongo each beat costs ~10 RU, so an active run burns ~40 RU/min just to stay alive. The heartbeat is inherently ephemeral — it has no value past its TTL — so it lives in Redis. The API enriches `processing` runs on response with an MGET so the portal still sees `run.lastHeartbeatAt` (transient, not persisted in Mongo).
**Why Redis for the heartbeat?** The previous design wrote `run.lastHeartbeatAt` to Mongo on every beat. With CosmosDB-compatible Mongo each beat costs ~10 RU, so an active run burns ~40 RU/min just to stay alive. The heartbeat is inherently ephemeral — it has no value past its TTL — so it lives in Redis. The API enriches `processing` runs on response with the latest beat (a batch read) so the portal still sees `run.lastHeartbeatAt` (transient, not persisted in Mongo).

> **Shard-safe batch reads (clustered Redis).** The batch read in [`RedisHeartbeatStore.mget`](../../packages/shared/src/queue/heartbeat-store.ts) delegates to the [`clusterSafeMget`](../../packages/shared/src/queue/cluster-safe-mget.ts) helper, which issues a **pipeline of single-key `GET`s**, not a native multi-key `MGET`. On a clustered Redis (e.g. Azure Cache for Redis with the Enterprise clustering policy) a multi-key `MGET` whose keys span hash slots fails with `CROSSSLOT`; the store's error handling would swallow that into an *empty* map, which silently made every `processing` run look heartbeat-less — falsely reaping healthy runs and blanking heartbeats on the runs **list** view, while single-key reads (run detail / side panel) kept working (issue #1064). Single-key `GET`s are always shard-routable, so the pipeline is correct on standalone and clustered Redis alike and isolates a per-key error instead of discarding the whole batch. The same helper backs both consumers of the batch read — the stuck-run reaper sweep and the API runs-list enrichment — so they cannot drift.

When a worker dequeues a message whose `run.status === "processing"`, it consults Redis:

Expand All @@ -166,8 +168,9 @@ Queue redelivery is best-effort: if a message ever expires (Azure queue TTL, def
2. **Finds candidates** with an indexed query on `run.status: "processing"` (+ not soft-deleted) only — no `run.startedAt` range predicate, to stay on the single-field `run.status` index and avoid needing a range index on CosmosDB. The `startedAt` cutoff (older than `staleThresholdMs`) is applied **in memory** to the small candidate batch (missing `startedAt` is treated as old).
3. **Reads heartbeats** for the candidates and keeps only those whose beat is stale/missing using the *same* staleness rule as the redelivery handler.
4. **Two-strikes:** a run must look stale in **two consecutive sweeps** before it is reaped (`toReap = currentStale ∩ previouslyStale`), absorbing transient blips.
5. **Circuit-breaker:** if a single sweep would reap more than `SCOPE_REAPER_MAX_PER_SWEEP` (default 30) runs, it skips and logs loudly — a high count implies a systemic slowdown (e.g. CosmosDB 429 storm), not N independent worker deaths.
6. **Atomic claim** per run via `findOneAndUpdate` gated on `_id` + `run._id` + `run.status: "processing"` + the current `run.worker.instanceId` (or `{$exists:false}`). The loser of any reaper↔worker / cancel / retry race no-ops. On success it sets a terminal `failed` outcome with a **reaper-specific** `run.error` (distinct from the redelivery handler's "presumed dead" wording, so the two recovery paths are distinguishable in forensics) and deletes the Redis key. It leaves `postProcessorStatus` **unset** so the existing `PostProcessorDispatcher` enqueues post-processing (the reaper has no queue client — this avoids a set-then-send rollback hazard).
5. **Confirmation re-read:** immediately before failing a run, its heartbeat is re-read with a **single-key `GET`**. The sweep verdict comes from a batch read; this direct read is always shard-routable, so a fresh beat here aborts the reap regardless of why the batch read missed it (defense-in-depth against the clustered-Redis `MGET` failure above — issue #1064).
6. **Circuit-breaker:** if a single sweep would reap more than `SCOPE_REAPER_MAX_PER_SWEEP` (default 30) runs, it skips and logs loudly — a high count implies a systemic slowdown (e.g. CosmosDB 429 storm), not N independent worker deaths.
7. **Atomic claim** per run via `findOneAndUpdate` gated on `_id` + `run._id` + `run.status: "processing"` + the current `run.worker.instanceId` (or `{$exists:false}`). The loser of any reaper↔worker / cancel / retry race no-ops. On success it sets a terminal `failed` outcome with a **reaper-specific** `run.error` (distinct from the redelivery handler's "presumed dead" wording, so the two recovery paths are distinguishable in forensics) and deletes the Redis key. It leaves `postProcessorStatus` **unset** so the existing `PostProcessorDispatcher` enqueues post-processing (the reaper has no queue client — this avoids a set-then-send rollback hazard).

**Redis is non-fatal to the scheduler.** The reaper is constructed only when `SCOPE_REAPER_ENABLED !== "false"` *and* `REDIS_HOST` is set, wrapped in try/catch; a construction or connection failure self-disables the reaper while the dispatch loop keeps running, and the `/` health probe stays Redis-independent. A false positive (the reaper reaping a slow-but-alive owner) is **safe, not corrupting**: the worker's terminal success write is status-gated on `run.status: "processing"`, so once the reaper flips the run to `failed` the worker's final write no-ops. The cost is a wasted in-flight run (the user retries), never a double-fail or a revived run.

Expand Down
12 changes: 7 additions & 5 deletions packages/shared/src/queue/base-queue-processor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -264,11 +264,13 @@ export abstract class BaseQueueProcessor<TDocument extends { _id: string } = any
undefined,
undefined,
{ documentId, runId: logRunId },
// Bump the per-run liveness heartbeat in Redis on every successful
// tick so the redelivery handler can distinguish a real worker crash
// from a spurious queue redelivery. Stored in Redis (not Mongo) to
// avoid the recurring CosmosDB RU cost of writing every 15 s for
// every active run.
// Bump the per-run liveness heartbeat in Redis on a dedicated interval
// (decoupled from the queue-visibility extension) so the redelivery
// handler and StuckRunReaper can distinguish a real worker crash from a
// spurious queue redelivery — or from a transient Azure Queue
// visibility-extension failure under load (issue #1064). Stored in
// Redis (not Mongo) to avoid the recurring CosmosDB RU cost of writing
// every 15 s for every active run.
async () => {
await this.heartbeatStore.set(logRunId, new Date());
// Fallback cancel check: if the Pub/Sub message was missed (e.g.
Expand Down
99 changes: 99 additions & 0 deletions packages/shared/src/queue/cluster-safe-mget.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

import { describe, it, expect, vi } from "vitest";
import { clusterSafeMget, type MgetCapableRedis } from "./cluster-safe-mget.js";

/**
* Unit coverage for the cluster-safe multi-key read helper (issue #1064).
* The helper must read each key with a single-key GET pipeline — never a
* native cross-slot MGET — and recombine results aligned by input index,
* isolating per-key errors and misses.
*/

type ExecResult = Array<[Error | null, unknown]> | null;

/** Build a controllable fake Redis whose pipeline records the keys it GETs. */
function fakeRedis(exec: () => ExecResult | Promise<ExecResult>) {
const gets: string[] = [];
const pipeline = {
get: vi.fn((k: string) => {
gets.push(k);
return pipeline;
}),
exec: vi.fn(async () => exec()),
};
const redis = {
pipeline: vi.fn(() => pipeline),
// Present so a regression that calls native MGET would be visibly wrong.
mget: vi.fn(),
};
return { redis: redis as unknown as MgetCapableRedis, raw: redis, pipeline, gets };
}

describe("clusterSafeMget", () => {
it("reads each key with a single-key GET pipeline and never a multi-key MGET", async () => {
const { redis, raw, pipeline, gets } = fakeRedis(() => [
[null, "v1"],
[null, "v2"],
]);

const out = await clusterSafeMget(redis, ["k1", "k2"]);

expect(raw.pipeline).toHaveBeenCalledTimes(1);
expect(raw.mget).not.toHaveBeenCalled();
expect(pipeline.get).toHaveBeenCalledTimes(2);
expect(gets).toEqual(["k1", "k2"]);
expect(out).toEqual(["v1", "v2"]);
});

it("returns values aligned by input index", async () => {
const { redis } = fakeRedis(() => [
[null, "a"],
[null, "b"],
[null, "c"],
]);

const out = await clusterSafeMget(redis, ["k0", "k1", "k2"]);

expect(out).toEqual(["a", "b", "c"]);
});

it("degrades a missing key, a per-key error, and a non-string value to null", async () => {
const { redis } = fakeRedis(() => [
[null, "present"], // present
[null, null], // missing key
[new Error("MOVED 1234 10.0.0.2:6379"), null], // per-key redirect/error
[null, 42], // unexpected non-string value
]);

const out = await clusterSafeMget(redis, ["a", "b", "c", "d"]);

expect(out).toEqual(["present", null, null, null]);
});

it("returns all-null (same length) when the pipeline exec returns null", async () => {
const { redis } = fakeRedis(() => null);

const out = await clusterSafeMget(redis, ["a", "b", "c"]);

expect(out).toEqual([null, null, null]);
});

it("pads with null when exec returns fewer entries than keys", async () => {
const { redis } = fakeRedis(() => [[null, "only-one"]]);

const out = await clusterSafeMget(redis, ["a", "b"]);

expect(out).toEqual(["only-one", null]);
});

it("returns an empty array without touching Redis for an empty key list", async () => {
const { redis, raw } = fakeRedis(() => []);

const out = await clusterSafeMget(redis, []);

expect(out).toEqual([]);
expect(raw.pipeline).not.toHaveBeenCalled();
});
});
Loading