From 0a1a81c2dbb061f3153834779a8dc92e1d0d3a39 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Wed, 10 Jun 2026 17:18:38 -0700 Subject: [PATCH] fix(scheduler): stop StuckRunReaper from reaping healthy runs (#1064) The reaper falsely failed healthy, actively-progressing runs in integration with "stuck in 'processing' with stale/missing heartbeat". Root cause: RedisHeartbeatStore.mget used a native multi-key MGET. Integration Redis runs the Azure Cache Enterprise clustering policy, where a cross-slot MGET fails with CROSSSLOT; the store swallowed that into an empty map, so every processing run looked heartbeat-less and was reaped. Single-key GETs (run detail / side panel) kept working, which is why heartbeats looked fresh there while the reaper killed the run. The runs-list view was blank for the same reason. - Extract the canonical cluster-safe read into clusterSafeMget (pipeline of single-key GETs, recombined by index, per-key errors isolated). The store delegates to it, so the reaper sweep and the API runs-list enrichment share one tested implementation. - Add a single-key confirmation re-read in the reaper before failing a run, so a fresh beat aborts the reap regardless of why the batch read missed it, plus per-sweep diagnostics. - Decouple the liveness signal onto a dedicated setInterval so a slow/hung queue visibility extension can no longer starve heartbeats. Tests: new cluster-safe-mget (6) and heartbeat-store (4) suites, +2 reaper confirmation-read tests, +3 liveness-decoupling tests. shared/scheduler/api tsc clean. Fixes: #1064 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- apps/api/src/routes/requests/index.ts | 4 +- apps/scheduler/src/stuck-run-reaper.test.ts | 43 ++++++++ apps/scheduler/src/stuck-run-reaper.ts | 50 +++++++++- docs/architecture/queue-scheduler.md | 11 ++- .../shared/src/queue/base-queue-processor.ts | 12 ++- .../src/queue/cluster-safe-mget.test.ts | 99 +++++++++++++++++++ .../shared/src/queue/cluster-safe-mget.ts | 59 +++++++++++ .../shared/src/queue/heartbeat-store.test.ts | 88 +++++++++++++++++ packages/shared/src/queue/heartbeat-store.ts | 8 +- packages/shared/src/queue/index.ts | 1 + .../src/queue/visibility-heartbeat.test.ts | 59 +++++++++++ .../shared/src/queue/visibility-heartbeat.ts | 91 ++++++++++++----- 12 files changed, 483 insertions(+), 42 deletions(-) create mode 100644 packages/shared/src/queue/cluster-safe-mget.test.ts create mode 100644 packages/shared/src/queue/cluster-safe-mget.ts create mode 100644 packages/shared/src/queue/heartbeat-store.test.ts diff --git a/apps/api/src/routes/requests/index.ts b/apps/api/src/routes/requests/index.ts index cd3fa56f1..6e4c33b63 100644 --- a/apps/api/src/routes/requests/index.ts +++ b/apps/api/src/routes/requests/index.ts @@ -1134,7 +1134,9 @@ apiRoute(ctx.app, ctx.registry, { const data = resources.map((r) => ({ ...r, id: r._id })); // Enrich `processing` runs with the latest liveness heartbeat from - // Redis (single MGET; heartbeats live there, not Mongo). + // Redis. The store's mget is cluster-safe (a pipeline of single-key GETs, + // not a cross-slot MGET) so this works on clustered Redis too — see + // clusterSafeMget / issue #1064. Heartbeats live in Redis, not Mongo. const processingRunIds = data .filter((r) => r.run?.status === "processing" && r.run._id) .map((r) => r.run!._id!); diff --git a/apps/scheduler/src/stuck-run-reaper.test.ts b/apps/scheduler/src/stuck-run-reaper.test.ts index 2b18f9aaf..80e8af133 100644 --- a/apps/scheduler/src/stuck-run-reaper.test.ts +++ b/apps/scheduler/src/stuck-run-reaper.test.ts @@ -78,6 +78,49 @@ describe("StuckRunReaper", () => { expect(collection.findOneAndUpdate).toHaveBeenCalledTimes(1); }); + it("aborts the reap when a single-key confirmation read finds a fresh beat (cross-slot mget under-reported)", async () => { + // Reproduces issue #1064: on a clustered Redis the multi-key mget fails + // cross-slot and is swallowed into an empty map, so a healthy run looks + // heartbeat-less across both sweeps — but a single-key get (what the run + // detail view uses) still returns the fresh beat. The reaper must not fail + // the run. + const docs = [ + { _id: "req-1", run: { _id: "run-1", startedAt: OLD(), worker: { instanceId: "w-1" } } }, + ]; + const collection = mockCollection(docs); + const store = { + ping: vi.fn().mockResolvedValue(true), + mget: vi.fn().mockResolvedValue(new Map()), // cross-slot failure → empty + get: vi.fn().mockResolvedValue(new Date()), // single-key read works → fresh + delete: vi.fn(), + } as any; + const reaper = makeReaper(collection, store); + + await reaper.sweep(); // strike 1 (looks missing via mget) + await reaper.sweep(); // strike 2 → reapOne, but confirmation get() aborts + expect(store.get).toHaveBeenCalledWith("run-1"); + expect(collection.findOneAndUpdate).not.toHaveBeenCalled(); + expect(store.delete).not.toHaveBeenCalled(); + }); + + it("still reaps when the confirmation read also reports a stale/missing beat", async () => { + const docs = [ + { _id: "req-1", run: { _id: "run-1", startedAt: OLD(), worker: { instanceId: "w-1" } } }, + ]; + const collection = mockCollection(docs); + const store = { + ping: vi.fn().mockResolvedValue(true), + mget: vi.fn().mockResolvedValue(new Map()), + get: vi.fn().mockResolvedValue(null), // genuinely dead — no beat + delete: vi.fn(), + } as any; + const reaper = makeReaper(collection, store); + + await reaper.sweep(); + await reaper.sweep(); + expect(collection.findOneAndUpdate).toHaveBeenCalledTimes(1); + }); + it("does NOT reap a run with a fresh heartbeat", async () => { const docs = [ { _id: "req-1", run: { _id: "run-1", startedAt: OLD(), worker: { instanceId: "w-1" } } }, diff --git a/apps/scheduler/src/stuck-run-reaper.ts b/apps/scheduler/src/stuck-run-reaper.ts index 8f90a973a..f07bc483d 100644 --- a/apps/scheduler/src/stuck-run-reaper.ts +++ b/apps/scheduler/src/stuck-run-reaper.ts @@ -33,6 +33,9 @@ import type { HeartbeatStore, RequestDocument } from "shared"; * is reaped. A transient Redis read blip (which makes the whole fleet look * stale for one sweep) is absorbed because nothing was stale in the prior * sweep; only genuinely-dead runs persist across both. + * - Confirmation re-read: immediately before failing a run, its heartbeat is + * re-read with a single-key GET. A fresh beat here aborts the reap + * regardless of why the batch read missed it (issue #1064). * - Per-sweep circuit-breaker: if more runs would be reaped than * `maxPerSweep`, the sweep is skipped and logged loudly — a correlated * spike is almost certainly systemic (CosmosDB throttling, network) rather @@ -170,12 +173,41 @@ export class StuckRunReaper { return; } + // Zero readable beats for a non-empty candidate set points at a systemic + // heartbeat-read/Redis-misconfig problem, not N simultaneous worker deaths + // (genuine deaths still expose present-but-stale beats inside the TTL). + // Surface it loudly; the per-candidate confirmation read in reapOne() is + // the actual false-positive guard (issue #1064). + if (heartbeats.size === 0 && candidates.length > 0) { + console.warn( + `[StuckRunReaper] Read 0 live heartbeats for ${candidates.length} processing candidate(s) ` + + `while Redis is reachable — likely a heartbeat read failure or Redis misconfiguration, ` + + `not worker deaths. Each candidate is re-verified with a direct read before any reap. ` + + `Verify the scheduler's REDIS_HOST/PORT/PASSWORD/TLS match the workers and API.`, + ); + } + const now = Date.now(); + let missingCount = 0; + let staleCount = 0; const currentStale = candidates.filter((c) => { const hb = heartbeats.get(c.runId); - if (!hb) return true; // missing & startedAt already old (filtered in findCandidates) - return now - hb.getTime() > this.staleThresholdMs; + if (!hb) { + missingCount++; + return true; // missing & startedAt already old (filtered in findCandidates) + } + if (now - hb.getTime() > this.staleThresholdMs) { + staleCount++; + return true; + } + return false; }); + if (currentStale.length > 0) { + console.log( + `[StuckRunReaper] Sweep: ${candidates.length} candidate(s), ${heartbeats.size} live beat(s), ` + + `${staleCount} stale, ${missingCount} missing; ${this.previouslyStale.size} stale in prior sweep.`, + ); + } const currentStaleIds = new Set(currentStale.map((c) => c.runId)); // Two-strikes: only reap runs that were ALSO stale in the previous sweep. @@ -262,6 +294,20 @@ export class StuckRunReaper { } private async reapOne(c: Candidate, now: number): Promise { + // Confirmation re-read (issue #1064): a single-key GET is always + // shard-routable, so a fresh beat here means the worker is alive — abort the + // reap regardless of why the batch read under-reported it. + const confirmBeat = await this.heartbeatStore.get(c.runId); + if (confirmBeat && now - confirmBeat.getTime() <= this.staleThresholdMs) { + console.warn( + `[StuckRunReaper] Aborting reap of request=${c._id} (runId=${c.runId}) — confirmation read found a ` + + `fresh heartbeat (${Math.round((now - confirmBeat.getTime()) / 1000)}s ago). The batch heartbeat ` + + `read under-reported liveness (likely a cross-slot MGET failure on clustered Redis); not reaping.`, + ); + this.previouslyStale.delete(c.runId); + return; + } + const ageDesc = c.startedAt ? `${Math.round((now - c.startedAt.getTime()) / 1000)}s` : "unknown"; diff --git a/docs/architecture/queue-scheduler.md b/docs/architecture/queue-scheduler.md index 8f4e93841..c61b3a489 100644 --- a/docs/architecture/queue-scheduler.md +++ b/docs/architecture/queue-scheduler.md @@ -144,9 +144,11 @@ Azure Storage Queues guarantee at-least-once delivery, so a worker may receive a |-------|-------|------|---------| | Mongo (`run.*`) | `worker` | `{ instanceId, podName? }` | Identity of the worker process currently processing the run. `instanceId` is a per-process UUID; `podName` is the K8s pod name (`HOSTNAME`) when running in a pod. Stamped atomically at the `queued → processing` pickup. Persisted for forensics; never updated by heartbeat ticks. | | Mongo (`run.*`) | `startedAt` | `Date` | Wall-clock time of the pickup. Used as the missing-heartbeat fallback (see below). | -| Redis | `run-heartbeat:` | `Date` (ISO string, TTL ≈ 5×visibility) | Wall-clock time the owning worker last beat. Refreshed every 15s by the per-run heartbeat callback wired into [`startVisibilityHeartbeat`](../../packages/shared/src/queue/visibility-heartbeat.ts). | +| Redis | `run-heartbeat:` | `Date` (ISO string, TTL ≈ 5×visibility) | Wall-clock time the owning worker last beat. Refreshed every 15s by the per-run liveness loop in [`startVisibilityHeartbeat`](../../packages/shared/src/queue/visibility-heartbeat.ts). This loop runs on a **dedicated `setInterval`, fully decoupled from the queue-visibility extension** — so a transient/slow `queueClient.updateMessage` (e.g. Azure Storage under concurrent load) can never starve the liveness write and get a healthy, busy worker reaped (issue #1064). | -**Why Redis for the heartbeat?** The previous design wrote `run.lastHeartbeatAt` to Mongo on every beat. With CosmosDB-compatible Mongo each beat costs ~10 RU, so an active run burns ~40 RU/min just to stay alive. The heartbeat is inherently ephemeral — it has no value past its TTL — so it lives in Redis. The API enriches `processing` runs on response with an MGET so the portal still sees `run.lastHeartbeatAt` (transient, not persisted in Mongo). +**Why Redis for the heartbeat?** The previous design wrote `run.lastHeartbeatAt` to Mongo on every beat. With CosmosDB-compatible Mongo each beat costs ~10 RU, so an active run burns ~40 RU/min just to stay alive. The heartbeat is inherently ephemeral — it has no value past its TTL — so it lives in Redis. The API enriches `processing` runs on response with the latest beat (a batch read) so the portal still sees `run.lastHeartbeatAt` (transient, not persisted in Mongo). + +> **Shard-safe batch reads (clustered Redis).** The batch read in [`RedisHeartbeatStore.mget`](../../packages/shared/src/queue/heartbeat-store.ts) delegates to the [`clusterSafeMget`](../../packages/shared/src/queue/cluster-safe-mget.ts) helper, which issues a **pipeline of single-key `GET`s**, not a native multi-key `MGET`. On a clustered Redis (e.g. Azure Cache for Redis with the Enterprise clustering policy) a multi-key `MGET` whose keys span hash slots fails with `CROSSSLOT`; the store's error handling would swallow that into an *empty* map, which silently made every `processing` run look heartbeat-less — falsely reaping healthy runs and blanking heartbeats on the runs **list** view, while single-key reads (run detail / side panel) kept working (issue #1064). Single-key `GET`s are always shard-routable, so the pipeline is correct on standalone and clustered Redis alike and isolates a per-key error instead of discarding the whole batch. The same helper backs both consumers of the batch read — the stuck-run reaper sweep and the API runs-list enrichment — so they cannot drift. When a worker dequeues a message whose `run.status === "processing"`, it consults Redis: @@ -166,8 +168,9 @@ Queue redelivery is best-effort: if a message ever expires (Azure queue TTL, def 2. **Finds candidates** with an indexed query on `run.status: "processing"` (+ not soft-deleted) only — no `run.startedAt` range predicate, to stay on the single-field `run.status` index and avoid needing a range index on CosmosDB. The `startedAt` cutoff (older than `staleThresholdMs`) is applied **in memory** to the small candidate batch (missing `startedAt` is treated as old). 3. **Reads heartbeats** for the candidates and keeps only those whose beat is stale/missing using the *same* staleness rule as the redelivery handler. 4. **Two-strikes:** a run must look stale in **two consecutive sweeps** before it is reaped (`toReap = currentStale ∩ previouslyStale`), absorbing transient blips. -5. **Circuit-breaker:** if a single sweep would reap more than `SCOPE_REAPER_MAX_PER_SWEEP` (default 30) runs, it skips and logs loudly — a high count implies a systemic slowdown (e.g. CosmosDB 429 storm), not N independent worker deaths. -6. **Atomic claim** per run via `findOneAndUpdate` gated on `_id` + `run._id` + `run.status: "processing"` + the current `run.worker.instanceId` (or `{$exists:false}`). The loser of any reaper↔worker / cancel / retry race no-ops. On success it sets a terminal `failed` outcome with a **reaper-specific** `run.error` (distinct from the redelivery handler's "presumed dead" wording, so the two recovery paths are distinguishable in forensics) and deletes the Redis key. It leaves `postProcessorStatus` **unset** so the existing `PostProcessorDispatcher` enqueues post-processing (the reaper has no queue client — this avoids a set-then-send rollback hazard). +5. **Confirmation re-read:** immediately before failing a run, its heartbeat is re-read with a **single-key `GET`**. The sweep verdict comes from a batch read; this direct read is always shard-routable, so a fresh beat here aborts the reap regardless of why the batch read missed it (defense-in-depth against the clustered-Redis `MGET` failure above — issue #1064). +6. **Circuit-breaker:** if a single sweep would reap more than `SCOPE_REAPER_MAX_PER_SWEEP` (default 30) runs, it skips and logs loudly — a high count implies a systemic slowdown (e.g. CosmosDB 429 storm), not N independent worker deaths. +7. **Atomic claim** per run via `findOneAndUpdate` gated on `_id` + `run._id` + `run.status: "processing"` + the current `run.worker.instanceId` (or `{$exists:false}`). The loser of any reaper↔worker / cancel / retry race no-ops. On success it sets a terminal `failed` outcome with a **reaper-specific** `run.error` (distinct from the redelivery handler's "presumed dead" wording, so the two recovery paths are distinguishable in forensics) and deletes the Redis key. It leaves `postProcessorStatus` **unset** so the existing `PostProcessorDispatcher` enqueues post-processing (the reaper has no queue client — this avoids a set-then-send rollback hazard). **Redis is non-fatal to the scheduler.** The reaper is constructed only when `SCOPE_REAPER_ENABLED !== "false"` *and* `REDIS_HOST` is set, wrapped in try/catch; a construction or connection failure self-disables the reaper while the dispatch loop keeps running, and the `/` health probe stays Redis-independent. A false positive (the reaper reaping a slow-but-alive owner) is **safe, not corrupting**: the worker's terminal success write is status-gated on `run.status: "processing"`, so once the reaper flips the run to `failed` the worker's final write no-ops. The cost is a wasted in-flight run (the user retries), never a double-fail or a revived run. diff --git a/packages/shared/src/queue/base-queue-processor.ts b/packages/shared/src/queue/base-queue-processor.ts index da7fc51c3..ed1420b59 100644 --- a/packages/shared/src/queue/base-queue-processor.ts +++ b/packages/shared/src/queue/base-queue-processor.ts @@ -264,11 +264,13 @@ export abstract class BaseQueueProcessor { await this.heartbeatStore.set(logRunId, new Date()); // Fallback cancel check: if the Pub/Sub message was missed (e.g. diff --git a/packages/shared/src/queue/cluster-safe-mget.test.ts b/packages/shared/src/queue/cluster-safe-mget.test.ts new file mode 100644 index 000000000..16f2501f7 --- /dev/null +++ b/packages/shared/src/queue/cluster-safe-mget.test.ts @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, it, expect, vi } from "vitest"; +import { clusterSafeMget, type MgetCapableRedis } from "./cluster-safe-mget.js"; + +/** + * Unit coverage for the cluster-safe multi-key read helper (issue #1064). + * The helper must read each key with a single-key GET pipeline — never a + * native cross-slot MGET — and recombine results aligned by input index, + * isolating per-key errors and misses. + */ + +type ExecResult = Array<[Error | null, unknown]> | null; + +/** Build a controllable fake Redis whose pipeline records the keys it GETs. */ +function fakeRedis(exec: () => ExecResult | Promise) { + const gets: string[] = []; + const pipeline = { + get: vi.fn((k: string) => { + gets.push(k); + return pipeline; + }), + exec: vi.fn(async () => exec()), + }; + const redis = { + pipeline: vi.fn(() => pipeline), + // Present so a regression that calls native MGET would be visibly wrong. + mget: vi.fn(), + }; + return { redis: redis as unknown as MgetCapableRedis, raw: redis, pipeline, gets }; +} + +describe("clusterSafeMget", () => { + it("reads each key with a single-key GET pipeline and never a multi-key MGET", async () => { + const { redis, raw, pipeline, gets } = fakeRedis(() => [ + [null, "v1"], + [null, "v2"], + ]); + + const out = await clusterSafeMget(redis, ["k1", "k2"]); + + expect(raw.pipeline).toHaveBeenCalledTimes(1); + expect(raw.mget).not.toHaveBeenCalled(); + expect(pipeline.get).toHaveBeenCalledTimes(2); + expect(gets).toEqual(["k1", "k2"]); + expect(out).toEqual(["v1", "v2"]); + }); + + it("returns values aligned by input index", async () => { + const { redis } = fakeRedis(() => [ + [null, "a"], + [null, "b"], + [null, "c"], + ]); + + const out = await clusterSafeMget(redis, ["k0", "k1", "k2"]); + + expect(out).toEqual(["a", "b", "c"]); + }); + + it("degrades a missing key, a per-key error, and a non-string value to null", async () => { + const { redis } = fakeRedis(() => [ + [null, "present"], // present + [null, null], // missing key + [new Error("MOVED 1234 10.0.0.2:6379"), null], // per-key redirect/error + [null, 42], // unexpected non-string value + ]); + + const out = await clusterSafeMget(redis, ["a", "b", "c", "d"]); + + expect(out).toEqual(["present", null, null, null]); + }); + + it("returns all-null (same length) when the pipeline exec returns null", async () => { + const { redis } = fakeRedis(() => null); + + const out = await clusterSafeMget(redis, ["a", "b", "c"]); + + expect(out).toEqual([null, null, null]); + }); + + it("pads with null when exec returns fewer entries than keys", async () => { + const { redis } = fakeRedis(() => [[null, "only-one"]]); + + const out = await clusterSafeMget(redis, ["a", "b"]); + + expect(out).toEqual(["only-one", null]); + }); + + it("returns an empty array without touching Redis for an empty key list", async () => { + const { redis, raw } = fakeRedis(() => []); + + const out = await clusterSafeMget(redis, []); + + expect(out).toEqual([]); + expect(raw.pipeline).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/shared/src/queue/cluster-safe-mget.ts b/packages/shared/src/queue/cluster-safe-mget.ts new file mode 100644 index 000000000..90759e9d7 --- /dev/null +++ b/packages/shared/src/queue/cluster-safe-mget.ts @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Cluster-safe multi-key read for Redis. + * + * A native `MGET` is a single multi-key command, so on a clustered Redis (e.g. + * Azure Cache for Redis Enterprise clustering policy) keys that hash to + * different slots make it fail with `CROSSSLOT`. This helper instead issues one + * single-key `GET` per key inside a pipeline (one round-trip). Single-key + * commands are always routable to their owning slot, so this is correct on + * standalone and clustered Redis alike. Results are aligned by index with + * `keys`; a per-key error or non-string value degrades that one entry to `null` + * rather than discarding the whole batch. + * + * Swallowing a cross-slot `MGET` failure into "all missing" is what made the + * stuck-run reaper falsely reap healthy runs and blanked the runs-list + * heartbeats (issue #1064). Canonical pattern per StackExchange.Redis#1650: + * split a cross-slot multi-key op into per-key commands and recombine by index. + */ + +/** Minimal pipeline surface used by {@link clusterSafeMget}. */ +export interface MgetPipeline { + get(key: string): unknown; + exec(): Promise | null>; +} + +/** Minimal Redis client surface used by {@link clusterSafeMget}. */ +export interface MgetCapableRedis { + pipeline(): MgetPipeline; +} + +/** + * Read multiple keys in a single round-trip without a cross-slot `MGET`. + * + * @returns An array the same length and order as `keys`. Each entry is the + * string value for a present key, or `null` when the key is missing, the + * value is not a string, or that key's command errored. If the pipeline + * itself returns `null` (e.g. the connection was torn down), every entry is + * `null`. + */ +export async function clusterSafeMget( + redis: MgetCapableRedis, + keys: string[], +): Promise> { + if (keys.length === 0) return []; + + const pipeline = redis.pipeline(); + for (const key of keys) pipeline.get(key); + const results = await pipeline.exec(); + if (!results) return keys.map(() => null); + + return keys.map((_, i) => { + const entry = results[i]; + if (!entry) return null; + const [err, value] = entry; + return err || typeof value !== "string" ? null : value; + }); +} diff --git a/packages/shared/src/queue/heartbeat-store.test.ts b/packages/shared/src/queue/heartbeat-store.test.ts new file mode 100644 index 000000000..1c81e84f7 --- /dev/null +++ b/packages/shared/src/queue/heartbeat-store.test.ts @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, it, expect, vi } from "vitest"; +import { RedisHeartbeatStore } from "./heartbeat-store.js"; + +/** + * Regression coverage for issue #1064: the heartbeat batch read must NOT use a + * native multi-key MGET. On a clustered Redis a cross-slot MGET fails and is + * swallowed into an empty map, which made the stuck-run reaper falsely reap + * healthy runs and dropped heartbeats from the runs list view. The store reads + * each key with a single-key GET pipeline, which is always shard-routable. + */ + +type ExecResult = Array<[Error | null, unknown]> | null; + +/** Build a store whose internal Redis client is a controllable fake. */ +function storeWithFakeRedis(exec: () => ExecResult | Promise) { + const store = new RedisHeartbeatStore({ + redisHost: "localhost", + redisPort: 6379, + redisPassword: "", + }); + // Drop the real client created in the constructor so it doesn't leak a + // connection attempt, then swap in the fake. + const real = (store as any).redis; + real?.disconnect?.(); + + const gets: string[] = []; + const pipeline: any = { + get: vi.fn((k: string) => { + gets.push(k); + return pipeline; + }), + exec: vi.fn(async () => exec()), + }; + const fake = { + pipeline: vi.fn(() => pipeline), + mget: vi.fn(), // must never be called — MGET is not shard-safe + on: vi.fn(), + }; + (store as any).redis = fake; + return { store, fake, pipeline, gets }; +} + +describe("RedisHeartbeatStore.mget", () => { + it("reads each key with a single-key GET pipeline and never a multi-key MGET", async () => { + const ts = new Date("2026-01-01T00:00:00.000Z"); + const { store, fake, gets } = storeWithFakeRedis(() => [[null, ts.toISOString()]]); + + const out = await store.mget(["run-1"]); + + expect(fake.pipeline).toHaveBeenCalledTimes(1); + expect(fake.mget).not.toHaveBeenCalled(); + expect(gets).toEqual(["run-heartbeat:run-1"]); + expect(out.get("run-1")).toEqual(ts); + }); + + it("recombines results by index and isolates per-key errors and misses", async () => { + const ts = new Date("2026-02-02T12:00:00.000Z"); + const { store, gets } = storeWithFakeRedis(() => [ + [null, ts.toISOString()], // present + [null, null], // missing key + [new Error("MOVED 1234 10.0.0.2:6379"), null], // per-key redirect/error + ]); + + const out = await store.mget(["a", "b", "c"]); + + expect(gets).toEqual(["run-heartbeat:a", "run-heartbeat:b", "run-heartbeat:c"]); + expect(out.size).toBe(1); + expect(out.get("a")).toEqual(ts); + expect(out.has("b")).toBe(false); + expect(out.has("c")).toBe(false); + }); + + it("returns an empty map (and does not throw) when the pipeline exec returns null", async () => { + const { store } = storeWithFakeRedis(() => null); + const out = await store.mget(["a", "b"]); + expect(out.size).toBe(0); + }); + + it("returns an empty map without touching Redis for an empty id list", async () => { + const { store, fake } = storeWithFakeRedis(() => []); + const out = await store.mget([]); + expect(out.size).toBe(0); + expect(fake.pipeline).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/shared/src/queue/heartbeat-store.ts b/packages/shared/src/queue/heartbeat-store.ts index a33f4b1e3..40f5159dc 100644 --- a/packages/shared/src/queue/heartbeat-store.ts +++ b/packages/shared/src/queue/heartbeat-store.ts @@ -6,6 +6,7 @@ const require = createRequire(import.meta.url); const Redis = require("ioredis"); import type { RedisConfig } from "../logging/log-publisher.js"; +import { clusterSafeMget } from "./cluster-safe-mget.js"; /** * Per-run liveness heartbeat storage. @@ -162,9 +163,10 @@ export class RedisHeartbeatStore implements HeartbeatStore { const out = new Map(); if (runIds.length === 0) return out; try { - const vals: (string | null)[] = await this.redis.mget(runIds.map(keyFor)); - vals.forEach((v, i) => { - if (!v) return; + // Cluster-safe batch read (see {@link clusterSafeMget}, issue #1064). + const values = await clusterSafeMget(this.redis, runIds.map(keyFor)); + values.forEach((v, i) => { + if (typeof v !== "string") return; const ms = Date.parse(v); if (Number.isFinite(ms)) out.set(runIds[i], new Date(ms)); }); diff --git a/packages/shared/src/queue/index.ts b/packages/shared/src/queue/index.ts index bcc63fb1e..fffc9a4db 100644 --- a/packages/shared/src/queue/index.ts +++ b/packages/shared/src/queue/index.ts @@ -5,3 +5,4 @@ export * from "./base-queue-processor.js"; export * from "./queue-processor.js"; export * from "./visibility-heartbeat.js"; export * from "./heartbeat-store.js"; +export * from "./cluster-safe-mget.js"; diff --git a/packages/shared/src/queue/visibility-heartbeat.test.ts b/packages/shared/src/queue/visibility-heartbeat.test.ts index 6f43c90f9..7b702df1b 100644 --- a/packages/shared/src/queue/visibility-heartbeat.test.ts +++ b/packages/shared/src/queue/visibility-heartbeat.test.ts @@ -130,6 +130,65 @@ describe("visibility heartbeat", () => { hb.stop(); }); + it("fires the liveness onTick on a dedicated interval even when updateMessage keeps failing", async () => { + const qc = createMockQueueClient(); + // Visibility extension fails on every tick — liveness must still fire. + (qc.updateMessage as ReturnType).mockRejectedValue(new Error("transient 500")); + const onTick = vi.fn().mockResolvedValue(undefined); + + const hb = startVisibilityHeartbeat( + qc, "msg-1", "receipt-0", "test-worker", 100, 120, {}, onTick, + ); + + await vi.advanceTimersByTimeAsync(350); + + // updateMessage failed every time, but liveness kept beating. + expect(qc.updateMessage).toHaveBeenCalled(); + expect(onTick).toHaveBeenCalledTimes(3); + expect(onTick).toHaveBeenLastCalledWith({ tickCount: 3 }); + + hb.stop(); + }); + + it("keeps beating liveness when a single onTick rejects (and does not abort)", async () => { + const qc = createMockQueueClient(); + (qc.updateMessage as ReturnType).mockResolvedValue({ popReceipt: "receipt-1" }); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const onTick = vi + .fn() + .mockRejectedValueOnce(new Error("redis blip")) + .mockResolvedValue(undefined); + + const hb = startVisibilityHeartbeat( + qc, "msg-1", "receipt-0", "test-worker", 100, 120, {}, onTick, + ); + + await vi.advanceTimersByTimeAsync(250); + + expect(onTick).toHaveBeenCalledTimes(2); + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining("Liveness heartbeat tick failed"), + expect.any(Error), + ); + + hb.stop(); + warnSpy.mockRestore(); + }); + + it("does not fire onTick after stop", async () => { + const qc = createMockQueueClient(); + (qc.updateMessage as ReturnType).mockResolvedValue({ popReceipt: "receipt-1" }); + const onTick = vi.fn().mockResolvedValue(undefined); + + const hb = startVisibilityHeartbeat( + qc, "msg-1", "receipt-0", "test-worker", 100, 120, {}, onTick, + ); + hb.stop(); + + await vi.advanceTimersByTimeAsync(500); + expect(onTick).not.toHaveBeenCalled(); + }); + it("exports correct default constants", () => { expect(HEARTBEAT_INTERVAL_MS).toBe(15_000); expect(HEARTBEAT_VISIBILITY_SECONDS).toBe(60); diff --git a/packages/shared/src/queue/visibility-heartbeat.ts b/packages/shared/src/queue/visibility-heartbeat.ts index da318fab6..534e13fc3 100644 --- a/packages/shared/src/queue/visibility-heartbeat.ts +++ b/packages/shared/src/queue/visibility-heartbeat.ts @@ -31,23 +31,34 @@ export interface HeartbeatLogContext { } /** - * Optional callback invoked on every successful heartbeat tick. Used by the - * queue processor to also bump a per-run liveness timestamp in MongoDB so - * the redelivery handler can tell a real worker crash apart from a spurious - * Azure Storage Queue redelivery while the original worker is still alive. - * - * The callback is awaited but its errors are caught and logged — a Mongo - * write failure must never abort the visibility extension itself. + * Optional callback invoked on every liveness tick. Used by the queue + * processor to bump a per-run liveness timestamp in Redis so the redelivery + * handler and the StuckRunReaper can tell a real worker crash apart from a + * spurious redelivery while the original worker is still alive. Runs on a + * dedicated interval (see {@link startVisibilityHeartbeat}); its errors are + * caught and logged so a Redis write failure never aborts the loops. */ export type HeartbeatTickCallback = (tickInfo: { tickCount: number }) => Promise | void; /** - * Start a background loop that periodically extends a queue message's - * visibility timeout via `QueueClient.updateMessage`. This keeps the message - * hidden from other workers as long as this process is alive. If the worker - * crashes, the heartbeat dies and the message reappears after at most - * {@link HEARTBEAT_VISIBILITY_SECONDS} (default 30 s) instead of the - * previous 35-minute single-shot extension. + * Start the heartbeat for an in-flight queue message. This drives two + * INDEPENDENT loops so that a failure in one can never starve the other: + * + * 1. Visibility loop — periodically extends the queue message's visibility + * timeout via `QueueClient.updateMessage`, keeping the message hidden from + * other workers as long as this process is alive. If the worker crashes, + * the heartbeat dies and the message reappears after at most + * {@link HEARTBEAT_VISIBILITY_SECONDS} instead of the previous 35-minute + * single-shot extension. + * + * 2. Liveness loop — when `onTick` is provided, fires it on a DEDICATED + * `setInterval` to record per-run liveness (a Redis timestamp the reaper / + * redelivery handler read). This is deliberately decoupled from the + * visibility extension: a transient `updateMessage` failure under + * concurrent load (or a slow extension await) must NOT prevent the + * liveness write, otherwise a healthy, busy worker would be reaped as + * "stale heartbeat" (see issue #1064). Each loop tolerates the other + * failing. * * @returns A handle to stop the heartbeat and retrieve the latest pop receipt. */ @@ -64,8 +75,11 @@ export function startVisibilityHeartbeat( let popReceipt = initialPopReceipt; let tickCount = 0; let failureCount = 0; + let livenessTickCount = 0; + let livenessInFlight = false; const abort = new AbortController(); const startedAt = Date.now(); + let livenessTimer: ReturnType | undefined; // Build a stable `key=value` suffix so log lines are easy to grep / parse. // Always includes messageId; documentId and runId are added when known. @@ -75,9 +89,12 @@ export function startVisibilityHeartbeat( const ctx = ctxParts.join(" "); console.log( - `[${workerName}] Visibility heartbeat started (every ${intervalMs / 1000}s, extending by ${visibilityTimeoutSeconds}s) ${ctx}`, + `[${workerName}] Visibility heartbeat started (every ${intervalMs / 1000}s, extending by ${visibilityTimeoutSeconds}s${onTick ? ", liveness on dedicated interval" : ""}) ${ctx}`, ); + // Loop 1 — queue visibility extension. Owns only the pop receipt; it no + // longer drives the liveness write so a failed/slow updateMessage cannot + // starve liveness. const loop = async () => { while (!abort.signal.aborted) { await new Promise((resolve) => { @@ -95,18 +112,6 @@ export function startVisibilityHeartbeat( console.log( `[${workerName}] Visibility heartbeat tick #${tickCount} extended by ${visibilityTimeoutSeconds}s ${ctx}`, ); - if (onTick) { - try { - await onTick({ tickCount }); - } catch (cbError) { - // Never let a callback failure abort the heartbeat loop — the - // queue visibility extension already succeeded. - console.warn( - `[${workerName}] Visibility heartbeat onTick callback failed ${ctx}:`, - cbError, - ); - } - } } catch (error) { if (abort.signal.aborted) break; failureCount++; @@ -120,6 +125,34 @@ export function startVisibilityHeartbeat( loop().catch(() => {}); // fire-and-forget + // Loop 2 — per-run liveness. Independent timer so it keeps writing even when + // the visibility extension is failing or its await is slow. A re-entrancy + // guard prevents overlapping ticks if a single onTick runs long. + if (onTick) { + const runLiveness = async () => { + if (abort.signal.aborted || livenessInFlight) return; + livenessInFlight = true; + const seq = livenessTickCount + 1; + try { + await onTick({ tickCount: seq }); + livenessTickCount = seq; + console.log( + `[${workerName}] Liveness heartbeat tick #${livenessTickCount} ${ctx}`, + ); + } catch (cbError) { + // Never let a callback failure abort the heartbeat — surface it so a + // persistently-failing liveness write is visible in pod stdout. + console.warn( + `[${workerName}] Liveness heartbeat tick failed ${ctx}:`, + cbError, + ); + } finally { + livenessInFlight = false; + } + }; + livenessTimer = setInterval(() => { void runLiveness(); }, intervalMs); + } + return { stop: () => { if (abort.signal.aborted) { @@ -130,9 +163,13 @@ export function startVisibilityHeartbeat( return popReceipt; } abort.abort(); + if (livenessTimer) { + clearInterval(livenessTimer); + livenessTimer = undefined; + } const elapsedMs = Date.now() - startedAt; console.log( - `[${workerName}] Visibility heartbeat stopped after ${tickCount} tick(s), ${failureCount} consecutive failure(s), elapsed=${(elapsedMs / 1000).toFixed(1)}s ${ctx}`, + `[${workerName}] Visibility heartbeat stopped after ${tickCount} tick(s), ${failureCount} consecutive failure(s), ${livenessTickCount} liveness tick(s), elapsed=${(elapsedMs / 1000).toFixed(1)}s ${ctx}`, ); return popReceipt; },