feat(relay): measure the managed tunnel backlog

infra/relay/package.json

@@ -6,6 +6,7 @@
     "deploy": "alchemy deploy",
     "push:android:smoke": "node scripts/android-push-smoke.ts",
     "push:android:watch": "node scripts/android-push-watch.ts",
+    "tunnels:census": "node scripts/tunnel-census.ts",
     "destroy": "alchemy destroy",
     "test": "vp test run --config ../../vite.config.ts --dir .",
     "typecheck": "tsc --noEmit"

infra/relay/scripts/tunnel-census.ts

@@ -0,0 +1,97 @@
+// Read-only count of Cloudflare tunnels in the managed-endpoint account,
+// grouped by relay stage, status, and how long they have been idle. The reaper
+// only sees its own stage's prefix, so this shows how much of the backlog
+// belongs to other stages.
+//
+//   CLOUDFLARE_ACCOUNT_ID=... CLOUDFLARE_API_TOKEN=... node scripts/tunnel-census.ts
+//
+// The token needs Cloudflare Tunnel read access. Nothing is modified.
+import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
+import * as Clock from "effect/Clock";
+import * as Config from "effect/Config";
+import * as Console from "effect/Console";
+import * as Effect from "effect/Effect";
+import * as Redacted from "effect/Redacted";
+import * as Schema from "effect/Schema";
+import * as FetchHttpClient from "effect/http/FetchHttpClient";
+import * as HttpClient from "effect/http/HttpClient";
+import * as HttpClientRequest from "effect/http/HttpClientRequest";
+import * as HttpClientResponse from "effect/http/HttpClientResponse";
+
+const PREFIX = "t3coderelay-managedendpoint-";
+const PAGE_SIZE = 1_000;
+const DAY_MS = 86_400_000;
+
+// Cloudflare documents every field here as optional, so a tunnel missing one
+// must not fail the whole page.
+const Tunnel = Schema.Struct({
+  name: Schema.optional(Schema.NullOr(Schema.String)),
+  status: Schema.optional(Schema.NullOr(Schema.String)),
+  created_at: Schema.optional(Schema.NullOr(Schema.String)),
+  conns_inactive_at: Schema.optional(Schema.NullOr(Schema.String)),
+});
+type Tunnel = typeof Tunnel.Type;
+const TunnelPage = Schema.Struct({ result: Schema.Array(Tunnel) });
+
+// "t3coderelay-managedendpoint-<stage>-<16 hex>"; stages may contain hyphens.
+const stageOf = (name: string | null | undefined) =>
+  name ? name.slice(PREFIX.length).replace(/-[a-f0-9]{16}$/u, "") : "unknown";
+
+// Like the reaper: a down tunnel is aged from when it lost its connector, and a
+// never-connected (inactive) one from when it was created.
+const ageBucket = (tunnel: Tunnel, now: number) => {
+  const since = Date.parse(
+    (tunnel.status === "down" ? tunnel.conns_inactive_at : tunnel.created_at) ?? "",
+  );
+  if (Number.isNaN(since)) return "unknown";
+  const days = (now - since) / DAY_MS;
+  if (days > 90) return ">90d";
+  if (days > 30) return ">30d";
+  if (days > 7) return ">7d";
+  return "<=7d";
+};
+
+const main = Effect.gen(function* () {
+  const accountId = yield* Config.String("CLOUDFLARE_ACCOUNT_ID");
+  const token = yield* Config.Redacted("CLOUDFLARE_API_TOKEN");
+  const client = yield* HttpClient.HttpClient;
+  const now = yield* Clock.currentTimeMillis;
+
+  const listPage = (page: number) =>
+    HttpClientRequest.get(`https://api.cloudflare.com/client/v4/accounts/${accountId}/cfd_tunnel`, {
+      urlParams: {
+        is_deleted: "false",
+        include_prefix: PREFIX,
+        per_page: String(PAGE_SIZE),
+        page: String(page),
+      },
+    }).pipe(
+      HttpClientRequest.bearerToken(Redacted.value(token)),
+      client.execute,
+      Effect.flatMap(HttpClientResponse.filterStatusOk),
+      Effect.flatMap(HttpClientResponse.schemaBodyJson(TunnelPage)),
+      Effect.map((body) => body.result),
+    );
+
+  const counts = new Map<string, number>();
+  let total = 0;
+  for (let page = 1; ; page += 1) {
+    const tunnels = yield* listPage(page);
+    for (const tunnel of tunnels) {
+      total += 1;
+      const idle =
+        tunnel.status === "down" || tunnel.status === "inactive" ? ageBucket(tunnel, now) : "-";
+      const key = `${stageOf(tunnel.name)}\t${tunnel.status ?? "unknown"}\t${idle}`;
+      counts.set(key, (counts.get(key) ?? 0) + 1);
+    }
+    if (tunnels.length < PAGE_SIZE) break;
+  }
+
+  yield* Console.log("stage\tstatus\tidle\tcount");
+  for (const [key, count] of [...counts].sort(([a], [b]) => a.localeCompare(b))) {
+    yield* Console.log(`${key}\t${count}`);
+  }
+  yield* Console.log(`total\t\t\t${total}`);
+}).pipe(Effect.provide(FetchHttpClient.layer));
+
+NodeRuntime.runMain(main);

infra/relay/src/environments/ManagedEndpointReaper.test.ts

@@ -436,6 +436,60 @@ describe("ManagedEndpointReaper", () => {
     }).pipe(Effect.provide(state.layer));
   });
 
+  it.effect("measures legacy age, uncounted skips, and Cloudflare totals", () => {
+    const down = (id: string, suffix: string, timestamp: string) =>
+      tunnel({ id, suffix, status: "down", timestamp });
+    const tunnels = [
+      // Legacy owners, down for 1, 10, 40, and 100 days, plus one just past
+      // seven days, which must already count as over seven.
+      down("legacy-1d", "1111111111111111", "2026-08-24T11:00:00.000Z"),
+      down("legacy-7d1h", "7777777777777777", "2026-08-18T11:00:00.000Z"),
+      down("legacy-10d", "2222222222222222", "2026-08-15T11:00:00.000Z"),
+      down("legacy-40d", "3333333333333333", "2026-07-16T11:00:00.000Z"),
+      down("legacy-100d", "4444444444444444", "2026-05-17T11:00:00.000Z"),
+      // The allocation now records a different tunnel under this name.
+      down("stale", "5555555555555555", "2026-08-25T11:00:00.000Z"),
+      // The allocation has not recorded a tunnel yet.
+      down("pending", "6666666666666666", "2026-08-25T11:00:00.000Z"),
+    ];
+    const state = harness({
+      tunnels,
+      allocations: [
+        allocation({ tunnelId: "legacy-1d", recoveryEnabled: false }),
+        allocation({ tunnelId: "legacy-7d1h", recoveryEnabled: false }),
+        allocation({ tunnelId: "legacy-10d", recoveryEnabled: false }),
+        allocation({ tunnelId: "legacy-40d", recoveryEnabled: false }),
+        allocation({ tunnelId: "legacy-100d", recoveryEnabled: false }),
+        {
+          ...allocation({ tunnelId: "current", recoveryEnabled: true }),
+          tunnelName: `${PREFIX}5555555555555555`,
+        },
+        {
+          ...allocation({ tunnelId: null, recoveryEnabled: false }),
+          tunnelName: `${PREFIX}6666666666666666`,
+        },
+      ],
+      cleanupMode: "dry-run",
+    });
+
+    return Effect.gen(function* () {
+      yield* TestClock.setTime(NOW_MILLIS);
+      const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
+      expect(yield* reaper.sweep).toMatchObject({
+        scanned: 7,
+        skippedLegacy: 5,
+        legacyOver7Days: 4,
+        legacyOver30Days: 2,
+        legacyOver90Days: 1,
+        skippedReplaced: 1,
+        skippedUnrecorded: 1,
+        totalDown: 7,
+        totalInactive: 0,
+        wouldDelete: 0,
+      });
+    }).pipe(Effect.provide(state.layer));
+  });
+
   it.effect("does not count a tunnel that was replaced before its release", () => {
     const state = harness({
       tunnels: [
@@ -649,7 +703,14 @@ describe("ManagedEndpointReaper", () => {
         deleted: 0,
         wouldDelete: 0,
         skippedLegacy: 0,
+        legacyOver7Days: 0,
+        legacyOver30Days: 0,
+        legacyOver90Days: 0,
         skippedOrphan: 0,
+        skippedReplaced: 0,
+        skippedUnrecorded: 0,
+        totalDown: null,
+        totalInactive: null,
         failed: 0,
         truncated: false,
       });

infra/relay/src/environments/ManagedEndpointReaper.ts

@@ -16,6 +16,9 @@ export const MANAGED_ENDPOINT_INACTIVE_GRACE_PERIOD_MINUTES = 60;
 export const MANAGED_ENDPOINT_SWEEP_PAGE_SIZE = 100;
 export const MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT = 100;
 export const MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT = 10;
+// Age buckets for legacy candidates, in days since the tunnel went down (or
+// was created, for one that never connected).
+const MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS = [7, 30, 90] as const;
 
 export interface ManagedEndpointSweepResult {
   readonly mode: RelayConfiguration.ManagedEndpointCleanupMode;
@@ -25,7 +28,18 @@ export interface ManagedEndpointSweepResult {
   readonly deleted: number;
   readonly wouldDelete: number;
   readonly skippedLegacy: number;
+  /** Legacy candidates, by days since they went down: over 7, 30, and 90. */
+  readonly legacyOver7Days: number;
+  readonly legacyOver30Days: number;
+  readonly legacyOver90Days: number;
   readonly skippedOrphan: number;
+  /** The allocation row records a different tunnel under this name. */
+  readonly skippedReplaced: number;
+  /** The allocation row has not recorded a tunnel yet. */
+  readonly skippedUnrecorded: number;
+  /** Cloudflare's count of matching tunnels, before page limits. */
+  readonly totalDown: number | null;
+  readonly totalInactive: number | null;
   readonly failed: number;
   readonly truncated: boolean;
 }
@@ -70,6 +84,19 @@ function isExpiredManagedTunnel(input: {
   return Option.isSome(timestamp) && timestamp.value.epochMilliseconds <= cutoff.epochMilliseconds;
 }
 
+/** Days (fractional) since the tunnel went down, or was created if it never connected. */
+function inactiveDaysAt(
+  tunnel: ManagedEndpointProvider.ManagedEndpointTunnel,
+  status: "down" | "inactive",
+  now: DateTime.Utc,
+): number | null {
+  const since = status === "down" ? tunnel.connsInactiveAt : tunnel.createdAt;
+  if (typeof since !== "string") return null;
+  const timestamp = DateTime.make(since);
+  if (Option.isNone(timestamp)) return null;
+  return (now.epochMilliseconds - timestamp.value.epochMilliseconds) / 86_400_000;
+}
+
 function isRateLimited(cause: unknown): boolean {
   if (typeof cause !== "object" || cause === null) {
     return false;
@@ -112,7 +139,14 @@ const emptyResult = (
   deleted: 0,
   wouldDelete: 0,
   skippedLegacy: 0,
+  legacyOver7Days: 0,
+  legacyOver30Days: 0,
+  legacyOver90Days: 0,
   skippedOrphan: 0,
+  skippedReplaced: 0,
+  skippedUnrecorded: 0,
+  totalDown: null,
+  totalInactive: null,
   failed: 0,
   truncated: false,
 });
@@ -142,6 +176,7 @@ export const make = Effect.gen(function* () {
     );
     let listRequests = 0;
     let truncated = false;
+    const totals: Record<"down" | "inactive", number | null> = { down: null, inactive: null };
     const expired: Array<{
       readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & {
         readonly id: string;
@@ -169,6 +204,7 @@ export const make = Effect.gen(function* () {
       const first = yield* listPage(1);
       const totalCount =
         typeof first.resultInfo?.totalCount === "number" ? first.resultInfo.totalCount : undefined;
+      totals[status] = totalCount ?? null;
       const pages = rotatedPages({
         totalCount,
         slot,
@@ -213,7 +249,12 @@ export const make = Effect.gen(function* () {
     let deleted = 0;
     let wouldDelete = 0;
     let skippedLegacy = 0;
+    const legacyOverDays = new Map<number, number>(
+      MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS.map((days) => [days, 0]),
+    );
     let skippedOrphan = 0;
+    let skippedReplaced = 0;
+    let skippedUnrecorded = 0;
     let failed = 0;
 
     for (const { tunnel, status, cutoff } of uniqueExpired) {
@@ -224,14 +265,24 @@ export const make = Effect.gen(function* () {
         allocation.tunnelId !== null &&
         allocation.tunnelId !== tunnel.id
       ) {
+        skippedReplaced += 1;
         continue;
       }
       const owner = allocation?.tunnelId === tunnel.id ? allocation : undefined;
       if (owner !== undefined && !owner.recoveryEnabled) {
         skippedLegacy += 1;
+        const inactiveDays = inactiveDaysAt(tunnel, status, now);
+        for (const days of MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS) {
+          if (inactiveDays !== null && inactiveDays > days) {
+            legacyOverDays.set(days, (legacyOverDays.get(days) ?? 0) + 1);
+          }
+        }
+        continue;
+      }
+      if (allocation !== undefined && owner === undefined) {
+        skippedUnrecorded += 1;
         continue;
       }
-      if (allocation !== undefined && owner === undefined) continue;
       // A tunnel with no allocation row cannot be claimed, so a relink that
       // adopts it by name races any delete here. Count it and leave it for a
       // manual sweep instead.
@@ -284,7 +335,14 @@ export const make = Effect.gen(function* () {
       deleted,
       wouldDelete,
       skippedLegacy,
+      legacyOver7Days: legacyOverDays.get(7) ?? 0,
+      legacyOver30Days: legacyOverDays.get(30) ?? 0,
+      legacyOver90Days: legacyOverDays.get(90) ?? 0,
       skippedOrphan,
+      skippedReplaced,
+      skippedUnrecorded,
+      totalDown: totals.down,
+      totalInactive: totals.inactive,
       failed,
       truncated,
     };