diff --git a/infra/relay/package.json b/infra/relay/package.json index ae3c806c4d24..472f2606f20a 100644 --- a/infra/relay/package.json +++ b/infra/relay/package.json @@ -6,6 +6,7 @@ "deploy": "alchemy deploy", "push:android:smoke": "node scripts/android-push-smoke.ts", "push:android:watch": "node scripts/android-push-watch.ts", + "tunnels:census": "node scripts/tunnel-census.ts", "destroy": "alchemy destroy", "test": "vp test run --config ../../vite.config.ts --dir .", "typecheck": "tsc --noEmit" diff --git a/infra/relay/scripts/tunnel-census.ts b/infra/relay/scripts/tunnel-census.ts new file mode 100644 index 000000000000..4e7611dce1a4 --- /dev/null +++ b/infra/relay/scripts/tunnel-census.ts @@ -0,0 +1,97 @@ +// Read-only count of Cloudflare tunnels in the managed-endpoint account, +// grouped by relay stage, status, and how long they have been idle. The reaper +// only sees its own stage's prefix, so this shows how much of the backlog +// belongs to other stages. +// +// CLOUDFLARE_ACCOUNT_ID=... CLOUDFLARE_API_TOKEN=... node scripts/tunnel-census.ts +// +// The token needs Cloudflare Tunnel read access. Nothing is modified. +import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; +import * as Clock from "effect/Clock"; +import * as Config from "effect/Config"; +import * as Console from "effect/Console"; +import * as Effect from "effect/Effect"; +import * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import * as FetchHttpClient from "effect/http/FetchHttpClient"; +import * as HttpClient from "effect/http/HttpClient"; +import * as HttpClientRequest from "effect/http/HttpClientRequest"; +import * as HttpClientResponse from "effect/http/HttpClientResponse"; + +const PREFIX = "t3coderelay-managedendpoint-"; +const PAGE_SIZE = 1_000; +const DAY_MS = 86_400_000; + +// Cloudflare documents every field here as optional, so a tunnel missing one +// must not fail the whole page. +const Tunnel = Schema.Struct({ + name: Schema.optional(Schema.NullOr(Schema.String)), + status: Schema.optional(Schema.NullOr(Schema.String)), + created_at: Schema.optional(Schema.NullOr(Schema.String)), + conns_inactive_at: Schema.optional(Schema.NullOr(Schema.String)), +}); +type Tunnel = typeof Tunnel.Type; +const TunnelPage = Schema.Struct({ result: Schema.Array(Tunnel) }); + +// "t3coderelay-managedendpoint--<16 hex>"; stages may contain hyphens. +const stageOf = (name: string | null | undefined) => + name ? name.slice(PREFIX.length).replace(/-[a-f0-9]{16}$/u, "") : "unknown"; + +// Like the reaper: a down tunnel is aged from when it lost its connector, and a +// never-connected (inactive) one from when it was created. +const ageBucket = (tunnel: Tunnel, now: number) => { + const since = Date.parse( + (tunnel.status === "down" ? tunnel.conns_inactive_at : tunnel.created_at) ?? "", + ); + if (Number.isNaN(since)) return "unknown"; + const days = (now - since) / DAY_MS; + if (days > 90) return ">90d"; + if (days > 30) return ">30d"; + if (days > 7) return ">7d"; + return "<=7d"; +}; + +const main = Effect.gen(function* () { + const accountId = yield* Config.String("CLOUDFLARE_ACCOUNT_ID"); + const token = yield* Config.Redacted("CLOUDFLARE_API_TOKEN"); + const client = yield* HttpClient.HttpClient; + const now = yield* Clock.currentTimeMillis; + + const listPage = (page: number) => + HttpClientRequest.get(`https://api.cloudflare.com/client/v4/accounts/${accountId}/cfd_tunnel`, { + urlParams: { + is_deleted: "false", + include_prefix: PREFIX, + per_page: String(PAGE_SIZE), + page: String(page), + }, + }).pipe( + HttpClientRequest.bearerToken(Redacted.value(token)), + client.execute, + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap(HttpClientResponse.schemaBodyJson(TunnelPage)), + Effect.map((body) => body.result), + ); + + const counts = new Map(); + let total = 0; + for (let page = 1; ; page += 1) { + const tunnels = yield* listPage(page); + for (const tunnel of tunnels) { + total += 1; + const idle = + tunnel.status === "down" || tunnel.status === "inactive" ? ageBucket(tunnel, now) : "-"; + const key = `${stageOf(tunnel.name)}\t${tunnel.status ?? "unknown"}\t${idle}`; + counts.set(key, (counts.get(key) ?? 0) + 1); + } + if (tunnels.length < PAGE_SIZE) break; + } + + yield* Console.log("stage\tstatus\tidle\tcount"); + for (const [key, count] of [...counts].sort(([a], [b]) => a.localeCompare(b))) { + yield* Console.log(`${key}\t${count}`); + } + yield* Console.log(`total\t\t\t${total}`); +}).pipe(Effect.provide(FetchHttpClient.layer)); + +NodeRuntime.runMain(main); diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts index 6f4d38d6a06d..06d38fa9395e 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -436,6 +436,60 @@ describe("ManagedEndpointReaper", () => { }).pipe(Effect.provide(state.layer)); }); + it.effect("measures legacy age, uncounted skips, and Cloudflare totals", () => { + const down = (id: string, suffix: string, timestamp: string) => + tunnel({ id, suffix, status: "down", timestamp }); + const tunnels = [ + // Legacy owners, down for 1, 10, 40, and 100 days, plus one just past + // seven days, which must already count as over seven. + down("legacy-1d", "1111111111111111", "2026-08-24T11:00:00.000Z"), + down("legacy-7d1h", "7777777777777777", "2026-08-18T11:00:00.000Z"), + down("legacy-10d", "2222222222222222", "2026-08-15T11:00:00.000Z"), + down("legacy-40d", "3333333333333333", "2026-07-16T11:00:00.000Z"), + down("legacy-100d", "4444444444444444", "2026-05-17T11:00:00.000Z"), + // The allocation now records a different tunnel under this name. + down("stale", "5555555555555555", "2026-08-25T11:00:00.000Z"), + // The allocation has not recorded a tunnel yet. + down("pending", "6666666666666666", "2026-08-25T11:00:00.000Z"), + ]; + const state = harness({ + tunnels, + allocations: [ + allocation({ tunnelId: "legacy-1d", recoveryEnabled: false }), + allocation({ tunnelId: "legacy-7d1h", recoveryEnabled: false }), + allocation({ tunnelId: "legacy-10d", recoveryEnabled: false }), + allocation({ tunnelId: "legacy-40d", recoveryEnabled: false }), + allocation({ tunnelId: "legacy-100d", recoveryEnabled: false }), + { + ...allocation({ tunnelId: "current", recoveryEnabled: true }), + tunnelName: `${PREFIX}5555555555555555`, + }, + { + ...allocation({ tunnelId: null, recoveryEnabled: false }), + tunnelName: `${PREFIX}6666666666666666`, + }, + ], + cleanupMode: "dry-run", + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toMatchObject({ + scanned: 7, + skippedLegacy: 5, + legacyOver7Days: 4, + legacyOver30Days: 2, + legacyOver90Days: 1, + skippedReplaced: 1, + skippedUnrecorded: 1, + totalDown: 7, + totalInactive: 0, + wouldDelete: 0, + }); + }).pipe(Effect.provide(state.layer)); + }); + it.effect("does not count a tunnel that was replaced before its release", () => { const state = harness({ tunnels: [ @@ -649,7 +703,14 @@ describe("ManagedEndpointReaper", () => { deleted: 0, wouldDelete: 0, skippedLegacy: 0, + legacyOver7Days: 0, + legacyOver30Days: 0, + legacyOver90Days: 0, skippedOrphan: 0, + skippedReplaced: 0, + skippedUnrecorded: 0, + totalDown: null, + totalInactive: null, failed: 0, truncated: false, }); diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts index 4a3d408b49be..7590d253401a 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -16,6 +16,9 @@ export const MANAGED_ENDPOINT_INACTIVE_GRACE_PERIOD_MINUTES = 60; export const MANAGED_ENDPOINT_SWEEP_PAGE_SIZE = 100; export const MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT = 100; export const MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT = 10; +// Age buckets for legacy candidates, in days since the tunnel went down (or +// was created, for one that never connected). +const MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS = [7, 30, 90] as const; export interface ManagedEndpointSweepResult { readonly mode: RelayConfiguration.ManagedEndpointCleanupMode; @@ -25,7 +28,18 @@ export interface ManagedEndpointSweepResult { readonly deleted: number; readonly wouldDelete: number; readonly skippedLegacy: number; + /** Legacy candidates, by days since they went down: over 7, 30, and 90. */ + readonly legacyOver7Days: number; + readonly legacyOver30Days: number; + readonly legacyOver90Days: number; readonly skippedOrphan: number; + /** The allocation row records a different tunnel under this name. */ + readonly skippedReplaced: number; + /** The allocation row has not recorded a tunnel yet. */ + readonly skippedUnrecorded: number; + /** Cloudflare's count of matching tunnels, before page limits. */ + readonly totalDown: number | null; + readonly totalInactive: number | null; readonly failed: number; readonly truncated: boolean; } @@ -70,6 +84,19 @@ function isExpiredManagedTunnel(input: { return Option.isSome(timestamp) && timestamp.value.epochMilliseconds <= cutoff.epochMilliseconds; } +/** Days (fractional) since the tunnel went down, or was created if it never connected. */ +function inactiveDaysAt( + tunnel: ManagedEndpointProvider.ManagedEndpointTunnel, + status: "down" | "inactive", + now: DateTime.Utc, +): number | null { + const since = status === "down" ? tunnel.connsInactiveAt : tunnel.createdAt; + if (typeof since !== "string") return null; + const timestamp = DateTime.make(since); + if (Option.isNone(timestamp)) return null; + return (now.epochMilliseconds - timestamp.value.epochMilliseconds) / 86_400_000; +} + function isRateLimited(cause: unknown): boolean { if (typeof cause !== "object" || cause === null) { return false; @@ -112,7 +139,14 @@ const emptyResult = ( deleted: 0, wouldDelete: 0, skippedLegacy: 0, + legacyOver7Days: 0, + legacyOver30Days: 0, + legacyOver90Days: 0, skippedOrphan: 0, + skippedReplaced: 0, + skippedUnrecorded: 0, + totalDown: null, + totalInactive: null, failed: 0, truncated: false, }); @@ -142,6 +176,7 @@ export const make = Effect.gen(function* () { ); let listRequests = 0; let truncated = false; + const totals: Record<"down" | "inactive", number | null> = { down: null, inactive: null }; const expired: Array<{ readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & { readonly id: string; @@ -169,6 +204,7 @@ export const make = Effect.gen(function* () { const first = yield* listPage(1); const totalCount = typeof first.resultInfo?.totalCount === "number" ? first.resultInfo.totalCount : undefined; + totals[status] = totalCount ?? null; const pages = rotatedPages({ totalCount, slot, @@ -213,7 +249,12 @@ export const make = Effect.gen(function* () { let deleted = 0; let wouldDelete = 0; let skippedLegacy = 0; + const legacyOverDays = new Map( + MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS.map((days) => [days, 0]), + ); let skippedOrphan = 0; + let skippedReplaced = 0; + let skippedUnrecorded = 0; let failed = 0; for (const { tunnel, status, cutoff } of uniqueExpired) { @@ -224,14 +265,24 @@ export const make = Effect.gen(function* () { allocation.tunnelId !== null && allocation.tunnelId !== tunnel.id ) { + skippedReplaced += 1; continue; } const owner = allocation?.tunnelId === tunnel.id ? allocation : undefined; if (owner !== undefined && !owner.recoveryEnabled) { skippedLegacy += 1; + const inactiveDays = inactiveDaysAt(tunnel, status, now); + for (const days of MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS) { + if (inactiveDays !== null && inactiveDays > days) { + legacyOverDays.set(days, (legacyOverDays.get(days) ?? 0) + 1); + } + } + continue; + } + if (allocation !== undefined && owner === undefined) { + skippedUnrecorded += 1; continue; } - if (allocation !== undefined && owner === undefined) continue; // A tunnel with no allocation row cannot be claimed, so a relink that // adopts it by name races any delete here. Count it and leave it for a // manual sweep instead. @@ -284,7 +335,14 @@ export const make = Effect.gen(function* () { deleted, wouldDelete, skippedLegacy, + legacyOver7Days: legacyOverDays.get(7) ?? 0, + legacyOver30Days: legacyOverDays.get(30) ?? 0, + legacyOver90Days: legacyOverDays.get(90) ?? 0, skippedOrphan, + skippedReplaced, + skippedUnrecorded, + totalDown: totals.down, + totalInactive: totals.inactive, failed, truncated, };