From 76eb4a53f96fc522fac63758ad94c01b83793f56 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 24 Aug 2026 18:12:09 -0700 Subject: [PATCH 01/14] fix(connect): remove tunnels after hosts go offline --- .../src/cloud/ManagedEndpointRuntime.ts | 7 + apps/server/src/cloud/http.test.ts | 126 +- apps/server/src/cloud/http.ts | 103 +- apps/server/src/server.test.ts | 15 + apps/server/src/server.ts | 60 +- docs/internals/t3-connect.md | 14 + docs/user/remote-access.md | 4 + .../migration.sql | 1 + .../snapshot.json | 1516 +++++++++++++++++ infra/relay/src/deploymentConfig.test.ts | 4 + infra/relay/src/deploymentConfig.ts | 6 +- .../environments/EnvironmentConnector.test.ts | 2 + .../ManagedEndpointAllocations.test.ts | 85 + .../ManagedEndpointAllocations.ts | 71 +- .../ManagedEndpointProvider.test.ts | 35 + .../environments/ManagedEndpointProvider.ts | 38 +- .../ManagedEndpointReaper.test.ts | 419 +++++ .../src/environments/ManagedEndpointReaper.ts | 166 ++ infra/relay/src/http/Api.test.ts | 162 +- infra/relay/src/http/Api.ts | 76 +- infra/relay/src/persistence/schema.ts | 1 + infra/relay/src/worker.ts | 49 +- packages/contracts/src/relay.ts | 20 + 23 files changed, 2924 insertions(+), 56 deletions(-) create mode 100644 infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/migration.sql create mode 100644 infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/snapshot.json create mode 100644 infra/relay/src/environments/ManagedEndpointReaper.test.ts create mode 100644 infra/relay/src/environments/ManagedEndpointReaper.ts diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index 89c0a23783c0..1bbdd5f08e69 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -5,6 +5,7 @@ import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; import * as Result from "effect/Result"; import * as Semaphore from "effect/Semaphore"; @@ -58,6 +59,8 @@ export class CloudManagedEndpointRuntime extends Context.Service< readonly applyConfig: ( config: RelayManagedEndpointRuntimeConfig | null, ) => Effect.Effect; + readonly recoveryRequests: Stream.Stream; + readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect; } >()("t3/cloud/ManagedEndpointRuntime/CloudManagedEndpointRuntime") {} @@ -104,6 +107,7 @@ export const make = Effect.gen(function* () { const relayClient = yield* RelayClient.RelayClient; const activeRef = yield* Ref.make(null); const desiredConfigRef = yield* Ref.make(null); + const recoveryRequests = yield* PubSub.sliding(1); const reconcileSemaphore = yield* Semaphore.make(1); let reconcileConfig: CloudManagedEndpointRuntime["Service"]["applyConfig"]; @@ -144,6 +148,7 @@ export const make = Effect.gen(function* () { tunnelId: connector.config.tunnelId, tunnelName: connector.config.tunnelName, }); + yield* PubSub.publish(recoveryRequests, connector.config); yield* reconcileConfig(desiredConfig); }), ); @@ -305,6 +310,8 @@ export const make = Effect.gen(function* () { const runtime = CloudManagedEndpointRuntime.of({ applyConfig, + recoveryRequests: Stream.fromPubSub(recoveryRequests), + requestRecovery: (config) => PubSub.publish(recoveryRequests, config).pipe(Effect.asVoid), }); const initialConfig = yield* readRuntimeConfig.pipe( diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index 0f24e6f34176..2f217f7bed0a 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -7,6 +7,7 @@ import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; import * as Tracer from "effect/Tracer"; +import * as Stream from "effect/Stream"; import { HttpClient, HttpClientResponse, @@ -30,13 +31,20 @@ import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts"; import * as CliTokenManager from "./CliTokenManager.ts"; import type { RelayLinkProofRequest } from "@t3tools/contracts/relay"; -import { CLOUD_ENDPOINT_RUNTIME_CONFIG, RELAY_URL_SECRET } from "./config.ts"; +import { + CLOUD_ENDPOINT_RUNTIME_CONFIG, + CLOUD_LINKED_USER_ID, + decodeRuntimeConfig, + RELAY_ENVIRONMENT_CREDENTIAL_SECRET, + RELAY_URL_SECRET, +} from "./config.ts"; import { consumeCloudReplayGuards, isSupportedLinkProviderKind, linkProofScopes, pendingServiceUpdateExists, reconcileDesiredCloudLink, + recoverManagedCloudTunnel, releaseManagedTunnelOnShutdown, } from "./http.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; @@ -209,6 +217,8 @@ describe("reconcileDesiredCloudLink", () => { ManagedEndpointRuntime.CloudManagedEndpointRuntime, ManagedEndpointRuntime.CloudManagedEndpointRuntime.of({ applyConfig: unusedSecretStoreOperation, + recoveryRequests: Stream.empty, + requestRecovery: () => Effect.void, } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"]), ), Effect.provideService( @@ -303,10 +313,18 @@ describe("releaseManagedTunnelOnShutdown", () => { applyConfig: (config) => Effect.sync(() => { harness.applyConfigCalls.push(config); - return { - status: "disabled", - } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus; + return config === null + ? ({ + status: "disabled", + } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus) + : ({ + status: "running", + providerKind: "cloudflare_tunnel", + pid: 123, + } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus); }), + recoveryRequests: Stream.empty, + requestRecovery: () => Effect.void, }), ), Effect.provideService( @@ -579,6 +597,106 @@ describe("releaseManagedTunnelOnShutdown", () => { }), ); }); + + it.effect("recovers a web-linked tunnel with its environment credential", () => { + const oldConfig = + '{"providerKind":"cloudflare_tunnel","connectorToken":"old-token","tunnelId":"old-tunnel"}'; + const nextConfig = { + providerKind: "cloudflare_tunnel", + connectorToken: "new-token", + tunnelId: "new-tunnel", + } as const; + const { store, values } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, oldConfig], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(true); + expect(requests).toHaveLength(1); + expect(requests[0]?.method).toBe("POST"); + expect(requests[0]?.url).toBe("https://relay.example.test/v1/environments/env_123/tunnel"); + expect(requests[0]?.headers.authorization).toBe("Bearer environment-credential"); + expect(applyConfigCalls).toEqual([nextConfig]); + expect( + Option.getOrNull( + decodeRuntimeConfig(new TextDecoder().decode(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG))), + ), + ).toEqual(nextConfig); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respond: () => + Response.json({ + endpoint: { + httpBaseUrl: "https://environment.example.test/", + wsBaseUrl: "wss://environment.example.test/ws", + providerKind: "cloudflare_tunnel", + }, + endpointRuntime: nextConfig, + }), + }), + ); + }); + + it.effect("does not recover an environment without a managed tunnel credential", () => { + const { store } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"], + [RELAY_URL_SECRET, "https://relay.example.test"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(false); + expect(applyConfigCalls).toEqual([]); + expect(requests).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + + it.effect("keeps a tunnel configuration replaced during recovery", () => { + const { store, values } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + const freshConfig = new TextEncoder().encode("fresh-config"); + + return Effect.gen(function* () { + expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(false); + expect(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(freshConfig); + expect(applyConfigCalls).toEqual([]); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respond: () => { + values.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, freshConfig); + return Response.json({ + endpoint: { + httpBaseUrl: "https://environment.example.test/", + wsBaseUrl: "wss://environment.example.test/ws", + providerKind: "cloudflare_tunnel", + }, + endpointRuntime: { + providerKind: "cloudflare_tunnel", + connectorToken: "replacement-token", + }, + }); + }, + }), + ); + }); }); describe("link proof provider kinds", () => { diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index 29fdfe8ece2f..4d40c5d59299 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -28,6 +28,7 @@ import { RelayEnvironmentLinkProofPayload, RelayLinkProofRequest, RelayManagedEndpointOrigin, + RelayManagedEndpointRecoveryResponse, RelayOkResponse, } from "@t3tools/contracts/relay"; import { withRelayClientTracing } from "@t3tools/shared/relayTracing"; @@ -453,6 +454,7 @@ const cloudLinkProofHandler = Effect.fn("environment.cloud.linkProof")( const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(function* ( dependencies: CloudHttpDependencies, payload: RelayEnvironmentConfigRequest, + options?: { readonly requestRecovery?: boolean }, ) { yield* validateRelayConfigPayload(payload); yield* validateLinkedCloudUser({ @@ -489,6 +491,9 @@ const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(fu CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(endpointRuntimeJson), ); + if (options?.requestRecovery !== false) { + yield* dependencies.endpointRuntime.requestRecovery(payload.endpointRuntime); + } } else { yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); } @@ -607,14 +612,18 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi schema: RelayEnvironmentLinkResponse, }); yield* setCliDesiredCloudLink(true, mode); - return yield* applyCloudRelayConfig(dependencies, { - relayUrl, - relayIssuer: link.relayIssuer, - cloudUserId: link.cloudUserId, - environmentCredential: link.environmentCredential, - cloudMintPublicKey: link.cloudMintPublicKey, - endpointRuntime: link.endpointRuntime, - }); + return yield* applyCloudRelayConfig( + dependencies, + { + relayUrl, + relayIssuer: link.relayIssuer, + cloudUserId: link.cloudUserId, + environmentCredential: link.environmentCredential, + cloudMintPublicKey: link.cloudMintPublicKey, + endpointRuntime: link.endpointRuntime, + }, + { requestRecovery: false }, + ); }, Effect.catchIf( ServerSecretStore.isSecretStoreError, @@ -635,6 +644,84 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD }, ); +export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverManagedCloudTunnel")( + function* (localOrigin: string) { + const dependencies = yield* cloudHttpDependencies; + const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([ + dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), + dependencies.secrets.get(RELAY_URL_SECRET), + dependencies.secrets.get(CLOUD_LINKED_USER_ID), + dependencies.secrets.get(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), + ]); + if ( + Option.isNone(runtimeConfig) || + Option.isNone(relayUrl) || + Option.isNone(cloudUserId) || + Option.isNone(environmentCredential) + ) { + return false; + } + + const localUrl = yield* Effect.try({ + try: () => new URL(localOrigin), + catch: () => + new EnvironmentHttpBadRequestError({ + message: "Could not resolve local environment origin.", + }), + }); + if (localUrl.origin !== localOrigin) { + return yield* new EnvironmentHttpBadRequestError({ + message: "Could not resolve local environment origin.", + }); + } + + const environmentId = yield* dependencies.environment.getEnvironmentId; + const recovered = yield* relayClientRequest(dependencies, { + url: `${bytesToString(relayUrl.value)}/v1/environments/${encodeURIComponent(environmentId)}/tunnel`, + token: bytesToString(environmentCredential.value), + payload: { + cloudUserId: bytesToString(cloudUserId.value), + origin: { + localHttpHost: localUrl.hostname, + localHttpPort: endpointRequestPort(localUrl), + }, + }, + schema: RelayManagedEndpointRecoveryResponse, + }); + if (recovered.endpointRuntime.providerKind !== "cloudflare_tunnel") { + return yield* new EnvironmentHttpInternalServerError({ + message: "T3 Connect returned an unsupported managed tunnel configuration.", + }); + } + + const currentConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); + if ( + Option.isNone(currentConfig) || + bytesToString(currentConfig.value) !== bytesToString(runtimeConfig.value) + ) { + return false; + } + + const status = yield* dependencies.endpointRuntime.applyConfig(recovered.endpointRuntime); + if (status.status !== "running") { + return yield* new EnvironmentCloudEndpointUnavailableError({ + message: "Managed endpoint runtime could not be started.", + endpointRuntimeStatus: status, + }); + } + const encoded = yield* encodeEndpointRuntimeConfigJson(recovered.endpointRuntime).pipe( + Effect.mapError( + () => + new EnvironmentHttpInternalServerError({ + message: "Could not persist the recovered managed tunnel configuration.", + }), + ), + ); + yield* dependencies.secrets.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(encoded)); + return true; + }, +); + // The launcher owns this durable state, so read it directly both when a trial // decides whether it owns pre-activation cleanup and while a server tears down. export const pendingServiceUpdateExists = Effect.gen(function* () { diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index eff5ec2c16ec..5dd10b374d9f 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -948,6 +948,8 @@ const buildAppUnderTest = (options?: { CloudManagedEndpointRuntime.CloudManagedEndpointRuntime, CloudManagedEndpointRuntime.CloudManagedEndpointRuntime.of({ applyConfig: () => Effect.succeed({ status: "disabled" }), + recoveryRequests: Stream.empty, + requestRecovery: () => Effect.void, ...options?.layers?.cloudManagedEndpointRuntime, }), ), @@ -2514,6 +2516,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => { it.effect("unlinks local cloud state and disables the managed endpoint runtime", () => Effect.gen(function* () { const appliedRuntimeConfigs: Array = []; + const requestedRecoveryConfigs: Array = []; yield* buildAppUnderTest({ layers: { cloudManagedEndpointRuntime: { @@ -2530,6 +2533,10 @@ it.layer(NodeServices.layer)("server router seam", (it) => { ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}), }); }, + requestRecovery: (config) => + Effect.sync(() => { + requestedRecoveryConfigs.push(config); + }), }, }, }); @@ -2604,6 +2611,14 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }, null, ]); + assert.deepEqual(requestedRecoveryConfigs, [ + { + providerKind: "cloudflare_tunnel", + connectorToken: "connector-token", + tunnelId: "tunnel-id", + tunnelName: "tunnel-name", + }, + ]); }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index d5bebe3d5000..ca80ec268dce 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -4,6 +4,7 @@ import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schedule from "effect/Schedule"; +import * as Stream from "effect/Stream"; import { FetchHttpClient, HttpRouter, HttpServer } from "effect/unstable/http"; import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder"; @@ -93,6 +94,7 @@ import { connectHttpApiLayer, pendingServiceUpdateExists, reconcileDesiredCloudLink, + recoverManagedCloudTunnel, releaseManagedTunnelOnShutdown, } from "./cloud/http.ts"; import { serverRelayBrokerTracingLayer } from "./cloud/relayTracing.ts"; @@ -620,22 +622,13 @@ export const makeServerLayer = Layer.unwrap( if (!cleanupBeforeActivation) { yield* Effect.addFinalizer(() => releaseManagedTunnel); } - if (!(yield* CloudCliState.readCliDesiredCloudLink)) return; const server = yield* HttpServer.HttpServer; const address = server.address; if (typeof address === "string" || !("port" in address)) return; - // No settling delay before the first attempt: routes are already - // serving by the time activation opens this gate (the startup - // sequence awaits routesReady), and the retry schedule below - // covers anything this sleep used to hedge against. Every - // millisecond here is dead time on the path to remote - // reachability after a restart. - yield* reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`).pipe( + const localOrigin = `http://127.0.0.1:${address.port}`; + const endpointRuntime = yield* CloudManagedEndpointRuntime.CloudManagedEndpointRuntime; + const recoverManagedTunnel = recoverManagedCloudTunnel(localOrigin).pipe( Effect.retry({ - while: (error) => - error._tag !== "EnvironmentHttpBadRequestError" && - error._tag !== "EnvironmentHttpUnauthorizedError" && - error._tag !== "EnvironmentHttpConflictError", schedule: Schedule.exponential("1 second").pipe( Schedule.modifyDelay(({ duration }) => Effect.succeed(Duration.min(duration, Duration.seconds(30))), @@ -643,13 +636,46 @@ export const makeServerLayer = Layer.unwrap( Schedule.upTo({ duration: "10 minutes" }), ), }), - Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")), - Effect.catch((cause) => - Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", { - cause, - }), + Effect.tap((recovered) => + recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void, + ), + Effect.catchCause((cause) => + Effect.logWarning("Failed to recover the T3 Connect managed tunnel", { cause }), ), ); + yield* endpointRuntime.recoveryRequests.pipe( + Stream.runForEach(() => recoverManagedTunnel), + Effect.forkScoped, + ); + // No settling delay before the first attempt: routes are already + // serving by the time activation opens this gate (the startup + // sequence awaits routesReady), and the retry schedule below + // covers anything this sleep used to hedge against. Every + // millisecond here is dead time on the path to remote + // reachability after a restart. + if (yield* CloudCliState.readCliDesiredCloudLink) { + yield* reconcileDesiredCloudLink(localOrigin).pipe( + Effect.retry({ + while: (error) => + error._tag !== "EnvironmentHttpBadRequestError" && + error._tag !== "EnvironmentHttpUnauthorizedError" && + error._tag !== "EnvironmentHttpConflictError", + schedule: Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + Schedule.upTo({ duration: "10 minutes" }), + ), + }), + Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")), + Effect.catch((cause) => + Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", { + cause, + }), + ), + ); + } + yield* recoverManagedTunnel; }), ); yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie); diff --git a/docs/internals/t3-connect.md b/docs/internals/t3-connect.md index 6f796123e98b..e342f5adba62 100644 --- a/docs/internals/t3-connect.md +++ b/docs/internals/t3-connect.md @@ -125,6 +125,20 @@ connector, and attempts to revoke the relay-side environment record. It retains authorization so `t3 connect link` can re-enable exposure without another browser flow. `t3 connect logout` performs the same cleanup and removes the stored CLI authorization. +### Managed tunnel lifecycle + +Every linked environment stores a relay-issued environment credential. When a managed environment +starts or its connector exits, the server uses that credential to request a tunnel from the relay. +This also covers environments linked through web or mobile settings, which do not have a stored CLI +credential. The relay keeps the existing hostname and DNS record, so a replacement tunnel does not +change the public endpoint. + +After a host completes this recovery request, the relay records that the host can recreate its own +tunnel. The existing five-minute maintenance job removes tunnels from those hosts when Cloudflare +reports that they have been down for at least five minutes. Tunnels that never connected are removed +when they are at least five minutes old. The job leaves older hosts alone until they complete a +recovery request, and it only removes tunnels that belong to its own deployment stage. + The background service has an independent lifecycle. Connect setup may offer to install it, but logout leaves it running; manage it with `t3 service status`, `install`, `update`, and `uninstall`. diff --git a/docs/user/remote-access.md b/docs/user/remote-access.md index 5993fca5b352..ff6fc4285fb5 100644 --- a/docs/user/remote-access.md +++ b/docs/user/remote-access.md @@ -229,6 +229,10 @@ works for a server that was wiped or is no longer reachable. Device-local connec controls remain in **Settings** → **Connections** on web and desktop or **Settings** → **Environments** on mobile. +If a linked environment stays offline for several minutes, T3 Connect removes its unused tunnel. +The environment stays linked to your account and keeps the same address. When the server starts +again, T3 Connect creates a replacement tunnel automatically. You do not need to pair it again. + ## Security Notes - Treat pairing URLs and pairing tokens like passwords. diff --git a/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/migration.sql b/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/migration.sql new file mode 100644 index 000000000000..f528c09fbc7f --- /dev/null +++ b/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/migration.sql @@ -0,0 +1 @@ +ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "recovery_enabled_at" varchar(64); \ No newline at end of file diff --git a/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/snapshot.json b/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/snapshot.json new file mode 100644 index 000000000000..648638a0e17d --- /dev/null +++ b/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/snapshot.json @@ -0,0 +1,1516 @@ +{ + "version": "8", + "dialect": "postgres", + "id": "f3c6f2a5-2ecf-43cb-b381-98790fdca66e", + "prevIds": ["2374caff-40bf-423c-9255-55e76dddbc2a"], + "ddl": [ + { + "isRlsEnabled": false, + "name": "relay_agent_activity_rows", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_delivery_attempts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_dpop_proofs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_environment_credentials", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_environment_links", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_live_activities", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_managed_endpoint_allocations", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_managed_tunnel_limits", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_mobile_devices", + "entityType": "tables", + "schema": "public" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thread_id", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "state_json", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(36)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thread_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_job_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_suffix", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_status", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_reason", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(128)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "transport_error", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(128)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thumbprint", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "jti", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "iat", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credential_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credential_hash", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'T3 Environment'", + "generated": null, + "identity": null, + "name": "environment_label", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_http_base_url", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_ws_base_url", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(32)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_provider_kind", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "notifications_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "live_activities_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "managed_tunnels_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by_device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "activity_push_token", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remote_start_queued_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remote_started_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ended_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_aggregate_json", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_live_activity_delivery_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "hostname", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_name", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "dns_record_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ready_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "recovery_enabled_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "max_tunnels", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'iOS device'", + "generated": null, + "identity": null, + "name": "label", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "platform", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ios_major_version", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "app_version", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "bundle_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "aps_environment", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "push_token", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "push_to_start_token", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "preferences_json", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "updated_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_agent_activity_rows_updated", + "entityType": "indexes", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "thread_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_delivery_attempts_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "source_job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_delivery_attempts_source_job", + "entityType": "indexes", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_dpop_proofs_expires_at", + "entityType": "indexes", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "credential_hash", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_hash", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "environment_public_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_environment_key", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_links_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_links" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "activity_push_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_live_activities_activity_push_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_live_activities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "hostname", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_managed_endpoint_allocations_hostname", + "entityType": "indexes", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "tunnel_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_managed_endpoint_allocations_tunnel_name", + "entityType": "indexes", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "push_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_mobile_devices_push_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "push_to_start_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_mobile_devices_push_to_start_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "columns": ["environment_id", "environment_public_key", "thread_id"], + "nameExplicit": false, + "name": "relay_agent_activity_rows_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "columns": ["thumbprint", "jti"], + "nameExplicit": false, + "name": "relay_dpop_proofs_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "columns": ["user_id", "environment_id"], + "nameExplicit": false, + "name": "relay_environment_links_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_environment_links" + }, + { + "columns": ["user_id", "device_id"], + "nameExplicit": false, + "name": "relay_live_activities_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_live_activities" + }, + { + "columns": ["user_id", "environment_id"], + "nameExplicit": false, + "name": "relay_managed_endpoint_allocations_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "columns": ["user_id", "device_id"], + "nameExplicit": false, + "name": "relay_mobile_devices_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "relay_delivery_attempts_pkey", + "schema": "public", + "table": "relay_delivery_attempts", + "entityType": "pks" + }, + { + "columns": ["credential_id"], + "nameExplicit": false, + "name": "relay_environment_credentials_pkey", + "schema": "public", + "table": "relay_environment_credentials", + "entityType": "pks" + }, + { + "columns": ["user_id"], + "nameExplicit": false, + "name": "relay_managed_tunnel_limits_pkey", + "schema": "public", + "table": "relay_managed_tunnel_limits", + "entityType": "pks" + } + ], + "renames": [] +} diff --git a/infra/relay/src/deploymentConfig.test.ts b/infra/relay/src/deploymentConfig.test.ts index 44c7627a4daf..175e36ca527d 100644 --- a/infra/relay/src/deploymentConfig.test.ts +++ b/infra/relay/src/deploymentConfig.test.ts @@ -7,6 +7,7 @@ import { managedEndpointHostname, isManagedEndpointHostname, managedEndpointTunnelName, + managedEndpointTunnelNamePrefix, relayOwnsManagedEndpointZone, RelayPublicDomainLabelTooLongError, relayPublicDomainForStage, @@ -91,6 +92,9 @@ describe("managed endpoint names", () => { expect(managedEndpointTunnelName("dev_julius", hash)).toBe( "t3coderelay-managedendpoint-dev-julius-abcdef0123456789", ); + expect(managedEndpointTunnelNamePrefix("dev_julius")).toBe( + "t3coderelay-managedendpoint-dev-julius-", + ); }); it("keeps the DNS label within the provider limit for long stage names", () => { diff --git a/infra/relay/src/deploymentConfig.ts b/infra/relay/src/deploymentConfig.ts index fe9d37b29988..1dab362a9f09 100644 --- a/infra/relay/src/deploymentConfig.ts +++ b/infra/relay/src/deploymentConfig.ts @@ -117,6 +117,10 @@ export function managedEndpointForHostname(hostname: string): RelayManagedEndpoi }; } +export function managedEndpointTunnelNamePrefix(stage: string): string { + return `${MANAGED_ENDPOINT_TUNNEL_PREFIX}-${relayStageSlug(stage)}-`; +} + export function managedEndpointTunnelName(stage: string, hash: string): string { - return `${MANAGED_ENDPOINT_TUNNEL_PREFIX}-${relayStageSlug(stage)}-${stableSuffix(hash)}`; + return `${managedEndpointTunnelNamePrefix(stage)}${stableSuffix(hash)}`; } diff --git a/infra/relay/src/environments/EnvironmentConnector.test.ts b/infra/relay/src/environments/EnvironmentConnector.test.ts index 7f536bafb375..dfa809a9f151 100644 --- a/infra/relay/src/environments/EnvironmentConnector.test.ts +++ b/infra/relay/src/environments/EnvironmentConnector.test.ts @@ -197,6 +197,8 @@ function makeAllocations( recordTunnel: () => Effect.die("unused"), recordDns: () => Effect.die("unused"), markReady: () => Effect.die("unused"), + enableRecovery: () => Effect.die("unused"), + listByTunnelNames: () => Effect.die("unused"), claimRelease: () => Effect.die("unused"), claimDeprovision: () => Effect.die("unused"), remove: () => Effect.die("unused"), diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts index ebf51de100c1..c4c06e9e723e 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts @@ -10,6 +10,91 @@ const layerWithDb = (db: RelayDb.RelayDb["Service"]) => ManagedEndpointAllocations.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, db))); describe("ManagedEndpointAllocations", () => { + it.effect("records recovery support and advances the allocation generation", () => { + let updated: + | { + readonly recoveryEnabledAt: string; + readonly updatedAt: string; + } + | undefined; + const fakeDb = { + update: (table: unknown) => { + expect(table).toBe(relayManagedEndpointAllocations); + return { + set: (values: { readonly recoveryEnabledAt: string; readonly updatedAt: string }) => { + updated = values; + return { + where: () => Effect.void, + }; + }, + }; + }, + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + yield* allocations.enableRecovery({ + userId: "user-1", + environmentId: "environment-1", + }); + + expect(updated?.recoveryEnabledAt).toBe(updated?.updatedAt); + expect(updated?.recoveryEnabledAt).toBeDefined(); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + + it.effect("returns recovery support with tunnel allocation lookups", () => { + const base = { + userId: "user-1", + hostname: "environment.example.test", + tunnelName: "managed-tunnel", + dnsRecordId: "dns-1", + readyAt: "2026-08-25T12:00:00.000Z", + updatedAt: "2026-08-25T12:00:00.000Z", + }; + const fakeDb = { + select: () => ({ + from: (table: unknown) => { + expect(table).toBe(relayManagedEndpointAllocations); + return { + where: () => + Effect.succeed([ + { + ...base, + environmentId: "environment-1", + tunnelId: "tunnel-1", + recoveryEnabledAt: "2026-08-25T12:00:00.000Z", + }, + { + ...base, + environmentId: "environment-2", + tunnelId: "tunnel-2", + recoveryEnabledAt: null, + }, + ]), + }; + }, + }), + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const result = yield* allocations.listByTunnelNames(["first-tunnel", "second-tunnel"]); + + expect(result.map((entry) => [entry.tunnelId, entry.recoveryEnabled])).toEqual([ + ["tunnel-1", true], + ["tunnel-2", false], + ]); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + + it.effect("skips the database for an empty tunnel lookup", () => + Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + expect(yield* allocations.listByTunnelNames([])).toEqual([]); + }).pipe(Effect.provide(layerWithDb({} as RelayDb.RelayDb["Service"]))), + ); + it.effect("returns a claim generation only when deprovision wins the allocation CAS", () => { let claimedAt: string | undefined; const fakeDb = { diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index 4320eeea3b72..4fbdadd20239 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -1,5 +1,5 @@ import type { RelayManagedEndpoint } from "@t3tools/contracts/relay"; -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray, isNull } from "drizzle-orm"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -25,6 +25,10 @@ export interface ManagedEndpointAllocation { readonly updatedAt: string; } +export interface ManagedEndpointTunnelAllocation extends ManagedEndpointAllocation { + readonly recoveryEnabled: boolean; +} + export function resolveReadyManagedEndpoint(input: { readonly allocation: ManagedEndpointAllocation; readonly baseDomain: string | undefined; @@ -50,6 +54,8 @@ export class ManagedEndpointAllocationPersistenceError extends Schema.TaggedErro "record-tunnel", "record-dns", "mark-ready", + "enable-recovery", + "list-tunnels", "claim-release", "claim-deprovision", "remove", @@ -119,6 +125,15 @@ export class ManagedEndpointAllocations extends Context.Service< readonly markReady: ( input: ManagedEndpointAllocationKey, ) => Effect.Effect; + readonly enableRecovery: ( + input: ManagedEndpointAllocationKey, + ) => Effect.Effect; + readonly listByTunnelNames: ( + tunnelNames: ReadonlyArray, + ) => Effect.Effect< + ReadonlyArray, + ManagedEndpointAllocationPersistenceError + >; /** * Atomically claims the right to delete the allocation's tunnel: succeeds * only while the recorded tunnel and generation still match what the @@ -312,6 +327,60 @@ export const make = Effect.gen(function* () { ), ); }), + enableRecovery: Effect.fn("relay.managed_endpoint_allocations.enable_recovery")(function* ( + input: ManagedEndpointAllocationKey, + ) { + const now = DateTime.formatIso(yield* DateTime.now); + yield* db + .update(relayManagedEndpointAllocations) + .set({ recoveryEnabledAt: now, updatedAt: now }) + .where( + and(whereAllocation(input), isNull(relayManagedEndpointAllocations.recoveryEnabledAt)), + ) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointAllocationPersistenceError({ + operation: "enable-recovery", + stage: "database-request", + ...input, + cause, + }), + ), + ); + }), + listByTunnelNames: Effect.fn("relay.managed_endpoint_allocations.list_by_tunnel_names")( + function* (tunnelNames: ReadonlyArray) { + if (tunnelNames.length === 0) { + return []; + } + return yield* db + .select({ + ...allocationSelection, + recoveryEnabledAt: relayManagedEndpointAllocations.recoveryEnabledAt, + }) + .from(relayManagedEndpointAllocations) + .where(inArray(relayManagedEndpointAllocations.tunnelName, tunnelNames)) + .pipe( + Effect.map((rows) => + rows.map(({ recoveryEnabledAt, ...allocation }) => ({ + ...allocation, + recoveryEnabled: recoveryEnabledAt !== null, + })), + ), + Effect.mapError( + (cause) => + new ManagedEndpointAllocationPersistenceError({ + operation: "list-tunnels", + stage: "database-request", + userId: "*", + environmentId: "*", + cause, + }), + ), + ); + }, + ), claimRelease: Effect.fn("relay.managed_endpoint_allocations.claim_release")(function* ( input: ClaimManagedEndpointReleaseInput, ) { diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 4d136658c8fd..8ec3f3762b75 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -159,6 +159,7 @@ function makeDnsClient( function makeAllocations(calls: AllocationCall[] = []) { const allocations = new Map(); + const recoveryEnabled = new Set(); let generation = 0; const mutate = ( key: string, @@ -214,6 +215,19 @@ function makeAllocations(calls: AllocationCall[] = []) { readyAt: "2026-06-02T00:00:00.000Z", })); }), + enableRecovery: (input) => + Effect.sync(() => { + recoveryEnabled.add(allocationKey(input)); + }), + listByTunnelNames: (tunnelNames) => + Effect.sync(() => + [...allocations.values()] + .filter((allocation) => tunnelNames.includes(allocation.tunnelName)) + .map((allocation) => ({ + ...allocation, + recoveryEnabled: recoveryEnabled.has(allocationKey(allocation)), + })), + ), claimRelease: (input) => Effect.sync(() => { calls.push({ operation: "claimRelease", input }); @@ -939,6 +953,27 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.provide(layer)); }); + it.effect("does not release a tunnel when the requested tunnel id is outdated", () => { + const tunnelCalls: TunnelCall[] = []; + const layer = providerLayer( + makePersistentTunnelClient(tunnelCalls), + makeDnsClient(), + makeAllocations(), + ); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + + expect(yield* provider.release({ ...key, expectedTunnelId: "old-tunnel-id" })).toBe(false); + expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete"); + }).pipe(Effect.provide(layer)); + }); + it.effect("treats an already deleted tunnel as successfully released", () => { const notFound = { _tag: "NotFound" } as const; const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 9a578874844e..1789c014ae5d 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -166,13 +166,28 @@ export class ManagedEndpointProvider extends Context.Service< readonly release: (input: { readonly userId: string; readonly environmentId: string; + readonly expectedTunnelId?: string; }) => Effect.Effect; } >()("t3code-relay/environments/ManagedEndpointProvider") {} -interface ManagedEndpointTunnel { +export interface ManagedEndpointTunnel { readonly id?: string | null; readonly name?: string | null; + readonly status?: string | null; + readonly createdAt?: string | null; + readonly connsInactiveAt?: string | null; +} + +export interface ManagedEndpointTunnelListRequest { + readonly isDeleted: false; + readonly name?: string; + readonly includePrefix?: string; + readonly status?: "inactive" | "down"; + readonly existedAt?: string; + readonly wasInactiveAt?: string; + readonly page?: number; + readonly perPage?: number; } const ManagedEndpointTunnelClientOperation = Schema.Literals([ @@ -201,11 +216,15 @@ export class ManagedEndpointTunnelClientError extends Schema.TaggedErrorClass Effect.Effect< - { readonly result: ReadonlyArray }, + readonly list: (request: ManagedEndpointTunnelListRequest) => Effect.Effect< + { + readonly result: ReadonlyArray; + readonly resultInfo?: { + readonly page?: number | null; + readonly perPage?: number | null; + readonly totalCount?: number | null; + } | null; + }, ManagedEndpointTunnelClientError >; readonly create: (request: { @@ -554,6 +573,9 @@ export const make = Effect.gen(function* () { if (allocation === null || tunnelId === null) { return true; } + if (input.expectedTunnelId !== undefined && input.expectedTunnelId !== tunnelId) { + return false; + } // Claim the release against the allocation's current generation before // touching Cloudflare. A provision racing this release (fast environment // restart) rewrites updatedAt when it records its tunnel, so a stale @@ -865,7 +887,7 @@ export const layerCloudflareBindings = ( alchemyRuntimeContext: Alchemy.BaseRuntimeContext, ) => layer.pipe( - Layer.provide( + Layer.provideMerge( Layer.mergeAll( layerTunnelClient({ list: (request) => @@ -874,7 +896,7 @@ export const layerCloudflareBindings = ( (cause) => new ManagedEndpointTunnelClientError({ operation: "list", - tunnelName: request.name, + ...(request.name === undefined ? {} : { tunnelName: request.name }), cause, }), ), diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts new file mode 100644 index 000000000000..c8e4816b348e --- /dev/null +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -0,0 +1,419 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Redacted from "effect/Redacted"; +import * as TestClock from "effect/testing/TestClock"; + +import * as RelayConfiguration from "../Config.ts"; +import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts"; +import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts"; +import * as ManagedEndpointReaper from "./ManagedEndpointReaper.ts"; + +const NOW = "2026-08-25T12:00:00.000Z"; +const NOW_MILLIS = DateTime.makeUnsafe(NOW).epochMilliseconds; +const PREFIX = "t3coderelay-managedendpoint-prod-"; + +function tunnel(input: { + readonly id: string; + readonly suffix: string; + readonly status: "down" | "inactive" | "healthy" | "degraded"; + readonly timestamp?: string | null; + readonly prefix?: string; +}): ManagedEndpointProvider.ManagedEndpointTunnel { + return { + id: input.id, + name: `${input.prefix ?? PREFIX}${input.suffix}`, + status: input.status, + ...(input.timestamp === undefined + ? {} + : input.status === "inactive" + ? { createdAt: input.timestamp } + : { connsInactiveAt: input.timestamp }), + }; +} + +function allocation(input: { + readonly tunnelId: string; + readonly recoveryEnabled: boolean; +}): ManagedEndpointAllocations.ManagedEndpointTunnelAllocation { + return { + userId: "user-1", + environmentId: `environment-${input.tunnelId}`, + hostname: `${input.tunnelId}.example.test`, + tunnelId: input.tunnelId, + tunnelName: `${PREFIX}aaaaaaaaaaaaaaaa`, + dnsRecordId: "dns-1", + readyAt: "2026-08-25T11:00:00.000Z", + updatedAt: "2026-08-25T11:00:00.000Z", + recoveryEnabled: input.recoveryEnabled, + }; +} + +function harness(input?: { + readonly tunnels?: ReadonlyArray; + readonly allocations?: ReadonlyArray; + readonly namespace?: string; + readonly failTunnelId?: string; + readonly skipTunnelId?: string; +}) { + const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = []; + const deleted: string[] = []; + const releases: Array<{ + readonly userId: string; + readonly environmentId: string; + readonly expectedTunnelId?: string; + }> = []; + const recorded = (input?.allocations ?? []).map((entry) => { + const matching = input?.tunnels?.find((candidate) => candidate.id === entry.tunnelId); + return typeof matching?.name === "string" ? { ...entry, tunnelName: matching.name } : entry; + }); + const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + list: (request) => + Effect.sync(() => { + listRequests.push(request); + const matching = (input?.tunnels ?? []).filter((entry) => entry.status === request.status); + const start = ((request.page ?? 1) - 1) * (request.perPage ?? 100); + return { + result: matching.slice(start, start + (request.perPage ?? 100)), + resultInfo: { + page: request.page ?? 1, + perPage: request.perPage ?? 100, + totalCount: matching.length, + }, + }; + }), + create: () => Effect.die("unused"), + putConfiguration: () => Effect.die("unused"), + getToken: () => Effect.die("unused"), + delete: (tunnelId) => + tunnelId === input?.failTunnelId + ? Effect.fail( + new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "delete", + tunnelId, + cause: "Cloudflare refused the deletion", + }), + ) + : Effect.sync(() => { + deleted.push(tunnelId); + }), + }); + const allocationService = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + get: () => Effect.die("unused"), + reserve: () => Effect.die("unused"), + recordTunnel: () => Effect.die("unused"), + recordDns: () => Effect.die("unused"), + markReady: () => Effect.die("unused"), + enableRecovery: () => Effect.die("unused"), + listByTunnelNames: (tunnelNames) => + Effect.succeed(recorded.filter((entry) => tunnelNames.includes(entry.tunnelName))), + claimRelease: () => Effect.die("unused"), + claimDeprovision: () => Effect.die("unused"), + remove: () => Effect.die("unused"), + removeClaimed: () => Effect.die("unused"), + }); + const provider = ManagedEndpointProvider.ManagedEndpointProvider.of({ + provision: () => Effect.die("unused"), + prepareDeprovision: () => Effect.die("unused"), + deprovision: () => Effect.die("unused"), + release: (request) => + Effect.sync(() => { + releases.push(request); + if (request.expectedTunnelId === input?.skipTunnelId) { + return false; + } + if (request.expectedTunnelId !== undefined) { + deleted.push(request.expectedTunnelId); + } + return true; + }), + }); + const config = RelayConfiguration.RelayConfiguration.of({ + relayIssuer: "https://relay.example.test", + apns: { + environment: "sandbox", + teamId: "team-id", + keyId: "key-id", + privateKey: Redacted.make("private-key"), + bundleId: "com.t3tools.t3code.dev", + }, + apnsDeliveryJobSigningSecret: Redacted.make("job-secret"), + clerkSecretKey: Redacted.make("clerk-secret"), + clerkPublishableKey: "pk_test_test", + clerkJwtAudience: "t3-code-relay", + cloudMintPrivateKey: Redacted.make("cloud-private-key"), + cloudMintPublicKey: "cloud-public-key", + managedEndpointBaseDomain: "example.test", + managedEndpointNamespace: input?.namespace ?? "prod", + }); + + return { + listRequests, + deleted, + releases, + layer: ManagedEndpointReaper.layer.pipe( + Layer.provide( + Layer.mergeAll( + RelayConfiguration.layer(config), + ManagedEndpointProvider.layerTunnelClient(tunnelClient), + Layer.succeed(ManagedEndpointProvider.ManagedEndpointProvider, provider), + Layer.succeed(ManagedEndpointAllocations.ManagedEndpointAllocations, allocationService), + ), + ), + ), + }; +} + +describe("ManagedEndpointReaper", () => { + it.effect("removes expired down and inactive tunnels from recoverable environments", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "down-1", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:55:00.000Z", + }), + tunnel({ + id: "inactive-1", + suffix: "bbbbbbbbbbbbbbbb", + status: "inactive", + timestamp: "2026-08-25T11:54:00.000Z", + }), + ], + allocations: [ + allocation({ tunnelId: "down-1", recoveryEnabled: true }), + allocation({ tunnelId: "inactive-1", recoveryEnabled: true }), + ], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 2, + deleted: 2, + skippedLegacy: 0, + failed: 0, + }); + expect(state.deleted).toEqual(["down-1", "inactive-1"]); + expect(state.releases.map((request) => request.expectedTunnelId)).toEqual([ + "down-1", + "inactive-1", + ]); + expect(state.listRequests).toEqual([ + { + isDeleted: false, + includePrefix: PREFIX, + status: "down", + existedAt: "2026-08-25T11:55:00.000Z", + wasInactiveAt: "2026-08-25T11:55:00.000Z", + page: 1, + perPage: 100, + }, + { + isDeleted: false, + includePrefix: PREFIX, + status: "inactive", + existedAt: "2026-08-25T11:55:00.000Z", + page: 1, + perPage: 100, + }, + ]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("keeps recent tunnels, other stages, and tunnels without valid timestamps", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "recent", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:55:01.000Z", + }), + tunnel({ + id: "other-stage", + prefix: `${PREFIX}julius-`, + suffix: "bbbbbbbbbbbbbbbb", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + tunnel({ + id: "missing-time", + suffix: "cccccccccccccccc", + status: "inactive", + timestamp: null, + }), + ], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 0, + deleted: 0, + skippedLegacy: 0, + failed: 0, + }); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("keeps tunnels owned by environments that cannot recover yet", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "legacy", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + allocations: [allocation({ tunnelId: "legacy", recoveryEnabled: false })], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 1, + deleted: 0, + skippedLegacy: 1, + failed: 0, + }); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("removes expired tunnels that no longer have an allocation", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "orphan", + suffix: "aaaaaaaaaaaaaaaa", + status: "inactive", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(1); + expect(state.deleted).toEqual(["orphan"]); + expect(state.releases).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("does not count a tunnel that was replaced before its release", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "replaced", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + allocations: [allocation({ tunnelId: "replaced", recoveryEnabled: true })], + skipTunnelId: "replaced", + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(0); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("continues after an orphan tunnel deletion fails", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "failed", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + tunnel({ + id: "next", + suffix: "bbbbbbbbbbbbbbbb", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + failTunnelId: "failed", + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 2, + deleted: 1, + skippedLegacy: 0, + failed: 1, + }); + expect(state.deleted).toEqual(["next"]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("continues past a page of older hosts to find recoverable tunnels", () => { + const entries = Array.from({ length: 101 }, (_, index) => + tunnel({ + id: `tunnel-${index}`, + suffix: index.toString(16).padStart(16, "0"), + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ); + const state = harness({ + tunnels: entries, + allocations: entries.map((entry, index) => + allocation({ tunnelId: entry.id!, recoveryEnabled: index === 100 }), + ), + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 101, + deleted: 1, + skippedLegacy: 100, + failed: 0, + }); + expect(state.deleted).toEqual(["tunnel-100"]); + expect( + state.listRequests + .filter((request) => request.status === "down") + .map((request) => request.page), + ).toEqual([1, 2]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("limits each cleanup run to 100 tunnel deletions", () => { + const state = harness({ + tunnels: Array.from({ length: 105 }, (_, index) => + tunnel({ + id: `tunnel-${index}`, + suffix: index.toString(16).padStart(16, "0"), + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ), + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(100); + expect(state.deleted).toHaveLength(100); + }).pipe(Effect.provide(state.layer)); + }); +}); diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts new file mode 100644 index 000000000000..cfd933ebe650 --- /dev/null +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -0,0 +1,166 @@ +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; + +import * as RelayConfiguration from "../Config.ts"; +import { managedEndpointTunnelNamePrefix } from "../deploymentConfig.ts"; +import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts"; +import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts"; + +export const MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES = 5; +export const MANAGED_ENDPOINT_SWEEP_PAGE_SIZE = 100; +export const MANAGED_ENDPOINT_SWEEP_DELETE_LIMIT = 100; + +export interface ManagedEndpointSweepResult { + readonly scanned: number; + readonly deleted: number; + readonly skippedLegacy: number; + readonly failed: number; +} + +export class ManagedEndpointReaper extends Context.Service< + ManagedEndpointReaper, + { + readonly sweep: Effect.Effect< + ManagedEndpointSweepResult, + | ManagedEndpointProvider.ManagedEndpointTunnelClientError + | ManagedEndpointAllocations.ManagedEndpointAllocationPersistenceError + >; + } +>()("t3code-relay/environments/ManagedEndpointReaper") {} + +function isExpiredManagedTunnel(input: { + readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel; + readonly status: "down" | "inactive"; + readonly prefix: string; + readonly cutoff: DateTime.Utc; +}): input is typeof input & { + readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & { + readonly id: string; + readonly name: string; + }; +} { + const { tunnel, status, prefix, cutoff } = input; + if ( + typeof tunnel.id !== "string" || + typeof tunnel.name !== "string" || + tunnel.status !== status || + !tunnel.name.startsWith(prefix) || + !/^[a-f0-9]{16}$/u.test(tunnel.name.slice(prefix.length)) + ) { + return false; + } + + const inactiveAt = status === "down" ? tunnel.connsInactiveAt : tunnel.createdAt; + if (typeof inactiveAt !== "string") { + return false; + } + const timestamp = DateTime.make(inactiveAt); + return Option.isSome(timestamp) && timestamp.value.epochMilliseconds <= cutoff.epochMilliseconds; +} + +export const make = Effect.gen(function* () { + const config = yield* RelayConfiguration.RelayConfiguration; + const tunnels = yield* ManagedEndpointProvider.ManagedEndpointTunnelClient; + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + + const sweep = Effect.gen(function* () { + const namespace = config.managedEndpointNamespace; + if (!namespace) { + return { scanned: 0, deleted: 0, skippedLegacy: 0, failed: 0 }; + } + + const now = yield* DateTime.now; + const cutoff = DateTime.subtract(now, { minutes: MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES }); + const cutoffIso = DateTime.formatIso(cutoff); + const prefix = managedEndpointTunnelNamePrefix(namespace); + let scanned = 0; + let deleted = 0; + let skippedLegacy = 0; + let failed = 0; + + for (const status of ["down", "inactive"] as const) { + let page = 1; + while (deleted < MANAGED_ENDPOINT_SWEEP_DELETE_LIMIT) { + const response = yield* tunnels.list({ + isDeleted: false, + includePrefix: prefix, + status, + existedAt: cutoffIso, + ...(status === "down" ? { wasInactiveAt: cutoffIso } : {}), + page, + perPage: MANAGED_ENDPOINT_SWEEP_PAGE_SIZE, + }); + const expired = response.result + .map((tunnel) => ({ tunnel, status, prefix, cutoff })) + .filter(isExpiredManagedTunnel) + .map(({ tunnel }) => tunnel); + scanned += expired.length; + + const recorded = yield* allocations.listByTunnelNames(expired.map((tunnel) => tunnel.name)); + const recordedByTunnelId = new Map( + recorded + .filter((allocation) => allocation.tunnelId !== null) + .map((allocation) => [allocation.tunnelId, allocation]), + ); + + for (const tunnel of expired) { + if (deleted >= MANAGED_ENDPOINT_SWEEP_DELETE_LIMIT) { + break; + } + const allocation = recordedByTunnelId.get(tunnel.id); + if (allocation !== undefined && !allocation.recoveryEnabled) { + skippedLegacy += 1; + continue; + } + + const result = + allocation === undefined + ? yield* tunnels.delete(tunnel.id).pipe(Effect.as(true), Effect.result) + : yield* provider + .release({ + userId: allocation.userId, + environmentId: allocation.environmentId, + expectedTunnelId: tunnel.id, + }) + .pipe(Effect.result); + if (result._tag === "Failure") { + failed += 1; + yield* Effect.logWarning("Failed to delete an inactive managed tunnel", { + tunnelId: tunnel.id, + tunnelName: tunnel.name, + cause: result.failure, + }); + } else if (result.success) { + deleted += 1; + yield* Effect.logInfo("Deleted an inactive managed tunnel", { + tunnelId: tunnel.id, + tunnelName: tunnel.name, + status, + }); + } + } + + const totalCount = response.resultInfo?.totalCount; + if ( + response.result.length === 0 || + (typeof totalCount === "number" + ? page * MANAGED_ENDPOINT_SWEEP_PAGE_SIZE >= totalCount + : response.result.length < MANAGED_ENDPOINT_SWEEP_PAGE_SIZE) + ) { + break; + } + page += 1; + } + } + + return { scanned, deleted, skippedLegacy, failed }; + }).pipe(Effect.withSpan("relay.managed_endpoint_reaper.sweep")); + + return ManagedEndpointReaper.of({ sweep }); +}); + +export const layer = Layer.effect(ManagedEndpointReaper, make); diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index daf756a2b7cc..12b39dd3e7e9 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -23,6 +23,7 @@ import { relayDocsRedirectRoute, relayEnvironmentAuthLayer, relayNotFoundRoute, + recoverEnvironmentTunnelRecord, revokeEnvironmentLinkRecord, traceRelayHttpRequestWith, unlinkEnvironmentRecord, @@ -33,6 +34,7 @@ import * as RelayConfiguration from "../Config.ts"; import * as RelayDb from "../db.ts"; import * as EnvironmentCredentials from "../environments/EnvironmentCredentials.ts"; import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts"; +import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts"; import * as ManagedEndpointProvider from "../environments/ManagedEndpointProvider.ts"; vi.mock("@clerk/backend", () => ({ @@ -168,6 +170,7 @@ function relayUnlinkTestLayer(input?: { readonly revokeCredential?: EnvironmentCredentials.EnvironmentCredentials["Service"]["revokeForEnvironmentPublicKey"]; readonly prepareDeprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["prepareDeprovision"]; readonly deprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["deprovision"]; + readonly provision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["provision"]; }) { return Layer.mergeAll( Layer.succeed( @@ -199,7 +202,7 @@ function relayUnlinkTestLayer(input?: { Layer.succeed( ManagedEndpointProvider.ManagedEndpointProvider, ManagedEndpointProvider.ManagedEndpointProvider.of({ - provision: () => Effect.die("unused provision"), + provision: input?.provision ?? (() => Effect.die("unused provision")), prepareDeprovision: input?.prepareDeprovision ?? (() => Effect.succeed(null)), deprovision: input?.deprovision ?? (() => Effect.void), release: () => Effect.die("unused release"), @@ -220,6 +223,163 @@ const linkedEnvironmentRecord = { linkedAt: "2026-07-28T00:00:00.000Z", } as const; +describe("relay managed tunnel recovery", () => { + it.effect("recovers a linked environment and marks its tunnel as recoverable", () => { + let recoveryEnabledFor: { readonly userId: string; readonly environmentId: string } | null = + null; + const runtime = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "replacement-token", + tunnelId: "replacement-tunnel", + }; + + return Effect.gen(function* () { + expect( + yield* recoverEnvironmentTunnelRecord({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ).toEqual({ + endpoint: linkedEnvironmentRecord.endpoint, + endpointRuntime: runtime, + }); + expect(recoveryEnabledFor).toEqual({ + userId: "user-1", + environmentId: "environment-1", + }); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + provision: () => + Effect.succeed({ + endpoint: linkedEnvironmentRecord.endpoint, + runtime, + }), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: (input) => + Effect.sync(() => { + recoveryEnabledFor = input; + }), + }), + ), + ), + ); + }); + + it.effect("rejects a credential from a different environment owner", () => { + let provisioned = false; + + return Effect.gen(function* () { + const error = yield* Effect.flip( + recoverEnvironmentTunnelRecord({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "different-public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + expect(error).toMatchObject({ _tag: "Unauthorized" }); + expect(provisioned).toBe(false); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + provision: () => + Effect.sync(() => { + provisioned = true; + return { + endpoint: linkedEnvironmentRecord.endpoint, + runtime: { providerKind: "cloudflare_tunnel", connectorToken: "token" }, + }; + }), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => Effect.die("unused"), + }), + ), + ), + ); + }); + + it.effect("does not recover a publish-only environment", () => + Effect.gen(function* () { + const error = yield* Effect.flip( + recoverEnvironmentTunnelRecord({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + expect(error).toMatchObject({ _tag: "Unauthorized" }); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => + Effect.succeed({ + ...linkedEnvironmentRecord, + endpoint: { + ...linkedEnvironmentRecord.endpoint, + providerKind: "manual" as const, + }, + }), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => Effect.die("unused"), + }), + ), + ), + ), + ); + + it.effect("rejects a recovered tunnel that changes the linked endpoint", () => { + let recoveryEnabled = false; + + return Effect.gen(function* () { + const error = yield* Effect.flip( + recoverEnvironmentTunnelRecord({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + expect(error).toMatchObject({ _tag: "Unauthorized" }); + expect(recoveryEnabled).toBe(false); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + provision: () => + Effect.succeed({ + endpoint: { + httpBaseUrl: "https://different.example.test/", + wsBaseUrl: "wss://different.example.test/ws", + providerKind: "cloudflare_tunnel", + }, + runtime: { providerKind: "cloudflare_tunnel", connectorToken: "token" }, + }), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => + Effect.sync(() => { + recoveryEnabled = true; + }), + }), + ), + ), + ); + }); +}); + describe("relay environment unlink", () => { it.effect("revokes the link and its credentials in one database transaction", () => { const calls: Array = []; diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 50bcff665a9b..001e0de7e7d3 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -46,6 +46,7 @@ import { RelayEnvironmentLinkLimitExceededError, RelayEnvironmentPrincipal, type RelayEnvironmentConnectRequest, + type RelayManagedEndpointOrigin, type RelayDpopAccessTokenScope, RelayInternalError, } from "@t3tools/contracts/relay"; @@ -464,6 +465,51 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron }, ); +export const recoverEnvironmentTunnelRecord = Effect.fn( + "relay.api.server.recoverEnvironmentTunnelRecord", +)(function* (input: { + readonly userId: string; + readonly environmentId: string; + readonly environmentPublicKey: string; + readonly origin: RelayManagedEndpointOrigin; +}) { + const links = yield* EnvironmentLinks.EnvironmentLinks; + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const managedEndpointProvider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const link = yield* links.getForUser({ + userId: input.userId, + environmentId: input.environmentId, + }); + if ( + link === null || + link.environmentPublicKey !== input.environmentPublicKey || + link.endpoint.providerKind !== "cloudflare_tunnel" + ) { + return yield* new HttpApiError.Unauthorized({}); + } + + const recovered = yield* managedEndpointProvider.provision({ + userId: input.userId, + environmentId: input.environmentId, + origin: input.origin, + }); + if ( + recovered.endpoint.httpBaseUrl !== link.endpoint.httpBaseUrl || + recovered.endpoint.wsBaseUrl !== link.endpoint.wsBaseUrl + ) { + return yield* new HttpApiError.Unauthorized({}); + } + + yield* allocations.enableRecovery({ + userId: input.userId, + environmentId: input.environmentId, + }); + return { + endpoint: recovered.endpoint, + endpointRuntime: recovered.runtime, + }; +}); + export const mobileApi = HttpApiBuilder.group( RelayApi, "mobile", @@ -856,7 +902,7 @@ export const serverApi = HttpApiBuilder.group( Effect.fnUntraced(function* (handlers) { const publisher = yield* AgentActivityPublisher.AgentActivityPublisher; const publishSignatures = yield* EnvironmentPublishSignatures.EnvironmentPublishSignatures; - return handlers.handle( + const activityHandlers = handlers.handle( "publishAgentActivity", Effect.fn("relay.api.server.publishAgentActivity")( function* (args) { @@ -984,6 +1030,34 @@ export const serverApi = HttpApiBuilder.group( mapRelayCommonApiErrors("not_authorized"), ), ); + + return activityHandlers.handle( + "recoverManagedEndpoint", + Effect.fn("relay.api.server.recoverManagedEndpoint")( + function* ({ params, payload }) { + const principal = yield* RelayEnvironmentPrincipal; + if (principal.environmentId !== params.environmentId) { + return yield* new HttpApiError.Unauthorized({}); + } + yield* appendRelayCredentialResponseHeaders; + return yield* recoverEnvironmentTunnelRecord({ + userId: payload.cloudUserId, + environmentId: params.environmentId, + environmentPublicKey: principal.environmentPublicKey, + origin: payload.origin, + }); + }, + Effect.catchTags({ + ManagedEndpointOriginNotAllowed: () => Effect.fail(new HttpApiError.Unauthorized({})), + ManagedEndpointProvisioningNotConfigured: () => + relayInternalErrorResponse("upstream_unavailable"), + ManagedEndpointProvisioningFailed: () => + relayInternalErrorResponse("upstream_unavailable"), + ManagedTunnelLimitExceeded: () => relayInternalErrorResponse("upstream_unavailable"), + }), + mapRelayCommonApiErrors("not_authorized"), + ), + ); }), ); diff --git a/infra/relay/src/persistence/schema.ts b/infra/relay/src/persistence/schema.ts index 61b72f2df868..ddd276ccfd70 100644 --- a/infra/relay/src/persistence/schema.ts +++ b/infra/relay/src/persistence/schema.ts @@ -93,6 +93,7 @@ export const relayManagedEndpointAllocations = pgTable( tunnelName: text("tunnel_name").notNull(), dnsRecordId: varchar("dns_record_id", { length: 191 }), readyAt: varchar("ready_at", { length: 64 }), + recoveryEnabledAt: varchar("recovery_enabled_at", { length: 64 }), createdAt: varchar("created_at", { length: 64 }).notNull(), updatedAt: varchar("updated_at", { length: 64 }).notNull(), }, diff --git a/infra/relay/src/worker.ts b/infra/relay/src/worker.ts index 77dfd845c5bc..2f09fd3e5b9c 100644 --- a/infra/relay/src/worker.ts +++ b/infra/relay/src/worker.ts @@ -55,6 +55,7 @@ import * as EnvironmentConnector from "./environments/EnvironmentConnector.ts"; import * as EnvironmentLinker from "./environments/EnvironmentLinker.ts"; import * as EnvironmentPublishSignatures from "./environments/EnvironmentPublishSignatures.ts"; import * as ManagedEndpointProvider from "./environments/ManagedEndpointProvider.ts"; +import * as ManagedEndpointReaper from "./environments/ManagedEndpointReaper.ts"; import * as ManagedTunnelLimits from "./environments/ManagedTunnelLimits.ts"; import * as MobileRegistrations from "./agentActivity/MobileRegistrations.ts"; @@ -195,7 +196,9 @@ export const ApiLive = Api.make( Layer.provideMerge(AgentActivityPublisher.layer), Layer.provideMerge(EnvironmentConnector.layer), Layer.provideMerge(EnvironmentLinker.layer), - Layer.provideMerge(EnvironmentPublishSignatures.layer), + Layer.provideMerge( + Layer.merge(EnvironmentPublishSignatures.layer, ManagedEndpointReaper.layer), + ), Layer.provideMerge( ManagedEndpointProvider.layerCloudflareBindings( managedEndpointTunnelBinding, @@ -261,22 +264,38 @@ export const ApiLive = Api.make( ); yield* Cloudflare.Workers.cron("*/5 * * * *", () => - DpopProofs.DpopProofReplay.pipe( - Effect.flatMap((dpopProofs) => dpopProofs.pruneExpired), - // Terminal thread rows are kept briefly so finished agents show as - // Done/Failed in the Live Activity; sweep them once they age out. - Effect.andThen( - Effect.all([AgentActivityRows.AgentActivityRows, DateTime.now]).pipe( - Effect.flatMap(([activityRows, now]) => - activityRows.pruneTerminal({ - updatedBefore: DateTime.formatIso(DateTime.subtract(now, { minutes: 30 })), - }), + Effect.all( + [ + DpopProofs.DpopProofReplay.pipe( + Effect.flatMap((dpopProofs) => dpopProofs.pruneExpired), + // Keep completed thread rows long enough to show their final state. + Effect.andThen( + Effect.all([AgentActivityRows.AgentActivityRows, DateTime.now]).pipe( + Effect.flatMap(([activityRows, now]) => + activityRows.pruneTerminal({ + updatedBefore: DateTime.formatIso(DateTime.subtract(now, { minutes: 30 })), + }), + ), + ), + ), + Effect.catchCause((cause) => + Effect.logWarning("Failed to prune expired relay state", { cause }), ), ), - ), - Effect.withSpan("relay.cron.prune_expired_state"), - Effect.provide(runtimeLayer), - ), + ManagedEndpointReaper.ManagedEndpointReaper.pipe( + Effect.flatMap((reaper) => reaper.sweep), + Effect.tap((result) => + result.scanned > 0 + ? Effect.logInfo("Finished managed tunnel cleanup", result) + : Effect.void, + ), + Effect.catchCause((cause) => + Effect.logWarning("Failed to clean up inactive managed tunnels", { cause }), + ), + ), + ], + { concurrency: 2, discard: true }, + ).pipe(Effect.withSpan("relay.cron.prune_expired_state"), Effect.provide(runtimeLayer)), ); const fetch = Layer.merge( diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index 52f7d7d43550..76b0b0ef7438 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -159,6 +159,18 @@ export const RelayManagedEndpointRuntimeConfig = Schema.Struct({ }); export type RelayManagedEndpointRuntimeConfig = typeof RelayManagedEndpointRuntimeConfig.Type; +export const RelayManagedEndpointRecoveryRequest = Schema.Struct({ + cloudUserId: TrimmedNonEmptyString, + origin: RelayManagedEndpointOrigin, +}); +export type RelayManagedEndpointRecoveryRequest = typeof RelayManagedEndpointRecoveryRequest.Type; + +export const RelayManagedEndpointRecoveryResponse = Schema.Struct({ + endpoint: RelayManagedEndpoint, + endpointRuntime: RelayManagedEndpointRuntimeConfig, +}); +export type RelayManagedEndpointRecoveryResponse = typeof RelayManagedEndpointRecoveryResponse.Type; + export const RelayLinkProofRequest = Schema.Struct({ challenge: Schema.String, relayIssuer: Schema.String, @@ -1053,6 +1065,14 @@ export const RelayDpopClientGroup = HttpApiGroup.make("dpopClient") export const RelayServerGroup = HttpApiGroup.make("server") .add( + HttpApiEndpoint.post("recoverManagedEndpoint", "/v1/environments/:environmentId/tunnel", { + params: Schema.Struct({ + environmentId: EnvironmentId, + }), + payload: RelayManagedEndpointRecoveryRequest, + success: RelayManagedEndpointRecoveryResponse, + error: RelayAuthAndInternalErrors, + }).annotate(OpenApi.Summary, "Recover an environment's managed tunnel"), HttpApiEndpoint.post( "publishAgentActivity", "/v1/environments/:environmentId/threads/:threadId/agent-activity", From 51308315235250e7a6c6a4c1e69e2427df9f3d19 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 24 Aug 2026 20:15:43 -0700 Subject: [PATCH 02/14] fix(connect): protect tunnel recovery and cleanup from races --- .../src/cloud/ManagedEndpointRuntime.test.ts | 30 ++++ .../src/cloud/ManagedEndpointRuntime.ts | 3 + apps/server/src/cloud/http.test.ts | 2 + apps/server/src/cloud/http.ts | 169 ++++++++++-------- apps/server/src/server.test.ts | 1 + apps/server/src/server.ts | 31 ++-- .../environments/EnvironmentLinker.test.ts | 3 +- .../ManagedEndpointAllocations.test.ts | 52 +++++- .../ManagedEndpointAllocations.ts | 46 +++-- .../ManagedEndpointProvider.test.ts | 115 +++++++++++- .../environments/ManagedEndpointProvider.ts | 103 +++++++++-- .../ManagedEndpointReaper.test.ts | 169 +++++++++++++++++- .../src/environments/ManagedEndpointReaper.ts | 148 +++++++++------ infra/relay/src/http/Api.test.ts | 117 +++++++++++- infra/relay/src/http/Api.ts | 28 ++- 15 files changed, 830 insertions(+), 187 deletions(-) diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index b45b5099252a..54a12373c732 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -80,6 +80,36 @@ function makeHandle(input: { } describe("CloudManagedEndpointRuntime", () => { + it.effect("serializes updates to persisted cloud link state", () => + Effect.gen(function* () { + const firstEntered = yield* Deferred.make(); + const releaseFirst = yield* Deferred.make(); + const secondEntered = yield* Deferred.make(); + const runtime = yield* buildCloudManagedEndpointRuntime( + ChildProcessSpawner.make(() => Effect.die("unused")), + ); + + const first = yield* runtime + .withLinkStateLock( + Deferred.succeed(firstEntered, undefined).pipe( + Effect.andThen(Deferred.await(releaseFirst)), + ), + ) + .pipe(Effect.forkChild); + yield* Deferred.await(firstEntered); + + const second = yield* runtime + .withLinkStateLock(Deferred.succeed(secondEntered, undefined)) + .pipe(Effect.forkChild); + expect(yield* Deferred.isDone(secondEntered)).toBe(false); + + yield* Deferred.succeed(releaseFirst, undefined); + yield* Fiber.join(first); + yield* Fiber.join(second); + expect(yield* Deferred.isDone(secondEntered)).toBe(true); + }), + ); + it("classifies Cloudflare connection and warning output", () => { expect( ManagedEndpointRuntime.classifyRelayClientOutput( diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index 1bbdd5f08e69..cf566ac0a4c8 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -61,6 +61,7 @@ export class CloudManagedEndpointRuntime extends Context.Service< ) => Effect.Effect; readonly recoveryRequests: Stream.Stream; readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect; + readonly withLinkStateLock: (effect: Effect.Effect) => Effect.Effect; } >()("t3/cloud/ManagedEndpointRuntime/CloudManagedEndpointRuntime") {} @@ -109,6 +110,7 @@ export const make = Effect.gen(function* () { const desiredConfigRef = yield* Ref.make(null); const recoveryRequests = yield* PubSub.sliding(1); const reconcileSemaphore = yield* Semaphore.make(1); + const linkStateSemaphore = yield* Semaphore.make(1); let reconcileConfig: CloudManagedEndpointRuntime["Service"]["applyConfig"]; const stopActive = Effect.gen(function* () { @@ -312,6 +314,7 @@ export const make = Effect.gen(function* () { applyConfig, recoveryRequests: Stream.fromPubSub(recoveryRequests), requestRecovery: (config) => PubSub.publish(recoveryRequests, config).pipe(Effect.asVoid), + withLinkStateLock: linkStateSemaphore.withPermits(1), }); const initialConfig = yield* readRuntimeConfig.pipe( diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index 2f217f7bed0a..fd0328b2c414 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -219,6 +219,7 @@ describe("reconcileDesiredCloudLink", () => { applyConfig: unusedSecretStoreOperation, recoveryRequests: Stream.empty, requestRecovery: () => Effect.void, + withLinkStateLock: (effect) => effect, } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"]), ), Effect.provideService( @@ -325,6 +326,7 @@ describe("releaseManagedTunnelOnShutdown", () => { }), recoveryRequests: Stream.empty, requestRecovery: () => Effect.void, + withLinkStateLock: (effect) => effect, }), ), Effect.provideService( diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index 4d40c5d59299..9ae34e6d0d5c 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -456,48 +456,55 @@ const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(fu payload: RelayEnvironmentConfigRequest, options?: { readonly requestRecovery?: boolean }, ) { - yield* validateRelayConfigPayload(payload); - yield* validateLinkedCloudUser({ - secrets: dependencies.secrets, - cloudUserId: payload.cloudUserId, - }); - yield* validateCloudMintPublicKey(payload.cloudMintPublicKey); - const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig( - payload.endpointRuntime, - ); - const ok = - endpointRuntimeStatus.status === "disabled" || endpointRuntimeStatus.status === "running"; - if (!ok) { - return yield* new EnvironmentCloudEndpointUnavailableError({ - message: "Managed endpoint runtime could not be started.", - endpointRuntimeStatus, - }); - } + return yield* dependencies.endpointRuntime.withLinkStateLock( + Effect.gen(function* () { + yield* validateRelayConfigPayload(payload); + yield* validateLinkedCloudUser({ + secrets: dependencies.secrets, + cloudUserId: payload.cloudUserId, + }); + yield* validateCloudMintPublicKey(payload.cloudMintPublicKey); + const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig( + payload.endpointRuntime, + ); + const ok = + endpointRuntimeStatus.status === "disabled" || endpointRuntimeStatus.status === "running"; + if (!ok) { + return yield* new EnvironmentCloudEndpointUnavailableError({ + message: "Managed endpoint runtime could not be started.", + endpointRuntimeStatus, + }); + } - yield* dependencies.secrets.set(RELAY_URL_SECRET, stringToBytes(payload.relayUrl)); - yield* dependencies.secrets.set( - RELAY_ISSUER_SECRET, - stringToBytes(payload.relayIssuer ?? payload.relayUrl), - ); - yield* dependencies.secrets.set(CLOUD_LINKED_USER_ID, stringToBytes(payload.cloudUserId)); - yield* dependencies.secrets.set( - RELAY_ENVIRONMENT_CREDENTIAL_SECRET, - stringToBytes(payload.environmentCredential), + yield* dependencies.secrets.set(RELAY_URL_SECRET, stringToBytes(payload.relayUrl)); + yield* dependencies.secrets.set( + RELAY_ISSUER_SECRET, + stringToBytes(payload.relayIssuer ?? payload.relayUrl), + ); + yield* dependencies.secrets.set(CLOUD_LINKED_USER_ID, stringToBytes(payload.cloudUserId)); + yield* dependencies.secrets.set( + RELAY_ENVIRONMENT_CREDENTIAL_SECRET, + stringToBytes(payload.environmentCredential), + ); + yield* dependencies.secrets.set( + CLOUD_MINT_PUBLIC_KEY, + stringToBytes(payload.cloudMintPublicKey), + ); + if (payload.endpointRuntime) { + const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime); + yield* dependencies.secrets.set( + CLOUD_ENDPOINT_RUNTIME_CONFIG, + stringToBytes(endpointRuntimeJson), + ); + if (options?.requestRecovery !== false) { + yield* dependencies.endpointRuntime.requestRecovery(payload.endpointRuntime); + } + } else { + yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); + } + return { ok, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; + }), ); - yield* dependencies.secrets.set(CLOUD_MINT_PUBLIC_KEY, stringToBytes(payload.cloudMintPublicKey)); - if (payload.endpointRuntime) { - const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime); - yield* dependencies.secrets.set( - CLOUD_ENDPOINT_RUNTIME_CONFIG, - stringToBytes(endpointRuntimeJson), - ); - if (options?.requestRecovery !== false) { - yield* dependencies.endpointRuntime.requestRecovery(payload.endpointRuntime); - } - } else { - yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); - } - return { ok, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; }); const cloudRelayConfigHandler = Effect.fn("environment.cloud.relayConfig")( @@ -694,31 +701,35 @@ export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverMan }); } - const currentConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); - if ( - Option.isNone(currentConfig) || - bytesToString(currentConfig.value) !== bytesToString(runtimeConfig.value) - ) { - return false; - } + return yield* dependencies.endpointRuntime.withLinkStateLock( + Effect.gen(function* () { + const currentConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); + if ( + Option.isNone(currentConfig) || + bytesToString(currentConfig.value) !== bytesToString(runtimeConfig.value) + ) { + return false; + } - const status = yield* dependencies.endpointRuntime.applyConfig(recovered.endpointRuntime); - if (status.status !== "running") { - return yield* new EnvironmentCloudEndpointUnavailableError({ - message: "Managed endpoint runtime could not be started.", - endpointRuntimeStatus: status, - }); - } - const encoded = yield* encodeEndpointRuntimeConfigJson(recovered.endpointRuntime).pipe( - Effect.mapError( - () => - new EnvironmentHttpInternalServerError({ - message: "Could not persist the recovered managed tunnel configuration.", - }), - ), + const status = yield* dependencies.endpointRuntime.applyConfig(recovered.endpointRuntime); + if (status.status !== "running") { + return yield* new EnvironmentCloudEndpointUnavailableError({ + message: "Managed endpoint runtime could not be started.", + endpointRuntimeStatus: status, + }); + } + const encoded = yield* encodeEndpointRuntimeConfigJson(recovered.endpointRuntime).pipe( + Effect.mapError( + () => + new EnvironmentHttpInternalServerError({ + message: "Could not persist the recovered managed tunnel configuration.", + }), + ), + ); + yield* dependencies.secrets.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(encoded)); + return true; + }), ); - yield* dependencies.secrets.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(encoded)); - return true; }, ); @@ -875,21 +886,25 @@ const cloudLinkStateHandler = Effect.fn("environment.cloud.linkState")( const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")( function* (dependencies: CloudHttpDependencies) { yield* requireEnvironmentScope(AuthRelayWriteScope); - const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig(null); - yield* Effect.all( - [ - dependencies.secrets.remove(CLOUD_LINKED_USER_ID), - dependencies.secrets.remove(RELAY_URL_SECRET), - dependencies.secrets.remove(RELAY_ISSUER_SECRET), - dependencies.secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), - dependencies.secrets.remove(CLOUD_MINT_PUBLIC_KEY), - dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG), - dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET), - ], - { concurrency: 7 }, + return yield* dependencies.endpointRuntime.withLinkStateLock( + Effect.gen(function* () { + const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig(null); + yield* Effect.all( + [ + dependencies.secrets.remove(CLOUD_LINKED_USER_ID), + dependencies.secrets.remove(RELAY_URL_SECRET), + dependencies.secrets.remove(RELAY_ISSUER_SECRET), + dependencies.secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), + dependencies.secrets.remove(CLOUD_MINT_PUBLIC_KEY), + dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG), + dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET), + ], + { concurrency: 7 }, + ); + yield* setCliDesiredCloudLink(false); + return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; + }), ); - yield* setCliDesiredCloudLink(false); - return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; }, Effect.catchIf( ServerSecretStore.isSecretStoreError, diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 5dd10b374d9f..ed29599f029f 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -950,6 +950,7 @@ const buildAppUnderTest = (options?: { applyConfig: () => Effect.succeed({ status: "disabled" }), recoveryRequests: Stream.empty, requestRecovery: () => Effect.void, + withLinkStateLock: (effect) => effect, ...options?.layers?.cloudManagedEndpointRuntime, }), ), diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index ca80ec268dce..7e66e400218d 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -4,6 +4,7 @@ import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schedule from "effect/Schedule"; +import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import { FetchHttpClient, HttpRouter, HttpServer } from "effect/unstable/http"; import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder"; @@ -627,20 +628,26 @@ export const makeServerLayer = Layer.unwrap( if (typeof address === "string" || !("port" in address)) return; const localOrigin = `http://127.0.0.1:${address.port}`; const endpointRuntime = yield* CloudManagedEndpointRuntime.CloudManagedEndpointRuntime; - const recoverManagedTunnel = recoverManagedCloudTunnel(localOrigin).pipe( - Effect.retry({ - schedule: Schedule.exponential("1 second").pipe( - Schedule.modifyDelay(({ duration }) => - Effect.succeed(Duration.min(duration, Duration.seconds(30))), + const recoveryLock = yield* Semaphore.make(1); + const recoverManagedTunnel = recoveryLock.withPermits(1)( + recoverManagedCloudTunnel(localOrigin).pipe( + Effect.retry({ + while: (error) => + error._tag !== "EnvironmentHttpBadRequestError" && + error._tag !== "EnvironmentCloudEndpointUnavailableError", + schedule: Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + Schedule.upTo({ duration: "10 minutes" }), ), - Schedule.upTo({ duration: "10 minutes" }), + }), + Effect.tap((recovered) => + recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void, + ), + Effect.catchCause((cause) => + Effect.logWarning("Failed to recover the T3 Connect managed tunnel", { cause }), ), - }), - Effect.tap((recovered) => - recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void, - ), - Effect.catchCause((cause) => - Effect.logWarning("Failed to recover the T3 Connect managed tunnel", { cause }), ), ); yield* endpointRuntime.recoveryRequests.pipe( diff --git a/infra/relay/src/environments/EnvironmentLinker.test.ts b/infra/relay/src/environments/EnvironmentLinker.test.ts index c0811e82d923..536c4e289695 100644 --- a/infra/relay/src/environments/EnvironmentLinker.test.ts +++ b/infra/relay/src/environments/EnvironmentLinker.test.ts @@ -137,7 +137,7 @@ function testLayer(input?: { }), Layer.succeed(ManagedEndpointProvider.ManagedEndpointProvider, { prepareDeprovision: () => Effect.succeed(null), - deprovision: input?.deprovision ?? (() => Effect.void), + deprovision: input?.deprovision ?? (() => Effect.succeed(true)), release: () => Effect.succeed(true), provision: () => Effect.succeed({ @@ -243,6 +243,7 @@ describe("EnvironmentLinker", () => { deprovision: (input) => Effect.sync(() => { deprovisionedEnvironmentId = input.environmentId; + return true; }), }), ), diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts index c4c06e9e723e..ba014dd8d576 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import { PgDialect } from "drizzle-orm/pg-core"; import * as RelayDb from "../db.ts"; import { relayManagedEndpointAllocations } from "../persistence/schema.ts"; @@ -17,6 +18,7 @@ describe("ManagedEndpointAllocations", () => { readonly updatedAt: string; } | undefined; + let condition: unknown; const fakeDb = { update: (table: unknown) => { expect(table).toBe(relayManagedEndpointAllocations); @@ -24,7 +26,12 @@ describe("ManagedEndpointAllocations", () => { set: (values: { readonly recoveryEnabledAt: string; readonly updatedAt: string }) => { updated = values; return { - where: () => Effect.void, + where: (where: unknown) => { + condition = where; + return { + returning: () => Effect.succeed([{ environmentId: "environment-1" }]), + }; + }, }; }, }; @@ -33,13 +40,48 @@ describe("ManagedEndpointAllocations", () => { return Effect.gen(function* () { const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; - yield* allocations.enableRecovery({ - userId: "user-1", - environmentId: "environment-1", - }); + expect( + yield* allocations.enableRecovery({ + userId: "user-1", + environmentId: "environment-1", + tunnelId: "tunnel-1", + environmentPublicKey: "public-key", + }), + ).toBe(true); expect(updated?.recoveryEnabledAt).toBe(updated?.updatedAt); expect(updated?.recoveryEnabledAt).toBeDefined(); + const query = new PgDialect().sqlToQuery(condition as never); + expect(query.sql).toContain('"relay_managed_endpoint_allocations"."tunnel_id"'); + expect(query.sql).toContain('"relay_environment_links"."environment_public_key"'); + expect(query.sql).toContain('"relay_environment_links"."revoked_at" is null'); + expect(query.sql).toContain("for update"); + expect(query.params).toContain("tunnel-1"); + expect(query.params).toContain("public-key"); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + + it.effect("rejects recovery when the tunnel or active link no longer matches", () => { + const fakeDb = { + update: () => ({ + set: () => ({ + where: () => ({ + returning: () => Effect.succeed([]), + }), + }), + }), + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + expect( + yield* allocations.enableRecovery({ + userId: "user-1", + environmentId: "environment-1", + tunnelId: "missing-tunnel", + environmentPublicKey: "public-key", + }), + ).toBe(false); }).pipe(Effect.provide(layerWithDb(fakeDb))); }); diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index 4fbdadd20239..6b67c6be93f1 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -1,5 +1,6 @@ import type { RelayManagedEndpoint } from "@t3tools/contracts/relay"; -import { and, eq, inArray, isNull } from "drizzle-orm"; +import { and, eq, exists, inArray, isNull } from "drizzle-orm"; +import { QueryBuilder } from "drizzle-orm/pg-core"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -8,7 +9,7 @@ import * as Schema from "effect/Schema"; import * as RelayDb from "../db.ts"; import { isManagedEndpointHostname, managedEndpointForHostname } from "../deploymentConfig.ts"; -import { relayManagedEndpointAllocations } from "../persistence/schema.ts"; +import { relayEnvironmentLinks, relayManagedEndpointAllocations } from "../persistence/schema.ts"; export interface ManagedEndpointAllocation { readonly userId: string; @@ -99,6 +100,11 @@ interface ClaimManagedEndpointReleaseInput extends ManagedEndpointAllocationKey readonly updatedAt: string; } +interface EnableManagedEndpointRecoveryInput extends ManagedEndpointAllocationKey { + readonly tunnelId: string; + readonly environmentPublicKey: string; +} + interface ClaimManagedEndpointDeprovisionInput extends ManagedEndpointAllocationKey { readonly updatedAt: string; } @@ -126,8 +132,8 @@ export class ManagedEndpointAllocations extends Context.Service< input: ManagedEndpointAllocationKey, ) => Effect.Effect; readonly enableRecovery: ( - input: ManagedEndpointAllocationKey, - ) => Effect.Effect; + input: EnableManagedEndpointRecoveryInput, + ) => Effect.Effect; readonly listByTunnelNames: ( tunnelNames: ReadonlyArray, ) => Effect.Effect< @@ -143,7 +149,7 @@ export class ManagedEndpointAllocations extends Context.Service< */ readonly claimRelease: ( input: ClaimManagedEndpointReleaseInput, - ) => Effect.Effect; + ) => Effect.Effect; /** * Claims the complete allocation for teardown only if its generation still * matches the snapshot captured by the unlink operation. @@ -328,16 +334,35 @@ export const make = Effect.gen(function* () { ); }), enableRecovery: Effect.fn("relay.managed_endpoint_allocations.enable_recovery")(function* ( - input: ManagedEndpointAllocationKey, + input: EnableManagedEndpointRecoveryInput, ) { const now = DateTime.formatIso(yield* DateTime.now); - yield* db + return yield* db .update(relayManagedEndpointAllocations) .set({ recoveryEnabledAt: now, updatedAt: now }) .where( - and(whereAllocation(input), isNull(relayManagedEndpointAllocations.recoveryEnabledAt)), + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), + exists( + new QueryBuilder() + .select({ userId: relayEnvironmentLinks.userId }) + .from(relayEnvironmentLinks) + .where( + and( + eq(relayEnvironmentLinks.userId, input.userId), + eq(relayEnvironmentLinks.environmentId, input.environmentId), + eq(relayEnvironmentLinks.environmentPublicKey, input.environmentPublicKey), + isNull(relayEnvironmentLinks.revokedAt), + ), + ) + .for("update"), + ), + ), ) + .returning({ environmentId: relayManagedEndpointAllocations.environmentId }) .pipe( + Effect.map((rows) => rows.length > 0), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -384,10 +409,11 @@ export const make = Effect.gen(function* () { claimRelease: Effect.fn("relay.managed_endpoint_allocations.claim_release")(function* ( input: ClaimManagedEndpointReleaseInput, ) { + const claimedAt = DateTime.formatIso(yield* DateTime.now); const claimed = yield* db .update(relayManagedEndpointAllocations) .set({ - updatedAt: DateTime.formatIso(yield* DateTime.now), + updatedAt: claimedAt, }) .where( and( @@ -411,7 +437,7 @@ export const make = Effect.gen(function* () { }), ), ); - return claimed; + return claimed ? claimedAt : null; }), claimDeprovision: Effect.fn("relay.managed_endpoint_allocations.claim_deprovision")(function* ( input: ClaimManagedEndpointDeprovisionInput, diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 8ec3f3762b75..769466554c68 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -33,7 +33,7 @@ const config = RelayConfiguration.RelayConfiguration.of({ }); interface TunnelCall { - readonly operation: "list" | "create" | "putConfiguration" | "getToken" | "delete"; + readonly operation: "get" | "list" | "create" | "putConfiguration" | "getToken" | "delete"; readonly input: unknown; } @@ -62,6 +62,16 @@ function allocationKey(input: { readonly userId: string; readonly environmentId: function makeTunnelClient(calls: TunnelCall[] = []) { return ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + get: (tunnelId) => + Effect.sync(() => { + calls.push({ operation: "get", input: tunnelId }); + return { + id: tunnelId, + name: "managed-tunnel", + status: "down", + connsInactiveAt: "2026-06-01T00:00:00.000Z", + }; + }), list: (request) => Effect.sync(() => { calls.push({ operation: "list", input: request }); @@ -91,6 +101,23 @@ function makeTunnelClient(calls: TunnelCall[] = []) { function makePersistentTunnelClient(calls: TunnelCall[] = []) { let tunnel: { readonly id: string; readonly name: string } | null = null; return ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + get: (tunnelId) => + Effect.suspend(() => { + calls.push({ operation: "get", input: tunnelId }); + return tunnel === null + ? Effect.fail( + new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "get", + tunnelId, + cause: { _tag: "NotFound" }, + }), + ) + : Effect.succeed({ + ...tunnel, + status: "down", + connsInactiveAt: "2026-06-01T00:00:00.000Z", + }); + }), list: (request) => Effect.sync(() => { calls.push({ operation: "list", input: request }); @@ -217,7 +244,12 @@ function makeAllocations(calls: AllocationCall[] = []) { }), enableRecovery: (input) => Effect.sync(() => { + const allocation = allocations.get(allocationKey(input)); + if (allocation?.tunnelId !== input.tunnelId) { + return false; + } recoveryEnabled.add(allocationKey(input)); + return true; }), listByTunnelNames: (tunnelNames) => Effect.sync(() => @@ -237,10 +269,10 @@ function makeAllocations(calls: AllocationCall[] = []) { allocation.tunnelId !== input.tunnelId || allocation.updatedAt !== input.updatedAt ) { - return false; + return null; } mutate(allocationKey(input), (current) => current); - return true; + return allocations.get(allocationKey(input))?.updatedAt ?? null; }), claimDeprovision: (input) => Effect.sync(() => { @@ -320,6 +352,7 @@ function expectedManagedTunnelName(environmentId: string, userId = "user_ABC"): describe("ManagedEndpointProvider", () => { it.effect("does not require the deployment RuntimeContext when building the Worker layer", () => { const tunnelClient = { + get: () => Effect.succeed({ id: "tunnel-id", name: "managed-tunnel" }), list: () => Effect.succeed({ result: [] }), create: (request: { readonly name: string }) => Effect.succeed({ id: "tunnel-id", name: request.name }), @@ -928,7 +961,7 @@ describe("ManagedEndpointProvider", () => { // longer matches what the release loaded, so the claim fails. const outdated = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ ...allocations, - claimRelease: () => Effect.succeed(false), + claimRelease: () => Effect.succeed(null), }); const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), outdated); @@ -974,6 +1007,80 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.provide(layer)); }); + it.effect("keeps a tunnel that reconnects before scheduled deletion", () => { + const tunnelCalls: TunnelCall[] = []; + const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + ...makePersistentTunnelClient(tunnelCalls), + get: (tunnelId) => + Effect.succeed({ + id: tunnelId, + name: expectedManagedTunnelName("env_ABC"), + status: "healthy", + connsInactiveAt: null, + }), + }); + const layer = providerLayer(tunnelClient, makeDnsClient(), makeAllocations()); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + + expect( + yield* provider.release({ + ...key, + expectedTunnelId: "tunnel-id", + expectedStatus: "down", + expectedInactiveBefore: "2026-06-01T00:05:00.000Z", + }), + ).toBe(false); + expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete"); + }).pipe(Effect.provide(layer)); + }); + + it.effect("keeps a tunnel when a new provision replaces the release generation", () => { + const tunnelCalls: TunnelCall[] = []; + const allocations = makeAllocations(); + const replaced = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + claimRelease: (input) => + allocations.claimRelease(input).pipe( + Effect.tap((claimedAt) => + claimedAt === null + ? Effect.void + : allocations.recordTunnel({ + userId: input.userId, + environmentId: input.environmentId, + tunnelId: "replacement-tunnel", + }), + ), + ), + }); + const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), replaced); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + + expect( + yield* provider.release({ + ...key, + expectedTunnelId: "tunnel-id", + expectedStatus: "down", + expectedInactiveBefore: "2026-06-01T00:05:00.000Z", + }), + ).toBe(false); + expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete"); + }).pipe(Effect.provide(layer)); + }); + it.effect("treats an already deleted tunnel as successfully released", () => { const notFound = { _tag: "NotFound" } as const; const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 1789c014ae5d..41852f20f282 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -3,6 +3,7 @@ import * as Cloudflare from "alchemy/Cloudflare"; import * as Arr from "effect/Array"; import * as Context from "effect/Context"; import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Encoding from "effect/Encoding"; import * as Layer from "effect/Layer"; @@ -76,6 +77,7 @@ export class ManagedEndpointProvisioningFailed extends Schema.TaggedErrorClass Effect.Effect; + }) => Effect.Effect; /** * Deletes the provisioned Cloudflare tunnel while keeping the allocation * (hostname + tunnel name reservation) and DNS record. Cloudflare bills per @@ -167,6 +169,8 @@ export class ManagedEndpointProvider extends Context.Service< readonly userId: string; readonly environmentId: string; readonly expectedTunnelId?: string; + readonly expectedInactiveBefore?: string; + readonly expectedStatus?: "inactive" | "down"; }) => Effect.Effect; } >()("t3code-relay/environments/ManagedEndpointProvider") {} @@ -191,6 +195,7 @@ export interface ManagedEndpointTunnelListRequest { } const ManagedEndpointTunnelClientOperation = Schema.Literals([ + "get", "list", "create", "put-configuration", @@ -216,6 +221,9 @@ export class ManagedEndpointTunnelClientError extends Schema.TaggedErrorClass Effect.Effect; readonly list: (request: ManagedEndpointTunnelListRequest) => Effect.Effect< { readonly result: ReadonlyArray; @@ -352,17 +360,17 @@ function isLoopbackOrigin(origin: RelayManagedEndpointOrigin): boolean { ); } -function isNotFoundCause(cause: unknown): boolean { +export function isManagedEndpointNotFound(cause: unknown): boolean { if (typeof cause !== "object" || cause === null) { return false; } - if ("_tag" in cause && cause._tag === "NotFound") { + if ("_tag" in cause && (cause._tag === "NotFound" || cause._tag === "TunnelNotFound")) { return true; } if ("status" in cause && cause.status === 404) { return true; } - return "cause" in cause && isNotFoundCause(cause.cause); + return "cause" in cause && isManagedEndpointNotFound(cause.cause); } type ManagedEndpointClientError = ManagedEndpointTunnelClientError | ManagedEndpointDnsClientError; @@ -374,9 +382,9 @@ const ignoreNotFound = ( Effect.asVoid, Effect.catchTags({ ManagedEndpointTunnelClientError: (error) => - isNotFoundCause(error.cause) ? Effect.void : Effect.fail(error), + isManagedEndpointNotFound(error.cause) ? Effect.void : Effect.fail(error), ManagedEndpointDnsClientError: (error) => - isNotFoundCause(error.cause) ? Effect.void : Effect.fail(error), + isManagedEndpointNotFound(error.cause) ? Effect.void : Effect.fail(error), }), ); @@ -418,7 +426,7 @@ export const make = Effect.gen(function* () { Effect.as(true), Effect.catchTags({ ManagedEndpointDnsClientError: (error) => - isNotFoundCause(error.cause) ? Effect.succeed(false) : Effect.fail(error), + isManagedEndpointNotFound(error.cause) ? Effect.succeed(false) : Effect.fail(error), }), ); if (checkpointedRecordUpdated) { @@ -484,7 +492,7 @@ export const make = Effect.gen(function* () { const allocation = input.target === undefined ? yield* prepareDeprovision(input) : input.target; if (allocation === null) { - return; + return true; } const claimedAt = yield* allocations .claimDeprovision({ @@ -505,7 +513,7 @@ export const make = Effect.gen(function* () { ), ); if (claimedAt === null) { - return; + return false; } const dnsRecordId = allocation.dnsRecordId; if (dnsRecordId !== null) { @@ -535,7 +543,7 @@ export const make = Effect.gen(function* () { ), ); } - yield* allocations + return yield* allocations .removeClaimed({ userId: input.userId, environmentId: input.environmentId, @@ -583,7 +591,7 @@ export const make = Effect.gen(function* () { // must be left alive. A provision that starts after the claim instead // fails loudly on the deleted tunnel and the client-side retry // provisions a replacement. - const claimed = yield* allocations + const claimedAt = yield* allocations .claimRelease({ userId: input.userId, environmentId: input.environmentId, @@ -601,9 +609,68 @@ export const make = Effect.gen(function* () { }), ), ); - if (!claimed) { + if (claimedAt === null) { return false; } + if (input.expectedInactiveBefore !== undefined && input.expectedStatus !== undefined) { + const currentAllocation = yield* allocations.get(input).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "load-allocation", + tunnelId, + cause, + }), + ), + ); + if ( + currentAllocation === null || + currentAllocation.tunnelId !== tunnelId || + currentAllocation.updatedAt !== claimedAt + ) { + return false; + } + + const currentTunnel = yield* tunnels.get(tunnelId).pipe( + Effect.map(Option.some), + Effect.catchTag("ManagedEndpointTunnelClientError", (cause) => + isManagedEndpointNotFound(cause.cause) + ? Effect.succeed(Option.none()) + : Effect.fail( + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "load-tunnel", + tunnelId, + cause, + }), + ), + ), + ); + if (Option.isNone(currentTunnel)) { + return true; + } + const inactiveAt = + input.expectedStatus === "down" + ? currentTunnel.value.connsInactiveAt + : currentTunnel.value.createdAt; + if ( + currentTunnel.value.id !== tunnelId || + currentTunnel.value.status !== input.expectedStatus || + typeof inactiveAt !== "string" + ) { + return false; + } + const inactiveTime = DateTime.make(inactiveAt); + const cutoff = DateTime.make(input.expectedInactiveBefore); + if ( + Option.isNone(inactiveTime) || + Option.isNone(cutoff) || + inactiveTime.value.epochMilliseconds > cutoff.value.epochMilliseconds + ) { + return false; + } + } yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe( Effect.mapError( (cause) => @@ -890,6 +957,18 @@ export const layerCloudflareBindings = ( Layer.provideMerge( Layer.mergeAll( layerTunnelClient({ + get: (tunnelId) => + tunnelClient.get(tunnelId).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointTunnelClientError({ + operation: "get", + tunnelId, + cause, + }), + ), + Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), + ), list: (request) => tunnelClient.list(request).pipe( Effect.mapError( diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts index c8e4816b348e..cc4c43b6ae23 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -55,24 +55,58 @@ function harness(input?: { readonly allocations?: ReadonlyArray; readonly namespace?: string; readonly failTunnelId?: string; + readonly missingOnDeleteTunnelId?: string; + readonly missingOnGetTunnelId?: string; + readonly reserveOnGetTunnelId?: string; + readonly refreshedTunnels?: ReadonlyMap; readonly skipTunnelId?: string; }) { const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = []; const deleted: string[] = []; - const releases: Array<{ - readonly userId: string; - readonly environmentId: string; - readonly expectedTunnelId?: string; - }> = []; + const releases: Array< + Parameters[0] + > = []; + const remaining = [...(input?.tunnels ?? [])]; const recorded = (input?.allocations ?? []).map((entry) => { - const matching = input?.tunnels?.find((candidate) => candidate.id === entry.tunnelId); + const matching = remaining.find((candidate) => candidate.id === entry.tunnelId); return typeof matching?.name === "string" ? { ...entry, tunnelName: matching.name } : entry; }); const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + get: (tunnelId) => + Effect.suspend(() => { + if (tunnelId === input?.missingOnGetTunnelId) { + return Effect.fail( + new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "get", + tunnelId, + cause: { _tag: "NotFound" }, + }), + ); + } + const found = + input?.refreshedTunnels?.get(tunnelId) ?? + remaining.find((candidate) => candidate.id === tunnelId); + if (found === undefined) { + return Effect.fail( + new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "get", + tunnelId, + cause: { _tag: "NotFound" }, + }), + ); + } + if (tunnelId === input?.reserveOnGetTunnelId && typeof found.name === "string") { + recorded.push({ + ...allocation({ tunnelId, recoveryEnabled: false }), + tunnelName: found.name, + }); + } + return Effect.succeed(found); + }), list: (request) => Effect.sync(() => { listRequests.push(request); - const matching = (input?.tunnels ?? []).filter((entry) => entry.status === request.status); + const matching = remaining.filter((entry) => entry.status === request.status); const start = ((request.page ?? 1) - 1) * (request.perPage ?? 100); return { result: matching.slice(start, start + (request.perPage ?? 100)), @@ -87,16 +121,23 @@ function harness(input?: { putConfiguration: () => Effect.die("unused"), getToken: () => Effect.die("unused"), delete: (tunnelId) => - tunnelId === input?.failTunnelId + tunnelId === input?.failTunnelId || tunnelId === input?.missingOnDeleteTunnelId ? Effect.fail( new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ operation: "delete", tunnelId, - cause: "Cloudflare refused the deletion", + cause: + tunnelId === input?.missingOnDeleteTunnelId + ? { _tag: "NotFound" } + : "Cloudflare refused the deletion", }), ) : Effect.sync(() => { deleted.push(tunnelId); + const index = remaining.findIndex((candidate) => candidate.id === tunnelId); + if (index !== -1) { + remaining.splice(index, 1); + } }), }); const allocationService = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ @@ -125,6 +166,12 @@ function harness(input?: { } if (request.expectedTunnelId !== undefined) { deleted.push(request.expectedTunnelId); + const index = remaining.findIndex( + (candidate) => candidate.id === request.expectedTunnelId, + ); + if (index !== -1) { + remaining.splice(index, 1); + } } return true; }), @@ -309,6 +356,81 @@ describe("ManagedEndpointReaper", () => { }).pipe(Effect.provide(state.layer)); }); + it.effect("keeps an orphan tunnel that reconnects before deletion", () => { + const listed = tunnel({ + id: "reconnected", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }); + const state = harness({ + tunnels: [listed], + refreshedTunnels: new Map([ + [ + "reconnected", + tunnel({ + id: "reconnected", + suffix: "aaaaaaaaaaaaaaaa", + status: "healthy", + }), + ], + ]), + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(0); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("treats an already deleted orphan tunnel as successfully removed", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "gone", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + missingOnDeleteTunnelId: "gone", + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 1, + deleted: 1, + skippedLegacy: 0, + failed: 0, + }); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("does not delete an orphan tunnel reserved during the status check", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "reserved", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + reserveOnGetTunnelId: "reserved", + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(0); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + it.effect("does not count a tunnel that was replaced before its release", () => { const state = harness({ tunnels: [ @@ -397,6 +519,35 @@ describe("ManagedEndpointReaper", () => { }).pipe(Effect.provide(state.layer)); }); + it.effect("collects every page before deletions shift Cloudflare pagination", () => { + const entries = Array.from({ length: 120 }, (_, index) => + tunnel({ + id: `tunnel-${index}`, + suffix: index.toString(16).padStart(16, "0"), + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ); + const state = harness({ + tunnels: entries, + allocations: entries + .slice(50, 100) + .map((entry) => allocation({ tunnelId: entry.id!, recoveryEnabled: false })), + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 120, + deleted: 70, + skippedLegacy: 50, + failed: 0, + }); + expect(state.deleted).toContain("tunnel-119"); + }).pipe(Effect.provide(state.layer)); + }); + it.effect("limits each cleanup run to 100 tunnel deletions", () => { const state = harness({ tunnels: Array.from({ length: 105 }, (_, index) => diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts index cfd933ebe650..da039f64030a 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -67,6 +67,42 @@ export const make = Effect.gen(function* () { const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const deleteOrphan = Effect.fn("relay.managed_endpoint_reaper.delete_orphan")(function* (input: { + readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & { + readonly id: string; + readonly name: string; + }; + readonly status: "down" | "inactive"; + readonly prefix: string; + readonly cutoff: DateTime.Utc; + }) { + const current = yield* tunnels.get(input.tunnel.id).pipe( + Effect.map(Option.some), + Effect.catchTag("ManagedEndpointTunnelClientError", (error) => + ManagedEndpointProvider.isManagedEndpointNotFound(error.cause) + ? Effect.succeed(Option.none()) + : Effect.fail(error), + ), + ); + if (Option.isNone(current)) { + return true; + } + if (!isExpiredManagedTunnel({ ...input, tunnel: current.value })) { + return false; + } + if ((yield* allocations.listByTunnelNames([input.tunnel.name])).length > 0) { + return false; + } + return yield* tunnels.delete(input.tunnel.id).pipe( + Effect.as(true), + Effect.catchTag("ManagedEndpointTunnelClientError", (error) => + ManagedEndpointProvider.isManagedEndpointNotFound(error.cause) + ? Effect.succeed(true) + : Effect.fail(error), + ), + ); + }); + const sweep = Effect.gen(function* () { const namespace = config.managedEndpointNamespace; if (!namespace) { @@ -77,14 +113,20 @@ export const make = Effect.gen(function* () { const cutoff = DateTime.subtract(now, { minutes: MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES }); const cutoffIso = DateTime.formatIso(cutoff); const prefix = managedEndpointTunnelNamePrefix(namespace); - let scanned = 0; let deleted = 0; let skippedLegacy = 0; let failed = 0; + const expired: Array<{ + readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & { + readonly id: string; + readonly name: string; + }; + readonly status: "down" | "inactive"; + }> = []; for (const status of ["down", "inactive"] as const) { let page = 1; - while (deleted < MANAGED_ENDPOINT_SWEEP_DELETE_LIMIT) { + while (true) { const response = yield* tunnels.list({ isDeleted: false, includePrefix: prefix, @@ -94,56 +136,13 @@ export const make = Effect.gen(function* () { page, perPage: MANAGED_ENDPOINT_SWEEP_PAGE_SIZE, }); - const expired = response.result - .map((tunnel) => ({ tunnel, status, prefix, cutoff })) - .filter(isExpiredManagedTunnel) - .map(({ tunnel }) => tunnel); - scanned += expired.length; - - const recorded = yield* allocations.listByTunnelNames(expired.map((tunnel) => tunnel.name)); - const recordedByTunnelId = new Map( - recorded - .filter((allocation) => allocation.tunnelId !== null) - .map((allocation) => [allocation.tunnelId, allocation]), + expired.push( + ...response.result + .map((tunnel) => ({ tunnel, status, prefix, cutoff })) + .filter(isExpiredManagedTunnel) + .map(({ tunnel }) => ({ tunnel, status })), ); - for (const tunnel of expired) { - if (deleted >= MANAGED_ENDPOINT_SWEEP_DELETE_LIMIT) { - break; - } - const allocation = recordedByTunnelId.get(tunnel.id); - if (allocation !== undefined && !allocation.recoveryEnabled) { - skippedLegacy += 1; - continue; - } - - const result = - allocation === undefined - ? yield* tunnels.delete(tunnel.id).pipe(Effect.as(true), Effect.result) - : yield* provider - .release({ - userId: allocation.userId, - environmentId: allocation.environmentId, - expectedTunnelId: tunnel.id, - }) - .pipe(Effect.result); - if (result._tag === "Failure") { - failed += 1; - yield* Effect.logWarning("Failed to delete an inactive managed tunnel", { - tunnelId: tunnel.id, - tunnelName: tunnel.name, - cause: result.failure, - }); - } else if (result.success) { - deleted += 1; - yield* Effect.logInfo("Deleted an inactive managed tunnel", { - tunnelId: tunnel.id, - tunnelName: tunnel.name, - status, - }); - } - } - const totalCount = response.resultInfo?.totalCount; if ( response.result.length === 0 || @@ -157,7 +156,54 @@ export const make = Effect.gen(function* () { } } - return { scanned, deleted, skippedLegacy, failed }; + const recorded = yield* allocations.listByTunnelNames(expired.map(({ tunnel }) => tunnel.name)); + const recordedByTunnelName = new Map( + recorded.map((allocation) => [allocation.tunnelName, allocation]), + ); + + for (const { tunnel, status } of expired) { + if (deleted >= MANAGED_ENDPOINT_SWEEP_DELETE_LIMIT) { + break; + } + const allocation = recordedByTunnelName.get(tunnel.name); + if (allocation !== undefined && allocation.tunnelId !== tunnel.id) { + continue; + } + if (allocation !== undefined && !allocation.recoveryEnabled) { + skippedLegacy += 1; + continue; + } + + const result = + allocation === undefined + ? yield* deleteOrphan({ tunnel, status, prefix, cutoff }).pipe(Effect.result) + : yield* provider + .release({ + userId: allocation.userId, + environmentId: allocation.environmentId, + expectedTunnelId: tunnel.id, + expectedInactiveBefore: cutoffIso, + expectedStatus: status, + }) + .pipe(Effect.result); + if (result._tag === "Failure") { + failed += 1; + yield* Effect.logWarning("Failed to delete an inactive managed tunnel", { + tunnelId: tunnel.id, + tunnelName: tunnel.name, + cause: result.failure, + }); + } else if (result.success) { + deleted += 1; + yield* Effect.logInfo("Deleted an inactive managed tunnel", { + tunnelId: tunnel.id, + tunnelName: tunnel.name, + status, + }); + } + } + + return { scanned: expired.length, deleted, skippedLegacy, failed }; }).pipe(Effect.withSpan("relay.managed_endpoint_reaper.sweep")); return ManagedEndpointReaper.of({ sweep }); diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index 12b39dd3e7e9..5453d492bb22 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -204,7 +204,7 @@ function relayUnlinkTestLayer(input?: { ManagedEndpointProvider.ManagedEndpointProvider.of({ provision: input?.provision ?? (() => Effect.die("unused provision")), prepareDeprovision: input?.prepareDeprovision ?? (() => Effect.succeed(null)), - deprovision: input?.deprovision ?? (() => Effect.void), + deprovision: input?.deprovision ?? (() => Effect.succeed(true)), release: () => Effect.die("unused release"), }), ), @@ -225,8 +225,12 @@ const linkedEnvironmentRecord = { describe("relay managed tunnel recovery", () => { it.effect("recovers a linked environment and marks its tunnel as recoverable", () => { - let recoveryEnabledFor: { readonly userId: string; readonly environmentId: string } | null = - null; + let recoveryEnabledFor: { + readonly userId: string; + readonly environmentId: string; + readonly tunnelId: string; + readonly environmentPublicKey: string; + } | null = null; const runtime = { providerKind: "cloudflare_tunnel" as const, connectorToken: "replacement-token", @@ -248,6 +252,8 @@ describe("relay managed tunnel recovery", () => { expect(recoveryEnabledFor).toEqual({ userId: "user-1", environmentId: "environment-1", + tunnelId: "replacement-tunnel", + environmentPublicKey: "public-key", }); }).pipe( Effect.provide( @@ -264,6 +270,7 @@ describe("relay managed tunnel recovery", () => { enableRecovery: (input) => Effect.sync(() => { recoveryEnabledFor = input; + return true; }), }), ), @@ -365,15 +372,76 @@ describe("relay managed tunnel recovery", () => { wsBaseUrl: "wss://different.example.test/ws", providerKind: "cloudflare_tunnel", }, - runtime: { providerKind: "cloudflare_tunnel", connectorToken: "token" }, + runtime: { + providerKind: "cloudflare_tunnel", + connectorToken: "token", + tunnelId: "replacement-tunnel", + }, }), }), Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ enableRecovery: () => Effect.sync(() => { recoveryEnabled = true; + return true; + }), + }), + ), + ), + ); + }); + + it.effect("removes a recovered tunnel when its link disappears before registration", () => { + let lookups = 0; + const cleaned: Array = []; + const target = { + userId: "user-1", + environmentId: "environment-1", + hostname: "environment-1.example.test", + tunnelId: "replacement-tunnel", + tunnelName: "environment-1-tunnel", + dnsRecordId: "dns-1", + readyAt: "2026-07-28T00:00:00.000Z", + updatedAt: "replacement-generation", + } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; + + return Effect.gen(function* () { + const error = yield* Effect.flip( + recoverEnvironmentTunnelRecord({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + expect(error).toMatchObject({ _tag: "Unauthorized" }); + expect(cleaned).toEqual(["replacement-tunnel"]); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.sync(() => (++lookups === 1 ? linkedEnvironmentRecord : null)), + provision: () => + Effect.succeed({ + endpoint: linkedEnvironmentRecord.endpoint, + runtime: { + providerKind: "cloudflare_tunnel", + connectorToken: "replacement-token", + tunnelId: "replacement-tunnel", + }, + }), + prepareDeprovision: () => Effect.succeed(target), + deprovision: ({ target: captured }) => + Effect.sync(() => { + if (captured?.tunnelId) { + cleaned.push(captured.tunnelId); + } + return true; }), }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => Effect.succeed(false), + }), ), ), ); @@ -473,6 +541,7 @@ describe("relay environment unlink", () => { Effect.sync(() => { expect(request.target).toBe(deprovisionTarget); calls.push("deprovision"); + return true; }), }), ), @@ -521,6 +590,7 @@ describe("relay environment unlink", () => { deprovision: () => Effect.sync(() => { calls.push("deprovision"); + return true; }), }), ), @@ -548,6 +618,45 @@ describe("relay environment unlink", () => { deprovision: () => Effect.sync(() => { calls.push("deprovision"); + return true; + }), + }), + ), + ); + }); + + it.effect("retries unlink cleanup when a concurrent tunnel release wins the first claim", () => { + let lookups = 0; + const targets: Array = []; + const target = { + userId: "user-1", + environmentId: "environment-1", + hostname: "environment-1.example.test", + tunnelId: "tunnel-1", + tunnelName: "environment-1-tunnel", + dnsRecordId: "dns-1", + readyAt: "2026-07-28T00:00:00.000Z", + updatedAt: "original-generation", + } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; + + return Effect.gen(function* () { + expect( + yield* unlinkEnvironmentRecord({ + userId: "user-1", + environmentId: "environment-1", + }), + ).toBe(true); + expect(targets).toEqual([target, undefined]); + }).pipe( + Effect.provide( + relayUnlinkTestLayer({ + getForUser: () => Effect.sync(() => (++lookups === 1 ? linkedEnvironmentRecord : null)), + revokeForUser: () => Effect.succeed(true), + prepareDeprovision: () => Effect.succeed(target), + deprovision: ({ target: captured }) => + Effect.sync(() => { + targets.push(captured ?? undefined); + return targets.length > 1; }), }), ), diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 001e0de7e7d3..5f66dec9d6dc 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -456,11 +456,14 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron // revocation commits so a database failure leaves a fully usable active // link. Still run teardown when the link is already revoked, allowing a // retry to finish cleanup after an earlier Cloudflare failure. - yield* managedEndpointProvider.deprovision({ + const deprovisioned = yield* managedEndpointProvider.deprovision({ userId: input.userId, environmentId: input.environmentId, target: deprovisionTarget, }); + if (!deprovisioned && (yield* links.getForUser(input)) === null) { + yield* managedEndpointProvider.deprovision(input); + } return unlinked; }, ); @@ -493,17 +496,36 @@ export const recoverEnvironmentTunnelRecord = Effect.fn( environmentId: input.environmentId, origin: input.origin, }); + const recoveredTunnelId = recovered.runtime.tunnelId; if ( + recoveredTunnelId === undefined || recovered.endpoint.httpBaseUrl !== link.endpoint.httpBaseUrl || recovered.endpoint.wsBaseUrl !== link.endpoint.wsBaseUrl ) { return yield* new HttpApiError.Unauthorized({}); } - yield* allocations.enableRecovery({ + const enabled = yield* allocations.enableRecovery({ userId: input.userId, environmentId: input.environmentId, + tunnelId: recoveredTunnelId, + environmentPublicKey: input.environmentPublicKey, }); + if (!enabled) { + const target = yield* managedEndpointProvider.prepareDeprovision(input); + if (target !== null && (yield* links.getForUser(input)) === null) { + yield* managedEndpointProvider.deprovision({ ...input, target }).pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to clean up a tunnel after its link was removed", { + userId: input.userId, + environmentId: input.environmentId, + cause, + }), + ), + ); + } + return yield* new HttpApiError.Unauthorized({}); + } return { endpoint: recovered.endpoint, endpointRuntime: recovered.runtime, @@ -1053,6 +1075,8 @@ export const serverApi = HttpApiBuilder.group( relayInternalErrorResponse("upstream_unavailable"), ManagedEndpointProvisioningFailed: () => relayInternalErrorResponse("upstream_unavailable"), + ManagedEndpointDeprovisioningFailed: () => + relayInternalErrorResponse("upstream_unavailable"), ManagedTunnelLimitExceeded: () => relayInternalErrorResponse("upstream_unavailable"), }), mapRelayCommonApiErrors("not_authorized"), From 7e4b7b443de568053b364a388a7fa4c00b6a38d5 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 24 Aug 2026 20:50:44 -0700 Subject: [PATCH 03/14] fix(connect): serialize tunnel deletion and recovery ownership --- .../src/cloud/ManagedEndpointRuntime.test.ts | 24 +++ .../src/cloud/ManagedEndpointRuntime.ts | 15 ++ apps/server/src/cloud/http.ts | 96 ++++----- apps/server/src/server.ts | 7 +- .../migration.sql | 1 - .../migration.sql | 2 + .../snapshot.json | 15 +- .../environments/EnvironmentConnector.test.ts | 3 + .../ManagedEndpointAllocations.test.ts | 58 +++++- .../ManagedEndpointAllocations.ts | 118 +++++++++--- .../ManagedEndpointProvider.test.ts | 60 +++++- .../environments/ManagedEndpointProvider.ts | 182 +++++++++++------- .../ManagedEndpointReaper.test.ts | 29 ++- .../src/environments/ManagedEndpointReaper.ts | 43 +++-- infra/relay/src/http/Api.test.ts | 5 +- infra/relay/src/http/Api.ts | 12 +- infra/relay/src/persistence/schema.ts | 1 + infra/relay/src/worker.ts | 4 +- 18 files changed, 486 insertions(+), 189 deletions(-) delete mode 100644 infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/migration.sql create mode 100644 infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/migration.sql rename infra/relay/migrations/postgres/{20260825010804_managed_endpoint_recovery => 20260825034308_managed_endpoint_recovery}/snapshot.json (99%) diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index 54a12373c732..4dc9cc2ca7b0 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -80,6 +80,30 @@ function makeHandle(input: { } describe("CloudManagedEndpointRuntime", () => { + it("retries connector startup failures but stops for unsupported runtimes", () => { + expect( + ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ + status: "failed", + reason: "The relay client is not installed.", + }), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ + status: "failed", + reason: "spawn failed", + }), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ + status: "failed", + reason: "Relay client is unsupported on linux-arm.", + }), + ).toBe(false); + expect( + ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ status: "unsupported" }), + ).toBe(false); + }); + it.effect("serializes updates to persisted cloud link state", () => Effect.gen(function* () { const firstEntered = yield* Deferred.make(); diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index cf566ac0a4c8..44ef29ecf8a3 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -82,6 +82,21 @@ export function classifyRelayClientOutput(line: string): "connected" | "warning" return /\b(?:ERR|WRN|FTL|PNC)\b/u.test(line) ? "warning" : "debug"; } +/** Connector startup failures can clear after installation or a later spawn attempt. */ +export function isRetryableManagedEndpointRuntimeStatus(status: unknown): boolean { + if (typeof status !== "object" || status === null || !("status" in status)) { + return false; + } + if (status.status !== "failed") { + return false; + } + return !( + "reason" in status && + typeof status.reason === "string" && + status.reason.startsWith("Relay client is unsupported on ") + ); +} + function runtimeConfigKey(config: RelayManagedEndpointRuntimeConfig): string { return JSON.stringify({ providerKind: config.providerKind, diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index 9ae34e6d0d5c..14d913cd105c 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -454,57 +454,56 @@ const cloudLinkProofHandler = Effect.fn("environment.cloud.linkProof")( const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(function* ( dependencies: CloudHttpDependencies, payload: RelayEnvironmentConfigRequest, - options?: { readonly requestRecovery?: boolean }, + options?: { readonly requestRecovery?: boolean; readonly lockHeld?: boolean }, ) { - return yield* dependencies.endpointRuntime.withLinkStateLock( - Effect.gen(function* () { - yield* validateRelayConfigPayload(payload); - yield* validateLinkedCloudUser({ - secrets: dependencies.secrets, - cloudUserId: payload.cloudUserId, + const apply = Effect.gen(function* () { + yield* validateRelayConfigPayload(payload); + yield* validateLinkedCloudUser({ + secrets: dependencies.secrets, + cloudUserId: payload.cloudUserId, + }); + yield* validateCloudMintPublicKey(payload.cloudMintPublicKey); + const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig( + payload.endpointRuntime, + ); + const ok = + endpointRuntimeStatus.status === "disabled" || endpointRuntimeStatus.status === "running"; + if (!ok) { + return yield* new EnvironmentCloudEndpointUnavailableError({ + message: "Managed endpoint runtime could not be started.", + endpointRuntimeStatus, }); - yield* validateCloudMintPublicKey(payload.cloudMintPublicKey); - const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig( - payload.endpointRuntime, - ); - const ok = - endpointRuntimeStatus.status === "disabled" || endpointRuntimeStatus.status === "running"; - if (!ok) { - return yield* new EnvironmentCloudEndpointUnavailableError({ - message: "Managed endpoint runtime could not be started.", - endpointRuntimeStatus, - }); - } + } - yield* dependencies.secrets.set(RELAY_URL_SECRET, stringToBytes(payload.relayUrl)); - yield* dependencies.secrets.set( - RELAY_ISSUER_SECRET, - stringToBytes(payload.relayIssuer ?? payload.relayUrl), - ); - yield* dependencies.secrets.set(CLOUD_LINKED_USER_ID, stringToBytes(payload.cloudUserId)); - yield* dependencies.secrets.set( - RELAY_ENVIRONMENT_CREDENTIAL_SECRET, - stringToBytes(payload.environmentCredential), - ); + yield* dependencies.secrets.set(RELAY_URL_SECRET, stringToBytes(payload.relayUrl)); + yield* dependencies.secrets.set( + RELAY_ISSUER_SECRET, + stringToBytes(payload.relayIssuer ?? payload.relayUrl), + ); + yield* dependencies.secrets.set(CLOUD_LINKED_USER_ID, stringToBytes(payload.cloudUserId)); + yield* dependencies.secrets.set( + RELAY_ENVIRONMENT_CREDENTIAL_SECRET, + stringToBytes(payload.environmentCredential), + ); + yield* dependencies.secrets.set( + CLOUD_MINT_PUBLIC_KEY, + stringToBytes(payload.cloudMintPublicKey), + ); + if (payload.endpointRuntime) { + const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime); yield* dependencies.secrets.set( - CLOUD_MINT_PUBLIC_KEY, - stringToBytes(payload.cloudMintPublicKey), + CLOUD_ENDPOINT_RUNTIME_CONFIG, + stringToBytes(endpointRuntimeJson), ); - if (payload.endpointRuntime) { - const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime); - yield* dependencies.secrets.set( - CLOUD_ENDPOINT_RUNTIME_CONFIG, - stringToBytes(endpointRuntimeJson), - ); - if (options?.requestRecovery !== false) { - yield* dependencies.endpointRuntime.requestRecovery(payload.endpointRuntime); - } - } else { - yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); + if (options?.requestRecovery !== false) { + yield* dependencies.endpointRuntime.requestRecovery(payload.endpointRuntime); } - return { ok, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; - }), - ); + } else { + yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); + } + return { ok, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; + }); + return yield* options?.lockHeld ? apply : dependencies.endpointRuntime.withLinkStateLock(apply); }); const cloudRelayConfigHandler = Effect.fn("environment.cloud.relayConfig")( @@ -629,7 +628,7 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi cloudMintPublicKey: link.cloudMintPublicKey, endpointRuntime: link.endpointRuntime, }, - { requestRecovery: false }, + { requestRecovery: false, lockHeld: true }, ); }, Effect.catchIf( @@ -647,7 +646,10 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileDesiredLink")( function* (localOrigin: string) { - return yield* reconcileDesiredCloudLinkWith(yield* cloudHttpDependencies, localOrigin); + const dependencies = yield* cloudHttpDependencies; + return yield* dependencies.endpointRuntime.withLinkStateLock( + reconcileDesiredCloudLinkWith(dependencies, localOrigin), + ); }, ); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 7e66e400218d..2bf4e0ac11dd 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -634,7 +634,10 @@ export const makeServerLayer = Layer.unwrap( Effect.retry({ while: (error) => error._tag !== "EnvironmentHttpBadRequestError" && - error._tag !== "EnvironmentCloudEndpointUnavailableError", + (error._tag !== "EnvironmentCloudEndpointUnavailableError" || + CloudManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus( + error.endpointRuntimeStatus, + )), schedule: Schedule.exponential("1 second").pipe( Schedule.modifyDelay(({ duration }) => Effect.succeed(Duration.min(duration, Duration.seconds(30))), @@ -645,7 +648,7 @@ export const makeServerLayer = Layer.unwrap( Effect.tap((recovered) => recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void, ), - Effect.catchCause((cause) => + Effect.catch((cause) => Effect.logWarning("Failed to recover the T3 Connect managed tunnel", { cause }), ), ), diff --git a/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/migration.sql b/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/migration.sql deleted file mode 100644 index f528c09fbc7f..000000000000 --- a/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/migration.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "recovery_enabled_at" varchar(64); \ No newline at end of file diff --git a/infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/migration.sql b/infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/migration.sql new file mode 100644 index 000000000000..4da6b524efa4 --- /dev/null +++ b/infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/migration.sql @@ -0,0 +1,2 @@ +ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "recovery_enabled_at" varchar(64);--> statement-breakpoint +ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "generation" integer DEFAULT 0 NOT NULL; \ No newline at end of file diff --git a/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/snapshot.json b/infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/snapshot.json similarity index 99% rename from infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/snapshot.json rename to infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/snapshot.json index 648638a0e17d..21afe569ac4e 100644 --- a/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/snapshot.json +++ b/infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/snapshot.json @@ -1,7 +1,7 @@ { "version": "8", "dialect": "postgres", - "id": "f3c6f2a5-2ecf-43cb-b381-98790fdca66e", + "id": "a0128e5a-4bba-4f2d-9851-82c3744dae2c", "prevIds": ["2374caff-40bf-423c-9255-55e76dddbc2a"], "ddl": [ { @@ -877,6 +877,19 @@ "schema": "public", "table": "relay_managed_endpoint_allocations" }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "generation", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, { "type": "varchar(64)", "typeSchema": null, diff --git a/infra/relay/src/environments/EnvironmentConnector.test.ts b/infra/relay/src/environments/EnvironmentConnector.test.ts index dfa809a9f151..31e6d27c8fcd 100644 --- a/infra/relay/src/environments/EnvironmentConnector.test.ts +++ b/infra/relay/src/environments/EnvironmentConnector.test.ts @@ -189,6 +189,7 @@ function makeAllocations( dnsRecordId: "dns-record-id", readyAt: "2026-05-25T00:00:00.000Z", updatedAt: "2026-05-25T00:00:00.000Z", + generation: 1, }, ): ManagedEndpointAllocations.ManagedEndpointAllocations["Service"] { return { @@ -200,6 +201,7 @@ function makeAllocations( enableRecovery: () => Effect.die("unused"), listByTunnelNames: () => Effect.die("unused"), claimRelease: () => Effect.die("unused"), + withClaimedTunnel: () => Effect.die("unused"), claimDeprovision: () => Effect.die("unused"), remove: () => Effect.die("unused"), removeClaimed: () => Effect.die("unused"), @@ -474,6 +476,7 @@ describe("EnvironmentConnector", () => { dnsRecordId: "dns-record-id", readyAt: null, updatedAt: "2026-05-25T00:00:00.000Z", + generation: 1, }), }), ), diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts index ba014dd8d576..73612590e5fa 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import { PgDialect } from "drizzle-orm/pg-core"; import * as RelayDb from "../db.ts"; @@ -93,6 +94,7 @@ describe("ManagedEndpointAllocations", () => { dnsRecordId: "dns-1", readyAt: "2026-08-25T12:00:00.000Z", updatedAt: "2026-08-25T12:00:00.000Z", + generation: 1, }; const fakeDb = { select: () => ({ @@ -138,16 +140,14 @@ describe("ManagedEndpointAllocations", () => { ); it.effect("returns a claim generation only when deprovision wins the allocation CAS", () => { - let claimedAt: string | undefined; const fakeDb = { update: (table: unknown) => { expect(table).toBe(relayManagedEndpointAllocations); return { - set: (values: { readonly updatedAt: string }) => { - claimedAt = values.updatedAt; + set: (_values: { readonly updatedAt: string }) => { return { where: () => ({ - returning: () => Effect.succeed([{ userId: "user-1" }]), + returning: () => Effect.succeed([{ generation: 8 }]), }), }; }, @@ -160,14 +160,58 @@ describe("ManagedEndpointAllocations", () => { const generation = yield* allocations.claimDeprovision({ userId: "user-1", environmentId: "environment-1", - updatedAt: "captured-generation", + generation: 7, }); - expect(generation).toBe(claimedAt); + expect(generation).toBe(8); expect(generation).not.toBeNull(); }).pipe(Effect.provide(layerWithDb(fakeDb))); }); + it.effect("holds the claimed allocation row while deleting its tunnel", () => { + const operations: string[] = []; + const fakeDb = { + $client: { + withTransaction: (effect: Effect.Effect) => + Effect.sync(() => { + operations.push("transaction"); + }).pipe(Effect.andThen(effect)), + }, + select: () => ({ + from: () => ({ + where: () => ({ + limit: () => ({ + for: (strength: string) => + Effect.sync(() => { + operations.push(`lock:${strength}`); + return [{ generation: 7 }]; + }), + }), + }), + }), + }), + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const result = yield* allocations.withClaimedTunnel( + { + userId: "user-1", + environmentId: "environment-1", + tunnelId: "tunnel-1", + generation: 7, + }, + Effect.sync(() => { + operations.push("delete"); + return true; + }), + ); + + expect(Option.getOrNull(result)).toBe(true); + expect(operations).toEqual(["transaction", "lock:update", "delete"]); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + it.effect("does not remove an allocation superseded after a deprovision claim", () => { const fakeDb = { delete: (table: unknown) => { @@ -186,7 +230,7 @@ describe("ManagedEndpointAllocations", () => { yield* allocations.removeClaimed({ userId: "user-1", environmentId: "environment-1", - updatedAt: "outdated-claim-generation", + generation: 7, }), ).toBe(false); }).pipe(Effect.provide(layerWithDb(fakeDb))); diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index 6b67c6be93f1..4e5edd1b1bb0 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -1,11 +1,13 @@ import type { RelayManagedEndpoint } from "@t3tools/contracts/relay"; -import { and, eq, exists, inArray, isNull } from "drizzle-orm"; +import { and, eq, exists, inArray, isNull, sql } from "drizzle-orm"; import { QueryBuilder } from "drizzle-orm/pg-core"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; +import { isSqlError } from "effect/unstable/sql/SqlError"; import * as RelayDb from "../db.ts"; import { isManagedEndpointHostname, managedEndpointForHostname } from "../deploymentConfig.ts"; @@ -19,11 +21,8 @@ export interface ManagedEndpointAllocation { readonly tunnelName: string; readonly dnsRecordId: string | null; readonly readyAt: string | null; - /** - * Doubles as the allocation's generation marker: every mutation rewrites it, - * so `claimRelease` can detect a provision that raced a release. - */ readonly updatedAt: string; + readonly generation: number; } export interface ManagedEndpointTunnelAllocation extends ManagedEndpointAllocation { @@ -57,6 +56,7 @@ export class ManagedEndpointAllocationPersistenceError extends Schema.TaggedErro "mark-ready", "enable-recovery", "list-tunnels", + "lock-tunnel", "claim-release", "claim-deprovision", "remove", @@ -89,6 +89,7 @@ interface ReserveManagedEndpointAllocationInput extends ManagedEndpointAllocatio interface RecordManagedEndpointTunnelInput extends ManagedEndpointAllocationKey { readonly tunnelId: string; + readonly generation: number; } interface RecordManagedEndpointDnsInput extends ManagedEndpointAllocationKey { @@ -97,7 +98,7 @@ interface RecordManagedEndpointDnsInput extends ManagedEndpointAllocationKey { interface ClaimManagedEndpointReleaseInput extends ManagedEndpointAllocationKey { readonly tunnelId: string; - readonly updatedAt: string; + readonly generation: number; } interface EnableManagedEndpointRecoveryInput extends ManagedEndpointAllocationKey { @@ -106,11 +107,11 @@ interface EnableManagedEndpointRecoveryInput extends ManagedEndpointAllocationKe } interface ClaimManagedEndpointDeprovisionInput extends ManagedEndpointAllocationKey { - readonly updatedAt: string; + readonly generation: number; } interface RemoveClaimedManagedEndpointAllocationInput extends ManagedEndpointAllocationKey { - readonly updatedAt: string; + readonly generation: number; } export class ManagedEndpointAllocations extends Context.Service< @@ -124,7 +125,7 @@ export class ManagedEndpointAllocations extends Context.Service< ) => Effect.Effect; readonly recordTunnel: ( input: RecordManagedEndpointTunnelInput, - ) => Effect.Effect; + ) => Effect.Effect; readonly recordDns: ( input: RecordManagedEndpointDnsInput, ) => Effect.Effect; @@ -143,13 +144,17 @@ export class ManagedEndpointAllocations extends Context.Service< /** * Atomically claims the right to delete the allocation's tunnel: succeeds * only while the recorded tunnel and generation still match what the - * caller loaded. A concurrent provision rewrites `updatedAt` when it + * caller loaded. A concurrent provision increments `generation` when it * records its tunnel, which makes a stale claim fail and keeps the freshly * issued tunnel alive. */ readonly claimRelease: ( input: ClaimManagedEndpointReleaseInput, - ) => Effect.Effect; + ) => Effect.Effect; + readonly withClaimedTunnel: ( + input: ClaimManagedEndpointReleaseInput, + effect: Effect.Effect, + ) => Effect.Effect, E | ManagedEndpointAllocationPersistenceError, R>; /** * Claims the complete allocation for teardown only if its generation still * matches the snapshot captured by the unlink operation. @@ -159,7 +164,7 @@ export class ManagedEndpointAllocations extends Context.Service< */ readonly claimDeprovision: ( input: ClaimManagedEndpointDeprovisionInput, - ) => Effect.Effect; + ) => Effect.Effect; readonly remove: ( input: ManagedEndpointAllocationKey, ) => Effect.Effect; @@ -178,6 +183,7 @@ const allocationSelection = { dnsRecordId: relayManagedEndpointAllocations.dnsRecordId, readyAt: relayManagedEndpointAllocations.readyAt, updatedAt: relayManagedEndpointAllocations.updatedAt, + generation: relayManagedEndpointAllocations.generation, }; const whereAllocation = (input: ManagedEndpointAllocationKey) => @@ -269,14 +275,22 @@ export const make = Effect.gen(function* () { recordTunnel: Effect.fn("relay.managed_endpoint_allocations.record_tunnel")(function* ( input: RecordManagedEndpointTunnelInput, ) { - yield* db + return yield* db .update(relayManagedEndpointAllocations) .set({ tunnelId: input.tunnelId, updatedAt: DateTime.formatIso(yield* DateTime.now), + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) - .where(whereAllocation(input)) + .where( + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.generation, input.generation), + ), + ) + .returning({ environmentId: relayManagedEndpointAllocations.environmentId }) .pipe( + Effect.map((rows) => rows.length > 0), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -296,6 +310,7 @@ export const make = Effect.gen(function* () { .set({ dnsRecordId: input.dnsRecordId, updatedAt: DateTime.formatIso(yield* DateTime.now), + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) .where(whereAllocation(input)) .pipe( @@ -319,6 +334,7 @@ export const make = Effect.gen(function* () { .set({ readyAt: now, updatedAt: now, + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) .where(whereAllocation(input)) .pipe( @@ -339,7 +355,11 @@ export const make = Effect.gen(function* () { const now = DateTime.formatIso(yield* DateTime.now); return yield* db .update(relayManagedEndpointAllocations) - .set({ recoveryEnabledAt: now, updatedAt: now }) + .set({ + recoveryEnabledAt: now, + updatedAt: now, + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, + }) .where( and( whereAllocation(input), @@ -409,22 +429,22 @@ export const make = Effect.gen(function* () { claimRelease: Effect.fn("relay.managed_endpoint_allocations.claim_release")(function* ( input: ClaimManagedEndpointReleaseInput, ) { - const claimedAt = DateTime.formatIso(yield* DateTime.now); const claimed = yield* db .update(relayManagedEndpointAllocations) .set({ - updatedAt: claimedAt, + updatedAt: DateTime.formatIso(yield* DateTime.now), + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) .where( and( whereAllocation(input), eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), - eq(relayManagedEndpointAllocations.updatedAt, input.updatedAt), + eq(relayManagedEndpointAllocations.generation, input.generation), ), ) - .returning({ userId: relayManagedEndpointAllocations.userId }) + .returning({ generation: relayManagedEndpointAllocations.generation }) .pipe( - Effect.map((rows) => rows.length > 0), + Effect.map((rows) => rows[0]?.generation ?? null), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -437,24 +457,66 @@ export const make = Effect.gen(function* () { }), ), ); - return claimed ? claimedAt : null; + return claimed; }), + withClaimedTunnel: Effect.fn("relay.managed_endpoint_allocations.with_claimed_tunnel")( + function* ( + input: ClaimManagedEndpointReleaseInput, + effect: Effect.Effect, + ): Effect.fn.Return, E | ManagedEndpointAllocationPersistenceError, R> { + const lockError = (cause: unknown) => + new ManagedEndpointAllocationPersistenceError({ + operation: "lock-tunnel", + stage: "database-request", + userId: input.userId, + environmentId: input.environmentId, + tunnelId: input.tunnelId, + cause, + }); + return yield* db.$client + .withTransaction( + db + .select({ generation: relayManagedEndpointAllocations.generation }) + .from(relayManagedEndpointAllocations) + .where( + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), + eq(relayManagedEndpointAllocations.generation, input.generation), + ), + ) + .limit(1) + .for("update") + .pipe( + Effect.mapError(lockError), + Effect.flatMap((rows) => + rows.length === 0 + ? Effect.succeed(Option.none()) + : effect.pipe(Effect.map(Option.some)), + ), + ), + ) + .pipe(Effect.mapError((cause) => (isSqlError(cause) ? lockError(cause) : cause))); + }, + ), claimDeprovision: Effect.fn("relay.managed_endpoint_allocations.claim_deprovision")(function* ( input: ClaimManagedEndpointDeprovisionInput, ) { - const claimedAt = DateTime.formatIso(yield* DateTime.now); const claimed = yield* db .update(relayManagedEndpointAllocations) - .set({ updatedAt: claimedAt }) + .set({ + updatedAt: DateTime.formatIso(yield* DateTime.now), + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, + }) .where( and( whereAllocation(input), - eq(relayManagedEndpointAllocations.updatedAt, input.updatedAt), + eq(relayManagedEndpointAllocations.generation, input.generation), ), ) - .returning({ userId: relayManagedEndpointAllocations.userId }) + .returning({ generation: relayManagedEndpointAllocations.generation }) .pipe( - Effect.map((rows) => rows.length > 0), + Effect.map((rows) => rows[0]?.generation ?? null), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -466,7 +528,7 @@ export const make = Effect.gen(function* () { }), ), ); - return claimed ? claimedAt : null; + return claimed; }), remove: Effect.fn("relay.managed_endpoint_allocations.remove")(function* ( input: ManagedEndpointAllocationKey, @@ -494,7 +556,7 @@ export const make = Effect.gen(function* () { .where( and( whereAllocation(input), - eq(relayManagedEndpointAllocations.updatedAt, input.updatedAt), + eq(relayManagedEndpointAllocations.generation, input.generation), ), ) .returning({ userId: relayManagedEndpointAllocations.userId }) diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 769466554c68..e1168543d393 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -6,6 +6,7 @@ import * as Alchemy from "alchemy"; import * as Cloudflare from "alchemy/Cloudflare"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Redacted from "effect/Redacted"; import * as RelayConfiguration from "../Config.ts"; @@ -196,7 +197,11 @@ function makeAllocations(calls: AllocationCall[] = []) { ) => { const allocation = allocations.get(key); if (allocation !== undefined) { - allocations.set(key, { ...change(allocation), updatedAt: `generation-${++generation}` }); + allocations.set(key, { + ...change(allocation), + generation: allocation.generation + 1, + updatedAt: `generation-${++generation}`, + }); } }; return ManagedEndpointAllocations.ManagedEndpointAllocations.of({ @@ -214,6 +219,7 @@ function makeAllocations(calls: AllocationCall[] = []) { dnsRecordId: null, readyAt: null, updatedAt: `generation-${++generation}`, + generation: 0, }; allocations.set(allocationKey(input), allocation); return allocation; @@ -221,10 +227,15 @@ function makeAllocations(calls: AllocationCall[] = []) { recordTunnel: (input) => Effect.sync(() => { calls.push({ operation: "recordTunnel", input }); + const current = allocations.get(allocationKey(input)); + if (current?.generation !== input.generation) { + return false; + } mutate(allocationKey(input), (allocation) => ({ ...allocation, tunnelId: input.tunnelId, })); + return true; }), recordDns: (input) => Effect.sync(() => { @@ -267,22 +278,29 @@ function makeAllocations(calls: AllocationCall[] = []) { if ( allocation === undefined || allocation.tunnelId !== input.tunnelId || - allocation.updatedAt !== input.updatedAt + allocation.generation !== input.generation ) { return null; } mutate(allocationKey(input), (current) => current); - return allocations.get(allocationKey(input))?.updatedAt ?? null; + return allocations.get(allocationKey(input))?.generation ?? null; + }), + withClaimedTunnel: (input, effect) => + Effect.suspend(() => { + const current = allocations.get(allocationKey(input)); + return current?.tunnelId === input.tunnelId && current.generation === input.generation + ? effect.pipe(Effect.map(Option.some)) + : Effect.succeed(Option.none()); }), claimDeprovision: (input) => Effect.sync(() => { calls.push({ operation: "claimDeprovision", input }); const allocation = allocations.get(allocationKey(input)); - if (allocation === undefined || allocation.updatedAt !== input.updatedAt) { + if (allocation === undefined || allocation.generation !== input.generation) { return null; } mutate(allocationKey(input), (current) => current); - return allocations.get(allocationKey(input))?.updatedAt ?? null; + return allocations.get(allocationKey(input))?.generation ?? null; }), remove: (input) => Effect.sync(() => { @@ -293,7 +311,7 @@ function makeAllocations(calls: AllocationCall[] = []) { Effect.sync(() => { calls.push({ operation: "removeClaimed", input }); const allocation = allocations.get(allocationKey(input)); - if (allocation === undefined || allocation.updatedAt !== input.updatedAt) { + if (allocation === undefined || allocation.generation !== input.generation) { return false; } allocations.delete(allocationKey(input)); @@ -1007,6 +1025,31 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.provide(layer)); }); + it.effect("rejects a tunnel recorded after its allocation generation changed", () => { + const allocations = makeAllocations(); + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + recordTunnel: () => Effect.succeed(false), + }); + const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const error = yield* Effect.flip( + provider.provision({ + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "record-tunnel", + }); + }).pipe(Effect.provide(layer)); + }); + it.effect("keeps a tunnel that reconnects before scheduled deletion", () => { const tunnelCalls: TunnelCall[] = []; const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ @@ -1048,13 +1091,14 @@ describe("ManagedEndpointProvider", () => { ...allocations, claimRelease: (input) => allocations.claimRelease(input).pipe( - Effect.tap((claimedAt) => - claimedAt === null + Effect.tap((claimedGeneration) => + claimedGeneration === null ? Effect.void : allocations.recordTunnel({ userId: input.userId, environmentId: input.environmentId, tunnelId: "replacement-tunnel", + generation: claimedGeneration, }), ), ), diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 41852f20f282..f61a853f52bf 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -494,11 +494,11 @@ export const make = Effect.gen(function* () { if (allocation === null) { return true; } - const claimedAt = yield* allocations + const claimedGeneration = yield* allocations .claimDeprovision({ userId: input.userId, environmentId: input.environmentId, - updatedAt: allocation.updatedAt, + generation: allocation.generation, }) .pipe( Effect.mapError( @@ -512,7 +512,7 @@ export const make = Effect.gen(function* () { }), ), ); - if (claimedAt === null) { + if (claimedGeneration === null) { return false; } const dnsRecordId = allocation.dnsRecordId; @@ -547,7 +547,7 @@ export const make = Effect.gen(function* () { .removeClaimed({ userId: input.userId, environmentId: input.environmentId, - updatedAt: claimedAt, + generation: claimedGeneration, }) .pipe( Effect.mapError( @@ -586,17 +586,17 @@ export const make = Effect.gen(function* () { } // Claim the release against the allocation's current generation before // touching Cloudflare. A provision racing this release (fast environment - // restart) rewrites updatedAt when it records its tunnel, so a stale + // restart) increments the generation when it records its tunnel, so a stale // claim means the recorded tunnel may already back a fresh connector and // must be left alive. A provision that starts after the claim instead // fails loudly on the deleted tunnel and the client-side retry // provisions a replacement. - const claimedAt = yield* allocations + const claimedGeneration = yield* allocations .claimRelease({ userId: input.userId, environmentId: input.environmentId, tunnelId, - updatedAt: allocation.updatedAt, + generation: allocation.generation, }) .pipe( Effect.mapError( @@ -609,69 +609,10 @@ export const make = Effect.gen(function* () { }), ), ); - if (claimedAt === null) { + if (claimedGeneration === null) { return false; } - if (input.expectedInactiveBefore !== undefined && input.expectedStatus !== undefined) { - const currentAllocation = yield* allocations.get(input).pipe( - Effect.mapError( - (cause) => - new ManagedEndpointDeprovisioningFailed({ - ...input, - stage: "load-allocation", - tunnelId, - cause, - }), - ), - ); - if ( - currentAllocation === null || - currentAllocation.tunnelId !== tunnelId || - currentAllocation.updatedAt !== claimedAt - ) { - return false; - } - - const currentTunnel = yield* tunnels.get(tunnelId).pipe( - Effect.map(Option.some), - Effect.catchTag("ManagedEndpointTunnelClientError", (cause) => - isManagedEndpointNotFound(cause.cause) - ? Effect.succeed(Option.none()) - : Effect.fail( - new ManagedEndpointDeprovisioningFailed({ - ...input, - stage: "load-tunnel", - tunnelId, - cause, - }), - ), - ), - ); - if (Option.isNone(currentTunnel)) { - return true; - } - const inactiveAt = - input.expectedStatus === "down" - ? currentTunnel.value.connsInactiveAt - : currentTunnel.value.createdAt; - if ( - currentTunnel.value.id !== tunnelId || - currentTunnel.value.status !== input.expectedStatus || - typeof inactiveAt !== "string" - ) { - return false; - } - const inactiveTime = DateTime.make(inactiveAt); - const cutoff = DateTime.make(input.expectedInactiveBefore); - if ( - Option.isNone(inactiveTime) || - Option.isNone(cutoff) || - inactiveTime.value.epochMilliseconds > cutoff.value.epochMilliseconds - ) { - return false; - } - } - yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe( + const deleteTunnel = ignoreNotFound(tunnels.delete(tunnelId)).pipe( Effect.mapError( (cause) => new ManagedEndpointDeprovisioningFailed({ @@ -682,6 +623,97 @@ export const make = Effect.gen(function* () { }), ), ); + if (input.expectedInactiveBefore !== undefined && input.expectedStatus !== undefined) { + const released = yield* allocations + .withClaimedTunnel( + { + userId: input.userId, + environmentId: input.environmentId, + tunnelId, + generation: claimedGeneration, + }, + Effect.gen(function* () { + const currentTunnel = yield* tunnels.get(tunnelId).pipe( + Effect.map(Option.some), + Effect.catchTags({ + ManagedEndpointTunnelClientError: (cause) => + isManagedEndpointNotFound(cause.cause) + ? Effect.succeed(Option.none()) + : Effect.fail( + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "load-tunnel", + tunnelId, + cause, + }), + ), + }), + ); + if (Option.isNone(currentTunnel)) { + return true; + } + const inactiveAt = + input.expectedStatus === "down" + ? currentTunnel.value.connsInactiveAt + : currentTunnel.value.createdAt; + if ( + currentTunnel.value.id !== tunnelId || + currentTunnel.value.status !== input.expectedStatus || + typeof inactiveAt !== "string" + ) { + return false; + } + const inactiveTime = DateTime.make(inactiveAt); + const cutoff = DateTime.make(input.expectedInactiveBefore!); + if ( + Option.isNone(inactiveTime) || + Option.isNone(cutoff) || + inactiveTime.value.epochMilliseconds > cutoff.value.epochMilliseconds + ) { + return false; + } + + const finalGeneration = yield* allocations + .claimRelease({ + userId: input.userId, + environmentId: input.environmentId, + tunnelId, + generation: claimedGeneration, + }) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "claim-release", + tunnelId, + cause, + }), + ), + ); + if (finalGeneration === null) { + return false; + } + yield* deleteTunnel; + return true; + }), + ) + .pipe( + Effect.catchTags({ + ManagedEndpointAllocationPersistenceError: (cause) => + Effect.fail( + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "claim-release", + tunnelId, + cause, + }), + ), + }), + ); + return Option.getOrElse(released, () => false); + } + yield* deleteTunnel; // The recorded tunnelId is now stale, but the allocation row is left // untouched deliberately: connect/status authorization requires a fully // recorded allocation, and an offline environment must keep reporting @@ -806,11 +838,12 @@ export const make = Effect.gen(function* () { }); } const tunnel = { id: tunnelResponse.id, name: tunnelResponse.name }; - yield* allocations + const recordedTunnel = yield* allocations .recordTunnel({ userId: input.userId, environmentId: input.environmentId, tunnelId: tunnel.id, + generation: allocation.generation, }) .pipe( Effect.mapError( @@ -826,6 +859,17 @@ export const make = Effect.gen(function* () { }), ), ); + if (!recordedTunnel) { + return yield* new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "record-tunnel", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause: "The tunnel allocation changed during provisioning.", + }); + } yield* tunnels .putConfiguration(tunnel.id, { diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts index cc4c43b6ae23..335d58c3c217 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -34,18 +34,19 @@ function tunnel(input: { } function allocation(input: { - readonly tunnelId: string; + readonly tunnelId: string | null; readonly recoveryEnabled: boolean; }): ManagedEndpointAllocations.ManagedEndpointTunnelAllocation { return { userId: "user-1", - environmentId: `environment-${input.tunnelId}`, - hostname: `${input.tunnelId}.example.test`, + environmentId: `environment-${input.tunnelId ?? "pending"}`, + hostname: `${input.tunnelId ?? "pending"}.example.test`, tunnelId: input.tunnelId, tunnelName: `${PREFIX}aaaaaaaaaaaaaaaa`, dnsRecordId: "dns-1", readyAt: "2026-08-25T11:00:00.000Z", updatedAt: "2026-08-25T11:00:00.000Z", + generation: 1, recoveryEnabled: input.recoveryEnabled, }; } @@ -150,6 +151,7 @@ function harness(input?: { listByTunnelNames: (tunnelNames) => Effect.succeed(recorded.filter((entry) => tunnelNames.includes(entry.tunnelName))), claimRelease: () => Effect.die("unused"), + withClaimedTunnel: () => Effect.die("unused"), claimDeprovision: () => Effect.die("unused"), remove: () => Effect.die("unused"), removeClaimed: () => Effect.die("unused"), @@ -356,6 +358,27 @@ describe("ManagedEndpointReaper", () => { }).pipe(Effect.provide(state.layer)); }); + it.effect("removes an expired tunnel that was created but never recorded", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "unrecorded", + suffix: "aaaaaaaaaaaaaaaa", + status: "inactive", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + allocations: [allocation({ tunnelId: null, recoveryEnabled: false })], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(1); + expect(state.deleted).toEqual(["unrecorded"]); + }).pipe(Effect.provide(state.layer)); + }); + it.effect("keeps an orphan tunnel that reconnects before deletion", () => { const listed = tunnel({ id: "reconnected", diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts index da039f64030a..0442d5e58a80 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -78,11 +78,12 @@ export const make = Effect.gen(function* () { }) { const current = yield* tunnels.get(input.tunnel.id).pipe( Effect.map(Option.some), - Effect.catchTag("ManagedEndpointTunnelClientError", (error) => - ManagedEndpointProvider.isManagedEndpointNotFound(error.cause) - ? Effect.succeed(Option.none()) - : Effect.fail(error), - ), + Effect.catchTags({ + ManagedEndpointTunnelClientError: (error) => + ManagedEndpointProvider.isManagedEndpointNotFound(error.cause) + ? Effect.succeed(Option.none()) + : Effect.fail(error), + }), ); if (Option.isNone(current)) { return true; @@ -90,16 +91,21 @@ export const make = Effect.gen(function* () { if (!isExpiredManagedTunnel({ ...input, tunnel: current.value })) { return false; } - if ((yield* allocations.listByTunnelNames([input.tunnel.name])).length > 0) { + if ( + (yield* allocations.listByTunnelNames([input.tunnel.name])).some( + (allocation) => allocation.tunnelId !== null, + ) + ) { return false; } return yield* tunnels.delete(input.tunnel.id).pipe( Effect.as(true), - Effect.catchTag("ManagedEndpointTunnelClientError", (error) => - ManagedEndpointProvider.isManagedEndpointNotFound(error.cause) - ? Effect.succeed(true) - : Effect.fail(error), - ), + Effect.catchTags({ + ManagedEndpointTunnelClientError: (error) => + ManagedEndpointProvider.isManagedEndpointNotFound(error.cause) + ? Effect.succeed(true) + : Effect.fail(error), + }), ); }); @@ -166,21 +172,26 @@ export const make = Effect.gen(function* () { break; } const allocation = recordedByTunnelName.get(tunnel.name); - if (allocation !== undefined && allocation.tunnelId !== tunnel.id) { + if ( + allocation !== undefined && + allocation.tunnelId !== null && + allocation.tunnelId !== tunnel.id + ) { continue; } - if (allocation !== undefined && !allocation.recoveryEnabled) { + const owner = allocation?.tunnelId === tunnel.id ? allocation : undefined; + if (owner !== undefined && !owner.recoveryEnabled) { skippedLegacy += 1; continue; } const result = - allocation === undefined + owner === undefined ? yield* deleteOrphan({ tunnel, status, prefix, cutoff }).pipe(Effect.result) : yield* provider .release({ - userId: allocation.userId, - environmentId: allocation.environmentId, + userId: owner.userId, + environmentId: owner.environmentId, expectedTunnelId: tunnel.id, expectedInactiveBefore: cutoffIso, expectedStatus: status, diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index 5453d492bb22..f759c9166e2e 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -403,6 +403,7 @@ describe("relay managed tunnel recovery", () => { dnsRecordId: "dns-1", readyAt: "2026-07-28T00:00:00.000Z", updatedAt: "replacement-generation", + generation: 3, } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; return Effect.gen(function* () { @@ -493,6 +494,7 @@ describe("relay environment unlink", () => { dnsRecordId: "dns-1", readyAt: "2026-07-28T00:00:00.000Z", updatedAt: "generation-before-unlink", + generation: 1, } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; return Effect.gen(function* () { @@ -637,6 +639,7 @@ describe("relay environment unlink", () => { dnsRecordId: "dns-1", readyAt: "2026-07-28T00:00:00.000Z", updatedAt: "original-generation", + generation: 1, } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; return Effect.gen(function* () { @@ -646,7 +649,7 @@ describe("relay environment unlink", () => { environmentId: "environment-1", }), ).toBe(true); - expect(targets).toEqual([target, undefined]); + expect(targets).toEqual([target, target]); }).pipe( Effect.provide( relayUnlinkTestLayer({ diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 5f66dec9d6dc..58aefda36b60 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -461,8 +461,11 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron environmentId: input.environmentId, target: deprovisionTarget, }); - if (!deprovisioned && (yield* links.getForUser(input)) === null) { - yield* managedEndpointProvider.deprovision(input); + if (!deprovisioned) { + const retryTarget = yield* managedEndpointProvider.prepareDeprovision(input); + if (retryTarget !== null && (yield* links.getForUser(input)) === null) { + yield* managedEndpointProvider.deprovision({ ...input, target: retryTarget }); + } } return unlinked; }, @@ -512,9 +515,10 @@ export const recoverEnvironmentTunnelRecord = Effect.fn( environmentPublicKey: input.environmentPublicKey, }); if (!enabled) { - const target = yield* managedEndpointProvider.prepareDeprovision(input); + const owner = { userId: input.userId, environmentId: input.environmentId }; + const target = yield* managedEndpointProvider.prepareDeprovision(owner); if (target !== null && (yield* links.getForUser(input)) === null) { - yield* managedEndpointProvider.deprovision({ ...input, target }).pipe( + yield* managedEndpointProvider.deprovision({ ...owner, target }).pipe( Effect.catch((cause) => Effect.logWarning("Failed to clean up a tunnel after its link was removed", { userId: input.userId, diff --git a/infra/relay/src/persistence/schema.ts b/infra/relay/src/persistence/schema.ts index ddd276ccfd70..a53be1ea1a30 100644 --- a/infra/relay/src/persistence/schema.ts +++ b/infra/relay/src/persistence/schema.ts @@ -94,6 +94,7 @@ export const relayManagedEndpointAllocations = pgTable( dnsRecordId: varchar("dns_record_id", { length: 191 }), readyAt: varchar("ready_at", { length: 64 }), recoveryEnabledAt: varchar("recovery_enabled_at", { length: 64 }), + generation: integer("generation").notNull().default(0), createdAt: varchar("created_at", { length: 64 }).notNull(), updatedAt: varchar("updated_at", { length: 64 }).notNull(), }, diff --git a/infra/relay/src/worker.ts b/infra/relay/src/worker.ts index 2f09fd3e5b9c..b9f5a947957f 100644 --- a/infra/relay/src/worker.ts +++ b/infra/relay/src/worker.ts @@ -278,7 +278,7 @@ export const ApiLive = Api.make( ), ), ), - Effect.catchCause((cause) => + Effect.catch((cause) => Effect.logWarning("Failed to prune expired relay state", { cause }), ), ), @@ -289,7 +289,7 @@ export const ApiLive = Api.make( ? Effect.logInfo("Finished managed tunnel cleanup", result) : Effect.void, ), - Effect.catchCause((cause) => + Effect.catch((cause) => Effect.logWarning("Failed to clean up inactive managed tunnels", { cause }), ), ), From f877a24112e3d02a8519bbd508c5b3f730a457a4 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 24 Aug 2026 21:09:09 -0700 Subject: [PATCH 04/14] fix(connect): keep tunnel setup and teardown in sync --- .../src/cloud/ManagedEndpointRuntime.test.ts | 4 + .../src/cloud/ManagedEndpointRuntime.ts | 12 +- apps/server/src/server.test.ts | 1 + apps/server/src/server.ts | 15 +- .../ManagedEndpointAllocations.ts | 45 ++- .../ManagedEndpointProvider.test.ts | 84 +++++- .../environments/ManagedEndpointProvider.ts | 272 ++++++++++++------ infra/relay/src/worker.ts | 13 +- 8 files changed, 319 insertions(+), 127 deletions(-) diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index 4dc9cc2ca7b0..40f8a271dd5a 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -84,18 +84,21 @@ describe("CloudManagedEndpointRuntime", () => { expect( ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ status: "failed", + failure: "not-installed", reason: "The relay client is not installed.", }), ).toBe(true); expect( ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ status: "failed", + failure: "spawn-failed", reason: "spawn failed", }), ).toBe(true); expect( ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ status: "failed", + failure: "unsupported-platform", reason: "Relay client is unsupported on linux-arm.", }), ).toBe(false); @@ -442,6 +445,7 @@ describe("CloudManagedEndpointRuntime", () => { expect(status).toEqual({ status: "failed", providerKind: "cloudflare_tunnel", + failure: "not-installed", reason: "The relay client is not installed.", }); expect(spawn).not.toHaveBeenCalled(); diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index 44ef29ecf8a3..ca12b430649f 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -37,6 +37,7 @@ export type CloudManagedEndpointRuntimeStatus = | { readonly status: "failed"; readonly providerKind: RelayManagedEndpointRuntimeConfig["providerKind"]; + readonly failure: "unsupported-platform" | "not-installed" | "spawn-failed"; readonly reason: string; readonly tunnelId?: string; readonly tunnelName?: string; @@ -87,14 +88,10 @@ export function isRetryableManagedEndpointRuntimeStatus(status: unknown): boolea if (typeof status !== "object" || status === null || !("status" in status)) { return false; } - if (status.status !== "failed") { + if (status.status !== "failed" || !("failure" in status)) { return false; } - return !( - "reason" in status && - typeof status.reason === "string" && - status.reason.startsWith("Relay client is unsupported on ") - ); + return status.failure === "not-installed" || status.failure === "spawn-failed"; } function runtimeConfigKey(config: RelayManagedEndpointRuntimeConfig): string { @@ -236,6 +233,7 @@ export const make = Effect.gen(function* () { return { status: "failed", providerKind: "cloudflare_tunnel", + failure: executable.status === "unsupported" ? "unsupported-platform" : "not-installed", reason: executable.status === "unsupported" ? `Relay client is unsupported on ${executable.platform}-${executable.arch}.` @@ -278,6 +276,7 @@ export const make = Effect.gen(function* () { Effect.as({ status: "failed", providerKind: "cloudflare_tunnel", + failure: "spawn-failed", reason: String(cause), ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}), ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}), @@ -312,6 +311,7 @@ export const make = Effect.gen(function* () { return { status: "failed", providerKind: "cloudflare_tunnel", + failure: "spawn-failed", reason: "Relay client did not start.", ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}), ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}), diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index ed29599f029f..d728adf74712 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -2939,6 +2939,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => { Effect.succeed({ status: "failed", providerKind: "cloudflare_tunnel", + failure: "not-installed", reason: "cloudflared missing", tunnelId: "tunnel-1", }), diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 2bf4e0ac11dd..7c95be8c85a7 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -1,4 +1,5 @@ import { EnvironmentHttpApi } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; import * as Duration from "effect/Duration"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; @@ -591,10 +592,6 @@ export const makeServerLayer = Layer.unwrap( : Layer.empty; const cloudDesiredLinkReconcileLayer = Layer.effectDiscard( Effect.gen(function* () { - if (!hasCloudPublicConfig) { - yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie); - return; - } const releaseManagedTunnel = releaseManagedTunnelOnShutdown().pipe( Effect.timeout("10 seconds"), Effect.tap((released) => @@ -648,8 +645,12 @@ export const makeServerLayer = Layer.unwrap( Effect.tap((recovered) => recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void, ), - Effect.catch((cause) => - Effect.logWarning("Failed to recover the T3 Connect managed tunnel", { cause }), + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.logWarning("Failed to recover the T3 Connect managed tunnel", { + cause, + }), ), ), ); @@ -663,7 +664,7 @@ export const makeServerLayer = Layer.unwrap( // covers anything this sleep used to hedge against. Every // millisecond here is dead time on the path to remote // reachability after a restart. - if (yield* CloudCliState.readCliDesiredCloudLink) { + if (hasCloudPublicConfig && (yield* CloudCliState.readCliDesiredCloudLink)) { yield* reconcileDesiredCloudLink(localOrigin).pipe( Effect.retry({ while: (error) => diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index 4e5edd1b1bb0..afd77151a34a 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -94,6 +94,13 @@ interface RecordManagedEndpointTunnelInput extends ManagedEndpointAllocationKey interface RecordManagedEndpointDnsInput extends ManagedEndpointAllocationKey { readonly dnsRecordId: string; + readonly tunnelId: string; + readonly generation: number; +} + +interface MarkManagedEndpointReadyInput extends ManagedEndpointAllocationKey { + readonly tunnelId: string; + readonly generation: number; } interface ClaimManagedEndpointReleaseInput extends ManagedEndpointAllocationKey { @@ -125,13 +132,13 @@ export class ManagedEndpointAllocations extends Context.Service< ) => Effect.Effect; readonly recordTunnel: ( input: RecordManagedEndpointTunnelInput, - ) => Effect.Effect; + ) => Effect.Effect; readonly recordDns: ( input: RecordManagedEndpointDnsInput, - ) => Effect.Effect; + ) => Effect.Effect; readonly markReady: ( - input: ManagedEndpointAllocationKey, - ) => Effect.Effect; + input: MarkManagedEndpointReadyInput, + ) => Effect.Effect; readonly enableRecovery: ( input: EnableManagedEndpointRecoveryInput, ) => Effect.Effect; @@ -288,9 +295,9 @@ export const make = Effect.gen(function* () { eq(relayManagedEndpointAllocations.generation, input.generation), ), ) - .returning({ environmentId: relayManagedEndpointAllocations.environmentId }) + .returning({ generation: relayManagedEndpointAllocations.generation }) .pipe( - Effect.map((rows) => rows.length > 0), + Effect.map((rows) => rows[0]?.generation ?? null), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -305,15 +312,23 @@ export const make = Effect.gen(function* () { recordDns: Effect.fn("relay.managed_endpoint_allocations.record_dns")(function* ( input: RecordManagedEndpointDnsInput, ) { - yield* db + return yield* db .update(relayManagedEndpointAllocations) .set({ dnsRecordId: input.dnsRecordId, updatedAt: DateTime.formatIso(yield* DateTime.now), generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) - .where(whereAllocation(input)) + .where( + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), + eq(relayManagedEndpointAllocations.generation, input.generation), + ), + ) + .returning({ generation: relayManagedEndpointAllocations.generation }) .pipe( + Effect.map((rows) => rows[0]?.generation ?? null), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -326,18 +341,26 @@ export const make = Effect.gen(function* () { ); }), markReady: Effect.fn("relay.managed_endpoint_allocations.mark_ready")(function* ( - input: ManagedEndpointAllocationKey, + input: MarkManagedEndpointReadyInput, ) { const now = DateTime.formatIso(yield* DateTime.now); - yield* db + return yield* db .update(relayManagedEndpointAllocations) .set({ readyAt: now, updatedAt: now, generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) - .where(whereAllocation(input)) + .where( + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), + eq(relayManagedEndpointAllocations.generation, input.generation), + ), + ) + .returning({ environmentId: relayManagedEndpointAllocations.environmentId }) .pipe( + Effect.map((rows) => rows.length > 0), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index e1168543d393..69d208a276ba 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -229,29 +229,39 @@ function makeAllocations(calls: AllocationCall[] = []) { calls.push({ operation: "recordTunnel", input }); const current = allocations.get(allocationKey(input)); if (current?.generation !== input.generation) { - return false; + return null; } mutate(allocationKey(input), (allocation) => ({ ...allocation, tunnelId: input.tunnelId, })); - return true; + return allocations.get(allocationKey(input))?.generation ?? null; }), recordDns: (input) => Effect.sync(() => { calls.push({ operation: "recordDns", input }); + const current = allocations.get(allocationKey(input)); + if (current?.generation !== input.generation || current.tunnelId !== input.tunnelId) { + return null; + } mutate(allocationKey(input), (allocation) => ({ ...allocation, dnsRecordId: input.dnsRecordId, })); + return allocations.get(allocationKey(input))?.generation ?? null; }), markReady: (input) => Effect.sync(() => { calls.push({ operation: "markReady", input }); + const current = allocations.get(allocationKey(input)); + if (current?.generation !== input.generation || current.tunnelId !== input.tunnelId) { + return false; + } mutate(allocationKey(input), (allocation) => ({ ...allocation, readyAt: "2026-06-02T00:00:00.000Z", })); + return true; }), enableRecovery: (input) => Effect.sync(() => { @@ -1026,12 +1036,13 @@ describe("ManagedEndpointProvider", () => { }); it.effect("rejects a tunnel recorded after its allocation generation changed", () => { + const tunnelCalls: TunnelCall[] = []; const allocations = makeAllocations(); const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ ...allocations, - recordTunnel: () => Effect.succeed(false), + recordTunnel: () => Effect.succeed(null), }); - const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(), changed); + const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), changed); return Effect.gen(function* () { const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; @@ -1047,6 +1058,57 @@ describe("ManagedEndpointProvider", () => { _tag: "ManagedEndpointProvisioningFailed", stage: "record-tunnel", }); + expect(tunnelCalls.map((call) => call.operation)).toEqual(["list", "create", "delete"]); + }).pipe(Effect.provide(layer)); + }); + + it.effect("does not overwrite DNS when tunnel ownership changes during provisioning", () => { + const allocations = makeAllocations(); + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + recordDns: () => Effect.succeed(null), + }); + const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const error = yield* Effect.flip( + provider.provision({ + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "record-dns", + }); + }).pipe(Effect.provide(layer)); + }); + + it.effect("does not mark a superseded tunnel allocation as ready", () => { + const allocations = makeAllocations(); + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + markReady: () => Effect.succeed(false), + }); + const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const error = yield* Effect.flip( + provider.provision({ + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "mark-allocation-ready", + }); }).pipe(Effect.provide(layer)); }); @@ -1094,12 +1156,14 @@ describe("ManagedEndpointProvider", () => { Effect.tap((claimedGeneration) => claimedGeneration === null ? Effect.void - : allocations.recordTunnel({ - userId: input.userId, - environmentId: input.environmentId, - tunnelId: "replacement-tunnel", - generation: claimedGeneration, - }), + : allocations + .recordTunnel({ + userId: input.userId, + environmentId: input.environmentId, + tunnelId: "replacement-tunnel", + generation: claimedGeneration, + }) + .pipe(Effect.asVoid), ), ), }); diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index f61a853f52bf..b3752a3b994e 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -515,52 +515,83 @@ export const make = Effect.gen(function* () { if (claimedGeneration === null) { return false; } - const dnsRecordId = allocation.dnsRecordId; - if (dnsRecordId !== null) { - yield* ignoreNotFound(dns.deleteRecord(dnsRecordId)).pipe( - Effect.mapError( - (cause) => - new ManagedEndpointDeprovisioningFailed({ - ...input, - stage: "delete-dns-record", - dnsRecordId, - cause, - }), - ), - ); - } const tunnelId = allocation.tunnelId; - if (tunnelId !== null) { - yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe( - Effect.mapError( - (cause) => - new ManagedEndpointDeprovisioningFailed({ - ...input, - stage: "delete-tunnel", - tunnelId, - cause, - }), - ), - ); + const deprovision = Effect.gen(function* () { + const dnsRecordId = allocation.dnsRecordId; + if (dnsRecordId !== null) { + yield* ignoreNotFound(dns.deleteRecord(dnsRecordId)).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "delete-dns-record", + dnsRecordId, + cause, + }), + ), + ); + } + if (tunnelId !== null) { + yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "delete-tunnel", + tunnelId, + cause, + }), + ), + ); + } + return yield* allocations + .removeClaimed({ + userId: input.userId, + environmentId: input.environmentId, + generation: claimedGeneration, + }) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "remove-allocation", + ...(allocation.tunnelId === null ? {} : { tunnelId: allocation.tunnelId }), + ...(allocation.dnsRecordId === null + ? {} + : { dnsRecordId: allocation.dnsRecordId }), + cause, + }), + ), + ); + }); + if (tunnelId === null) { + return yield* deprovision; } - return yield* allocations - .removeClaimed({ - userId: input.userId, - environmentId: input.environmentId, - generation: claimedGeneration, - }) + const removed = yield* allocations + .withClaimedTunnel( + { + userId: input.userId, + environmentId: input.environmentId, + tunnelId, + generation: claimedGeneration, + }, + deprovision, + ) .pipe( - Effect.mapError( - (cause) => - new ManagedEndpointDeprovisioningFailed({ - ...input, - stage: "remove-allocation", - ...(allocation.tunnelId === null ? {} : { tunnelId: allocation.tunnelId }), - ...(allocation.dnsRecordId === null ? {} : { dnsRecordId: allocation.dnsRecordId }), - cause, - }), - ), + Effect.catchTags({ + ManagedEndpointAllocationPersistenceError: (cause) => + Effect.fail( + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "claim-deprovision", + tunnelId, + cause, + }), + ), + }), ); + return Option.getOrElse(removed, () => false); }), release: Effect.fn("relay.managed_endpoint_provider.release")(function* (input) { yield* Effect.annotateCurrentSpan({ @@ -624,6 +655,48 @@ export const make = Effect.gen(function* () { ), ); if (input.expectedInactiveBefore !== undefined && input.expectedStatus !== undefined) { + const expectedStatus = input.expectedStatus; + const inactiveBefore = input.expectedInactiveBefore; + const currentTunnel = yield* tunnels.get(tunnelId).pipe( + Effect.map(Option.some), + Effect.catchTags({ + ManagedEndpointTunnelClientError: (cause) => + isManagedEndpointNotFound(cause.cause) + ? Effect.succeed(Option.none()) + : Effect.fail( + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "load-tunnel", + tunnelId, + cause, + }), + ), + }), + ); + if (Option.isNone(currentTunnel)) { + return true; + } + const inactiveAt = + expectedStatus === "down" + ? currentTunnel.value.connsInactiveAt + : currentTunnel.value.createdAt; + if ( + currentTunnel.value.id !== tunnelId || + currentTunnel.value.status !== expectedStatus || + typeof inactiveAt !== "string" + ) { + return false; + } + const inactiveTime = DateTime.make(inactiveAt); + const cutoff = DateTime.make(inactiveBefore); + if ( + Option.isNone(inactiveTime) || + Option.isNone(cutoff) || + inactiveTime.value.epochMilliseconds > cutoff.value.epochMilliseconds + ) { + return false; + } + const released = yield* allocations .withClaimedTunnel( { @@ -633,46 +706,6 @@ export const make = Effect.gen(function* () { generation: claimedGeneration, }, Effect.gen(function* () { - const currentTunnel = yield* tunnels.get(tunnelId).pipe( - Effect.map(Option.some), - Effect.catchTags({ - ManagedEndpointTunnelClientError: (cause) => - isManagedEndpointNotFound(cause.cause) - ? Effect.succeed(Option.none()) - : Effect.fail( - new ManagedEndpointDeprovisioningFailed({ - ...input, - stage: "load-tunnel", - tunnelId, - cause, - }), - ), - }), - ); - if (Option.isNone(currentTunnel)) { - return true; - } - const inactiveAt = - input.expectedStatus === "down" - ? currentTunnel.value.connsInactiveAt - : currentTunnel.value.createdAt; - if ( - currentTunnel.value.id !== tunnelId || - currentTunnel.value.status !== input.expectedStatus || - typeof inactiveAt !== "string" - ) { - return false; - } - const inactiveTime = DateTime.make(inactiveAt); - const cutoff = DateTime.make(input.expectedInactiveBefore!); - if ( - Option.isNone(inactiveTime) || - Option.isNone(cutoff) || - inactiveTime.value.epochMilliseconds > cutoff.value.epochMilliseconds - ) { - return false; - } - const finalGeneration = yield* allocations .claimRelease({ userId: input.userId, @@ -694,6 +727,8 @@ export const make = Effect.gen(function* () { if (finalGeneration === null) { return false; } + // Keep only the final delete inside the row lock so a concurrent + // provision cannot record this tunnel while Cloudflare removes it. yield* deleteTunnel; return true; }), @@ -805,13 +840,16 @@ export const make = Effect.gen(function* () { ); const { hostname, tunnelName } = allocation; - const tunnelResponse = yield* tunnels.list({ name: tunnelName, isDeleted: false }).pipe( + const selectedTunnel = yield* tunnels.list({ name: tunnelName, isDeleted: false }).pipe( Effect.map((tunnels) => tunnels.result), Effect.map(Arr.findFirst((tunnel) => tunnel.name === tunnelName)), Effect.flatMap( Option.match({ - onSome: (tunnel) => Effect.succeed(tunnel), - onNone: () => tunnels.create({ name: tunnelName, configSrc: "cloudflare" }), + onSome: (tunnel) => Effect.succeed({ tunnel, created: false }), + onNone: () => + tunnels + .create({ name: tunnelName, configSrc: "cloudflare" }) + .pipe(Effect.map((tunnel) => ({ tunnel, created: true }))), }), ), Effect.mapError( @@ -826,6 +864,7 @@ export const make = Effect.gen(function* () { }), ), ); + const tunnelResponse = selectedTunnel.tunnel; if (!tunnelResponse.id || tunnelResponse.name !== tunnelName) { return yield* new ManagedEndpointProvisioningFailed({ userId: input.userId, @@ -838,7 +877,7 @@ export const make = Effect.gen(function* () { }); } const tunnel = { id: tunnelResponse.id, name: tunnelResponse.name }; - const recordedTunnel = yield* allocations + const tunnelGeneration = yield* allocations .recordTunnel({ userId: input.userId, environmentId: input.environmentId, @@ -859,7 +898,34 @@ export const make = Effect.gen(function* () { }), ), ); - if (!recordedTunnel) { + if (tunnelGeneration === null) { + if (selectedTunnel.created) { + const current = yield* allocations.get(input).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "record-tunnel", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause, + }), + ), + ); + if (current?.tunnelId !== tunnel.id) { + yield* ignoreNotFound(tunnels.delete(tunnel.id)).pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to remove a tunnel that lost its allocation", { + tunnelId: tunnel.id, + tunnelName, + cause, + }), + ), + ); + } + } return yield* new ManagedEndpointProvisioningFailed({ userId: input.userId, environmentId: input.environmentId, @@ -919,11 +985,13 @@ export const make = Effect.gen(function* () { }), ), ); - yield* allocations + const dnsGeneration = yield* allocations .recordDns({ userId: input.userId, environmentId: input.environmentId, dnsRecordId, + tunnelId: tunnel.id, + generation: tunnelGeneration, }) .pipe( Effect.mapError( @@ -940,6 +1008,18 @@ export const make = Effect.gen(function* () { }), ), ); + if (dnsGeneration === null) { + return yield* new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "record-dns", + hostname, + tunnelName, + tunnelId: tunnel.id, + dnsRecordId, + cause: "The tunnel allocation changed before its DNS record was saved.", + }); + } const connectorToken = yield* tunnels.getToken(tunnel.id).pipe( Effect.mapError( @@ -956,10 +1036,12 @@ export const make = Effect.gen(function* () { }), ), ); - yield* allocations + const ready = yield* allocations .markReady({ userId: input.userId, environmentId: input.environmentId, + tunnelId: tunnel.id, + generation: dnsGeneration, }) .pipe( Effect.mapError( @@ -976,6 +1058,18 @@ export const make = Effect.gen(function* () { }), ), ); + if (!ready) { + return yield* new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "mark-allocation-ready", + hostname, + tunnelName, + tunnelId: tunnel.id, + dnsRecordId, + cause: "The tunnel allocation changed before it became ready.", + }); + } return { endpoint: managedEndpointForHostname(hostname), diff --git a/infra/relay/src/worker.ts b/infra/relay/src/worker.ts index b9f5a947957f..fe9c085fdea8 100644 --- a/infra/relay/src/worker.ts +++ b/infra/relay/src/worker.ts @@ -2,6 +2,7 @@ import * as Alchemy from "alchemy"; import * as Cloudflare from "alchemy/Cloudflare"; import * as Drizzle from "alchemy/Drizzle"; import * as Config from "effect/Config"; +import * as Cause from "effect/Cause"; import * as DateTime from "effect/DateTime"; import * as Crypto from "effect/Crypto"; import * as Effect from "effect/Effect"; @@ -278,8 +279,10 @@ export const ApiLive = Api.make( ), ), ), - Effect.catch((cause) => - Effect.logWarning("Failed to prune expired relay state", { cause }), + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.logWarning("Failed to prune expired relay state", { cause }), ), ), ManagedEndpointReaper.ManagedEndpointReaper.pipe( @@ -289,8 +292,10 @@ export const ApiLive = Api.make( ? Effect.logInfo("Finished managed tunnel cleanup", result) : Effect.void, ), - Effect.catch((cause) => - Effect.logWarning("Failed to clean up inactive managed tunnels", { cause }), + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.logWarning("Failed to clean up inactive managed tunnels", { cause }), ), ), ], From 71fc7769720a8838b8fc9d1af2c29e6d436ad6dc Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 24 Aug 2026 21:23:53 -0700 Subject: [PATCH 05/14] fix(connect): guard tunnel configuration and batch cleanup --- apps/server/src/server.ts | 11 +- .../ManagedEndpointAllocations.test.ts | 21 ++ .../ManagedEndpointAllocations.ts | 64 +++--- .../ManagedEndpointProvider.test.ts | 58 ++++++ .../environments/ManagedEndpointProvider.ts | 190 +++++++++++++----- 5 files changed, 265 insertions(+), 79 deletions(-) diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 7c95be8c85a7..44eaf202faf6 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -664,7 +664,16 @@ export const makeServerLayer = Layer.unwrap( // covers anything this sleep used to hedge against. Every // millisecond here is dead time on the path to remote // reachability after a restart. - if (hasCloudPublicConfig && (yield* CloudCliState.readCliDesiredCloudLink)) { + const wantsCliLink = hasCloudPublicConfig + ? yield* CloudCliState.readCliDesiredCloudLink.pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to read the desired T3 Connect link", { cause }).pipe( + Effect.as(false), + ), + ), + ) + : false; + if (wantsCliLink) { yield* reconcileDesiredCloudLink(localOrigin).pipe( Effect.retry({ while: (error) => diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts index 73612590e5fa..103b6c840a82 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts @@ -139,6 +139,27 @@ describe("ManagedEndpointAllocations", () => { }).pipe(Effect.provide(layerWithDb({} as RelayDb.RelayDb["Service"]))), ); + it.effect("splits large tunnel lookups into bounded database queries", () => { + const batchSizes: number[] = []; + const fakeDb = { + select: () => ({ + from: () => ({ + where: (condition: unknown) => { + batchSizes.push(new PgDialect().sqlToQuery(condition as never).params.length); + return Effect.succeed([]); + }, + }), + }), + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const names = Array.from({ length: 1_001 }, (_, index) => `tunnel-${index}`); + expect(yield* allocations.listByTunnelNames(names)).toEqual([]); + expect(batchSizes).toEqual([500, 500, 1]); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + it.effect("returns a claim generation only when deprovision wins the allocation CAS", () => { const fakeDb = { update: (table: unknown) => { diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index afd77151a34a..ee1bf35e7990 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -29,6 +29,8 @@ export interface ManagedEndpointTunnelAllocation extends ManagedEndpointAllocati readonly recoveryEnabled: boolean; } +export const MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE = 500; + export function resolveReadyManagedEndpoint(input: { readonly allocation: ManagedEndpointAllocation; readonly baseDomain: string | undefined; @@ -422,31 +424,45 @@ export const make = Effect.gen(function* () { if (tunnelNames.length === 0) { return []; } - return yield* db - .select({ - ...allocationSelection, - recoveryEnabledAt: relayManagedEndpointAllocations.recoveryEnabledAt, - }) - .from(relayManagedEndpointAllocations) - .where(inArray(relayManagedEndpointAllocations.tunnelName, tunnelNames)) - .pipe( - Effect.map((rows) => - rows.map(({ recoveryEnabledAt, ...allocation }) => ({ - ...allocation, - recoveryEnabled: recoveryEnabledAt !== null, - })), - ), - Effect.mapError( - (cause) => - new ManagedEndpointAllocationPersistenceError({ - operation: "list-tunnels", - stage: "database-request", - userId: "*", - environmentId: "*", - cause, - }), + const batches = Array.from( + { length: Math.ceil(tunnelNames.length / MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE) }, + (_, index) => + tunnelNames.slice( + index * MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE, + (index + 1) * MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE, ), - ); + ); + const results = yield* Effect.forEach( + batches, + (batch) => + db + .select({ + ...allocationSelection, + recoveryEnabledAt: relayManagedEndpointAllocations.recoveryEnabledAt, + }) + .from(relayManagedEndpointAllocations) + .where(inArray(relayManagedEndpointAllocations.tunnelName, batch)) + .pipe( + Effect.map((rows) => + rows.map(({ recoveryEnabledAt, ...allocation }) => ({ + ...allocation, + recoveryEnabled: recoveryEnabledAt !== null, + })), + ), + Effect.mapError( + (cause) => + new ManagedEndpointAllocationPersistenceError({ + operation: "list-tunnels", + stage: "database-request", + userId: "*", + environmentId: "*", + cause, + }), + ), + ), + { concurrency: 1 }, + ); + return results.flat(); }, ), claimRelease: Effect.fn("relay.managed_endpoint_allocations.claim_release")(function* ( diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 69d208a276ba..cc0ddc8b5edc 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -1087,6 +1087,64 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.provide(layer)); }); + it.effect("does not change tunnel ingress after another provision takes ownership", () => { + const tunnelCalls: TunnelCall[] = []; + const allocations = makeAllocations(); + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + withClaimedTunnel: () => Effect.succeed(Option.none()), + }); + const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const error = yield* Effect.flip( + provider.provision({ + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "configure-tunnel", + }); + expect(tunnelCalls.map((call) => call.operation)).not.toContain("putConfiguration"); + }).pipe(Effect.provide(layer)); + }); + + it.effect("does not change DNS after another provision takes ownership", () => { + const dnsCalls: DnsCall[] = []; + const allocations = makeAllocations(); + let lockCount = 0; + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + withClaimedTunnel: (input, effect) => + ++lockCount === 1 + ? allocations.withClaimedTunnel(input, effect) + : Effect.succeed(Option.none()), + }); + const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(dnsCalls), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const error = yield* Effect.flip( + provider.provision({ + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "record-dns", + }); + expect(dnsCalls).toEqual([]); + }).pipe(Effect.provide(layer)); + }); + it.effect("does not mark a superseded tunnel allocation as ready", () => { const allocations = makeAllocations(); const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index b3752a3b994e..507707213d5a 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -937,30 +937,66 @@ export const make = Effect.gen(function* () { }); } - yield* tunnels - .putConfiguration(tunnel.id, { - ingress: [ - { - hostname, - service: formatOriginService(input.origin), - }, - { service: "http_status:404" }, - ], - }) + const configured = yield* allocations + .withClaimedTunnel( + { + userId: input.userId, + environmentId: input.environmentId, + tunnelId: tunnel.id, + generation: tunnelGeneration, + }, + tunnels + .putConfiguration(tunnel.id, { + ingress: [ + { + hostname, + service: formatOriginService(input.origin), + }, + { service: "http_status:404" }, + ], + }) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "configure-tunnel", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause, + }), + ), + ), + ) .pipe( - Effect.mapError( - (cause) => - new ManagedEndpointProvisioningFailed({ - userId: input.userId, - environmentId: input.environmentId, - stage: "configure-tunnel", - hostname, - tunnelName, - tunnelId: tunnel.id, - cause, - }), - ), + Effect.catchTags({ + ManagedEndpointAllocationPersistenceError: (cause) => + Effect.fail( + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "configure-tunnel", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause, + }), + ), + }), ); + if (Option.isNone(configured)) { + return yield* new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "configure-tunnel", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause: "The tunnel allocation changed before its configuration was updated.", + }); + } const dnsRecord = { type: "CNAME", @@ -970,33 +1006,61 @@ export const make = Effect.gen(function* () { proxied: true, } as const; - const dnsRecordId = yield* ensureDnsRecord(hostname, allocation.dnsRecordId, dnsRecord).pipe( - Effect.mapError( - (cause) => - new ManagedEndpointProvisioningFailed({ - userId: input.userId, - environmentId: input.environmentId, - stage: "ensure-dns-record", + const recordedDns = yield* allocations + .withClaimedTunnel( + { + userId: input.userId, + environmentId: input.environmentId, + tunnelId: tunnel.id, + generation: tunnelGeneration, + }, + Effect.gen(function* () { + const dnsRecordId = yield* ensureDnsRecord( hostname, - tunnelName, - tunnelId: tunnel.id, - ...(allocation.dnsRecordId === null ? {} : { dnsRecordId: allocation.dnsRecordId }), - cause, - }), - ), - ); - const dnsGeneration = yield* allocations - .recordDns({ - userId: input.userId, - environmentId: input.environmentId, - dnsRecordId, - tunnelId: tunnel.id, - generation: tunnelGeneration, - }) - .pipe( - Effect.mapError( - (cause) => - new ManagedEndpointProvisioningFailed({ + allocation.dnsRecordId, + dnsRecord, + ).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "ensure-dns-record", + hostname, + tunnelName, + tunnelId: tunnel.id, + ...(allocation.dnsRecordId === null + ? {} + : { dnsRecordId: allocation.dnsRecordId }), + cause, + }), + ), + ); + const dnsGeneration = yield* allocations + .recordDns({ + userId: input.userId, + environmentId: input.environmentId, + dnsRecordId, + tunnelId: tunnel.id, + generation: tunnelGeneration, + }) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "record-dns", + hostname, + tunnelName, + tunnelId: tunnel.id, + dnsRecordId, + cause, + }), + ), + ); + if (dnsGeneration === null) { + return yield* new ManagedEndpointProvisioningFailed({ userId: input.userId, environmentId: input.environmentId, stage: "record-dns", @@ -1004,11 +1068,29 @@ export const make = Effect.gen(function* () { tunnelName, tunnelId: tunnel.id, dnsRecordId, - cause, - }), - ), + cause: "The tunnel allocation changed before its DNS record was saved.", + }); + } + return { dnsRecordId, dnsGeneration }; + }), + ) + .pipe( + Effect.catchTags({ + ManagedEndpointAllocationPersistenceError: (cause) => + Effect.fail( + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "record-dns", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause, + }), + ), + }), ); - if (dnsGeneration === null) { + if (Option.isNone(recordedDns)) { return yield* new ManagedEndpointProvisioningFailed({ userId: input.userId, environmentId: input.environmentId, @@ -1016,10 +1098,10 @@ export const make = Effect.gen(function* () { hostname, tunnelName, tunnelId: tunnel.id, - dnsRecordId, - cause: "The tunnel allocation changed before its DNS record was saved.", + cause: "The tunnel allocation changed before its DNS record was updated.", }); } + const { dnsRecordId, dnsGeneration } = recordedDns.value; const connectorToken = yield* tunnels.getToken(tunnel.id).pipe( Effect.mapError( From 645416cf1e42a3e1cc17b082ccbf20c464e5b4fd Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 24 Aug 2026 21:44:56 -0700 Subject: [PATCH 06/14] fix(connect): bind tunnel recovery to the current environment --- apps/server/src/cloud/http.test.ts | 30 +++++ apps/server/src/cloud/http.ts | 28 ++++- apps/server/src/server.ts | 2 + .../migration.sql | 1 + .../snapshot.json | 15 ++- .../ManagedEndpointAllocations.test.ts | 107 ++++++++++++++---- .../ManagedEndpointAllocations.ts | 34 +++++- .../ManagedEndpointProvider.test.ts | 1 + infra/relay/src/persistence/schema.ts | 1 + 9 files changed, 187 insertions(+), 32 deletions(-) rename infra/relay/migrations/postgres/{20260825034308_managed_endpoint_recovery => 20260825044249_managed_endpoint_recovery}/migration.sql (63%) rename infra/relay/migrations/postgres/{20260825034308_managed_endpoint_recovery => 20260825044249_managed_endpoint_recovery}/snapshot.json (98%) diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index fd0328b2c414..418230da2d35 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -662,6 +662,36 @@ describe("releaseManagedTunnelOnShutdown", () => { }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); }); + it.effect.each([ + { status: 401, errorTag: "EnvironmentHttpUnauthorizedError" }, + { status: 403, errorTag: "EnvironmentHttpUnauthorizedError" }, + { status: 409, errorTag: "EnvironmentHttpConflictError" }, + ])("preserves a permanent $status relay recovery failure", ({ status, errorTag }) => { + const { store } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + const error = yield* Effect.flip(recoverManagedCloudTunnel("http://127.0.0.1:3773")); + + expect(error._tag).toBe(errorTag); + expect(requests).toHaveLength(1); + expect(applyConfigCalls).toEqual([]); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respond: () => Response.json({}, { status }), + }), + ); + }); + it.effect("keeps a tunnel configuration replaced during recovery", () => { const { store, values } = makeMemorySecretStore([ [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"], diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index 14d913cd105c..fdd59c6d05f0 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -537,13 +537,29 @@ const relayClientRequest = ( HttpClientRequest.bearerToken(input.token), HttpClientRequest.bodyJson(input.payload), Effect.flatMap(dependencies.httpClient.execute), - Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap((response) => + Effect.gen(function* () { + if (response.status === 401 || response.status === 403) { + return yield* new EnvironmentHttpUnauthorizedError({ + message: "T3 Connect rejected the stored environment credential.", + }); + } + if (response.status === 409) { + return yield* new EnvironmentHttpConflictError({ + message: "T3 Connect rejected the current environment link.", + }); + } + return yield* HttpClientResponse.filterStatusOk(response); + }), + ), Effect.flatMap(HttpClientResponse.schemaBodyJson(input.schema)), - Effect.mapError( - (cause) => - new EnvironmentHttpInternalServerError({ - message: `T3 Connect relay request failed: ${String(cause)}`, - }), + Effect.mapError((cause) => + cause._tag === "EnvironmentHttpUnauthorizedError" || + cause._tag === "EnvironmentHttpConflictError" + ? cause + : new EnvironmentHttpInternalServerError({ + message: `T3 Connect relay request failed: ${String(cause)}`, + }), ), withRelayClientTracing, ); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 44eaf202faf6..b63758021096 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -631,6 +631,8 @@ export const makeServerLayer = Layer.unwrap( Effect.retry({ while: (error) => error._tag !== "EnvironmentHttpBadRequestError" && + error._tag !== "EnvironmentHttpUnauthorizedError" && + error._tag !== "EnvironmentHttpConflictError" && (error._tag !== "EnvironmentCloudEndpointUnavailableError" || CloudManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus( error.endpointRuntimeStatus, diff --git a/infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/migration.sql b/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/migration.sql similarity index 63% rename from infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/migration.sql rename to infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/migration.sql index 4da6b524efa4..a9480ed4c93b 100644 --- a/infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/migration.sql +++ b/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/migration.sql @@ -1,2 +1,3 @@ ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "recovery_enabled_at" varchar(64);--> statement-breakpoint +ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "recovery_environment_public_key" text;--> statement-breakpoint ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "generation" integer DEFAULT 0 NOT NULL; \ No newline at end of file diff --git a/infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/snapshot.json b/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/snapshot.json similarity index 98% rename from infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/snapshot.json rename to infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/snapshot.json index 21afe569ac4e..ce549ba1c6c0 100644 --- a/infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/snapshot.json +++ b/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/snapshot.json @@ -1,7 +1,7 @@ { "version": "8", "dialect": "postgres", - "id": "a0128e5a-4bba-4f2d-9851-82c3744dae2c", + "id": "7e85c554-d61a-4253-bd7b-17f92e98e665", "prevIds": ["2374caff-40bf-423c-9255-55e76dddbc2a"], "ddl": [ { @@ -877,6 +877,19 @@ "schema": "public", "table": "relay_managed_endpoint_allocations" }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "recovery_environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, { "type": "integer", "typeSchema": null, diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts index 103b6c840a82..de14b4b82a97 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts @@ -12,10 +12,49 @@ const layerWithDb = (db: RelayDb.RelayDb["Service"]) => ManagedEndpointAllocations.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, db))); describe("ManagedEndpointAllocations", () => { + it.effect("clears endpoint readiness when recording a replacement tunnel", () => { + let updated: + | { + readonly tunnelId: string; + readonly readyAt: string | null; + } + | undefined; + const fakeDb = { + update: (table: unknown) => { + expect(table).toBe(relayManagedEndpointAllocations); + return { + set: (values: { readonly tunnelId: string; readonly readyAt: string | null }) => { + updated = values; + return { + where: () => ({ + returning: () => Effect.succeed([{ generation: 8 }]), + }), + }; + }, + }; + }, + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + expect( + yield* allocations.recordTunnel({ + userId: "user-1", + environmentId: "environment-1", + tunnelId: "replacement-tunnel", + generation: 7, + }), + ).toBe(8); + expect(updated?.tunnelId).toBe("replacement-tunnel"); + expect(updated?.readyAt).toBeNull(); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + it.effect("records recovery support and advances the allocation generation", () => { let updated: | { readonly recoveryEnabledAt: string; + readonly recoveryEnvironmentPublicKey: string; readonly updatedAt: string; } | undefined; @@ -24,7 +63,11 @@ describe("ManagedEndpointAllocations", () => { update: (table: unknown) => { expect(table).toBe(relayManagedEndpointAllocations); return { - set: (values: { readonly recoveryEnabledAt: string; readonly updatedAt: string }) => { + set: (values: { + readonly recoveryEnabledAt: string; + readonly recoveryEnvironmentPublicKey: string; + readonly updatedAt: string; + }) => { updated = values; return { where: (where: unknown) => { @@ -52,6 +95,7 @@ describe("ManagedEndpointAllocations", () => { expect(updated?.recoveryEnabledAt).toBe(updated?.updatedAt); expect(updated?.recoveryEnabledAt).toBeDefined(); + expect(updated?.recoveryEnvironmentPublicKey).toBe("public-key"); const query = new PgDialect().sqlToQuery(condition as never); expect(query.sql).toContain('"relay_managed_endpoint_allocations"."tunnel_id"'); expect(query.sql).toContain('"relay_environment_links"."environment_public_key"'); @@ -101,21 +145,35 @@ describe("ManagedEndpointAllocations", () => { from: (table: unknown) => { expect(table).toBe(relayManagedEndpointAllocations); return { - where: () => - Effect.succeed([ - { - ...base, - environmentId: "environment-1", - tunnelId: "tunnel-1", - recoveryEnabledAt: "2026-08-25T12:00:00.000Z", - }, - { - ...base, - environmentId: "environment-2", - tunnelId: "tunnel-2", - recoveryEnabledAt: null, - }, - ]), + leftJoin: () => ({ + where: () => + Effect.succeed([ + { + ...base, + environmentId: "environment-1", + tunnelId: "tunnel-1", + recoveryEnabledAt: "2026-08-25T12:00:00.000Z", + recoveryEnvironmentPublicKey: "current-key", + linkedEnvironmentPublicKey: "current-key", + }, + { + ...base, + environmentId: "environment-2", + tunnelId: "tunnel-2", + recoveryEnabledAt: null, + recoveryEnvironmentPublicKey: null, + linkedEnvironmentPublicKey: "current-key", + }, + { + ...base, + environmentId: "environment-3", + tunnelId: "tunnel-3", + recoveryEnabledAt: "2026-08-25T12:00:00.000Z", + recoveryEnvironmentPublicKey: "old-key", + linkedEnvironmentPublicKey: "new-key", + }, + ]), + }), }; }, }), @@ -123,11 +181,16 @@ describe("ManagedEndpointAllocations", () => { return Effect.gen(function* () { const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; - const result = yield* allocations.listByTunnelNames(["first-tunnel", "second-tunnel"]); + const result = yield* allocations.listByTunnelNames([ + "first-tunnel", + "second-tunnel", + "third-tunnel", + ]); expect(result.map((entry) => [entry.tunnelId, entry.recoveryEnabled])).toEqual([ ["tunnel-1", true], ["tunnel-2", false], + ["tunnel-3", false], ]); }).pipe(Effect.provide(layerWithDb(fakeDb))); }); @@ -144,10 +207,12 @@ describe("ManagedEndpointAllocations", () => { const fakeDb = { select: () => ({ from: () => ({ - where: (condition: unknown) => { - batchSizes.push(new PgDialect().sqlToQuery(condition as never).params.length); - return Effect.succeed([]); - }, + leftJoin: () => ({ + where: (condition: unknown) => { + batchSizes.push(new PgDialect().sqlToQuery(condition as never).params.length); + return Effect.succeed([]); + }, + }), }), }), } as unknown as RelayDb.RelayDb["Service"]; diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index ee1bf35e7990..fda5aae35a50 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -288,6 +288,7 @@ export const make = Effect.gen(function* () { .update(relayManagedEndpointAllocations) .set({ tunnelId: input.tunnelId, + readyAt: null, updatedAt: DateTime.formatIso(yield* DateTime.now), generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) @@ -382,6 +383,7 @@ export const make = Effect.gen(function* () { .update(relayManagedEndpointAllocations) .set({ recoveryEnabledAt: now, + recoveryEnvironmentPublicKey: input.environmentPublicKey, updatedAt: now, generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) @@ -439,15 +441,39 @@ export const make = Effect.gen(function* () { .select({ ...allocationSelection, recoveryEnabledAt: relayManagedEndpointAllocations.recoveryEnabledAt, + recoveryEnvironmentPublicKey: + relayManagedEndpointAllocations.recoveryEnvironmentPublicKey, + linkedEnvironmentPublicKey: relayEnvironmentLinks.environmentPublicKey, }) .from(relayManagedEndpointAllocations) + .leftJoin( + relayEnvironmentLinks, + and( + eq(relayEnvironmentLinks.userId, relayManagedEndpointAllocations.userId), + eq( + relayEnvironmentLinks.environmentId, + relayManagedEndpointAllocations.environmentId, + ), + isNull(relayEnvironmentLinks.revokedAt), + ), + ) .where(inArray(relayManagedEndpointAllocations.tunnelName, batch)) .pipe( Effect.map((rows) => - rows.map(({ recoveryEnabledAt, ...allocation }) => ({ - ...allocation, - recoveryEnabled: recoveryEnabledAt !== null, - })), + rows.map( + ({ + recoveryEnabledAt, + recoveryEnvironmentPublicKey, + linkedEnvironmentPublicKey, + ...allocation + }) => ({ + ...allocation, + recoveryEnabled: + recoveryEnabledAt !== null && + recoveryEnvironmentPublicKey !== null && + recoveryEnvironmentPublicKey === linkedEnvironmentPublicKey, + }), + ), ), Effect.mapError( (cause) => diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index cc0ddc8b5edc..8131b20a635b 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -234,6 +234,7 @@ function makeAllocations(calls: AllocationCall[] = []) { mutate(allocationKey(input), (allocation) => ({ ...allocation, tunnelId: input.tunnelId, + readyAt: null, })); return allocations.get(allocationKey(input))?.generation ?? null; }), diff --git a/infra/relay/src/persistence/schema.ts b/infra/relay/src/persistence/schema.ts index a53be1ea1a30..88196edc4fd6 100644 --- a/infra/relay/src/persistence/schema.ts +++ b/infra/relay/src/persistence/schema.ts @@ -94,6 +94,7 @@ export const relayManagedEndpointAllocations = pgTable( dnsRecordId: varchar("dns_record_id", { length: 191 }), readyAt: varchar("ready_at", { length: 64 }), recoveryEnabledAt: varchar("recovery_enabled_at", { length: 64 }), + recoveryEnvironmentPublicKey: text("recovery_environment_public_key"), generation: integer("generation").notNull().default(0), createdAt: varchar("created_at", { length: 64 }).notNull(), updatedAt: varchar("updated_at", { length: 64 }).notNull(), From 2f235cf8b83185e070820c8ffc09d5e5f3338faa Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 24 Aug 2026 21:53:33 -0700 Subject: [PATCH 07/14] fix(connect): keep existing tunnels ready during recovery --- .../environments/ManagedEndpointAllocations.test.ts | 12 ++++++++---- .../src/environments/ManagedEndpointAllocations.ts | 2 +- .../src/environments/ManagedEndpointProvider.test.ts | 9 +++++++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts index de14b4b82a97..843604e5c4eb 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts @@ -12,18 +12,18 @@ const layerWithDb = (db: RelayDb.RelayDb["Service"]) => ManagedEndpointAllocations.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, db))); describe("ManagedEndpointAllocations", () => { - it.effect("clears endpoint readiness when recording a replacement tunnel", () => { + it.effect("clears endpoint readiness only when the recorded tunnel changes", () => { let updated: | { readonly tunnelId: string; - readonly readyAt: string | null; + readonly readyAt: unknown; } | undefined; const fakeDb = { update: (table: unknown) => { expect(table).toBe(relayManagedEndpointAllocations); return { - set: (values: { readonly tunnelId: string; readonly readyAt: string | null }) => { + set: (values: { readonly tunnelId: string; readonly readyAt: unknown }) => { updated = values; return { where: () => ({ @@ -46,7 +46,11 @@ describe("ManagedEndpointAllocations", () => { }), ).toBe(8); expect(updated?.tunnelId).toBe("replacement-tunnel"); - expect(updated?.readyAt).toBeNull(); + const query = new PgDialect().sqlToQuery(updated?.readyAt as never); + expect(query.sql).toBe( + 'case when "relay_managed_endpoint_allocations"."tunnel_id" = $1 then "relay_managed_endpoint_allocations"."ready_at" else null end', + ); + expect(query.params).toEqual(["replacement-tunnel"]); }).pipe(Effect.provide(layerWithDb(fakeDb))); }); diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index fda5aae35a50..bd764071c308 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -288,7 +288,7 @@ export const make = Effect.gen(function* () { .update(relayManagedEndpointAllocations) .set({ tunnelId: input.tunnelId, - readyAt: null, + readyAt: sql`case when ${relayManagedEndpointAllocations.tunnelId} = ${input.tunnelId} then ${relayManagedEndpointAllocations.readyAt} else null end`, updatedAt: DateTime.formatIso(yield* DateTime.now), generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 8131b20a635b..c7ae9c0d1569 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -234,7 +234,7 @@ function makeAllocations(calls: AllocationCall[] = []) { mutate(allocationKey(input), (allocation) => ({ ...allocation, tunnelId: input.tunnelId, - readyAt: null, + readyAt: allocation.tunnelId === input.tunnelId ? allocation.readyAt : null, })); return allocations.get(allocationKey(input))?.generation ?? null; }), @@ -807,7 +807,8 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.andThen(Effect.fail(failure))), deleteRecord: () => Effect.void, }); - const layer = providerLayer(makePersistentTunnelClient(), dnsClient, makeAllocations()); + const allocations = makeAllocations(); + const layer = providerLayer(makePersistentTunnelClient(), dnsClient, allocations); return Effect.gen(function* () { const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; @@ -830,6 +831,10 @@ describe("ManagedEndpointProvider", () => { "createRecord", "updateRecord", ]); + expect(yield* allocations.get(request)).toMatchObject({ + tunnelId: "tunnel-id", + readyAt: "2026-06-02T00:00:00.000Z", + }); }).pipe(Effect.provide(layer)); }); From 563ba54efb53a2e0be7f3fa0b28e598b96558082 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 25 Aug 2026 00:10:56 -0700 Subject: [PATCH 08/14] fix(connect): recover tunnels without startup provisioning --- .../src/cloud/ManagedEndpointRuntime.test.ts | 114 +++++++++++++++- .../src/cloud/ManagedEndpointRuntime.ts | 47 ++++++- apps/server/src/cloud/http.test.ts | 65 ++++++++- apps/server/src/cloud/http.ts | 38 ++++++ apps/server/src/server.ts | 31 ++++- docs/internals/t3-connect.md | 27 ++-- docs/user/remote-access.md | 3 +- infra/relay/src/http/Api.test.ts | 123 ++++++++++++++++++ infra/relay/src/http/Api.ts | 103 ++++++++++++--- packages/contracts/src/relay.ts | 19 +++ 10 files changed, 525 insertions(+), 45 deletions(-) diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index 40f8a271dd5a..733c19e11442 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -6,9 +6,11 @@ import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as PlatformError from "effect/PlatformError"; +import * as Queue from "effect/Queue"; import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay"; import * as RelayClient from "@t3tools/shared/relayClient"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; @@ -60,6 +62,7 @@ function makeHandle(input: { readonly onKill: () => void; readonly isRunning?: () => boolean; readonly exitCode?: Effect.Effect; + readonly output?: Stream.Stream; }) { return ChildProcessSpawner.makeHandle({ pid: ChildProcessSpawner.ProcessId(input.pid), @@ -73,7 +76,7 @@ function makeHandle(input: { stdin: Sink.drain, stdout: Stream.empty, stderr: Stream.empty, - all: Stream.empty, + all: input.output ?? Stream.empty, getInputFd: () => Sink.drain, getOutputFd: () => Stream.empty, }); @@ -164,6 +167,115 @@ describe("CloudManagedEndpointRuntime", () => { ).toBe("warning"); }); + it("recognizes tunnel authorization failures without matching ordinary transport errors", () => { + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Failed to get tunnel" connIndex=0', + ), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Record for tunnel not found" connIndex=0', + ), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Invalid tunnel secret" connIndex=0', + ), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="connection timed out" connIndex=0', + ), + ).toBe(false); + }); + + it.effect("keeps recovery requests sent before the server starts consuming them", () => + Effect.gen(function* () { + const runtime = yield* buildCloudManagedEndpointRuntime( + ChildProcessSpawner.make(() => Effect.die("unused")), + ); + const config = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "token", + tunnelId: "tunnel-1", + }; + + yield* runtime.requestRecovery(config); + + expect(Option.getOrNull(yield* Stream.runHead(runtime.recoveryRequests))).toEqual(config); + }), + ); + + it.effect("recovers a rejected tunnel without waiting for the connector to exit", () => + Effect.gen(function* () { + const output = yield* Queue.unbounded(); + const firstBatchObserved = yield* Deferred.make(); + const secondBatchObserved = yield* Deferred.make(); + const recoveryRequested = yield* Deferred.make(); + const spawned: Array = []; + const encoder = new TextEncoder(); + const connectorOutput = Stream.fromQueue(output).pipe( + Stream.tap((chunk) => { + const line = new TextDecoder().decode(chunk); + if (line === "first checkpoint\n") { + return Deferred.succeed(firstBatchObserved, undefined).pipe(Effect.asVoid); + } + if (line === "second checkpoint\n") { + return Deferred.succeed(secondBatchObserved, undefined).pipe(Effect.asVoid); + } + return Effect.void; + }), + ); + const spawner = ChildProcessSpawner.make(() => + Effect.gen(function* () { + const pid = 600; + spawned.push(pid); + const handle = makeHandle({ pid, onKill: () => {}, output: connectorOutput }); + yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore)); + return handle; + }), + ); + const runtime = yield* buildCloudManagedEndpointRuntime(spawner); + const config = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "token", + tunnelId: "deleted-tunnel", + }; + const rejectedLine = + '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Failed to get tunnel" connIndex=0\n'; + + yield* runtime.recoveryRequests.pipe( + Stream.runForEach((requested) => + Deferred.succeed(recoveryRequested, requested).pipe(Effect.asVoid), + ), + Effect.forkChild, + ); + yield* runtime.applyConfig(config); + + yield* Queue.offer(output, encoder.encode(rejectedLine.repeat(3))); + yield* Queue.offer(output, encoder.encode("first checkpoint\n")); + yield* Deferred.await(firstBatchObserved); + expect(yield* Deferred.isDone(recoveryRequested)).toBe(false); + + yield* Queue.offer( + output, + encoder.encode( + "2026-06-17T02:00:00Z INF Registered tunnel connection connIndex=0\n" + + rejectedLine.repeat(3), + ), + ); + yield* Queue.offer(output, encoder.encode("second checkpoint\n")); + yield* Deferred.await(secondBatchObserved); + expect(yield* Deferred.isDone(recoveryRequested)).toBe(false); + + yield* Queue.offer(output, encoder.encode(rejectedLine)); + + expect(yield* Deferred.await(recoveryRequested)).toEqual(config); + expect(spawned).toEqual([600]); + }), + ); + it.effect("starts, deduplicates, rotates, and stops the Cloudflare connector", () => Effect.gen(function* () { const spawned: Array = []; diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index ca12b430649f..3079428bb117 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -5,7 +5,7 @@ import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; -import * as PubSub from "effect/PubSub"; +import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Result from "effect/Result"; import * as Semaphore from "effect/Semaphore"; @@ -73,6 +73,9 @@ interface ActiveConnector { readonly config: RelayManagedEndpointRuntimeConfig; } +// Newly created tunnels can fail authorization briefly while Cloudflare propagates their token. +const TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY = 4; + export function classifyRelayClientOutput(line: string): "connected" | "warning" | "debug" { if (/\bRegistered tunnel connection\b/iu.test(line)) { return "connected"; @@ -83,6 +86,15 @@ export function classifyRelayClientOutput(line: string): "connected" | "warning" return /\b(?:ERR|WRN|FTL|PNC)\b/u.test(line) ? "warning" : "debug"; } +export function isRejectedRelayClientTunnelOutput(line: string): boolean { + return ( + /\bRegister tunnel error from server side\b/iu.test(line) && + /\bUnauthorized:\s*(?:Failed to get tunnel|Record for tunnel not found|Invalid tunnel secret)\b/iu.test( + line, + ) + ); +} + /** Connector startup failures can clear after installation or a later spawn attempt. */ export function isRetryableManagedEndpointRuntimeStatus(status: unknown): boolean { if (typeof status !== "object" || status === null || !("status" in status)) { @@ -120,7 +132,7 @@ export const make = Effect.gen(function* () { const relayClient = yield* RelayClient.RelayClient; const activeRef = yield* Ref.make(null); const desiredConfigRef = yield* Ref.make(null); - const recoveryRequests = yield* PubSub.sliding(1); + const recoveryRequests = yield* Queue.sliding(1); const reconcileSemaphore = yield* Semaphore.make(1); const linkStateSemaphore = yield* Semaphore.make(1); let reconcileConfig: CloudManagedEndpointRuntime["Service"]["applyConfig"]; @@ -162,7 +174,7 @@ export const make = Effect.gen(function* () { tunnelId: connector.config.tunnelId, tunnelName: connector.config.tunnelName, }); - yield* PubSub.publish(recoveryRequests, connector.config); + yield* Queue.offer(recoveryRequests, connector.config); yield* reconcileConfig(desiredConfig); }), ); @@ -170,8 +182,11 @@ export const make = Effect.gen(function* () { Effect.catchCause((cause) => Effect.logWarning("Relay client supervisor failed", { cause })), ); - const observeConnectorOutput = (connector: ActiveConnector) => - connector.child.all.pipe( + const observeConnectorOutput = (connector: ActiveConnector) => { + let rejectedRegistrations = 0; + let recoveryRequested = false; + + return connector.child.all.pipe( Stream.decodeText(), Stream.splitLines, Stream.map((line) => line.trim()), @@ -186,8 +201,25 @@ export const make = Effect.gen(function* () { }; switch (classifyRelayClientOutput(line)) { case "connected": + rejectedRegistrations = 0; return Effect.logInfo("Relay client tunnel connection registered", attributes); case "warning": + if (isRejectedRelayClientTunnelOutput(line)) { + rejectedRegistrations += 1; + if ( + !recoveryRequested && + rejectedRegistrations >= TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY + ) { + recoveryRequested = true; + return Effect.logWarning( + "Relay client tunnel was rejected; requesting recovery", + attributes, + ).pipe( + Effect.andThen(Queue.offer(recoveryRequests, connector.config)), + Effect.asVoid, + ); + } + } return Effect.logWarning("Relay client reported a transport warning", attributes); case "debug": return Effect.logDebug("Relay client output", attributes); @@ -202,6 +234,7 @@ export const make = Effect.gen(function* () { }), ), ); + }; reconcileConfig = Effect.fn("CloudManagedEndpointRuntime.reconcileConfig")(function* (config) { if (!config || config.providerKind !== "cloudflare_tunnel") { @@ -327,8 +360,8 @@ export const make = Effect.gen(function* () { const runtime = CloudManagedEndpointRuntime.of({ applyConfig, - recoveryRequests: Stream.fromPubSub(recoveryRequests), - requestRecovery: (config) => PubSub.publish(recoveryRequests, config).pipe(Effect.asVoid), + recoveryRequests: Stream.fromQueue(recoveryRequests), + requestRecovery: (config) => Queue.offer(recoveryRequests, config).pipe(Effect.asVoid), withLinkStateLock: linkStateSemaphore.withPermits(1), }); diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index 418230da2d35..c27c6a64a4f8 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -6,6 +6,7 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; +import * as Schema from "effect/Schema"; import * as Tracer from "effect/Tracer"; import * as Stream from "effect/Stream"; import { @@ -30,7 +31,10 @@ import { import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts"; import * as CliTokenManager from "./CliTokenManager.ts"; -import type { RelayLinkProofRequest } from "@t3tools/contracts/relay"; +import { + RelayManagedEndpointRecoveryRegistrationRequest, + type RelayLinkProofRequest, +} from "@t3tools/contracts/relay"; import { CLOUD_ENDPOINT_RUNTIME_CONFIG, CLOUD_LINKED_USER_ID, @@ -45,6 +49,7 @@ import { pendingServiceUpdateExists, reconcileDesiredCloudLink, recoverManagedCloudTunnel, + registerManagedCloudTunnelRecovery, releaseManagedTunnelOnShutdown, } from "./http.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; @@ -62,6 +67,9 @@ const storeFailure = (tag: "AlreadyExists" | "PermissionDenied") => }); const unusedSecretStoreOperation = () => Effect.die("unused secret-store operation"); +const decodeManagedTunnelRecoveryRegistration = Schema.decodeUnknownEffect( + Schema.fromJsonString(RelayManagedEndpointRecoveryRegistrationRequest), +); function makeSecretStore( create: ServerSecretStore.ServerSecretStore["Service"]["create"], @@ -600,6 +608,61 @@ describe("releaseManagedTunnelOnShutdown", () => { ); }); + it.effect("registers an existing tunnel without provisioning or restarting it", () => { + const { store } = makeMemorySecretStore([ + [ + CLOUD_ENDPOINT_RUNTIME_CONFIG, + '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}', + ], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* registerManagedCloudTunnelRecovery()).toBe(true); + expect(requests).toHaveLength(1); + expect(requests[0]?.method).toBe("POST"); + expect(requests[0]?.url).toBe( + "https://relay.example.test/v1/environments/env_123/tunnel/recovery", + ); + expect(requests[0]?.headers.authorization).toBe("Bearer environment-credential"); + const body = requests[0]?.body; + expect(body?._tag).toBe("Uint8Array"); + if (body?._tag === "Uint8Array") { + expect( + yield* decodeManagedTunnelRecoveryRegistration(new TextDecoder().decode(body.body)), + ).toEqual({ + cloudUserId: "user-123", + tunnelId: "existing-tunnel", + }); + } + expect(applyConfigCalls).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + + it.effect("does not register recovery without a recorded tunnel ID", () => { + const { store } = makeMemorySecretStore([ + [ + CLOUD_ENDPOINT_RUNTIME_CONFIG, + '{"providerKind":"cloudflare_tunnel","connectorToken":"token"}', + ], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* registerManagedCloudTunnelRecovery()).toBe(false); + expect(requests).toEqual([]); + expect(applyConfigCalls).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + it.effect("recovers a web-linked tunnel with its environment credential", () => { const oldConfig = '{"providerKind":"cloudflare_tunnel","connectorToken":"old-token","tunnelId":"old-tunnel"}'; diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index fdd59c6d05f0..b99c77368d18 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -71,6 +71,7 @@ import { CLOUD_ENDPOINT_RUNTIME_CONFIG, CLOUD_LINKED_USER_ID, CLOUD_MINT_PUBLIC_KEY, + decodeRuntimeConfig, encodeEndpointRuntimeConfigJson, PUBLISH_AGENT_ACTIVITY_SECRET, RELAY_ENVIRONMENT_CREDENTIAL_SECRET, @@ -669,6 +670,43 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD }, ); +export const registerManagedCloudTunnelRecovery = Effect.fn( + "environment.cloud.registerManagedCloudTunnelRecovery", +)(function* () { + const dependencies = yield* cloudHttpDependencies; + const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([ + dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), + dependencies.secrets.get(RELAY_URL_SECRET), + dependencies.secrets.get(CLOUD_LINKED_USER_ID), + dependencies.secrets.get(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), + ]); + if ( + Option.isNone(runtimeConfig) || + Option.isNone(relayUrl) || + Option.isNone(cloudUserId) || + Option.isNone(environmentCredential) + ) { + return false; + } + + const config = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeConfig.value))); + if (config?.providerKind !== "cloudflare_tunnel" || config.tunnelId === undefined) { + return false; + } + + const environmentId = yield* dependencies.environment.getEnvironmentId; + const registered = yield* relayClientRequest(dependencies, { + url: `${bytesToString(relayUrl.value)}/v1/environments/${encodeURIComponent(environmentId)}/tunnel/recovery`, + token: bytesToString(environmentCredential.value), + payload: { + cloudUserId: bytesToString(cloudUserId.value), + tunnelId: config.tunnelId, + }, + schema: RelayOkResponse, + }); + return registered.ok; +}); + export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverManagedCloudTunnel")( function* (localOrigin: string) { const dependencies = yield* cloudHttpDependencies; diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index b63758021096..3cb10cec8655 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -97,6 +97,7 @@ import { pendingServiceUpdateExists, reconcileDesiredCloudLink, recoverManagedCloudTunnel, + registerManagedCloudTunnelRecovery, releaseManagedTunnelOnShutdown, } from "./cloud/http.ts"; import { serverRelayBrokerTracingLayer } from "./cloud/relayTracing.ts"; @@ -675,8 +676,27 @@ export const makeServerLayer = Layer.unwrap( ), ) : false; - if (wantsCliLink) { - yield* reconcileDesiredCloudLink(localOrigin).pipe( + const desiredCliLinkMode = wantsCliLink + ? yield* CloudCliState.readCliDesiredLinkMode + : null; + const registerManagedTunnel = registerManagedCloudTunnelRecovery().pipe( + Effect.tap((registered) => + registered + ? Effect.logInfo("T3 Connect managed tunnel recovery registered") + : Effect.void, + ), + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.logWarning("Failed to register T3 Connect managed tunnel recovery", { + cause, + }).pipe(Effect.as(false)), + ), + ); + const registered = + desiredCliLinkMode === "publish_only" ? false : yield* registerManagedTunnel; + if (wantsCliLink && !registered) { + const reconciled = yield* reconcileDesiredCloudLink(localOrigin).pipe( Effect.retry({ while: (error) => error._tag !== "EnvironmentHttpBadRequestError" && @@ -690,14 +710,17 @@ export const makeServerLayer = Layer.unwrap( ), }), Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")), + Effect.as(true), Effect.catch((cause) => Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", { cause, - }), + }).pipe(Effect.as(false)), ), ); + if (reconciled && desiredCliLinkMode === "managed") { + yield* registerManagedTunnel; + } } - yield* recoverManagedTunnel; }), ); yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie); diff --git a/docs/internals/t3-connect.md b/docs/internals/t3-connect.md index e342f5adba62..16edf0c65b55 100644 --- a/docs/internals/t3-connect.md +++ b/docs/internals/t3-connect.md @@ -127,17 +127,22 @@ logout` performs the same cleanup and removes the stored CLI authorization. ### Managed tunnel lifecycle -Every linked environment stores a relay-issued environment credential. When a managed environment -starts or its connector exits, the server uses that credential to request a tunnel from the relay. -This also covers environments linked through web or mobile settings, which do not have a stored CLI -credential. The relay keeps the existing hostname and DNS record, so a replacement tunnel does not -change the public endpoint. - -After a host completes this recovery request, the relay records that the host can recreate its own -tunnel. The existing five-minute maintenance job removes tunnels from those hosts when Cloudflare -reports that they have been down for at least five minutes. Tunnels that never connected are removed -when they are at least five minutes old. The job leaves older hosts alone until they complete a -recovery request, and it only removes tunnels that belong to its own deployment stage. +Every linked environment stores a relay-issued environment credential. At startup, the server uses +that credential to register recovery support for its existing tunnel. Registration only updates the +relay database and does not call Cloudflare. Healthy CLI links reuse the stored tunnel instead of +provisioning it again. + +If the connector exits or repeatedly reports that Cloudflare rejected its tunnel, the server uses +the same environment credential to request a replacement. This also recovers a tunnel deleted +while a laptop was asleep, even when the connector keeps running. Environments linked through web +or mobile settings do not need a stored CLI credential. The relay keeps the existing hostname and +DNS record, so a replacement tunnel does not change the public endpoint. + +After a host registers recovery support, the existing five-minute maintenance job removes its +tunnel when Cloudflare reports that the tunnel has been down for at least five minutes. Tunnels +that never connected are removed when they are at least five minutes old. The job leaves older +hosts alone until they register recovery support, and it only removes tunnels that belong to its +own deployment stage. The background service has an independent lifecycle. Connect setup may offer to install it, but logout leaves it running; manage it with `t3 service status`, `install`, `update`, and `uninstall`. diff --git a/docs/user/remote-access.md b/docs/user/remote-access.md index ff6fc4285fb5..67259f8cce26 100644 --- a/docs/user/remote-access.md +++ b/docs/user/remote-access.md @@ -231,7 +231,8 @@ controls remain in **Settings** → **Connections** on web and desktop or **Sett If a linked environment stays offline for several minutes, T3 Connect removes its unused tunnel. The environment stays linked to your account and keeps the same address. When the server starts -again, T3 Connect creates a replacement tunnel automatically. You do not need to pair it again. +again or the computer wakes, T3 Connect creates a replacement tunnel automatically. You do not +need to pair it again. ## Security Notes diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index f759c9166e2e..883c366a511a 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -24,6 +24,7 @@ import { relayEnvironmentAuthLayer, relayNotFoundRoute, recoverEnvironmentTunnelRecord, + registerEnvironmentTunnelRecovery, revokeEnvironmentLinkRecord, traceRelayHttpRequestWith, unlinkEnvironmentRecord, @@ -224,6 +225,107 @@ const linkedEnvironmentRecord = { } as const; describe("relay managed tunnel recovery", () => { + it.effect("registers recovery for an existing tunnel without provisioning it", () => { + let recoveryEnabledFor: { + readonly userId: string; + readonly environmentId: string; + readonly tunnelId: string; + readonly environmentPublicKey: string; + } | null = null; + + return Effect.gen(function* () { + expect( + yield* registerEnvironmentTunnelRecovery({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + tunnelId: "existing-tunnel", + }), + ).toEqual({ ok: true }); + expect(recoveryEnabledFor).toEqual({ + userId: "user-1", + environmentId: "environment-1", + tunnelId: "existing-tunnel", + environmentPublicKey: "public-key", + }); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + provision: () => Effect.die("registration must not provision a tunnel"), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: (input) => + Effect.sync(() => { + recoveryEnabledFor = input; + return true; + }), + }), + ), + ), + ); + }); + + it.effect("rejects recovery registration for a different environment key", () => { + let recoveryEnabled = false; + + return Effect.gen(function* () { + const error = yield* Effect.flip( + registerEnvironmentTunnelRecovery({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "different-public-key", + tunnelId: "existing-tunnel", + }), + ); + + expect(error).toMatchObject({ _tag: "Unauthorized" }); + expect(recoveryEnabled).toBe(false); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => + Effect.sync(() => { + recoveryEnabled = true; + return true; + }), + }), + ), + ), + ); + }); + + it.effect("rejects recovery registration when the recorded tunnel changed", () => + Effect.gen(function* () { + const error = yield* Effect.flip( + registerEnvironmentTunnelRecovery({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + tunnelId: "stale-tunnel", + }), + ); + + expect(error).toMatchObject({ _tag: "Unauthorized" }); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => Effect.succeed(false), + }), + ), + ), + ), + ); + it.effect("recovers a linked environment and marks its tunnel as recoverable", () => { let recoveryEnabledFor: { readonly userId: string; @@ -348,6 +450,18 @@ describe("relay managed tunnel recovery", () => { it.effect("rejects a recovered tunnel that changes the linked endpoint", () => { let recoveryEnabled = false; + const cleaned: Array = []; + const target = { + userId: "user-1", + environmentId: "environment-1", + hostname: "different.example.test", + tunnelId: "replacement-tunnel", + tunnelName: "environment-1-tunnel", + dnsRecordId: "dns-1", + readyAt: "2026-07-28T00:00:00.000Z", + updatedAt: "replacement-generation", + generation: 3, + } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; return Effect.gen(function* () { const error = yield* Effect.flip( @@ -360,6 +474,7 @@ describe("relay managed tunnel recovery", () => { ); expect(error).toMatchObject({ _tag: "Unauthorized" }); expect(recoveryEnabled).toBe(false); + expect(cleaned).toEqual(["replacement-tunnel"]); }).pipe( Effect.provide( Layer.merge( @@ -378,6 +493,14 @@ describe("relay managed tunnel recovery", () => { tunnelId: "replacement-tunnel", }, }), + prepareDeprovision: () => Effect.succeed(target), + deprovision: ({ target: captured }) => + Effect.sync(() => { + if (captured?.tunnelId) { + cleaned.push(captured.tunnelId); + } + return true; + }), }), Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ enableRecovery: () => diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 58aefda36b60..d537b15a5983 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -471,6 +471,33 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron }, ); +export const registerEnvironmentTunnelRecovery = Effect.fn( + "relay.api.server.registerEnvironmentTunnelRecovery", +)(function* (input: { + readonly userId: string; + readonly environmentId: string; + readonly environmentPublicKey: string; + readonly tunnelId: string; +}) { + const links = yield* EnvironmentLinks.EnvironmentLinks; + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const link = yield* links.getForUser({ + userId: input.userId, + environmentId: input.environmentId, + }); + if ( + link === null || + link.environmentPublicKey !== input.environmentPublicKey || + link.endpoint.providerKind !== "cloudflare_tunnel" + ) { + return yield* new HttpApiError.Unauthorized({}); + } + if (!(yield* allocations.enableRecovery(input))) { + return yield* new HttpApiError.Unauthorized({}); + } + return { ok: true }; +}); + export const recoverEnvironmentTunnelRecord = Effect.fn( "relay.api.server.recoverEnvironmentTunnelRecord", )(function* (input: { @@ -505,6 +532,22 @@ export const recoverEnvironmentTunnelRecord = Effect.fn( recovered.endpoint.httpBaseUrl !== link.endpoint.httpBaseUrl || recovered.endpoint.wsBaseUrl !== link.endpoint.wsBaseUrl ) { + if (recoveredTunnelId !== undefined) { + const owner = { userId: input.userId, environmentId: input.environmentId }; + const target = yield* managedEndpointProvider.prepareDeprovision(owner); + if (target?.tunnelId === recoveredTunnelId) { + yield* managedEndpointProvider.deprovision({ ...owner, target }).pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to clean up a tunnel with a mismatched endpoint", { + userId: input.userId, + environmentId: input.environmentId, + tunnelId: recoveredTunnelId, + cause, + }), + ), + ); + } + } return yield* new HttpApiError.Unauthorized({}); } @@ -1057,35 +1100,55 @@ export const serverApi = HttpApiBuilder.group( ), ); - return activityHandlers.handle( - "recoverManagedEndpoint", - Effect.fn("relay.api.server.recoverManagedEndpoint")( - function* ({ params, payload }) { + return activityHandlers + .handle( + "registerManagedEndpointRecovery", + Effect.fn("relay.api.server.registerManagedEndpointRecovery")(function* ({ + params, + payload, + }) { const principal = yield* RelayEnvironmentPrincipal; if (principal.environmentId !== params.environmentId) { return yield* new HttpApiError.Unauthorized({}); } yield* appendRelayCredentialResponseHeaders; - return yield* recoverEnvironmentTunnelRecord({ + return yield* registerEnvironmentTunnelRecovery({ userId: payload.cloudUserId, environmentId: params.environmentId, environmentPublicKey: principal.environmentPublicKey, - origin: payload.origin, + tunnelId: payload.tunnelId, }); - }, - Effect.catchTags({ - ManagedEndpointOriginNotAllowed: () => Effect.fail(new HttpApiError.Unauthorized({})), - ManagedEndpointProvisioningNotConfigured: () => - relayInternalErrorResponse("upstream_unavailable"), - ManagedEndpointProvisioningFailed: () => - relayInternalErrorResponse("upstream_unavailable"), - ManagedEndpointDeprovisioningFailed: () => - relayInternalErrorResponse("upstream_unavailable"), - ManagedTunnelLimitExceeded: () => relayInternalErrorResponse("upstream_unavailable"), - }), - mapRelayCommonApiErrors("not_authorized"), - ), - ); + }, mapRelayCommonApiErrors("not_authorized")), + ) + .handle( + "recoverManagedEndpoint", + Effect.fn("relay.api.server.recoverManagedEndpoint")( + function* ({ params, payload }) { + const principal = yield* RelayEnvironmentPrincipal; + if (principal.environmentId !== params.environmentId) { + return yield* new HttpApiError.Unauthorized({}); + } + yield* appendRelayCredentialResponseHeaders; + return yield* recoverEnvironmentTunnelRecord({ + userId: payload.cloudUserId, + environmentId: params.environmentId, + environmentPublicKey: principal.environmentPublicKey, + origin: payload.origin, + }); + }, + Effect.catchTags({ + ManagedEndpointOriginNotAllowed: () => Effect.fail(new HttpApiError.Unauthorized({})), + ManagedEndpointProvisioningNotConfigured: () => + relayInternalErrorResponse("upstream_unavailable"), + ManagedEndpointProvisioningFailed: () => + relayInternalErrorResponse("upstream_unavailable"), + ManagedEndpointDeprovisioningFailed: () => + relayInternalErrorResponse("upstream_unavailable"), + ManagedTunnelLimitExceeded: () => relayInternalErrorResponse("upstream_unavailable"), + }), + mapRelayCommonApiErrors("not_authorized"), + ), + ); }), ); diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index 76b0b0ef7438..cd1ea10228e1 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -165,6 +165,13 @@ export const RelayManagedEndpointRecoveryRequest = Schema.Struct({ }); export type RelayManagedEndpointRecoveryRequest = typeof RelayManagedEndpointRecoveryRequest.Type; +export const RelayManagedEndpointRecoveryRegistrationRequest = Schema.Struct({ + cloudUserId: TrimmedNonEmptyString, + tunnelId: TrimmedNonEmptyString, +}); +export type RelayManagedEndpointRecoveryRegistrationRequest = + typeof RelayManagedEndpointRecoveryRegistrationRequest.Type; + export const RelayManagedEndpointRecoveryResponse = Schema.Struct({ endpoint: RelayManagedEndpoint, endpointRuntime: RelayManagedEndpointRuntimeConfig, @@ -1065,6 +1072,18 @@ export const RelayDpopClientGroup = HttpApiGroup.make("dpopClient") export const RelayServerGroup = HttpApiGroup.make("server") .add( + HttpApiEndpoint.post( + "registerManagedEndpointRecovery", + "/v1/environments/:environmentId/tunnel/recovery", + { + params: Schema.Struct({ + environmentId: EnvironmentId, + }), + payload: RelayManagedEndpointRecoveryRegistrationRequest, + success: RelayOkResponse, + error: RelayAuthAndInternalErrors, + }, + ).annotate(OpenApi.Summary, "Register managed tunnel recovery without provisioning"), HttpApiEndpoint.post("recoverManagedEndpoint", "/v1/environments/:environmentId/tunnel", { params: Schema.Struct({ environmentId: EnvironmentId, From 3eaad479bd7cf3926c5ca463867f67ccf186c4ba Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 25 Aug 2026 00:34:53 -0700 Subject: [PATCH 09/14] fix(connect): sign tunnel recovery requests --- .../src/cloud/ManagedEndpointRuntime.test.ts | 16 ++- .../src/cloud/ManagedEndpointRuntime.ts | 8 +- apps/server/src/cloud/http.test.ts | 33 +++++- apps/server/src/cloud/http.ts | 110 ++++++++++++++++-- apps/server/src/server.ts | 58 ++++----- docs/internals/t3-connect.md | 12 +- infra/relay/src/http/Api.test.ts | 89 +++++++++++--- infra/relay/src/http/Api.ts | 89 +++++++++++++- packages/contracts/src/relay.ts | 21 ++++ packages/shared/src/relayJwt.ts | 1 + 10 files changed, 361 insertions(+), 76 deletions(-) diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index 733c19e11442..cb2441095f5d 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -213,6 +213,8 @@ describe("CloudManagedEndpointRuntime", () => { const firstBatchObserved = yield* Deferred.make(); const secondBatchObserved = yield* Deferred.make(); const recoveryRequested = yield* Deferred.make(); + const recoveryRetried = yield* Deferred.make(); + let recoveryRequestCount = 0; const spawned: Array = []; const encoder = new TextEncoder(); const connectorOutput = Stream.fromQueue(output).pipe( @@ -246,9 +248,13 @@ describe("CloudManagedEndpointRuntime", () => { '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Failed to get tunnel" connIndex=0\n'; yield* runtime.recoveryRequests.pipe( - Stream.runForEach((requested) => - Deferred.succeed(recoveryRequested, requested).pipe(Effect.asVoid), - ), + Stream.runForEach((requested) => { + recoveryRequestCount += 1; + return Deferred.succeed( + recoveryRequestCount === 1 ? recoveryRequested : recoveryRetried, + requested, + ).pipe(Effect.asVoid); + }), Effect.forkChild, ); yield* runtime.applyConfig(config); @@ -272,6 +278,10 @@ describe("CloudManagedEndpointRuntime", () => { yield* Queue.offer(output, encoder.encode(rejectedLine)); expect(yield* Deferred.await(recoveryRequested)).toEqual(config); + + yield* Queue.offer(output, encoder.encode(rejectedLine.repeat(4))); + + expect(yield* Deferred.await(recoveryRetried)).toEqual(config); expect(spawned).toEqual([600]); }), ); diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index 3079428bb117..7695327da63b 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -184,7 +184,6 @@ export const make = Effect.gen(function* () { const observeConnectorOutput = (connector: ActiveConnector) => { let rejectedRegistrations = 0; - let recoveryRequested = false; return connector.child.all.pipe( Stream.decodeText(), @@ -206,11 +205,8 @@ export const make = Effect.gen(function* () { case "warning": if (isRejectedRelayClientTunnelOutput(line)) { rejectedRegistrations += 1; - if ( - !recoveryRequested && - rejectedRegistrations >= TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY - ) { - recoveryRequested = true; + if (rejectedRegistrations >= TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY) { + rejectedRegistrations = 0; return Effect.logWarning( "Relay client tunnel was rejected; requesting recovery", attributes, diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index c27c6a64a4f8..6e500aec5816 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -270,7 +270,10 @@ describe("releaseManagedTunnelOnShutdown", () => { Effect.sync(() => { values.set(name, value); }), - create: unusedSecretStoreOperation, + create: (name, value) => + Effect.sync(() => { + values.set(name, value); + }), getOrCreateRandom: unusedSecretStoreOperation, remove: (name) => Effect.sync(() => { @@ -634,7 +637,7 @@ describe("releaseManagedTunnelOnShutdown", () => { if (body?._tag === "Uint8Array") { expect( yield* decodeManagedTunnelRecoveryRegistration(new TextDecoder().decode(body.body)), - ).toEqual({ + ).toMatchObject({ cloudUserId: "user-123", tunnelId: "existing-tunnel", }); @@ -725,6 +728,32 @@ describe("releaseManagedTunnelOnShutdown", () => { }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); }); + it.effect("ignores recovery requests for a tunnel that has already been replaced", () => { + const { store } = makeMemorySecretStore([ + [ + CLOUD_ENDPOINT_RUNTIME_CONFIG, + '{"providerKind":"cloudflare_tunnel","connectorToken":"current-token","tunnelId":"current-tunnel"}', + ], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect( + yield* recoverManagedCloudTunnel("http://127.0.0.1:3773", { + providerKind: "cloudflare_tunnel", + connectorToken: "old-token", + tunnelId: "old-tunnel", + }), + ).toBe(false); + expect(requests).toEqual([]); + expect(applyConfigCalls).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + it.effect.each([ { status: 401, errorTag: "EnvironmentHttpUnauthorizedError" }, { status: 403, errorTag: "EnvironmentHttpUnauthorizedError" }, diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index b99c77368d18..a0cbcafa5a4a 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -28,7 +28,9 @@ import { RelayEnvironmentLinkProofPayload, RelayLinkProofRequest, RelayManagedEndpointOrigin, + RelayManagedEndpointRecoveryProofPayload, RelayManagedEndpointRecoveryResponse, + type RelayManagedEndpointRuntimeConfig, RelayOkResponse, } from "@t3tools/contracts/relay"; import { withRelayClientTracing } from "@t3tools/shared/relayTracing"; @@ -37,6 +39,7 @@ import { RELAY_HEALTH_REQUEST_TYP, RELAY_HEALTH_RESPONSE_TYP, RELAY_LINK_PROOF_TYP, + RELAY_MANAGED_TUNNEL_RECOVERY_TYP, RELAY_MINT_REQUEST_TYP, RELAY_MINT_RESPONSE_TYP, signRelayJwt, @@ -496,7 +499,14 @@ const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(fu CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(endpointRuntimeJson), ); - if (options?.requestRecovery !== false) { + const registered = yield* registerManagedCloudTunnelRecovery().pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to register T3 Connect managed tunnel recovery", { + cause, + }).pipe(Effect.as(false)), + ), + ); + if (!registered && options?.requestRecovery !== false) { yield* dependencies.endpointRuntime.requestRecovery(payload.endpointRuntime); } } else { @@ -670,6 +680,53 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD }, ); +type ManagedTunnelRecoveryProofInput = { + readonly environmentId: RelayManagedEndpointRecoveryProofPayload["environmentId"]; + readonly cloudUserId: string; + readonly relayUrl: string; +} & ( + | { readonly action: "register"; readonly tunnelId: string } + | { readonly action: "recover"; readonly origin: RelayManagedEndpointOrigin } +); + +const makeManagedTunnelRecoveryProof = Effect.fn( + "environment.cloud.makeManagedTunnelRecoveryProof", +)(function* (dependencies: CloudHttpDependencies, input: ManagedTunnelRecoveryProofInput) { + const keyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(dependencies.secrets); + const configuredIssuer = yield* dependencies.secrets.get(RELAY_ISSUER_SECRET); + const now = yield* DateTime.now; + const issuedAt = Math.floor(now.epochMilliseconds / 1_000); + const claims = { + iss: `t3-env:${input.environmentId}`, + aud: normalizeRelayIssuer( + Option.isSome(configuredIssuer) ? bytesToString(configuredIssuer.value) : input.relayUrl, + ), + sub: input.environmentId, + jti: yield* Crypto.Crypto.pipe(Effect.flatMap((crypto) => crypto.randomUUIDv4)), + iat: issuedAt, + exp: issuedAt + 60, + environmentId: input.environmentId, + cloudUserId: input.cloudUserId, + }; + const payload = + input.action === "register" + ? { ...claims, action: "register" as const, tunnelId: input.tunnelId } + : { ...claims, action: "recover" as const, origin: input.origin }; + + return yield* signRelayJwt({ + privateKey: keyPair.privateKey, + typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP, + payload, + }).pipe( + Effect.mapError( + () => + new EnvironmentHttpInternalServerError({ + message: "Could not sign the managed tunnel recovery request.", + }), + ), + ); +}); + export const registerManagedCloudTunnelRecovery = Effect.fn( "environment.cloud.registerManagedCloudTunnelRecovery", )(function* () { @@ -695,12 +752,22 @@ export const registerManagedCloudTunnelRecovery = Effect.fn( } const environmentId = yield* dependencies.environment.getEnvironmentId; + const relayUrlValue = bytesToString(relayUrl.value); + const cloudUserIdValue = bytesToString(cloudUserId.value); + const proof = yield* makeManagedTunnelRecoveryProof(dependencies, { + action: "register", + environmentId, + cloudUserId: cloudUserIdValue, + relayUrl: relayUrlValue, + tunnelId: config.tunnelId, + }); const registered = yield* relayClientRequest(dependencies, { - url: `${bytesToString(relayUrl.value)}/v1/environments/${encodeURIComponent(environmentId)}/tunnel/recovery`, + url: `${relayUrlValue}/v1/environments/${encodeURIComponent(environmentId)}/tunnel/recovery`, token: bytesToString(environmentCredential.value), payload: { - cloudUserId: bytesToString(cloudUserId.value), + cloudUserId: cloudUserIdValue, tunnelId: config.tunnelId, + proof, }, schema: RelayOkResponse, }); @@ -708,7 +775,7 @@ export const registerManagedCloudTunnelRecovery = Effect.fn( }); export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverManagedCloudTunnel")( - function* (localOrigin: string) { + function* (localOrigin: string, expectedConfig?: RelayManagedEndpointRuntimeConfig) { const dependencies = yield* cloudHttpDependencies; const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([ dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), @@ -724,6 +791,18 @@ export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverMan ) { return false; } + if (expectedConfig !== undefined) { + const current = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeConfig.value))); + if ( + current === null || + current.providerKind !== expectedConfig.providerKind || + current.connectorToken !== expectedConfig.connectorToken || + current.tunnelId !== expectedConfig.tunnelId || + current.tunnelName !== expectedConfig.tunnelName + ) { + return false; + } + } const localUrl = yield* Effect.try({ try: () => new URL(localOrigin), @@ -739,15 +818,26 @@ export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverMan } const environmentId = yield* dependencies.environment.getEnvironmentId; + const relayUrlValue = bytesToString(relayUrl.value); + const cloudUserIdValue = bytesToString(cloudUserId.value); + const origin = { + localHttpHost: localUrl.hostname, + localHttpPort: endpointRequestPort(localUrl), + }; + const proof = yield* makeManagedTunnelRecoveryProof(dependencies, { + action: "recover", + environmentId, + cloudUserId: cloudUserIdValue, + relayUrl: relayUrlValue, + origin, + }); const recovered = yield* relayClientRequest(dependencies, { - url: `${bytesToString(relayUrl.value)}/v1/environments/${encodeURIComponent(environmentId)}/tunnel`, + url: `${relayUrlValue}/v1/environments/${encodeURIComponent(environmentId)}/tunnel`, token: bytesToString(environmentCredential.value), payload: { - cloudUserId: bytesToString(cloudUserId.value), - origin: { - localHttpHost: localUrl.hostname, - localHttpPort: endpointRequestPort(localUrl), - }, + cloudUserId: cloudUserIdValue, + origin, + proof, }, schema: RelayManagedEndpointRecoveryResponse, }); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 3cb10cec8655..4347223f12a5 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -1,4 +1,5 @@ import { EnvironmentHttpApi } from "@t3tools/contracts"; +import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay"; import * as Cause from "effect/Cause"; import * as Duration from "effect/Duration"; import * as Deferred from "effect/Deferred"; @@ -627,38 +628,39 @@ export const makeServerLayer = Layer.unwrap( const localOrigin = `http://127.0.0.1:${address.port}`; const endpointRuntime = yield* CloudManagedEndpointRuntime.CloudManagedEndpointRuntime; const recoveryLock = yield* Semaphore.make(1); - const recoverManagedTunnel = recoveryLock.withPermits(1)( - recoverManagedCloudTunnel(localOrigin).pipe( - Effect.retry({ - while: (error) => - error._tag !== "EnvironmentHttpBadRequestError" && - error._tag !== "EnvironmentHttpUnauthorizedError" && - error._tag !== "EnvironmentHttpConflictError" && - (error._tag !== "EnvironmentCloudEndpointUnavailableError" || - CloudManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus( - error.endpointRuntimeStatus, - )), - schedule: Schedule.exponential("1 second").pipe( - Schedule.modifyDelay(({ duration }) => - Effect.succeed(Duration.min(duration, Duration.seconds(30))), + const recoverManagedTunnel = (config: RelayManagedEndpointRuntimeConfig) => + recoveryLock.withPermits(1)( + recoverManagedCloudTunnel(localOrigin, config).pipe( + Effect.retry({ + while: (error) => + error._tag !== "EnvironmentHttpBadRequestError" && + error._tag !== "EnvironmentHttpUnauthorizedError" && + error._tag !== "EnvironmentHttpConflictError" && + (error._tag !== "EnvironmentCloudEndpointUnavailableError" || + CloudManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus( + error.endpointRuntimeStatus, + )), + schedule: Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + Schedule.upTo({ duration: "10 minutes" }), ), - Schedule.upTo({ duration: "10 minutes" }), + }), + Effect.tap((recovered) => + recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void, + ), + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.logWarning("Failed to recover the T3 Connect managed tunnel", { + cause, + }), ), - }), - Effect.tap((recovered) => - recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void, - ), - Effect.catchCause((cause) => - Cause.hasInterrupts(cause) - ? Effect.interrupt - : Effect.logWarning("Failed to recover the T3 Connect managed tunnel", { - cause, - }), ), - ), - ); + ); yield* endpointRuntime.recoveryRequests.pipe( - Stream.runForEach(() => recoverManagedTunnel), + Stream.runForEach(recoverManagedTunnel), Effect.forkScoped, ); // No settling delay before the first attempt: routes are already diff --git a/docs/internals/t3-connect.md b/docs/internals/t3-connect.md index 16edf0c65b55..0c7ed2ba3096 100644 --- a/docs/internals/t3-connect.md +++ b/docs/internals/t3-connect.md @@ -127,16 +127,18 @@ logout` performs the same cleanup and removes the stored CLI authorization. ### Managed tunnel lifecycle -Every linked environment stores a relay-issued environment credential. At startup, the server uses -that credential to register recovery support for its existing tunnel. Registration only updates the -relay database and does not call Cloudflare. Healthy CLI links reuse the stored tunnel instead of -provisioning it again. +Every linked environment stores a relay-issued environment credential. When setup installs a tunnel +and when the server starts, the server uses that credential to register recovery support for the +existing tunnel. Registration only updates the relay database and does not call Cloudflare. +Healthy CLI links reuse the stored tunnel instead of provisioning it again. If the connector exits or repeatedly reports that Cloudflare rejected its tunnel, the server uses the same environment credential to request a replacement. This also recovers a tunnel deleted while a laptop was asleep, even when the connector keeps running. Environments linked through web or mobile settings do not need a stored CLI credential. The relay keeps the existing hostname and -DNS record, so a replacement tunnel does not change the public endpoint. +DNS record, so a replacement tunnel does not change the public endpoint. Each registration and +recovery request includes a short-lived host signature that binds the cloud user and the current +tunnel or local T3 server address. After a host registers recovery support, the existing five-minute maintenance job removes its tunnel when Cloudflare reports that the tunnel has been down for at least five minutes. Tunnels diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index 883c366a511a..4f1c15ff6bcf 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -1,7 +1,9 @@ +import * as NodeCrypto from "node:crypto"; import { createClerkClient, verifyToken } from "@clerk/backend"; import { describe, expect, it } from "@effect/vitest"; import { vi } from "vite-plus/test"; import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; @@ -16,6 +18,7 @@ import * as HttpServerRequest from "effect/unstable/http/HttpServerRequest"; import * as HttpServerResponse from "effect/unstable/http/HttpServerResponse"; import { EnvironmentId } from "@t3tools/contracts"; import { RelayEnvironmentAuth } from "@t3tools/contracts/relay"; +import { RELAY_MANAGED_TUNNEL_RECOVERY_TYP, signRelayJwt } from "@t3tools/shared/relayJwt"; import { RELAY_REQUEST_DEADLINE_MS, @@ -29,6 +32,7 @@ import { traceRelayHttpRequestWith, unlinkEnvironmentRecord, verifyRelayClientBearerToken, + verifyEnvironmentTunnelRecoveryProof, withoutCapturedParentSpan, } from "./Api.ts"; import * as RelayConfiguration from "../Config.ts"; @@ -172,6 +176,7 @@ function relayUnlinkTestLayer(input?: { readonly prepareDeprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["prepareDeprovision"]; readonly deprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["deprovision"]; readonly provision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["provision"]; + readonly release?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["release"]; }) { return Layer.mergeAll( Layer.succeed( @@ -206,7 +211,7 @@ function relayUnlinkTestLayer(input?: { provision: input?.provision ?? (() => Effect.die("unused provision")), prepareDeprovision: input?.prepareDeprovision ?? (() => Effect.succeed(null)), deprovision: input?.deprovision ?? (() => Effect.succeed(true)), - release: () => Effect.die("unused release"), + release: input?.release ?? (() => Effect.die("unused release")), }), ), ); @@ -225,6 +230,68 @@ const linkedEnvironmentRecord = { } as const; describe("relay managed tunnel recovery", () => { + it.effect("binds recovery requests to the host, cloud user, and T3 service origin", () => + Effect.gen(function* () { + const keyPair = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, + }); + const now = yield* DateTime.now; + const issuedAt = Math.floor(now.epochMilliseconds / 1_000); + const proof = yield* signRelayJwt({ + privateKey: keyPair.privateKey, + typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP, + payload: { + iss: "t3-env:environment-1", + aud: "https://relay.example.test", + sub: "environment-1", + jti: "recovery-proof", + iat: issuedAt, + exp: issuedAt + 60, + action: "recover", + environmentId: "environment-1", + cloudUserId: "user-1", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }, + }); + const request = { + action: "recover" as const, + proof, + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: keyPair.publicKey, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }; + + yield* verifyEnvironmentTunnelRecoveryProof(request); + + const wrongOwner = yield* Effect.flip( + verifyEnvironmentTunnelRecoveryProof({ ...request, userId: "user-2" }), + ); + expect(wrongOwner).toMatchObject({ _tag: "Unauthorized" }); + + const wrongOrigin = yield* Effect.flip( + verifyEnvironmentTunnelRecoveryProof({ + ...request, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 5432 }, + }), + ); + expect(wrongOrigin).toMatchObject({ _tag: "Unauthorized" }); + + const wrongAction = yield* Effect.flip( + verifyEnvironmentTunnelRecoveryProof({ + action: "register", + proof, + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: keyPair.publicKey, + tunnelId: "existing-tunnel", + }), + ); + expect(wrongAction).toMatchObject({ _tag: "Unauthorized" }); + }).pipe(Effect.provideService(RelayConfiguration.RelayConfiguration, relaySettings)), + ); + it.effect("registers recovery for an existing tunnel without provisioning it", () => { let recoveryEnabledFor: { readonly userId: string; @@ -451,17 +518,6 @@ describe("relay managed tunnel recovery", () => { it.effect("rejects a recovered tunnel that changes the linked endpoint", () => { let recoveryEnabled = false; const cleaned: Array = []; - const target = { - userId: "user-1", - environmentId: "environment-1", - hostname: "different.example.test", - tunnelId: "replacement-tunnel", - tunnelName: "environment-1-tunnel", - dnsRecordId: "dns-1", - readyAt: "2026-07-28T00:00:00.000Z", - updatedAt: "replacement-generation", - generation: 3, - } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; return Effect.gen(function* () { const error = yield* Effect.flip( @@ -493,11 +549,12 @@ describe("relay managed tunnel recovery", () => { tunnelId: "replacement-tunnel", }, }), - prepareDeprovision: () => Effect.succeed(target), - deprovision: ({ target: captured }) => + prepareDeprovision: () => Effect.die("must keep the active allocation"), + deprovision: () => Effect.die("must keep the active link DNS"), + release: ({ expectedTunnelId }) => Effect.sync(() => { - if (captured?.tunnelId) { - cleaned.push(captured.tunnelId); + if (expectedTunnelId) { + cleaned.push(expectedTunnelId); } return true; }), diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index d537b15a5983..ce568652efd9 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -47,10 +47,15 @@ import { RelayEnvironmentPrincipal, type RelayEnvironmentConnectRequest, type RelayManagedEndpointOrigin, + RelayManagedEndpointRecoveryProofPayload, type RelayDpopAccessTokenScope, RelayInternalError, } from "@t3tools/contracts/relay"; -import { normalizeRelayIssuer } from "@t3tools/shared/relayJwt"; +import { + normalizeRelayIssuer, + RELAY_MANAGED_TUNNEL_RECOVERY_TYP, + verifyRelayJwt, +} from "@t3tools/shared/relayJwt"; import * as DeliveryAttempts from "../agentActivity/DeliveryAttempts.ts"; import * as AgentActivityRows from "../agentActivity/AgentActivityRows.ts"; @@ -93,6 +98,10 @@ const relayCorsPreflightHeaders = { "access-control-max-age": "86400", } as const; +const decodeManagedTunnelRecoveryProof = Schema.decodeUnknownEffect( + RelayManagedEndpointRecoveryProofPayload, +); + const appendRelayCredentialResponseHeaders = HttpEffect.appendPreResponseHandler( (_request, response) => Effect.succeed( @@ -471,6 +480,56 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron }, ); +type EnvironmentTunnelRecoveryProofInput = { + readonly proof: string; + readonly userId: string; + readonly environmentId: string; + readonly environmentPublicKey: string; +} & ( + | { readonly action: "register"; readonly tunnelId: string } + | { readonly action: "recover"; readonly origin: RelayManagedEndpointOrigin } +); + +export const verifyEnvironmentTunnelRecoveryProof = Effect.fn( + "relay.api.server.verifyEnvironmentTunnelRecoveryProof", +)(function* (input: EnvironmentTunnelRecoveryProofInput) { + const config = yield* RelayConfiguration.RelayConfiguration; + const now = yield* DateTime.now; + const verified = yield* verifyRelayJwt({ + publicKey: input.environmentPublicKey, + token: input.proof, + typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP, + issuer: `t3-env:${input.environmentId}`, + audience: normalizeRelayIssuer(config.relayIssuer), + nowEpochSeconds: Math.floor(now.epochMilliseconds / 1_000), + }).pipe( + Effect.flatMap(decodeManagedTunnelRecoveryProof), + Effect.mapError(() => new HttpApiError.Unauthorized({})), + ); + + if ( + verified.environmentId !== input.environmentId || + verified.sub !== input.environmentId || + verified.cloudUserId !== input.userId || + verified.action !== input.action + ) { + return yield* new HttpApiError.Unauthorized({}); + } + if (input.action === "register") { + if (verified.action !== "register" || verified.tunnelId !== input.tunnelId) { + return yield* new HttpApiError.Unauthorized({}); + } + return; + } + if ( + verified.action !== "recover" || + verified.origin.localHttpHost !== input.origin.localHttpHost || + verified.origin.localHttpPort !== input.origin.localHttpPort + ) { + return yield* new HttpApiError.Unauthorized({}); + } +}); + export const registerEnvironmentTunnelRecovery = Effect.fn( "relay.api.server.registerEnvironmentTunnelRecovery", )(function* (input: { @@ -533,10 +592,13 @@ export const recoverEnvironmentTunnelRecord = Effect.fn( recovered.endpoint.wsBaseUrl !== link.endpoint.wsBaseUrl ) { if (recoveredTunnelId !== undefined) { - const owner = { userId: input.userId, environmentId: input.environmentId }; - const target = yield* managedEndpointProvider.prepareDeprovision(owner); - if (target?.tunnelId === recoveredTunnelId) { - yield* managedEndpointProvider.deprovision({ ...owner, target }).pipe( + yield* managedEndpointProvider + .release({ + userId: input.userId, + environmentId: input.environmentId, + expectedTunnelId: recoveredTunnelId, + }) + .pipe( Effect.catch((cause) => Effect.logWarning("Failed to clean up a tunnel with a mismatched endpoint", { userId: input.userId, @@ -546,7 +608,6 @@ export const recoverEnvironmentTunnelRecord = Effect.fn( }), ), ); - } } return yield* new HttpApiError.Unauthorized({}); } @@ -1111,6 +1172,14 @@ export const serverApi = HttpApiBuilder.group( if (principal.environmentId !== params.environmentId) { return yield* new HttpApiError.Unauthorized({}); } + yield* verifyEnvironmentTunnelRecoveryProof({ + action: "register", + proof: payload.proof, + userId: payload.cloudUserId, + environmentId: params.environmentId, + environmentPublicKey: principal.environmentPublicKey, + tunnelId: payload.tunnelId, + }); yield* appendRelayCredentialResponseHeaders; return yield* registerEnvironmentTunnelRecovery({ userId: payload.cloudUserId, @@ -1128,6 +1197,14 @@ export const serverApi = HttpApiBuilder.group( if (principal.environmentId !== params.environmentId) { return yield* new HttpApiError.Unauthorized({}); } + yield* verifyEnvironmentTunnelRecoveryProof({ + action: "recover", + proof: payload.proof, + userId: payload.cloudUserId, + environmentId: params.environmentId, + environmentPublicKey: principal.environmentPublicKey, + origin: payload.origin, + }); yield* appendRelayCredentialResponseHeaders; return yield* recoverEnvironmentTunnelRecord({ userId: payload.cloudUserId, diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index cd1ea10228e1..8db9dd50aac0 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -162,12 +162,14 @@ export type RelayManagedEndpointRuntimeConfig = typeof RelayManagedEndpointRunti export const RelayManagedEndpointRecoveryRequest = Schema.Struct({ cloudUserId: TrimmedNonEmptyString, origin: RelayManagedEndpointOrigin, + proof: TrimmedNonEmptyString, }); export type RelayManagedEndpointRecoveryRequest = typeof RelayManagedEndpointRecoveryRequest.Type; export const RelayManagedEndpointRecoveryRegistrationRequest = Schema.Struct({ cloudUserId: TrimmedNonEmptyString, tunnelId: TrimmedNonEmptyString, + proof: TrimmedNonEmptyString, }); export type RelayManagedEndpointRecoveryRegistrationRequest = typeof RelayManagedEndpointRecoveryRegistrationRequest.Type; @@ -205,6 +207,25 @@ const RelaySignedJwtRegisteredClaims = { exp: Schema.Int, } as const; +export const RelayManagedEndpointRecoveryProofPayload = Schema.Union([ + Schema.Struct({ + ...RelaySignedJwtRegisteredClaims, + action: Schema.Literal("register"), + environmentId: EnvironmentId, + cloudUserId: TrimmedNonEmptyString, + tunnelId: TrimmedNonEmptyString, + }), + Schema.Struct({ + ...RelaySignedJwtRegisteredClaims, + action: Schema.Literal("recover"), + environmentId: EnvironmentId, + cloudUserId: TrimmedNonEmptyString, + origin: RelayManagedEndpointOrigin, + }), +]); +export type RelayManagedEndpointRecoveryProofPayload = + typeof RelayManagedEndpointRecoveryProofPayload.Type; + export const RelayAgentActivityPublishProofPayload = Schema.Struct({ ...RelaySignedJwtRegisteredClaims, environmentId: EnvironmentId, diff --git a/packages/shared/src/relayJwt.ts b/packages/shared/src/relayJwt.ts index 9e848bedfb02..986bd982e622 100644 --- a/packages/shared/src/relayJwt.ts +++ b/packages/shared/src/relayJwt.ts @@ -10,6 +10,7 @@ export const RELAY_HEALTH_REQUEST_TYP = "t3-cloud-health+jwt"; export const RELAY_MINT_RESPONSE_TYP = "t3-env-mint+jwt"; export const RELAY_HEALTH_RESPONSE_TYP = "t3-env-health+jwt"; export const RELAY_ACTIVITY_PUBLISH_TYP = "t3-env-activity+jwt"; +export const RELAY_MANAGED_TUNNEL_RECOVERY_TYP = "t3-env-managed-tunnel-recovery+jwt"; export class RelayJwtError extends Schema.TaggedErrorClass()("RelayJwtError", { operation: Schema.Literals(["sign", "verify"]), From 8c6e41408dc8ac243cb6736a56e377b1268d79e3 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 25 Aug 2026 00:48:03 -0700 Subject: [PATCH 10/14] fix(connect): keep recovery limited to managed links --- apps/server/src/cloud/http.ts | 17 ++++++++++------- apps/server/src/server.test.ts | 9 +-------- .../ManagedEndpointAllocations.test.ts | 2 ++ .../environments/ManagedEndpointAllocations.ts | 1 + infra/relay/src/http/Api.test.ts | 17 +++++++++++++++-- infra/relay/src/http/Api.ts | 8 ++++++-- 6 files changed, 35 insertions(+), 19 deletions(-) diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index a0cbcafa5a4a..2adc803bb4db 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -458,7 +458,7 @@ const cloudLinkProofHandler = Effect.fn("environment.cloud.linkProof")( const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(function* ( dependencies: CloudHttpDependencies, payload: RelayEnvironmentConfigRequest, - options?: { readonly requestRecovery?: boolean; readonly lockHeld?: boolean }, + options?: { readonly lockHeld?: boolean }, ) { const apply = Effect.gen(function* () { yield* validateRelayConfigPayload(payload); @@ -499,16 +499,19 @@ const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(fu CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(endpointRuntimeJson), ); - const registered = yield* registerManagedCloudTunnelRecovery().pipe( + yield* registerManagedCloudTunnelRecovery().pipe( + Effect.retry({ + times: 2, + while: (error) => + error._tag !== "EnvironmentHttpUnauthorizedError" && + error._tag !== "EnvironmentHttpConflictError", + }), Effect.catch((cause) => Effect.logWarning("Failed to register T3 Connect managed tunnel recovery", { cause, - }).pipe(Effect.as(false)), + }), ), ); - if (!registered && options?.requestRecovery !== false) { - yield* dependencies.endpointRuntime.requestRecovery(payload.endpointRuntime); - } } else { yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); } @@ -655,7 +658,7 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi cloudMintPublicKey: link.cloudMintPublicKey, endpointRuntime: link.endpointRuntime, }, - { requestRecovery: false, lockHeld: true }, + { lockHeld: true }, ); }, Effect.catchIf( diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index d728adf74712..f20777cdd60c 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -2612,14 +2612,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }, null, ]); - assert.deepEqual(requestedRecoveryConfigs, [ - { - providerKind: "cloudflare_tunnel", - connectorToken: "connector-token", - tunnelId: "tunnel-id", - tunnelName: "tunnel-name", - }, - ]); + assert.deepEqual(requestedRecoveryConfigs, []); }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts index 843604e5c4eb..5484b3ae519d 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts @@ -103,10 +103,12 @@ describe("ManagedEndpointAllocations", () => { const query = new PgDialect().sqlToQuery(condition as never); expect(query.sql).toContain('"relay_managed_endpoint_allocations"."tunnel_id"'); expect(query.sql).toContain('"relay_environment_links"."environment_public_key"'); + expect(query.sql).toContain('"relay_environment_links"."endpoint_provider_kind"'); expect(query.sql).toContain('"relay_environment_links"."revoked_at" is null'); expect(query.sql).toContain("for update"); expect(query.params).toContain("tunnel-1"); expect(query.params).toContain("public-key"); + expect(query.params).toContain("cloudflare_tunnel"); }).pipe(Effect.provide(layerWithDb(fakeDb))); }); diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index bd764071c308..c26b71d285c1 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -400,6 +400,7 @@ export const make = Effect.gen(function* () { eq(relayEnvironmentLinks.userId, input.userId), eq(relayEnvironmentLinks.environmentId, input.environmentId), eq(relayEnvironmentLinks.environmentPublicKey, input.environmentPublicKey), + eq(relayEnvironmentLinks.endpointProviderKind, "cloudflare_tunnel"), isNull(relayEnvironmentLinks.revokedAt), ), ) diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index 4f1c15ff6bcf..ca0a321b4ded 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -571,7 +571,19 @@ describe("relay managed tunnel recovery", () => { ); }); - it.effect("removes a recovered tunnel when its link disappears before registration", () => { + it.effect.each([ + { state: "removed", currentLink: null }, + { + state: "publish-only", + currentLink: { + ...linkedEnvironmentRecord, + endpoint: { + ...linkedEnvironmentRecord.endpoint, + providerKind: "manual" as const, + }, + }, + }, + ])("removes a recovered tunnel when its link becomes $state", ({ currentLink }) => { let lookups = 0; const cleaned: Array = []; const target = { @@ -601,7 +613,8 @@ describe("relay managed tunnel recovery", () => { Effect.provide( Layer.merge( relayUnlinkTestLayer({ - getForUser: () => Effect.sync(() => (++lookups === 1 ? linkedEnvironmentRecord : null)), + getForUser: () => + Effect.sync(() => (++lookups === 1 ? linkedEnvironmentRecord : currentLink)), provision: () => Effect.succeed({ endpoint: linkedEnvironmentRecord.endpoint, diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index ce568652efd9..df1d1661279b 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -621,10 +621,14 @@ export const recoverEnvironmentTunnelRecord = Effect.fn( if (!enabled) { const owner = { userId: input.userId, environmentId: input.environmentId }; const target = yield* managedEndpointProvider.prepareDeprovision(owner); - if (target !== null && (yield* links.getForUser(input)) === null) { + const currentLink = target === null ? null : yield* links.getForUser(input); + if ( + target !== null && + (currentLink === null || currentLink.endpoint.providerKind !== "cloudflare_tunnel") + ) { yield* managedEndpointProvider.deprovision({ ...owner, target }).pipe( Effect.catch((cause) => - Effect.logWarning("Failed to clean up a tunnel after its link was removed", { + Effect.logWarning("Failed to clean up a tunnel after its managed link was removed", { userId: input.userId, environmentId: input.environmentId, cause, From 6a0992d89db060c96f101e91e361115f995010e4 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Fri, 28 Aug 2026 02:02:56 -0700 Subject: [PATCH 11/14] fix(relay): harden managed tunnel recovery --- .github/workflows/deploy-relay.yml | 1 + apps/server/src/cloud/CliState.test.ts | 2 + apps/server/src/cloud/CliState.ts | 2 + .../src/cloud/ManagedEndpointRuntime.test.ts | 4 +- .../src/cloud/ManagedEndpointRuntime.ts | 47 +-- apps/server/src/cloud/config.ts | 19 +- apps/server/src/cloud/http.test.ts | 137 ++++++- apps/server/src/cloud/http.ts | 355 +++++++++++++++--- .../src/cloud/managedTunnelStartup.test.ts | 91 +++++ apps/server/src/cloud/managedTunnelStartup.ts | 51 +++ apps/server/src/server.test.ts | 192 ++++++++-- apps/server/src/server.ts | 72 +++- docs/internals/t3-connect.md | 40 +- docs/operations/release.md | 54 +++ docs/user/remote-access.md | 10 +- infra/relay/.env.example | 4 + .../migration.sql | 1 + .../snapshot.json | 15 +- infra/relay/src/Config.test.ts | 35 ++ infra/relay/src/Config.ts | 17 + .../environments/EnvironmentConnector.test.ts | 2 + .../environments/EnvironmentLinker.test.ts | 1 + .../ManagedEndpointAllocations.test.ts | 2 + .../ManagedEndpointAllocations.ts | 9 +- .../ManagedEndpointProvider.test.ts | 205 ++++++++++ .../environments/ManagedEndpointProvider.ts | 246 +++++++++--- .../ManagedEndpointReaper.test.ts | 228 ++++++++++- .../src/environments/ManagedEndpointReaper.ts | 166 +++++--- infra/relay/src/http/Api.test.ts | 89 ++++- infra/relay/src/http/Api.ts | 86 +++-- infra/relay/src/persistence/schema.ts | 2 + infra/relay/src/worker.ts | 4 +- packages/contracts/src/relay.ts | 10 +- 33 files changed, 1900 insertions(+), 299 deletions(-) create mode 100644 apps/server/src/cloud/managedTunnelStartup.test.ts create mode 100644 apps/server/src/cloud/managedTunnelStartup.ts create mode 100644 infra/relay/src/Config.test.ts diff --git a/.github/workflows/deploy-relay.yml b/.github/workflows/deploy-relay.yml index f652844a54f3..59a53c9cd384 100644 --- a/.github/workflows/deploy-relay.yml +++ b/.github/workflows/deploy-relay.yml @@ -28,6 +28,7 @@ jobs: RELAY_DOMAIN: ${{ vars.RELAY_DOMAIN }} RELAY_API_ZONE_NAME: ${{ vars.RELAY_API_ZONE_NAME }} RELAY_TUNNEL_ZONE_NAME: ${{ vars.RELAY_TUNNEL_ZONE_NAME }} + RELAY_TUNNEL_CLEANUP_MODE: ${{ vars.RELAY_TUNNEL_CLEANUP_MODE }} CLERK_PUBLISHABLE_KEY: ${{ vars.CLERK_PUBLISHABLE_KEY }} CLERK_JWT_AUDIENCE: ${{ vars.CLERK_JWT_AUDIENCE }} APNS_ENVIRONMENT: ${{ vars.APNS_ENVIRONMENT }} diff --git a/apps/server/src/cloud/CliState.test.ts b/apps/server/src/cloud/CliState.test.ts index 39f904b47b89..d59b89875e79 100644 --- a/apps/server/src/cloud/CliState.test.ts +++ b/apps/server/src/cloud/CliState.test.ts @@ -8,6 +8,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import { ServerConfig } from "../config.ts"; import * as CliState from "./CliState.ts"; import { + CLOUD_ENDPOINT_CONFIRMED_ORIGIN, CLOUD_ENDPOINT_RUNTIME_CONFIG, CLOUD_LINKED_USER_ID, CLOUD_MINT_PUBLIC_KEY, @@ -24,6 +25,7 @@ const persistedCloudLinkSecrets = [ RELAY_ENVIRONMENT_CREDENTIAL_SECRET, CLOUD_MINT_PUBLIC_KEY, CLOUD_ENDPOINT_RUNTIME_CONFIG, + CLOUD_ENDPOINT_CONFIRMED_ORIGIN, PUBLISH_AGENT_ACTIVITY_SECRET, ] as const; diff --git a/apps/server/src/cloud/CliState.ts b/apps/server/src/cloud/CliState.ts index 9af9a032f856..dc77609ccc92 100644 --- a/apps/server/src/cloud/CliState.ts +++ b/apps/server/src/cloud/CliState.ts @@ -3,6 +3,7 @@ import * as Option from "effect/Option"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import { + CLOUD_ENDPOINT_CONFIRMED_ORIGIN, CLOUD_ENDPOINT_RUNTIME_CONFIG, CLOUD_LINKED_USER_ID, CLOUD_MINT_PUBLIC_KEY, @@ -67,6 +68,7 @@ export const clearPersistedCloudLink = Effect.gen(function* () { secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), secrets.remove(CLOUD_MINT_PUBLIC_KEY), secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG), + secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN), secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET), ], { concurrency: "unbounded" }, diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index cb2441095f5d..000285a5bcb5 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -333,8 +333,8 @@ describe("CloudManagedEndpointRuntime", () => { expect(spawned.map((command) => command.command)).toEqual(["cloudflared", "cloudflared"]); expect(spawned.map((command) => command.args)).toEqual([ - ["tunnel", "run"], - ["tunnel", "run"], + ["tunnel", "--no-autoupdate", "--loglevel", "info", "--output", "default", "run"], + ["tunnel", "--no-autoupdate", "--loglevel", "info", "--output", "default", "run"], ]); expect(spawned.map((command) => command.options.env?.TUNNEL_TOKEN)).toEqual([ "token-1", diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index 7695327da63b..e2bf9a113672 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -4,7 +4,6 @@ import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; -import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Result from "effect/Result"; @@ -14,22 +13,6 @@ import * as Stream from "effect/Stream"; import * as ChildProcess from "effect/unstable/process/ChildProcess"; import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; -import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; -import { CLOUD_ENDPOINT_RUNTIME_CONFIG, decodeRuntimeConfig } from "./config.ts"; - -function bytesToString(bytes: Uint8Array): string { - return new TextDecoder().decode(bytes); -} - -const readRuntimeConfig = Effect.gen(function* () { - const secrets = yield* ServerSecretStore.ServerSecretStore; - const bytes = yield* secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); - if (Option.isNone(bytes)) { - return null; - } - return Option.getOrNull(decodeRuntimeConfig(bytesToString(bytes.value))); -}); - export type CloudManagedEndpointRuntimeStatus = | { readonly status: "disabled"; @@ -275,16 +258,20 @@ export const make = Effect.gen(function* () { const connectorScope = yield* Scope.make("sequential"); const child = yield* spawner .spawn( - ChildProcess.make(executable.executablePath, ["tunnel", "run"], { - detached: false, - env: { - ...process.env, - TUNNEL_TOKEN: config.connectorToken, + ChildProcess.make( + executable.executablePath, + ["tunnel", "--no-autoupdate", "--loglevel", "info", "--output", "default", "run"], + { + detached: false, + env: { + ...process.env, + TUNNEL_TOKEN: config.connectorToken, + }, + shell: false, + stderr: "pipe", + stdout: "pipe", }, - shell: false, - stderr: "pipe", - stdout: "pipe", - }), + ), ) .pipe( Effect.provideService(Scope.Scope, connectorScope), @@ -361,14 +348,6 @@ export const make = Effect.gen(function* () { withLinkStateLock: linkStateSemaphore.withPermits(1), }); - const initialConfig = yield* readRuntimeConfig.pipe( - Effect.catch((cause) => - Effect.logWarning("Failed to read managed endpoint runtime config", { cause }).pipe( - Effect.as(null), - ), - ), - ); - yield* runtime.applyConfig(initialConfig); yield* Effect.addFinalizer(() => runtime.applyConfig(null)); return runtime; }); diff --git a/apps/server/src/cloud/config.ts b/apps/server/src/cloud/config.ts index 2eff693f61e6..9b1b281ba2da 100644 --- a/apps/server/src/cloud/config.ts +++ b/apps/server/src/cloud/config.ts @@ -1,4 +1,7 @@ -import { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay"; +import { + RelayManagedEndpointOrigin, + RelayManagedEndpointRuntimeConfig, +} from "@t3tools/contracts/relay"; import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; @@ -7,6 +10,7 @@ import type * as ServerSecretStore from "../auth/ServerSecretStore.ts"; export const CLOUD_MINT_PUBLIC_KEY = "cloud-mint-ed25519-public-key"; export const CLOUD_ENDPOINT_RUNTIME_CONFIG = "cloud-endpoint-runtime-config"; +export const CLOUD_ENDPOINT_CONFIRMED_ORIGIN = "cloud-endpoint-confirmed-origin"; export const CLOUD_LINKED_USER_ID = "cloud-linked-user-id"; export const RELAY_URL_SECRET = "cloud-relay-url"; export const RELAY_ISSUER_SECRET = "cloud-relay-issuer"; @@ -21,6 +25,19 @@ export const decodeRuntimeConfig = Schema.decodeUnknownOption( Schema.fromJsonString(RelayManagedEndpointRuntimeConfig), ); +export const ManagedEndpointConfirmedOrigin = Schema.Struct({ + config: RelayManagedEndpointRuntimeConfig, + origin: RelayManagedEndpointOrigin, +}); + +export const encodeConfirmedOriginJson = Schema.encodeEffect( + Schema.fromJsonString(ManagedEndpointConfirmedOrigin), +); + +export const decodeConfirmedOrigin = Schema.decodeUnknownOption( + Schema.fromJsonString(ManagedEndpointConfirmedOrigin), +); + export function isAgentActivityPublishingEnabledValue(value: string | null): boolean { return value === "true"; } diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index 6e500aec5816..4c260e0554d7 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -36,6 +36,7 @@ import { type RelayLinkProofRequest, } from "@t3tools/contracts/relay"; import { + CLOUD_ENDPOINT_CONFIRMED_ORIGIN, CLOUD_ENDPOINT_RUNTIME_CONFIG, CLOUD_LINKED_USER_ID, decodeRuntimeConfig, @@ -48,9 +49,11 @@ import { linkProofScopes, pendingServiceUpdateExists, reconcileDesiredCloudLink, + reconcileDesiredCloudLinkIfStillDesired, recoverManagedCloudTunnel, registerManagedCloudTunnelRecovery, releaseManagedTunnelOnShutdown, + startManagedCloudTunnelIfOriginConfirmed, } from "./http.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; import { traceAuthenticatedRelayRequest, traceRelayRequest } from "./traceRelayRequest.ts"; @@ -379,10 +382,27 @@ describe("releaseManagedTunnelOnShutdown", () => { // The persisted state of a CLI-managed link whose tunnel is releasable. const managedLinkSecrets = [ [CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"], + [CLOUD_ENDPOINT_CONFIRMED_ORIGIN, "confirmed-origin"], [RELAY_URL_SECRET, "https://relay.example.test"], [CLOUD_CLI_DESIRED_LINK_SECRET, "managed"], ] as const; + it.effect("does not recreate a link that was unlinked while startup registration retried", () => { + const { store, values } = makeMemorySecretStore(managedLinkSecrets); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + // Registration started while this marker existed. Unlink removes it + // before startup receives the relay's final not_linked response. + values.delete(CLOUD_CLI_DESIRED_LINK_SECRET); + + expect(yield* reconcileDesiredCloudLinkIfStillDesired("http://127.0.0.1:3773")).toBeNull(); + expect(requests).toEqual([]); + expect(applyConfigCalls).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + it.effect("stops the connector, releases the relay tunnel, and drops the dead token", () => { const { store, values } = makeMemorySecretStore(managedLinkSecrets); const applyConfigCalls: Array = []; @@ -401,6 +421,7 @@ describe("releaseManagedTunnelOnShutdown", () => { ); expect(request.headers.authorization).toBe("Bearer cli-access-token"); expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false); + expect(values.has(CLOUD_ENDPOINT_CONFIRMED_ORIGIN)).toBe(false); }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); }); @@ -611,7 +632,7 @@ describe("releaseManagedTunnelOnShutdown", () => { ); }); - it.effect("registers an existing tunnel without provisioning or restarting it", () => { + it.effect("registers an existing tunnel and starts the confirmed connector", () => { const { store } = makeMemorySecretStore([ [ CLOUD_ENDPOINT_RUNTIME_CONFIG, @@ -625,7 +646,9 @@ describe("releaseManagedTunnelOnShutdown", () => { const requests: Array = []; return Effect.gen(function* () { - expect(yield* registerManagedCloudTunnelRecovery()).toBe(true); + expect(yield* registerManagedCloudTunnelRecovery("http://127.0.0.1:3773")).toMatchObject({ + status: "ready", + }); expect(requests).toHaveLength(1); expect(requests[0]?.method).toBe("POST"); expect(requests[0]?.url).toBe( @@ -640,12 +663,118 @@ describe("releaseManagedTunnelOnShutdown", () => { ).toMatchObject({ cloudUserId: "user-123", tunnelId: "existing-tunnel", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, }); } + expect(applyConfigCalls).toHaveLength(1); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respond: () => Response.json({ status: "ready" }), + }), + ); + }); + + it.effect( + "starts a connector with a marker for the current origin without contacting relay", + () => { + const configJson = + '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}'; + const config = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "existing-token", + tunnelId: "existing-tunnel", + }; + const { store } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, configJson], + [ + CLOUD_ENDPOINT_CONFIRMED_ORIGIN, + `{"config":${configJson},"origin":{"localHttpHost":"127.0.0.1","localHttpPort":3773}}`, + ], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* startManagedCloudTunnelIfOriginConfirmed("http://127.0.0.1:3773")).toBe(true); + expect(applyConfigCalls).toEqual([config]); + expect(requests).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }, + ); + + it.effect.each([ + { name: "missing", marker: undefined, origin: "http://127.0.0.1:3773" }, + { + name: "stale", + marker: + '{"config":{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"},"origin":{"localHttpHost":"127.0.0.1","localHttpPort":3773}}', + origin: "http://127.0.0.1:4884", + }, + ])("does not start a connector with a $name origin marker", ({ marker, origin }) => { + const entries: Array = [ + [ + CLOUD_ENDPOINT_RUNTIME_CONFIG, + '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}', + ], + ]; + if (marker !== undefined) entries.push([CLOUD_ENDPOINT_CONFIRMED_ORIGIN, marker]); + const { store } = makeMemorySecretStore(entries); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* startManagedCloudTunnelIfOriginConfirmed(origin)).toBe(false); expect(applyConfigCalls).toEqual([]); + expect(requests).toEqual([]); }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); }); + it.effect.each(["replaced", "removed"] as const)( + "does not activate a tunnel when its runtime config is %s during registration", + (mutation) => { + const originalConfig = + '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}'; + const { store, values } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, originalConfig], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* registerManagedCloudTunnelRecovery("http://127.0.0.1:3773")).toEqual({ + status: "superseded", + }); + expect(applyConfigCalls).toEqual([]); + expect(values.has(CLOUD_ENDPOINT_CONFIRMED_ORIGIN)).toBe(false); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respond: () => { + if (mutation === "replaced") { + values.set( + CLOUD_ENDPOINT_RUNTIME_CONFIG, + new TextEncoder().encode( + '{"providerKind":"cloudflare_tunnel","connectorToken":"fresh-token","tunnelId":"fresh-tunnel"}', + ), + ); + } else { + values.delete(CLOUD_ENDPOINT_RUNTIME_CONFIG); + } + return Response.json({ status: "ready" }); + }, + }), + ); + }, + ); + it.effect("does not register recovery without a recorded tunnel ID", () => { const { store } = makeMemorySecretStore([ [ @@ -660,7 +789,9 @@ describe("releaseManagedTunnelOnShutdown", () => { const requests: Array = []; return Effect.gen(function* () { - expect(yield* registerManagedCloudTunnelRecovery()).toBe(false); + expect(yield* registerManagedCloudTunnelRecovery("http://127.0.0.1:3773")).toEqual({ + status: "not_linked", + }); expect(requests).toEqual([]); expect(applyConfigCalls).toEqual([]); }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index 2adc803bb4db..b6e8ec58c865 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -29,6 +29,7 @@ import { RelayLinkProofRequest, RelayManagedEndpointOrigin, RelayManagedEndpointRecoveryProofPayload, + RelayManagedEndpointRecoveryRegistrationResponse, RelayManagedEndpointRecoveryResponse, type RelayManagedEndpointRuntimeConfig, RelayOkResponse, @@ -54,10 +55,12 @@ import * as FileSystem from "effect/FileSystem"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import * as Schedule from "effect/Schedule"; import * as HttpEffect from "effect/unstable/http/HttpEffect"; import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder"; +import * as HttpServer from "effect/unstable/http/HttpServer"; import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; @@ -72,10 +75,13 @@ import { } from "./serviceProtocol.ts"; import { CLOUD_ENDPOINT_RUNTIME_CONFIG, + CLOUD_ENDPOINT_CONFIRMED_ORIGIN, + decodeConfirmedOrigin, CLOUD_LINKED_USER_ID, CLOUD_MINT_PUBLIC_KEY, decodeRuntimeConfig, encodeEndpointRuntimeConfigJson, + encodeConfirmedOriginJson, PUBLISH_AGENT_ACTIVITY_SECRET, RELAY_ENVIRONMENT_CREDENTIAL_SECRET, RELAY_ISSUER_SECRET, @@ -455,10 +461,137 @@ const cloudLinkProofHandler = Effect.fn("environment.cloud.linkProof")( ), ); +function managedEndpointOriginFromLocalUrl(localOrigin: string): RelayManagedEndpointOrigin { + const localUrl = new URL(localOrigin); + if (localUrl.origin !== localOrigin) { + throw new Error("Invalid local origin"); + } + return { + localHttpHost: localUrl.hostname, + localHttpPort: endpointRequestPort(localUrl), + }; +} + +function managedEndpointRuntimeConfigsMatch( + left: RelayManagedEndpointRuntimeConfig, + right: RelayManagedEndpointRuntimeConfig, +): boolean { + return ( + left.providerKind === right.providerKind && + left.connectorToken === right.connectorToken && + left.tunnelId === right.tunnelId && + left.tunnelName === right.tunnelName + ); +} + +const activateManagedTunnel = Effect.fn("environment.cloud.activateManagedTunnel")(function* ( + dependencies: CloudHttpDependencies, + input: { + readonly config: RelayManagedEndpointRuntimeConfig; + readonly configJson: string; + readonly origin: RelayManagedEndpointOrigin; + }, +) { + return yield* dependencies.endpointRuntime.withLinkStateLock( + Effect.gen(function* () { + const currentConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); + if (Option.isNone(currentConfig) || bytesToString(currentConfig.value) !== input.configJson) { + return null; + } + const status = yield* dependencies.endpointRuntime.applyConfig(input.config); + if (status.status !== "running") { + return yield* new EnvironmentCloudEndpointUnavailableError({ + message: "Managed endpoint runtime could not be started.", + endpointRuntimeStatus: status, + }); + } + const marker = yield* encodeConfirmedOriginJson({ + config: input.config, + origin: input.origin, + }); + yield* dependencies.secrets.set(CLOUD_ENDPOINT_CONFIRMED_ORIGIN, stringToBytes(marker)); + return status; + }), + ); +}); + +const activateManagedTunnelWithRetry = ( + dependencies: CloudHttpDependencies, + input: { + readonly config: RelayManagedEndpointRuntimeConfig; + readonly configJson: string; + readonly origin: RelayManagedEndpointOrigin; + }, + retryRuntimeFailures: boolean, +) => { + const activate = activateManagedTunnel(dependencies, input); + return retryRuntimeFailures + ? activate.pipe( + Effect.retry({ + while: (error) => + error._tag === "EnvironmentCloudEndpointUnavailableError" && + ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus( + error.endpointRuntimeStatus, + ), + schedule: Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + Schedule.jittered, + ), + }), + ) + : activate; +}; + +export const startManagedCloudTunnelIfOriginConfirmed = Effect.fn( + "environment.cloud.startManagedCloudTunnelIfOriginConfirmed", +)(function* (localOrigin: string) { + const dependencies = yield* cloudHttpDependencies; + const origin = yield* Effect.try({ + try: () => managedEndpointOriginFromLocalUrl(localOrigin), + catch: () => + new EnvironmentHttpBadRequestError({ + message: "Could not resolve local environment origin.", + }), + }); + return yield* dependencies.endpointRuntime.withLinkStateLock( + Effect.gen(function* () { + const [runtimeBytes, markerBytes] = yield* Effect.all([ + dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), + dependencies.secrets.get(CLOUD_ENDPOINT_CONFIRMED_ORIGIN), + ]); + if (Option.isNone(runtimeBytes) || Option.isNone(markerBytes)) return false; + const config = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeBytes.value))); + const marker = Option.getOrNull(decodeConfirmedOrigin(bytesToString(markerBytes.value))); + if ( + config === null || + marker === null || + !managedEndpointRuntimeConfigsMatch(marker.config, config) || + marker.origin.localHttpHost !== origin.localHttpHost || + marker.origin.localHttpPort !== origin.localHttpPort + ) { + return false; + } + const status = yield* dependencies.endpointRuntime.applyConfig(config); + if (status.status !== "running") { + return yield* new EnvironmentCloudEndpointUnavailableError({ + message: "Managed endpoint runtime could not be started.", + endpointRuntimeStatus: status, + }); + } + return true; + }), + ); +}); + const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(function* ( dependencies: CloudHttpDependencies, payload: RelayEnvironmentConfigRequest, - options?: { readonly lockHeld?: boolean }, + options?: { + readonly lockHeld?: boolean; + readonly confirmedOrigin?: RelayManagedEndpointOrigin; + }, ) { const apply = Effect.gen(function* () { yield* validateRelayConfigPayload(payload); @@ -467,17 +600,20 @@ const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(fu cloudUserId: payload.cloudUserId, }); yield* validateCloudMintPublicKey(payload.cloudMintPublicKey); - const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig( - payload.endpointRuntime, - ); - const ok = - endpointRuntimeStatus.status === "disabled" || endpointRuntimeStatus.status === "running"; - if (!ok) { + if ( + payload.endpointRuntime !== null && + payload.endpointRuntime.providerKind !== "cloudflare_tunnel" + ) { + const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig( + payload.endpointRuntime, + ); return yield* new EnvironmentCloudEndpointUnavailableError({ message: "Managed endpoint runtime could not be started.", endpointRuntimeStatus, }); } + yield* dependencies.endpointRuntime.applyConfig(null); + yield* dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN); yield* dependencies.secrets.set(RELAY_URL_SECRET, stringToBytes(payload.relayUrl)); yield* dependencies.secrets.set( @@ -499,23 +635,30 @@ const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(fu CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(endpointRuntimeJson), ); - yield* registerManagedCloudTunnelRecovery().pipe( - Effect.retry({ - times: 2, - while: (error) => - error._tag !== "EnvironmentHttpUnauthorizedError" && - error._tag !== "EnvironmentHttpConflictError", - }), - Effect.catch((cause) => - Effect.logWarning("Failed to register T3 Connect managed tunnel recovery", { - cause, - }), - ), - ); } else { yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); } - return { ok, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; + if (payload.endpointRuntime === null || options?.confirmedOrigin === undefined) { + return { + ok: true, + endpointRuntimeStatus: { status: "disabled" }, + } satisfies EnvironmentCloudRelayConfigResult; + } + const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig( + payload.endpointRuntime, + ); + if (endpointRuntimeStatus.status !== "running") { + return yield* new EnvironmentCloudEndpointUnavailableError({ + message: "Managed endpoint runtime could not be started.", + endpointRuntimeStatus, + }); + } + const marker = yield* encodeConfirmedOriginJson({ + config: payload.endpointRuntime, + origin: options.confirmedOrigin, + }); + yield* dependencies.secrets.set(CLOUD_ENDPOINT_CONFIRMED_ORIGIN, stringToBytes(marker)); + return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; }); return yield* options?.lockHeld ? apply : dependencies.endpointRuntime.withLinkStateLock(apply); }); @@ -523,7 +666,43 @@ const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(fu const cloudRelayConfigHandler = Effect.fn("environment.cloud.relayConfig")( function* (dependencies: CloudHttpDependencies, payload: RelayEnvironmentConfigRequest) { yield* requireEnvironmentScope(AuthRelayWriteScope); - return yield* applyCloudRelayConfig(dependencies, payload); + const result = yield* applyCloudRelayConfig(dependencies, payload); + if (payload.endpointRuntime?.providerKind === "cloudflare_tunnel") { + const server = yield* HttpServer.HttpServer; + const address = server.address; + if (typeof address === "string" || !("port" in address)) { + return yield* new EnvironmentHttpInternalServerError({ + message: "Could not resolve the local server origin.", + }); + } + const registration = yield* registerManagedCloudTunnelRecovery( + `http://127.0.0.1:${address.port}`, + ).pipe( + Effect.retry({ + times: 2, + while: (error) => + error._tag !== "EnvironmentCloudEndpointUnavailableError" && + error._tag !== "EnvironmentHttpUnauthorizedError" && + error._tag !== "EnvironmentHttpConflictError", + }), + ); + if (registration.status === "superseded") { + return yield* new EnvironmentHttpConflictError({ + message: "The managed tunnel configuration changed during registration.", + }); + } + if (registration.status !== "ready") { + return yield* new EnvironmentCloudEndpointUnavailableError({ + message: "Managed endpoint origin could not be confirmed.", + endpointRuntimeStatus: { status: "disabled" }, + }); + } + return { + ok: true, + endpointRuntimeStatus: registration.endpointRuntimeStatus, + } satisfies EnvironmentCloudRelayConfigResult; + } + return result; }, Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) => failEnvironmentCloudInternalError(error.message)(error), @@ -536,6 +715,10 @@ const cloudRelayConfigHandler = Effect.fn("environment.cloud.relayConfig")( "SchemaError", failEnvironmentCloudInternalError("Could not persist environment relay configuration."), ), + Effect.catchTag( + "PlatformError", + failEnvironmentCloudInternalError("Could not register the managed endpoint origin."), + ), ); const relayClientRequest = ( @@ -567,6 +750,7 @@ const relayClientRequest = ( }), ), Effect.flatMap(HttpClientResponse.schemaBodyJson(input.schema)), + Effect.timeout("10 seconds"), Effect.mapError((cause) => cause._tag === "EnvironmentHttpUnauthorizedError" || cause._tag === "EnvironmentHttpConflictError" @@ -658,7 +842,13 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi cloudMintPublicKey: link.cloudMintPublicKey, endpointRuntime: link.endpointRuntime, }, - { lockHeld: true }, + { + lockHeld: true, + confirmedOrigin: { + localHttpHost: localUrl.hostname, + localHttpPort: endpointRequestPort(localUrl), + }, + }, ); }, Effect.catchIf( @@ -683,12 +873,32 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD }, ); +export const reconcileDesiredCloudLinkIfStillDesired = Effect.fn( + "environment.cloud.reconcileDesiredLinkIfStillDesired", +)(function* (localOrigin: string) { + const dependencies = yield* cloudHttpDependencies; + return yield* dependencies.endpointRuntime.withLinkStateLock( + Effect.gen(function* () { + if (!(yield* readCliDesiredCloudLink)) { + return null; + } + const mode = yield* readCliDesiredLinkMode; + yield* reconcileDesiredCloudLinkWith(dependencies, localOrigin); + return mode; + }), + ); +}); + type ManagedTunnelRecoveryProofInput = { readonly environmentId: RelayManagedEndpointRecoveryProofPayload["environmentId"]; readonly cloudUserId: string; readonly relayUrl: string; } & ( - | { readonly action: "register"; readonly tunnelId: string } + | { + readonly action: "register"; + readonly tunnelId: string; + readonly origin: RelayManagedEndpointOrigin; + } | { readonly action: "recover"; readonly origin: RelayManagedEndpointOrigin } ); @@ -713,7 +923,12 @@ const makeManagedTunnelRecoveryProof = Effect.fn( }; const payload = input.action === "register" - ? { ...claims, action: "register" as const, tunnelId: input.tunnelId } + ? { + ...claims, + action: "register" as const, + tunnelId: input.tunnelId, + origin: input.origin, + } : { ...claims, action: "recover" as const, origin: input.origin }; return yield* signRelayJwt({ @@ -732,7 +947,7 @@ const makeManagedTunnelRecoveryProof = Effect.fn( export const registerManagedCloudTunnelRecovery = Effect.fn( "environment.cloud.registerManagedCloudTunnelRecovery", -)(function* () { +)(function* (localOrigin: string, options?: { readonly retryRuntimeFailures?: boolean }) { const dependencies = yield* cloudHttpDependencies; const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([ dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), @@ -746,14 +961,21 @@ export const registerManagedCloudTunnelRecovery = Effect.fn( Option.isNone(cloudUserId) || Option.isNone(environmentCredential) ) { - return false; + return { status: "not_linked" as const }; } const config = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeConfig.value))); if (config?.providerKind !== "cloudflare_tunnel" || config.tunnelId === undefined) { - return false; + return { status: "not_linked" as const }; } + const origin = yield* Effect.try({ + try: () => managedEndpointOriginFromLocalUrl(localOrigin), + catch: () => + new EnvironmentHttpBadRequestError({ + message: "Could not resolve local environment origin.", + }), + }); const environmentId = yield* dependencies.environment.getEnvironmentId; const relayUrlValue = bytesToString(relayUrl.value); const cloudUserIdValue = bytesToString(cloudUserId.value); @@ -763,6 +985,7 @@ export const registerManagedCloudTunnelRecovery = Effect.fn( cloudUserId: cloudUserIdValue, relayUrl: relayUrlValue, tunnelId: config.tunnelId, + origin, }); const registered = yield* relayClientRequest(dependencies, { url: `${relayUrlValue}/v1/environments/${encodeURIComponent(environmentId)}/tunnel/recovery`, @@ -770,15 +993,34 @@ export const registerManagedCloudTunnelRecovery = Effect.fn( payload: { cloudUserId: cloudUserIdValue, tunnelId: config.tunnelId, + origin, proof, }, - schema: RelayOkResponse, + schema: RelayManagedEndpointRecoveryRegistrationResponse, }); - return registered.ok; + if (registered.status === "recovery_required") { + return { status: registered.status, config }; + } + const endpointRuntimeStatus = yield* activateManagedTunnelWithRetry( + dependencies, + { + config, + configJson: bytesToString(runtimeConfig.value), + origin, + }, + options?.retryRuntimeFailures === true, + ); + return endpointRuntimeStatus === null + ? { status: "superseded" as const } + : { status: "ready" as const, endpointRuntimeStatus }; }); export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverManagedCloudTunnel")( - function* (localOrigin: string, expectedConfig?: RelayManagedEndpointRuntimeConfig) { + function* ( + localOrigin: string, + expectedConfig?: RelayManagedEndpointRuntimeConfig, + options?: { readonly retryRuntimeFailures?: boolean }, + ) { const dependencies = yield* cloudHttpDependencies; const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([ dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), @@ -850,7 +1092,15 @@ export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverMan }); } - return yield* dependencies.endpointRuntime.withLinkStateLock( + const encoded = yield* encodeEndpointRuntimeConfigJson(recovered.endpointRuntime).pipe( + Effect.mapError( + () => + new EnvironmentHttpInternalServerError({ + message: "Could not persist the recovered managed tunnel configuration.", + }), + ), + ); + const stored = yield* dependencies.endpointRuntime.withLinkStateLock( Effect.gen(function* () { const currentConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); if ( @@ -859,26 +1109,22 @@ export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverMan ) { return false; } - - const status = yield* dependencies.endpointRuntime.applyConfig(recovered.endpointRuntime); - if (status.status !== "running") { - return yield* new EnvironmentCloudEndpointUnavailableError({ - message: "Managed endpoint runtime could not be started.", - endpointRuntimeStatus: status, - }); - } - const encoded = yield* encodeEndpointRuntimeConfigJson(recovered.endpointRuntime).pipe( - Effect.mapError( - () => - new EnvironmentHttpInternalServerError({ - message: "Could not persist the recovered managed tunnel configuration.", - }), - ), - ); yield* dependencies.secrets.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(encoded)); + yield* dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN); return true; }), ); + if (!stored) return false; + const status = yield* activateManagedTunnelWithRetry( + dependencies, + { + config: recovered.endpointRuntime, + configJson: encoded, + origin, + }, + options?.retryRuntimeFailures === true, + ); + return status !== null; }, ); @@ -928,11 +1174,10 @@ export const releaseManagedTunnelOnShutdown = Effect.fn( if (Option.isNone(runtimeConfig)) { return false; } - // Only CLI-desired managed links release on shutdown, because the startup - // reconcile that provisions the replacement tunnel only runs for them. A - // link installed by a web/mobile client comes back after a restart by - // reapplying the stored connector token — it has no boot-time re-provision - // path — so its tunnel must survive the restart. (Unlink still deletes it.) + // Only CLI-desired managed links release eagerly because this request uses + // CLI authorization. Web/mobile links register startup recovery with their + // environment credential, and the relay reaper removes them after they are + // down for the configured grace period. Unlink still deletes either kind. if (!(yield* readCliDesiredCloudLink) || (yield* readCliDesiredLinkMode) !== "managed") { return false; } @@ -989,6 +1234,7 @@ export const releaseManagedTunnelOnShutdown = Effect.fn( bytesToString(storedConfig.value) === bytesToString(runtimeConfig.value) ) { yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); + yield* dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN); } return true; }); @@ -1046,9 +1292,10 @@ const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")( dependencies.secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), dependencies.secrets.remove(CLOUD_MINT_PUBLIC_KEY), dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG), + dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN), dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET), ], - { concurrency: 7 }, + { concurrency: 8 }, ); yield* setCliDesiredCloudLink(false); return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; diff --git a/apps/server/src/cloud/managedTunnelStartup.test.ts b/apps/server/src/cloud/managedTunnelStartup.test.ts new file mode 100644 index 000000000000..223c5defe0ef --- /dev/null +++ b/apps/server/src/cloud/managedTunnelStartup.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as TestClock from "effect/testing/TestClock"; + +import { + managedTunnelStartupAction, + retryManagedTunnelRegistration, +} from "./managedTunnelStartup.ts"; + +describe("managedTunnelStartupAction", () => { + const config = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "connector-token", + tunnelId: "tunnel-1", + }; + + it("requests tunnel recovery only when the relay proves it is needed", () => { + expect( + managedTunnelStartupAction({ + wantsCliLink: true, + registration: { status: "recovery_required", config }, + }), + ).toEqual({ action: "request_recovery", config }); + }); + + it("creates a desired CLI link only when no local managed link exists", () => { + expect( + managedTunnelStartupAction({ + wantsCliLink: true, + registration: { status: "not_linked" }, + }), + ).toEqual({ action: "reconcile_link" }); + }); + + it.each(["ready", "unavailable"] as const)( + "does not provision after a %s registration result", + (status) => { + expect( + managedTunnelStartupAction({ + wantsCliLink: true, + registration: { status }, + }), + ).toEqual({ action: "none" }); + }, + ); +}); + +describe("retryManagedTunnelRegistration", () => { + it.effect("waits for successful registration before it activates the connector", () => + Effect.gen(function* () { + const firstAttempt = yield* Deferred.make(); + let attempts = 0; + let activations = 0; + let reconciliations = 0; + const registration = Effect.suspend(() => { + attempts += 1; + if (attempts === 1) { + return Deferred.succeed(firstAttempt, undefined).pipe( + Effect.andThen(Effect.fail("relay unavailable" as const)), + ); + } + return Effect.succeed({ status: "ready" as const }); + }); + const startup = retryManagedTunnelRegistration(registration, () => true).pipe( + Effect.tap((result) => + Effect.sync(() => { + const action = managedTunnelStartupAction({ wantsCliLink: true, registration: result }); + if (action.action === "reconcile_link") { + reconciliations += 1; + } + activations += 1; + }), + ), + ); + + const fiber = yield* Effect.forkChild(startup, { startImmediately: true }); + yield* Deferred.await(firstAttempt); + expect(attempts).toBe(1); + expect(activations).toBe(0); + + yield* TestClock.adjust("2 seconds"); + yield* Fiber.join(fiber); + + expect(attempts).toBe(2); + expect(activations).toBe(1); + expect(reconciliations).toBe(0); + }), + ); +}); diff --git a/apps/server/src/cloud/managedTunnelStartup.ts b/apps/server/src/cloud/managedTunnelStartup.ts new file mode 100644 index 000000000000..619a27c1aedd --- /dev/null +++ b/apps/server/src/cloud/managedTunnelStartup.ts @@ -0,0 +1,51 @@ +import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Schedule from "effect/Schedule"; + +export type ManagedTunnelRegistrationResult = + | { readonly status: "not_linked" | "ready" | "unavailable" | "superseded" } + | { + readonly status: "recovery_required"; + readonly config: RelayManagedEndpointRuntimeConfig; + }; + +export type ManagedTunnelStartupAction = + | { readonly action: "none" } + | { readonly action: "reconcile_link" } + | { + readonly action: "request_recovery"; + readonly config: RelayManagedEndpointRuntimeConfig; + }; + +export function managedTunnelStartupAction(input: { + readonly wantsCliLink: boolean; + readonly registration: ManagedTunnelRegistrationResult; +}): ManagedTunnelStartupAction { + if (input.registration.status === "recovery_required") { + return { + action: "request_recovery", + config: input.registration.config, + }; + } + if (input.wantsCliLink && input.registration.status === "not_linked") { + return { action: "reconcile_link" }; + } + return { action: "none" }; +} + +export const retryManagedTunnelRegistration = ( + registration: Effect.Effect, + isRetryable: (error: E) => boolean, +) => + registration.pipe( + Effect.retry({ + while: isRetryable, + schedule: Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + Schedule.jittered, + ), + }), + ); diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index f20777cdd60c..fd9deee5675e 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -425,6 +425,7 @@ const buildAppUnderTest = (options?: { >; relayClient?: Partial; cloudCliTokenManager?: Partial; + httpClient?: HttpClient.HttpClient; nativeTelemetryClient?: Partial; desktopTelemetryReceiver?: Partial< DesktopTelemetryReceiver.DesktopTelemetryReceiver["Service"] @@ -981,7 +982,11 @@ const buildAppUnderTest = (options?: { Layer.provideMerge(makeAuthTestLayer()), Layer.provideMerge(ServerSecretStore.layer), Layer.provide(workspaceAndProjectServicesLayer), - Layer.provideMerge(FetchHttpClient.layer), + Layer.provideMerge( + options?.layers?.httpClient === undefined + ? FetchHttpClient.layer + : Layer.succeed(HttpClient.HttpClient, options.layers.httpClient), + ), Layer.provide(layerConfig), ); @@ -2439,6 +2444,69 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + it.effect("rejects a non-Cloudflare managed endpoint runtime without persisting the link", () => + Effect.gen(function* () { + const appliedRuntimeConfigs: Array = []; + yield* buildAppUnderTest({ + layers: { + cloudManagedEndpointRuntime: { + applyConfig: (config) => + Effect.sync(() => { + appliedRuntimeConfigs.push(config); + return config === null + ? ({ status: "disabled" } as const) + : ({ status: "unsupported", providerKind: config.providerKind } as const); + }), + }, + }, + }); + + const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, + }); + const ownerCookie = yield* getAuthenticatedSessionCookieHeader(); + const relayConfigUrl = yield* getHttpServerUrl("/api/connect/relay-config"); + const relayConfigResponse = yield* fetchEffect(relayConfigUrl, { + method: "POST", + headers: { + cookie: ownerCookie, + "content-type": "application/json", + }, + body: jsonRequestBody({ + relayUrl: "https://relay.example.test", + cloudUserId: "user_123", + environmentCredential: "t3env_test_credential", + cloudMintPublicKey: cloudKeyPair.publicKey, + endpointRuntime: { + providerKind: "manual", + connectorToken: "manual-token", + }, + }), + }); + const relayConfigBody = yield* responseJsonEffect<{ + readonly _tag?: string; + readonly endpointRuntimeStatus?: { readonly status?: string }; + }>(relayConfigResponse); + const linkStateUrl = yield* getHttpServerUrl("/api/connect/link-state"); + const linkStateResponse = yield* fetchEffect(linkStateUrl, { + headers: { cookie: ownerCookie }, + }); + const linkStateBody = yield* responseJsonEffect<{ readonly linked?: boolean }>( + linkStateResponse, + ); + + assert.equal(relayConfigResponse.status, 503); + assert.equal(relayConfigBody._tag, "EnvironmentCloudEndpointUnavailableError"); + assert.equal(relayConfigBody.endpointRuntimeStatus?.status, "unsupported"); + assert.deepEqual(appliedRuntimeConfigs, [ + { providerKind: "manual", connectorToken: "manual-token" }, + ]); + assert.equal(linkStateResponse.status, 200); + assert.equal(linkStateBody.linked, false); + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ); + it.effect("reports local cloud link state from persisted relay config", () => Effect.gen(function* () { yield* buildAppUnderTest(); @@ -2539,6 +2607,9 @@ it.layer(NodeServices.layer)("server router seam", (it) => { requestedRecoveryConfigs.push(config); }), }, + httpClient: HttpClient.make((request) => + Effect.succeed(HttpClientResponse.fromWeb(request, Response.json({ status: "ready" }))), + ), }, }); @@ -2604,6 +2675,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => { assert.equal(linkStateBody.relayUrl, null); assert.equal(linkStateBody.relayIssuer, null); assert.deepEqual(appliedRuntimeConfigs, [ + null, { providerKind: "cloudflare_tunnel", connectorToken: "connector-token", @@ -2923,20 +2995,91 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + it.effect("keeps a managed connector stopped when relay registration fails", () => + Effect.gen(function* () { + const appliedRuntimeConfigs: Array = []; + const relayRequests: Array = []; + yield* buildAppUnderTest({ + layers: { + cloudManagedEndpointRuntime: { + applyConfig: (config) => + Effect.sync(() => { + appliedRuntimeConfigs.push(config); + return config === null + ? ({ status: "disabled" } as const) + : ({ status: "running", providerKind: "cloudflare_tunnel", pid: 123 } as const); + }), + }, + httpClient: HttpClient.make((request) => + Effect.sync(() => { + relayRequests.push(request); + return HttpClientResponse.fromWeb( + request, + Response.json({ message: "relay unavailable" }, { status: 503 }), + ); + }), + ), + }, + }); + + const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, + }); + const ownerCookie = yield* getAuthenticatedSessionCookieHeader(); + const relayConfigUrl = yield* getHttpServerUrl("/api/connect/relay-config"); + const relayConfigResponse = yield* fetchEffect(relayConfigUrl, { + method: "POST", + headers: { + cookie: ownerCookie, + "content-type": "application/json", + }, + body: jsonRequestBody({ + relayUrl: "https://relay.example.test", + cloudUserId: "user_123", + environmentCredential: "t3env_test_credential", + cloudMintPublicKey: cloudKeyPair.publicKey, + endpointRuntime: { + providerKind: "cloudflare_tunnel", + connectorToken: "connector-token", + tunnelId: "tunnel-1", + }, + }), + }); + const relayConfigBody = yield* responseJsonEffect<{ readonly _tag?: string }>( + relayConfigResponse, + ); + + assert.equal(relayConfigResponse.status, 500); + assert.equal(relayConfigBody._tag, "EnvironmentHttpInternalServerError"); + assert.equal(relayRequests.length, 3); + assert.deepEqual(appliedRuntimeConfigs, [null]); + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ); + it.effect("fails relay config when the managed endpoint connector cannot start", () => Effect.gen(function* () { + const appliedRuntimeConfigs: Array = []; yield* buildAppUnderTest({ layers: { cloudManagedEndpointRuntime: { - applyConfig: () => - Effect.succeed({ - status: "failed", - providerKind: "cloudflare_tunnel", - failure: "not-installed", - reason: "cloudflared missing", - tunnelId: "tunnel-1", + applyConfig: (config) => + Effect.sync(() => { + appliedRuntimeConfigs.push(config); + return config === null + ? ({ status: "disabled" } as const) + : ({ + status: "failed", + providerKind: "cloudflare_tunnel", + failure: "not-installed", + reason: "cloudflared missing", + tunnelId: "tunnel-1", + } as const); }), }, + httpClient: HttpClient.make((request) => + Effect.succeed(HttpClientResponse.fromWeb(request, Response.json({ status: "ready" }))), + ), }, }); @@ -2975,33 +3118,14 @@ it.layer(NodeServices.layer)("server router seam", (it) => { assert.equal(relayConfigBody.message, "Managed endpoint runtime could not be started."); assert.equal(relayConfigBody.endpointRuntimeStatus?.status, "failed"); assert.equal(relayConfigBody.endpointRuntimeStatus?.reason, "cloudflared missing"); - - const now = yield* DateTime.now; - const healthRequest = makeCloudEnvironmentHealthRequest({ - privateKey: cloudKeyPair.privateKey, - environmentId: testEnvironmentDescriptor.environmentId, - nonce: "cloud-health-after-failed-runtime", - issuedAt: DateTime.formatIso(now), - expiresAt: DateTime.formatIso(DateTime.add(now, { minutes: 5 })), - }); - const healthUrl = yield* getHttpServerUrl("/api/t3-connect/health"); - const healthResponse = yield* fetchEffect(healthUrl, { - method: "POST", - headers: { - "content-type": "application/json", + assert.deepEqual(appliedRuntimeConfigs, [ + null, + { + providerKind: "cloudflare_tunnel", + connectorToken: "connector-token", + tunnelId: "tunnel-1", }, - body: jsonRequestBody(healthRequest), - }); - const healthBody = yield* responseJsonEffect<{ - _tag?: string; - message?: string; - }>(healthResponse); - assert.equal(healthResponse.status, 500); - assert.equal(healthBody._tag, "EnvironmentHttpInternalServerError"); - assert.equal( - healthBody.message, - "Cloud mint public key is not installed for this environment.", - ); + ]); }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 4347223f12a5..27668667c4ea 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -96,13 +96,18 @@ import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; import { connectHttpApiLayer, pendingServiceUpdateExists, - reconcileDesiredCloudLink, + reconcileDesiredCloudLinkIfStillDesired, recoverManagedCloudTunnel, registerManagedCloudTunnelRecovery, + startManagedCloudTunnelIfOriginConfirmed, releaseManagedTunnelOnShutdown, } from "./cloud/http.ts"; import { serverRelayBrokerTracingLayer } from "./cloud/relayTracing.ts"; import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts"; +import { + managedTunnelStartupAction, + retryManagedTunnelRegistration, +} from "./cloud/managedTunnelStartup.ts"; import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts"; import * as CloudCliState from "./cloud/CliState.ts"; import * as ServerSelfUpdate from "./cloud/selfUpdate.ts"; @@ -630,21 +635,20 @@ export const makeServerLayer = Layer.unwrap( const recoveryLock = yield* Semaphore.make(1); const recoverManagedTunnel = (config: RelayManagedEndpointRuntimeConfig) => recoveryLock.withPermits(1)( - recoverManagedCloudTunnel(localOrigin, config).pipe( + recoverManagedCloudTunnel(localOrigin, config, { + retryRuntimeFailures: true, + }).pipe( Effect.retry({ while: (error) => error._tag !== "EnvironmentHttpBadRequestError" && error._tag !== "EnvironmentHttpUnauthorizedError" && error._tag !== "EnvironmentHttpConflictError" && - (error._tag !== "EnvironmentCloudEndpointUnavailableError" || - CloudManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus( - error.endpointRuntimeStatus, - )), + error._tag !== "EnvironmentCloudEndpointUnavailableError", schedule: Schedule.exponential("1 second").pipe( Schedule.modifyDelay(({ duration }) => Effect.succeed(Duration.min(duration, Duration.seconds(30))), ), - Schedule.upTo({ duration: "10 minutes" }), + Schedule.jittered, ), }), Effect.tap((recovered) => @@ -681,9 +685,18 @@ export const makeServerLayer = Layer.unwrap( const desiredCliLinkMode = wantsCliLink ? yield* CloudCliState.readCliDesiredLinkMode : null; - const registerManagedTunnel = registerManagedCloudTunnelRecovery().pipe( - Effect.tap((registered) => - registered + const registerManagedTunnel = retryManagedTunnelRegistration( + registerManagedCloudTunnelRecovery(localOrigin, { + retryRuntimeFailures: true, + }), + (error) => + error._tag !== "EnvironmentCloudEndpointUnavailableError" && + error._tag !== "EnvironmentHttpBadRequestError" && + error._tag !== "EnvironmentHttpUnauthorizedError" && + error._tag !== "EnvironmentHttpConflictError", + ).pipe( + Effect.tap((result) => + result.status === "ready" ? Effect.logInfo("T3 Connect managed tunnel recovery registered") : Effect.void, ), @@ -692,13 +705,26 @@ export const makeServerLayer = Layer.unwrap( ? Effect.interrupt : Effect.logWarning("Failed to register T3 Connect managed tunnel recovery", { cause, - }).pipe(Effect.as(false)), + }).pipe(Effect.as({ status: "unavailable" as const })), + ), + ); + yield* startManagedCloudTunnelIfOriginConfirmed(localOrigin).pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to start the confirmed T3 Connect tunnel", { cause }), ), ); - const registered = - desiredCliLinkMode === "publish_only" ? false : yield* registerManagedTunnel; - if (wantsCliLink && !registered) { - const reconciled = yield* reconcileDesiredCloudLink(localOrigin).pipe( + const registration = + desiredCliLinkMode === "publish_only" + ? { status: "not_linked" as const } + : yield* registerManagedTunnel; + const startupAction = managedTunnelStartupAction({ wantsCliLink, registration }); + if (startupAction.action === "request_recovery") { + yield* endpointRuntime.requestRecovery(startupAction.config); + } + if (startupAction.action === "reconcile_link") { + const reconciledMode = yield* reconcileDesiredCloudLinkIfStillDesired( + localOrigin, + ).pipe( Effect.retry({ while: (error) => error._tag !== "EnvironmentHttpBadRequestError" && @@ -711,16 +737,22 @@ export const makeServerLayer = Layer.unwrap( Schedule.upTo({ duration: "10 minutes" }), ), }), - Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")), - Effect.as(true), + Effect.tap((mode) => + mode === null + ? Effect.void + : Effect.logInfo("T3 Connect desired link reconciled on startup"), + ), Effect.catch((cause) => Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", { cause, - }).pipe(Effect.as(false)), + }).pipe(Effect.as(null)), ), ); - if (reconciled && desiredCliLinkMode === "managed") { - yield* registerManagedTunnel; + if (reconciledMode === "managed") { + const afterReconcile = yield* registerManagedTunnel; + if (afterReconcile.status === "recovery_required") { + yield* endpointRuntime.requestRecovery(afterReconcile.config); + } } } }), diff --git a/docs/internals/t3-connect.md b/docs/internals/t3-connect.md index 0c7ed2ba3096..a02b032b44a2 100644 --- a/docs/internals/t3-connect.md +++ b/docs/internals/t3-connect.md @@ -129,8 +129,18 @@ logout` performs the same cleanup and removes the stored CLI authorization. Every linked environment stores a relay-issued environment credential. When setup installs a tunnel and when the server starts, the server uses that credential to register recovery support for the -existing tunnel. Registration only updates the relay database and does not call Cloudflare. -Healthy CLI links reuse the stored tunnel instead of provisioning it again. +existing tunnel and its current loopback HTTP origin. A healthy boot with a previously confirmed +origin makes no Cloudflare API calls. The first registration of an older allocation, or a +registration after the local server port changes, makes one logical ingress configuration request. +The host retries transient registration failures for the life of the server with jittered exponential +backoff capped at 30 seconds. Each retry can make one logical ingress configuration request after an +origin change. The Cloudflare SDK can also retry each logical request internally. + +The host stores a confirmed-origin marker with the full connector config. A later boot starts the +connector before relay registration only when that marker matches both the current config and the +current local origin. A missing marker or changed port keeps the connector stopped until the relay +confirms or updates the ingress route. This lets a known configuration start during a relay outage +without sending traffic to an old local port. If the connector exits or repeatedly reports that Cloudflare rejected its tunnel, the server uses the same environment credential to request a replacement. This also recovers a tunnel deleted @@ -140,11 +150,27 @@ DNS record, so a replacement tunnel does not change the public endpoint. Each re recovery request includes a short-lived host signature that binds the cloud user and the current tunnel or local T3 server address. -After a host registers recovery support, the existing five-minute maintenance job removes its -tunnel when Cloudflare reports that the tunnel has been down for at least five minutes. Tunnels -that never connected are removed when they are at least five minutes old. The job leaves older -hosts alone until they register recovery support, and it only removes tunnels that belong to its -own deployment stage. +The existing five-minute maintenance job can run tunnel cleanup in `off`, `dry-run`, or `enabled` +mode through `RELAY_TUNNEL_CLEANUP_MODE`. The default is `off`. Dry-run mode lists and counts +candidates without deleting them. Enabled mode removes a recoverable host's tunnel when Cloudflare +reports that it has been down for at least five minutes. Tunnels that never connected are candidates +when they are at least five minutes old. Cleanup usually runs five to ten minutes after a tunnel goes +down because the five-minute grace period and five-minute schedule can align. Backlogs can add more +time. + +One sweep makes at most ten logical tunnel-list requests and attempts at most 100 deletions. A +deletion can use a status read and a delete, for at most 210 logical Cloudflare SDK operations in a +full sweep. The SDK can retry each operation internally, so this is not an exact network request +limit. The whole sweep has a two-minute deadline and stops early after a structured Cloudflare rate +limit response. Page rotation and alternating `down` and `inactive` priority keep large backlogs +moving across later sweeps. + +Cleanup only deletes tunnels for hosts that registered recovery with the public key on their current +environment link. Older hosts remain untouched. It also skips incomplete allocations with no +recorded tunnel ID because provisioning can be between tunnel creation and the database update. +Allocations that point at another tunnel ID are skipped for the same ownership reason. These skips +can leave stale tunnels for manual cleanup, but they prevent a cleanup sweep from deleting a tunnel +that a concurrent provision owns. The background service has an independent lifecycle. Connect setup may offer to install it, but logout leaves it running; manage it with `t3 service status`, `install`, `update`, and `uninstall`. diff --git a/docs/operations/release.md b/docs/operations/release.md index 68f2e8da578f..9fd6041074ee 100644 --- a/docs/operations/release.md +++ b/docs/operations/release.md @@ -83,6 +83,8 @@ Required `production` environment variables: Optional `production` environment variables: - `RELAY_DOMAIN` when overriding the derived `relay.` domain +- `RELAY_TUNNEL_CLEANUP_MODE` with `off`, `dry-run`, or `enabled`. Missing and blank values use + `off`. Required `production` environment secrets: @@ -102,6 +104,58 @@ Developers deploy personal stages locally rather than through pull-request autom vp run --filter t3code-relay deploy -- --stage "$USER" --env-file .env.local ``` +### Managed tunnel cleanup rollout + +Keep `RELAY_TUNNEL_CLEANUP_MODE=off` for the first production deploy. That deploy applies the +nullable allocation-origin migration and adds the registration and recovery endpoints. Web and +mobile clients do not need a coordinated deploy. Server builds in CLI and desktop releases must +reach users before cleanup is enabled because those builds register recovery and replace a deleted +tunnel after wake. + +Use this rollout order: + +1. Deploy the relay and database migration with cleanup set to `off`. +2. Release the server build and confirm that current hosts register recovery. Older hosts remain + marked as legacy and are not cleanup candidates. +3. Set cleanup to `dry-run`, deploy the relay, and inspect the cleanup log counters. Check + `scanned`, `wouldDelete`, `skippedLegacy`, `failed`, and `truncated` across several sweeps. +4. Run the disposable-host canary below. +5. Set cleanup to `enabled` only after the canary recovers without a server restart. + +The cleanup job runs every five minutes with a five-minute inactivity grace period. A candidate is +usually removed between five and ten minutes after it goes down. A backlog takes longer because one +sweep attempts at most 100 deletions. A full sweep makes at most ten logical list requests and 200 +logical status and delete operations. The Cloudflare SDK can retry each logical operation up to five +times, so these budgets are not exact network request counts. A two-minute sweep deadline keeps the +job below its five-minute schedule. A Cloudflare rate limit stops the current deletion loop early. + +### Disposable-host canary + +This test has not been run against a real Cloudflare account for this change. Run every step against +a disposable relay stage, test Cloudflare account, disposable host, disposable T3 home, and test +environment link. Keep production cleanup at `off` or `dry-run` until this canary passes. Do not stop +a daily-use T3 server or disable the whole machine's network. + +1. Deploy the disposable relay stage with cleanup set to `dry-run`. Link the disposable environment + and confirm that its tunnel is healthy and recovery is registered. +2. Capture the PID of that environment's managed `cloudflared` child from its server logs. Confirm + that the PID belongs to the disposable T3 process. +3. Pause only that captured child with `kill -STOP `. Wait until Cloudflare reports the tunnel + as down for more than five minutes. +4. Confirm that dry-run logs count the tunnel in `wouldDelete` without deleting it. +5. Set cleanup to `enabled` on the disposable relay stage and deploy it. Confirm through the test + Cloudflare account that the old tunnel is deleted. Allow up to ten minutes plus any reported + backlog. +6. Resume only the captured child with `kill -CONT `. Confirm that the running server detects + repeated tunnel authorization rejection, requests recovery, starts a replacement tunnel, and + becomes reachable at the same public hostname without a server restart. +7. Repeat with a physical sleep and wake cycle on a disposable laptop before broad rollout. + +To roll back, set cleanup to `off` and deploy the relay before downgrading any host. Keep the recovery +endpoints deployed while current server builds are in use. Downgrading a host that has registered +recovery while cleanup remains enabled is unsupported because the older host cannot replace a +deleted tunnel. The nullable database columns can remain in place. + ## Hosted web app release deployment The hosted app is intentionally not deployed by Vercel's Git integration. The diff --git a/docs/user/remote-access.md b/docs/user/remote-access.md index 67259f8cce26..4899bda7601f 100644 --- a/docs/user/remote-access.md +++ b/docs/user/remote-access.md @@ -229,10 +229,12 @@ works for a server that was wiped or is no longer reachable. Device-local connec controls remain in **Settings** → **Connections** on web and desktop or **Settings** → **Environments** on mobile. -If a linked environment stays offline for several minutes, T3 Connect removes its unused tunnel. -The environment stays linked to your account and keeps the same address. When the server starts -again or the computer wakes, T3 Connect creates a replacement tunnel automatically. You do not -need to pair it again. +When inactive tunnel cleanup is enabled, T3 Connect removes a linked environment's unused tunnel +after it stays offline for several minutes. The environment stays linked to your account and keeps +the same address. When the server starts again or the computer wakes, T3 Connect creates a +replacement tunnel automatically. You do not need to pair it again. Cleanup usually takes between +five and ten minutes after the tunnel goes down. It can take longer when many tunnels are waiting +for cleanup. ## Security Notes diff --git a/infra/relay/.env.example b/infra/relay/.env.example index 7ab2a5d6e44f..eb1500d3cab0 100644 --- a/infra/relay/.env.example +++ b/infra/relay/.env.example @@ -5,6 +5,10 @@ RELAY_API_ZONE_NAME=example.com RELAY_TUNNEL_ZONE_NAME=tunnels.example.com +# Optional: inactive tunnel cleanup. Start with dry-run, verify the cleanup +# logs, then set enabled. Unset and off both disable cleanup. +# RELAY_TUNNEL_CLEANUP_MODE=off + # Optional: Relay domain override # Set this only when the derived relay hostname should not be used. # RELAY_DOMAIN=relay.example.com diff --git a/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/migration.sql b/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/migration.sql index a9480ed4c93b..66cc66564b8d 100644 --- a/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/migration.sql +++ b/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/migration.sql @@ -1,3 +1,4 @@ ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "recovery_enabled_at" varchar(64);--> statement-breakpoint ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "recovery_environment_public_key" text;--> statement-breakpoint +ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "origin" jsonb;--> statement-breakpoint ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "generation" integer DEFAULT 0 NOT NULL; \ No newline at end of file diff --git a/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/snapshot.json b/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/snapshot.json index ce549ba1c6c0..d50cb8724843 100644 --- a/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/snapshot.json +++ b/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/snapshot.json @@ -1,7 +1,7 @@ { "version": "8", "dialect": "postgres", - "id": "7e85c554-d61a-4253-bd7b-17f92e98e665", + "id": "652b5aca-ea64-49e4-aa44-ea1e9a9f20bd", "prevIds": ["2374caff-40bf-423c-9255-55e76dddbc2a"], "ddl": [ { @@ -890,6 +890,19 @@ "schema": "public", "table": "relay_managed_endpoint_allocations" }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "origin", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, { "type": "integer", "typeSchema": null, diff --git a/infra/relay/src/Config.test.ts b/infra/relay/src/Config.test.ts new file mode 100644 index 000000000000..e37d605d0644 --- /dev/null +++ b/infra/relay/src/Config.test.ts @@ -0,0 +1,35 @@ +import { expect, it } from "@effect/vitest"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as Effect from "effect/Effect"; + +import { managedEndpointCleanupModeConfig } from "./Config.ts"; + +it.effect.each([ + { name: "missing", env: {}, expected: "off" }, + { name: "empty", env: { RELAY_TUNNEL_CLEANUP_MODE: "" }, expected: "off" }, + { name: "whitespace", env: { RELAY_TUNNEL_CLEANUP_MODE: " \t" }, expected: "off" }, + { name: "off", env: { RELAY_TUNNEL_CLEANUP_MODE: "off" }, expected: "off" }, + { + name: "dry-run", + env: { RELAY_TUNNEL_CLEANUP_MODE: "dry-run" }, + expected: "dry-run", + }, + { name: "enabled", env: { RELAY_TUNNEL_CLEANUP_MODE: "enabled" }, expected: "enabled" }, +] as const)("loads $name cleanup mode as $expected", ({ env, expected }) => + Effect.gen(function* () { + const provider = ConfigProvider.fromEnv({ env }); + expect(yield* managedEndpointCleanupModeConfig.parse(provider)).toBe(expected); + }), +); + +it.effect("rejects an invalid cleanup mode", () => + Effect.gen(function* () { + const provider = ConfigProvider.fromEnv({ + env: { RELAY_TUNNEL_CLEANUP_MODE: "delete-everything" }, + }); + const error = yield* Effect.flip(managedEndpointCleanupModeConfig.parse(provider)); + + expect(error._tag).toBe("ConfigError"); + expect(error.message).toContain('Expected "off" | "dry-run" | "enabled"'); + }), +); diff --git a/infra/relay/src/Config.ts b/infra/relay/src/Config.ts index e7c7d42f2ae1..1eadf4181463 100644 --- a/infra/relay/src/Config.ts +++ b/infra/relay/src/Config.ts @@ -1,4 +1,6 @@ +import * as Config from "effect/Config"; import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Redacted from "effect/Redacted"; import * as Schema from "effect/Schema"; @@ -6,6 +8,20 @@ import * as Schema from "effect/Schema"; export const ApnsEnvironment = Schema.Literals(["sandbox", "production"]); export type ApnsEnvironment = typeof ApnsEnvironment.Type; +export const ManagedEndpointCleanupMode = Schema.Literals(["off", "dry-run", "enabled"]); +export type ManagedEndpointCleanupMode = typeof ManagedEndpointCleanupMode.Type; +const decodeManagedEndpointCleanupMode = Schema.decodeUnknownEffect(ManagedEndpointCleanupMode); + +export const managedEndpointCleanupModeConfig = Config.string("RELAY_TUNNEL_CLEANUP_MODE").pipe( + Config.withDefault("off"), + Config.map((value) => value.trim() || "off"), + Config.mapOrFail((value) => + decodeManagedEndpointCleanupMode(value).pipe( + Effect.mapError((error) => new Config.ConfigError(error)), + ), + ), +); + export interface ApnsCredentials { readonly teamId: string; readonly keyId: string; @@ -27,6 +43,7 @@ export class RelayConfiguration extends Context.Service< readonly cloudMintPublicKey: string; readonly managedEndpointBaseDomain: string | undefined; readonly managedEndpointNamespace: string | undefined; + readonly managedEndpointCleanupMode?: ManagedEndpointCleanupMode; } >()("t3code-relay/Config/RelayConfiguration") {} diff --git a/infra/relay/src/environments/EnvironmentConnector.test.ts b/infra/relay/src/environments/EnvironmentConnector.test.ts index 31e6d27c8fcd..624d1241a287 100644 --- a/infra/relay/src/environments/EnvironmentConnector.test.ts +++ b/infra/relay/src/environments/EnvironmentConnector.test.ts @@ -188,6 +188,7 @@ function makeAllocations( tunnelName: "tunnel-name", dnsRecordId: "dns-record-id", readyAt: "2026-05-25T00:00:00.000Z", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, updatedAt: "2026-05-25T00:00:00.000Z", generation: 1, }, @@ -475,6 +476,7 @@ describe("EnvironmentConnector", () => { tunnelName: "tunnel-name", dnsRecordId: "dns-record-id", readyAt: null, + origin: null, updatedAt: "2026-05-25T00:00:00.000Z", generation: 1, }), diff --git a/infra/relay/src/environments/EnvironmentLinker.test.ts b/infra/relay/src/environments/EnvironmentLinker.test.ts index 536c4e289695..3de7363c7154 100644 --- a/infra/relay/src/environments/EnvironmentLinker.test.ts +++ b/infra/relay/src/environments/EnvironmentLinker.test.ts @@ -136,6 +136,7 @@ function testLayer(input?: { revokeForEnvironmentPublicKey: () => Effect.succeed(false), }), Layer.succeed(ManagedEndpointProvider.ManagedEndpointProvider, { + reconcileOrigin: () => Effect.succeed("ready"), prepareDeprovision: () => Effect.succeed(null), deprovision: input?.deprovision ?? (() => Effect.succeed(true)), release: () => Effect.succeed(true), diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts index 5484b3ae519d..1378c3bf5984 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts @@ -94,6 +94,7 @@ describe("ManagedEndpointAllocations", () => { environmentId: "environment-1", tunnelId: "tunnel-1", environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, }), ).toBe(true); @@ -131,6 +132,7 @@ describe("ManagedEndpointAllocations", () => { environmentId: "environment-1", tunnelId: "missing-tunnel", environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, }), ).toBe(false); }).pipe(Effect.provide(layerWithDb(fakeDb))); diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index c26b71d285c1..8fa85a376e8e 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -1,4 +1,4 @@ -import type { RelayManagedEndpoint } from "@t3tools/contracts/relay"; +import type { RelayManagedEndpoint, RelayManagedEndpointOrigin } from "@t3tools/contracts/relay"; import { and, eq, exists, inArray, isNull, sql } from "drizzle-orm"; import { QueryBuilder } from "drizzle-orm/pg-core"; import * as Context from "effect/Context"; @@ -21,6 +21,7 @@ export interface ManagedEndpointAllocation { readonly tunnelName: string; readonly dnsRecordId: string | null; readonly readyAt: string | null; + readonly origin: RelayManagedEndpointOrigin | null; readonly updatedAt: string; readonly generation: number; } @@ -103,6 +104,7 @@ interface RecordManagedEndpointDnsInput extends ManagedEndpointAllocationKey { interface MarkManagedEndpointReadyInput extends ManagedEndpointAllocationKey { readonly tunnelId: string; readonly generation: number; + readonly origin: RelayManagedEndpointOrigin; } interface ClaimManagedEndpointReleaseInput extends ManagedEndpointAllocationKey { @@ -113,6 +115,7 @@ interface ClaimManagedEndpointReleaseInput extends ManagedEndpointAllocationKey interface EnableManagedEndpointRecoveryInput extends ManagedEndpointAllocationKey { readonly tunnelId: string; readonly environmentPublicKey: string; + readonly origin: RelayManagedEndpointOrigin; } interface ClaimManagedEndpointDeprovisionInput extends ManagedEndpointAllocationKey { @@ -191,6 +194,7 @@ const allocationSelection = { tunnelName: relayManagedEndpointAllocations.tunnelName, dnsRecordId: relayManagedEndpointAllocations.dnsRecordId, readyAt: relayManagedEndpointAllocations.readyAt, + origin: relayManagedEndpointAllocations.origin, updatedAt: relayManagedEndpointAllocations.updatedAt, generation: relayManagedEndpointAllocations.generation, }; @@ -289,6 +293,7 @@ export const make = Effect.gen(function* () { .set({ tunnelId: input.tunnelId, readyAt: sql`case when ${relayManagedEndpointAllocations.tunnelId} = ${input.tunnelId} then ${relayManagedEndpointAllocations.readyAt} else null end`, + origin: sql`case when ${relayManagedEndpointAllocations.tunnelId} = ${input.tunnelId} then ${relayManagedEndpointAllocations.origin} else null end`, updatedAt: DateTime.formatIso(yield* DateTime.now), generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) @@ -351,6 +356,7 @@ export const make = Effect.gen(function* () { .update(relayManagedEndpointAllocations) .set({ readyAt: now, + origin: input.origin, updatedAt: now, generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) @@ -391,6 +397,7 @@ export const make = Effect.gen(function* () { and( whereAllocation(input), eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), + eq(relayManagedEndpointAllocations.origin, input.origin), exists( new QueryBuilder() .select({ userId: relayEnvironmentLinks.userId }) diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index c7ae9c0d1569..96385917fec0 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -218,6 +218,7 @@ function makeAllocations(calls: AllocationCall[] = []) { tunnelId: null, dnsRecordId: null, readyAt: null, + origin: null, updatedAt: `generation-${++generation}`, generation: 0, }; @@ -261,6 +262,7 @@ function makeAllocations(calls: AllocationCall[] = []) { mutate(allocationKey(input), (allocation) => ({ ...allocation, readyAt: "2026-06-02T00:00:00.000Z", + origin: input.origin, })); return true; }), @@ -1041,6 +1043,209 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.provide(layer)); }); + it.effect("keeps a tunnel when a provision replaces an ordinary release generation", () => { + const tunnelCalls: TunnelCall[] = []; + const allocations = makeAllocations(); + let replaceAfterClaim = false; + const replaced = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + claimRelease: (input) => + allocations.claimRelease(input).pipe( + Effect.tap((claimedGeneration) => { + if (claimedGeneration === null || replaceAfterClaim) return Effect.void; + replaceAfterClaim = true; + return allocations + .recordTunnel({ + userId: input.userId, + environmentId: input.environmentId, + tunnelId: "replacement-tunnel", + generation: claimedGeneration, + }) + .pipe(Effect.asVoid); + }), + ), + }); + const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), replaced); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + + expect(yield* provider.release(key)).toBe(false); + expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete"); + }).pipe(Effect.provide(layer)); + }); + + it.effect("does no Cloudflare work when the registered origin is unchanged", () => { + const tunnelCalls: TunnelCall[] = []; + const layer = providerLayer(makePersistentTunnelClient(tunnelCalls)); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + const origin = { localHttpHost: "127.0.0.1", localHttpPort: 3773 } as const; + const provisioned = yield* provider.provision({ ...key, origin }); + tunnelCalls.length = 0; + + expect( + yield* provider.reconcileOrigin({ + ...key, + tunnelId: provisioned.runtime.tunnelId!, + origin, + endpoint: provisioned.endpoint, + }), + ).toBe("ready"); + expect(tunnelCalls).toEqual([]); + }).pipe(Effect.provide(layer)); + }); + + it.effect("updates Cloudflare ingress once when the registered port changes", () => { + const tunnelCalls: TunnelCall[] = []; + const layer = providerLayer(makePersistentTunnelClient(tunnelCalls)); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + const provisioned = yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + tunnelCalls.length = 0; + + expect( + yield* provider.reconcileOrigin({ + ...key, + tunnelId: provisioned.runtime.tunnelId!, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 4884 }, + endpoint: provisioned.endpoint, + }), + ).toBe("ready"); + expect(tunnelCalls).toEqual([ + { + operation: "putConfiguration", + input: { + tunnelId: "tunnel-id", + tunnelConfig: { + ingress: [ + { + hostname: expectedManagedHostname("env_ABC"), + service: "http://127.0.0.1:4884", + }, + { service: "http_status:404" }, + ], + }, + }, + }, + ]); + }).pipe(Effect.provide(layer)); + }); + + it.effect("requires recovery when Cloudflare reports the registered tunnel is missing", () => { + const tunnelCalls: TunnelCall[] = []; + const baseTunnelClient = makePersistentTunnelClient(tunnelCalls); + let tunnelMissing = false; + const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + ...baseTunnelClient, + putConfiguration: (tunnelId, tunnelConfig) => + tunnelMissing + ? Effect.fail( + new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "put-configuration", + tunnelId, + cause: { _tag: "TunnelNotFound" }, + }), + ) + : baseTunnelClient.putConfiguration(tunnelId, tunnelConfig), + }); + const layer = providerLayer(tunnelClient); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + const provisioned = yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + tunnelMissing = true; + + expect( + yield* provider.reconcileOrigin({ + ...key, + tunnelId: provisioned.runtime.tunnelId!, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 4884 }, + endpoint: provisioned.endpoint, + }), + ).toBe("recovery_required"); + }).pipe(Effect.provide(layer)); + }); + + it.effect("fails origin sync when the allocation generation changes", () => { + const allocations = makeAllocations(); + let loseClaim = false; + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + withClaimedTunnel: (input, effect) => + loseClaim ? Effect.succeed(Option.none()) : allocations.withClaimedTunnel(input, effect), + }); + const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + const provisioned = yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + loseClaim = true; + + const error = yield* Effect.flip( + provider.reconcileOrigin({ + ...key, + tunnelId: provisioned.runtime.tunnelId!, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 4884 }, + endpoint: provisioned.endpoint, + }), + ); + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "sync-origin", + }); + }).pipe(Effect.provide(layer)); + }); + + it.effect("rejects an active endpoint that does not match the allocation hostname", () => { + const layer = providerLayer(makePersistentTunnelClient()); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + const provisioned = yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + + const error = yield* Effect.flip( + provider.reconcileOrigin({ + ...key, + tunnelId: provisioned.runtime.tunnelId!, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + endpoint: { + ...provisioned.endpoint, + httpBaseUrl: "https://different-host.t3code.test/", + }, + }), + ); + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "sync-origin", + }); + }).pipe(Effect.provide(layer)); + }); + it.effect("rejects a tunnel recorded after its allocation generation changed", () => { const tunnelCalls: TunnelCall[] = []; const allocations = makeAllocations(); diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 507707213d5a..6cf6c8d8903b 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -53,6 +53,8 @@ const ManagedEndpointProvisioningStage = Schema.Literals([ "record-dns", "get-tunnel-token", "mark-allocation-ready", + "load-allocation", + "sync-origin", ]); export class ManagedEndpointProvisioningFailed extends Schema.TaggedErrorClass()( @@ -126,6 +128,8 @@ export interface ManagedEndpointProvisioningResult { readonly runtime: RelayManagedEndpointRuntimeConfig; } +export type ManagedEndpointOriginSyncResult = "ready" | "recovery_required"; + export type ManagedEndpointDeprovisionTarget = ManagedEndpointAllocations.ManagedEndpointAllocation; export class ManagedEndpointProvider extends Context.Service< @@ -136,6 +140,13 @@ export class ManagedEndpointProvider extends Context.Service< readonly environmentId: string; readonly origin: RelayManagedEndpointOrigin; }) => Effect.Effect; + readonly reconcileOrigin: (input: { + readonly userId: string; + readonly environmentId: string; + readonly tunnelId: string; + readonly origin: RelayManagedEndpointOrigin; + readonly endpoint: RelayManagedEndpoint; + }) => Effect.Effect; /** * Captures the allocation generation owned by an unlink before its link * revocation commits. Passing this target to `deprovision` prevents a @@ -482,8 +493,131 @@ export const make = Effect.gen(function* () { }, ); + const reconcileOrigin = Effect.fn("relay.managed_endpoint_provider.reconcile_origin")( + function* (input: { + readonly userId: string; + readonly environmentId: string; + readonly tunnelId: string; + readonly origin: RelayManagedEndpointOrigin; + readonly endpoint: RelayManagedEndpoint; + }) { + if (!isLoopbackOrigin(input.origin)) { + return yield* new ManagedEndpointOriginNotAllowed({ + userId: input.userId, + environmentId: input.environmentId, + host: input.origin.localHttpHost, + port: input.origin.localHttpPort, + }); + } + const allocation = yield* allocations.get(input).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointProvisioningFailed({ + ...input, + stage: "load-allocation", + cause, + }), + ), + ); + if ( + allocation === null || + allocation.tunnelId !== input.tunnelId || + allocation.dnsRecordId === null || + allocation.readyAt === null + ) { + return "recovery_required"; + } + const cf = yield* requireCloudflareSettings(config, input); + const recordedEndpoint = ManagedEndpointAllocations.resolveReadyManagedEndpoint({ + allocation, + baseDomain: cf.baseDomain, + }); + if ( + recordedEndpoint === null || + recordedEndpoint.httpBaseUrl !== input.endpoint.httpBaseUrl || + recordedEndpoint.wsBaseUrl !== input.endpoint.wsBaseUrl || + recordedEndpoint.providerKind !== input.endpoint.providerKind + ) { + return yield* new ManagedEndpointProvisioningFailed({ + ...input, + stage: "sync-origin", + hostname: allocation.hostname, + cause: "The recorded tunnel endpoint does not match the active environment link.", + }); + } + if ( + allocation.origin?.localHttpHost === input.origin.localHttpHost && + allocation.origin.localHttpPort === input.origin.localHttpPort + ) { + return "ready"; + } + + const updated = yield* allocations + .withClaimedTunnel( + { + userId: input.userId, + environmentId: input.environmentId, + tunnelId: input.tunnelId, + generation: allocation.generation, + }, + tunnels + .putConfiguration(input.tunnelId, { + ingress: [ + { + hostname: allocation.hostname, + service: formatOriginService(input.origin), + }, + { service: "http_status:404" }, + ], + }) + .pipe( + Effect.as("configured" as const), + Effect.catchTags({ + ManagedEndpointTunnelClientError: (error) => + isManagedEndpointNotFound(error.cause) + ? Effect.succeed("missing" as const) + : Effect.fail(error), + }), + Effect.flatMap((result) => + result === "missing" + ? Effect.succeed(result) + : allocations + .markReady({ + ...input, + generation: allocation.generation, + }) + .pipe( + Effect.map((updated) => + updated ? ("configured" as const) : ("stale" as const), + ), + ), + ), + ), + ) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointProvisioningFailed({ + ...input, + stage: "sync-origin", + cause, + }), + ), + ); + if (Option.isNone(updated) || updated.value === "stale") { + return yield* new ManagedEndpointProvisioningFailed({ + ...input, + stage: "sync-origin", + cause: "The tunnel allocation changed while its origin was updated.", + }); + } + return updated.value === "configured" ? "ready" : "recovery_required"; + }, + ); + return ManagedEndpointProvider.of({ prepareDeprovision, + reconcileOrigin, deprovision: Effect.fn("relay.managed_endpoint_provider.deprovision")(function* (input) { yield* Effect.annotateCurrentSpan({ "relay.user_id": input.userId, @@ -696,66 +830,61 @@ export const make = Effect.gen(function* () { ) { return false; } - - const released = yield* allocations - .withClaimedTunnel( - { - userId: input.userId, - environmentId: input.environmentId, - tunnelId, - generation: claimedGeneration, - }, - Effect.gen(function* () { - const finalGeneration = yield* allocations - .claimRelease({ - userId: input.userId, - environmentId: input.environmentId, - tunnelId, - generation: claimedGeneration, - }) - .pipe( - Effect.mapError( - (cause) => - new ManagedEndpointDeprovisioningFailed({ - ...input, - stage: "claim-release", - tunnelId, - cause, - }), - ), - ); - if (finalGeneration === null) { - return false; - } - // Keep only the final delete inside the row lock so a concurrent - // provision cannot record this tunnel while Cloudflare removes it. - yield* deleteTunnel; - return true; - }), - ) - .pipe( - Effect.catchTags({ - ManagedEndpointAllocationPersistenceError: (cause) => - Effect.fail( - new ManagedEndpointDeprovisioningFailed({ - ...input, - stage: "claim-release", - tunnelId, - cause, - }), - ), - }), - ); - return Option.getOrElse(released, () => false); } - yield* deleteTunnel; + const released = yield* allocations + .withClaimedTunnel( + { + userId: input.userId, + environmentId: input.environmentId, + tunnelId, + generation: claimedGeneration, + }, + Effect.gen(function* () { + const finalGeneration = yield* allocations + .claimRelease({ + userId: input.userId, + environmentId: input.environmentId, + tunnelId, + generation: claimedGeneration, + }) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "claim-release", + tunnelId, + cause, + }), + ), + ); + if (finalGeneration === null) { + return false; + } + yield* deleteTunnel; + return true; + }), + ) + .pipe( + Effect.catchTags({ + ManagedEndpointAllocationPersistenceError: (cause) => + Effect.fail( + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "claim-release", + tunnelId, + cause, + }), + ), + }), + ); // The recorded tunnelId is now stale, but the allocation row is left // untouched deliberately: connect/status authorization requires a fully // recorded allocation, and an offline environment must keep reporting // "offline" (health probe fails) rather than "not authorized". The next // provision lists tunnels by name, finds none, creates a replacement and // re-records the fresh id. - return true; + return Option.getOrElse(released, () => false); }), provision: Effect.fn("relay.managed_endpoint_provider.provision")(function* (input) { yield* Effect.annotateCurrentSpan({ @@ -1124,6 +1253,7 @@ export const make = Effect.gen(function* () { environmentId: input.environmentId, tunnelId: tunnel.id, generation: dnsGeneration, + origin: input.origin, }) .pipe( Effect.mapError( @@ -1179,6 +1309,7 @@ export const layerCloudflareBindings = ( layerTunnelClient({ get: (tunnelId) => tunnelClient.get(tunnelId).pipe( + Effect.timeout("8 seconds"), Effect.mapError( (cause) => new ManagedEndpointTunnelClientError({ @@ -1191,6 +1322,7 @@ export const layerCloudflareBindings = ( ), list: (request) => tunnelClient.list(request).pipe( + Effect.timeout("8 seconds"), Effect.mapError( (cause) => new ManagedEndpointTunnelClientError({ @@ -1203,6 +1335,7 @@ export const layerCloudflareBindings = ( ), create: (request) => tunnelClient.create(request).pipe( + Effect.timeout("8 seconds"), Effect.mapError( (cause) => new ManagedEndpointTunnelClientError({ @@ -1215,6 +1348,7 @@ export const layerCloudflareBindings = ( ), putConfiguration: (tunnelId, config) => tunnelClient.putConfiguration(tunnelId, config).pipe( + Effect.timeout("8 seconds"), Effect.mapError( (cause) => new ManagedEndpointTunnelClientError({ @@ -1227,6 +1361,7 @@ export const layerCloudflareBindings = ( ), getToken: (tunnelId) => tunnelClient.getToken(tunnelId).pipe( + Effect.timeout("8 seconds"), Effect.mapError( (cause) => new ManagedEndpointTunnelClientError({ @@ -1239,6 +1374,7 @@ export const layerCloudflareBindings = ( ), delete: (tunnelId) => tunnelClient.delete(tunnelId).pipe( + Effect.timeout("8 seconds"), Effect.mapError( (cause) => new ManagedEndpointTunnelClientError({ @@ -1253,6 +1389,7 @@ export const layerCloudflareBindings = ( layerDnsClient({ listRecords: (hostname) => dnsClient.listDnsRecords({ search: hostname }).pipe( + Effect.timeout("8 seconds"), Effect.map((response) => response.result.filter( (record): record is typeof record & { readonly id: string } => @@ -1272,6 +1409,7 @@ export const layerCloudflareBindings = ( ), createRecord: (request) => dnsClient.createDnsRecord(request).pipe( + Effect.timeout("8 seconds"), Effect.map((response) => ({ id: response.id })), Effect.mapError( (cause) => @@ -1285,6 +1423,7 @@ export const layerCloudflareBindings = ( ), updateRecord: (dnsRecordId, request) => dnsClient.updateDnsRecord(dnsRecordId, request).pipe( + Effect.timeout("8 seconds"), Effect.mapError( (cause) => new ManagedEndpointDnsClientError({ @@ -1298,6 +1437,7 @@ export const layerCloudflareBindings = ( ), deleteRecord: (dnsRecordId) => dnsClient.deleteDnsRecord(dnsRecordId).pipe( + Effect.timeout("8 seconds"), Effect.mapError( (cause) => new ManagedEndpointDnsClientError({ diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts index 335d58c3c217..71181dae04ab 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -45,6 +45,7 @@ function allocation(input: { tunnelName: `${PREFIX}aaaaaaaaaaaaaaaa`, dnsRecordId: "dns-1", readyAt: "2026-08-25T11:00:00.000Z", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, updatedAt: "2026-08-25T11:00:00.000Z", generation: 1, recoveryEnabled: input.recoveryEnabled, @@ -56,11 +57,14 @@ function harness(input?: { readonly allocations?: ReadonlyArray; readonly namespace?: string; readonly failTunnelId?: string; + readonly rateLimitedTunnelId?: string; + readonly failAllDeletes?: boolean; readonly missingOnDeleteTunnelId?: string; readonly missingOnGetTunnelId?: string; readonly reserveOnGetTunnelId?: string; readonly refreshedTunnels?: ReadonlyMap; readonly skipTunnelId?: string; + readonly cleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode; }) { const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = []; const deleted: string[] = []; @@ -122,7 +126,10 @@ function harness(input?: { putConfiguration: () => Effect.die("unused"), getToken: () => Effect.die("unused"), delete: (tunnelId) => - tunnelId === input?.failTunnelId || tunnelId === input?.missingOnDeleteTunnelId + input?.failAllDeletes === true || + tunnelId === input?.failTunnelId || + tunnelId === input?.rateLimitedTunnelId || + tunnelId === input?.missingOnDeleteTunnelId ? Effect.fail( new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ operation: "delete", @@ -130,7 +137,15 @@ function harness(input?: { cause: tunnelId === input?.missingOnDeleteTunnelId ? { _tag: "NotFound" } - : "Cloudflare refused the deletion", + : tunnelId === input?.rateLimitedTunnelId + ? { + cause: { + _tag: "TooManyRequests", + message: "Cloudflare rate limit exceeded", + retryAfter: 60, + }, + } + : "Cloudflare refused the deletion", }), ) : Effect.sync(() => { @@ -158,6 +173,7 @@ function harness(input?: { }); const provider = ManagedEndpointProvider.ManagedEndpointProvider.of({ provision: () => Effect.die("unused"), + reconcileOrigin: () => Effect.die("unused"), prepareDeprovision: () => Effect.die("unused"), deprovision: () => Effect.die("unused"), release: (request) => @@ -195,6 +211,7 @@ function harness(input?: { cloudMintPublicKey: "cloud-public-key", managedEndpointBaseDomain: "example.test", managedEndpointNamespace: input?.namespace ?? "prod", + managedEndpointCleanupMode: input?.cleanupMode ?? "enabled", }); return { @@ -240,7 +257,7 @@ describe("ManagedEndpointReaper", () => { return Effect.gen(function* () { yield* TestClock.setTime(NOW_MILLIS); const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; - expect(yield* reaper.sweep).toEqual({ + expect(yield* reaper.sweep).toMatchObject({ scanned: 2, deleted: 2, skippedLegacy: 0, @@ -301,7 +318,7 @@ describe("ManagedEndpointReaper", () => { return Effect.gen(function* () { yield* TestClock.setTime(NOW_MILLIS); const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; - expect(yield* reaper.sweep).toEqual({ + expect(yield* reaper.sweep).toMatchObject({ scanned: 0, deleted: 0, skippedLegacy: 0, @@ -327,7 +344,7 @@ describe("ManagedEndpointReaper", () => { return Effect.gen(function* () { yield* TestClock.setTime(NOW_MILLIS); const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; - expect(yield* reaper.sweep).toEqual({ + expect(yield* reaper.sweep).toMatchObject({ scanned: 1, deleted: 0, skippedLegacy: 1, @@ -358,7 +375,7 @@ describe("ManagedEndpointReaper", () => { }).pipe(Effect.provide(state.layer)); }); - it.effect("removes an expired tunnel that was created but never recorded", () => { + it.effect("keeps an expired tunnel while its allocation is incomplete", () => { const state = harness({ tunnels: [ tunnel({ @@ -374,8 +391,8 @@ describe("ManagedEndpointReaper", () => { return Effect.gen(function* () { yield* TestClock.setTime(NOW_MILLIS); const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; - expect((yield* reaper.sweep).deleted).toBe(1); - expect(state.deleted).toEqual(["unrecorded"]); + expect((yield* reaper.sweep).deleted).toBe(0); + expect(state.deleted).toEqual([]); }).pipe(Effect.provide(state.layer)); }); @@ -424,7 +441,7 @@ describe("ManagedEndpointReaper", () => { return Effect.gen(function* () { yield* TestClock.setTime(NOW_MILLIS); const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; - expect(yield* reaper.sweep).toEqual({ + expect(yield* reaper.sweep).toMatchObject({ scanned: 1, deleted: 1, skippedLegacy: 0, @@ -498,7 +515,7 @@ describe("ManagedEndpointReaper", () => { return Effect.gen(function* () { yield* TestClock.setTime(NOW_MILLIS); const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; - expect(yield* reaper.sweep).toEqual({ + expect(yield* reaper.sweep).toMatchObject({ scanned: 2, deleted: 1, skippedLegacy: 0, @@ -508,6 +525,38 @@ describe("ManagedEndpointReaper", () => { }).pipe(Effect.provide(state.layer)); }); + it.effect("stops the sweep after a structured Cloudflare rate limit error", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "limited", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + tunnel({ + id: "next", + suffix: "bbbbbbbbbbbbbbbb", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + rateLimitedTunnelId: "limited", + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toMatchObject({ + attempted: 1, + deleted: 0, + failed: 1, + truncated: true, + }); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + it.effect("continues past a page of older hosts to find recoverable tunnels", () => { const entries = Array.from({ length: 101 }, (_, index) => tunnel({ @@ -527,7 +576,7 @@ describe("ManagedEndpointReaper", () => { return Effect.gen(function* () { yield* TestClock.setTime(NOW_MILLIS); const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; - expect(yield* reaper.sweep).toEqual({ + expect(yield* reaper.sweep).toMatchObject({ scanned: 101, deleted: 1, skippedLegacy: 100, @@ -561,7 +610,7 @@ describe("ManagedEndpointReaper", () => { return Effect.gen(function* () { yield* TestClock.setTime(NOW_MILLIS); const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; - expect(yield* reaper.sweep).toEqual({ + expect(yield* reaper.sweep).toMatchObject({ scanned: 120, deleted: 70, skippedLegacy: 50, @@ -590,4 +639,159 @@ describe("ManagedEndpointReaper", () => { expect(state.deleted).toHaveLength(100); }).pipe(Effect.provide(state.layer)); }); + + it.effect("does no Cloudflare work while cleanup is off", () => { + const state = harness({ + cleanupMode: "off", + tunnels: [ + tunnel({ + id: "expired", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + mode: "off", + listRequests: 0, + scanned: 0, + attempted: 0, + deleted: 0, + wouldDelete: 0, + skippedLegacy: 0, + failed: 0, + truncated: false, + }); + expect(state.listRequests).toEqual([]); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("reports candidates without mutating them in dry-run mode", () => { + const state = harness({ + cleanupMode: "dry-run", + tunnels: [ + tunnel({ + id: "expired", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toMatchObject({ + mode: "dry-run", + scanned: 1, + attempted: 0, + deleted: 0, + wouldDelete: 1, + }); + expect(state.deleted).toEqual([]); + expect(state.releases).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("caps failed deletion attempts and Cloudflare list pages", () => { + const entries = Array.from({ length: 250 }, (_, index) => + tunnel({ + id: `failed-${index}`, + suffix: index.toString(16).padStart(16, "0"), + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ); + const state = harness({ tunnels: entries, failAllDeletes: true }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toMatchObject({ + attempted: 100, + deleted: 0, + failed: 100, + truncated: true, + }); + expect(state.listRequests.length).toBeLessThanOrEqual( + ManagedEndpointReaper.MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT, + ); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("rotates bounded pages across a large legacy prefix", () => { + const entries = Array.from({ length: 1_000 }, (_, index) => + tunnel({ + id: `legacy-${index}`, + suffix: index.toString(16).padStart(16, "0"), + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ); + const state = harness({ + cleanupMode: "dry-run", + tunnels: entries, + allocations: entries.map((entry) => + allocation({ tunnelId: entry.id!, recoveryEnabled: false }), + ), + }); + + return Effect.gen(function* () { + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + yield* TestClock.setTime(NOW_MILLIS); + yield* reaper.sweep; + const firstPages = state.listRequests + .filter((request) => request.status === "down") + .map((request) => request.page); + state.listRequests.length = 0; + yield* TestClock.setTime(NOW_MILLIS + 5 * 60 * 1_000); + yield* reaper.sweep; + const secondPages = state.listRequests + .filter((request) => request.status === "down") + .map((request) => request.page); + expect(firstPages).not.toEqual(secondPages); + expect(firstPages).toHaveLength(5); + expect(secondPages).toHaveLength(5); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("alternates status priority when the attempt budget is full", () => { + const entries = (["down", "inactive"] as const).flatMap((status) => + Array.from({ length: 100 }, (_, index) => + tunnel({ + id: `${status}-${index}`, + suffix: `${status === "down" ? "a" : "b"}${index.toString(16).padStart(15, "0")}`, + status, + timestamp: "2026-08-25T11:00:00.000Z", + }), + ), + ); + const first = harness({ tunnels: entries }); + const second = harness({ tunnels: entries }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const firstReaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + yield* firstReaper.sweep; + expect(first.deleted.every((id) => id.startsWith("down-"))).toBe(true); + }) + .pipe(Effect.provide(first.layer)) + .pipe( + Effect.andThen( + Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS + 5 * 60 * 1_000); + const secondReaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + yield* secondReaper.sweep; + expect(second.deleted.every((id) => id.startsWith("inactive-"))).toBe(true); + }).pipe(Effect.provide(second.layer)), + ), + ); + }); }); diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts index 0442d5e58a80..26fe5cc7e9ae 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -4,6 +4,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import type { ManagedEndpointCleanupMode } from "../Config.ts"; import * as RelayConfiguration from "../Config.ts"; import { managedEndpointTunnelNamePrefix } from "../deploymentConfig.ts"; import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts"; @@ -11,13 +12,19 @@ import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts"; export const MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES = 5; export const MANAGED_ENDPOINT_SWEEP_PAGE_SIZE = 100; -export const MANAGED_ENDPOINT_SWEEP_DELETE_LIMIT = 100; +export const MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT = 100; +export const MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT = 10; export interface ManagedEndpointSweepResult { + readonly mode: ManagedEndpointCleanupMode; + readonly listRequests: number; readonly scanned: number; + readonly attempted: number; readonly deleted: number; + readonly wouldDelete: number; readonly skippedLegacy: number; readonly failed: number; + readonly truncated: boolean; } export class ManagedEndpointReaper extends Context.Service< @@ -52,7 +59,6 @@ function isExpiredManagedTunnel(input: { ) { return false; } - const inactiveAt = status === "down" ? tunnel.connsInactiveAt : tunnel.createdAt; if (typeof inactiveAt !== "string") { return false; @@ -61,6 +67,50 @@ function isExpiredManagedTunnel(input: { return Option.isSome(timestamp) && timestamp.value.epochMilliseconds <= cutoff.epochMilliseconds; } +function isRateLimited(cause: unknown): boolean { + if (typeof cause !== "object" || cause === null) { + return false; + } + if ("_tag" in cause && cause._tag === "TooManyRequests") { + return true; + } + if ("status" in cause && cause.status === 429) { + return true; + } + return "cause" in cause && isRateLimited(cause.cause); +} + +function rotatedPages(input: { + readonly totalCount: number | undefined; + readonly slot: number; + readonly limit: number; +}): ReadonlyArray { + if (input.limit <= 0) return []; + if (input.totalCount === undefined) { + return Array.from({ length: input.limit }, (_, index) => index + 2); + } + const laterPageCount = Math.max( + 0, + Math.ceil(input.totalCount / MANAGED_ENDPOINT_SWEEP_PAGE_SIZE) - 1, + ); + if (laterPageCount === 0) return []; + const count = Math.min(input.limit, laterPageCount); + const start = input.slot % laterPageCount; + return Array.from({ length: count }, (_, index) => 2 + ((start + index) % laterPageCount)); +} + +const emptyResult = (mode: ManagedEndpointCleanupMode): ManagedEndpointSweepResult => ({ + mode, + listRequests: 0, + scanned: 0, + attempted: 0, + deleted: 0, + wouldDelete: 0, + skippedLegacy: 0, + failed: 0, + truncated: false, +}); + export const make = Effect.gen(function* () { const config = yield* RelayConfiguration.RelayConfiguration; const tunnels = yield* ManagedEndpointProvider.ManagedEndpointTunnelClient; @@ -85,19 +135,9 @@ export const make = Effect.gen(function* () { : Effect.fail(error), }), ); - if (Option.isNone(current)) { - return true; - } - if (!isExpiredManagedTunnel({ ...input, tunnel: current.value })) { - return false; - } - if ( - (yield* allocations.listByTunnelNames([input.tunnel.name])).some( - (allocation) => allocation.tunnelId !== null, - ) - ) { - return false; - } + if (Option.isNone(current)) return true; + if (!isExpiredManagedTunnel({ ...input, tunnel: current.value })) return false; + if ((yield* allocations.listByTunnelNames([input.tunnel.name])).length > 0) return false; return yield* tunnels.delete(input.tunnel.id).pipe( Effect.as(true), Effect.catchTags({ @@ -110,18 +150,19 @@ export const make = Effect.gen(function* () { }); const sweep = Effect.gen(function* () { + const mode = config.managedEndpointCleanupMode ?? "off"; const namespace = config.managedEndpointNamespace; - if (!namespace) { - return { scanned: 0, deleted: 0, skippedLegacy: 0, failed: 0 }; - } + if (mode === "off" || !namespace) return emptyResult(mode); const now = yield* DateTime.now; const cutoff = DateTime.subtract(now, { minutes: MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES }); const cutoffIso = DateTime.formatIso(cutoff); const prefix = managedEndpointTunnelNamePrefix(namespace); - let deleted = 0; - let skippedLegacy = 0; - let failed = 0; + const slot = Math.floor( + now.epochMilliseconds / (MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES * 60 * 1_000), + ); + let listRequests = 0; + let truncated = false; const expired: Array<{ readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & { readonly id: string; @@ -130,10 +171,12 @@ export const make = Effect.gen(function* () { readonly status: "down" | "inactive"; }> = []; - for (const status of ["down", "inactive"] as const) { - let page = 1; - while (true) { - const response = yield* tunnels.list({ + const statuses = + slot % 2 === 0 ? (["down", "inactive"] as const) : (["inactive", "down"] as const); + for (const status of statuses) { + const listPage = (page: number) => { + listRequests += 1; + return tunnels.list({ isDeleted: false, includePrefix: prefix, status, @@ -142,35 +185,51 @@ export const make = Effect.gen(function* () { page, perPage: MANAGED_ENDPOINT_SWEEP_PAGE_SIZE, }); + }; + const first = yield* listPage(1); + const totalCount = + typeof first.resultInfo?.totalCount === "number" ? first.resultInfo.totalCount : undefined; + const pages = rotatedPages({ + totalCount, + slot, + limit: Math.floor(MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT / 2) - 1, + }); + const responses = [first, ...(yield* Effect.forEach(pages, listPage, { concurrency: 1 }))]; + if ( + totalCount !== undefined && + Math.ceil(totalCount / MANAGED_ENDPOINT_SWEEP_PAGE_SIZE) > responses.length + ) { + truncated = true; + } else if ( + totalCount === undefined && + responses.at(-1)?.result.length === MANAGED_ENDPOINT_SWEEP_PAGE_SIZE + ) { + truncated = true; + } + for (const response of responses) { expired.push( ...response.result .map((tunnel) => ({ tunnel, status, prefix, cutoff })) .filter(isExpiredManagedTunnel) .map(({ tunnel }) => ({ tunnel, status })), ); - - const totalCount = response.resultInfo?.totalCount; - if ( - response.result.length === 0 || - (typeof totalCount === "number" - ? page * MANAGED_ENDPOINT_SWEEP_PAGE_SIZE >= totalCount - : response.result.length < MANAGED_ENDPOINT_SWEEP_PAGE_SIZE) - ) { - break; - } - page += 1; } } - const recorded = yield* allocations.listByTunnelNames(expired.map(({ tunnel }) => tunnel.name)); + const uniqueExpired = [...new Map(expired.map((entry) => [entry.tunnel.id, entry])).values()]; + const recorded = yield* allocations.listByTunnelNames( + uniqueExpired.map(({ tunnel }) => tunnel.name), + ); const recordedByTunnelName = new Map( recorded.map((allocation) => [allocation.tunnelName, allocation]), ); + let attempted = 0; + let deleted = 0; + let wouldDelete = 0; + let skippedLegacy = 0; + let failed = 0; - for (const { tunnel, status } of expired) { - if (deleted >= MANAGED_ENDPOINT_SWEEP_DELETE_LIMIT) { - break; - } + for (const { tunnel, status } of uniqueExpired) { const allocation = recordedByTunnelName.get(tunnel.name); if ( allocation !== undefined && @@ -184,7 +243,14 @@ export const make = Effect.gen(function* () { skippedLegacy += 1; continue; } - + if (allocation !== undefined && owner === undefined) continue; + wouldDelete += 1; + if (mode === "dry-run") continue; + if (attempted >= MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) { + truncated = true; + break; + } + attempted += 1; const result = owner === undefined ? yield* deleteOrphan({ tunnel, status, prefix, cutoff }).pipe(Effect.result) @@ -204,6 +270,10 @@ export const make = Effect.gen(function* () { tunnelName: tunnel.name, cause: result.failure, }); + if (isRateLimited(result.failure)) { + truncated = true; + break; + } } else if (result.success) { deleted += 1; yield* Effect.logInfo("Deleted an inactive managed tunnel", { @@ -214,7 +284,17 @@ export const make = Effect.gen(function* () { } } - return { scanned: expired.length, deleted, skippedLegacy, failed }; + return { + mode, + listRequests, + scanned: uniqueExpired.length, + attempted, + deleted, + wouldDelete, + skippedLegacy, + failed, + truncated, + }; }).pipe(Effect.withSpan("relay.managed_endpoint_reaper.sweep")); return ManagedEndpointReaper.of({ sweep }); diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index ca0a321b4ded..60330d20ef97 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -176,6 +176,7 @@ function relayUnlinkTestLayer(input?: { readonly prepareDeprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["prepareDeprovision"]; readonly deprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["deprovision"]; readonly provision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["provision"]; + readonly reconcileOrigin?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["reconcileOrigin"]; readonly release?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["release"]; }) { return Layer.mergeAll( @@ -209,6 +210,7 @@ function relayUnlinkTestLayer(input?: { ManagedEndpointProvider.ManagedEndpointProvider, ManagedEndpointProvider.ManagedEndpointProvider.of({ provision: input?.provision ?? (() => Effect.die("unused provision")), + reconcileOrigin: input?.reconcileOrigin ?? (() => Effect.succeed("ready")), prepareDeprovision: input?.prepareDeprovision ?? (() => Effect.succeed(null)), deprovision: input?.deprovision ?? (() => Effect.succeed(true)), release: input?.release ?? (() => Effect.die("unused release")), @@ -286,18 +288,62 @@ describe("relay managed tunnel recovery", () => { environmentId: "environment-1", environmentPublicKey: keyPair.publicKey, tunnelId: "existing-tunnel", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, }), ); expect(wrongAction).toMatchObject({ _tag: "Unauthorized" }); }).pipe(Effect.provideService(RelayConfiguration.RelayConfiguration, relaySettings)), ); + it.effect("rejects a signed registration proof for a different origin", () => + Effect.gen(function* () { + const keyPair = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, + }); + const now = yield* DateTime.now; + const issuedAt = Math.floor(now.epochMilliseconds / 1_000); + const proof = yield* signRelayJwt({ + privateKey: keyPair.privateKey, + typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP, + payload: { + iss: "t3-env:environment-1", + aud: "https://relay.example.test", + sub: "environment-1", + jti: "registration-origin-proof", + iat: issuedAt, + exp: issuedAt + 60, + action: "register", + environmentId: "environment-1", + cloudUserId: "user-1", + tunnelId: "existing-tunnel", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }, + }); + + const error = yield* Effect.flip( + verifyEnvironmentTunnelRecoveryProof({ + action: "register", + proof, + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: keyPair.publicKey, + tunnelId: "existing-tunnel", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 5432 }, + }), + ); + + expect(error).toMatchObject({ _tag: "Unauthorized" }); + }).pipe(Effect.provideService(RelayConfiguration.RelayConfiguration, relaySettings)), + ); + it.effect("registers recovery for an existing tunnel without provisioning it", () => { let recoveryEnabledFor: { readonly userId: string; readonly environmentId: string; readonly tunnelId: string; readonly environmentPublicKey: string; + readonly origin: { readonly localHttpHost: string; readonly localHttpPort: number }; } | null = null; return Effect.gen(function* () { @@ -307,13 +353,15 @@ describe("relay managed tunnel recovery", () => { environmentId: "environment-1", environmentPublicKey: "public-key", tunnelId: "existing-tunnel", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, }), - ).toEqual({ ok: true }); + ).toEqual({ status: "ready" }); expect(recoveryEnabledFor).toEqual({ userId: "user-1", environmentId: "environment-1", tunnelId: "existing-tunnel", environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, }); }).pipe( Effect.provide( @@ -334,6 +382,39 @@ describe("relay managed tunnel recovery", () => { ); }); + it.effect("requests recovery without enabling a stale tunnel", () => { + let recoveryEnabled = false; + + return Effect.gen(function* () { + expect( + yield* registerEnvironmentTunnelRecovery({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + tunnelId: "deleted-tunnel", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ).toEqual({ status: "recovery_required" }); + expect(recoveryEnabled).toBe(false); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + reconcileOrigin: () => Effect.succeed("recovery_required"), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => + Effect.sync(() => { + recoveryEnabled = true; + return true; + }), + }), + ), + ), + ); + }); + it.effect("rejects recovery registration for a different environment key", () => { let recoveryEnabled = false; @@ -344,6 +425,7 @@ describe("relay managed tunnel recovery", () => { environmentId: "environment-1", environmentPublicKey: "different-public-key", tunnelId: "existing-tunnel", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, }), ); @@ -375,6 +457,7 @@ describe("relay managed tunnel recovery", () => { environmentId: "environment-1", environmentPublicKey: "public-key", tunnelId: "stale-tunnel", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, }), ); @@ -423,6 +506,7 @@ describe("relay managed tunnel recovery", () => { environmentId: "environment-1", tunnelId: "replacement-tunnel", environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, }); }).pipe( Effect.provide( @@ -594,6 +678,7 @@ describe("relay managed tunnel recovery", () => { tunnelName: "environment-1-tunnel", dnsRecordId: "dns-1", readyAt: "2026-07-28T00:00:00.000Z", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, updatedAt: "replacement-generation", generation: 3, } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; @@ -686,6 +771,7 @@ describe("relay environment unlink", () => { tunnelName: "environment-1-tunnel", dnsRecordId: "dns-1", readyAt: "2026-07-28T00:00:00.000Z", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, updatedAt: "generation-before-unlink", generation: 1, } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; @@ -831,6 +917,7 @@ describe("relay environment unlink", () => { tunnelName: "environment-1-tunnel", dnsRecordId: "dns-1", readyAt: "2026-07-28T00:00:00.000Z", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, updatedAt: "original-generation", generation: 1, } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index df1d1661279b..dac1f6ec7db2 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -486,7 +486,11 @@ type EnvironmentTunnelRecoveryProofInput = { readonly environmentId: string; readonly environmentPublicKey: string; } & ( - | { readonly action: "register"; readonly tunnelId: string } + | { + readonly action: "register"; + readonly tunnelId: string; + readonly origin: RelayManagedEndpointOrigin; + } | { readonly action: "recover"; readonly origin: RelayManagedEndpointOrigin } ); @@ -516,7 +520,12 @@ export const verifyEnvironmentTunnelRecoveryProof = Effect.fn( return yield* new HttpApiError.Unauthorized({}); } if (input.action === "register") { - if (verified.action !== "register" || verified.tunnelId !== input.tunnelId) { + if ( + verified.action !== "register" || + verified.tunnelId !== input.tunnelId || + verified.origin.localHttpHost !== input.origin.localHttpHost || + verified.origin.localHttpPort !== input.origin.localHttpPort + ) { return yield* new HttpApiError.Unauthorized({}); } return; @@ -537,9 +546,11 @@ export const registerEnvironmentTunnelRecovery = Effect.fn( readonly environmentId: string; readonly environmentPublicKey: string; readonly tunnelId: string; + readonly origin: RelayManagedEndpointOrigin; }) { const links = yield* EnvironmentLinks.EnvironmentLinks; const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const managedEndpointProvider = yield* ManagedEndpointProvider.ManagedEndpointProvider; const link = yield* links.getForUser({ userId: input.userId, environmentId: input.environmentId, @@ -551,10 +562,20 @@ export const registerEnvironmentTunnelRecovery = Effect.fn( ) { return yield* new HttpApiError.Unauthorized({}); } + const status = yield* managedEndpointProvider.reconcileOrigin({ + userId: input.userId, + environmentId: input.environmentId, + tunnelId: input.tunnelId, + origin: input.origin, + endpoint: link.endpoint, + }); + if (status === "recovery_required") { + return { status }; + } if (!(yield* allocations.enableRecovery(input))) { return yield* new HttpApiError.Unauthorized({}); } - return { ok: true }; + return { status }; }); export const recoverEnvironmentTunnelRecord = Effect.fn( @@ -617,6 +638,7 @@ export const recoverEnvironmentTunnelRecord = Effect.fn( environmentId: input.environmentId, tunnelId: recoveredTunnelId, environmentPublicKey: input.environmentPublicKey, + origin: input.origin, }); if (!enabled) { const owner = { userId: input.userId, environmentId: input.environmentId }; @@ -1168,30 +1190,40 @@ export const serverApi = HttpApiBuilder.group( return activityHandlers .handle( "registerManagedEndpointRecovery", - Effect.fn("relay.api.server.registerManagedEndpointRecovery")(function* ({ - params, - payload, - }) { - const principal = yield* RelayEnvironmentPrincipal; - if (principal.environmentId !== params.environmentId) { - return yield* new HttpApiError.Unauthorized({}); - } - yield* verifyEnvironmentTunnelRecoveryProof({ - action: "register", - proof: payload.proof, - userId: payload.cloudUserId, - environmentId: params.environmentId, - environmentPublicKey: principal.environmentPublicKey, - tunnelId: payload.tunnelId, - }); - yield* appendRelayCredentialResponseHeaders; - return yield* registerEnvironmentTunnelRecovery({ - userId: payload.cloudUserId, - environmentId: params.environmentId, - environmentPublicKey: principal.environmentPublicKey, - tunnelId: payload.tunnelId, - }); - }, mapRelayCommonApiErrors("not_authorized")), + Effect.fn("relay.api.server.registerManagedEndpointRecovery")( + function* ({ params, payload }) { + const principal = yield* RelayEnvironmentPrincipal; + if (principal.environmentId !== params.environmentId) { + return yield* new HttpApiError.Unauthorized({}); + } + yield* verifyEnvironmentTunnelRecoveryProof({ + action: "register", + proof: payload.proof, + userId: payload.cloudUserId, + environmentId: params.environmentId, + environmentPublicKey: principal.environmentPublicKey, + tunnelId: payload.tunnelId, + origin: payload.origin, + }); + yield* appendRelayCredentialResponseHeaders; + return yield* registerEnvironmentTunnelRecovery({ + userId: payload.cloudUserId, + environmentId: params.environmentId, + environmentPublicKey: principal.environmentPublicKey, + tunnelId: payload.tunnelId, + origin: payload.origin, + }); + }, + Effect.catchTags({ + ManagedEndpointOriginNotAllowed: () => Effect.fail(new HttpApiError.Unauthorized({})), + ManagedEndpointProvisioningNotConfigured: () => + relayInternalErrorResponse("upstream_unavailable"), + ManagedEndpointProvisioningFailed: () => + relayInternalErrorResponse("upstream_unavailable"), + ManagedTunnelLimitExceeded: () => relayInternalErrorResponse("upstream_unavailable"), + }), + mapRelayCommonApiErrors("not_authorized"), + ), ) .handle( "recoverManagedEndpoint", diff --git a/infra/relay/src/persistence/schema.ts b/infra/relay/src/persistence/schema.ts index 88196edc4fd6..4ecc9e8115c6 100644 --- a/infra/relay/src/persistence/schema.ts +++ b/infra/relay/src/persistence/schema.ts @@ -2,6 +2,7 @@ import type { RelayAgentActivityAggregateState, RelayAgentActivityState, RelayAgentAwarenessPreferences, + RelayManagedEndpointOrigin, } from "@t3tools/contracts/relay"; import { boolean, @@ -95,6 +96,7 @@ export const relayManagedEndpointAllocations = pgTable( readyAt: varchar("ready_at", { length: 64 }), recoveryEnabledAt: varchar("recovery_enabled_at", { length: 64 }), recoveryEnvironmentPublicKey: text("recovery_environment_public_key"), + origin: jsonb("origin").$type(), generation: integer("generation").notNull().default(0), createdAt: varchar("created_at", { length: 64 }).notNull(), updatedAt: varchar("updated_at", { length: 64 }).notNull(), diff --git a/infra/relay/src/worker.ts b/infra/relay/src/worker.ts index fe9c085fdea8..54b3467b6054 100644 --- a/infra/relay/src/worker.ts +++ b/infra/relay/src/worker.ts @@ -157,6 +157,7 @@ export const ApiLive = Api.make( yield* yield* relayApiZone.zoneId; const managedEndpointDnsBinding = yield* Cloudflare.DNS.ReadWriteDns(managedEndpointZone); const managedEndpointZoneName = yield* managedEndpointZone.name; + const managedEndpointCleanupMode = yield* RelayConfiguration.managedEndpointCleanupModeConfig; // // 3. Runtime layers and app construction @@ -181,6 +182,7 @@ export const ApiLive = Api.make( cloudMintPublicKey: yield* cloudMintPublicKey, managedEndpointBaseDomain: yield* managedEndpointZoneName, managedEndpointNamespace: stage, + managedEndpointCleanupMode, }); }); @@ -286,7 +288,7 @@ export const ApiLive = Api.make( ), ), ManagedEndpointReaper.ManagedEndpointReaper.pipe( - Effect.flatMap((reaper) => reaper.sweep), + Effect.flatMap((reaper) => reaper.sweep.pipe(Effect.timeout("2 minutes"))), Effect.tap((result) => result.scanned > 0 ? Effect.logInfo("Finished managed tunnel cleanup", result) diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index 8db9dd50aac0..477ca0f2dafd 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -169,11 +169,18 @@ export type RelayManagedEndpointRecoveryRequest = typeof RelayManagedEndpointRec export const RelayManagedEndpointRecoveryRegistrationRequest = Schema.Struct({ cloudUserId: TrimmedNonEmptyString, tunnelId: TrimmedNonEmptyString, + origin: RelayManagedEndpointOrigin, proof: TrimmedNonEmptyString, }); export type RelayManagedEndpointRecoveryRegistrationRequest = typeof RelayManagedEndpointRecoveryRegistrationRequest.Type; +export const RelayManagedEndpointRecoveryRegistrationResponse = Schema.Struct({ + status: Schema.Literals(["ready", "recovery_required"]), +}); +export type RelayManagedEndpointRecoveryRegistrationResponse = + typeof RelayManagedEndpointRecoveryRegistrationResponse.Type; + export const RelayManagedEndpointRecoveryResponse = Schema.Struct({ endpoint: RelayManagedEndpoint, endpointRuntime: RelayManagedEndpointRuntimeConfig, @@ -214,6 +221,7 @@ export const RelayManagedEndpointRecoveryProofPayload = Schema.Union([ environmentId: EnvironmentId, cloudUserId: TrimmedNonEmptyString, tunnelId: TrimmedNonEmptyString, + origin: RelayManagedEndpointOrigin, }), Schema.Struct({ ...RelaySignedJwtRegisteredClaims, @@ -1101,7 +1109,7 @@ export const RelayServerGroup = HttpApiGroup.make("server") environmentId: EnvironmentId, }), payload: RelayManagedEndpointRecoveryRegistrationRequest, - success: RelayOkResponse, + success: RelayManagedEndpointRecoveryRegistrationResponse, error: RelayAuthAndInternalErrors, }, ).annotate(OpenApi.Summary, "Register managed tunnel recovery without provisioning"), From e02577fd8646508942547f6e1928b899121f2406 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Fri, 28 Aug 2026 02:13:13 -0700 Subject: [PATCH 12/14] fix(server): follow Effect error conventions --- apps/server/src/cloud/http.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index b6e8ec58c865..54cdb4a12c5a 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -711,14 +711,14 @@ const cloudRelayConfigHandler = Effect.fn("environment.cloud.relayConfig")( ServerSecretStore.isSecretStoreError, failEnvironmentCloudInternalError("Could not persist environment relay configuration."), ), - Effect.catchTag( - "SchemaError", - failEnvironmentCloudInternalError("Could not persist environment relay configuration."), - ), - Effect.catchTag( - "PlatformError", - failEnvironmentCloudInternalError("Could not register the managed endpoint origin."), - ), + Effect.catchTags({ + SchemaError: failEnvironmentCloudInternalError( + "Could not persist environment relay configuration.", + ), + PlatformError: failEnvironmentCloudInternalError( + "Could not register the managed endpoint origin.", + ), + }), ); const relayClientRequest = ( From dcfdd3d7893f83e48371d79b173757ca9e4b92b0 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Fri, 28 Aug 2026 02:19:11 -0700 Subject: [PATCH 13/14] docs(relay): complete tunnel cleanup canary --- docs/operations/release.md | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/docs/operations/release.md b/docs/operations/release.md index 9fd6041074ee..e6b6a2d29096 100644 --- a/docs/operations/release.md +++ b/docs/operations/release.md @@ -136,20 +136,31 @@ a disposable relay stage, test Cloudflare account, disposable host, disposable T environment link. Keep production cleanup at `off` or `dry-run` until this canary passes. Do not stop a daily-use T3 server or disable the whole machine's network. -1. Deploy the disposable relay stage with cleanup set to `dry-run`. Link the disposable environment - and confirm that its tunnel is healthy and recovery is registered. -2. Capture the PID of that environment's managed `cloudflared` child from its server logs. Confirm - that the PID belongs to the disposable T3 process. -3. Pause only that captured child with `kill -STOP `. Wait until Cloudflare reports the tunnel +1. Deploy the disposable relay stage with cleanup set to `dry-run`. Link the first disposable + environment through web or mobile settings, then confirm that its tunnel is healthy and recovery + is registered. +2. Stop the first test host and restart the same T3 home on a different local port. A web or mobile + link preserves its existing tunnel during this restart, so this exercises origin reconciliation. + Confirm the public hostname reaches the new port after registration. Keep the prior port closed or + attach a request logger to it, and confirm the public hostname does not send requests to that port. +3. Link a second disposable environment with a server build that predates recovery registration. + Capture its managed `cloudflared` child PID and confirm that it belongs to the second test host. + Pause only that child with `kill -STOP ` while its T3 server stays running. Wait until + Cloudflare reports the legacy tunnel as down for more than five minutes. +4. Capture the PID of the first environment's managed `cloudflared` child from its server logs. + Confirm that the PID belongs to the first test host. +5. Pause only that captured child with `kill -STOP `. Wait until Cloudflare reports the tunnel as down for more than five minutes. -4. Confirm that dry-run logs count the tunnel in `wouldDelete` without deleting it. -5. Set cleanup to `enabled` on the disposable relay stage and deploy it. Confirm through the test - Cloudflare account that the old tunnel is deleted. Allow up to ten minutes plus any reported - backlog. -6. Resume only the captured child with `kill -CONT `. Confirm that the running server detects +6. Confirm that dry-run logs count the first tunnel in `wouldDelete` without deleting it. Confirm the + second tunnel remains `skippedLegacy`. +7. Set cleanup to `enabled` on the disposable relay stage and deploy it. Confirm through the test + Cloudflare account that the first tunnel is deleted and the legacy tunnel still exists. Allow up + to ten minutes plus any reported backlog. +8. Resume only the first child with `kill -CONT `. Confirm that the running server detects repeated tunnel authorization rejection, requests recovery, starts a replacement tunnel, and becomes reachable at the same public hostname without a server restart. -7. Repeat with a physical sleep and wake cycle on a disposable laptop before broad rollout. +9. Resume the legacy child with `kill -CONT ` and confirm its original tunnel reconnects. +10. Repeat with a physical sleep and wake cycle on a disposable laptop before broad rollout. To roll back, set cleanup to `off` and deploy the relay before downgrading any host. Keep the recovery endpoints deployed while current server builds are in use. Downgrading a host that has registered From 185a29f7ba7fcf509cf7954bae9859ec40d64939 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Fri, 28 Aug 2026 02:35:51 -0700 Subject: [PATCH 14/14] fix(server): close managed tunnel recovery gaps --- apps/server/src/cloud/http.test.ts | 122 +++++++++++++++++++++++++++-- apps/server/src/cloud/http.ts | 105 +++++++++++++------------ apps/server/src/server.test.ts | 78 ++++++++++++++++++ docs/internals/t3-connect.md | 15 ++-- 4 files changed, 257 insertions(+), 63 deletions(-) diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index 4c260e0554d7..26c2bb3c2a52 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -1,6 +1,8 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; @@ -9,6 +11,7 @@ import * as PlatformError from "effect/PlatformError"; import * as Schema from "effect/Schema"; import * as Tracer from "effect/Tracer"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import { HttpClient, HttpClientResponse, @@ -48,6 +51,7 @@ import { isSupportedLinkProviderKind, linkProofScopes, pendingServiceUpdateExists, + parseManagedEndpointLocalOrigin, reconcileDesiredCloudLink, reconcileDesiredCloudLinkIfStillDesired, recoverManagedCloudTunnel, @@ -55,6 +59,7 @@ import { releaseManagedTunnelOnShutdown, startManagedCloudTunnelIfOriginConfirmed, } from "./http.ts"; +import { managedTunnelStartupAction } from "./managedTunnelStartup.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; import { traceAuthenticatedRelayRequest, traceRelayRequest } from "./traceRelayRequest.ts"; @@ -256,6 +261,39 @@ describe("reconcileDesiredCloudLink", () => { ); }); +describe("parseManagedEndpointLocalOrigin", () => { + it.each([ + { + input: "http://127.0.0.1:80", + httpBaseUrl: "http://127.0.0.1", + wsBaseUrl: "ws://127.0.0.1", + port: 80, + }, + { + input: "https://127.0.0.1:443", + httpBaseUrl: "https://127.0.0.1", + wsBaseUrl: "wss://127.0.0.1", + port: 443, + }, + ])("accepts an explicit default port in $input", ({ input, httpBaseUrl, wsBaseUrl, port }) => { + expect(parseManagedEndpointLocalOrigin(input)).toEqual({ + httpBaseUrl, + wsBaseUrl, + origin: { localHttpHost: "127.0.0.1", localHttpPort: port }, + }); + }); + + it.each([ + "ftp://127.0.0.1:3773", + "http://user:password@127.0.0.1:3773", + "http://127.0.0.1:3773/api", + "http://127.0.0.1:3773?mode=test", + "http://127.0.0.1:3773#fragment", + ])("rejects non-origin URL %s", (input) => { + expect(() => parseManagedEndpointLocalOrigin(input)).toThrow("Invalid local origin"); + }); +}); + describe("releaseManagedTunnelOnShutdown", () => { const cliToken: CliTokenManager.PersistedToken = { accessToken: "cli-access-token", @@ -290,7 +328,9 @@ describe("releaseManagedTunnelOnShutdown", () => { readonly store: ServerSecretStore.ServerSecretStore["Service"]; readonly applyConfigCalls: Array; readonly requests: Array; + readonly onRequest?: (request: HttpClientRequest.HttpClientRequest) => Effect.Effect; readonly respond?: () => Response; + readonly respondEffect?: Effect.Effect; } // Writes the launcher's durable state file into this test's baseDir with @@ -362,11 +402,14 @@ describe("releaseManagedTunnelOnShutdown", () => { HttpClient.make((request) => Effect.sync(() => { harness.requests.push(request); - return HttpClientResponse.fromWeb( - request, - (harness.respond ?? (() => Response.json({ ok: true })))(), - ); - }), + }).pipe( + Effect.andThen(harness.onRequest?.(request) ?? Effect.void), + Effect.andThen( + harness.respondEffect ?? + Effect.sync(() => (harness.respond ?? (() => Response.json({ ok: true })))()), + ), + Effect.map((response) => HttpClientResponse.fromWeb(request, response)), + ), ), ), // The release consults the launcher state file under the configured @@ -775,7 +818,7 @@ describe("releaseManagedTunnelOnShutdown", () => { }, ); - it.effect("does not register recovery without a recorded tunnel ID", () => { + it.effect("requests startup recovery for a legacy config without a recorded tunnel ID", () => { const { store } = makeMemorySecretStore([ [ CLOUD_ENDPOINT_RUNTIME_CONFIG, @@ -789,8 +832,19 @@ describe("releaseManagedTunnelOnShutdown", () => { const requests: Array = []; return Effect.gen(function* () { - expect(yield* registerManagedCloudTunnelRecovery("http://127.0.0.1:3773")).toEqual({ - status: "not_linked", + const registration = yield* registerManagedCloudTunnelRecovery("http://127.0.0.1:3773"); + expect(registration).toEqual({ + status: "recovery_required", + config: { providerKind: "cloudflare_tunnel", connectorToken: "token" }, + }); + expect( + managedTunnelStartupAction({ + wantsCliLink: false, + registration, + }), + ).toEqual({ + action: "request_recovery", + config: { providerKind: "cloudflare_tunnel", connectorToken: "token" }, }); expect(requests).toEqual([]); expect(applyConfigCalls).toEqual([]); @@ -844,6 +898,58 @@ describe("releaseManagedTunnelOnShutdown", () => { ); }); + it.effect("allows managed tunnel provisioning to take longer than ten seconds", () => + Effect.gen(function* () { + const oldConfig = + '{"providerKind":"cloudflare_tunnel","connectorToken":"old-token","tunnelId":"old-tunnel"}'; + const nextConfig = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "new-token", + tunnelId: "new-tunnel", + }; + const { store } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, oldConfig], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + const requestStarted = yield* Deferred.make(); + const response = yield* Deferred.make(); + const recovery = yield* recoverManagedCloudTunnel("http://127.0.0.1:3773").pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + onRequest: () => Deferred.succeed(requestStarted, undefined), + respondEffect: Deferred.await(response), + }), + Effect.forkChild({ startImmediately: true }), + ); + + yield* Deferred.await(requestStarted); + expect(requests).toHaveLength(1); + yield* TestClock.adjust("11 seconds"); + yield* Effect.yieldNow; + yield* Deferred.succeed( + response, + Response.json({ + endpoint: { + httpBaseUrl: "https://environment.example.test/", + wsBaseUrl: "wss://environment.example.test/ws", + providerKind: "cloudflare_tunnel", + }, + endpointRuntime: nextConfig, + }), + ); + + expect(yield* Fiber.join(recovery)).toBe(true); + expect(requests).toHaveLength(1); + expect(applyConfigCalls).toEqual([nextConfig]); + }), + ); + it.effect("does not recover an environment without a managed tunnel credential", () => { const { store } = makeMemorySecretStore([ [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"], diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index 54cdb4a12c5a..7c044bfc005b 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -103,6 +103,7 @@ const CLOUD_HEALTH_NONCE_PREFIX = "cloud-health-nonce-"; const CLOUD_HEALTH_JTI_PREFIX = "cloud-health-jti-"; const CLOUD_PROOF_MAX_LIFETIME_SECONDS = 5 * 60; const CLOUD_PROOF_CLOCK_SKEW_SECONDS = 60; +const MANAGED_ENDPOINT_PROVISION_REQUEST_TIMEOUT = Duration.minutes(2); const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "::1", "localhost"]); const CLOUD_CREDENTIAL_RESPONSE_HEADERS = { "cache-control": "no-store", @@ -307,6 +308,33 @@ function endpointRequestPort(url: URL): number { return Number(url.port || (url.protocol === "https:" ? 443 : 80)); } +export function parseManagedEndpointLocalOrigin(localOrigin: string) { + const url = new URL(localOrigin); + if ( + localOrigin !== localOrigin.trim() || + (url.protocol !== "http:" && url.protocol !== "https:") || + url.username !== "" || + url.password !== "" || + url.pathname !== "/" || + url.search !== "" || + url.hash !== "" || + localOrigin.includes("?") || + localOrigin.includes("#") + ) { + throw new Error("Invalid local origin"); + } + const wsUrl = new URL(url.origin); + wsUrl.protocol = url.protocol === "https:" ? "wss:" : "ws:"; + return { + httpBaseUrl: url.origin, + wsBaseUrl: wsUrl.origin, + origin: { + localHttpHost: url.hostname, + localHttpPort: endpointRequestPort(url), + } satisfies RelayManagedEndpointOrigin, + }; +} + function isAllowedEndpointOrigin(input: { readonly origin: RelayManagedEndpointOrigin; readonly requestUrl: string; @@ -461,17 +489,6 @@ const cloudLinkProofHandler = Effect.fn("environment.cloud.linkProof")( ), ); -function managedEndpointOriginFromLocalUrl(localOrigin: string): RelayManagedEndpointOrigin { - const localUrl = new URL(localOrigin); - if (localUrl.origin !== localOrigin) { - throw new Error("Invalid local origin"); - } - return { - localHttpHost: localUrl.hostname, - localHttpPort: endpointRequestPort(localUrl), - }; -} - function managedEndpointRuntimeConfigsMatch( left: RelayManagedEndpointRuntimeConfig, right: RelayManagedEndpointRuntimeConfig, @@ -548,8 +565,8 @@ export const startManagedCloudTunnelIfOriginConfirmed = Effect.fn( "environment.cloud.startManagedCloudTunnelIfOriginConfirmed", )(function* (localOrigin: string) { const dependencies = yield* cloudHttpDependencies; - const origin = yield* Effect.try({ - try: () => managedEndpointOriginFromLocalUrl(localOrigin), + const parsedOrigin = yield* Effect.try({ + try: () => parseManagedEndpointLocalOrigin(localOrigin), catch: () => new EnvironmentHttpBadRequestError({ message: "Could not resolve local environment origin.", @@ -568,8 +585,8 @@ export const startManagedCloudTunnelIfOriginConfirmed = Effect.fn( config === null || marker === null || !managedEndpointRuntimeConfigsMatch(marker.config, config) || - marker.origin.localHttpHost !== origin.localHttpHost || - marker.origin.localHttpPort !== origin.localHttpPort + marker.origin.localHttpHost !== parsedOrigin.origin.localHttpHost || + marker.origin.localHttpPort !== parsedOrigin.origin.localHttpPort ) { return false; } @@ -691,6 +708,9 @@ const cloudRelayConfigHandler = Effect.fn("environment.cloud.relayConfig")( message: "The managed tunnel configuration changed during registration.", }); } + if (registration.status === "recovery_required") { + yield* dependencies.endpointRuntime.requestRecovery(registration.config); + } if (registration.status !== "ready") { return yield* new EnvironmentCloudEndpointUnavailableError({ message: "Managed endpoint origin could not be confirmed.", @@ -728,6 +748,7 @@ const relayClientRequest = ( readonly token: string; readonly payload: unknown; readonly schema: Schema.Decoder; + readonly timeout?: Duration.Input; }, ) => HttpClientRequest.post(input.url).pipe( @@ -750,7 +771,7 @@ const relayClientRequest = ( }), ), Effect.flatMap(HttpClientResponse.schemaBodyJson(input.schema)), - Effect.timeout("10 seconds"), + Effect.timeout(input.timeout ?? "10 seconds"), Effect.mapError((cause) => cause._tag === "EnvironmentHttpUnauthorizedError" || cause._tag === "EnvironmentHttpConflictError" @@ -764,19 +785,13 @@ const relayClientRequest = ( const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesiredLinkWith")( function* (dependencies: CloudHttpDependencies, localOrigin: string) { - const localUrl = yield* Effect.try({ - try: () => new URL(localOrigin), + const parsedOrigin = yield* Effect.try({ + try: () => parseManagedEndpointLocalOrigin(localOrigin), catch: () => new EnvironmentHttpBadRequestError({ message: "Could not resolve local environment origin.", }), }); - if (localUrl.origin !== localOrigin) { - return yield* new EnvironmentHttpBadRequestError({ - message: "Could not resolve local environment origin.", - }); - } - const localWsOrigin = localOrigin.replace(/^http/u, "ws"); const token = yield* dependencies.cliTokenManager.getExisting.pipe( Effect.flatMap( Option.match({ @@ -809,16 +824,13 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi challenge: challenge.challenge, relayIssuer: relayUrl, endpoint: { - httpBaseUrl: localOrigin, - wsBaseUrl: localWsOrigin, + httpBaseUrl: parsedOrigin.httpBaseUrl, + wsBaseUrl: parsedOrigin.wsBaseUrl, providerKind: managedTunnelsEnabled ? "cloudflare_tunnel" : "manual", }, - origin: { - localHttpHost: localUrl.hostname, - localHttpPort: endpointRequestPort(localUrl), - }, + origin: parsedOrigin.origin, }, - localOrigin, + parsedOrigin.httpBaseUrl, ); const link = yield* relayClientRequest(dependencies, { url: `${relayUrl}/v1/client/environment-links`, @@ -830,6 +842,7 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi managedTunnelsEnabled, }, schema: RelayEnvironmentLinkResponse, + timeout: MANAGED_ENDPOINT_PROVISION_REQUEST_TIMEOUT, }); yield* setCliDesiredCloudLink(true, mode); return yield* applyCloudRelayConfig( @@ -844,10 +857,7 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi }, { lockHeld: true, - confirmedOrigin: { - localHttpHost: localUrl.hostname, - localHttpPort: endpointRequestPort(localUrl), - }, + confirmedOrigin: parsedOrigin.origin, }, ); }, @@ -965,17 +975,21 @@ export const registerManagedCloudTunnelRecovery = Effect.fn( } const config = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeConfig.value))); - if (config?.providerKind !== "cloudflare_tunnel" || config.tunnelId === undefined) { + if (config?.providerKind !== "cloudflare_tunnel") { return { status: "not_linked" as const }; } - const origin = yield* Effect.try({ - try: () => managedEndpointOriginFromLocalUrl(localOrigin), + const parsedOrigin = yield* Effect.try({ + try: () => parseManagedEndpointLocalOrigin(localOrigin), catch: () => new EnvironmentHttpBadRequestError({ message: "Could not resolve local environment origin.", }), }); + if (config.tunnelId === undefined) { + return { status: "recovery_required" as const, config }; + } + const origin = parsedOrigin.origin; const environmentId = yield* dependencies.environment.getEnvironmentId; const relayUrlValue = bytesToString(relayUrl.value); const cloudUserIdValue = bytesToString(cloudUserId.value); @@ -1049,26 +1063,18 @@ export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverMan } } - const localUrl = yield* Effect.try({ - try: () => new URL(localOrigin), + const parsedOrigin = yield* Effect.try({ + try: () => parseManagedEndpointLocalOrigin(localOrigin), catch: () => new EnvironmentHttpBadRequestError({ message: "Could not resolve local environment origin.", }), }); - if (localUrl.origin !== localOrigin) { - return yield* new EnvironmentHttpBadRequestError({ - message: "Could not resolve local environment origin.", - }); - } const environmentId = yield* dependencies.environment.getEnvironmentId; const relayUrlValue = bytesToString(relayUrl.value); const cloudUserIdValue = bytesToString(cloudUserId.value); - const origin = { - localHttpHost: localUrl.hostname, - localHttpPort: endpointRequestPort(localUrl), - }; + const origin = parsedOrigin.origin; const proof = yield* makeManagedTunnelRecoveryProof(dependencies, { action: "recover", environmentId, @@ -1085,6 +1091,7 @@ export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverMan proof, }, schema: RelayManagedEndpointRecoveryResponse, + timeout: MANAGED_ENDPOINT_PROVISION_REQUEST_TIMEOUT, }); if (recovered.endpointRuntime.providerKind !== "cloudflare_tunnel") { return yield* new EnvironmentHttpInternalServerError({ diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index fd9deee5675e..02faafe797f6 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -3057,6 +3057,84 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + it.effect( + "queues recovery without starting a connector when relay registration requires it", + () => + Effect.gen(function* () { + const appliedRuntimeConfigs: Array = []; + const requestedRecoveryConfigs: Array = []; + const relayRequests: Array = []; + yield* buildAppUnderTest({ + layers: { + cloudManagedEndpointRuntime: { + applyConfig: (config) => + Effect.sync(() => { + appliedRuntimeConfigs.push(config); + return config === null + ? ({ status: "disabled" } as const) + : ({ status: "running", providerKind: "cloudflare_tunnel", pid: 123 } as const); + }), + requestRecovery: (config) => + Effect.sync(() => { + requestedRecoveryConfigs.push(config); + }), + }, + httpClient: HttpClient.make((request) => + Effect.sync(() => { + relayRequests.push(request); + return HttpClientResponse.fromWeb( + request, + Response.json({ status: "recovery_required" }), + ); + }), + ), + }, + }); + + const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, + }); + const ownerCookie = yield* getAuthenticatedSessionCookieHeader(); + const relayConfigUrl = yield* getHttpServerUrl("/api/connect/relay-config"); + const relayConfigResponse = yield* fetchEffect(relayConfigUrl, { + method: "POST", + headers: { + cookie: ownerCookie, + "content-type": "application/json", + }, + body: jsonRequestBody({ + relayUrl: "https://relay.example.test", + cloudUserId: "user_123", + environmentCredential: "t3env_test_credential", + cloudMintPublicKey: cloudKeyPair.publicKey, + endpointRuntime: { + providerKind: "cloudflare_tunnel", + connectorToken: "connector-token", + tunnelId: "tunnel-1", + }, + }), + }); + const relayConfigBody = yield* responseJsonEffect<{ + readonly _tag?: string; + readonly endpointRuntimeStatus?: { readonly status?: string }; + }>(relayConfigResponse); + + assert.equal(relayConfigResponse.status, 503); + assert.equal(relayConfigBody._tag, "EnvironmentCloudEndpointUnavailableError"); + assert.equal(relayConfigBody.endpointRuntimeStatus?.status, "disabled"); + assert.equal(relayRequests.length, 1); + assert.deepEqual(appliedRuntimeConfigs, [null]); + assert.deepEqual(requestedRecoveryConfigs, [ + { + providerKind: "cloudflare_tunnel", + connectorToken: "connector-token", + tunnelId: "tunnel-1", + }, + ]); + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ); + it.effect("fails relay config when the managed endpoint connector cannot start", () => Effect.gen(function* () { const appliedRuntimeConfigs: Array = []; diff --git a/docs/internals/t3-connect.md b/docs/internals/t3-connect.md index a02b032b44a2..1f400efe6bc6 100644 --- a/docs/internals/t3-connect.md +++ b/docs/internals/t3-connect.md @@ -165,12 +165,15 @@ limit. The whole sweep has a two-minute deadline and stops early after a structu limit response. Page rotation and alternating `down` and `inactive` priority keep large backlogs moving across later sweeps. -Cleanup only deletes tunnels for hosts that registered recovery with the public key on their current -environment link. Older hosts remain untouched. It also skips incomplete allocations with no -recorded tunnel ID because provisioning can be between tunnel creation and the database update. -Allocations that point at another tunnel ID are skipped for the same ownership reason. These skips -can leave stale tunnels for manual cleanup, but they prevent a cleanup sweep from deleting a tunnel -that a concurrent provision owns. +For tunnels with existing allocation records, cleanup only deletes tunnels for hosts that registered +recovery with the public key on their current environment link. Older allocation records remain +untouched. The reaper treats an expired same-namespace tunnel with no allocation record as an orphan +because legacy-host protection requires an allocation record. It rechecks the tunnel's Cloudflare +status and deletes it after the inactivity grace period. It still skips incomplete allocations with +no recorded tunnel ID because provisioning can be between tunnel creation and the database update. +Allocations that point at another tunnel ID are skipped for the same ownership reason. These +allocation skips can leave stale tunnels for manual cleanup, but they prevent a cleanup sweep from +deleting a tunnel that a concurrent provision owns. The background service has an independent lifecycle. Connect setup may offer to install it, but logout leaves it running; manage it with `t3 service status`, `install`, `update`, and `uninstall`.