diff --git a/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts b/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts index 83189e1ee0af..ff2410c629f3 100644 --- a/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts +++ b/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts @@ -989,6 +989,116 @@ describe("DesktopBackendConfiguration", () => { }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); + it.effect("resolveWsl forwards the standard OTEL variables into the distro", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-backend-config-test-", + }); + + const previousWslEnv = process.env.WSLENV; + const previousEndpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT; + const previousHeaders = process.env.OTEL_EXPORTER_OTLP_HEADERS; + const previousTemporality = process.env.OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE; + try { + // The bootstrap carries the resolved URLs but nothing else these + // variables say, and it is the lowest-priority source. Without the + // names crossing too, a Windows machine would reach the collector + // inside the distro unauthenticated, in the wrong wire format, and + // with an aggregation the receiver drops. + delete process.env.WSLENV; + process.env.OTEL_EXPORTER_OTLP_ENDPOINT = "https://collector.example.com"; + process.env.OTEL_EXPORTER_OTLP_HEADERS = "authorization=Bearer%20ambient"; + process.env.OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "delta"; + + yield* Effect.gen(function* () { + const configuration = yield* DesktopBackendConfiguration.DesktopBackendConfiguration; + const config = yield* configuration.resolveWsl({ port: 5050, distro: null }); + + assert.equal(config.env.OTEL_EXPORTER_OTLP_ENDPOINT, "https://collector.example.com"); + const declared = (config.env.WSLENV ?? "").split(":"); + assert.include(declared, "OTEL_EXPORTER_OTLP_ENDPOINT"); + assert.include(declared, "OTEL_EXPORTER_OTLP_HEADERS"); + assert.include(declared, "OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE"); + // A bare entry crosses verbatim. A path flag would rewrite a URL. + assert.notInclude(config.env.WSLENV ?? "", "OTEL_EXPORTER_OTLP_ENDPOINT/"); + }).pipe( + Effect.provide( + DesktopBackendConfiguration.layer.pipe( + Layer.provideMerge(serverExposureLayer), + Layer.provideMerge(DesktopAppSettings.layerTest()), + Layer.provideMerge(DesktopWslServerTree.layerTest()), + Layer.provideMerge( + DesktopWslEnvironment.layerTest({ + isAvailable: true, + windowsToWslPath: () => Option.some("/mnt/c/repo/apps/server/src/index.ts"), + getDistroIp: () => Option.some("172.27.0.99"), + }), + ), + Layer.provideMerge(makeEnvironmentLayer(baseDir, { platform: "win32" })), + ), + ), + ); + } finally { + restoreEnv("WSLENV", previousWslEnv); + restoreEnv("OTEL_EXPORTER_OTLP_ENDPOINT", previousEndpoint); + restoreEnv("OTEL_EXPORTER_OTLP_HEADERS", previousHeaders); + restoreEnv("OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE", previousTemporality); + } + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("resolveWsl forwards T3 Code's own endpoint under its own name", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-backend-config-test-", + }); + + const previousWslEnv = process.env.WSLENV; + const previousTracesUrl = process.env.T3CODE_OTLP_TRACES_URL; + const previousEndpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT; + try { + // The bootstrap carries this URL too, but it cannot say which variable + // put it there, and the bootstrap is the lowest-priority source. Only + // the name crossing keeps T3 Code's own variable outranking an ambient + // endpoint inside the distro the way it does everywhere else. + delete process.env.WSLENV; + process.env.T3CODE_OTLP_TRACES_URL = "http://localhost:4318/v1/traces"; + process.env.OTEL_EXPORTER_OTLP_ENDPOINT = "https://collector.example.com"; + + yield* Effect.gen(function* () { + const configuration = yield* DesktopBackendConfiguration.DesktopBackendConfiguration; + const config = yield* configuration.resolveWsl({ port: 5050, distro: null }); + + assert.equal(config.env.T3CODE_OTLP_TRACES_URL, "http://localhost:4318/v1/traces"); + assert.include((config.env.WSLENV ?? "").split(":"), "T3CODE_OTLP_TRACES_URL"); + assert.notInclude(config.env.WSLENV ?? "", "T3CODE_OTLP_TRACES_URL/"); + }).pipe( + Effect.provide( + DesktopBackendConfiguration.layer.pipe( + Layer.provideMerge(serverExposureLayer), + Layer.provideMerge(DesktopAppSettings.layerTest()), + Layer.provideMerge(DesktopWslServerTree.layerTest()), + Layer.provideMerge( + DesktopWslEnvironment.layerTest({ + isAvailable: true, + windowsToWslPath: () => Option.some("/mnt/c/repo/apps/server/src/index.ts"), + getDistroIp: () => Option.some("172.27.0.99"), + }), + ), + Layer.provideMerge(makeEnvironmentLayer(baseDir, { platform: "win32" })), + ), + ), + ); + } finally { + restoreEnv("WSLENV", previousWslEnv); + restoreEnv("T3CODE_OTLP_TRACES_URL", previousTracesUrl); + restoreEnv("OTEL_EXPORTER_OTLP_ENDPOINT", previousEndpoint); + } + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + it.effect("resolveWsl preserves existing WSLENV entries when forwarding backend secrets", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -1001,7 +1111,10 @@ describe("DesktopBackendConfiguration", () => { const previousAnthropicKey = process.env.ANTHROPIC_API_KEY; const previousOtlpHeaders = process.env.T3CODE_OTLP_HEADERS; const previousOtlpProtocol = process.env.T3CODE_OTLP_PROTOCOL; + // A developer's own OTEL_* variables would be forwarded too. + const ambientOtel = Object.entries(process.env).filter(([name]) => name.startsWith("OTEL_")); try { + for (const [name] of ambientOtel) delete process.env[name]; process.env.WSLENV = "GOPATH/p:OPENAI_API_KEY/u:EMPTY::AZURE_DEVOPS_EXT_PAT/u"; process.env.OPENAI_API_KEY = "openai-key"; process.env.ANTHROPIC_API_KEY = "anthropic-key"; @@ -1058,6 +1171,7 @@ describe("DesktopBackendConfiguration", () => { restoreEnv("ANTHROPIC_API_KEY", previousAnthropicKey); restoreEnv("T3CODE_OTLP_HEADERS", previousOtlpHeaders); restoreEnv("T3CODE_OTLP_PROTOCOL", previousOtlpProtocol); + for (const [name, value] of ambientOtel) restoreEnv(name, value); } }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); diff --git a/apps/desktop/src/backend/DesktopBackendConfiguration.ts b/apps/desktop/src/backend/DesktopBackendConfiguration.ts index ff693d1ac9e3..df4af85e05f7 100644 --- a/apps/desktop/src/backend/DesktopBackendConfiguration.ts +++ b/apps/desktop/src/backend/DesktopBackendConfiguration.ts @@ -89,18 +89,55 @@ const DESKTOP_BACKEND_ENV_NAMES = [ "T3CODE_TAILSCALE_SERVE_PORT", ] as const; +// Every name the server reads to decide what it exports and where. Forwarded +// under their own names, not folded into the bootstrap envelope, so precedence +// inside a WSL distro is the same as on every other platform and a collector's +// headers and wire format travel with its endpoint. Declared in WSLENV without +// a flag, which is what makes URL-shaped values safe: only a `/p`, `/l`, `/u`, +// or `/w` entry is path-translated. +const OBSERVABILITY_FORWARDED_ENV_NAMES = [ + "T3CODE_OTEL_SDK_DISABLED", + "T3CODE_OTLP_TRACES_URL", + "T3CODE_OTLP_METRICS_URL", + "T3CODE_OTLP_LOGS_URL", + "T3CODE_OTLP_HEADERS", + "T3CODE_OTLP_PROTOCOL", + "T3CODE_OTLP_EXPORT_INTERVAL_MS", + "T3CODE_OTLP_SERVICE_NAME", + "OTEL_SDK_DISABLED", + "OTEL_EXPORTER_OTLP_ENDPOINT", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", + "OTEL_EXPORTER_OTLP_HEADERS", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS", + "OTEL_EXPORTER_OTLP_METRICS_HEADERS", + "OTEL_EXPORTER_OTLP_LOGS_HEADERS", + "OTEL_EXPORTER_OTLP_PROTOCOL", + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", + "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL", + "OTEL_EXPORTER_OTLP_LOGS_PROTOCOL", + "OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE", + "OTEL_TRACES_EXPORTER", + "OTEL_METRICS_EXPORTER", + "OTEL_LOGS_EXPORTER", + "OTEL_BSP_SCHEDULE_DELAY", + "OTEL_BSP_MAX_EXPORT_BATCH_SIZE", + "OTEL_BLRP_SCHEDULE_DELAY", + "OTEL_BLRP_MAX_EXPORT_BATCH_SIZE", + "OTEL_METRIC_EXPORT_INTERVAL", + "OTEL_SERVICE_NAME", + "OTEL_SERVICE_VERSION", + "OTEL_RESOURCE_ATTRIBUTES", +] as const; + // Env vars that the WSL backend needs but Windows process.env won't forward // across the wsl.exe boundary without WSLENV. The dev-server URL is handled -// separately via a `--dev-url` CLI flag because WSLENV translation of -// URL-shaped values (colons / slashes) is unreliable. +// separately via a `--dev-url` CLI flag. const WSL_FORWARDED_ENV_NAMES = [ "OPENAI_API_KEY", "ANTHROPIC_API_KEY", - // Otherwise the WSL server keeps exporting to endpoints from the bootstrap. - "T3CODE_OTEL_SDK_DISABLED", - "OTEL_SDK_DISABLED", - "T3CODE_OTLP_HEADERS", - "T3CODE_OTLP_PROTOCOL", + ...OBSERVABILITY_FORWARDED_ENV_NAMES, ] as const; const WSL_SERVER_SYSTEM_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; @@ -221,12 +258,12 @@ const readPersistedBackendObservabilitySettings = Effect.gen(function* () { }; }); -// The bootstrap is the only channel that carries an OTLP endpoint to every -// backend. A Windows-native child inherits the desktop process's env, but a -// WSL child gets nothing across wsl.exe that WSLENV does not declare, and -// WSLENV translation of URL-shaped values is unreliable, so the endpoints are -// deliberately not forwarded that way. Env beats the persisted settings file, -// matching the precedence resolveServerConfig and DesktopObservability apply. +// The bootstrap envelope is the channel that carries an endpoint to every +// backend whatever its platform, and it is the lowest-priority source the +// server consults. Env beats the persisted settings file here, matching the +// precedence resolveServerConfig and DesktopObservability apply. The variables +// themselves reach a WSL backend under their own names through WSLENV, which is +// what keeps that precedence identical inside the distro. const readBackendObservabilitySettings = Effect.gen(function* () { const environment = yield* DesktopEnvironment.DesktopEnvironment; const persisted = yield* readPersistedBackendObservabilitySettings; @@ -733,10 +770,8 @@ const resolveWslStartConfig = Effect.fn("desktop.backendConfiguration.resolveWsl }; // Forward the dev-server URL as an explicit CLI flag so the WSL backend's - // config resolution lands in dev/ instead of userdata/. Inheriting through - // WSLENV is unreliable in practice (URL-shaped values with colons / - // slashes get translated unpredictably depending on flags), and the - // packaged build leaves devServerUrl as None anyway. + // config resolution lands in dev/ instead of userdata/. The packaged build + // leaves devServerUrl as None anyway. const devUrlArgs = Option.match(environment.devServerUrl, { onNone: () => [] as ReadonlyArray, onSome: (url) => ["--dev-url", url.href], diff --git a/apps/server/src/cli/config.test.ts b/apps/server/src/cli/config.test.ts index f9b45caf7442..9fa223c61cae 100644 --- a/apps/server/src/cli/config.test.ts +++ b/apps/server/src/cli/config.test.ts @@ -1,6 +1,7 @@ // @effect-diagnostics nodeBuiltinImport:off import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; import { assert, expect, it } from "@effect/vitest"; import * as ConfigProvider from "effect/ConfigProvider"; @@ -596,6 +597,223 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => { }), ); + // Resolving a config reads the settings file and creates the trace + // directory, so a shared home would let one case see another's writes and + // would race when these run in parallel. + const resolveWithEnv = (env: Record) => { + const baseDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3code-otel-config-")); + return resolveServerConfig( + { + mode: Option.some("web"), + port: Option.some(4888), + host: Option.none(), + baseDir: Option.some(baseDir), + cwd: Option.none(), + devUrl: Option.none(), + noBrowser: Option.none(), + bootstrapFd: Option.none(), + autoBootstrapProjectFromCwd: Option.none(), + logWebSocketEvents: Option.none(), + tailscaleServeEnabled: Option.none(), + tailscaleServePort: Option.none(), + }, + Option.none(), + ).pipe( + Effect.provide( + Layer.mergeAll(ConfigProvider.layer(ConfigProvider.fromEnv({ env })), NetService.layer), + ), + Effect.ensuring( + Effect.sync(() => { + NodeFS.rmSync(baseDir, { recursive: true, force: true }); + }), + ), + ); + }; + + it.effect("exports to the endpoint the rest of the machine already uses", () => + Effect.gen(function* () { + const resolved = yield* resolveWithEnv({ + OTEL_EXPORTER_OTLP_ENDPOINT: "https://collector.example.com", + OTEL_SERVICE_NAME: "t3", + }); + + expect(resolved.otlpTracesUrl).toBe("https://collector.example.com/v1/traces"); + expect(resolved.otlpMetricsUrl).toBe("https://collector.example.com/v1/metrics"); + expect(resolved.otlpLogsUrl).toBe("https://collector.example.com/v1/logs"); + // The endpoint is the machine's to name. The service is not. + expect(resolved.otlpServiceName).toBe("t3-server"); + }), + ); + + it.effect("cannot be renamed by the environment", () => + Effect.gen(function* () { + // A shell profile that names the app it was written for must not decide + // what T3 Code calls itself. + const resolved = yield* resolveWithEnv({ + OTEL_SERVICE_NAME: "some-other-app", + OTEL_RESOURCE_ATTRIBUTES: "service.name=some-other-app", + }); + + expect(resolved.otlpServiceName).toBe("t3-server"); + }), + ); + + it.effect("does not let an empty T3 Code name stand in for an answer", () => + Effect.gen(function* () { + // An empty variable is set without saying anything, so the ambient + // endpoint still answers. + const resolved = yield* resolveWithEnv({ + OTEL_EXPORTER_OTLP_ENDPOINT: "https://collector.example.com", + OTEL_SERVICE_NAME: "t3", + T3CODE_OTLP_TRACES_URL: "", + T3CODE_OTLP_METRICS_URL: " ", + T3CODE_OTLP_LOGS_URL: "", + T3CODE_OTLP_SERVICE_NAME: "", + }); + + expect(resolved.otlpTracesUrl).toBe("https://collector.example.com/v1/traces"); + expect(resolved.otlpMetricsUrl).toBe("https://collector.example.com/v1/metrics"); + expect(resolved.otlpLogsUrl).toBe("https://collector.example.com/v1/logs"); + expect(resolved.otlpServiceName).toBe("t3-server"); + }), + ); + + it.effect("keeps T3 Code's own names as the explicit answer", () => + Effect.gen(function* () { + const resolved = yield* resolveWithEnv({ + OTEL_EXPORTER_OTLP_ENDPOINT: "https://collector.example.com", + OTEL_SERVICE_NAME: "t3", + T3CODE_OTLP_TRACES_URL: "http://localhost:4318/v1/traces", + T3CODE_OTLP_LOGS_URL: "http://localhost:4318/v1/logs", + T3CODE_OTLP_SERVICE_NAME: "t3-local", + }); + + expect(resolved.otlpTracesUrl).toBe("http://localhost:4318/v1/traces"); + expect(resolved.otlpMetricsUrl).toBe("https://collector.example.com/v1/metrics"); + expect(resolved.otlpLogsUrl).toBe("http://localhost:4318/v1/logs"); + expect(resolved.otlpServiceName).toBe("t3-local"); + }), + ); + + it.effect("leaves a T3 Code endpoint alone when the environment names another", () => + Effect.gen(function* () { + // An ambient endpoint that lost the URL keeps its wire format, headers, + // and batching on the endpoint it named. + const resolved = yield* resolveWithEnv({ + OTEL_EXPORTER_OTLP_ENDPOINT: "https://collector.example.com", + T3CODE_OTLP_TRACES_URL: "http://localhost:4318/v1/traces", + }); + + expect(resolved.otelEnvironment.traces.settings).toBeUndefined(); + expect(resolved.otelEnvironment.metrics.settings?.url).toBe( + "https://collector.example.com/v1/metrics", + ); + expect(resolved.otlpTracesExport.exportIntervalMs).toBe(10_000); + }), + ); + + it.effect("keeps an ambient aggregation and schedule off a T3 Code metric endpoint", () => + Effect.gen(function* () { + const resolved = yield* resolveWithEnv({ + OTEL_EXPORTER_OTLP_ENDPOINT: "https://collector.example.com", + OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE: "delta", + T3CODE_OTLP_METRICS_URL: "http://localhost:4318/v1/metrics", + }); + + expect(resolved.otelEnvironment.metrics.settings).toBeUndefined(); + expect(resolved.otelEnvironment.traces.settings?.temporality).toBeUndefined(); + expect(resolved.otlpTracesExport.exportIntervalMs).toBe(5_000); + expect(resolved.otlpMetricsExport.exportIntervalMs).toBe(10_000); + }), + ); + + it.effect("keeps a T3 Code credential off an endpoint the standard variables named", () => + Effect.gen(function* () { + // An `OTEL_*` endpoint that says nothing about headers is asking for + // none, not to borrow the token a T3 Code variable carries. + const resolved = yield* resolveWithEnv({ + OTEL_EXPORTER_OTLP_ENDPOINT: "https://collector.example.com", + T3CODE_OTLP_HEADERS: "authorization=Bearer%20t3-token", + T3CODE_OTLP_PROTOCOL: "http/json", + }); + + expect(resolved.otlpTracesExport.headers).toBeUndefined(); + expect(resolved.otlpTracesExport.protocol).toBe("http/protobuf"); + }), + ); + + it.effect("carries a T3 Code credential to the endpoint T3 Code named", () => + Effect.gen(function* () { + const resolved = yield* resolveWithEnv({ + T3CODE_OTLP_TRACES_URL: "http://localhost:4318/v1/traces", + T3CODE_OTLP_HEADERS: "authorization=Bearer%20t3-token", + }); + + expect(resolved.otlpTracesExport.headers).toEqual({ + authorization: "Bearer t3-token", + }); + expect(resolved.otlpTracesExport.protocol).toBe("http/json"); + }), + ); + + it.effect("keeps a span's schedule off a T3 Code log endpoint", () => + Effect.gen(function* () { + // A log endpoint that came from a T3 Code name keeps T3 Code's interval + // instead of the span delay standing beside the ambient endpoint. + const resolved = yield* resolveWithEnv({ + OTEL_EXPORTER_OTLP_ENDPOINT: "https://collector.example.com", + OTEL_BSP_SCHEDULE_DELAY: "7000", + T3CODE_OTLP_LOGS_URL: "http://localhost:4318/v1/logs", + }); + + expect(resolved.otelEnvironment.logs.settings).toBeUndefined(); + expect(resolved.otlpTracesExport.exportIntervalMs).toBe(7_000); + expect(resolved.otlpLogsExport.exportIntervalMs).toBe(10_000); + }), + ); + + it.effect("does not report a signal as declined while it is exporting", () => + Effect.gen(function* () { + // grpc turns off only the export the variable that named it configures. + const resolved = yield* resolveWithEnv({ + OTEL_EXPORTER_OTLP_ENDPOINT: "https://collector.example.com", + OTEL_EXPORTER_OTLP_PROTOCOL: "grpc", + T3CODE_OTLP_TRACES_URL: "http://localhost:4318/v1/traces", + }); + + expect(resolved.otlpTracesUrl).toBe("http://localhost:4318/v1/traces"); + expect(resolved.otelEnvironment.traces.declined).toBeUndefined(); + expect(resolved.otelEnvironment.metrics.declined).toContain("grpc"); + expect(resolved.otelEnvironment.logs.declined).toContain("grpc"); + }), + ); + + it.effect("exports nothing at all once the SDK is switched off", () => + Effect.gen(function* () { + const resolved = yield* resolveWithEnv({ + OTEL_SDK_DISABLED: "true", + OTEL_EXPORTER_OTLP_ENDPOINT: "https://collector.example.com", + T3CODE_OTLP_TRACES_URL: "http://localhost:4318/v1/traces", + }); + + expect(resolved.otlpTracesUrl).toBeUndefined(); + expect(resolved.otlpMetricsUrl).toBeUndefined(); + expect(resolved.otlpLogsUrl).toBeUndefined(); + }), + ); + + it.effect("keeps exporting when T3 Code's own name says to, whatever the standard one says", () => + Effect.gen(function* () { + const resolved = yield* resolveWithEnv({ + T3CODE_OTEL_SDK_DISABLED: "false", + OTEL_SDK_DISABLED: "true", + T3CODE_OTLP_TRACES_URL: "http://localhost:4318/v1/traces", + }); + + expect(resolved.otlpTracesUrl).toBe("http://localhost:4318/v1/traces"); + }), + ); + it.effect("falls back to persisted observability settings when env vars are absent", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -771,6 +989,236 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => { }), ); + it.effect("does not let a blank bootstrap endpoint hide the stored one", () => + Effect.gen(function* () { + // The desktop sends the envelope whether or not it resolved an endpoint, + // so an empty string means "I found nothing", not "export nowhere". It + // must not stand in front of the Settings endpoint underneath it. + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-cli-config-blank-" }); + const derivedPaths = yield* deriveExplicitServerPaths(baseDir, undefined); + yield* fs.makeDirectory(path.dirname(derivedPaths.settingsPath), { recursive: true }); + yield* fs.writeFileString( + derivedPaths.settingsPath, + // @effect-diagnostics-next-line preferSchemaOverJson:off + `${JSON.stringify({ + observability: { otlpTracesUrl: "http://stored.example.com/v1/traces" }, + })}\n`, + ); + const fd = yield* openBootstrapFd( + makeDesktopBootstrap({ t3Home: baseDir, otlpTracesUrl: "" }), + ); + + const resolved = yield* resolveServerConfig( + { + mode: Option.none(), + port: Option.none(), + host: Option.none(), + baseDir: Option.none(), + cwd: Option.none(), + devUrl: Option.none(), + noBrowser: Option.none(), + bootstrapFd: Option.none(), + autoBootstrapProjectFromCwd: Option.none(), + logWebSocketEvents: Option.none(), + tailscaleServeEnabled: Option.none(), + tailscaleServePort: Option.none(), + }, + Option.none(), + ).pipe( + Effect.provide( + Layer.mergeAll( + ConfigProvider.layer( + ConfigProvider.fromEnv({ env: { T3CODE_BOOTSTRAP_FD: String(fd) } }), + ), + NetService.layer, + ), + ), + ); + + expect(resolved.otlpTracesUrl).toBe("http://stored.example.com/v1/traces"); + }), + ); + + it.effect("reads an exported endpoint before a stored one", () => + Effect.gen(function* () { + // An exported variable is what the operator asked for now; Settings is + // what somebody asked for once. The standard names sit directly under + // T3 Code's own, not under the file, which is the order every setting + // here follows. + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-cli-config-order-" }); + const derivedPaths = yield* deriveExplicitServerPaths(baseDir, undefined); + yield* fs.makeDirectory(path.dirname(derivedPaths.settingsPath), { recursive: true }); + yield* fs.writeFileString( + derivedPaths.settingsPath, + // @effect-diagnostics-next-line preferSchemaOverJson:off + `${JSON.stringify({ + observability: { + otlpTracesUrl: "http://stored.example.com/v1/traces", + otlpMetricsUrl: "http://stored.example.com/v1/metrics", + otlpLogsUrl: "http://stored.example.com/v1/logs", + }, + })}\n`, + ); + + const resolved = yield* resolveServerConfig( + { + mode: Option.some("desktop"), + port: Option.some(4888), + host: Option.none(), + baseDir: Option.some(baseDir), + cwd: Option.none(), + devUrl: Option.none(), + noBrowser: Option.none(), + bootstrapFd: Option.none(), + autoBootstrapProjectFromCwd: Option.none(), + logWebSocketEvents: Option.none(), + tailscaleServeEnabled: Option.none(), + tailscaleServePort: Option.none(), + }, + Option.none(), + ).pipe( + Effect.provide( + Layer.mergeAll( + ConfigProvider.layer( + ConfigProvider.fromEnv({ + env: { + OTEL_EXPORTER_OTLP_ENDPOINT: "https://collector.example.com", + T3CODE_OTLP_LOGS_URL: "http://localhost:4318/v1/logs", + }, + }), + ), + NetService.layer, + ), + ), + ); + + expect(resolved.otlpTracesUrl).toBe("https://collector.example.com/v1/traces"); + expect(resolved.otlpMetricsUrl).toBe("https://collector.example.com/v1/metrics"); + // T3 Code's own name still outranks both, and taking the signal with it + // leaves the ambient wire format on the endpoint that asked for it. + expect(resolved.otlpLogsUrl).toBe("http://localhost:4318/v1/logs"); + expect(resolved.otelEnvironment.traces.settings?.protocol).toBe("http/protobuf"); + expect(resolved.otelEnvironment.logs.settings).toBeUndefined(); + }), + ); + + it.effect("keeps a stored endpoint from re-enabling a signal turned off by name", () => + Effect.gen(function* () { + // Turning one signal off is the most common reason to touch an exporter + // list, and a Settings endpoint underneath used to quietly keep sending + // it, which is the failure the operator was trying to prevent. + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-cli-config-off-" }); + const derivedPaths = yield* deriveExplicitServerPaths(baseDir, undefined); + yield* fs.makeDirectory(path.dirname(derivedPaths.settingsPath), { recursive: true }); + yield* fs.writeFileString( + derivedPaths.settingsPath, + // @effect-diagnostics-next-line preferSchemaOverJson:off + `${JSON.stringify({ + observability: { + otlpTracesUrl: "http://stored.example.com/v1/traces", + otlpLogsUrl: "http://stored.example.com/v1/logs", + }, + })}\n`, + ); + + const resolved = yield* resolveServerConfig( + { + mode: Option.none(), + port: Option.none(), + host: Option.none(), + baseDir: Option.some(baseDir), + cwd: Option.none(), + devUrl: Option.none(), + noBrowser: Option.none(), + bootstrapFd: Option.none(), + autoBootstrapProjectFromCwd: Option.none(), + logWebSocketEvents: Option.none(), + tailscaleServeEnabled: Option.none(), + tailscaleServePort: Option.none(), + }, + Option.none(), + ).pipe( + Effect.provide( + Layer.mergeAll( + ConfigProvider.layer( + ConfigProvider.fromEnv({ + env: { + OTEL_EXPORTER_OTLP_ENDPOINT: "https://collector.example.com", + OTEL_LOGS_EXPORTER: "none", + }, + }), + ), + NetService.layer, + ), + ), + ); + + expect(resolved.otlpLogsUrl).toBeUndefined(); + expect(resolved.otlpTracesUrl).toBe("https://collector.example.com/v1/traces"); + }), + ); + + it.effect("falls back to a stored endpoint for the signals nothing exported", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-cli-config-order-signal-" }); + const derivedPaths = yield* deriveExplicitServerPaths(baseDir, undefined); + yield* fs.makeDirectory(path.dirname(derivedPaths.settingsPath), { recursive: true }); + yield* fs.writeFileString( + derivedPaths.settingsPath, + // @effect-diagnostics-next-line preferSchemaOverJson:off + `${JSON.stringify({ + observability: { otlpMetricsUrl: "http://stored.example.com/v1/metrics" }, + })}\n`, + ); + + const resolved = yield* resolveServerConfig( + { + mode: Option.some("desktop"), + port: Option.some(4888), + host: Option.none(), + baseDir: Option.some(baseDir), + cwd: Option.none(), + devUrl: Option.none(), + noBrowser: Option.none(), + bootstrapFd: Option.none(), + autoBootstrapProjectFromCwd: Option.none(), + logWebSocketEvents: Option.none(), + tailscaleServeEnabled: Option.none(), + tailscaleServePort: Option.none(), + }, + Option.none(), + ).pipe( + Effect.provide( + Layer.mergeAll( + ConfigProvider.layer( + ConfigProvider.fromEnv({ + env: { + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "https://collector.example.com/v1/traces", + }, + }), + ), + NetService.layer, + ), + ), + ); + + // The three signals are answered separately, so a variable that named + // one endpoint does not decide where the others go. + expect(resolved.otlpTracesUrl).toBe("https://collector.example.com/v1/traces"); + expect(resolved.otlpMetricsUrl).toBe("http://stored.example.com/v1/metrics"); + expect(resolved.otlpLogsUrl).toBeUndefined(); + expect(resolved.otelEnvironment.metrics.settings).toBeUndefined(); + }), + ); + it.effect("forces noBrowser and disables auto-bootstrap for headless startup presentation", () => Effect.gen(function* () { const { join } = yield* Path.Path; diff --git a/apps/server/src/cli/config.ts b/apps/server/src/cli/config.ts index 1b6449433139..b3242c47b108 100644 --- a/apps/server/src/cli/config.ts +++ b/apps/server/src/cli/config.ts @@ -1,8 +1,8 @@ import * as NetService from "@t3tools/shared/Net"; import { + DEFAULT_SIGNAL_EXPORT, OtlpHeadersFromString, OtlpProtocol, - type SignalExport, } from "@t3tools/shared/observability"; import * as OtelEnvironment from "@t3tools/shared/otelEnvironment"; import { parsePersistedServerObservabilitySettings } from "@t3tools/shared/serverSettings"; @@ -106,9 +106,13 @@ const EnvServerConfig = Config.all({ Config.map(Option.getOrUndefined), ), otlpExportIntervalMs: Config.Int("T3CODE_OTLP_EXPORT_INTERVAL_MS").pipe( - Config.withDefault(10_000), + Config.option, + Config.map(Option.getOrUndefined), + ), + otlpServiceName: Config.String("T3CODE_OTLP_SERVICE_NAME").pipe( + Config.option, + Config.map(Option.getOrUndefined), ), - otlpServiceName: Config.String("T3CODE_OTLP_SERVICE_NAME").pipe(Config.withDefault("t3-server")), otlpHeaders: Config.schema(OtlpHeadersFromString, "T3CODE_OTLP_HEADERS").pipe( Config.option, Config.map(Option.getOrUndefined), @@ -256,6 +260,7 @@ export const resolveServerConfig = ( const path = yield* Path.Path; const fs = yield* FileSystem.FileSystem; const env = yield* EnvServerConfig; + const otel = yield* OtelEnvironment.load; const normalizedFlags = { mode: flags.mode ?? Option.none(), port: flags.port ?? Option.none(), @@ -387,15 +392,57 @@ export const resolveServerConfig = ( ); const logLevel = Option.getOrElse(cliLogLevel, () => env.logLevel); - const otel = yield* OtelEnvironment.load; + // A blank bootstrap value is not an endpoint, so it must not stand in + // front of the Settings endpoint underneath it. `??` alone keeps the empty + // string and would discard both. + const persistedUrl = ( + bootstrapUrl: string | undefined, + settingsUrl: string | undefined, + ): string | undefined => + OtelEnvironment.blankAsUnset(bootstrapUrl) ?? OtelEnvironment.blankAsUnset(settingsUrl); - // T3 Code's own OTLP variables name no signal, so the one answer they give - // is the answer for all three. - const signalExport: SignalExport = { - protocol: env.otlpProtocol, - headers: env.otlpHeaders, - exportIntervalMs: env.otlpExportIntervalMs, - }; + const traces = OtelEnvironment.resolveSignalSource({ + t3Url: env.otlpTracesUrl, + signal: otel.traces, + persistedUrl: persistedUrl( + bootstrap?.otlpTracesUrl, + persistedObservabilitySettings.otlpTracesUrl, + ), + }); + const metrics = OtelEnvironment.resolveSignalSource({ + t3Url: env.otlpMetricsUrl, + signal: otel.metrics, + persistedUrl: persistedUrl( + bootstrap?.otlpMetricsUrl, + persistedObservabilitySettings.otlpMetricsUrl, + ), + }); + const logs = OtelEnvironment.resolveSignalSource({ + t3Url: env.otlpLogsUrl, + signal: otel.logs, + persistedUrl: persistedUrl( + bootstrap?.otlpLogsUrl, + persistedObservabilitySettings.otlpLogsUrl, + ), + }); + const otelEnvironment = { + ...otel, + traces: traces.signal, + metrics: metrics.signal, + logs: logs.signal, + } satisfies OtelEnvironment.OtelEnvironment; + + // T3 Code has one interval, one header set, and one wire format, and they + // deliberately cover every signal. They apply to the signals the standard + // variables did not claim; the standard variables bring their own + // per-signal defaults for the ones they did. + const signalExport = (settings: OtelEnvironment.OtlpSignalSettings | undefined) => + OtelEnvironment.resolveSignalExport({ + settings, + t3Protocol: env.otlpProtocol, + t3Headers: env.otlpHeaders, + t3ExportIntervalMs: env.otlpExportIntervalMs ?? DEFAULT_SIGNAL_EXPORT.exportIntervalMs, + }); const config: ServerConfig.ServerConfig["Service"] = { logLevel, @@ -404,24 +451,14 @@ export const resolveServerConfig = ( traceBatchWindowMs: env.traceBatchWindowMs, traceMaxBytes: env.traceMaxBytes, traceMaxFiles: env.traceMaxFiles, - otlpTracesUrl: otel.disabled - ? undefined - : (env.otlpTracesUrl ?? - bootstrap?.otlpTracesUrl ?? - persistedObservabilitySettings.otlpTracesUrl), - otlpMetricsUrl: otel.disabled - ? undefined - : (env.otlpMetricsUrl ?? - bootstrap?.otlpMetricsUrl ?? - persistedObservabilitySettings.otlpMetricsUrl), - otlpLogsUrl: otel.disabled - ? undefined - : (env.otlpLogsUrl ?? bootstrap?.otlpLogsUrl ?? persistedObservabilitySettings.otlpLogsUrl), - otlpTracesExport: signalExport, - otlpMetricsExport: signalExport, - otlpLogsExport: signalExport, - otlpServiceName: env.otlpServiceName, - otelEnvironment: otel, + otlpTracesUrl: otelEnvironment.disabled ? undefined : traces.url, + otlpMetricsUrl: otelEnvironment.disabled ? undefined : metrics.url, + otlpLogsUrl: otelEnvironment.disabled ? undefined : logs.url, + otlpTracesExport: signalExport(otelEnvironment.traces.settings), + otlpMetricsExport: signalExport(otelEnvironment.metrics.settings), + otlpLogsExport: signalExport(otelEnvironment.logs.settings), + otlpServiceName: OtelEnvironment.blankAsUnset(env.otlpServiceName) ?? "t3-server", + otelEnvironment, mode, port, cwd, diff --git a/apps/server/src/config.ts b/apps/server/src/config.ts index 344619c0eb93..0d1a7b0507dc 100644 --- a/apps/server/src/config.ts +++ b/apps/server/src/config.ts @@ -6,6 +6,8 @@ * * @module ServerConfig */ +import { DEFAULT_SIGNAL_EXPORT, type SignalExport } from "@t3tools/shared/observability"; +import * as OtelEnvironment from "@t3tools/shared/otelEnvironment"; import * as Context from "effect/Context"; import * as Clock from "effect/Clock"; import * as Effect from "effect/Effect"; @@ -17,8 +19,6 @@ import type * as Redacted from "effect/Redacted"; import * as Schema from "effect/Schema"; import { sweepStalePendingAttachments } from "./attachmentStore.ts"; -import { DEFAULT_SIGNAL_EXPORT, type SignalExport } from "@t3tools/shared/observability"; -import * as OtelEnvironment from "@t3tools/shared/otelEnvironment"; export const DEFAULT_PORT = 3773; @@ -75,14 +75,21 @@ export class ServerConfig extends Context.Service< readonly otlpMetricsUrl: string | undefined; readonly otlpLogsUrl: string | undefined; /** - * How each signal is exported. Read instead of a process-wide setting so - * the wire format, credential, and schedule travel with the endpoint they - * were configured beside. + * How each signal is exported, already resolved to the source that named + * that signal's endpoint. This is the only place the wire format, headers, + * batching, and aggregation are read from, so a setting cannot be paired + * by hand with an endpoint that came from somewhere else. */ readonly otlpTracesExport: SignalExport; readonly otlpMetricsExport: SignalExport; readonly otlpLogsExport: SignalExport; readonly otlpServiceName: string; + /** + * What the standard `OTEL_*` variables asked for. The endpoints above are + * already resolved from it; this carries the rest, which T3 Code has no + * names of its own for per signal: headers, wire format, resource + * attributes, and the batching knobs. + */ readonly otelEnvironment: OtelEnvironment.OtelEnvironment; readonly mode: RuntimeMode; readonly port: number; @@ -120,7 +127,11 @@ export const make = (config: ServerConfig["Service"]) => ServerConfig.of(config) */ export const otlpResource = (config: ServerConfig["Service"]) => ({ serviceName: config.otlpServiceName, + ...(config.otelEnvironment.serviceVersion === undefined + ? {} + : { serviceVersion: config.otelEnvironment.serviceVersion }), attributes: { + ...config.otelEnvironment.resourceAttributes, "service.runtime": "t3-server", "service.mode": config.mode, }, diff --git a/apps/server/src/observability/Layers/Observability.ts b/apps/server/src/observability/Layers/Observability.ts index 77ebe8410a91..cdaf67d0e38e 100644 --- a/apps/server/src/observability/Layers/Observability.ts +++ b/apps/server/src/observability/Layers/Observability.ts @@ -3,6 +3,7 @@ import { makeLocalFileTracer, makeTraceSink, otlpSerializationLayer, + type SignalExport, } from "@t3tools/shared/observability"; import * as OtelEnvironment from "@t3tools/shared/otelEnvironment"; import * as Effect from "effect/Effect"; @@ -21,14 +22,22 @@ import * as BrowserTraceCollector from "../BrowserTraceCollector.ts"; export const ObservabilityLive = Layer.unwrap( Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; - - const traces = config.otlpTracesExport; - const metrics = config.otlpMetricsExport; - // The trace serializer stays in the returned context because the browser - // trace forwarder exports on the same signal. - const serializationLayer = otlpSerializationLayer(traces.protocol); - const resource = ServerConfig.otlpResource(config); const attribution = yield* ResourceAttribution.ResourceAttribution; + const otel = config.otelEnvironment; + + // One variable can decline every signal, and saying so three times reads + // like three separate problems. + const declined = new Set( + [otel.traces.declined, otel.metrics.declined, otel.logs.declined].filter( + (reason) => reason !== undefined, + ), + ); + + // Each signal builds its own serializer, so the wire format travels with + // the endpoint that asked for it rather than with this process. + const serializationFor = (signal: SignalExport) => otlpSerializationLayer(signal.protocol); + + const otlpResource = ServerConfig.otlpResource(config); const traceReferencesLayer = Layer.mergeAll( Layer.succeed(Tracer.MinimumTraceLevel, config.traceMinLevel), @@ -57,9 +66,12 @@ export const ObservabilityLive = Layer.unwrap( ? undefined : yield* OtlpTracer.make({ url: config.otlpTracesUrl, - exportInterval: `${traces.exportIntervalMs} millis`, - headers: traces.headers, - resource, + exportInterval: `${config.otlpTracesExport.exportIntervalMs} millis`, + resource: otlpResource, + headers: config.otlpTracesExport.headers, + ...(config.otlpTracesExport.maxBatchSize === undefined + ? {} + : { maxBatchSize: config.otlpTracesExport.maxBatchSize }), }); const tracer = yield* makeLocalFileTracer({ @@ -76,21 +88,28 @@ export const ObservabilityLive = Layer.unwrap( BrowserTraceCollector.layer(sink), ); }), - ).pipe(Layer.provide(OtlpExporter.layerFlusher), Layer.provideMerge(serializationLayer)); + ).pipe( + Layer.provide(OtlpExporter.layerFlusher), + // The trace serializer is also the one this layer hands out, because the + // proxy in http.ts re-encodes browser spans and has to reach the trace + // collector in the format that collector was configured for. + Layer.provideMerge(serializationFor(config.otlpTracesExport)), + ); const metricsLayer = config.otlpMetricsUrl === undefined ? Layer.empty : OtlpMetrics.layer({ url: config.otlpMetricsUrl, - exportInterval: `${metrics.exportIntervalMs} millis`, - headers: metrics.headers, - resource, - }).pipe(Layer.provide(otlpSerializationLayer(metrics.protocol))); + exportInterval: `${config.otlpMetricsExport.exportIntervalMs} millis`, + resource: otlpResource, + headers: config.otlpMetricsExport.headers, + temporality: config.otlpMetricsExport.temporality, + }).pipe(Layer.provide(serializationFor(config.otlpMetricsExport))); // Logged once the server's loggers are installed, so the warnings use them. const otelWarningsLayer = Layer.effectDiscard( - Effect.forEach(config.otelEnvironment.warnings, (warning) => Effect.logWarning(warning)), + Effect.forEach([...otel.warnings, ...declined], (warning) => Effect.logWarning(warning)), ); return otelWarningsLayer.pipe( diff --git a/apps/server/src/serverLogger.test.ts b/apps/server/src/serverLogger.test.ts index a5437582d2e3..059237fd2dbd 100644 --- a/apps/server/src/serverLogger.test.ts +++ b/apps/server/src/serverLogger.test.ts @@ -161,12 +161,17 @@ describe("ServerLoggerLive", () => { it.effect("sends the headers and wire format the log signal asked for", () => Effect.gen(function* () { + // Which source won the log signal is settled before this point, so the + // logger reads the resolved export rather than pairing the URL with a + // header set that may belong to a different collector. const requests = yield* logThrough({ otlpLogsUrl: "https://collector.example.com/v1/logs", otlpLogsExport: { ...DEFAULT_SIGNAL_EXPORT, protocol: "http/protobuf", headers: { "x-scope": "logs" }, + exportIntervalMs: 1_000, + maxBatchSize: 512, }, }); diff --git a/apps/server/src/serverLogger.ts b/apps/server/src/serverLogger.ts index 6d19907c20ca..3bbf86d5202e 100644 --- a/apps/server/src/serverLogger.ts +++ b/apps/server/src/serverLogger.ts @@ -19,8 +19,9 @@ export const ServerLoggerLive = Effect.gen(function* () { : OtlpLogger.make({ url: config.otlpLogsUrl, exportInterval: `${logs.exportIntervalMs} millis`, - headers: logs.headers, resource: otlpResource(config), + ...(logs.headers === undefined ? {} : { headers: logs.headers }), + ...(logs.maxBatchSize === undefined ? {} : { maxBatchSize: logs.maxBatchSize }), }); // `Logger.layer` writes the whole logger set rather than adding to it, so diff --git a/docs/operations/observability.md b/docs/operations/observability.md index 97c6936ba608..7be072907720 100644 --- a/docs/operations/observability.md +++ b/docs/operations/observability.md @@ -189,6 +189,163 @@ Do not rely on launching from Finder, Spotlight, the dock, or the Start menu aft The backend reads observability config at process start. If you change OTLP env vars, stop the app completely and start it again. +### Option 3: The Standard `OTEL_*` Variables + +If your machine already exports the OpenTelemetry environment variables for everything else running +on it, the server joins in without being told twice. Nothing above is required: + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +export OTEL_RESOURCE_ATTRIBUTES=deployment.environment=lab +``` + +The base endpoint is a base and not a full URL: traces go to `/v1/traces`, metrics to +`/v1/metrics`, and log records to `/v1/logs`. Set +`OTEL_EXPORTER_OTLP_{TRACES,METRICS,LOGS}_ENDPOINT` when a signal needs a full URL of its own. + +Ambient `OTEL_*` variables turn export on by themselves, so a work collector in your shell profile +means T3 Code exports to it, and `OTEL_SDK_DISABLED=true` is how you stop that. For the reverse, a +profile that disables every other SDK on the machine, `T3CODE_OTEL_SDK_DISABLED=false` keeps T3 Code +exporting. + +Only the server reads these. The desktop app passes its environment to the backend it spawns, so a +variable exported before launching it reaches the server, but the Electron main process is a second +producer that still configures itself from `T3CODE_OTLP_*` and Settings. Wiring it to the same +reader is follow-up work. + +#### Precedence + +For each signal, the first source that names its endpoint wins: + +1. `T3CODE_OTLP_*` +2. `OTEL_*` +3. the desktop bootstrap envelope +4. Settings, under `observability` + +An exported variable is what the operator asked for now and a stored one is what somebody asked for +once, and T3 Code's own spelling of a variable outranks the standard spelling of it. So an ambient +`OTEL_EXPORTER_OTLP_ENDPOINT` overrides an endpoint saved in Settings, and a `T3CODE_OTLP_*` URL is +the endpoint nothing on the machine can redirect. + +Whichever source wins takes the whole signal and not the URL alone, because a variable describes the +collector it named rather than some other one. Traces sent to a `T3CODE_OTLP_TRACES_URL` endpoint +keep T3 Code's own wire format, headers, batching, and interval even when `OTEL_*` variables are +set. The same rule runs inside the `OTEL_*` group: a signal's own variable owns that signal once it +is set at all, so an unreadable `OTEL_EXPORTER_OTLP_TRACES_HEADERS` leaves traces without headers +rather than sending them the generic collector's credential. `T3CODE_OTLP_HEADERS`, +`T3CODE_OTLP_PROTOCOL`, and `T3CODE_OTLP_EXPORT_INTERVAL_MS` belong to no single signal and +configure every endpoint T3 Code's own names, the bootstrap envelope, or Settings placed. + +The three signals are resolved separately, so traces can come from one source and metrics or logs +from another. + +A source that wins a signal can also decide not to export it: +`OTEL_{TRACES,METRICS,LOGS}_EXPORTER=none`, a list naming an exporter T3 Code does not have, and +`OTEL_EXPORTER_OTLP_{TRACES,METRICS,LOGS}_PROTOCOL=grpc` each turn off the signal they describe, and +a stored endpoint does not take it back over. The exporter list is read only for a signal these +variables pointed somewhere, so on a machine that exports no `OTEL_*` endpoint, +`OTEL_LOGS_EXPORTER=none` says nothing about a logs endpoint saved in Settings. + +Whether anything is exported at all is one setting read in that same order. +`T3CODE_OTEL_SDK_DISABLED` answers it, `OTEL_SDK_DISABLED` answers it only when ours is unset, and +either answer stops every server export, including one configured through Settings. + +#### What Is Read + +| Variable | Effect | +| ------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------- | +| `OTEL_SDK_DISABLED` | Stops all export, unless `T3CODE_OTEL_SDK_DISABLED` answered first | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | Base URL for every signal | +| `OTEL_EXPORTER_OTLP_{TRACES,METRICS,LOGS}_ENDPOINT` | Full URL for one signal | +| `OTEL_EXPORTER_OTLP_HEADERS`, `OTEL_EXPORTER_OTLP_{TRACES,METRICS,LOGS}_HEADERS` | Export headers, per signal overriding the shared ones | +| `OTEL_EXPORTER_OTLP_PROTOCOL`, `OTEL_EXPORTER_OTLP_{TRACES,METRICS,LOGS}_PROTOCOL` | `http/protobuf` (default) or `http/json` | +| `OTEL_{TRACES,METRICS,LOGS}_EXPORTER` | A list; the signal is exported when it contains `otlp`, which is the default | +| `OTEL_SERVICE_VERSION`, `OTEL_RESOURCE_ATTRIBUTES` | Resource identity attached to every span, metric, and log record | +| `OTEL_SERVICE_NAME` | Refused with a warning; see below | +| `OTEL_BSP_SCHEDULE_DELAY`, `OTEL_METRIC_EXPORT_INTERVAL`, `OTEL_BLRP_SCHEDULE_DELAY` | Export interval, one per signal | +| `OTEL_BSP_MAX_EXPORT_BATCH_SIZE`, `OTEL_BLRP_MAX_EXPORT_BATCH_SIZE` | Spans per batch, log records per batch | +| `OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE` | `cumulative` (default), `delta`, or `lowmemory`, which resolves to `delta` | + +The wire format defaults to `http/protobuf` when the endpoint came from `OTEL_*`, matching the +specification, and follows `T3CODE_OTLP_PROTOCOL` otherwise, which defaults to `http/json`. Once +these variables are the ones configuring the exporter, the specification's own defaults apply: +`OTEL_BSP_SCHEDULE_DELAY` 5s, `OTEL_METRIC_EXPORT_INTERVAL` 60s, `OTEL_BLRP_SCHEDULE_DELAY` 1s, and +512 records per batch. A `T3CODE_OTLP_*` setup keeps the numbers T3 Code has always used. + +Header and resource-attribute pairs are percent decoded on both sides, so +`OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer%20abc` sends the space, a base64 credential keeps +its `=` padding, and `OTEL_RESOURCE_ATTRIBUTES=team%2Fname=blue` becomes the attribute `team/name`. + +#### The Service Name Is Not Read + +`OTEL_SERVICE_NAME` and a `service.name` inside `OTEL_RESOURCE_ATTRIBUTES` are both refused, with a +warning naming the one you set, and the attribute is dropped rather than passed through. Every T3 +Code process names itself, so a shell profile written for another app cannot rename it and merge two +services in every dashboard built on them. `service.runtime` and `service.mode` are fixed for the +same reason. Use `OTEL_RESOURCE_ATTRIBUTES` to tell instances apart: + +```bash +export OTEL_RESOURCE_ATTRIBUTES=service.instance.id=laptop-01,deployment.environment=lab +``` + +#### Known Gaps + +- **No gRPC.** `OTEL_EXPORTER_OTLP_PROTOCOL=grpc` is refused rather than downgraded, since posting + an HTTP body to a gRPC endpoint fails in a way that is harder to read than exporting nothing. It + turns off only the signal that named it, and only when these variables named where that signal + goes. +- **No exporter but OTLP.** `OTEL_{TRACES,METRICS,LOGS}_EXPORTER` honors `otlp` and `none`. A list + naming an exporter the specification defines for that signal and not `otlp` reads as a deliberate + "not this one" and the signal is not exported. The names are read per signal, so + `OTEL_LOGS_EXPORTER=prometheus` is the mistake it is rather than a choice, and a list naming + nothing recognizable is reported as a typo while the signal keeps exporting. +- **No compression and no client TLS.** `OTEL_EXPORTER_OTLP_COMPRESSION`, + `OTEL_EXPORTER_OTLP_CERTIFICATE`, `OTEL_EXPORTER_OTLP_CLIENT_KEY`, and + `OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE` are ignored. A collector that requires mutual TLS needs a + proxy in front of it. +- **No export timeouts.** `OTEL_EXPORTER_OTLP_TIMEOUT`, + `OTEL_EXPORTER_OTLP_{TRACES,METRICS,LOGS}_TIMEOUT`, and `OTEL_METRIC_EXPORT_TIMEOUT` are + per-request deadlines and this exporter has no per-request knob, so they are ignored. Spending + them on the shutdown flush instead would let a generous collector timeout hold the server open on + every restart. +- **Only an `OTEL_*` metrics endpoint can choose its aggregation.** A signal is configured by + whichever source named its endpoint, so a metrics endpoint from `T3CODE_OTLP_METRICS_URL`, the + bootstrap envelope, or Settings exports `cumulative`, and there is no `T3CODE_*` spelling of + `OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE` to change that. It matters for a receiver that + accepts delta histograms only, which drops the cumulative ones without an error, so every + `_duration` timer goes missing while the `_total` counters keep arriving. Point such a backend at + `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` and set the preference to `delta`. `lowmemory` asks for a + choice per instrument kind that this exporter cannot express and resolves to `delta` with a + warning, which is what it asks for on the counters and timers T3 Code records. +- **`OTEL_SERVICE_VERSION` is not a specification variable.** It is read because the exporter + library reads it too. `OTEL_RESOURCE_ATTRIBUTES=service.version=...` is the portable spelling. + +Everything else is ignored, including `OTEL_BSP_MAX_QUEUE_SIZE`, `OTEL_BLRP_MAX_QUEUE_SIZE`, +`OTEL_BSP_EXPORT_TIMEOUT`, `OTEL_BLRP_EXPORT_TIMEOUT`, sampler variables, propagator variables, and +the attribute and span limit variables. + +#### When A Value Cannot Be Used + +A variable T3 Code cannot act on never stops it from starting. The value is reported once at startup +and the default applies, so one bad value never costs you the other variables. Zero is refused for +an interval or a batch size because the exporter acts on it every time around the loop: a batch size +of zero posts one HTTP request per span, and an interval of zero exports continuously. + +An empty value means the same as an unset one, so `OTEL_SERVICE_VERSION=` reads as absent and an +empty `OTEL_SERVICE_NAME` is not an attempt to rename anything. `OTEL_SDK_DISABLED` follows the +specification's one rule for booleans, where the case-insensitive string `true` is the only value +that switches export off and anything else, `yes` and `1` included, leaves it on and is reported. +`T3CODE_OTEL_SDK_DISABLED` is T3 Code's own name, so it takes `true`, `1`, `yes`, `on` and their +negatives, and a value it cannot read is left to `OTEL_SDK_DISABLED` to answer. + +An `OTEL_EXPORTER_OTLP_HEADERS` or `OTEL_RESOURCE_ATTRIBUTES` value is discarded whole rather than +partly, whether a member fails to decode or carries no `key=value` pair at all. Keeping the members +that did parse is what makes a bad variable read like a bad token: the collector answers a +half-parsed credential with the authentication error a wrong one gets, and +`authorization=token,x-tenant` would have authenticated and then routed to the wrong tenant. A +trailing or doubled comma is spacing rather than a member, so `authorization=token,` is read as the +one pair it contains. + ## How To Use Traces And Metrics To Debug The Server ### Start With The Local Trace File @@ -551,6 +708,9 @@ OTLP export: - `T3CODE_OTLP_HEADERS`: extra headers for all three exporters, same format as `OTEL_EXPORTER_OTLP_HEADERS`: comma-separated `key=value` pairs with percent-encoded values. - `T3CODE_OTLP_PROTOCOL`: `http/json` (default) or `http/protobuf` +- `T3CODE_OTEL_SDK_DISABLED`: stops every server export, whatever configured it, including Settings. + Read before `OTEL_SDK_DISABLED`, so `false` here keeps T3 Code exporting on a machine that sets + the standard name. If the OTLP URLs are unset, local tracing still works, metrics stay in-process only, and logs stay on stdout only. @@ -558,14 +718,9 @@ on stdout only. ### The Kill Switch `T3CODE_OTEL_SDK_DISABLED` and `OTEL_SDK_DISABLED` turn off every OTLP export in both the server and -the desktop main process, overriding any endpoint from the environment or Settings. Local trace -files and stdout logs are unaffected. - -`T3CODE_OTEL_SDK_DISABLED` wins when set, so `T3CODE_OTEL_SDK_DISABLED=false` re-enables export on a -machine that sets `OTEL_SDK_DISABLED` for everything else. It accepts the usual boolean spellings -(`true`/`false`, `yes`/`no`, `on`/`off`, `1`/`0`, `y`/`n`). `OTEL_SDK_DISABLED` follows the -OpenTelemetry specification and only `true` disables export, so `OTEL_SDK_DISABLED=1` does not. -Values are case-insensitive and trimmed. An unrecognized value is ignored with a startup warning. +the desktop main process, overriding any endpoint from the environment or Settings; see +[precedence](#precedence) above for how the two names and their accepted values are read. Local +trace files and stdout logs are unaffected. ### What Is Instrumented Today diff --git a/packages/shared/src/observability.ts b/packages/shared/src/observability.ts index 6393ed0879db..52bbddfe0700 100644 --- a/packages/shared/src/observability.ts +++ b/packages/shared/src/observability.ts @@ -16,16 +16,25 @@ export type OtlpProtocol = typeof OtlpProtocol.Type; export const otlpSerializationLayer = (protocol: OtlpProtocol) => protocol === "http/protobuf" ? OtlpSerialization.layerProtobuf : OtlpSerialization.layerJson; +/** `OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE`. */ +export type MetricsTemporality = "cumulative" | "delta"; + +/** What metrics are aggregated as when nothing asks for anything. */ +export const DEFAULT_METRICS_TEMPORALITY: MetricsTemporality = "cumulative"; + /** * How one signal is exported, once whichever source named that signal's - * endpoint has been resolved. Held per signal rather than per process, so a - * wire format or a credential cannot be paired by hand with an endpoint that - * came from somewhere else. + * endpoint has been resolved, and after its owner has been decided. Held per + * signal rather than per process, so a wire format or a credential cannot be + * paired by hand with an endpoint that came from somewhere else. */ export interface SignalExport { readonly protocol: OtlpProtocol; readonly headers: Readonly> | undefined; readonly exportIntervalMs: number; + readonly maxBatchSize: number | undefined; + /** Metrics only. Spans and log records have no aggregation to prefer. */ + readonly temporality: MetricsTemporality; } /** What T3 Code exports with when nothing configured a signal. */ @@ -33,6 +42,8 @@ export const DEFAULT_SIGNAL_EXPORT: SignalExport = { protocol: "http/json", headers: undefined, exportIntervalMs: 10_000, + maxBatchSize: undefined, + temporality: DEFAULT_METRICS_TEMPORALITY, }; const FLUSH_BUFFER_THRESHOLD = 256; diff --git a/packages/shared/src/otelEnvironment.test.ts b/packages/shared/src/otelEnvironment.test.ts index 548f2a78b776..6ae603ca6263 100644 --- a/packages/shared/src/otelEnvironment.test.ts +++ b/packages/shared/src/otelEnvironment.test.ts @@ -6,17 +6,608 @@ import * as OtlpResource from "effect/unstable/observability/OtlpResource"; import * as OtelEnvironment from "./otelEnvironment.ts"; -const load = (env: Record) => - OtelEnvironment.load.pipe(Effect.provide(ConfigProvider.layer(ConfigProvider.fromEnv({ env })))); +const withEnv = (env: Record) => + Effect.provide(Layer.mergeAll(ConfigProvider.layer(ConfigProvider.fromEnv({ env })))); -const SPEC_OFF = - "OTEL_SDK_DISABLED is set, so no telemetry is exported, whatever configured it; set T3CODE_OTEL_SDK_DISABLED=false to export anyway"; -const T3_OFF = - "T3CODE_OTEL_SDK_DISABLED is set, so no telemetry is exported, whatever configured it"; -const specIgnored = (value: string) => - `OTEL_SDK_DISABLED=${value} was read as false; the OpenTelemetry specification recognizes only the string true, so use OTEL_SDK_DISABLED=true or T3CODE_OTEL_SDK_DISABLED to say it any other way`; +const COLLECTOR = "https://collector.example.com"; + +/** A collector every signal can reach, plus whatever the case under test adds. */ +const read = (env: Record = {}) => + OtelEnvironment.load.pipe(withEnv({ OTEL_EXPORTER_OTLP_ENDPOINT: COLLECTOR, ...env })); + +const warnings = (resolved: OtelEnvironment.OtelEnvironment) => resolved.warnings.join("\n"); describe("OtelEnvironment", () => { + it.effect("stays off when nothing is configured", () => + Effect.gen(function* () { + const resolved = yield* OtelEnvironment.load.pipe(withEnv({})); + assert.strictEqual(resolved.traces.settings, undefined); + assert.strictEqual(resolved.metrics.settings, undefined); + assert.strictEqual(resolved.logs.settings, undefined); + assert.isFalse(resolved.disabled); + }), + ); + + it.effect("reads the specification's own defaults for a collector named this way", () => + Effect.gen(function* () { + // Once these variables are the ones configuring the exporter, the numbers + // that apply are the specification's and not the ones T3 Code picked for + // itself, and an aggregation nobody asked for is left for the exporter. + const resolved = yield* read(); + assert.strictEqual(resolved.traces.settings?.url, `${COLLECTOR}/v1/traces`); + assert.strictEqual(resolved.metrics.settings?.url, `${COLLECTOR}/v1/metrics`); + assert.strictEqual(resolved.logs.settings?.url, `${COLLECTOR}/v1/logs`); + assert.strictEqual(resolved.traces.settings?.protocol, "http/protobuf"); + assert.strictEqual(resolved.traces.settings?.exportIntervalMs, 5000); + assert.strictEqual(resolved.traces.settings?.maxBatchSize, 512); + assert.strictEqual(resolved.metrics.settings?.exportIntervalMs, 60_000); + assert.strictEqual(resolved.metrics.settings?.temporality, undefined); + assert.strictEqual(resolved.logs.settings?.exportIntervalMs, 1000); + assert.deepStrictEqual(resolved.warnings, []); + }), + ); + + const ENDPOINT_JOINS = [ + { given: "a bare host", endpoint: COLLECTOR, url: `${COLLECTOR}/v1/traces` }, + { given: "a trailing slash", endpoint: `${COLLECTOR}/`, url: `${COLLECTOR}/v1/traces` }, + { given: "a base path", endpoint: `${COLLECTOR}/otel`, url: `${COLLECTOR}/otel/v1/traces` }, + { + // Several vendor intakes take their API key in the query string, where + // joining the strings would make the path part of the key's value. + given: "a query string", + endpoint: "https://intake.example.com/otlp?key=abc", + url: "https://intake.example.com/otlp/v1/traces?key=abc", + }, + { + // A shell profile that lined up its exports did not mean the padding to + // land in the middle of the URL, where nothing would report it. + given: "padding", + endpoint: ` ${COLLECTOR}/ `, + url: `${COLLECTOR}/v1/traces`, + }, + ]; + + for (const join of ENDPOINT_JOINS) { + it.effect(`appends the signal path to a generic endpoint with ${join.given}`, () => + Effect.gen(function* () { + const resolved = yield* OtelEnvironment.load.pipe( + withEnv({ OTEL_EXPORTER_OTLP_ENDPOINT: join.endpoint }), + ); + assert.strictEqual(resolved.traces.settings?.url, join.url); + }), + ); + } + + const UNUSABLE = [ + { + given: "a disable flag the specification does not define", + env: { OTEL_SDK_DISABLED: "yes" }, + named: "OTEL_SDK_DISABLED=yes", + check: (resolved: OtelEnvironment.OtelEnvironment) => { + assert.isFalse(resolved.disabled); + assert.isDefined(resolved.traces.settings); + }, + }, + { + given: "a delay that is not a number", + env: { OTEL_BSP_SCHEDULE_DELAY: "abc" }, + named: "OTEL_BSP_SCHEDULE_DELAY=abc", + check: (resolved: OtelEnvironment.OtelEnvironment) => { + assert.strictEqual(resolved.traces.settings?.exportIntervalMs, 5000); + assert.isDefined(resolved.metrics.settings); + }, + }, + { + given: "a misspelled wire format", + env: { OTEL_EXPORTER_OTLP_PROTOCOL: "htp/json" }, + named: "htp/json", + check: (resolved: OtelEnvironment.OtelEnvironment) => { + assert.strictEqual(resolved.traces.settings?.protocol, "http/protobuf"); + assert.strictEqual(resolved.traces.declined, undefined); + }, + }, + { + given: "an aggregation that is not a preference", + env: { OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE: "hourly" }, + named: "hourly", + check: (resolved: OtelEnvironment.OtelEnvironment) => { + assert.isDefined(resolved.metrics.settings); + assert.strictEqual(resolved.metrics.settings?.temporality, undefined); + }, + }, + { + // Half a header set is worse than none: the collector answers a partial + // credential with the same 401 it gives a wrong one, and routes the + // stream to whichever tenant the readable members named. + given: "a header list with a member that carries no pair", + env: { OTEL_EXPORTER_OTLP_HEADERS: "authorization=token,x-tenant" }, + named: "OTEL_EXPORTER_OTLP_HEADERS", + check: (resolved: OtelEnvironment.OtelEnvironment) => + assert.strictEqual(resolved.traces.settings?.headers, undefined), + }, + { + given: "a header value that is not valid percent encoding", + env: { OTEL_EXPORTER_OTLP_HEADERS: "x-token=100%zz,x-other=100%25" }, + named: "OTEL_EXPORTER_OTLP_HEADERS", + check: (resolved: OtelEnvironment.OtelEnvironment) => + assert.strictEqual(resolved.traces.settings?.headers, undefined), + }, + { + given: "a resource attribute that does not decode", + env: { OTEL_RESOURCE_ATTRIBUTES: "team=100%zz,deployment=prod" }, + named: "OTEL_RESOURCE_ATTRIBUTES", + check: (resolved: OtelEnvironment.OtelEnvironment) => + assert.deepStrictEqual(resolved.resourceAttributes, {}), + }, + ]; + + for (const unusable of UNUSABLE) { + it.effect(`keeps exporting and reports ${unusable.given}`, () => + Effect.gen(function* () { + const resolved = yield* read(unusable.env); + unusable.check(resolved); + assert.include(warnings(resolved), unusable.named); + }), + ); + } + + it.effect("treats an empty value as an unset one", () => + Effect.gen(function* () { + const resolved = yield* read({ + OTEL_SERVICE_NAME: "", + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "", + }); + // An empty rename is not a rename, so it is not worth a warning either. + assert.deepStrictEqual(resolved.warnings, []); + assert.strictEqual(resolved.traces.settings?.url, `${COLLECTOR}/v1/traces`); + }), + ); + + const HEADER_LISTS = [ + { + given: "the percent encoding the specification asks for", + value: "Authorization=Bearer%20abc123, x-scope=team%2Fplatform", + headers: { Authorization: "Bearer abc123", "x-scope": "team/platform" }, + }, + { + given: "a credential that contains its own separator", + value: "Authorization=Basic YWJjOmRlZg==", + headers: { Authorization: "Basic YWJjOmRlZg==" }, + }, + { + given: "a trailing comma as spacing rather than as a member", + value: "authorization=token,", + headers: { authorization: "token" }, + }, + ]; + + for (const list of HEADER_LISTS) { + it.effect(`reads a header list written with ${list.given}`, () => + Effect.gen(function* () { + const resolved = yield* read({ OTEL_EXPORTER_OTLP_HEADERS: list.value }); + assert.deepStrictEqual(resolved.traces.settings?.headers, list.headers); + }), + ); + } + + it.effect("reads the service version and the resource attributes beside it", () => + Effect.gen(function* () { + const resolved = yield* OtelEnvironment.load.pipe( + withEnv({ + OTEL_RESOURCE_ATTRIBUTES: "team=platform%20eng,deployment.environment=prod", + OTEL_SERVICE_VERSION: " 1.2.3 ", + }), + ); + assert.strictEqual(resolved.serviceVersion, "1.2.3"); + // service.version becomes a named field, so leaving it in the attribute + // bag too would send it twice. + assert.deepStrictEqual(resolved.resourceAttributes, { + team: "platform eng", + "deployment.environment": "prod", + }); + }), + ); + + const SERVICE_NAME_REFUSALS = [ + { + given: "OTEL_SERVICE_NAME", + env: { OTEL_SERVICE_NAME: "some-other-app" }, + attributes: {}, + named: "OTEL_SERVICE_NAME was ignored", + }, + { + given: "a service.name hidden in the resource attributes", + env: { OTEL_RESOURCE_ATTRIBUTES: "service.name=some-other-app,host.name=lab-01" }, + attributes: { "host.name": "lab-01" }, + named: "service.name was ignored", + }, + { + given: "a percent-encoded service.name", + env: { OTEL_RESOURCE_ATTRIBUTES: "team%2Fname=blue,service%2Ename=impostor" }, + attributes: { "team/name": "blue" }, + named: "service.name was ignored", + }, + { + given: "both names at once", + env: { + OTEL_SERVICE_NAME: "explicit", + OTEL_RESOURCE_ATTRIBUTES: "service.name=from-attributes", + }, + attributes: {}, + named: "OTEL_SERVICE_NAME was ignored", + }, + ]; + + for (const refusal of SERVICE_NAME_REFUSALS) { + it.effect(`refuses to rename the service through ${refusal.given}`, () => + Effect.gen(function* () { + // Each T3 Code process names itself, so a fleet-wide rename would merge + // two of them into one service, and passing the attribute through would + // send a second service.name beside the one the process chose. + const resolved = yield* OtelEnvironment.load.pipe(withEnv(refusal.env)); + assert.deepStrictEqual(resolved.resourceAttributes, refusal.attributes); + assert.lengthOf(resolved.warnings, 1); + assert.include(resolved.warnings[0] ?? "", refusal.named); + }), + ); + } + + it.effect("refuses a zero interval and a zero batch size on every signal", () => + Effect.gen(function* () { + // The exporter sleeps for the interval before each run, so zero is a busy + // loop, and a zero batch posts one request per span. + const resolved = yield* read({ + OTEL_BSP_SCHEDULE_DELAY: "0", + OTEL_METRIC_EXPORT_INTERVAL: "0", + OTEL_BLRP_SCHEDULE_DELAY: "0", + OTEL_BSP_MAX_EXPORT_BATCH_SIZE: "0", + OTEL_BLRP_MAX_EXPORT_BATCH_SIZE: "0", + }); + assert.strictEqual(resolved.traces.settings?.exportIntervalMs, 5000); + assert.strictEqual(resolved.metrics.settings?.exportIntervalMs, 60_000); + assert.strictEqual(resolved.logs.settings?.exportIntervalMs, 1000); + assert.strictEqual(resolved.traces.settings?.maxBatchSize, 512); + assert.strictEqual(resolved.logs.settings?.maxBatchSize, 512); + for (const name of [ + "OTEL_BSP_SCHEDULE_DELAY", + "OTEL_METRIC_EXPORT_INTERVAL", + "OTEL_BLRP_SCHEDULE_DELAY", + "OTEL_BSP_MAX_EXPORT_BATCH_SIZE", + "OTEL_BLRP_MAX_EXPORT_BATCH_SIZE", + ]) { + assert.include(warnings(resolved), `${name}=0`); + } + }), + ); + + it.effect("declines gRPC on the signal that asked for it and leaves the others alone", () => + Effect.gen(function* () { + // A metric endpoint that speaks gRPC says nothing about where traces go, + // and switching traces off over it loses telemetry nobody asked to lose. + const resolved = yield* read({ OTEL_EXPORTER_OTLP_METRICS_PROTOCOL: "grpc" }); + assert.isDefined(resolved.traces.settings); + assert.isDefined(resolved.logs.settings); + assert.strictEqual(resolved.metrics.settings, undefined); + assert.include(resolved.metrics.declined ?? "", "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL"); + assert.isTrue(resolved.metrics.off); + }), + ); + + it.effect("does not blame gRPC for a signal that was never going to export", () => + Effect.gen(function* () { + // Nothing named an endpoint, so the protocol is beside the point and + // reporting it would send someone looking for a collector problem. + const resolved = yield* OtelEnvironment.load.pipe( + withEnv({ OTEL_EXPORTER_OTLP_PROTOCOL: "grpc" }), + ); + assert.strictEqual(resolved.traces.declined, undefined); + assert.strictEqual(resolved.metrics.declined, undefined); + assert.deepStrictEqual(resolved.warnings, []); + }), + ); + + const EXPORTER_LISTS = [ + { given: "none", value: "none", exports: false, reported: false }, + // Reading `otlpp` as "not OTLP" would turn one transposed letter into a + // signal that silently stops exporting. + { given: "a misspelling of otlp", value: "otlpp", exports: true, reported: true }, + { + given: "otlp beside an exporter T3 Code has none of", + value: "console, otlp", + exports: true, + reported: true, + }, + // A real exporter for this signal that this build does not have, so the + // operator asked for something deliberate that cannot be served here. + { given: "an exporter T3 Code has none of", value: "zipkin", exports: false, reported: true }, + ]; + + for (const list of EXPORTER_LISTS) { + it.effect(`reads an exporter list of ${list.given}`, () => + Effect.gen(function* () { + const resolved = yield* read({ OTEL_TRACES_EXPORTER: list.value }); + assert.strictEqual(resolved.traces.settings === undefined, !list.exports); + assert.strictEqual(warnings(resolved).includes("OTEL_TRACES_EXPORTER"), list.reported); + assert.isDefined(resolved.metrics.settings); + }), + ); + } + + it.effect("says nothing about an exporter list on a signal with nowhere to go", () => + Effect.gen(function* () { + const resolved = yield* OtelEnvironment.load.pipe( + withEnv({ OTEL_TRACES_EXPORTER: "zipkin" }), + ); + assert.deepStrictEqual(resolved.warnings, []); + }), + ); + + it.effect("resolves lowmemory to the aggregation it asks for on these metrics", () => + Effect.gen(function* () { + // Falling back to the default here would invert the request rather than + // decline it, and invert it toward the value a delta-only receiver drops + // without an error, so the timers would vanish and the counters would not. + const resolved = yield* read({ + OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE: "lowmemory", + }); + assert.strictEqual(resolved.metrics.settings?.temporality, "delta"); + assert.include(warnings(resolved), "lowmemory"); + }), + ); + + it.effect("exports nothing at all when the SDK is disabled", () => + Effect.gen(function* () { + const resolved = yield* read({ OTEL_SDK_DISABLED: "true" }); + assert.isTrue(resolved.disabled); + assert.strictEqual(resolved.traces.settings, undefined); + assert.strictEqual(resolved.metrics.settings, undefined); + assert.strictEqual(resolved.logs.settings, undefined); + // Someone who inherited this from a shell profile has somewhere to go. + assert.include(warnings(resolved), "T3CODE_OTEL_SDK_DISABLED=false"); + }), + ); + + it.effect("lets T3 Code's own name answer before the standard one", () => + Effect.gen(function* () { + const off = yield* read({ T3CODE_OTEL_SDK_DISABLED: "true" }); + assert.isTrue(off.disabled); + assert.strictEqual(off.traces.settings, undefined); + assert.deepStrictEqual(off.warnings, [ + "T3CODE_OTEL_SDK_DISABLED is set, so no telemetry is exported, whatever configured it", + ]); + + // The point of reading ours first: a machine that disables every other + // SDK in its shell profile can still ask for T3 Code's telemetry. + const on = yield* read({ T3CODE_OTEL_SDK_DISABLED: "false", OTEL_SDK_DISABLED: "true" }); + assert.isFalse(on.disabled); + assert.isDefined(on.traces.settings); + assert.deepStrictEqual(on.warnings, []); + }), + ); + + it.effect("reads T3 Code's own name the way T3 Code reads a boolean", () => + Effect.gen(function* () { + // Ours to define, so it takes the affirmatives people type. The + // specification's single spelling stays with the OTEL_* name. + const numeric = yield* OtelEnvironment.load.pipe(withEnv({ T3CODE_OTEL_SDK_DISABLED: "1" })); + assert.isTrue(numeric.disabled); + + // A value that answers nothing leaves the source under it to answer. + const nonsense = yield* read({ + T3CODE_OTEL_SDK_DISABLED: "maybe", + OTEL_SDK_DISABLED: "true", + }); + assert.isTrue(nonsense.disabled); + assert.include(warnings(nonsense), "T3CODE_OTEL_SDK_DISABLED=maybe"); + }), + ); + + it.effect("takes a signal endpoint exactly as written and leaves the others generic", () => + Effect.gen(function* () { + // The per-signal variable is a whole URL. Appending to it would send + // traces to a path the collector does not serve. + const resolved = yield* read({ + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "https://traces.example.com/ingest", + }); + assert.strictEqual(resolved.traces.settings?.url, "https://traces.example.com/ingest"); + assert.strictEqual(resolved.metrics.settings?.url, `${COLLECTOR}/v1/metrics`); + }), + ); + + it.effect("keeps a signal's own endpoint from falling back to the generic collector", () => + Effect.gen(function* () { + // A variable that names this signal owns it once it is set at all, so a + // value nobody can use is not an invitation to post the signal somewhere + // else that was configured for the other signals. + const resolved = yield* read({ + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "collector.example.com", + }); + assert.strictEqual(resolved.traces.settings, undefined); + assert.strictEqual(resolved.metrics.settings?.url, `${COLLECTOR}/v1/metrics`); + assert.include(warnings(resolved), "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"); + assert.include(warnings(resolved), "OTEL_EXPORTER_OTLP_ENDPOINT was not used in its place"); + }), + ); + + it.effect("replaces the generic headers with the signal's own rather than merging", () => + Effect.gen(function* () { + // What the specification says, and what a collector holding two different + // keys depends on. + const resolved = yield* read({ + OTEL_EXPORTER_OTLP_HEADERS: "api-key=abc123,x-tenant=acme", + OTEL_EXPORTER_OTLP_TRACES_HEADERS: "api-key=traces-only", + }); + assert.deepStrictEqual(resolved.traces.settings?.headers, { "api-key": "traces-only" }); + assert.deepStrictEqual(resolved.metrics.settings?.headers, { + "api-key": "abc123", + "x-tenant": "acme", + }); + }), + ); + + it.effect("keeps a signal's own header list from sending the generic credential", () => + Effect.gen(function* () { + // The signal's list is malformed rather than absent, and the generic + // credential belongs to whoever was told to accept it, so falling back + // would authorize this stream as a tenant nobody named for it. + const resolved = yield* read({ + OTEL_EXPORTER_OTLP_TRACES_HEADERS: "junk", + OTEL_EXPORTER_OTLP_HEADERS: "Authorization=Bearer%20abc123", + }); + assert.strictEqual(resolved.traces.settings?.headers, undefined); + assert.deepStrictEqual(resolved.metrics.settings?.headers, { + Authorization: "Bearer abc123", + }); + assert.include(warnings(resolved), "OTEL_EXPORTER_OTLP_TRACES_HEADERS"); + assert.include(warnings(resolved), "OTEL_EXPORTER_OTLP_HEADERS was not used in its place"); + }), + ); + + it.effect("does not write the credential it refused into the startup log", () => + Effect.gen(function* () { + // These warnings are logged, so a variable that carries a token is named + // without quoting what was in it. + const resolved = yield* read({ + OTEL_EXPORTER_OTLP_HEADERS: "authorization=Bearer%20super-secret-token,x-tenant", + OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "https://someone:hunter2@collector.example.com:port", + }); + + assert.include(warnings(resolved), "OTEL_EXPORTER_OTLP_HEADERS"); + assert.include(warnings(resolved), "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT"); + assert.notInclude(warnings(resolved), "super-secret-token"); + assert.notInclude(warnings(resolved), "hunter2"); + }), + ); + + it.effect("names a shared variable once even though every signal reads it", () => + Effect.gen(function* () { + const resolved = yield* read({ OTEL_EXPORTER_OTLP_HEADERS: "x-token=100%zz" }); + assert.lengthOf( + resolved.warnings.filter((warning) => warning.includes("OTEL_EXPORTER_OTLP_HEADERS")), + 1, + ); + }), + ); + + it.effect("says nothing about an aggregation for metrics these variables did not place", () => + Effect.gen(function* () { + // The preference travels with the endpoint that asked for it, so on a + // machine whose metrics endpoint comes from somewhere else this warning + // would claim an aggregation that never applied. + const resolved = yield* OtelEnvironment.load.pipe( + withEnv({ + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: `${COLLECTOR}/v1/traces`, + OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE: "lowmemory", + }), + ); + assert.strictEqual(resolved.metrics.settings, undefined); + assert.notInclude(warnings(resolved), "lowmemory"); + }), + ); + + it("does not lend T3 Code's own settings to a signal the standard names configured", () => { + const settings = { + url: `${COLLECTOR}/v1/traces`, + protocol: "http/protobuf", + headers: undefined, + exportIntervalMs: 5000, + maxBatchSize: 512, + temporality: undefined, + } as const; + const t3 = { + t3Protocol: "http/json", + t3Headers: { authorization: "t3-only" }, + t3ExportIntervalMs: 10_000, + } as const; + + // Settings that exist and say nothing about the headers are these variables + // owning the signal and being silent, which is an answer of its own. + const standard = OtelEnvironment.resolveSignalExport({ settings, ...t3 }); + assert.strictEqual(standard.headers, undefined); + assert.strictEqual(standard.protocol, "http/protobuf"); + assert.strictEqual(standard.exportIntervalMs, 5000); + + // Absent settings mean the standard variables named nothing, so T3 Code's + // own answers apply. + const own = OtelEnvironment.resolveSignalExport({ settings: undefined, ...t3 }); + assert.deepStrictEqual(own.headers, { authorization: "t3-only" }); + assert.strictEqual(own.protocol, "http/json"); + assert.strictEqual(own.exportIntervalMs, 10_000); + }); + + it.effect("lets T3 Code's own endpoint reach a signal the standard names turned off", () => + Effect.gen(function* () { + // T3 Code's own name outranks the standard ones, so an operator who set + // it is not overruled by a fleet-wide exporter list. + const resolved = yield* read({ OTEL_METRICS_EXPORTER: "none" }); + const metrics = OtelEnvironment.resolveSignalSource({ + t3Url: "https://t3.example.com/v1/metrics", + signal: resolved.metrics, + persistedUrl: undefined, + }); + assert.strictEqual(metrics.url, "https://t3.example.com/v1/metrics"); + assert.strictEqual(metrics.signal.settings, undefined); + }), + ); + + it.effect("keeps a stored endpoint from re-enabling a signal turned off by name", () => + Effect.gen(function* () { + // `none` is an answer about this signal, not an absence of one, so the + // endpoint somebody saved once does not get to give the opposite answer. + const resolved = yield* read({ OTEL_LOGS_EXPORTER: "none" }); + assert.isTrue(resolved.logs.off); + const logs = OtelEnvironment.resolveSignalSource({ + t3Url: undefined, + signal: resolved.logs, + persistedUrl: "https://stored.example.com/v1/logs", + }); + assert.strictEqual(logs.url, undefined); + }), + ); + + it.effect("keeps a stored endpoint from answering for a declined transport", () => + Effect.gen(function* () { + const resolved = yield* read({ OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: "grpc" }); + const traces = OtelEnvironment.resolveSignalSource({ + t3Url: undefined, + signal: resolved.traces, + persistedUrl: "https://stored.example.com/v1/traces", + }); + assert.strictEqual(traces.url, undefined); + assert.isDefined(traces.signal.declined); + }), + ); + + it.effect("leaves a stored endpoint alone when no standard endpoint named the signal", () => + Effect.gen(function* () { + // With nowhere for these variables to send anything, the exporter list is + // not read at all, so it cannot switch off an export it never described. + const resolved = yield* OtelEnvironment.load.pipe(withEnv({ OTEL_LOGS_EXPORTER: "none" })); + assert.isFalse(resolved.logs.off); + const logs = OtelEnvironment.resolveSignalSource({ + t3Url: undefined, + signal: resolved.logs, + persistedUrl: "https://stored.example.com/v1/logs", + }); + assert.strictEqual(logs.url, "https://stored.example.com/v1/logs"); + assert.strictEqual(logs.signal.settings, undefined); + }), + ); +}); + +describe("OtelEnvironment kill switch", () => { + const load = (env: Record) => + OtelEnvironment.load.pipe( + Effect.provide(ConfigProvider.layer(ConfigProvider.fromEnv({ env }))), + ); + + const SPEC_OFF = + "OTEL_SDK_DISABLED is set, so no telemetry is exported, whatever configured it; set T3CODE_OTEL_SDK_DISABLED=false to export anyway"; + const T3_OFF = + "T3CODE_OTEL_SDK_DISABLED is set, so no telemetry is exported, whatever configured it"; + const specIgnored = (value: string) => + `OTEL_SDK_DISABLED=${value} was read as false; the OpenTelemetry specification recognizes only the string true, so use OTEL_SDK_DISABLED=true or T3CODE_OTEL_SDK_DISABLED to say it any other way`; + it.effect.each([ { name: "nothing set", env: {}, disabled: false, warnings: [] }, // OTEL_SDK_DISABLED follows the specification: only `true`, case-insensitively. diff --git a/packages/shared/src/otelEnvironment.ts b/packages/shared/src/otelEnvironment.ts index a10b3be4e278..9cf06cc6c29d 100644 --- a/packages/shared/src/otelEnvironment.ts +++ b/packages/shared/src/otelEnvironment.ts @@ -1,43 +1,185 @@ /** - * otelEnvironment: the OpenTelemetry kill switch, shared by the server and the - * desktop main process so both agree on what turns export off. + * otelEnvironment: the OpenTelemetry environment variables, read the way the + * specification says to read them. Read by every T3 Code process that exports + * telemetry, so the server and the desktop app cannot disagree about what a + * variable means. * - * `T3CODE_OTEL_SDK_DISABLED` is read first, so a machine that sets - * `OTEL_SDK_DISABLED` for everything else can still opt T3 Code back in. + * Parsing is the Config algebra's. What this module adds on top of it, and the + * only reasoning that crosses the functions below, is: + * + * - `T3CODE_OTEL_SDK_DISABLED` is read before `OTEL_SDK_DISABLED`, so a machine + * that sets `OTEL_SDK_DISABLED` for everything else can still opt T3 Code + * back in. + * - An unusable value is a warning and the default, never a refusal to start, + * which is `ignoring`. + * - A signal's own variable owns that signal once it is set, which is `owned`. + * - The exporter speaks OTLP over HTTP, so `grpc` is declined loudly. + * + * Why each of those is the right answer, the precedence between the sources, + * and every variable this reads is in `docs/operations/observability.md`. * * @module otelEnvironment */ import * as Config from "effect/Config"; import * as ConfigProvider from "effect/ConfigProvider"; import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; +import * as SchemaIssue from "effect/SchemaIssue"; import * as SchemaTransformation from "effect/SchemaTransformation"; +import { + DEFAULT_METRICS_TEMPORALITY, + type MetricsTemporality, + type OtlpProtocol, + type SignalExport, +} from "./observability.ts"; + +/** The signals T3 Code exports, spelled as the variable names spell them. */ +export type OtlpSignalName = "TRACES" | "METRICS" | "LOGS"; + +/** Everything one signal's exporter needs, or `undefined` if it is off. */ +export interface OtlpSignalSettings { + readonly url: string; + /** Per signal, since each signal builds its own serializer. */ + readonly protocol: OtlpProtocol; + readonly headers: Readonly> | undefined; + readonly exportIntervalMs: number | undefined; + readonly maxBatchSize: number | undefined; + /** Metrics only. Spans and log records have no aggregation to prefer. */ + readonly temporality: MetricsTemporality | undefined; +} + +/** + * One signal's whole answer from these variables, so a caller whose endpoint + * came from somewhere else drops this one value and leaves nothing behind. + */ +export interface OtlpSignal { + readonly settings: OtlpSignalSettings | undefined; + /** Why a configured endpoint is not used, for the caller to report at startup. */ + readonly declined: string | undefined; + /** + * Whether these variables named this signal and then asked for no export. Kept + * apart from absent `settings`, which is what lets a stored endpoint answer. + */ + readonly off: boolean; +} + export interface OtelEnvironment { - /** Whether OTLP export is off, whatever endpoint is configured. */ + /** Whether anything is exported at all. */ readonly disabled: boolean; - /** Messages for the caller to log once at startup. */ + /** Settings that were named but could not be used, phrased for a human. */ readonly warnings: ReadonlyArray; - /** `OTEL_RESOURCE_ATTRIBUTES`, or nothing when it could not be read. */ + readonly traces: OtlpSignal; + readonly metrics: OtlpSignal; + readonly logs: OtlpSignal; + /** + * `OTEL_RESOURCE_ATTRIBUTES`, or nothing when it could not be read. + * `service.name` is deliberately absent: each T3 Code process names itself, + * and an attempt to set it through here is reported through `warnings`. + */ readonly resourceAttributes: Readonly>; + readonly serviceVersion: string | undefined; } +/** A set but blank value reads as unset, so the source under it can answer. */ +export const blankAsUnset = (value: string | undefined) => { + const trimmed = value?.trim(); + return trimmed === undefined || trimmed === "" ? undefined : trimmed; +}; + +/** A value this reader could use, and what to say about the ones it could not. */ +interface Parsed { + readonly value: A | undefined; + readonly warnings: ReadonlyArray; +} + +const usable = (value: A | undefined): Parsed => ({ value, warnings: [] }); + +/** + * Turns a read that would fail into the default plus a warning naming the value + * that caused it. Wrap every leaf read below in this. Pass `secret` for a + * variable whose value can carry a credential, and the warning names the + * variable without quoting what was in it: these warnings are written to the + * startup log, and a header list or a URL with userinfo in it would put the + * token there in plain text. + */ +const ignoring = ( + name: string, + expected: string, + config: Config.Config, + options?: { readonly secret?: boolean }, +): Config.Config> => + config.pipe( + Config.option, + Config.map((value) => usable(Option.getOrUndefined(value))), + Config.orElse(() => + Config.String(name).pipe( + Config.map((raw): Parsed => ({ + value: undefined, + warnings: [ + options?.secret === true + ? `${name} is not ${expected} and was ignored` + : `${name}=${raw} is not ${expected} and was ignored`, + ], + })), + ), + ), + ); + +/** + * Picks between a signal's own variable and the generic one. The signal's own + * owns it the moment it is set, unusable values included. + */ +const owned = ( + names: { readonly own: string; readonly generic: string }, + own: Parsed, + generic: Parsed, +): Parsed => + own.value === undefined && own.warnings.length === 0 + ? generic + : { + value: own.value, + warnings: [ + ...own.warnings, + ...generic.warnings, + ...(own.value === undefined && generic.value !== undefined + ? [`${names.own} names this signal, so ${names.generic} was not used in its place`] + : []), + ], + }; + +/** Tidies a raw value before a schema reads it. */ +const cleaned = >( + clean: (value: string) => string, + schema: S, +) => + Schema.String.pipe( + Schema.decodeTo( + schema, + SchemaTransformation.transform({ decode: clean, encode: (value: string) => value }), + ), + ); + +const folded = >(schema: S) => + cleaned((value) => value.trim().toLowerCase(), schema); + +const optionalString = (name: string) => + Config.String(name).pipe( + Config.option, + Config.map((value) => blankAsUnset(Option.getOrUndefined(value))), + ); + +/** `undefined` when the variable is unset, blank, or unreadable. */ interface Flag { - /** `undefined` when the variable is unset, blank, or unreadable. */ readonly value: boolean | undefined; readonly warning?: string; } -const TrimmedLowercase = Schema.String.pipe( - Schema.decodeTo( - Schema.String, - SchemaTransformation.trim().compose(SchemaTransformation.toLowerCase()), - ), -); - /** - * Reads a boolean that accepts `truthy` and `falsy`, ignoring case and padding. - * Any other value is ignored with a warning rather than failing startup. + * Reads a boolean that accepts an operator-chosen set of truthy and falsy + * spellings, ignoring case and padding. Any other value is ignored with a + * warning rather than failing startup. */ const flag = ( name: string, @@ -45,10 +187,7 @@ const flag = ( falsy: ReadonlyArray, invalid: (value: string) => string, ) => - Config.schema( - TrimmedLowercase.pipe(Schema.decodeTo(Schema.Literals([...truthy, ...falsy]))), - name, - ).pipe( + Config.schema(folded(Schema.Literals([...truthy, ...falsy])), name).pipe( Config.map((value): Flag => ({ value: truthy.includes(value) })), Config.orElse(() => Config.String(name).pipe( @@ -67,6 +206,339 @@ const flag = ( const T3CODE_TRUE = ["true", "yes", "on", "1", "y"]; const T3CODE_FALSE = ["false", "no", "off", "0", "n"]; +/** Intervals and batch sizes, where zero is a busy loop rather than a number. */ +const positiveInt = (name: string, subject: string) => + ignoring( + name, + `${subject} above zero`, + Config.schema(Schema.Int.check(Schema.isGreaterThan(0)), name), + ); + +/** + * Headers are a W3C Baggage string, read more strictly here than `Config.Record` + * reads one: its splitter divides on every `=`, which truncates base64 basic + * auth at its padding, and keeps the readable members of a malformed list, which + * is how `authorization=token,x-tenant` authenticates and then routes to the + * wrong tenant. + */ +const headerPairs = (raw: string): Readonly> | undefined => { + const entries: Record = {}; + for (const member of raw.split(",")) { + // Trailing and doubled commas are whitespace, not a member. + if (member.trim() === "") { + continue; + } + const separator = member.indexOf("="); + if (separator === -1) { + return undefined; + } + const key = member.slice(0, separator).trim(); + if (key === "") { + return undefined; + } + try { + entries[decodeURIComponent(key)] = decodeURIComponent(member.slice(separator + 1).trim()); + } catch { + return undefined; + } + } + // No pair at all is malformed rather than a request for no headers. + return Object.keys(entries).length === 0 ? undefined : entries; +}; + +const HeaderList = Schema.String.pipe( + Schema.decodeTo( + Schema.Record(Schema.String, Schema.String), + SchemaTransformation.transformEffect>, string>({ + decode: (raw, options) => { + const pairs = headerPairs(raw); + return pairs === undefined + ? Effect.fail( + new SchemaIssue.InvalidValue({ expected: "a list of key=value pairs" }, raw, options), + ) + : Effect.succeed(pairs); + }, + encode: (pairs) => + Effect.succeed( + Object.entries(pairs) + .map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(value)}`) + .join(","), + ), + }), + ), +); + +const headerList = (name: string) => + ignoring(name, "a valid list of key=value pairs", Config.schema(HeaderList, name), { + secret: true, + }); + +/** + * `OTEL_EXPORTER_OTLP__ENDPOINT` is a full URL and is used as given. The + * generic `OTEL_EXPORTER_OTLP_ENDPOINT` is a base each signal appends its own + * path to, on the path rather than on the end of the string, since the base may + * carry the query an intake takes its API key in. + */ +const signalEndpoint = (signal: OtlpSignalName) => { + const names = { + own: `OTEL_EXPORTER_OTLP_${signal}_ENDPOINT`, + generic: "OTEL_EXPORTER_OTLP_ENDPOINT", + }; + return Effect.gen(function* () { + const own = yield* ignoring(names.own, "a URL", Config.URL(names.own), { secret: true }); + const generic = yield* ignoring( + names.generic, + "a URL", + Config.URL(names.generic).pipe( + Config.map((base) => { + const url = new URL(base); + url.pathname = `${url.pathname.replace(/\/+$/, "")}/v1/${signal.toLowerCase()}`; + return url; + }), + ), + { secret: true }, + ); + const endpoint = owned(names, own, generic); + return { value: endpoint.value?.toString(), warnings: endpoint.warnings }; + }); +}; + +/** + * The exporters the specification names for each signal that T3 Code has no + * implementation of. Naming one is a deliberate "not OTLP". Per signal rather + * than pooled, so `OTEL_LOGS_EXPORTER=prometheus` reads as the mistake it is. + */ +const FOREIGN_EXPORTERS: Readonly>> = { + TRACES: new Set(["console", "logging", "zipkin", "jaeger"]), + METRICS: new Set(["console", "logging", "prometheus"]), + LOGS: new Set(["console", "logging"]), +}; + +/** + * `OTEL__EXPORTER` is a list, and `otlp` is its default. A list naming + * other exporters and not `otlp` is a deliberate "not this one"; a list naming + * nothing recognizable is a typo and leaves the default in place. + */ +const signalWantsOtlp = (signal: OtlpSignalName) => { + const name = `OTEL_${signal}_EXPORTER`; + return Effect.gen(function* () { + const raw = yield* optionalString(name); + if (raw === undefined) { + return usable(true); + } + const entries = yield* Config.Array(Schema.String, name).pipe( + Config.map((list) => + list.map((entry) => entry.trim().toLowerCase()).filter((entry) => entry !== ""), + ), + ); + if (entries.includes("otlp")) { + return { + value: true, + warnings: entries.some((entry) => entry !== "otlp") + ? [ + `${name}=${raw} names otlp, so this signal is exported over OTLP and nothing else in that list is honored`, + ] + : [], + }; + } + if (!entries.some((entry) => entry === "none" || FOREIGN_EXPORTERS[signal].has(entry))) { + return { + value: true, + warnings: [ + `${name}=${raw} names no exporter T3 Code recognizes and was ignored, so this signal is still exported over OTLP`, + ], + }; + } + // `none` is the specification's own way to say "export nothing" and needs no + // explanation. A foreign exporter does. + return { + value: false, + warnings: entries.includes("none") + ? [] + : [ + `${name}=${raw} asks for an exporter T3 Code does not have, so this signal is not exported`, + ], + }; + }); +}; + +/** The specification's own defaults, which apply once this route configures the exporter. */ +const SPEC_DEFAULT_PROTOCOL = "http/protobuf" as const; +const SPEC_DEFAULT_MAX_EXPORT_BATCH_SIZE = 512; + +/** + * How often each signal drains and how much it drains at once, under the + * variables and defaults the specification gives that signal. Metrics have no + * batch size because a collection cycle already bounds itself. + */ +const SIGNAL_BATCHING = { + TRACES: { + scheduleDelay: "OTEL_BSP_SCHEDULE_DELAY", + defaultScheduleDelayMs: 5_000, + maxExportBatchSize: "OTEL_BSP_MAX_EXPORT_BATCH_SIZE", + }, + METRICS: { + scheduleDelay: "OTEL_METRIC_EXPORT_INTERVAL", + defaultScheduleDelayMs: 60_000, + maxExportBatchSize: undefined, + }, + LOGS: { + scheduleDelay: "OTEL_BLRP_SCHEDULE_DELAY", + defaultScheduleDelayMs: 1_000, + maxExportBatchSize: "OTEL_BLRP_MAX_EXPORT_BATCH_SIZE", + }, +} as const satisfies Record< + OtlpSignalName, + { + readonly scheduleDelay: string; + readonly defaultScheduleDelayMs: number; + readonly maxExportBatchSize: string | undefined; + } +>; + +/** One signal as these variables read it, before the transport decision. */ +interface ReadSignal extends Parsed { + readonly off: boolean; +} + +const signalSettings = ( + signal: OtlpSignalName, + protocol: OtlpProtocol, + /** Metrics only, and passed in so an aggregation shares its endpoint's fate. */ + temporality: Parsed | undefined, +) => + Effect.gen(function* () { + const endpoint = yield* signalEndpoint(signal); + // Nothing else about a signal is read or reported until it has somewhere to go. + if (endpoint.value === undefined) { + return { value: undefined, warnings: endpoint.warnings, off: false } satisfies ReadSignal; + } + const wantsOtlp = yield* signalWantsOtlp(signal); + if (!wantsOtlp.value) { + return { value: undefined, warnings: wantsOtlp.warnings, off: true } satisfies ReadSignal; + } + const names = { + own: `OTEL_EXPORTER_OTLP_${signal}_HEADERS`, + generic: "OTEL_EXPORTER_OTLP_HEADERS", + }; + const headers = owned(names, yield* headerList(names.own), yield* headerList(names.generic)); + const batching = SIGNAL_BATCHING[signal]; + const interval = yield* positiveInt(batching.scheduleDelay, "an export interval"); + const batchSize = + batching.maxExportBatchSize === undefined + ? usable(undefined) + : yield* positiveInt(batching.maxExportBatchSize, "a batch size"); + return { + value: { + url: endpoint.value, + protocol, + headers: headers.value, + exportIntervalMs: interval.value ?? batching.defaultScheduleDelayMs, + maxBatchSize: + batching.maxExportBatchSize === undefined + ? undefined + : (batchSize.value ?? SPEC_DEFAULT_MAX_EXPORT_BATCH_SIZE), + temporality: temporality?.value, + }, + warnings: [ + ...wantsOtlp.warnings, + ...headers.warnings, + ...interval.warnings, + ...batchSize.warnings, + ...(temporality?.warnings ?? []), + ], + off: false, + } satisfies ReadSignal; + }); + +/** What one signal should do about its wire format. */ +interface SignalProtocol { + readonly protocol: OtlpProtocol; + readonly declined: string | undefined; +} + +interface ProtocolDecision { + readonly traces: SignalProtocol; + readonly metrics: SignalProtocol; + readonly logs: SignalProtocol; + readonly warnings: ReadonlyArray; +} + +const wireProtocol = (name: string) => + ignoring( + name, + "a known OTLP protocol", + Config.schema(folded(Schema.Literals(["http/json", "http/protobuf", "grpc"])), name), + ).pipe(Config.map((parsed) => ({ ...parsed, name }))); + +/** + * Each signal builds its own serializer, so all three are answered separately + * and are free to disagree, and `grpc` declines only the signal that named it. + */ +const resolveProtocol = Effect.gen(function* () { + const generic = yield* wireProtocol("OTEL_EXPORTER_OTLP_PROTOCOL"); + const named = { + traces: yield* wireProtocol("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"), + metrics: yield* wireProtocol("OTEL_EXPORTER_OTLP_METRICS_PROTOCOL"), + logs: yield* wireProtocol("OTEL_EXPORTER_OTLP_LOGS_PROTOCOL"), + }; + const decide = (own: typeof generic): SignalProtocol => { + const asked = own.value === undefined ? generic : own; + if (asked.value === undefined) { + return { protocol: SPEC_DEFAULT_PROTOCOL, declined: undefined }; + } + return asked.value === "grpc" + ? { + protocol: SPEC_DEFAULT_PROTOCOL, + declined: `${asked.name}=grpc is not supported; T3 Code exports OTLP over HTTP only, so this signal is not exported`, + } + : { protocol: asked.value, declined: undefined }; + }; + return { + traces: decide(named.traces), + metrics: decide(named.metrics), + logs: decide(named.logs), + warnings: [ + ...generic.warnings, + ...named.traces.warnings, + ...named.metrics.warnings, + ...named.logs.warnings, + ], + } satisfies ProtocolDecision; +}); + +const TEMPORALITY_PREFERENCE = "OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE"; + +/** + * `lowmemory` is a real preference this exporter cannot express per instrument + * kind, and resolves to `delta` because that is what it asks for on the counters + * and timers T3 Code records. + */ +const resolveMetricsTemporality = ignoring( + TEMPORALITY_PREFERENCE, + "a known preference", + Config.schema( + folded(Schema.Literals(["cumulative", "delta", "lowmemory"])), + TEMPORALITY_PREFERENCE, + ), +).pipe( + Config.map((preference): Parsed => + preference.value === "lowmemory" + ? { + value: "delta", + warnings: [ + `${TEMPORALITY_PREFERENCE}=lowmemory cannot be expressed per instrument kind here, so delta is used for every metric sent to the endpoint these variables named, which is what lowmemory asks for on the counters and timers T3 Code records`, + ], + } + : { + value: preference.value, + warnings: preference.warnings.map( + (warning) => `${warning}, so metrics are exported as ${DEFAULT_METRICS_TEMPORALITY}`, + ), + }, + ), +); + const RESOURCE_ATTRIBUTES = "OTEL_RESOURCE_ATTRIBUTES"; interface ResourceAttributes { @@ -93,43 +565,192 @@ const resourceAttributes = Config.Record( Config.withDefault({ value: {} }), ); -export const load: Effect.Effect = Config.all({ - t3: flag( +/** + * On top of the exporters' own read above: `OTEL_SERVICE_VERSION` becomes + * `serviceVersion`, and `service.name` is refused, from either variable, + * because every T3 Code process names itself. + */ +const resolveResource = Effect.gen(function* () { + const resource = yield* resourceAttributes; + const { "service.name": attributeName, ...attributes } = resource.value; + const serviceName = yield* optionalString("OTEL_SERVICE_NAME"); + const serviceVersion = yield* optionalString("OTEL_SERVICE_VERSION"); + // Read only to say it was refused, rather than dropped without a word. + const declinedName = + serviceName !== undefined + ? "OTEL_SERVICE_NAME" + : attributeName === undefined + ? undefined + : `${RESOURCE_ATTRIBUTES}=service.name`; + return { + attributes, + serviceVersion, + warnings: [ + ...(resource.warning === undefined ? [] : [resource.warning]), + ...(declinedName === undefined + ? [] + : [ + `${declinedName} was ignored; every T3 Code process names itself, so use OTEL_RESOURCE_ATTRIBUTES to tell instances apart instead`, + ]), + ], + }; +}); + +const UNREADABLE = "the OpenTelemetry environment could not be read"; + +/** Names whichever variable switched export off, and how to overrule it. */ +const disabledBy = (name: string) => + name === "OTEL_SDK_DISABLED" + ? "OTEL_SDK_DISABLED is set, so no telemetry is exported, whatever configured it; set T3CODE_OTEL_SDK_DISABLED=false to export anyway" + : `${name} is set, so no telemetry is exported, whatever configured it`; + +/** + * One signal's whole answer, with the transport decision folded in. A decline is + * reported only for a signal that had somewhere to go and asked for OTLP, since + * on any other signal gRPC would name the wrong cause. + */ +const signalOf = (signal: ReadSignal, transport: SignalProtocol): OtlpSignal => + signal.value === undefined || transport.declined === undefined + ? { settings: signal.value, declined: undefined, off: signal.off } + : { settings: undefined, declined: transport.declined, off: true }; + +/** + * Read the environment. Never fails: a variable T3 Code cannot honor leaves its + * setting unset and is reported through `warnings`. + */ +export const load: Effect.Effect = Effect.gen(function* () { + const t3 = yield* flag( "T3CODE_OTEL_SDK_DISABLED", T3CODE_TRUE, T3CODE_FALSE, (value) => `T3CODE_OTEL_SDK_DISABLED=${value} is not a yes or a no and was ignored`, - ), + ); // The specification: a boolean it defines is true "only by the // case-insensitive string `true`", implementations "MUST NOT" accept other // values as true, and should warn about unrecognized ones. - spec: flag( + const spec = yield* flag( "OTEL_SDK_DISABLED", ["true"], ["false"], (value) => `OTEL_SDK_DISABLED=${value} was read as false; the OpenTelemetry specification recognizes only the string true, so use OTEL_SDK_DISABLED=true or T3CODE_OTEL_SDK_DISABLED to say it any other way`, - ), - resource: resourceAttributes, + ); + const disabled = t3.value ?? spec.value ?? false; + const protocolDecision = yield* resolveProtocol; + const resource = yield* resolveResource; + const temporality = yield* resolveMetricsTemporality; + const silent: ReadSignal = { value: undefined, warnings: [], off: false }; + const traces = disabled + ? silent + : yield* signalSettings("TRACES", protocolDecision.traces.protocol, undefined); + const metrics = disabled + ? silent + : yield* signalSettings("METRICS", protocolDecision.metrics.protocol, temporality); + const logs = disabled + ? silent + : yield* signalSettings("LOGS", protocolDecision.logs.protocol, undefined); + return { + disabled, + // Every signal reads the generic variables, so one bad value arrives thrice. + warnings: [ + ...new Set([ + ...(t3.warning === undefined ? [] : [t3.warning]), + ...(spec.warning === undefined ? [] : [spec.warning]), + ...(disabled + ? [disabledBy(t3.value === true ? "T3CODE_OTEL_SDK_DISABLED" : "OTEL_SDK_DISABLED")] + : []), + ...protocolDecision.warnings, + ...resource.warnings, + ...traces.warnings, + ...metrics.warnings, + ...logs.warnings, + ]), + ], + traces: signalOf(traces, protocolDecision.traces), + metrics: signalOf(metrics, protocolDecision.metrics), + logs: signalOf(logs, protocolDecision.logs), + resourceAttributes: resource.attributes, + serviceVersion: resource.serviceVersion, + }; }).pipe( - Effect.map(({ t3, spec, resource }) => { - const disabled = t3.value ?? spec.value ?? false; - const warnings = [t3.warning, spec.warning, resource.warning].filter( - (warning) => warning !== undefined, - ); - if (disabled) { - warnings.push( - t3.value - ? "T3CODE_OTEL_SDK_DISABLED is set, so no telemetry is exported, whatever configured it" - : "OTEL_SDK_DISABLED is set, so no telemetry is exported, whatever configured it; set T3CODE_OTEL_SDK_DISABLED=false to export anyway", - ); - } - return { disabled, warnings, resourceAttributes: resource.value }; - }), - // Every read above falls back instead of failing, so this cannot happen. - Effect.orDie, + Effect.catchCause((cause) => + Effect.logWarning("Could not read the OpenTelemetry environment", cause).pipe( + Effect.as({ + disabled: false, + warnings: [], + traces: { settings: undefined, declined: UNREADABLE, off: false }, + metrics: { settings: undefined, declined: UNREADABLE, off: false }, + logs: { settings: undefined, declined: UNREADABLE, off: false }, + resourceAttributes: {}, + serviceVersion: undefined, + }), + ), + ), ); +/** A signal these variables said nothing usable about. */ +const noSignal: OtlpSignal = { settings: undefined, declined: undefined, off: false }; + +/** + * Applies the whole-signal rule to the knobs and not only to the URL. Call this + * rather than `settings?.headers ?? t3Headers`, which collapses the two cases it + * has to keep apart: absent settings mean the standard variables named nothing, + * while settings silent about one knob mean they own the signal and left that + * knob unset, which is an answer of its own. + */ +export const resolveSignalExport = (input: { + readonly settings: OtlpSignalSettings | undefined; + readonly t3Protocol: OtlpProtocol; + readonly t3Headers: Readonly> | undefined; + readonly t3ExportIntervalMs: number; +}): SignalExport => + input.settings === undefined + ? { + protocol: input.t3Protocol, + headers: input.t3Headers, + exportIntervalMs: input.t3ExportIntervalMs, + maxBatchSize: undefined, + temporality: DEFAULT_METRICS_TEMPORALITY, + } + : { + protocol: input.settings.protocol, + headers: input.settings.headers, + exportIntervalMs: input.settings.exportIntervalMs ?? input.t3ExportIntervalMs, + maxBatchSize: input.settings.maxBatchSize, + temporality: input.settings.temporality ?? DEFAULT_METRICS_TEMPORALITY, + }; + +/** + * Where one signal's endpoint comes from, and therefore which source configures + * the rest of it: T3 Code's own name, then the standard `OTEL_*` names, then + * whatever was persisted. Whichever source wins takes the whole signal, so the + * signal returned is `noSignal` unless `OTEL_*` is what won, and a signal those + * variables switched off is not one they said nothing about. + * + * Read by every process that exports, so two of them cannot resolve the same + * machine's variables differently. + */ +export const resolveSignalSource = (input: { + readonly t3Url: string | undefined; + readonly signal: OtlpSignal; + readonly persistedUrl: string | undefined; +}): { readonly url: string | undefined; readonly signal: OtlpSignal } => { + const t3Url = blankAsUnset(input.t3Url); + if (t3Url !== undefined) { + return { url: t3Url, signal: noSignal }; + } + if (input.signal.settings !== undefined) { + return { url: input.signal.settings.url, signal: input.signal }; + } + if (input.signal.off) { + return { url: undefined, signal: input.signal }; + } + const persistedUrl = blankAsUnset(input.persistedUrl); + return persistedUrl === undefined + ? { url: undefined, signal: input.signal } + : { url: persistedUrl, signal: noSignal }; +}; + /** * Provide this around Effect's OTLP exporters, which read * `OTEL_RESOURCE_ATTRIBUTES` for themselves and die when it does not decode, @@ -153,5 +774,9 @@ export const layerResourceAttributes = (attributes: Readonly