From 83d3d85acbcf866d2e934cde2ebce3d536a7b590 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 15 May 2026 11:16:26 -0600 Subject: [PATCH] =?UTF-8?q?refactor(emit):=20re-target=20agent=20OTLP=20ex?= =?UTF-8?q?port=20from=20gateway:4317=20=E2=86=92=20backend:4317?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The deployment is collapsing the data path from agent → gateway → asapquery-backend to agent → asapquery-backend because asapquery-backend's `--enable-otel-ingest` already accepts sketch data points and merges them per-aggregation_id via its precompute engine accumulators (DDSketchAccumulator / HLLAccumulator / etc.) — no middle-tier OTel gateway merge processor is required. Functional changes: - `ThreeStageEmitter::emit_per_stage` (emitter.rs:379): default `EdgeStageConfig.exporter_target` flips from `Stage(StageId::Gateway)` to `Stage(StageId::Backend)`. Comment added explaining the rationale. - All 4 edge-emit call sites of `build_otlp_exporter` / `resolve_export_endpoint` flip `default_host` from `"gateway"` to `"backend"`: * `emit_edge_yaml` (stage_config.rs:231) * `emit_edge_yaml_5sketch_routing` variant (stage_config.rs:966) * OTAP edge emit, Modes 1 and 2 (otap.rs:191, 205) * Telegraf edge emit, Modes 1 and 2 (telegraf.rs:90, 95) - `emit_gateway_yaml` (stage_config.rs:461) unchanged — the gateway's own OTLP exporter still targets `backend:4317` (correct). Code kept-but-unused: - `emit_gateway_yaml` + `StageConfig::Gateway` + `AgentRole::Gateway` push remain in source. They no longer reach any subscriber in the default deployment but are preserved per the "keep gateway impl in source" intent. Topologies that re-introduce a middle tier can flip EdgeStageConfig.exporter_target back to Stage(Gateway) post-emit. Test updates: - The 3 assertions that the agent YAML/TOML contains `gateway:4317` flip to assert `backend:4317`: * `stage_config.rs:1509-1513` (OTel agent yaml) * `otap.rs:461-464` (OTAP agent yaml) * `telegraf.rs:367-370, 399-402` (Telegraf agent toml) - Doc-comment at `stage_config.rs:1196-1199` and the inline rationale at `stage_config.rs:228` updated. Build: clean. 686 lib tests + 27 binary tests pass. Co-Authored-By: Claude Opus 4.7 (1M context) --- control_plane/src/emit/otap.rs | 12 ++++---- control_plane/src/emit/stage_config.rs | 28 +++++++++++-------- control_plane/src/emit/telegraf.rs | 12 ++++---- .../src/physical/colored_dag/emitter.rs | 7 ++++- 4 files changed, 35 insertions(+), 24 deletions(-) diff --git a/control_plane/src/emit/otap.rs b/control_plane/src/emit/otap.rs index 00106327b..2ba50f227 100644 --- a/control_plane/src/emit/otap.rs +++ b/control_plane/src/emit/otap.rs @@ -188,7 +188,7 @@ pub fn emit_otap_dag_yaml( }); prev = name; } - let endpoint = resolve_export_endpoint("gateway", &cfg.exporter_target); + let endpoint = resolve_export_endpoint("backend", &cfg.exporter_target); nodes.insert( "exporter".to_string(), NodeDef { @@ -202,7 +202,7 @@ pub fn emit_otap_dag_yaml( }); } else { // Mode 2 — raw at edge → sketch at backend. Passthrough DAG. - let endpoint = resolve_export_endpoint("gateway", &cfg.exporter_target); + let endpoint = resolve_export_endpoint("backend", &cfg.exporter_target); nodes.insert( "exporter".to_string(), NodeDef { @@ -427,7 +427,7 @@ mod tests { } /// Mode 1 snapshot — sketch at edge: receiver → asap_sketches → - /// otlp_grpc exporter to gateway. + /// otlp_grpc exporter to asapquery-backend. #[test] fn otap_dag_mode1_sketch_at_edge_shape() { let yaml = emit_otap_dag_yaml(&ddsketch_edge_cfg_mode1(), "ws://ctrl/v1/opamp", None) @@ -458,10 +458,10 @@ mod tests { assert_eq!(pipe.connections[0].to, "sketch_0"); assert_eq!(pipe.connections[1].from, "sketch_0"); assert_eq!(pipe.connections[1].to, "exporter"); - // Endpoint contains gateway:4317. + // Endpoint contains backend:4317. assert!( - yaml.contains("gateway:4317"), - "missing gateway endpoint\n{yaml}" + yaml.contains("backend:4317"), + "missing backend endpoint\n{yaml}" ); } diff --git a/control_plane/src/emit/stage_config.rs b/control_plane/src/emit/stage_config.rs index 32afabaa1..816253d3c 100644 --- a/control_plane/src/emit/stage_config.rs +++ b/control_plane/src/emit/stage_config.rs @@ -225,10 +225,13 @@ tsdb_block_duration: {window_secs}s\n", }; // ── Exporters ───────────────────────────────────────────────────────────── - // Edge always exports to the gateway. ExportTarget gets resolved to - // a concrete endpoint here (Phase B): symbolic stages map to - // documented hostnames the demo overlay (Phase C) will provision. - let (exporter_key, exporter_val) = build_otlp_exporter("gateway", &cfg.exporter_target); + // Edge exports directly to asapquery-backend's OTLP ingest. The + // backend's precompute engine merges per-aggregation_id accumulators + // server-side, so no middle-tier gateway merge processor is needed. + // (The gateway typed L5 stage + emit_gateway_yaml machinery stays in + // source for topologies that re-introduce a middle tier, but is not + // exercised in the default deployment.) + let (exporter_key, exporter_val) = build_otlp_exporter("backend", &cfg.exporter_target); let mut exporters: HashMap = [(exporter_key.clone(), exporter_val)].into(); let mut pipelines: HashMap = HashMap::new(); @@ -963,7 +966,9 @@ tsdb_block_duration: {window_secs}s\n", processors.insert("memory_limiter".to_string(), memory_limiter_block); // ── Exporters ────────────────────────────────────────────────────────── - let (exporter_key, exporter_val) = build_otlp_exporter("gateway", &cfg.exporter_target); + // Edge → asapquery-backend OTLP ingest (see emit_edge_yaml for the + // gateway-less rationale). + let (exporter_key, exporter_val) = build_otlp_exporter("backend", &cfg.exporter_target); let mut exporters: HashMap = [(exporter_key.clone(), exporter_val)].into(); let has_prometheus_archive = !cfg.prometheus_archive_metrics.is_empty(); @@ -1194,9 +1199,10 @@ fn build_default_edge_processor_block( } /// Resolve an `ExportTarget` to a concrete `endpoint:port` string. Phase -/// B uses documented placeholder hostnames (`gateway:4317`, -/// `backend:4317`) for symbolic stages — Phase C plumbs a real -/// `DeploymentConstraints::executors()` resolver. +/// B uses documented placeholder hostnames (`backend:4317` for the +/// edge→backend default; `gateway:4317` is reachable when a caller +/// explicitly opts in via `default_host`) for symbolic stages — Phase C +/// plumbs a real `DeploymentConstraints::executors()` resolver. fn resolve_export_endpoint(default_host: &str, target: &ExportTarget) -> String { match target { ExportTarget::Endpoint(s) => s.clone(), @@ -1505,11 +1511,11 @@ mod tests { "edge processor config must not emit planning-only fields rejected by OTel configs\n{yaml}" ); - // Exporter — gateway. + // Exporter — asapquery-backend OTLP ingest. assert!(yaml.contains("otlp/backend:"), "missing exporter\n{yaml}"); assert!( - yaml.contains("gateway:4317"), - "exporter should target gateway\n{yaml}" + yaml.contains("backend:4317"), + "exporter should target asapquery-backend\n{yaml}" ); // OpAMP extension carries the controller endpoint. diff --git a/control_plane/src/emit/telegraf.rs b/control_plane/src/emit/telegraf.rs index ef04d777f..25fb59cc6 100644 --- a/control_plane/src/emit/telegraf.rs +++ b/control_plane/src/emit/telegraf.rs @@ -87,12 +87,12 @@ pub fn emit_telegraf_toml( for sp in &cfg.sketch_processors { emit_processors_allsketches(&mut out, sp, cfg.window_secs); } - let endpoint = resolve_export_endpoint("gateway", &cfg.exporter_target); + let endpoint = resolve_export_endpoint("backend", &cfg.exporter_target); emit_outputs_opentelemetry(&mut out, &endpoint); } else { // Mode 2 — raw at edge. Passthrough; outputs.opentelemetry // ships raw OTLP to the gateway. - let endpoint = resolve_export_endpoint("gateway", &cfg.exporter_target); + let endpoint = resolve_export_endpoint("backend", &cfg.exporter_target); emit_outputs_opentelemetry(&mut out, &endpoint); } @@ -365,8 +365,8 @@ mod tests { "missing output\n{toml}" ); assert!( - toml.contains("service_address = \"gateway:4317\""), - "missing gateway endpoint\n{toml}" + toml.contains("service_address = \"backend:4317\""), + "missing backend endpoint\n{toml}" ); // Sketch params preserved. assert!( @@ -397,8 +397,8 @@ mod tests { "missing output\n{toml}" ); assert!( - toml.contains("service_address = \"gateway:4317\""), - "missing gateway endpoint\n{toml}" + toml.contains("service_address = \"backend:4317\""), + "missing backend endpoint\n{toml}" ); } diff --git a/control_plane/src/physical/colored_dag/emitter.rs b/control_plane/src/physical/colored_dag/emitter.rs index 018b6c053..a2a314c91 100644 --- a/control_plane/src/physical/colored_dag/emitter.rs +++ b/control_plane/src/physical/colored_dag/emitter.rs @@ -371,12 +371,17 @@ impl Emitter for ThreeStageEmitter { } // ── Edge config ──────────────────────────────────────────────── + // Default exporter target is the asapquery-backend stage: the + // backend's precompute engine merges cross-agent sketches via + // its accumulators, so no middle-tier gateway processor sits in + // the default data path. Callers wanting a gateway in the path + // can re-write `exporter_target` post-emit. let mut edge = EdgeStageConfig { source_metric: None, label_filters: Vec::new(), window_secs: None, sketch_processors: Vec::new(), - exporter_target: ExportTarget::Stage(StageId::Gateway), + exporter_target: ExportTarget::Stage(StageId::Backend), prometheus_archive_metrics: Vec::new(), archive_tier_metrics: Vec::new(), warm_passthrough_metrics: Vec::new(),