From c6d3784f5454eb8eaf6b63b60c23735da0f3e385 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 15 May 2026 12:11:39 -0600 Subject: [PATCH] =?UTF-8?q?refactor(deploy):=20retire=20gateway=20from=20d?= =?UTF-8?q?efault=20data=20path=20=E2=80=94=20agent=20=E2=86=92=20asapquer?= =?UTF-8?q?y-backend?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default data path is now: fake-exporter → agent → asapquery-backend collapsing the former: fake-exporter → agent → gateway → asapquery-backend Why this works: asapquery-backend's `--enable-otel-ingest` already accepts sketch data points and merges them per-aggregation_id via its precompute engine accumulators (DDSketchAccumulator / HLLAccumulator / etc.) — no middle-tier OTel gateway merge processor is required. The asapquery-backend control-plane PRs #241 / #242 / #243 retire the gateway-targeted emit pieces (legacy backend-collector emitter + shadow `BackendCollectorConfig` + agent OTLP default_host). Code kept-in-source per "keep gateway impl in repos" intent: - All `configs/gateway*.yaml` files preserved - `configs/asap/asap-otel-gateway-mvp-placeholder.yaml` preserved - Gateway service definition in `base.yml` profile-gated under `["gateway-legacy"]` — `docker compose --profile gateway-legacy …` still spins it up - `gateway_up` / `gateway_down` functions in multinode `run_demo.sh` defined-but-uncalled (lifecycle dropped from `arm_up` / `arm_down`) Files touched: Agent OTel configs (rename `otlp/gateway` exporter block to `otlp/backend`, change `endpoint: gateway:4317` → `endpoint: backend:4317`): - mvp-singlenode/configs/asap-otel-agent-{b0a-raw-stream, b0b-raw-batched, b2-full, b3-delta, b4-tunable, b6-asap-single-sketch}.yaml - mvp-multinode/configs/asap/asap-otel-agent-b6-asap-single-sketch.yaml Workload tags (assign_to_role: backend|gateway → agent): - mvp-singlenode/configs/workloads.yaml (2 rows) - mvp-singlenode/configs/mvp-workload.yaml (1 row) - mvp-multinode/configs/asap/mvp-workload.yaml (1 row) Compose: - mvp-singlenode/docker-compose/base.yml — gateway service `profiles: ["gateway-legacy"]`; fake-exporter default EXPORTER_TARGET=backend:4317; fake-exporter depends_on→backend - mvp-singlenode/docker-compose/e2e-overlay.yml — gateway override also `profiles: ["gateway-legacy"]` - mvp-singlenode/docker-compose/agents-N{1,10,100}.yml — agent depends_on→backend (was gateway) - mvp-singlenode/docker-compose/gen-agents.sh — template emits same - mvp-singlenode/docker-compose/mvp-multi-stage.yml — gateway override `profiles: ["gateway-legacy"]`; comment notes how to re-enable multi-stage flow (requires `--profile gateway-legacy` + gateway- targeting AGENT_CONFIG) Multinode orchestration: - mvp-multinode/topology.env — NODE1 reclassified "unused / reserved" (DNS alias preserved for opt-in) - mvp-multinode/scripts/run_demo.sh — gateway_up/down dropped from arm lifecycle (functions remain) - mvp-multinode/scripts/run_demo_sweep.sh — roles map node1 "gateway" → "unused" Docs: - mvp-singlenode/README.md — port table + topology blurb updated - mvp-multinode/README.md — topology diagram updated Co-Authored-By: Claude Opus 4.7 (1M context) --- deploy/mvp-multinode/README.md | 8 ++- ...asap-otel-agent-b6-asap-single-sketch.yaml | 2 +- .../configs/asap/mvp-workload.yaml | 8 +-- deploy/mvp-multinode/scripts/run_demo.sh | 20 ++++--- .../mvp-multinode/scripts/run_demo_sweep.sh | 2 +- deploy/mvp-multinode/topology.env | 8 ++- deploy/mvp-singlenode/README.md | 12 +++-- .../asap-otel-agent-b0a-raw-stream.yaml | 6 +-- .../asap-otel-agent-b0b-raw-batched.yaml | 6 +-- .../configs/asap-otel-agent-b2-full.yaml | 16 +++--- .../configs/asap-otel-agent-b3-delta.yaml | 6 +-- .../configs/asap-otel-agent-b4-tunable.yaml | 6 +-- ...asap-otel-agent-b6-asap-single-sketch.yaml | 2 +- .../mvp-singlenode/configs/mvp-workload.yaml | 8 +-- deploy/mvp-singlenode/configs/workloads.yaml | 7 ++- .../docker-compose/agents-N1.yml | 6 ++- .../docker-compose/agents-N10.yml | 3 +- .../docker-compose/agents-N100.yml | 3 +- deploy/mvp-singlenode/docker-compose/base.yml | 53 +++++++++++++------ .../docker-compose/e2e-overlay.yml | 14 ++--- .../docker-compose/gen-agents.sh | 6 ++- .../docker-compose/mvp-multi-stage.yml | 29 +++++++--- 22 files changed, 152 insertions(+), 79 deletions(-) diff --git a/deploy/mvp-multinode/README.md b/deploy/mvp-multinode/README.md index 650a990e..e766eb04 100644 --- a/deploy/mvp-multinode/README.md +++ b/deploy/mvp-multinode/README.md @@ -12,8 +12,12 @@ Single-host driver: `deploy/mvp-singlenode/scripts/run_mvp_demo.sh` (runs all co node0 ──────►│ producers + agent-a (data source) │ 10.10.1.1 │ │ │ │ - node1 ◄──────│ gateway (ASAP arm only) │ - 10.10.1.2 │ │ + node1 ── │ unused / reserved (former gateway slot; │ + 10.10.1.2 │ agent → asapquery- │ + │ backend is the default │ + │ path. `gateway_up()` │ + │ in run_demo.sh stays │ + │ defined-but-uncalled.) │ │ │ node2 ◄──────│ backend stack (asap-query-backend, │ 10.10.1.3 │ minio, thanos-*, │ diff --git a/deploy/mvp-multinode/configs/asap/asap-otel-agent-b6-asap-single-sketch.yaml b/deploy/mvp-multinode/configs/asap/asap-otel-agent-b6-asap-single-sketch.yaml index 482aaf89..4f5c52b2 100644 --- a/deploy/mvp-multinode/configs/asap/asap-otel-agent-b6-asap-single-sketch.yaml +++ b/deploy/mvp-multinode/configs/asap/asap-otel-agent-b6-asap-single-sketch.yaml @@ -301,7 +301,7 @@ connectors: exporters: otlp/backend: - endpoint: gateway:4317 + endpoint: backend:4317 tls: insecure: true # Backend's tonic gRPC server doesn't accept gzip-compressed bodies diff --git a/deploy/mvp-multinode/configs/asap/mvp-workload.yaml b/deploy/mvp-multinode/configs/asap/mvp-workload.yaml index 7dfe99dc..f6a9a8d9 100644 --- a/deploy/mvp-multinode/configs/asap/mvp-workload.yaml +++ b/deploy/mvp-multinode/configs/asap/mvp-workload.yaml @@ -54,12 +54,14 @@ # 2. LABEL AGGREGATION ACROSS SERIES AT ONE TIMESTAMP # Outer: sum-by(zone) at instant. Inner: identity. -# Stage: gateway (cross-host fan-in merge); backend stores -# the merged sum-by-zone vector. +# Stage: edge sketches per-host; asapquery-backend's precompute +# engine performs the cross-host sum-by(zone) fan-in merge via its +# per-aggregation_id accumulators. (No middle-tier gateway OTel +# collector role is deployed; the merge happens server-side.) - metric_name: http_requests_total query_string: "sum by (zone) (http_requests_total)" accuracy_sla: 0.0 - assign_to_role: gateway + assign_to_role: agent # 3. COMBINED WINDOW + LABEL AGGREGATION # Inner: rate per series over 5m. Outer: sum-by(zone). diff --git a/deploy/mvp-multinode/scripts/run_demo.sh b/deploy/mvp-multinode/scripts/run_demo.sh index 93166842..38c009f5 100755 --- a/deploy/mvp-multinode/scripts/run_demo.sh +++ b/deploy/mvp-multinode/scripts/run_demo.sh @@ -6,17 +6,21 @@ # freshness) but distributed across 4 hosts on 10.10.1.0/24: # # node0 (10.10.1.1) producers + agent-a (data source) -# node1 (10.10.1.2) gateway (ASAP arm only) +# node1 (10.10.1.2) unused / reserved (former gateway slot — see +# gateway_up() below, now +# defined-but-uncalled) # node2 (10.10.1.3) backend stack (controller, asap-query-backend, # prometheus, minio, # thanos-{query,store-gateway,compact}) # node3 (10.10.1.4) producers + agent-b (data source) # # Three arms run back-to-back over the same workload: -# b0 OTel agent → Prometheus (PRW) [no gateway, no backend] -# b1 OTel agent + serfprocessor → Prometheus [no gateway, no backend] -# asap OTel agent → gateway → asap-query-backend [+ controller, -# + Thanos/MinIO archive, sketches per controller plan] +# b0 OTel agent → Prometheus (PRW) [no backend] +# b1 OTel agent + serfprocessor → Prometheus [no backend] +# asap OTel agent → asap-query-backend [+ controller, +# + Thanos/MinIO archive, sketches per controller plan; +# asap-query-backend's --enable-otel-ingest merges sketches +# per-aggregation_id via precompute-engine accumulators] # # Scope notes vs. the canonical single-host `run_mvp_demo.sh`: # - Uses `docker run --network host --add-host` (no docker-compose, no @@ -332,11 +336,14 @@ agents_down() { } # ─── ARM lifecycle ────────────────────────────────────────────────── +# gateway_up / gateway_down are no longer called by arm lifecycle — +# the default data path is agent → asapquery-backend directly. The +# functions stay defined so a topology that re-introduces a middle +# tier can re-enable them by re-adding the calls. arm_up() { local arm=$1 log "=== ARM UP: ${arm} ===" backend_up "${arm}" - gateway_up "${arm}" sleep 5 agents_up "${arm}" log "=== arm ${arm} all containers started; waiting WARMUP_S=${WARMUP_S} ===" @@ -346,7 +353,6 @@ arm_up() { arm_down() { log "=== ARM DOWN ===" agents_down - gateway_down backend_down } diff --git a/deploy/mvp-multinode/scripts/run_demo_sweep.sh b/deploy/mvp-multinode/scripts/run_demo_sweep.sh index ec98f9d3..b4181986 100755 --- a/deploy/mvp-multinode/scripts/run_demo_sweep.sh +++ b/deploy/mvp-multinode/scripts/run_demo_sweep.sh @@ -120,7 +120,7 @@ print(f"\\nRun: \`${RUN_ID}\` | Soak: \`${SOAK}s\` | Workload: PER_AGENT_CAR print("## §1 Per-arm NIC bandwidth (cluster-wide, /sys/class/net/enp130s0f0)\\n") print("| arm | node | role | rx_MB/s | tx_MB/s | rx_total_MB | tx_total_MB |") print("|---|---|---|---|---|---|---|") -roles = {"node0":"producer+agent-a","node1":"gateway","node2":"backend","node3":"producer+agent-b"} +roles = {"node0":"producer+agent-a","node1":"unused","node2":"backend","node3":"producer+agent-b"} for f in sorted(glob.glob(os.path.join(RUN_DIR,"nic-*.csv"))): arm = os.path.basename(f).replace("nic-","").replace(".csv","") with open(f) as fh: diff --git a/deploy/mvp-multinode/topology.env b/deploy/mvp-multinode/topology.env index c64e66bd..a6c1356f 100644 --- a/deploy/mvp-multinode/topology.env +++ b/deploy/mvp-multinode/topology.env @@ -3,7 +3,13 @@ # ── Node IPs ── NODE0_IP=10.10.1.1 # producers + agent-a (data source) -NODE1_IP=10.10.1.2 # gateway (ASAP arm only) +NODE1_IP=10.10.1.2 # unused / reserved (former gateway-role host; the + # default data path is agent → asapquery-backend, so + # no container is brought up here. The `gateway:` + # DNS alias below still resolves to this IP for + # callers that opt back into the gateway tier — see + # gateway_up() in scripts/run_demo.sh, currently + # uncalled but preserved in source.) NODE2_IP=10.10.1.3 # backend stack (asap-query-backend, MinIO, Thanos, Prometheus, controller) NODE3_IP=10.10.1.4 # producers + agent-b (data source) diff --git a/deploy/mvp-singlenode/README.md b/deploy/mvp-singlenode/README.md index 9fd73039..93e078fe 100644 --- a/deploy/mvp-singlenode/README.md +++ b/deploy/mvp-singlenode/README.md @@ -13,9 +13,12 @@ _Last updated: 2026-04-23 (post N=10 sweep)._ N ∈ {1, 10, 100} # number of edge agents ``` -All other components (backend, gateway, controller, MinIO, -Prometheus, Grafana) stay at 1 replica — paper's claim is "one -controller coordinates N agents". +All other components (backend, controller, MinIO, Prometheus, +Grafana) stay at 1 replica — paper's claim is "one controller +coordinates N agents". The OTel **gateway** tier is opt-in via the +`gateway-legacy` profile (see `base.yml`); the default data path is +agent → asapquery-backend, with the precompute engine merging +sketches server-side via its per-aggregation_id accumulators. ## Compose (dev / small & mid scale) @@ -54,7 +57,8 @@ agent containers. Host ports: | 9090 | Prometheus | | | 3000 | Grafana | (admin/admin, anon viewer also allowed) | | 9000 / 9001 | MinIO S3 / console | raw-sample cold store (local disk via named volume) | -| 4317 / 4318 | gateway OTLP | where agents ship metrics | +| 4317 / 4318 | backend OTLP | where agents ship metrics (asapquery-backend's `--enable-otel-ingest`) | +| 14317 / 14318 | gateway OTLP | only when `--profile gateway-legacy` is set | MinIO is configured as a local cold store (data in the `minio-data` named volume under `/mydata/...`). Byte-layout diff --git a/deploy/mvp-singlenode/configs/asap-otel-agent-b0a-raw-stream.yaml b/deploy/mvp-singlenode/configs/asap-otel-agent-b0a-raw-stream.yaml index 7a03ece2..a8800b63 100644 --- a/deploy/mvp-singlenode/configs/asap-otel-agent-b0a-raw-stream.yaml +++ b/deploy/mvp-singlenode/configs/asap-otel-agent-b0a-raw-stream.yaml @@ -22,8 +22,8 @@ processors: send_batch_size: 1024 exporters: - otlp/gateway: - endpoint: gateway:4317 + otlp/backend: + endpoint: backend:4317 tls: insecure: true @@ -32,7 +32,7 @@ service: metrics: receivers: [otlp] processors: [batch] - exporters: [otlp/gateway] + exporters: [otlp/backend] telemetry: logs: diff --git a/deploy/mvp-singlenode/configs/asap-otel-agent-b0b-raw-batched.yaml b/deploy/mvp-singlenode/configs/asap-otel-agent-b0b-raw-batched.yaml index c1398d56..3788dc46 100644 --- a/deploy/mvp-singlenode/configs/asap-otel-agent-b0b-raw-batched.yaml +++ b/deploy/mvp-singlenode/configs/asap-otel-agent-b0b-raw-batched.yaml @@ -30,8 +30,8 @@ processors: send_batch_max_size: 1000000 exporters: - otlp/gateway: - endpoint: gateway:4317 + otlp/backend: + endpoint: backend:4317 tls: insecure: true @@ -40,7 +40,7 @@ service: metrics: receivers: [otlp] processors: [batch] - exporters: [otlp/gateway] + exporters: [otlp/backend] telemetry: logs: diff --git a/deploy/mvp-singlenode/configs/asap-otel-agent-b2-full.yaml b/deploy/mvp-singlenode/configs/asap-otel-agent-b2-full.yaml index 5004f277..fe2bb902 100644 --- a/deploy/mvp-singlenode/configs/asap-otel-agent-b2-full.yaml +++ b/deploy/mvp-singlenode/configs/asap-otel-agent-b2-full.yaml @@ -13,12 +13,12 @@ # │ agent (this config) │ # │ receivers: otlp │ # │ processors: ddsketch + hll + batch │ -# │ exporters: otlp/gateway, prometheus │ +# │ exporters: otlp/backend, prometheus │ # └──────────────────────────────────────────┘ # │ │ # │ OTLP :4317 │ /metrics :8889 # ▼ ▼ -# gateway → backend Prometheus scrape +# asapquery-backend Prometheus scrape # # Only DDSketch + HLL are enabled here: together they exercise # the "quantile + distinct-count" joint path the paper's B2 @@ -70,10 +70,12 @@ processors: send_batch_size: 1024 exporters: - # Forward sketched output to the gateway. Gateway config takes - # it from here — currently prometheusremotewrite → backend. - otlp/gateway: - endpoint: gateway:4317 + # Forward sketched output directly to asapquery-backend's OTLP ingest + # (the precompute engine merges per-aggregation_id sketches + # server-side via its accumulators — no middle-tier gateway + # processor is in the path). + otlp/backend: + endpoint: backend:4317 tls: insecure: true @@ -88,7 +90,7 @@ service: metrics: receivers: [otlp] processors: [ddsketch, HLL, batch] - exporters: [otlp/gateway, prometheus] + exporters: [otlp/backend, prometheus] telemetry: logs: diff --git a/deploy/mvp-singlenode/configs/asap-otel-agent-b3-delta.yaml b/deploy/mvp-singlenode/configs/asap-otel-agent-b3-delta.yaml index 6f5267e8..777f940c 100644 --- a/deploy/mvp-singlenode/configs/asap-otel-agent-b3-delta.yaml +++ b/deploy/mvp-singlenode/configs/asap-otel-agent-b3-delta.yaml @@ -41,8 +41,8 @@ processors: send_batch_size: 1024 exporters: - otlp/gateway: - endpoint: gateway:4317 + otlp/backend: + endpoint: backend:4317 tls: insecure: true @@ -51,7 +51,7 @@ service: metrics: receivers: [otlp] processors: [ddsketch, HLL, batch] - exporters: [otlp/gateway] + exporters: [otlp/backend] telemetry: logs: diff --git a/deploy/mvp-singlenode/configs/asap-otel-agent-b4-tunable.yaml b/deploy/mvp-singlenode/configs/asap-otel-agent-b4-tunable.yaml index 53e200b2..7f4c8f66 100644 --- a/deploy/mvp-singlenode/configs/asap-otel-agent-b4-tunable.yaml +++ b/deploy/mvp-singlenode/configs/asap-otel-agent-b4-tunable.yaml @@ -45,8 +45,8 @@ processors: send_batch_size: 1024 exporters: - otlp/gateway: - endpoint: gateway:4317 + otlp/backend: + endpoint: backend:4317 tls: insecure: true @@ -55,7 +55,7 @@ service: metrics: receivers: [otlp] processors: [ddsketch, HLL, batch] - exporters: [otlp/gateway] + exporters: [otlp/backend] telemetry: logs: diff --git a/deploy/mvp-singlenode/configs/asap-otel-agent-b6-asap-single-sketch.yaml b/deploy/mvp-singlenode/configs/asap-otel-agent-b6-asap-single-sketch.yaml index 482aaf89..4f5c52b2 100644 --- a/deploy/mvp-singlenode/configs/asap-otel-agent-b6-asap-single-sketch.yaml +++ b/deploy/mvp-singlenode/configs/asap-otel-agent-b6-asap-single-sketch.yaml @@ -301,7 +301,7 @@ connectors: exporters: otlp/backend: - endpoint: gateway:4317 + endpoint: backend:4317 tls: insecure: true # Backend's tonic gRPC server doesn't accept gzip-compressed bodies diff --git a/deploy/mvp-singlenode/configs/mvp-workload.yaml b/deploy/mvp-singlenode/configs/mvp-workload.yaml index 7dfe99dc..f6a9a8d9 100644 --- a/deploy/mvp-singlenode/configs/mvp-workload.yaml +++ b/deploy/mvp-singlenode/configs/mvp-workload.yaml @@ -54,12 +54,14 @@ # 2. LABEL AGGREGATION ACROSS SERIES AT ONE TIMESTAMP # Outer: sum-by(zone) at instant. Inner: identity. -# Stage: gateway (cross-host fan-in merge); backend stores -# the merged sum-by-zone vector. +# Stage: edge sketches per-host; asapquery-backend's precompute +# engine performs the cross-host sum-by(zone) fan-in merge via its +# per-aggregation_id accumulators. (No middle-tier gateway OTel +# collector role is deployed; the merge happens server-side.) - metric_name: http_requests_total query_string: "sum by (zone) (http_requests_total)" accuracy_sla: 0.0 - assign_to_role: gateway + assign_to_role: agent # 3. COMBINED WINDOW + LABEL AGGREGATION # Inner: rate per series over 5m. Outer: sum-by(zone). diff --git a/deploy/mvp-singlenode/configs/workloads.yaml b/deploy/mvp-singlenode/configs/workloads.yaml index 6b9d34ab..f4355d03 100644 --- a/deploy/mvp-singlenode/configs/workloads.yaml +++ b/deploy/mvp-singlenode/configs/workloads.yaml @@ -63,14 +63,17 @@ # min row sum; see PROGRESS "Single-pipeline multi-sketch" note). # Not exercised under b3-delta in the current cell; included so the # full-sweep matrix (E3) can pick this up by toggling AGENT_CONFIG. +# Role tagged `agent`: the asapquery-backend's precompute engine +# merges per-aggregation_id sketches arriving via OTLP, so a +# middle-tier backend OTel collector role is no longer deployed. - metric_name: http_requests_total query_string: "sum(http_requests_total)" accuracy_sla: 0.01 - assign_to_role: backend + assign_to_role: agent # 4. topk — CountSketch family. As above: lives here for the sweep, # not for the b3-delta cell. - metric_name: http_requests_total query_string: "topk(10, http_requests_total)" accuracy_sla: 0.05 - assign_to_role: backend + assign_to_role: agent diff --git a/deploy/mvp-singlenode/docker-compose/agents-N1.yml b/deploy/mvp-singlenode/docker-compose/agents-N1.yml index fcfe6bb6..74020f44 100644 --- a/deploy/mvp-singlenode/docker-compose/agents-N1.yml +++ b/deploy/mvp-singlenode/docker-compose/agents-N1.yml @@ -13,7 +13,11 @@ x-agent: &agent-base image: asap/asap-otel:dev depends_on: - - gateway + # Agents OTLP-export sketches directly to asapquery-backend + # (asapquery-backend's --enable-otel-ingest merges them + # per-aggregation_id server-side). The gateway tier is opt-in + # via base.yml's `gateway-legacy` profile. + - backend - controller # Agents are configured via the mounted sketch pipeline; overriding # the entrypoint isn't needed — the patched asap-otel binary diff --git a/deploy/mvp-singlenode/docker-compose/agents-N10.yml b/deploy/mvp-singlenode/docker-compose/agents-N10.yml index 4aeb1dfb..781a0c2c 100644 --- a/deploy/mvp-singlenode/docker-compose/agents-N10.yml +++ b/deploy/mvp-singlenode/docker-compose/agents-N10.yml @@ -13,7 +13,8 @@ x-agent: &agent-base image: asap/asap-otel:dev depends_on: - - gateway + # See agents-N1.yml for the agent → asapquery-backend rationale. + - backend - controller command: - "--config=/etc/otel/config.yaml" diff --git a/deploy/mvp-singlenode/docker-compose/agents-N100.yml b/deploy/mvp-singlenode/docker-compose/agents-N100.yml index 23b7075f..9944141e 100644 --- a/deploy/mvp-singlenode/docker-compose/agents-N100.yml +++ b/deploy/mvp-singlenode/docker-compose/agents-N100.yml @@ -13,7 +13,8 @@ x-agent: &agent-base image: asap/asap-otel:dev depends_on: - - gateway + # See agents-N1.yml for the agent → asapquery-backend rationale. + - backend - controller command: - "--config=/etc/otel/config.yaml" diff --git a/deploy/mvp-singlenode/docker-compose/base.yml b/deploy/mvp-singlenode/docker-compose/base.yml index b95e526e..3dcdf7ec 100644 --- a/deploy/mvp-singlenode/docker-compose/base.yml +++ b/deploy/mvp-singlenode/docker-compose/base.yml @@ -204,22 +204,36 @@ services: # send OTLP straight to the backend from the host. logging: *default-logging - # ───────── OpenTelemetry gateway (aggregator) ───────── + # ───────── OpenTelemetry gateway (aggregator) — opt-in only ───────── + # The default data path is agent → asapquery-backend (the backend's + # `--enable-otel-ingest` accepts sketch data points and merges them + # per-aggregation_id via its precompute-engine accumulators). The + # gateway tier is preserved in source under the `gateway-legacy` + # profile so A/B comparisons against an explicit middle-tier merge + # collector are still one flag away: + # + # docker compose --profile gateway-legacy -f base.yml -f agents-N1.yml up + # + # (Agents continue to target backend:4317 directly; flipping any + # data path back through the gateway requires also re-pointing the + # agent OTel config + fake-exporter EXPORTER_TARGET.) + # + # ASAP-patched OTel collector — same image the agents use. + # Required because stock contrib 0.108's pdata silently drops + # typed sketch data points (Metric.data tags 13–17) via + # `ConsumeUnknown` during OTLP receive → in-memory → OTLP + # export round-trip. The patched build's pdata knows about + # those variants and preserves them. See the comment block in + # `deploy/mvp-singlenode/configs/gateway.yaml` for the code reference. + # + # Three gateway configs in deploy/mvp-singlenode/configs/: + # gateway.yaml — pure forwarder (default) + # gateway-aggregate-from-raw.yaml — sketch raw points at gateway tier + # gateway-aggregate-from-sketches.yaml — merge already-sketched payloads + # Override the mounted file via the GATEWAY_CONFIG env var, + # same pattern agents use for AGENT_CONFIG. gateway: - # ASAP-patched OTel collector — same image the agents use. - # Required because stock contrib 0.108's pdata silently drops - # typed sketch data points (Metric.data tags 13–17) via - # `ConsumeUnknown` during OTLP receive → in-memory → OTLP - # export round-trip. The patched build's pdata knows about - # those variants and preserves them. See the comment block in - # `deploy/mvp-singlenode/configs/gateway.yaml` for the code reference. - # - # Three gateway configs in deploy/mvp-singlenode/configs/: - # gateway.yaml — pure forwarder (default) - # gateway-aggregate-from-raw.yaml — sketch raw points at gateway tier - # gateway-aggregate-from-sketches.yaml — merge already-sketched payloads - # Override the mounted file via the GATEWAY_CONFIG env var, - # same pattern agents use for AGENT_CONFIG. + profiles: ["gateway-legacy"] image: asap/asap-otel:dev command: ["--config=/etc/otel/config.yaml"] volumes: @@ -241,7 +255,12 @@ services: # DataCollector/datasets_eval/. image: asap/fake-exporter:dev environment: - EXPORTER_TARGET: "gateway:4317" + # Default target is asapquery-backend's OTLP ingest. Overlays + # that introduce agents (agents-N.yml) re-point this at the + # agent so the producer flows through the agent's sketch + # pipeline. The `--profile gateway-legacy` opt-in path also + # needs the target manually flipped back to gateway:4317. + EXPORTER_TARGET: "backend:4317" # Workload knobs (can be overridden from sweep scripts via # host env). Legacy EXPORTER_RATE is accepted but deprecated # — fake-exporter itself logs a warning when set. @@ -257,7 +276,7 @@ services: EXPORTER_SDK_AGG: "${EXPORTER_SDK_AGG:-default}" EXPORTER_MAX_BUFFER_PER_SERIES: "${EXPORTER_MAX_BUFFER_PER_SERIES:-0}" depends_on: - - gateway + - backend logging: *default-logging # Agents are defined in `agents-N.yml` overrides — keeping diff --git a/deploy/mvp-singlenode/docker-compose/e2e-overlay.yml b/deploy/mvp-singlenode/docker-compose/e2e-overlay.yml index fe9c43d1..deca5681 100644 --- a/deploy/mvp-singlenode/docker-compose/e2e-overlay.yml +++ b/deploy/mvp-singlenode/docker-compose/e2e-overlay.yml @@ -57,13 +57,15 @@ services: volumes: - ../configs/workloads.yaml:/etc/asap/workloads.yaml:ro - # The canonical gateway forwards everything as OTLP (sketches + - # raw) to the backend. The default `gateway.yaml` uses - # `prometheusremotewrite` which silently drops sketch-typed data - # points — fine for a raw-metrics-only deployment, wrong for an - # ASAP run. Override the mounted config in this overlay so the - # e2e harness gets sketch-through-gateway-to-backend by default. + # Gateway is opt-in (`profiles: ["gateway-legacy"]` in base.yml). + # When the profile IS enabled, this overlay swaps the canonical + # config for the OTLP-forwarder variant (so sketches survive the + # round-trip — the default `gateway.yaml`'s + # `prometheusremotewrite` exporter silently drops sketch-typed + # data points). Without `--profile gateway-legacy`, this override + # is inert. gateway: + profiles: ["gateway-legacy"] volumes: - ../configs/gateway-otlp-forward.yaml:/etc/otelcol-contrib/config.yaml:ro diff --git a/deploy/mvp-singlenode/docker-compose/gen-agents.sh b/deploy/mvp-singlenode/docker-compose/gen-agents.sh index df40de09..9dde7526 100755 --- a/deploy/mvp-singlenode/docker-compose/gen-agents.sh +++ b/deploy/mvp-singlenode/docker-compose/gen-agents.sh @@ -28,7 +28,11 @@ cat <