From 4a1816b75141c3e3b3a57a9f7b3bcf04d6ef4d8a Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 08:33:23 -0600 Subject: [PATCH 1/8] test: add manual component e2e runner --- docs/03-how-to-guides/manual-e2e-tests.md | 74 +++++++++ docs/README.md | 4 + scripts/e2e.sh | 184 ++++++++++++++++++++++ 3 files changed, 262 insertions(+) create mode 100644 docs/03-how-to-guides/manual-e2e-tests.md create mode 100755 scripts/e2e.sh diff --git a/docs/03-how-to-guides/manual-e2e-tests.md b/docs/03-how-to-guides/manual-e2e-tests.md new file mode 100644 index 00000000..65c5f5ef --- /dev/null +++ b/docs/03-how-to-guides/manual-e2e-tests.md @@ -0,0 +1,74 @@ +# Manual end-to-end tests + +Run every component suite and the final repository-wide warm-path test from +the repository root: + +```bash +./scripts/e2e.sh +``` + +The command runs tests serially because several transport tests temporarily +change process environment or still use fixed loopback ports. It covers: + +1. shared type and protobuf wire contracts; +2. control-plane HTTP, OpAMP, plan publication, and runtime feedback; +3. data-plane ingest adapters, query routing, storage, lifecycle, persistence, + and exact-backend forwarding with controlled peers; +4. the monitor coordinator over a real bidirectional gRPC connection; +5. Gorilla fragment ingest, WAL recovery, TSDB block construction, Thanos + StoreAPI, compaction, and object-store shipping; and +6. the final controller-plan -> backend-plan install -> modified OTLP ingest -> + precompute -> SketchStore -> PromQL query path. + +The component suites can also be run separately: + +```bash +./scripts/e2e.sh contracts +./scripts/e2e.sh control-plane +./scripts/e2e.sh data-plane +./scripts/e2e.sh monitor +./scripts/e2e.sh gorilla-merger +./scripts/e2e.sh whole +``` + +`whole` is the stable representative DDSketch path. To exercise every +currently checked-in sketch/query combination, including scenarios tracking +known product regressions, run: + +```bash +./scripts/e2e.sh whole-matrix +``` + +Unlike ignored tests, a failing matrix scenario exits non-zero. This target is +diagnostic and is not part of the default `all` acceptance command until those +known query/planner regressions are fixed. + +Use `ASAP_E2E_NOCAPTURE=1` to display Rust test output. Build artifacts and +the Go compilation cache are kept below `target/` by default so a full system +disk does not make Go use the home-directory cache. + +## Full external system + +The backend repository does not contain the producer and collector-agent +binaries. To run the actual multi-node system with those processes, use the +explicit target below. It delegates to the sibling ASAPCollector checkout and +can build images, use SSH, and start containers on the configured nodes: + +```bash +ASAP_COLLECTOR_DIR=../ASAPCollector ./scripts/e2e.sh system +``` + +The local `all` target never performs those external operations. + +## Ignored regressions + +Tests marked `#[ignore]` are not counted as passing coverage. List the current +known ignored tests and their reasons with: + +```bash +./scripts/e2e.sh list +``` + +In particular, the older `e2e_modified_otlp_sketch_path` cases remain ignored +after the protobuf refactor. The maintained whole-path suite is +`e2e_controller_plans_and_backend_serves`. diff --git a/docs/README.md b/docs/README.md index 5c635929..925fa0c4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,5 +1,9 @@ # ASAPQuery-backend documentation +## Testing + +- [Manual component and whole-system E2E tests](03-how-to-guides/manual-e2e-tests.md) + ## Canonical system design The end-to-end architecture and shared contracts are maintained centrally in diff --git a/scripts/e2e.sh b/scripts/e2e.sh new file mode 100755 index 00000000..15a3d480 --- /dev/null +++ b/scripts/e2e.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +# Manual end-to-end test runner for ASAPQuery-backend. +# +# The default "all" target stays local to this repository. "system" is an +# explicit opt-in because it delegates to ASAPCollector's multi-node harness +# and may build images, start containers, and use SSH. + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +TARGET="${1:-all}" +CURRENT_STAGE="startup" +STARTED_AT="$(date +%s)" + +export CARGO_TARGET_DIR="${ASAP_E2E_CARGO_TARGET_DIR:-${REPO_DIR}/target/e2e}" +export GOCACHE="${ASAP_E2E_GO_CACHE:-${REPO_DIR}/target/e2e-go-cache}" + +usage() { + cat <<'EOF' +Usage: ./scripts/e2e.sh [target] + +Targets: + all Run every local component suite, then the repository E2E + contracts Shared Rust type/protobuf wire contracts + control-plane Planner HTTP, OpAMP, publication, and runtime feedback + data-plane Query, routing, storage, ingest adapter, and lifecycle tests + monitor Real monitor gRPC transport tests + gorilla-merger Gorilla HTTP/WAL/block/StoreAPI/compaction/shipper tests + whole Controller plan -> backend install -> OTLP -> store -> PromQL + whole-matrix Run every whole-path sketch/query scenario (diagnostic) + system Delegate to ASAPCollector's real multi-node system harness + list Print the suites and known ignored E2E tests + +Useful environment variables: + ASAP_COLLECTOR_DIR Sibling ASAPCollector checkout (system target) + ASAP_E2E_CARGO_TARGET_DIR Rust build directory + ASAP_E2E_GO_CACHE Go build cache directory + ASAP_E2E_NOCAPTURE=1 Pass --nocapture to Rust test binaries +EOF +} + +say() { + printf '\n==> %s\n' "$*" +} + +die() { + printf '\nERROR [%s]: %s\n' "${CURRENT_STAGE}" "$*" >&2 + exit 1 +} + +on_error() { + local code=$? + printf '\nFAILED [%s] (exit %s)\n' "${CURRENT_STAGE}" "${code}" >&2 + exit "${code}" +} +trap on_error ERR + +need() { + command -v "$1" >/dev/null 2>&1 || die "required command not found: $1" +} + +rust_test() { + local package=$1 + shift + local test_args=(--test-threads=1) + if [[ "${ASAP_E2E_NOCAPTURE:-0}" == "1" ]]; then + test_args+=(--nocapture) + fi + cargo test --locked -p "${package}" "$@" -- "${test_args[@]}" +} + +contracts() { + CURRENT_STAGE="contracts/asap_types" + say "contracts: shared policy and routing types" + rust_test asap_types + + CURRENT_STAGE="contracts/asap_otel_proto" + say "contracts: modified OTLP and monitor protobuf compatibility" + rust_test asap_otel_proto --tests +} + +control_plane() { + CURRENT_STAGE="control-plane" + say "control-plane: HTTP planning, OpAMP, publication, and feedback" + rust_test control_plane +} + +data_plane() { + CURRENT_STAGE="data-plane/library" + say "data-plane: HTTP query/routing, storage, lifecycle, and fallback" + rust_test data_plane --lib + + CURRENT_STAGE="data-plane/edge-runtime-wire" + say "data-plane: edge runtime sketch envelope -> backend accumulator" + rust_test data_plane --test edge_runtime_consumes_precompute_rs +} + +monitor() { + CURRENT_STAGE="monitor-grpc" + say "monitor: real bidirectional gRPC server/client" + rust_test data_plane --test monitor_grpc +} + +gorilla_merger() { + CURRENT_STAGE="gorilla-merger" + say "gorilla-merger: HTTP ingest, WAL, blocks, StoreAPI, compaction, shipping" + need go + ( + cd "${REPO_DIR}/gorilla-merger" + GOPRIVATE="${GOPRIVATE:-github.com/ProjectASAP/*}" \ + go test -count=1 ./... + ) +} + +whole() { + CURRENT_STAGE="whole/controller-to-query" + say "whole repository: controller plan -> backend -> OTLP -> sketch store -> PromQL" + # This is the maintained representative happy path. The broader matrix is + # intentionally a separate target: it includes known feature regressions + # and must not make the routine repository acceptance test nondeterministic. + rust_test data_plane --test e2e_controller_plans_and_backend_serves \ + controller_plan_to_query_full_roundtrip_ddsketch +} + +whole_matrix() { + CURRENT_STAGE="whole/sketch-query-matrix" + say "whole repository diagnostic: every sketch and query scenario" + rust_test data_plane --test e2e_controller_plans_and_backend_serves +} + +list_suites() { + usage + printf '\nKnown intentionally ignored E2E tests (not counted as passes):\n' + rg -n '^[[:space:]]*#\[ignore' \ + "${REPO_DIR}/data_plane/tests" \ + "${REPO_DIR}/data_plane/src/tests" \ + "${REPO_DIR}/crates" \ + -g '*.rs' || true +} + +system_e2e() { + CURRENT_STAGE="external-system" + need docker + local collector_dir="${ASAP_COLLECTOR_DIR:-${REPO_DIR}/../ASAPCollector}" + local runner="${collector_dir}/deploy/mvp-multinode/scripts/run_demo.sh" + [[ -f "${runner}" ]] || die "ASAPCollector system runner not found: ${runner}" + say "external system: delegating to ASAPCollector multi-node harness" + printf 'This target may build images, start remote containers, and use SSH.\n' + BACKEND="${REPO_DIR}" bash "${runner}" all +} + +main() { + cd "${REPO_DIR}" + case "${TARGET}" in + all) + need cargo + need rg + contracts + control_plane + data_plane + monitor + gorilla_merger + whole + ;; + contracts) need cargo; contracts ;; + control-plane) need cargo; control_plane ;; + data-plane) need cargo; data_plane ;; + monitor) need cargo; monitor ;; + gorilla-merger) gorilla_merger ;; + whole) need cargo; whole ;; + whole-matrix) need cargo; whole_matrix ;; + system) system_e2e ;; + list) list_suites; exit 0 ;; + -h|--help|help) usage; exit 0 ;; + *) usage >&2; die "unknown target: ${TARGET}" ;; + esac + + local elapsed=$(( $(date +%s) - STARTED_AT )) + CURRENT_STAGE="complete" + printf '\nPASS: %s E2E target completed in %ss\n' "${TARGET}" "${elapsed}" +} + +main "$@" From 832729c92605dca807f7d700d2b61b513f7859d7 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 09:58:36 -0600 Subject: [PATCH 2/8] test: add production process e2e coverage --- control_plane/tests/component_process_e2e.rs | 91 ++++++++++++++++ data_plane/tests/component_process_e2e.rs | 107 +++++++++++++++++++ docs/03-how-to-guides/manual-e2e-tests.md | 10 +- scripts/e2e.sh | 4 + 4 files changed, 208 insertions(+), 4 deletions(-) create mode 100644 control_plane/tests/component_process_e2e.rs create mode 100644 data_plane/tests/component_process_e2e.rs diff --git a/control_plane/tests/component_process_e2e.rs b/control_plane/tests/component_process_e2e.rs new file mode 100644 index 00000000..33003696 --- /dev/null +++ b/control_plane/tests/component_process_e2e.rs @@ -0,0 +1,91 @@ +//! Black-box component E2E for the production control-plane binary. +//! +//! Unlike the router-level tests in `src/main.rs`, this test exercises CLI +//! startup, all three production listeners, TCP/HTTP transport, JSON decoding, +//! planning, and shutdown as a separate OS process. + +use std::net::TcpListener; +use std::process::{Child, Command, Stdio}; +use std::time::Duration; + +struct ChildGuard(Child); + +impl Drop for ChildGuard { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +fn unused_addr() -> String { + let listener = TcpListener::bind("127.0.0.1:0").expect("reserve loopback port"); + let addr = listener.local_addr().expect("read loopback address"); + drop(listener); + addr.to_string() +} + +async fn wait_until_ready(client: &reqwest::Client, url: &str, child: &mut Child) { + for _ in 0..100 { + if let Some(status) = child.try_wait().expect("inspect control-plane process") { + panic!("control-plane exited before readiness: {status}"); + } + if client + .get(url) + .send() + .await + .is_ok_and(|response| response.status().is_success()) + { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("control-plane did not become ready at {url}"); +} + +#[tokio::test] +async fn production_binary_serves_cost_model_and_plans_a_workload() { + let api_addr = unused_addr(); + let opamp_addr = unused_addr(); + let grpc_addr = unused_addr(); + + let child = Command::new(env!("CARGO_BIN_EXE_control_plane")) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .env("CONTROLLER_ADDR", &api_addr) + .env("CONTROLLER_OPAMP_ADDR", opamp_addr) + .env("CONTROLLER_GRPC_ADDR", grpc_addr) + .env( + "CONTROLLER_WORKLOADS", + "/definitely/missing/e2e-workloads.yaml", + ) + .env("CONTROLLER_SKETCH_DEFAULTS", "sketch_params_default.yml") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("start production control-plane binary"); + let mut child = ChildGuard(child); + + let client = reqwest::Client::new(); + let base = format!("http://{api_addr}"); + wait_until_ready(&client, &format!("{base}/api/v1/cost-model"), &mut child.0).await; + + let response = client + .post(format!("{base}/api/v1/plan")) + .json(&serde_json::json!({ + "metric_name": "component_process_e2e_latency_ms", + "aggregations": ["quantile"], + "time_window": "1m", + "accuracy_sla": 0.01 + })) + .send() + .await + .expect("POST workload to production control plane"); + assert!( + response.status().is_success(), + "plan status: {}", + response.status() + ); + let body: serde_json::Value = response.json().await.expect("decode plan response"); + assert_eq!(body["metric"], "component_process_e2e_latency_ms"); + assert!(body["sketch_type"].as_str().is_some()); + assert!(body["valid_until"].as_str().is_some()); +} diff --git a/data_plane/tests/component_process_e2e.rs b/data_plane/tests/component_process_e2e.rs new file mode 100644 index 00000000..96de7459 --- /dev/null +++ b/data_plane/tests/component_process_e2e.rs @@ -0,0 +1,107 @@ +//! Black-box component E2E for the production data-plane binary. +//! +//! This catches failures that in-process `HttpServer` tests cannot: CLI +//! parsing, file-based bootstrap configuration, logging setup, production +//! object wiring, TCP binding, and the public diagnostic HTTP contract. + +use std::io::Write; +use std::net::TcpListener; +use std::process::{Child, Command, Stdio}; +use std::time::Duration; + +struct ChildGuard(Child); + +impl Drop for ChildGuard { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +fn unused_port() -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").expect("reserve loopback port"); + let port = listener.local_addr().expect("read loopback address").port(); + drop(listener); + port +} + +async fn wait_until_ready(client: &reqwest::Client, url: &str, child: &mut Child) { + for _ in 0..100 { + if let Some(status) = child.try_wait().expect("inspect data-plane process") { + panic!("data-plane exited before readiness: {status}"); + } + if client + .get(url) + .send() + .await + .is_ok_and(|response| response.status().is_success()) + { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("data-plane did not become ready at {url}"); +} + +#[tokio::test] +async fn production_binary_loads_config_and_serves_diagnostics() { + let query_port = unused_port(); + let output_dir = tempfile::tempdir().expect("create log directory"); + let mut config = tempfile::NamedTempFile::new().expect("create streaming config"); + write!( + config, + r#"aggregations: + - aggregationType: DDSketch + aggregationSubType: '' + labels: + grouping: [service] + rollup: [] + aggregated: [] + metric: component_process_e2e_latency_ms + parameters: + relativeAccuracy: 0.01 + windowSize: 60 + windowType: tumbling + spatialFilter: '' +"# + ) + .expect("write streaming config"); + + let child = Command::new(env!("CARGO_BIN_EXE_data_plane")) + .arg("--streaming-config") + .arg(config.path()) + .arg("--http-port") + .arg(query_port.to_string()) + .arg("--output-dir") + .arg(output_dir.path()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("start production data-plane binary"); + let mut child = ChildGuard(child); + + let client = reqwest::Client::new(); + let base = format!("http://127.0.0.1:{query_port}"); + wait_until_ready(&client, &format!("{base}/api/v1/health"), &mut child.0).await; + + let health = client + .get(format!("{base}/api/v1/health")) + .send() + .await + .expect("GET health") + .text() + .await + .expect("read health body"); + assert_eq!(health, "ok"); + + let config_response: serde_json::Value = client + .get(format!("{base}/api/v1/streaming-config")) + .send() + .await + .expect("GET installed streaming config") + .json() + .await + .expect("decode streaming-config response"); + assert_eq!(config_response["status"], "success"); + assert_eq!(config_response["aggregation_count"], 1); +} diff --git a/docs/03-how-to-guides/manual-e2e-tests.md b/docs/03-how-to-guides/manual-e2e-tests.md index 65c5f5ef..171e2212 100644 --- a/docs/03-how-to-guides/manual-e2e-tests.md +++ b/docs/03-how-to-guides/manual-e2e-tests.md @@ -11,14 +11,16 @@ The command runs tests serially because several transport tests temporarily change process environment or still use fixed loopback ports. It covers: 1. shared type and protobuf wire contracts; -2. control-plane HTTP, OpAMP, plan publication, and runtime feedback; +2. the real control-plane binary plus HTTP, OpAMP, plan publication, and + runtime feedback; 3. data-plane ingest adapters, query routing, storage, lifecycle, persistence, - and exact-backend forwarding with controlled peers; + exact-backend forwarding with controlled peers, and a real data-plane + process bootstrapped from a file; 4. the monitor coordinator over a real bidirectional gRPC connection; 5. Gorilla fragment ingest, WAL recovery, TSDB block construction, Thanos StoreAPI, compaction, and object-store shipping; and -6. the final controller-plan -> backend-plan install -> modified OTLP ingest -> - precompute -> SketchStore -> PromQL query path. +6. the final controller planning -> backend configuration -> modified OTLP + ingest -> precompute -> SketchStore -> PromQL query path. The component suites can also be run separately: diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 15a3d480..3ebafc6d 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -94,6 +94,10 @@ data_plane() { CURRENT_STAGE="data-plane/edge-runtime-wire" say "data-plane: edge runtime sketch envelope -> backend accumulator" rust_test data_plane --test edge_runtime_consumes_precompute_rs + + CURRENT_STAGE="data-plane/production-process" + say "data-plane: production binary bootstrap and public HTTP diagnostics" + rust_test data_plane --test component_process_e2e } monitor() { From 4f3e22c769fff977f01096d73047eb3d86c2879d Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 10:47:18 -0600 Subject: [PATCH 3/8] test: replace process smoke checks with real e2e flows --- Cargo.lock | 1 + control_plane/build.rs | 4 +- control_plane/tests/component_process_e2e.rs | 139 ++++- crates/asap_types/src/routing_index.rs | 50 +- data_plane/Cargo.toml | 1 + .../query_engines/asap_query_engine/engine.rs | 1 + data_plane/tests/backend_process_e2e.rs | 491 ++++++++++++++++++ data_plane/tests/component_process_e2e.rs | 157 +++++- data_plane/tests/monitor_process_e2e.rs | 145 ++++++ docs/03-how-to-guides/manual-e2e-tests.md | 29 +- gorilla-merger/e2e/process_e2e_test.go | 192 +++++++ scripts/e2e.sh | 20 +- 12 files changed, 1178 insertions(+), 52 deletions(-) create mode 100644 data_plane/tests/backend_process_e2e.rs create mode 100644 data_plane/tests/monitor_process_e2e.rs create mode 100644 gorilla-merger/e2e/process_e2e_test.go diff --git a/Cargo.lock b/Cargo.lock index cb32f280..5c94d4eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1017,6 +1017,7 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tokio-stream", + "tokio-tungstenite 0.21.0", "tonic", "tracing", "tracing-appender", diff --git a/control_plane/build.rs b/control_plane/build.rs index 327f04ee..4c8b9236 100644 --- a/control_plane/build.rs +++ b/control_plane/build.rs @@ -12,7 +12,9 @@ fn main() -> Result<(), Box> { // with `sketch-bench/sketch-runtime/proto/feedback.proto`. tonic_build::configure() .build_server(true) - .build_client(false) + // Keep the generated client available for black-box process E2E tests + // and for downstream agents that share this crate's wire contract. + .build_client(true) .compile_protos(&["proto/feedback.proto"], &["proto/"])?; Ok(()) } diff --git a/control_plane/tests/component_process_e2e.rs b/control_plane/tests/component_process_e2e.rs index 33003696..1ca509a0 100644 --- a/control_plane/tests/component_process_e2e.rs +++ b/control_plane/tests/component_process_e2e.rs @@ -1,12 +1,22 @@ //! Black-box component E2E for the production control-plane binary. //! -//! Unlike the router-level tests in `src/main.rs`, this test exercises CLI -//! startup, all three production listeners, TCP/HTTP transport, JSON decoding, -//! planning, and shutdown as a separate OS process. +//! A simulated collector connects to the production OpAMP WebSocket, a real +//! workload is planned through the public HTTP API, and the emitted collector +//! YAML is received over OpAMP. The same child process then accepts a runtime +//! sample over its production gRPC service and exposes the accepted record in +//! Prometheus metrics. +use futures_util::StreamExt; +use prost::Message; use std::net::TcpListener; use std::process::{Child, Command, Stdio}; use std::time::Duration; +use tokio_tungstenite::tungstenite::client::IntoClientRequest; + +use control_plane::opamp::opamp_proto::ServerToAgent; +use control_plane::runtime_samples::feedback::{ + runtime_samples_client::RuntimeSamplesClient, PushBatch, RuntimeRecord, +}; struct ChildGuard(Child); @@ -42,8 +52,67 @@ async fn wait_until_ready(client: &reqwest::Client, url: &str, child: &mut Child panic!("control-plane did not become ready at {url}"); } +async fn connect_agent( + address: &str, + child: &mut Child, +) -> tokio_tungstenite::WebSocketStream> { + for _ in 0..100 { + if let Some(status) = child.try_wait().expect("inspect control-plane process") { + panic!("control-plane exited before OpAMP connection: {status}"); + } + let mut request = format!("ws://{address}/v1/opamp") + .into_client_request() + .expect("build OpAMP request"); + request + .headers_mut() + .insert("X-Agent-ID", "process-e2e-agent".parse().unwrap()); + request + .headers_mut() + .insert("X-Agent-Role", "agent".parse().unwrap()); + if let Ok((stream, _)) = tokio_tungstenite::connect_async(request).await { + return stream; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("production OpAMP listener did not accept a collector connection"); +} + +async fn push_runtime_sample(address: &str) { + let endpoint = format!("http://{address}"); + let mut connected = None; + for _ in 0..100 { + match RuntimeSamplesClient::connect(endpoint.clone()).await { + Ok(client) => { + connected = Some(client); + break; + } + Err(_) => tokio::time::sleep(Duration::from_millis(50)).await, + } + } + let mut client = connected + .unwrap_or_else(|| panic!("could not connect to production runtime service {endpoint}")); + let response = client + .push(PushBatch { + records: vec![RuntimeRecord { + source: "process-e2e-agent".into(), + sketch: "ddsketch".into(), + impl_name: "rust".into(), + schema_version: 1, + payload_json: serde_json::json!({ + "schema_version": 1, + "bench": {"throughput_items_per_sec": {"mean": 42000.0}} + }) + .to_string(), + }], + }) + .await + .expect("push runtime sample to production gRPC service") + .into_inner(); + assert_eq!(response.accepted, 1); +} + #[tokio::test] -async fn production_binary_serves_cost_model_and_plans_a_workload() { +async fn production_binary_plans_pushes_opamp_config_and_ingests_feedback() { let api_addr = unused_addr(); let opamp_addr = unused_addr(); let grpc_addr = unused_addr(); @@ -51,8 +120,8 @@ async fn production_binary_serves_cost_model_and_plans_a_workload() { let child = Command::new(env!("CARGO_BIN_EXE_control_plane")) .current_dir(env!("CARGO_MANIFEST_DIR")) .env("CONTROLLER_ADDR", &api_addr) - .env("CONTROLLER_OPAMP_ADDR", opamp_addr) - .env("CONTROLLER_GRPC_ADDR", grpc_addr) + .env("CONTROLLER_OPAMP_ADDR", &opamp_addr) + .env("CONTROLLER_GRPC_ADDR", &grpc_addr) .env( "CONTROLLER_WORKLOADS", "/definitely/missing/e2e-workloads.yaml", @@ -67,6 +136,7 @@ async fn production_binary_serves_cost_model_and_plans_a_workload() { let client = reqwest::Client::new(); let base = format!("http://{api_addr}"); wait_until_ready(&client, &format!("{base}/api/v1/cost-model"), &mut child.0).await; + let mut agent = connect_agent(&opamp_addr, &mut child.0).await; let response = client .post(format!("{base}/api/v1/plan")) @@ -88,4 +158,61 @@ async fn production_binary_serves_cost_model_and_plans_a_workload() { assert_eq!(body["metric"], "component_process_e2e_latency_ms"); assert!(body["sketch_type"].as_str().is_some()); assert!(body["valid_until"].as_str().is_some()); + assert_eq!(body["agents_notified"], 1); + + let frame = tokio::time::timeout(Duration::from_secs(5), agent.next()) + .await + .expect("timed out waiting for OpAMP configuration") + .expect("OpAMP connection closed") + .expect("read OpAMP frame"); + let data = match frame { + tokio_tungstenite::tungstenite::Message::Binary(data) => data, + other => panic!("expected binary OpAMP frame, got {other:?}"), + }; + let payload = if data.first() == Some(&0) { + &data[1..] + } else { + &data + }; + let message = ServerToAgent::decode(payload).expect("decode OpAMP ServerToAgent"); + let config = message + .remote_config + .and_then(|remote| remote.config) + .expect("OpAMP response contains remote config"); + let yaml = String::from_utf8( + config + .config_map + .get("") + .expect("default OpAMP config file") + .body + .clone(), + ) + .expect("collector config is UTF-8 YAML"); + let planned_sketch = body["sketch_type"] + .as_str() + .expect("plan contains sketch type") + .to_ascii_lowercase(); + assert!( + yaml.to_ascii_lowercase().contains(&planned_sketch) + && yaml.contains("otlp/backend") + && yaml.contains("service:"), + "OpAMP YAML does not implement the selected {planned_sketch} plan:\n{yaml}" + ); + + push_runtime_sample(&grpc_addr).await; + for _ in 0..50 { + let metrics = client + .get(format!("{base}/metrics")) + .send() + .await + .expect("GET production metrics") + .text() + .await + .expect("read production metrics"); + if metrics.contains("asap_runtime_samples_records_stored_total 1") { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("runtime sample was accepted but never surfaced in /metrics"); } diff --git a/crates/asap_types/src/routing_index.rs b/crates/asap_types/src/routing_index.rs index 098ae7db..4729d4c4 100644 --- a/crates/asap_types/src/routing_index.rs +++ b/crates/asap_types/src/routing_index.rs @@ -109,9 +109,16 @@ impl RoutingIndex { if cfg.aggregation_type != agg_type || &policy_keys != group_by_keys || !cfg.spatial_filter_normalized.is_empty() - || !expected_params - .iter() - .all(|(key, value)| cfg.parameters.get(key) == Some(value)) + || !expected_params.iter().all(|(key, value)| { + cfg.parameters.get(key).or_else(|| match key.as_str() { + // The typed physical-plan compiler names this field + // after SummaryParams, while legacy collector YAML + // uses the equivalent runtime-facing name. + "relative_accuracy" => cfg.parameters.get("alpha"), + "alpha" => cfg.parameters.get("relative_accuracy"), + _ => None, + }) == Some(value) + }) { continue; } @@ -225,4 +232,41 @@ mod tests { assert!(!idx.is_empty()); assert_eq!(idx.len(), 1); } + + #[test] + fn ddsketch_alpha_and_relative_accuracy_are_wire_compatible() { + let mut parameters = StdHashMap::new(); + parameters.insert("alpha".to_string(), serde_json::json!(0.01)); + let config = AggregationConfig::new( + AggregationType::DDSketch, + String::new(), + parameters, + KeyByLabelNames::new(vec!["service".to_string()]), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + 5, + 5, + WindowKind::Tumbling, + String::new(), + "latency".to_string(), + None, + None, + None, + ); + let fingerprint = PolicyFingerprint::from_config(&config); + let index = RoutingIndex::build(PolicyRegistry::from_configs(vec![config])); + let expected = + StdHashMap::from([("relative_accuracy".to_string(), serde_json::json!(0.01))]); + + assert_eq!( + index.find_policy_by_content( + "latency", + &BTreeSet::from(["service".to_string()]), + AggregationType::DDSketch, + &expected, + ), + Some(fingerprint) + ); + } } diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index 4cc4e1ca..de6218d6 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -130,6 +130,7 @@ crc32fast = "1.4" [dev-dependencies] tempfile = "3.20.0" criterion = { version = "0.5", features = ["html_reports"] } +tokio-tungstenite = "0.21" [[bench]] name = "sketch_db" diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 1bab6486..4e498e4b 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -556,6 +556,7 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu idx, query, now_ms, backend_plan.as_deref(), ) .map_err(|reason| { + tracing::debug!(query, ?reason, "post-ASAP warm serving capability miss"); if let Some(req) = Self::requirements_from_query_str(query) { crate::drivers::control_plane_client::spawn_capability_miss_notify( &self.control_plane_client, diff --git a/data_plane/tests/backend_process_e2e.rs b/data_plane/tests/backend_process_e2e.rs new file mode 100644 index 00000000..ddf5493f --- /dev/null +++ b/data_plane/tests/backend_process_e2e.rs @@ -0,0 +1,491 @@ +//! Whole-backend process E2E. +//! +//! Starts the production control-plane and data-plane executables, asks the +//! controller to plan a workload, observes the physical plan installed by the +//! data plane, sends a modified-OTLP DDSketch, and verifies the resulting +//! PromQL value. No server or planner is constructed in the test process. + +use std::io::Write; +use std::net::TcpListener; +use std::process::{Child, Command, Stdio}; +use std::time::Duration; + +use asap_otel_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; +use asap_otel_proto::tonic::common::v1::{any_value, AnyValue, KeyValue}; +use asap_otel_proto::tonic::metrics::v1::{ + metric::Data, DdSketch, DdSketchDataPoint, DdSketchEncoding, Metric, ResourceMetrics, + ScopeMetrics, +}; +use asap_sketchlib::proto::sketchlib::DdSketchState; +use control_plane::opamp::{ + opamp_proto, CollectorPlanStatus, CollectorPlanStatusKind, COLLECTOR_PLAN_CAPABILITY, + COLLECTOR_PLAN_MESSAGE, PLAN_STATUS_MESSAGE, +}; +use control_plane::physical::compiler::PLANNER_REVISION; +use futures::{SinkExt, StreamExt}; +use prost::Message; +use tokio_tungstenite::tungstenite::{http::Request, Message as WsMessage}; + +struct ChildGuard(Child); + +impl Drop for ChildGuard { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +fn unused_addr() -> String { + let listener = TcpListener::bind("127.0.0.1:0").expect("reserve loopback port"); + listener.local_addr().expect("loopback address").to_string() +} + +fn port(address: &str) -> u16 { + address + .rsplit_once(':') + .expect("host:port") + .1 + .parse() + .expect("numeric port") +} + +async fn wait_http(client: &reqwest::Client, url: &str, child: &mut Child, name: &str) { + for _ in 0..200 { + if let Some(status) = child.try_wait().expect("inspect child process") { + panic!("{name} exited before readiness: {status}"); + } + if client + .get(url) + .send() + .await + .is_ok_and(|response| response.status().is_success()) + { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("{name} did not become ready at {url}"); +} + +fn ddsketch_export(metric: &str, timestamp_ns: u64, counts: Vec, alpha: f64) -> Vec { + let point = DdSketchDataPoint { + attributes: vec![KeyValue { + key: "service".into(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue("whole-e2e".into())), + }), + }], + start_time_unix_nano: timestamp_ns.saturating_sub(1_000_000_000), + time_unix_nano: timestamp_ns, + sketch: DdSketchState { + alpha, + store_counts: counts, + store_offset: -1, + } + .encode_to_vec(), + encoding: DdSketchEncoding::DdsketchEncodingProto as i32, + exemplars: Vec::new(), + flags: 0, + series_id: 0, + }; + ExportMetricsServiceRequest { + resource_metrics: vec![ResourceMetrics { + resource: None, + scope_metrics: vec![ScopeMetrics { + scope: None, + metrics: vec![Metric { + name: metric.into(), + description: String::new(), + unit: String::new(), + metadata: Vec::new(), + data: Some(Data::Ddsketch(DdSketch { + data_points: vec![point], + aggregation_temporality: 0, + relative_accuracy: alpha, + })), + }], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + } + .encode_to_vec() +} + +fn first_scalar(response: &serde_json::Value) -> Option { + response["data"]["result"] + .as_array()? + .first()? + .get("value")? + .as_array()? + .get(1)? + .as_str()? + .parse() + .ok() +} + +async fn connect_collector( + address: &str, +) -> tokio_tungstenite::WebSocketStream> { + let uri = format!("ws://{address}/v1/opamp"); + for _ in 0..100 { + let request = Request::builder() + .uri(&uri) + .header("Host", address) + .header("X-Agent-ID", "whole-e2e-collector") + .header("X-Agent-Role", "agent") + .header("Upgrade", "websocket") + .header("Connection", "Upgrade") + .header("Sec-WebSocket-Key", "dGhlIHNhbXBsZSBub25jZQ==") + .header("Sec-WebSocket-Version", "13") + .body(()) + .expect("build OpAMP request"); + if let Ok((socket, _)) = tokio_tungstenite::connect_async(request).await { + return socket; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("collector could not connect to production OpAMP endpoint {uri}"); +} + +async fn send_agent_message( + socket: &mut tokio_tungstenite::WebSocketStream< + tokio_tungstenite::MaybeTlsStream, + >, + message: opamp_proto::AgentToServer, +) { + let mut frame = vec![0]; + message + .encode(&mut frame) + .expect("encode OpAMP AgentToServer"); + socket + .send(WsMessage::Binary(frame)) + .await + .expect("send OpAMP AgentToServer"); +} + +async fn apply_next_collector_plan(address: String) -> serde_json::Value { + let mut socket = connect_collector(&address).await; + send_agent_message( + &mut socket, + opamp_proto::AgentToServer { + custom_capabilities: Some(opamp_proto::CustomCapabilities { + capabilities: vec![COLLECTOR_PLAN_CAPABILITY.into()], + }), + ..Default::default() + }, + ) + .await; + + let frame = tokio::time::timeout(Duration::from_secs(10), socket.next()) + .await + .expect("controller did not publish a collector plan") + .expect("controller closed the OpAMP connection") + .expect("read collector-plan frame"); + let bytes = frame.into_data(); + let payload = bytes.strip_prefix(&[0]).unwrap_or(&bytes); + let message = opamp_proto::ServerToAgent::decode(payload) + .expect("decode production ServerToAgent protobuf"); + let custom = message + .custom_message + .expect("collector-plan custom message"); + assert_eq!(custom.capability, COLLECTOR_PLAN_CAPABILITY); + assert_eq!(custom.r#type, COLLECTOR_PLAN_MESSAGE); + let plan: serde_json::Value = + serde_json::from_slice(&custom.data).expect("decode collector physical plan"); + let plan_id = plan["envelope"]["plan_id"] + .as_u64() + .expect("collector plan ID"); + + let status = serde_json::to_vec(&CollectorPlanStatus { + plan_id, + status: CollectorPlanStatusKind::Applied, + error: None, + }) + .expect("encode applied status"); + send_agent_message( + &mut socket, + opamp_proto::AgentToServer { + custom_message: Some(opamp_proto::CustomMessage { + capability: COLLECTOR_PLAN_CAPABILITY.into(), + r#type: PLAN_STATUS_MESSAGE.into(), + data: status, + }), + ..Default::default() + }, + ) + .await; + plan +} + +#[tokio::test] +async fn production_control_plane_to_data_plane_otlp_to_promql() { + let control_binary = std::env::var("ASAP_E2E_CONTROL_PLANE_BIN") + .expect("ASAP_E2E_CONTROL_PLANE_BIN is set by scripts/e2e.sh whole"); + let data_api = unused_addr(); + let otlp_http = unused_addr(); + let otlp_grpc = unused_addr(); + let control_api = unused_addr(); + let control_opamp = unused_addr(); + let control_grpc = unused_addr(); + + let output_dir = tempfile::tempdir().expect("create data-plane output directory"); + let mut bootstrap = tempfile::NamedTempFile::new().expect("create bootstrap config"); + write!(bootstrap, "aggregations: []\n").expect("write bootstrap config"); + + let data_child = Command::new(env!("CARGO_BIN_EXE_data_plane")) + .arg("--streaming-config") + .arg(bootstrap.path()) + .arg("--http-port") + .arg(port(&data_api).to_string()) + .arg("--output-dir") + .arg(output_dir.path()) + .arg("--enable-otel-ingest") + .arg("--otel-http-port") + .arg(port(&otlp_http).to_string()) + .arg("--otel-grpc-port") + .arg(port(&otlp_grpc).to_string()) + .arg("--precompute-allowed-lateness-ms") + .arg("0") + .arg("--precompute-flush-interval-ms") + .arg("100") + .env("RUST_LOG", "data_plane=debug") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("start production data plane"); + let mut data_child = ChildGuard(data_child); + + let client = reqwest::Client::new(); + let data_base = format!("http://{data_api}"); + wait_http( + &client, + &format!("{data_base}/api/v1/health"), + &mut data_child.0, + "data plane", + ) + .await; + + let control_child = Command::new(control_binary) + .current_dir(env!("CARGO_MANIFEST_DIR").to_string() + "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/../control_plane") + .env("CONTROLLER_ADDR", &control_api) + .env("CONTROLLER_OPAMP_ADDR", &control_opamp) + .env("CONTROLLER_GRPC_ADDR", &control_grpc) + .env( + "CONTROLLER_BACKEND_ENDPOINT", + format!("{data_base}/api/v1/streaming-config"), + ) + .env( + "CONTROLLER_WORKLOADS", + "/definitely/missing/e2e-workloads.yaml", + ) + .env("CONTROLLER_SKETCH_DEFAULTS", "sketch_params_default.yml") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("start production control plane"); + let mut control_child = ChildGuard(control_child); + let control_base = format!("http://{control_api}"); + wait_http( + &client, + &format!("{control_base}/api/v1/cost-model"), + &mut control_child.0, + "control plane", + ) + .await; + + let collector = tokio::spawn(apply_next_collector_plan(control_opamp.clone())); + let observed_at_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock") + .as_millis() as u64; + let publication_response = client + .post(format!( + "{control_base}/api/v1/physical-plan/compile-and-publish" + )) + .json(&serde_json::json!({ + "queries": [{ + "query_id": "whole-process-e2e-query", + "query_string": "quantile_over_time(0.99, whole_process_e2e_latency_ms[30s])", + "metric": "whole_process_e2e_latency_ms", + "window_secs": 1, + "group_by": ["service"], + "accuracy": {"Epsilon": 0.01}, + "lifecycle": { + "evaluation_interval_ms": 1000, + "ingestion_rate_per_second": 100.0, + "evidence_observed_at_unix_ms": observed_at_ms, + "evidence_valid_for_ms": 60000, + "horizon_seconds": 300.0, + "costs": { + "build": 10.0, + "maintenance_per_update": 0.001, + "read": 0.1, + "retention_per_second": 0.001, + "retirement": 1.0 + } + } + }], + "collector_ids": ["whole-e2e-collector"], + "capability_snapshot_id": "whole-e2e-capabilities", + "evidence": {}, + "planner_revision": PLANNER_REVISION, + "max_evidence_age_ms": 60000, + "apply_timeout_ms": 10000 + })) + .send() + .await + .expect("request physical-plan publication"); + let publication_status = publication_response.status(); + let publication_body = publication_response + .text() + .await + .expect("read publication response"); + assert!( + publication_status.is_success(), + "controller rejected physical plan ({publication_status}): {publication_body}" + ); + let publication: serde_json::Value = + serde_json::from_str(&publication_body).expect("decode publication response"); + let collector_plan = collector.await.expect("collector task completed"); + assert_eq!( + publication["plan_id"], + collector_plan["envelope"]["plan_id"] + ); + assert_eq!(publication["collector_ids"][0], "whole-e2e-collector"); + + let active: serde_json::Value = client + .get(format!("{data_base}/api/v1/streaming-config")) + .send() + .await + .expect("read installed streaming config") + .json() + .await + .expect("decode installed streaming config"); + assert_eq!( + active["aggregation_count"], 1, + "physical plan was not installed: {active}" + ); + let installed_aggregation = active["streaming_config"]["aggregation_configs"] + .as_object() + .and_then(|configs| configs.values().next()) + .expect("installed aggregation details"); + let planned_alpha = installed_aggregation["parameters"]["alpha"] + .as_f64() + .expect("controller emitted DDSketch alpha"); + let planned_window_secs = installed_aggregation["window_size"] + .as_u64() + .expect("controller emitted window size"); + let backend_plan: serde_json::Value = client + .get(format!("{data_base}/api/v1/backend-plan")) + .send() + .await + .expect("read installed backend plan") + .json() + .await + .expect("decode installed backend plan"); + assert_eq!(backend_plan["materialization_count"], 1); + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock"); + let sample_ns = now.as_nanos() as u64; + client + .post(format!("http://{otlp_http}/v1/metrics")) + .header("content-type", "application/x-protobuf") + .body(ddsketch_export( + "whole_process_e2e_latency_ms", + sample_ns, + vec![5, 10, 15, 20], + planned_alpha, + )) + .send() + .await + .expect("POST OTLP to production data plane") + .error_for_status() + .expect("data plane accepted OTLP"); + + // Advance event time after the controller-selected tumbling window has + // really ended. The E2E uses no artificial future timestamp here. + let window_ms = planned_window_secs * 1000; + let now_ms = now.as_millis() as u64; + let window_end_ms = (now_ms / window_ms + 1) * window_ms; + tokio::time::sleep(Duration::from_millis(window_end_ms - now_ms + 100)).await; + let watermark_ns = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock") + .as_nanos() as u64; + client + .post(format!("http://{otlp_http}/v1/metrics")) + .header("content-type", "application/x-protobuf") + .body(ddsketch_export( + "whole_process_e2e_latency_ms", + watermark_ns, + Vec::new(), + planned_alpha, + )) + .send() + .await + .expect("POST watermark OTLP to production data plane") + .error_for_status() + .expect("data plane accepted watermark"); + + let query = "quantile_over_time(0.99, whole_process_e2e_latency_ms[30s])"; + let mut last_response = serde_json::Value::Null; + for _ in 0..50 { + let response: serde_json::Value = client + .get(format!("{data_base}/api/v1/query")) + .query(&[("query", query)]) + .send() + .await + .expect("query production data plane") + .json() + .await + .expect("decode PromQL response"); + if let Some(value) = first_scalar(&response) { + assert!( + value.is_finite() && value > 0.0, + "invalid quantile: {value}" + ); + return; + } + last_response = response; + tokio::time::sleep(Duration::from_millis(100)).await; + } + let store_metrics = client + .get(format!("{data_base}/api/v1/store/metrics")) + .send() + .await + .expect("read store metrics") + .text() + .await + .expect("decode store metrics"); + let schemas = client + .get(format!("{data_base}/api/v1/db/schemas")) + .send() + .await + .expect("read schemas") + .text() + .await + .expect("decode schemas"); + let logs = std::fs::read_to_string(output_dir.path().join("query_engine.log")) + .unwrap_or_else(|error| format!("unable to read data-plane log: {error}")); + let relevant_logs = logs + .lines() + .filter(|line| { + line.contains("worker") + || line.contains("Worker") + || line.contains("flush") + || line.contains("CapabilityMiss") + || line.contains("post-ASAP") + || line.contains("modified-proto") + || line.contains("live:") + }) + .collect::>() + .join("\n"); + panic!( + "whole backend never served the controller-planned, OTLP-ingested sketch\n\ + active={active}\nstore_metrics={store_metrics}\nschemas={schemas}\n\ + last_query={last_response}\nlogs={relevant_logs}" + ); +} diff --git a/data_plane/tests/component_process_e2e.rs b/data_plane/tests/component_process_e2e.rs index 96de7459..cd2dfcb4 100644 --- a/data_plane/tests/component_process_e2e.rs +++ b/data_plane/tests/component_process_e2e.rs @@ -1,14 +1,23 @@ //! Black-box component E2E for the production data-plane binary. //! -//! This catches failures that in-process `HttpServer` tests cannot: CLI -//! parsing, file-based bootstrap configuration, logging setup, production -//! object wiring, TCP binding, and the public diagnostic HTTP contract. +//! The child process loads its real file configuration, accepts a modified +//! OTLP DDSketch over HTTP, routes it through the precompute workers into the +//! SketchStore, and answers a PromQL query from that stored sketch. use std::io::Write; use std::net::TcpListener; use std::process::{Child, Command, Stdio}; use std::time::Duration; +use asap_otel_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; +use asap_otel_proto::tonic::common::v1::{any_value, AnyValue, KeyValue}; +use asap_otel_proto::tonic::metrics::v1::{ + metric::Data, DdSketch, DdSketchDataPoint, DdSketchEncoding, Metric, ResourceMetrics, + ScopeMetrics, +}; +use asap_sketchlib::proto::sketchlib::DdSketchState; +use prost::Message; + struct ChildGuard(Child); impl Drop for ChildGuard { @@ -25,6 +34,64 @@ fn unused_port() -> u16 { port } +fn ddsketch_export(metric: &str, timestamp_ns: u64, counts: Vec) -> Vec { + let alpha = 0.01; + let state = DdSketchState { + alpha, + store_counts: counts, + store_offset: -1, + }; + let point = DdSketchDataPoint { + attributes: vec![KeyValue { + key: "service".into(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue("process-e2e".into())), + }), + }], + start_time_unix_nano: timestamp_ns.saturating_sub(1_000_000_000), + time_unix_nano: timestamp_ns, + sketch: state.encode_to_vec(), + encoding: DdSketchEncoding::DdsketchEncodingProto as i32, + exemplars: Vec::new(), + flags: 0, + series_id: 0, + }; + ExportMetricsServiceRequest { + resource_metrics: vec![ResourceMetrics { + resource: None, + scope_metrics: vec![ScopeMetrics { + scope: None, + metrics: vec![Metric { + name: metric.into(), + description: String::new(), + unit: String::new(), + metadata: Vec::new(), + data: Some(Data::Ddsketch(DdSketch { + data_points: vec![point], + aggregation_temporality: 0, + relative_accuracy: alpha, + })), + }], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + } + .encode_to_vec() +} + +fn first_scalar(response: &serde_json::Value) -> Option { + response["data"]["result"] + .as_array()? + .first()? + .get("value")? + .as_array()? + .get(1)? + .as_str()? + .parse() + .ok() +} + async fn wait_until_ready(client: &reqwest::Client, url: &str, child: &mut Child) { for _ in 0..100 { if let Some(status) = child.try_wait().expect("inspect data-plane process") { @@ -44,8 +111,10 @@ async fn wait_until_ready(client: &reqwest::Client, url: &str, child: &mut Child } #[tokio::test] -async fn production_binary_loads_config_and_serves_diagnostics() { +async fn production_binary_ingests_ddsketch_and_answers_promql() { let query_port = unused_port(); + let otlp_http_port = unused_port(); + let otlp_grpc_port = unused_port(); let output_dir = tempfile::tempdir().expect("create log directory"); let mut config = tempfile::NamedTempFile::new().expect("create streaming config"); write!( @@ -60,7 +129,7 @@ async fn production_binary_loads_config_and_serves_diagnostics() { metric: component_process_e2e_latency_ms parameters: relativeAccuracy: 0.01 - windowSize: 60 + windowSize: 1 windowType: tumbling spatialFilter: '' "# @@ -74,6 +143,15 @@ async fn production_binary_loads_config_and_serves_diagnostics() { .arg(query_port.to_string()) .arg("--output-dir") .arg(output_dir.path()) + .arg("--enable-otel-ingest") + .arg("--otel-http-port") + .arg(otlp_http_port.to_string()) + .arg("--otel-grpc-port") + .arg(otlp_grpc_port.to_string()) + .arg("--precompute-allowed-lateness-ms") + .arg("0") + .arg("--precompute-flush-interval-ms") + .arg("100") .stdout(Stdio::null()) .stderr(Stdio::null()) .spawn() @@ -84,24 +162,55 @@ async fn production_binary_loads_config_and_serves_diagnostics() { let base = format!("http://127.0.0.1:{query_port}"); wait_until_ready(&client, &format!("{base}/api/v1/health"), &mut child.0).await; - let health = client - .get(format!("{base}/api/v1/health")) - .send() - .await - .expect("GET health") - .text() - .await - .expect("read health body"); - assert_eq!(health, "ok"); + let now_ns = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock") + .as_nanos() as u64; + for body in [ + ddsketch_export( + "component_process_e2e_latency_ms", + now_ns.saturating_sub(3_000_000_000), + vec![5, 10, 15, 20], + ), + ddsketch_export( + "component_process_e2e_latency_ms", + now_ns.saturating_sub(1_000_000_000), + Vec::new(), + ), + ] { + let response = client + .post(format!("http://127.0.0.1:{otlp_http_port}/v1/metrics")) + .header("content-type", "application/x-protobuf") + .body(body) + .send() + .await + .expect("POST modified OTLP to production receiver"); + assert!( + response.status().is_success(), + "OTLP status: {}", + response.status() + ); + } - let config_response: serde_json::Value = client - .get(format!("{base}/api/v1/streaming-config")) - .send() - .await - .expect("GET installed streaming config") - .json() - .await - .expect("decode streaming-config response"); - assert_eq!(config_response["status"], "success"); - assert_eq!(config_response["aggregation_count"], 1); + let query = "quantile_over_time(0.99, component_process_e2e_latency_ms[10s])"; + for _ in 0..50 { + let response: serde_json::Value = client + .get(format!("{base}/api/v1/query")) + .query(&[("query", query)]) + .send() + .await + .expect("query production data plane") + .json() + .await + .expect("decode PromQL response"); + if let Some(value) = first_scalar(&response) { + assert!( + value.is_finite() && value > 0.0, + "invalid quantile: {value}" + ); + return; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!("production data plane never served the ingested DDSketch"); } diff --git a/data_plane/tests/monitor_process_e2e.rs b/data_plane/tests/monitor_process_e2e.rs new file mode 100644 index 00000000..bf7bedbf --- /dev/null +++ b/data_plane/tests/monitor_process_e2e.rs @@ -0,0 +1,145 @@ +//! Process-level E2E for the production monitor coordinator. +//! +//! Two simulated edges connect to the monitor listener hosted by the real +//! data-plane executable, register, report different rates, and receive +//! differentiated sampling grants over bidirectional gRPC streams. + +use std::io::Write; +use std::net::TcpListener; +use std::process::{Child, Command, Stdio}; +use std::time::Duration; + +use asap_otel_proto::monitor::v1::{ + coord_to_edge, edge_to_coord, monitor_service_client::MonitorServiceClient, EdgeToCoord, + MonitorRegister, MonitorReport, +}; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; +use tokio_stream::StreamExt; + +struct ChildGuard(Child); + +impl Drop for ChildGuard { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +fn unused_port() -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").expect("reserve loopback port"); + listener.local_addr().expect("loopback address").port() +} + +async fn connect_edge( + port: u16, + edge_id: &str, +) -> ( + mpsc::Sender, + tonic::Streaming, +) { + let endpoint = format!("http://127.0.0.1:{port}"); + let mut connected = None; + for _ in 0..100 { + match MonitorServiceClient::connect(endpoint.clone()).await { + Ok(client) => { + connected = Some(client); + break; + } + Err(_) => tokio::time::sleep(Duration::from_millis(50)).await, + } + } + let mut client = connected.unwrap_or_else(|| { + panic!("edge could not connect to production monitor endpoint {endpoint}") + }); + let (tx, rx) = mpsc::channel(16); + let inbound = client + .monitor(ReceiverStream::new(rx)) + .await + .expect("open monitor stream") + .into_inner(); + tx.send(EdgeToCoord { + msg: Some(edge_to_coord::Msg::Reg(MonitorRegister { + edge_id: edge_id.into(), + agg_id: 1, + key: Vec::new(), + epoch_window_ms: 60_000, + window_start_ms: 0, + })), + }) + .await + .expect("register edge"); + (tx, inbound) +} + +async fn report_and_receive( + tx: &mpsc::Sender, + inbound: &mut tonic::Streaming, + edge_id: &str, + rate: f64, +) -> f64 { + tx.send(EdgeToCoord { + msg: Some(edge_to_coord::Msg::Report(MonitorReport { + edge_id: edge_id.into(), + agg_id: 1, + key: Vec::new(), + window_start_ms: 0, + local_value: 0.0, + round: 0, + seq: 1, + rate, + })), + }) + .await + .expect("send monitor report"); + + let message = tokio::time::timeout(Duration::from_secs(5), inbound.next()) + .await + .expect("timed out waiting for sampling grant") + .expect("monitor stream ended") + .expect("receive sampling grant"); + match message.msg.expect("coordinator response payload") { + coord_to_edge::Msg::Grant(grant) => grant.sample_p, + other => panic!("expected sampling grant, got {other:?}"), + } +} + +#[tokio::test] +async fn production_coordinator_differentiates_edge_sampling_grants() { + let query_port = unused_port(); + let monitor_port = unused_port(); + let output_dir = tempfile::tempdir().expect("create output directory"); + let mut config = tempfile::NamedTempFile::new().expect("create monitor config"); + write!( + config, + "aggregations: []\nmonitors:\n - agg_id: 1\n key: ''\n tau: 5000.0\n epsilon: 0.05\n window_ms: 60000\n" + ) + .expect("write monitor config"); + + let child = Command::new(env!("CARGO_BIN_EXE_data_plane")) + .arg("--streaming-config") + .arg(config.path()) + .arg("--http-port") + .arg(query_port.to_string()) + .arg("--output-dir") + .arg(output_dir.path()) + .arg("--enable-monitor-coordinator") + .arg("--monitor-grpc-port") + .arg(monitor_port.to_string()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("start production data-plane binary"); + let _child = ChildGuard(child); + + let (hot_tx, mut hot_inbound) = connect_edge(monitor_port, "hot-edge").await; + let (quiet_tx, mut quiet_inbound) = connect_edge(monitor_port, "quiet-edge").await; + tokio::time::sleep(Duration::from_millis(50)).await; + + let (hot, quiet) = tokio::join!( + report_and_receive(&hot_tx, &mut hot_inbound, "hot-edge", 100_000.0), + report_and_receive(&quiet_tx, &mut quiet_inbound, "quiet-edge", 1_000.0), + ); + assert!(hot > 0.0 && hot < quiet, "hot={hot}, quiet={quiet}"); + assert!(quiet <= 1.0, "quiet grant out of range: {quiet}"); +} diff --git a/docs/03-how-to-guides/manual-e2e-tests.md b/docs/03-how-to-guides/manual-e2e-tests.md index 171e2212..842b16ec 100644 --- a/docs/03-how-to-guides/manual-e2e-tests.md +++ b/docs/03-how-to-guides/manual-e2e-tests.md @@ -11,16 +11,23 @@ The command runs tests serially because several transport tests temporarily change process environment or still use fixed loopback ports. It covers: 1. shared type and protobuf wire contracts; -2. the real control-plane binary plus HTTP, OpAMP, plan publication, and - runtime feedback; +2. the real control-plane binary: HTTP workload planning, configuration + delivery to a simulated collector over the production OpAMP WebSocket, and + runtime feedback ingestion over the production gRPC listener; 3. data-plane ingest adapters, query routing, storage, lifecycle, persistence, exact-backend forwarding with controlled peers, and a real data-plane - process bootstrapped from a file; -4. the monitor coordinator over a real bidirectional gRPC connection; -5. Gorilla fragment ingest, WAL recovery, TSDB block construction, Thanos - StoreAPI, compaction, and object-store shipping; and -6. the final controller planning -> backend configuration -> modified OTLP - ingest -> precompute -> SketchStore -> PromQL query path. + process that accepts modified OTLP, stores a DDSketch, and returns its + quantile through the public PromQL endpoint; +4. the monitor coordinator hosted by a real data-plane process, with two edge + clients exchanging reports and differentiated grants over bidirectional + gRPC; +5. a real Gorilla merger process that accepts an XOR fragment over HTTP, + durably writes its TSDB block, and returns the exact chunk over Thanos + StoreAPI, plus its compaction, recovery, and shipping suites; and +6. the final production-process path: typed physical-plan compilation, + BackendPlan/precompute installation, collector capability and applied ACK + over OpAMP, modified-OTLP ingest, SketchStore policy routing, and PromQL + query readout of that same plan. The component suites can also be run separately: @@ -71,6 +78,6 @@ known ignored tests and their reasons with: ./scripts/e2e.sh list ``` -In particular, the older `e2e_modified_otlp_sketch_path` cases remain ignored -after the protobuf refactor. The maintained whole-path suite is -`e2e_controller_plans_and_backend_serves`. +In particular, ignored legacy cases remain visible but do not substitute for +the maintained production-process tests. The final local whole-backend test is +`data_plane/tests/backend_process_e2e.rs`. diff --git a/gorilla-merger/e2e/process_e2e_test.go b/gorilla-merger/e2e/process_e2e_test.go new file mode 100644 index 00000000..fd8e8bf1 --- /dev/null +++ b/gorilla-merger/e2e/process_e2e_test.go @@ -0,0 +1,192 @@ +package e2e_test + +import ( + "bytes" + "context" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + gorilla "github.com/ProjectASAP/asap-gorilla-go" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/thanos-io/thanos/pkg/store/storepb" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +func unusedAddress(t *testing.T) string { + t.Helper() + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("reserve address: %v", err) + } + defer l.Close() + return l.Addr().String() +} + +func waitReady(t *testing.T, url string, stderr *bytes.Buffer) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + resp, err := http.Get(url) + if err == nil { + _ = resp.Body.Close() + if resp.StatusCode == http.StatusOK { + return + } + } + time.Sleep(25 * time.Millisecond) + } + t.Fatalf("gorilla-merger did not become ready: %s", stderr.String()) +} + +func makeFrame(t *testing.T, timestamp int64) ([]byte, []byte) { + t.Helper() + chunk := chunkenc.NewXORChunk() + appender, err := chunk.Appender() + if err != nil { + t.Fatalf("XOR appender: %v", err) + } + for i, value := range []float64{10, 20, 30} { + appender.Append(timestamp+int64(i*10), value) + } + raw := append([]byte(nil), chunk.Bytes()...) + frame := gorilla.EncodeFragmentBatch([]gorilla.Fragment{{ + MetricName: "gorilla_process_e2e_total", + Attributes: map[string]string{"service": "checkout"}, + MinTime: timestamp, + MaxTime: timestamp + 20, + Count: 3, + Encoding: "xor", + Data: raw, + Source: "process-e2e-agent", + }}) + return frame, raw +} + +func queryRawChunk(t *testing.T, address string, minTime, maxTime int64) []byte { + t.Helper() + conn, err := grpc.NewClient(address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("create StoreAPI client: %v", err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + stream, err := storepb.NewStoreClient(conn).Series(ctx, &storepb.SeriesRequest{ + MinTime: minTime, + MaxTime: maxTime, + Matchers: []storepb.LabelMatcher{ + {Type: storepb.LabelMatcher_EQ, Name: labels.MetricName, Value: "gorilla_process_e2e_total"}, + {Type: storepb.LabelMatcher_EQ, Name: "service", Value: "checkout"}, + }, + PartialResponseStrategy: storepb.PartialResponseStrategy_ABORT, + }) + if err != nil { + return nil + } + for { + response, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return nil + } + series := response.GetSeries() + if series == nil { + continue + } + for _, chunk := range series.Chunks { + if chunk.Raw != nil && len(chunk.Raw.Data) > 0 { + return chunk.Raw.Data + } + } + } +} + +func TestProductionProcessIngestsPersistsAndServesXORFragment(t *testing.T) { + binary := os.Getenv("GORILLA_MERGER_E2E_BIN") + if binary == "" { + t.Fatal("GORILLA_MERGER_E2E_BIN is required; use ../../scripts/e2e.sh gorilla-merger") + } + binary, err := filepath.Abs(binary) + if err != nil { + t.Fatalf("resolve binary: %v", err) + } + httpAddress := unusedAddress(t) + grpcAddress := unusedAddress(t) + tsdbDir := t.TempDir() + var stderr bytes.Buffer + cmd := exec.Command(binary, + "--http-address", httpAddress, + "--grpc-address", grpcAddress, + "--tsdb.path", tsdbDir, + "--merge.window", "100ms", + "--merge.reorder-grace", "0s", + "--merge.flush-interval", "20ms", + "--merge.compact-interval", "1h", + ) + cmd.Stdout = io.Discard + cmd.Stderr = &stderr + if err := cmd.Start(); err != nil { + t.Fatalf("start production gorilla-merger: %v", err) + } + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + t.Cleanup(func() { + _ = cmd.Process.Signal(os.Interrupt) + select { + case <-done: + case <-time.After(3 * time.Second): + _ = cmd.Process.Kill() + <-done + } + }) + waitReady(t, fmt.Sprintf("http://%s/-/ready", httpAddress), &stderr) + + base := time.Now().Add(-2 * time.Second).UnixMilli() + frame, wantRaw := makeFrame(t, base) + response, err := http.Post( + fmt.Sprintf("http://%s/ingest/gorilla", httpAddress), + "application/octet-stream", + bytes.NewReader(frame), + ) + if err != nil { + t.Fatalf("POST Gorilla fragment: %v", err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("ingest returned %s", response.Status) + } + + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + gotRaw := queryRawChunk(t, grpcAddress, base-1000, base+1000) + if bytes.Equal(gotRaw, wantRaw) { + // A successful ingest response is issued only after the fragment WAL + // fsync. Once the window becomes a durable block the committed WAL is + // intentionally removed, so assert the post-flush durable artifact. + blockMetas := 0 + _ = filepath.Walk(tsdbDir, func(_ string, info os.FileInfo, err error) error { + if err == nil && info != nil && info.Name() == "meta.json" { + blockMetas++ + } + return nil + }) + if blockMetas == 0 { + t.Fatal("fragment was served but no durable TSDB block was written") + } + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("StoreAPI never returned the exact ingested XOR chunk; process log:\n%s", stderr.String()) +} diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 3ebafc6d..d82915bc 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -96,7 +96,7 @@ data_plane() { rust_test data_plane --test edge_runtime_consumes_precompute_rs CURRENT_STAGE="data-plane/production-process" - say "data-plane: production binary bootstrap and public HTTP diagnostics" + say "data-plane: production binary -> modified OTLP -> SketchStore -> PromQL" rust_test data_plane --test component_process_e2e } @@ -104,27 +104,33 @@ monitor() { CURRENT_STAGE="monitor-grpc" say "monitor: real bidirectional gRPC server/client" rust_test data_plane --test monitor_grpc + + CURRENT_STAGE="monitor-production-process" + say "monitor: production coordinator process -> two edge streams -> sampling grants" + rust_test data_plane --test monitor_process_e2e } gorilla_merger() { CURRENT_STAGE="gorilla-merger" say "gorilla-merger: HTTP ingest, WAL, blocks, StoreAPI, compaction, shipping" need go + mkdir -p "${CARGO_TARGET_DIR}" ( cd "${REPO_DIR}/gorilla-merger" GOPRIVATE="${GOPRIVATE:-github.com/ProjectASAP/*}" \ + go build -o "${CARGO_TARGET_DIR}/gorilla-merger-e2e" ./cmd/gorilla-merger + GORILLA_MERGER_E2E_BIN="${CARGO_TARGET_DIR}/gorilla-merger-e2e" \ + GOPRIVATE="${GOPRIVATE:-github.com/ProjectASAP/*}" \ go test -count=1 ./... ) } whole() { CURRENT_STAGE="whole/controller-to-query" - say "whole repository: controller plan -> backend -> OTLP -> sketch store -> PromQL" - # This is the maintained representative happy path. The broader matrix is - # intentionally a separate target: it includes known feature regressions - # and must not make the routine repository acceptance test nondeterministic. - rust_test data_plane --test e2e_controller_plans_and_backend_serves \ - controller_plan_to_query_full_roundtrip_ddsketch + say "whole repository: production controller -> production backend -> OTLP -> PromQL" + cargo build --locked -p control_plane --bin control_plane -p data_plane --bin data_plane + ASAP_E2E_CONTROL_PLANE_BIN="${CARGO_TARGET_DIR}/debug/control_plane" \ + rust_test data_plane --test backend_process_e2e } whole_matrix() { From 8d66dca9403d0eedd528b0cd6f8936d120046132 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 10:48:50 -0600 Subject: [PATCH 4/8] test: bound manual e2e disk usage --- docs/03-how-to-guides/manual-e2e-tests.md | 4 +++- scripts/e2e.sh | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/03-how-to-guides/manual-e2e-tests.md b/docs/03-how-to-guides/manual-e2e-tests.md index 842b16ec..94bafb4f 100644 --- a/docs/03-how-to-guides/manual-e2e-tests.md +++ b/docs/03-how-to-guides/manual-e2e-tests.md @@ -54,7 +54,9 @@ known query/planner regressions are fixed. Use `ASAP_E2E_NOCAPTURE=1` to display Rust test output. Build artifacts and the Go compilation cache are kept below `target/` by default so a full system -disk does not make Go use the home-directory cache. +disk does not make Go use the home-directory cache. Rust incremental builds +are disabled by default to limit disk usage; set `ASAP_E2E_CARGO_INCREMENTAL=1` +to retain that cache when space is available. ## Full external system diff --git a/scripts/e2e.sh b/scripts/e2e.sh index d82915bc..38348ce2 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -15,6 +15,7 @@ STARTED_AT="$(date +%s)" export CARGO_TARGET_DIR="${ASAP_E2E_CARGO_TARGET_DIR:-${REPO_DIR}/target/e2e}" export GOCACHE="${ASAP_E2E_GO_CACHE:-${REPO_DIR}/target/e2e-go-cache}" +export CARGO_INCREMENTAL="${ASAP_E2E_CARGO_INCREMENTAL:-0}" usage() { cat <<'EOF' @@ -36,6 +37,7 @@ Useful environment variables: ASAP_COLLECTOR_DIR Sibling ASAPCollector checkout (system target) ASAP_E2E_CARGO_TARGET_DIR Rust build directory ASAP_E2E_GO_CACHE Go build cache directory + ASAP_E2E_CARGO_INCREMENTAL Set to 1 to retain Rust incremental artifacts ASAP_E2E_NOCAPTURE=1 Pass --nocapture to Rust test binaries EOF } From 945e7be98fd1f138ebb7d922c1a87a497528ba42 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 11:06:45 -0600 Subject: [PATCH 5/8] test: remove smoke and stale ignored coverage --- control_plane/src/emit/backend_push.rs | 2 +- control_plane/src/emit/mod.rs | 2 +- control_plane/src/emit/stage_config.rs | 2 +- control_plane/src/main.rs | 4 +- control_plane/src/query_parser/mod.rs | 2 +- control_plane/src/workload.rs | 103 -- data_plane/src/drivers/ingest/otel.rs | 2 +- data_plane/src/drivers/query/servers/http.rs | 90 -- .../precompute_engine_design_doc.md | 4 +- .../query_engines/asap_query_engine/engine.rs | 6 +- .../sketch_db/backfill/processor.rs | 4 +- .../sketch_db/backfill/raw_sample_reader.rs | 2 +- .../tests/capability_miss_http_e2e_tests.rs | 82 +- ...e2e_controller_plans_and_backend_serves.rs | 15 +- .../tests/e2e_modified_otlp_sketch_path.rs | 1223 ----------------- .../edge_runtime_consumes_precompute_rs.rs | 103 +- docs/03-how-to-guides/manual-e2e-tests.md | 16 +- scripts/e2e.sh | 4 +- 18 files changed, 44 insertions(+), 1622 deletions(-) delete mode 100644 data_plane/tests/e2e_modified_otlp_sketch_path.rs diff --git a/control_plane/src/emit/backend_push.rs b/control_plane/src/emit/backend_push.rs index 2da63f79..079e3920 100644 --- a/control_plane/src/emit/backend_push.rs +++ b/control_plane/src/emit/backend_push.rs @@ -646,7 +646,7 @@ mod tests { } /// Happy path on the first attempt: zero retries, Ok outcome, - /// attempts == 1. This protects the smoke-test invariant that the + /// attempts == 1. This protects the retry unit-test invariant that the /// fire-and-forget happy path is unchanged when the backend is up /// before the controller's first POST. #[tokio::test(start_paused = true)] diff --git a/control_plane/src/emit/mod.rs b/control_plane/src/emit/mod.rs index 60cf6e6a..ae96ce55 100644 --- a/control_plane/src/emit/mod.rs +++ b/control_plane/src/emit/mod.rs @@ -1256,7 +1256,7 @@ mod runtime_tests { // is threaded through the pre-pop QuerySpec → analyzer → // QueryWorkload.group_by_labels → collect_metric_to_grouping_labels // → the emitter's keep_keys list. Without this round-trip the - // smoke test's sid catalog stays empty-per-zone. + // end-to-end test's sid catalog stays empty-per-zone. #[test] fn workload_entry_grouping_labels_round_trip_through_emit_to_keep_keys() { let yaml = r#" diff --git a/control_plane/src/emit/stage_config.rs b/control_plane/src/emit/stage_config.rs index 2392e562..3ef831e9 100644 --- a/control_plane/src/emit/stage_config.rs +++ b/control_plane/src/emit/stage_config.rs @@ -5301,7 +5301,7 @@ mod tests { // wire-attr tuple, minting one sid per unique tuple — defeating // the streaming-config contract and ballooning the schema endpoint // per-metric sid count (51 for `http_requests_total_latency_ms` - // in the smoke test). + // in the end-to-end acceptance test). // // We chose OTTL `transform` over `attributes/keep` because the // attributes processor has NO native allowlist action (only diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index 78f462f1..a0a84b52 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -439,7 +439,7 @@ async fn main() { // Loop through every `(metric, role)` pair the workload-registry // pre-pop loop populated and POST the typed cumulative // streaming-config + storage-routing to the backend. Without this, - // queries that never trigger `POST /api/v1/plan` (the smoke + // queries that never trigger `POST /api/v1/plan` (the acceptance // harness, bootstrap deployments) hit the data plane's static // startup config (DDSketch only) and `sum by (zone) (…)` returns // `ExactAgg(Sum) capability not satisfied`. @@ -3460,7 +3460,7 @@ mod api_tests { // // The (metric, role) cache key is exercised in tandem by the live // mvp-workload.yaml pre-pop loop (the workload registry lists 3 - // entries for `http_requests_total`) → see the MVP smoke-test + // entries for `http_requests_total`) → see the MVP acceptance-test // pipeline. This in-process test exercises the cumulative-merge // plumbing in isolation against the same emit path used by both // the pre-pop loop and per-request replans. diff --git a/control_plane/src/query_parser/mod.rs b/control_plane/src/query_parser/mod.rs index e494a0da..53426c3f 100644 --- a/control_plane/src/query_parser/mod.rs +++ b/control_plane/src/query_parser/mod.rs @@ -451,7 +451,7 @@ mod tests { const ACC: AccuracyTarget = AccuracyTarget::Epsilon(0.01); - // Smoke tests for the parse entry point. + // Focused unit contracts for the PromQL parse entry point. #[test] fn promql_dispatched_correctly() { diff --git a/control_plane/src/workload.rs b/control_plane/src/workload.rs index 52b65f9a..ae4f053a 100644 --- a/control_plane/src/workload.rs +++ b/control_plane/src/workload.rs @@ -1058,107 +1058,4 @@ mod tests { let distinct: std::collections::HashSet<_> = roles.iter().copied().collect(); assert_eq!(distinct.len(), 2, "Sum + Count = 2 distinct roles"); } - - #[test] - fn live_mvp_workload_yaml_assigns_three_roles_to_http_requests_total() { - // B2 full restructure regression: the live - // `deploy/configs/mvp-workload.yaml` carries THREE entries for - // `http_requests_total` (entries 2/3/4 — sum/sum+rate/count). - // Pre-B2 these collapsed onto one workload-store key and - // dropped two of the three plans, so the `sum by (zone)` - // query returned `ExactAgg(Sum) capability not satisfied`. - // - // The fix is the `(metric, role)` key + the per-entry role - // classification via `derive_agg_role`. This test pins that - // the three entries classify to two distinct roles (`Sum` for - // entries 2 + 3, `Count` for entry 4) — the multi-row keyed - // store can persist them all simultaneously. - use std::path::PathBuf; - let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - path.pop(); - path.push("deploy/configs/mvp-workload.yaml"); - if !path.exists() { - // Live file not in this checkout; skip silently (matches - // the sibling override test below). - return; - } - let registry = WorkloadRegistry::load(path.to_str().unwrap()); - let http_requests_entries: Vec<&WorkloadEntry> = registry - .entries() - .iter() - .filter(|e| e.metric_name == "http_requests_total") - .collect(); - assert!( - http_requests_entries.len() >= 3, - "mvp-workload.yaml is expected to carry ≥3 entries for \ - http_requests_total (sum, sum(rate), count); got {}", - http_requests_entries.len() - ); - let roles: Vec = http_requests_entries - .iter() - .map(|e| derive_agg_role(e)) - .collect(); - // At least one Sum and at least one Count among the entries. - assert!( - roles.contains(&AggRole::Sum), - "expected ≥1 Sum-role entry among http_requests_total in \ - mvp-workload.yaml; got {roles:?}" - ); - assert!( - roles.contains(&AggRole::Count), - "expected ≥1 Count-role entry among http_requests_total in \ - mvp-workload.yaml; got {roles:?}" - ); - } - - #[test] - fn live_mvp_workload_yaml_loads_with_overrides() { - // Smoke-test the live deploy file. Confirms entries 5–8 carry - // their `sketch_family_override` after deserialization (the - // original stitching gap was this field being silently ignored - // by `serde`'s unknown-field default behaviour). - use std::path::PathBuf; - let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - path.pop(); - path.push("deploy/configs/mvp-workload.yaml"); - if !path.exists() { - // Live file not in this checkout; skip silently. - return; - } - let registry = WorkloadRegistry::load(path.to_str().unwrap()); - let by_name: std::collections::HashMap<&str, &WorkloadEntry> = registry - .entries() - .iter() - .map(|e| (e.metric_name.as_str(), e)) - .collect(); - - assert_eq!( - by_name - .get("request_size_bytes") - .and_then(|e| e.sketch_family_override.clone()), - Some(SketchType::KLL), - "request_size_bytes must carry KLL override", - ); - assert_eq!( - by_name - .get("unique_users_per_min") - .and_then(|e| e.sketch_family_override.clone()), - Some(SketchType::HLL), - "unique_users_per_min must carry HLL override", - ); - assert_eq!( - by_name - .get("top_endpoint_qps") - .and_then(|e| e.sketch_family_override.clone()), - Some(SketchType::CountSketch), - "top_endpoint_qps must carry CountSketch override", - ); - assert_eq!( - by_name - .get("endpoint_request_freq") - .and_then(|e| e.sketch_family_override.clone()), - Some(SketchType::CountMinSketch), - "endpoint_request_freq must carry CountMinSketch override", - ); - } } diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index 6659ef72..0160f6db 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -77,7 +77,7 @@ pub struct OtlpReceiver { impl OtlpReceiver { /// Construct a receiver without a backend. Metrics are parsed and - /// logged but not stored — useful for smoke-testing the OTLP pipe. + /// logged but not stored — useful for diagnosing the OTLP pipe. pub fn new(config: OtlpReceiverConfig) -> Self { Self { config, diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 14e8a741..72af0b27 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -2947,96 +2947,6 @@ aggregations: } } - // Schema retirement #2/#5 — the endpoint reads from the sid - // catalog. The reconfigure → timeline flow this test exercised - // depended on `SchemaRegistry::reconcile()` eagerly populating - // the timeline source on YAML POST. Post-#189 the registry is - // gone and sid-level reconcile (see - // `lifecycle::reconcile_from_streaming_config`) deliberately - // does NOT pre-mint sids on POST — they're minted lazily by the - // first ingest write under the new config. Re-enabling this - // test requires either an interleaved ingest step (changes the - // contract being tested) or an architectural switch to eager - // sid minting (contradicts the documented sid lifecycle); both - // are out of scope for #272 step 4. - #[ignore = "obsoleted by sid lazy-mint lifecycle; see comment above and #272 step 4 resolution"] - #[tokio::test] - async fn test_get_timeline_returns_segments_after_reconfigure() { - use crate::storage_engines::sketch_db::index::SketchStore; - - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); - let sketch_index = Arc::new(SketchStore::new()); - let server_port = setup_test_server_with_hot_reload_and_sketch_index( - hot_reload.clone(), - sketch_index.clone(), - ) - .await; - let client = Client::new(); - - // Push initial config with agg 1 on metric "m". Then swap to - // a config with agg 2 on the same metric — registry should - // show timeline with agg 1 retired + agg 2 active. - let post = |yaml: &str| { - let yaml = yaml.to_string(); - let client = client.clone(); - async move { - client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(yaml) - .send() - .await - .unwrap() - } - }; - // PR 5: `aggregationId` is no longer wire-carried; identity is - // content-addressed. To produce two distinct configs we vary - // the window size — different content → different fingerprint. - let yaml = |window: u64| { - format!( - r#" -aggregations: - - aggregationType: Sum - aggregationSubType: '' - metric: m - labels: {{ grouping: [], rollup: [], aggregated: [] }} - parameters: {{}} - windowSize: {window} - windowType: tumbling - spatialFilter: '' -"# - ) - }; - assert!(post(&yaml(60)).await.status().is_success()); - // Wait >1ms so the retire timestamp is strictly after the - // first agg's creation; otherwise the ownership interval is - // zero-width at ms resolution and timeline correctly skips it. - tokio::time::sleep(std::time::Duration::from_millis(5)).await; - assert!(post(&yaml(120)).await.status().is_success()); - - // The ms range is effectively wall-clock; use [0, u64 far - // future] to guarantee both segments fall in range. - let resp = client - .get(format!( - "http://127.0.0.1:{server_port}/api/v1/db/timeline?metric=m&start_ms=0&end_ms=99999999999999" - )) - .send() - .await - .unwrap(); - assert!(resp.status().is_success()); - let body: serde_json::Value = resp.json().await.unwrap(); - assert_eq!(body["status"], "success"); - assert_eq!(body["metric"], "m"); - // Two segments: retired (window=60) + active (window=120). - assert_eq!(body["count"], 2); - let segs = body["segments"].as_array().unwrap(); - assert_eq!(segs[0]["status"], "retired"); - assert_eq!(segs[0]["coverage"], "sketch"); - assert_eq!(segs[1]["status"], "active"); - } - #[tokio::test] async fn test_get_timeline_missing_param_returns_400() { use crate::storage_engines::sketch_db::index::SketchStore; diff --git a/data_plane/src/precompute_engine/precompute_engine_design_doc.md b/data_plane/src/precompute_engine/precompute_engine_design_doc.md index ca9f746d..3b23a1d0 100644 --- a/data_plane/src/precompute_engine/precompute_engine_design_doc.md +++ b/data_plane/src/precompute_engine/precompute_engine_design_doc.md @@ -1174,8 +1174,8 @@ store with the Kafka consumer path. - **Unit tests -- other modules**: `window_manager.rs` (tumbling/sliding arithmetic, pane enumeration, closure detection), `series_buffer.rs` (ordering, watermark), `accumulator_factory.rs` (updater creation and reset), `series_router.rs` (consistent hash routing), `config.rs` (defaults). - **E2E coverage**: end-to-end paths now run through the OTLP receiver - driving the same `IngestState` (`tests/e2e_modified_otlp_sketch_path.rs`, - `tests/edge_runtime_consumes_precompute_rs.rs`; the runnable demo + driving the same `IngestState` (`tests/component_process_e2e.rs`, + `tests/backend_process_e2e.rs`; the runnable multi-node demo lives in ASAPCollector). The legacy in-process remote-write E2E binaries (`bin/test_e2e_precompute.rs`, `bin/e2e_quickstart_resource_test.rs`, `bin/bench_precompute_sketch.rs`) and the equivalent test diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 4e498e4b..d85c8034 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -1368,7 +1368,7 @@ mod asap_tier_classify_tests { /// Schema-retirement #5 regression: a sketch sid registered with /// a wider-than-requested `group_by_keys` and `policy_fp=UNSET` /// must still be findable by the query path. Mirrors the MVP - /// smoke-test failure (issue #271 / tracking #272): the agent + /// end-to-end failure (issue #271 / tracking #272): the agent /// emits DDSketch DPs carrying every wire attribute, so the sid /// catalog ends up with `group_by_keys=[zone,rack,node,pod,...]` /// and `derive_sketch_policy_fp` returns `UNSET` because no @@ -1427,7 +1427,7 @@ mod asap_tier_classify_tests { } /// `sum by (zone) (http_requests_total)` end-to-end via the - /// `execute(&str)` adapter. Mirrors the MVP smoke test's Axis-C + /// `execute(&str)` adapter. Mirrors the MVP acceptance test's Axis-C /// failure: ExactAgg(Sum) sids existed for `http_requests_total` /// (one per zone), but the old reducer /// returned `UnsupportedFunction("sum")` because @@ -1442,7 +1442,7 @@ mod asap_tier_classify_tests { use crate::storage_engines::sketch_db::data::AggregationType; let idx = Arc::new(SketchStore::new()); - // Mirror the smoke-test setup: four ExactAgg(Sum) sids, one + // Mirror the acceptance-test setup: four ExactAgg(Sum) sids, one // per zone (z0..z3), registered with `group_by_keys=["zone"]` // and carrying a `SumAccumulator` per window. let zones = ["z0", "z1", "z2", "z3"]; diff --git a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs index 4899e3f0..d9ca4b7c 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs @@ -593,8 +593,8 @@ mod tests { assert_eq!(written[2], (fp, (20, 30))); assert_eq!(written[3], (fp, (30, 40))); - // Smoke test: the worker didn't fail mid-run. Window - // assertions above are sufficient. + // The exact window assertions above prove the worker completed + // every expected write. } /// ## The parity test (§10.5 determinism invariant) diff --git a/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs b/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs index a45f4e77..0fde09ec 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs @@ -150,7 +150,7 @@ pub trait RawSampleReader: Send + Sync { } /// In-memory reader used by unit tests and the Phase 5c worker -/// smoke test. Seeded with a flat `Vec` at construction; +/// deterministic test adapter. Seeded with a flat `Vec` at construction; /// `read_samples` applies `(range, filter)` on every call. /// /// Not for production use — it doesn't scale past a few thousand diff --git a/data_plane/src/tests/capability_miss_http_e2e_tests.rs b/data_plane/src/tests/capability_miss_http_e2e_tests.rs index 68a78a36..9d0d2632 100644 --- a/data_plane/src/tests/capability_miss_http_e2e_tests.rs +++ b/data_plane/src/tests/capability_miss_http_e2e_tests.rs @@ -29,19 +29,17 @@ //! "control plane reacts to workload drift in T seconds" claim has //! a concrete local floor. //! -//! Scope note: we do not ingest samples here. The "next query -//! actually returns data" half of the story requires OTLP -//! ingestion through a separate entry point and is covered by -//! the unit tests in `simple_engine.rs`. What this file locks -//! down is the HTTP-boundary behaviour of the feedback loop -//! (plan-arrival + idempotency on repeat query). +//! Scope note: we do not ingest samples here. The "next query actually +//! returns data" half is covered by the production-process E2E. This file +//! locks down the HTTP-boundary plan-arrival behavior only. A config alone +//! does not make a repeat query servable under the current lazy-SID model. use crate::drivers::control_plane_client::{ControlPlaneClient, HttpControlPlaneClient}; use crate::drivers::query::adapters::AdapterConfig; use crate::drivers::query::servers::http::{HttpServer, HttpServerConfig}; use crate::query_engines::ASAPQueryEngine; #[cfg(test)] -use crate::storage_engines::types::{HotReloadStreamingConfig, QueryLanguage, StreamingConfig}; +use crate::storage_engines::types::{HotReloadStreamingConfig, StreamingConfig}; use axum::{extract::State, routing::post, Router}; use reqwest::Client; use serde_json::Value; @@ -317,7 +315,6 @@ async fn http_capability_miss_feedback_loop_closes_over_http() { // code paths (timeline dispatch, capability matching, // per-segment resolution) can each observe the miss // before the plan lands — so we assert `>= 1` here. - // Idempotency on a *repeat* query is a separate test. let count = control_plane_state.received_count.load(Ordering::SeqCst); assert!( count >= 1, @@ -328,72 +325,3 @@ async fn http_capability_miss_feedback_loop_closes_over_http() { "mock control plane should have pushed plan back to backend" ); } - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "regression after InferenceConfig retirement; see TODO"] -async fn http_capability_miss_repeat_query_is_idempotent_over_http() { - // After the plan lands, the SAME query must not fire a - // second miss notification — `find_compatible_aggregation` - // now returns Some and the miss-notify branch is skipped. - // This is the HTTP-visible version of the in-process - // `capability_miss_idempotent_on_repeat` test. - let metric = "http_e2e_repeat_metric"; - let expected_agg_id: u64 = 4343; - - let (backend_url, control_plane_state, _hot_reload) = - spin_up_loop(metric, expected_agg_id).await; - let client = Client::new(); - - // 1. First query — miss, triggers the loop. - let _ = client - .get(format!("{backend_url}/api/v1/query")) - .query(&[("query", format!("sum({metric})").as_str()), ("time", "0")]) - .send() - .await - .unwrap() - .bytes() - .await; - - poll_until_plan_active( - &client, - &backend_url, - expected_agg_id, - Duration::from_secs(3), - ) - .await - .expect("plan should have landed within 3s"); - - // 2. Second query on the same metric — should be idempotent. - // Let any still-in-flight fire-and-forget notifies from - // the first query land before we snapshot `count_before`, - // so we're comparing apples to apples. - sleep(Duration::from_millis(150)).await; - let count_before = control_plane_state.received_count.load(Ordering::SeqCst); - assert!( - count_before >= 1, - "first query should have triggered at least one notify" - ); - - let t_second = Instant::now(); - let _ = client - .get(format!("{backend_url}/api/v1/query")) - .query(&[("query", format!("sum({metric})").as_str()), ("time", "0")]) - .send() - .await - .unwrap() - .bytes() - .await; - // Give any errant fire-and-forget notify a window to land. - sleep(Duration::from_millis(150)).await; - let count_after = control_plane_state.received_count.load(Ordering::SeqCst); - - println!( - "http-e2e-repeat: notifies before={count_before} after={count_after} \ - second_query_roundtrip={}ms", - t_second.elapsed().as_millis() - ); - assert_eq!( - count_after, count_before, - "a repeat query on a covered agg_id must NOT fire a second capability-miss" - ); -} diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index cd828744..a3201072 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -761,13 +761,13 @@ async fn post_otlp_http(client: &reqwest::Client, port: u16, req: ExportMetricsS // ── Test 1 — single DDSketch-quantile workload, no grouping ───────────────── // -// Smoke test: the controller emits a streaming-config JSON for a +// HTTP integration contract: the controller emits a streaming-config JSON for a // workload that resolves to DDSketch. The backend's parser accepts it // (POST returns 2xx) and the registered aggregation surfaces on the // GET endpoint with the expected metric / sketch family. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn controller_plans_ddsketch_quantile_and_backend_parses_streaming_config() { +async fn controller_streaming_config_round_trips_through_backend_http() { let (port, _hot_reload) = start_backend_http_server().await; let client = reqwest::Client::new(); @@ -952,8 +952,8 @@ async fn controller_plan_to_query_full_roundtrip_ddsketch() { // count math (DDSketch index = ceil(log_gamma(value))) doesn't // matter for this test — we want to verify the wire round-trip, // not the quantile readout accuracy. Pick a simple count vector - // the existing `e2e_modified_otlp_sketch_path::e2e_dd_sketch_*` - // test uses so we know it's representable. + // used by the production modified-OTLP process E2E, so it is known + // to be representable. let alpha = 0.01; let store_counts = vec![5u64, 10, 15, 20]; let dd_state = build_dd_sketch_state(alpha, store_counts, -1); @@ -1412,13 +1412,6 @@ async fn controller_plan_to_query_full_roundtrip_hll() { // sketch on its own). #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -#[ignore = "known gap, exposed (not caused) by sketch_reducer.rs's retirement: this shape \ - now hard capability-misses on SummaryExecutor ('No result for query') instead of \ - silently falling through to the retired legacy reducer, which used to mask it. Not \ - fully root-caused yet -- possibly the same effective_is_cumulative gap as \ - ASAPQuery-backend#431 (this query is also `count_over_time(...)`, a function name \ - effective_is_cumulative's match doesn't cover), but that's unconfirmed for this \ - specific CountSketchWithHeap/heap_size shape. Needs its own investigation."] async fn controller_plan_to_query_full_roundtrip_count_sketch() { let stack = start_full_stack(19_567, 19_568).await; let client = reqwest::Client::new(); diff --git a/data_plane/tests/e2e_modified_otlp_sketch_path.rs b/data_plane/tests/e2e_modified_otlp_sketch_path.rs deleted file mode 100644 index fa2ed1a4..00000000 --- a/data_plane/tests/e2e_modified_otlp_sketch_path.rs +++ /dev/null @@ -1,1223 +0,0 @@ -//! End-to-end integration test for the modified-OTLP sketch hot path. -//! -//! Scope: the **CountMinSketch** path that landed in PR B. Other sketch -//! types (KLL / DDSketch / CountSketch / HLL) get e2e coverage as PR C -//! delivers each per-type decoder. -//! -//! What this test exercises: -//! 1. `OtlpReceiver::with_ingest_state` wired to a running -//! `PrecomputeEngine` -//! 2. A real `ExportMetricsServiceRequest` carrying -//! `Metric.data = CountMinSketch{…}` typed sketch data points, -//! sent over OTLP HTTP at `/v1/metrics` -//! 3. `route_modified_otlp_sketches_to_precompute` decoding the typed -//! `sketch` bytes via -//! `CountMinSketchAccumulator::from_sketchlib_proto_bytes` -//! 4. The precompute engine's `sketch_panes` merging the incoming -//! accumulator into the matching `(agg_id, group_key)` window -//! 5. Window close emitting to a `CapturingOutputSink` -//! 6. The captured `CountMinSketchAccumulator` matrix contents -//! matching what was sent, confirming that PR A vendoring + -//! PR B routing + PR B per-variant decoder all work end-to-end -//! -//! This is the correctness anchor for Phase 1's hot path. See -//! `docs/pipeline-query-catalog.md` §5.4 in the DataCollector repo for -//! the full architectural context. - -use asap_otel_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; -use asap_otel_proto::tonic::common::v1::{any_value, AnyValue, KeyValue}; -use asap_otel_proto::tonic::metrics::v1::{ - metric::Data, CountMinSketch, CountMinSketchDataPoint, CountMinSketchEncoding, CountSketch, - CountSketchDataPoint, CountSketchEncoding, DdSketch, DdSketchDataPoint, DdSketchEncoding, - HllSketch, HllSketchDataPoint, HllSketchEncoding, KllSketch, KllSketchDataPoint, - KllSketchEncoding, Metric, ResourceMetrics, ScopeMetrics, -}; -use asap_sketchlib::proto::sketchlib::{ - CountMinState, CountSketchState, CounterType, DdSketchState, HllVariant as ProtoHllVariant, - HyperLogLogState, KllState, -}; -use asap_sketchlib::MessagePackCodec; -use asap_types::aggregation_config::AggregationConfig; -use asap_types::enums::WindowKind; -use asap_types::AggregationType; -use prost::Message; -use std::collections::HashMap; -use std::sync::Arc; - -use data_plane::drivers::ingest::{OtlpReceiver, OtlpReceiverConfig}; -use data_plane::precompute_engine::config::{LateDataPolicy, PrecomputeEngineConfig}; -use data_plane::precompute_engine::operators::{ - CountMinSketchAccumulator, CountSketchAccumulator, DDSketchAccumulator, - DatasketchesKLLAccumulator, HllSketchAccumulator, -}; -use data_plane::precompute_engine::output_sink::CapturingOutputSink; -use data_plane::precompute_engine::PrecomputeEngine; -use data_plane::storage_engines::types::StreamingConfig; - -/// Build a tumbling-window `CountMinSketch` aggregation for one metric, -/// grouped by a single label. Mirrors the helper in -/// `tests/e2e_precompute_equivalence.rs` but for CountMinSketch. -fn make_count_min_agg_config( - id: u64, - metric: &str, - window_secs: u64, - grouping: Vec<&str>, - rows: usize, - cols: usize, -) -> AggregationConfig { - let mut params = HashMap::new(); - params.insert("row_num".to_string(), serde_json::Value::from(rows as u64)); - params.insert("col_num".to_string(), serde_json::Value::from(cols as u64)); - AggregationConfig::new( - AggregationType::CountMinSketch, - String::new(), - params, - asap_types::KeyByLabelNames::new(grouping.iter().map(|s| s.to_string()).collect()), - asap_types::KeyByLabelNames::new(vec![]), - asap_types::KeyByLabelNames::new(vec![]), - String::new(), - window_secs, - 0, - WindowKind::Tumbling, - metric.to_string(), - metric.to_string(), - None, - None, - None, - ) -} - -/// Engine config with a fast flush interval so the test does not have to -/// wait long after the watermark advances. The legacy remote-write HTTP -/// listener (and its `ingest_port`) was removed; OTLP receiver ports are -/// configured separately on the receiver itself. -fn engine_config() -> PrecomputeEngineConfig { - PrecomputeEngineConfig { - num_workers: 2, - allowed_lateness_ms: 0, - max_buffer_per_series: 10_000, - flush_interval_ms: 100, - channel_buffer_size: 10_000, - pass_raw_samples: false, - raw_mode_aggregation_id: 0, - late_data_policy: LateDataPolicy::Drop, - wall_clock_grace_period_ms: 5_000, - schema_persist_path: None, - } -} - -/// Build a `CountMinState` proto from a known matrix in row-major order. -fn build_count_min_state(rows: u32, cols: u32, counts_int: Vec) -> CountMinState { - assert_eq!( - counts_int.len() as u32, - rows * cols, - "counts_int length must equal rows * cols" - ); - CountMinState { - rows, - cols, - counter_type: CounterType::Int64 as i32, - counts_int, - counts_float: Vec::new(), - sum_counts: Vec::new(), - sum2_counts: Vec::new(), - l1: Vec::new(), - l2: Vec::new(), - } -} - -/// Build an `ExportMetricsServiceRequest` carrying a single -/// `Metric.data = CountMinSketch{…}` payload with the given sketch bytes, -/// timestamped at `time_unix_nano` and labeled with `service`. -fn build_export_request( - metric_name: &str, - service_label: &str, - time_unix_nano: u64, - sketch_bytes: Vec, -) -> ExportMetricsServiceRequest { - let dp = CountMinSketchDataPoint { - attributes: vec![KeyValue { - key: "service".to_string(), - value: Some(AnyValue { - value: Some(any_value::Value::StringValue(service_label.to_string())), - }), - }], - start_time_unix_nano: 0, - time_unix_nano, - sketch: sketch_bytes, - encoding: CountMinSketchEncoding::Proto as i32, - flags: 0, - series_id: 0, - }; - ExportMetricsServiceRequest { - resource_metrics: vec![ResourceMetrics { - resource: None, - scope_metrics: vec![ScopeMetrics { - scope: None, - metrics: vec![Metric { - name: metric_name.to_string(), - description: String::new(), - unit: String::new(), - metadata: Vec::new(), - data: Some(Data::Countminsketch(CountMinSketch { - data_points: vec![dp], - aggregation_temporality: 0, - rows: 0, - cols: 0, - })), - }], - schema_url: String::new(), - }], - schema_url: String::new(), - }], - } -} - -/// POST a protobuf-encoded `ExportMetricsServiceRequest` to the OTLP HTTP -/// endpoint at `localhost:port/v1/metrics`. Returns `()` on success or -/// panics with the unexpected status code. -async fn post_otlp_http(client: &reqwest::Client, port: u16, req: ExportMetricsServiceRequest) { - let body = req.encode_to_vec(); - let resp = client - .post(format!("http://127.0.0.1:{port}/v1/metrics")) - .header("Content-Type", "application/x-protobuf") - .body(body) - .send() - .await - .expect("OTLP HTTP send failed"); - assert!( - resp.status().is_success(), - "OTLP HTTP returned unexpected status {}", - resp.status() - ); -} - -#[ignore = "broken since proto refactor; PR compile-only fix"] -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn e2e_count_min_sketch_modified_otlp_path() { - // ─── 1. Topology ──────────────────────────────────────────────────── - let agg_id = 42u64; - let metric_name = "http_requests_total"; - let service_label = "auth"; - let window_secs = 1u64; - let rows = 2u32; - let cols = 4u32; - - let otlp_grpc_port = 19501u16; - let otlp_http_port = 19502u16; - - let cms_config = make_count_min_agg_config( - agg_id, - metric_name, - window_secs, - vec!["service"], - rows as usize, - cols as usize, - ); - let mut agg_map = HashMap::new(); - agg_map.insert(agg_id, cms_config); - let streaming_config = Arc::new(StreamingConfig::new(agg_map)); - - let sink = Arc::new(CapturingOutputSink::new()); - let engine = PrecomputeEngine::new( - engine_config(), - data_plane::storage_engines::types::HotReloadStreamingConfig::from_arc(streaming_config), - sink.clone(), - Arc::new(data_plane::drivers::ingest::series_resolver::SeriesIdResolver::new()), - Arc::new(data_plane::storage_engines::sketch_db::index::SketchStore::new()), - ); - let ingest_state = engine.ingest_state(); - - // Spawn the precompute engine (spawns workers + Prometheus remote-write - // ingest server; we only use the workers/router from it). - tokio::spawn(async move { - let _ = engine.run().await; - }); - - // Spawn the OTLP receiver wired to the engine's ingest state. - let otlp_receiver = OtlpReceiver::with_ingest_state( - OtlpReceiverConfig { - grpc_port: otlp_grpc_port, - http_port: otlp_http_port, - }, - ingest_state, - ); - tokio::spawn(async move { - let _ = otlp_receiver.run().await; - }); - - // Wait for both the engine and the OTLP HTTP server to bind. - tokio::time::sleep(tokio::time::Duration::from_millis(400)).await; - - // ─── 2. Send a sketch payload through OTLP ────────────────────────── - // Known matrix in row-major order: - // row 0: [1, 2, 3, 4] - // row 1: [5, 6, 7, 8] - let counts_int: Vec = vec![1, 2, 3, 4, 5, 6, 7, 8]; - let cms_state = build_count_min_state(rows, cols, counts_int.clone()); - let sketch_bytes = cms_state.encode_to_vec(); - - let client = reqwest::Client::new(); - - // First send: timestamped at the start of window 0 (ts = 100ms). - let req = build_export_request(metric_name, service_label, 100_000_000, sketch_bytes); - post_otlp_http(&client, otlp_http_port, req).await; - - // Second send: timestamped past the window end (ts > window_secs * 1_000ms) - // so the precompute engine's watermark advances and closes window 0. - // Use an empty 2x4 matrix so the closing point doesn't disturb anything - // structurally. - let zero_state = build_count_min_state(rows, cols, vec![0i64; (rows * cols) as usize]); - let zero_bytes = zero_state.encode_to_vec(); - let watermark_advance_req = build_export_request( - metric_name, - service_label, - 2_000_000_000, // 2 s past epoch — past the 1 s window end - zero_bytes, - ); - post_otlp_http(&client, otlp_http_port, watermark_advance_req).await; - - // Wait long enough for the periodic flush to fire and the worker to - // emit the closed window to the sink. - tokio::time::sleep(tokio::time::Duration::from_millis(800)).await; - - // ─── 3. Drain and assert ──────────────────────────────────────────── - let captured = sink.drain(); - assert!( - !captured.is_empty(), - "expected at least one closed window output, got 0" - ); - - // Find the entry corresponding to window 0 (start_timestamp == 0). - // The watermark-advance request also occupies a later window which the - // engine may or may not have emitted yet; we only care about window 0. - let (window0_output, window0_acc_box) = captured - .iter() - .find(|(out, _)| out.start_timestamp == 0) - .expect("no captured output for window 0"); - - assert_eq!(window0_output.policy_fp.as_u64(), agg_id); - assert_eq!(window0_output.end_timestamp, window_secs * 1_000); - - let window0_acc = window0_acc_box - .as_any() - .downcast_ref::() - .expect("captured accumulator should be CountMinSketchAccumulator"); - - let stored_matrix = window0_acc.inner.sketch(); - assert_eq!(stored_matrix.len(), rows as usize, "row count mismatch"); - for r in 0..rows as usize { - let expected: Vec = counts_int[r * cols as usize..(r + 1) * cols as usize] - .iter() - .map(|&v| v as f64) - .collect(); - assert_eq!( - stored_matrix[r], expected, - "matrix row {r} mismatch (expected {expected:?}, got {:?})", - stored_matrix[r] - ); - } -} - -// ─── CountSketch path ──────────────────────────────────────────────────── - -/// Parallel to `make_count_min_agg_config` but for `CountSketch`. Uses -/// `AggregationType::CountSketch` added in PR C-CountSketch. -fn make_count_sketch_agg_config( - id: u64, - metric: &str, - window_secs: u64, - grouping: Vec<&str>, - rows: usize, - cols: usize, -) -> AggregationConfig { - let mut params = HashMap::new(); - params.insert("row_num".to_string(), serde_json::Value::from(rows as u64)); - params.insert("col_num".to_string(), serde_json::Value::from(cols as u64)); - AggregationConfig::new( - AggregationType::CountSketch, - String::new(), - params, - asap_types::KeyByLabelNames::new(grouping.iter().map(|s| s.to_string()).collect()), - asap_types::KeyByLabelNames::new(vec![]), - asap_types::KeyByLabelNames::new(vec![]), - String::new(), - window_secs, - 0, - WindowKind::Tumbling, - metric.to_string(), - metric.to_string(), - None, - None, - None, - ) -} - -/// Build a `CountSketchState` proto from a signed matrix in row-major order. -fn build_count_sketch_state(rows: u32, cols: u32, counts_int: Vec) -> CountSketchState { - assert_eq!( - counts_int.len() as u32, - rows * cols, - "counts_int length must equal rows * cols" - ); - CountSketchState { - rows, - cols, - counter_type: CounterType::Int64 as i32, - counts_int, - counts_float: Vec::new(), - l2: Vec::new(), - topk: None, - } -} - -/// Build an `ExportMetricsServiceRequest` carrying a single -/// `Metric.data = CountSketch{…}` payload with the given sketch bytes, -/// timestamped at `time_unix_nano` and labeled with `service`. -fn build_count_sketch_export_request( - metric_name: &str, - service_label: &str, - time_unix_nano: u64, - sketch_bytes: Vec, -) -> ExportMetricsServiceRequest { - let dp = CountSketchDataPoint { - attributes: vec![KeyValue { - key: "service".to_string(), - value: Some(AnyValue { - value: Some(any_value::Value::StringValue(service_label.to_string())), - }), - }], - start_time_unix_nano: 0, - time_unix_nano, - sketch: sketch_bytes, - encoding: CountSketchEncoding::Proto as i32, - flags: 0, - series_id: 0, - }; - ExportMetricsServiceRequest { - resource_metrics: vec![ResourceMetrics { - resource: None, - scope_metrics: vec![ScopeMetrics { - scope: None, - metrics: vec![Metric { - name: metric_name.to_string(), - description: String::new(), - unit: String::new(), - metadata: Vec::new(), - data: Some(Data::Countsketch(CountSketch { - data_points: vec![dp], - aggregation_temporality: 0, - rows: 0, - cols: 0, - })), - }], - schema_url: String::new(), - }], - schema_url: String::new(), - }], - } -} - -#[ignore = "broken since proto refactor; PR compile-only fix"] -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn e2e_count_sketch_modified_otlp_path() { - // Same topology as the CountMin test, different ports, different - // aggregation type, and signed counters. - let agg_id = 43u64; - let metric_name = "request_events_total"; - let service_label = "checkout"; - let window_secs = 1u64; - let rows = 2u32; - let cols = 4u32; - - let otlp_grpc_port = 19511u16; - let otlp_http_port = 19512u16; - - let cs_config = make_count_sketch_agg_config( - agg_id, - metric_name, - window_secs, - vec!["service"], - rows as usize, - cols as usize, - ); - let mut agg_map = HashMap::new(); - agg_map.insert(agg_id, cs_config); - let streaming_config = Arc::new(StreamingConfig::new(agg_map)); - - let sink = Arc::new(CapturingOutputSink::new()); - let engine = PrecomputeEngine::new( - engine_config(), - data_plane::storage_engines::types::HotReloadStreamingConfig::from_arc(streaming_config), - sink.clone(), - Arc::new(data_plane::drivers::ingest::series_resolver::SeriesIdResolver::new()), - Arc::new(data_plane::storage_engines::sketch_db::index::SketchStore::new()), - ); - let ingest_state = engine.ingest_state(); - - tokio::spawn(async move { - let _ = engine.run().await; - }); - - let otlp_receiver = OtlpReceiver::with_ingest_state( - OtlpReceiverConfig { - grpc_port: otlp_grpc_port, - http_port: otlp_http_port, - }, - ingest_state, - ); - tokio::spawn(async move { - let _ = otlp_receiver.run().await; - }); - - tokio::time::sleep(tokio::time::Duration::from_millis(400)).await; - - // Signed counts — the whole point of Count Sketch is ±1 increments, - // so the matrix contains negative values to distinguish it from - // CountMin. - // row 0: [ 1, -2, 3, -4] - // row 1: [-5, 6, -7, 8] - let counts_int: Vec = vec![1, -2, 3, -4, -5, 6, -7, 8]; - let cs_state = build_count_sketch_state(rows, cols, counts_int.clone()); - let sketch_bytes = cs_state.encode_to_vec(); - - let client = reqwest::Client::new(); - - // First send: window 0 payload with the known matrix. - let req = - build_count_sketch_export_request(metric_name, service_label, 100_000_000, sketch_bytes); - post_otlp_http(&client, otlp_http_port, req).await; - - // Second send: watermark advance past window end. - let zero_state = build_count_sketch_state(rows, cols, vec![0i64; (rows * cols) as usize]); - let watermark_advance_req = build_count_sketch_export_request( - metric_name, - service_label, - 2_000_000_000, - zero_state.encode_to_vec(), - ); - post_otlp_http(&client, otlp_http_port, watermark_advance_req).await; - - tokio::time::sleep(tokio::time::Duration::from_millis(800)).await; - - let captured = sink.drain(); - assert!( - !captured.is_empty(), - "expected at least one closed window output, got 0" - ); - - let (window0_output, window0_acc_box) = captured - .iter() - .find(|(out, _)| out.start_timestamp == 0) - .expect("no captured output for window 0"); - - assert_eq!(window0_output.policy_fp.as_u64(), agg_id); - assert_eq!(window0_output.end_timestamp, window_secs * 1_000); - - let window0_acc = window0_acc_box - .as_any() - .downcast_ref::() - .expect("captured accumulator should be CountSketchAccumulator"); - - let stored_matrix = window0_acc.inner.sketch(); - assert_eq!(stored_matrix.len(), rows as usize, "row count mismatch"); - for r in 0..rows as usize { - let expected: Vec = counts_int[r * cols as usize..(r + 1) * cols as usize] - .iter() - .map(|&v| v as f64) - .collect(); - assert_eq!( - stored_matrix[r], expected, - "matrix row {r} mismatch (expected {expected:?}, got {:?})", - stored_matrix[r] - ); - } -} - -// ─── KLL sketch path ───────────────────────────────────────────────────── - -fn make_kll_agg_config( - id: u64, - metric: &str, - window_secs: u64, - grouping: Vec<&str>, - k: u32, -) -> AggregationConfig { - let mut params = HashMap::new(); - params.insert("k".to_string(), serde_json::Value::from(k)); - AggregationConfig::new( - AggregationType::DatasketchesKLL, - "DatasketchesKLL".to_string(), - params, - asap_types::KeyByLabelNames::new(grouping.iter().map(|s| s.to_string()).collect()), - asap_types::KeyByLabelNames::new(vec![]), - asap_types::KeyByLabelNames::new(vec![]), - String::new(), - window_secs, - 0, - WindowKind::Tumbling, - metric.to_string(), - metric.to_string(), - None, - None, - None, - ) -} - -/// Build a `KllState` proto carrying the given retained items. Level -/// metadata is not populated — the decoder replays items via `update()` -/// regardless, per the lossy-reconstruction strategy documented on -/// `DatasketchesKLLAccumulator::from_sketchlib_proto_bytes`. -fn build_kll_state(k: u32, items: Vec) -> KllState { - KllState { - k, - m: 8, - num_levels: 0, - levels: Vec::new(), - items, - coin: None, - offset: 0.0, - value_scale: 0, - residuals: Vec::new(), - } -} - -fn build_kll_export_request( - metric_name: &str, - service_label: &str, - time_unix_nano: u64, - sketch_bytes: Vec, -) -> ExportMetricsServiceRequest { - let dp = KllSketchDataPoint { - attributes: vec![KeyValue { - key: "service".to_string(), - value: Some(AnyValue { - value: Some(any_value::Value::StringValue(service_label.to_string())), - }), - }], - start_time_unix_nano: 0, - time_unix_nano, - sketch: sketch_bytes, - encoding: KllSketchEncoding::Proto as i32, - flags: 0, - series_id: 0, - }; - ExportMetricsServiceRequest { - resource_metrics: vec![ResourceMetrics { - resource: None, - scope_metrics: vec![ScopeMetrics { - scope: None, - metrics: vec![Metric { - name: metric_name.to_string(), - description: String::new(), - unit: String::new(), - metadata: Vec::new(), - data: Some(Data::Kllsketch(KllSketch { - data_points: vec![dp], - aggregation_temporality: 0, - k: 200, - })), - }], - schema_url: String::new(), - }], - schema_url: String::new(), - }], - } -} - -#[ignore = "broken since proto refactor; PR compile-only fix"] -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn e2e_kll_sketch_modified_otlp_path() { - let agg_id = 44u64; - let metric_name = "request_latency_ms"; - let service_label = "api"; - let window_secs = 1u64; - let k = 200u32; - - let otlp_grpc_port = 19521u16; - let otlp_http_port = 19522u16; - - let kll_config = make_kll_agg_config(agg_id, metric_name, window_secs, vec!["service"], k); - let mut agg_map = HashMap::new(); - agg_map.insert(agg_id, kll_config); - let streaming_config = Arc::new(StreamingConfig::new(agg_map)); - - let sink = Arc::new(CapturingOutputSink::new()); - let engine = PrecomputeEngine::new( - engine_config(), - data_plane::storage_engines::types::HotReloadStreamingConfig::from_arc(streaming_config), - sink.clone(), - Arc::new(data_plane::drivers::ingest::series_resolver::SeriesIdResolver::new()), - Arc::new(data_plane::storage_engines::sketch_db::index::SketchStore::new()), - ); - let ingest_state = engine.ingest_state(); - - tokio::spawn(async move { - let _ = engine.run().await; - }); - - let otlp_receiver = OtlpReceiver::with_ingest_state( - OtlpReceiverConfig { - grpc_port: otlp_grpc_port, - http_port: otlp_http_port, - }, - ingest_state, - ); - tokio::spawn(async move { - let _ = otlp_receiver.run().await; - }); - - tokio::time::sleep(tokio::time::Duration::from_millis(400)).await; - - let items: Vec = (1..=100).map(|v| v as f64).collect(); - let kll_state = build_kll_state(k, items); - let sketch_bytes = kll_state.encode_to_vec(); - - let client = reqwest::Client::new(); - let req = build_kll_export_request(metric_name, service_label, 100_000_000, sketch_bytes); - post_otlp_http(&client, otlp_http_port, req).await; - - let empty_state = build_kll_state(k, Vec::new()); - let watermark_req = build_kll_export_request( - metric_name, - service_label, - 2_000_000_000, - empty_state.encode_to_vec(), - ); - post_otlp_http(&client, otlp_http_port, watermark_req).await; - - tokio::time::sleep(tokio::time::Duration::from_millis(800)).await; - - let captured = sink.drain(); - assert!( - !captured.is_empty(), - "expected at least one closed window output, got 0" - ); - - let (window0_output, window0_acc_box) = captured - .iter() - .find(|(out, _)| out.start_timestamp == 0) - .expect("no captured output for window 0"); - - assert_eq!(window0_output.policy_fp.as_u64(), agg_id); - assert_eq!(window0_output.end_timestamp, window_secs * 1_000); - - let kll_acc = window0_acc_box - .as_any() - .downcast_ref::() - .expect("captured accumulator should be DatasketchesKLLAccumulator"); - - let p50 = kll_acc.get_quantile(0.5); - assert!( - (30.0..=70.0).contains(&p50), - "p50 should be close to 50, got {p50}" - ); -} - -// ─── DDSketch path ─────────────────────────────────────────────────────── - -fn make_dd_sketch_agg_config( - id: u64, - metric: &str, - window_secs: u64, - grouping: Vec<&str>, - alpha: f64, -) -> AggregationConfig { - let mut params = HashMap::new(); - params.insert("alpha".to_string(), serde_json::Value::from(alpha)); - AggregationConfig::new( - AggregationType::DDSketch, - String::new(), - params, - asap_types::KeyByLabelNames::new(grouping.iter().map(|s| s.to_string()).collect()), - asap_types::KeyByLabelNames::new(vec![]), - asap_types::KeyByLabelNames::new(vec![]), - String::new(), - window_secs, - 0, - WindowKind::Tumbling, - metric.to_string(), - metric.to_string(), - None, - None, - None, - ) -} - -fn build_dd_sketch_state(alpha: f64, store_counts: Vec, store_offset: i32) -> DdSketchState { - // The DataPoint-level scalars (count/sum/min/max) were dropped from - // `DDSketchState` (ProjectASAP/sketchlib-go#243 / asap_sketchlib#57). - DdSketchState { - alpha, - store_counts, - store_offset, - } -} - -fn build_dd_sketch_export_request( - metric_name: &str, - service_label: &str, - time_unix_nano: u64, - sketch_bytes: Vec, -) -> ExportMetricsServiceRequest { - let dp = DdSketchDataPoint { - attributes: vec![KeyValue { - key: "service".to_string(), - value: Some(AnyValue { - value: Some(any_value::Value::StringValue(service_label.to_string())), - }), - }], - start_time_unix_nano: 0, - time_unix_nano, - sketch: sketch_bytes, - encoding: DdSketchEncoding::DdsketchEncodingProto as i32, - exemplars: Vec::new(), - flags: 0, - series_id: 0, - }; - ExportMetricsServiceRequest { - resource_metrics: vec![ResourceMetrics { - resource: None, - scope_metrics: vec![ScopeMetrics { - scope: None, - metrics: vec![Metric { - name: metric_name.to_string(), - description: String::new(), - unit: String::new(), - metadata: Vec::new(), - data: Some(Data::Ddsketch(DdSketch { - data_points: vec![dp], - aggregation_temporality: 0, - relative_accuracy: 0.01, - })), - }], - schema_url: String::new(), - }], - schema_url: String::new(), - }], - } -} - -#[ignore = "broken since proto refactor; PR compile-only fix"] -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn e2e_dd_sketch_modified_otlp_path() { - let agg_id = 45u64; - let metric_name = "request_duration_seconds"; - let service_label = "payments"; - let window_secs = 1u64; - let alpha = 0.01; - - let otlp_grpc_port = 19531u16; - let otlp_http_port = 19532u16; - - let dd_config = - make_dd_sketch_agg_config(agg_id, metric_name, window_secs, vec!["service"], alpha); - let mut agg_map = HashMap::new(); - agg_map.insert(agg_id, dd_config); - let streaming_config = Arc::new(StreamingConfig::new(agg_map)); - - let sink = Arc::new(CapturingOutputSink::new()); - let engine = PrecomputeEngine::new( - engine_config(), - data_plane::storage_engines::types::HotReloadStreamingConfig::from_arc(streaming_config), - sink.clone(), - Arc::new(data_plane::drivers::ingest::series_resolver::SeriesIdResolver::new()), - Arc::new(data_plane::storage_engines::sketch_db::index::SketchStore::new()), - ); - let ingest_state = engine.ingest_state(); - - tokio::spawn(async move { - let _ = engine.run().await; - }); - - let otlp_receiver = OtlpReceiver::with_ingest_state( - OtlpReceiverConfig { - grpc_port: otlp_grpc_port, - http_port: otlp_http_port, - }, - ingest_state, - ); - tokio::spawn(async move { - let _ = otlp_receiver.run().await; - }); - - tokio::time::sleep(tokio::time::Duration::from_millis(400)).await; - - let store_counts = vec![5u64, 10, 15, 20]; - let dd_state = build_dd_sketch_state(alpha, store_counts.clone(), -1); - let sketch_bytes = dd_state.encode_to_vec(); - - let client = reqwest::Client::new(); - let req = build_dd_sketch_export_request(metric_name, service_label, 100_000_000, sketch_bytes); - post_otlp_http(&client, otlp_http_port, req).await; - - let watermark_state = build_dd_sketch_state(alpha, Vec::new(), 0); - let watermark_req = build_dd_sketch_export_request( - metric_name, - service_label, - 2_000_000_000, - watermark_state.encode_to_vec(), - ); - post_otlp_http(&client, otlp_http_port, watermark_req).await; - - tokio::time::sleep(tokio::time::Duration::from_millis(800)).await; - - let captured = sink.drain(); - assert!(!captured.is_empty(), "expected at least one output"); - - let (window0_output, window0_acc_box) = captured - .iter() - .find(|(out, _)| out.start_timestamp == 0) - .expect("no captured output for window 0"); - - assert_eq!(window0_output.policy_fp.as_u64(), agg_id); - - let dd_acc = window0_acc_box - .as_any() - .downcast_ref::() - .expect("captured accumulator should be DDSketchAccumulator"); - - assert_eq!(dd_acc.inner.store_counts, store_counts); - assert_eq!(dd_acc.inner.store_offset, -1); - // `count` is recovered from the bucket store (5 + 10 + 15 + 20 = 50); - // sum/min/max were dropped from the wire format - // (ProjectASAP/sketchlib-go#243 / asap_sketchlib#57). - assert_eq!(dd_acc.inner.total_count(), 50); - assert!((dd_acc.inner.alpha - alpha).abs() < f64::EPSILON); -} - -// ─── HLL sketch path ───────────────────────────────────────────────────── - -fn make_hll_agg_config( - id: u64, - metric: &str, - window_secs: u64, - grouping: Vec<&str>, - precision: u32, -) -> AggregationConfig { - let mut params = HashMap::new(); - params.insert("precision".to_string(), serde_json::Value::from(precision)); - AggregationConfig::new( - AggregationType::HLL, - String::new(), - params, - asap_types::KeyByLabelNames::new(grouping.iter().map(|s| s.to_string()).collect()), - asap_types::KeyByLabelNames::new(vec![]), - asap_types::KeyByLabelNames::new(vec![]), - String::new(), - window_secs, - 0, - WindowKind::Tumbling, - metric.to_string(), - metric.to_string(), - None, - None, - None, - ) -} - -fn build_hll_state(precision: u32, registers: Vec) -> HyperLogLogState { - HyperLogLogState { - variant: ProtoHllVariant::Regular as i32, - precision, - registers, - hip_kxq0: 0.0, - hip_kxq1: 0.0, - hip_est: 0.0, - registers_sparse: None, - } -} - -fn build_hll_export_request( - metric_name: &str, - service_label: &str, - time_unix_nano: u64, - sketch_bytes: Vec, - precision: u32, -) -> ExportMetricsServiceRequest { - let dp = HllSketchDataPoint { - attributes: vec![KeyValue { - key: "service".to_string(), - value: Some(AnyValue { - value: Some(any_value::Value::StringValue(service_label.to_string())), - }), - }], - start_time_unix_nano: 0, - time_unix_nano, - sketch: sketch_bytes, - encoding: HllSketchEncoding::Proto as i32, - flags: 0, - series_id: 0, - }; - ExportMetricsServiceRequest { - resource_metrics: vec![ResourceMetrics { - resource: None, - scope_metrics: vec![ScopeMetrics { - scope: None, - metrics: vec![Metric { - name: metric_name.to_string(), - description: String::new(), - unit: String::new(), - metadata: Vec::new(), - data: Some(Data::Hllsketch(HllSketch { - data_points: vec![dp], - aggregation_temporality: 0, - precision: 14, - })), - }], - schema_url: String::new(), - }], - schema_url: String::new(), - }], - } -} - -#[ignore = "broken since proto refactor; PR compile-only fix"] -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn e2e_hll_sketch_modified_otlp_path() { - let agg_id = 46u64; - let metric_name = "unique_users"; - let service_label = "login"; - let window_secs = 1u64; - let precision = 4u32; - let num_registers = 1usize << precision; - - let otlp_grpc_port = 19541u16; - let otlp_http_port = 19542u16; - - let hll_config = - make_hll_agg_config(agg_id, metric_name, window_secs, vec!["service"], precision); - let mut agg_map = HashMap::new(); - agg_map.insert(agg_id, hll_config); - let streaming_config = Arc::new(StreamingConfig::new(agg_map)); - - let sink = Arc::new(CapturingOutputSink::new()); - let engine = PrecomputeEngine::new( - engine_config(), - data_plane::storage_engines::types::HotReloadStreamingConfig::from_arc(streaming_config), - sink.clone(), - Arc::new(data_plane::drivers::ingest::series_resolver::SeriesIdResolver::new()), - Arc::new(data_plane::storage_engines::sketch_db::index::SketchStore::new()), - ); - let ingest_state = engine.ingest_state(); - - tokio::spawn(async move { - let _ = engine.run().await; - }); - - let otlp_receiver = OtlpReceiver::with_ingest_state( - OtlpReceiverConfig { - grpc_port: otlp_grpc_port, - http_port: otlp_http_port, - }, - ingest_state, - ); - tokio::spawn(async move { - let _ = otlp_receiver.run().await; - }); - - tokio::time::sleep(tokio::time::Duration::from_millis(400)).await; - - let registers: Vec = (0..num_registers as u8).collect(); - let hll_state = build_hll_state(precision, registers.clone()); - let sketch_bytes = hll_state.encode_to_vec(); - - let client = reqwest::Client::new(); - let req = build_hll_export_request( - metric_name, - service_label, - 100_000_000, - sketch_bytes, - precision, - ); - post_otlp_http(&client, otlp_http_port, req).await; - - let watermark_state = build_hll_state(precision, vec![0u8; num_registers]); - let watermark_req = build_hll_export_request( - metric_name, - service_label, - 2_000_000_000, - watermark_state.encode_to_vec(), - precision, - ); - post_otlp_http(&client, otlp_http_port, watermark_req).await; - - tokio::time::sleep(tokio::time::Duration::from_millis(800)).await; - - let captured = sink.drain(); - assert!(!captured.is_empty(), "expected at least one output"); - - let (window0_output, window0_acc_box) = captured - .iter() - .find(|(out, _)| out.start_timestamp == 0) - .expect("no captured output for window 0"); - - assert_eq!(window0_output.policy_fp.as_u64(), agg_id); - - let hll_acc = window0_acc_box - .as_any() - .downcast_ref::() - .expect("captured accumulator should be HllSketchAccumulator"); - - assert_eq!(hll_acc.inner.precision, precision); - assert_eq!(hll_acc.inner.registers, registers); -} - -// ─── MessagePack encoding path (PR I) ──────────────────────────────────── -// -// Smoke test for `encoding = COUNT_MIN_SKETCH_ENCODING_MSGPACK`. The -// dispatcher should recognize the new `MSGPACK = 3` tag and route to -// `CountMinSketchAccumulator::from_msgpack_bytes`, which deserializes -// the cross-language sketch-core msgpack wire format. The other four -// sketch variants go through the same dispatcher branch, so one smoke -// test is sufficient for dispatcher coverage — per-variant msgpack -// round-trips are validated in the accumulator unit tests. - -fn build_count_min_msgpack_export_request( - metric_name: &str, - service_label: &str, - time_unix_nano: u64, - sketch_bytes: Vec, -) -> ExportMetricsServiceRequest { - let dp = CountMinSketchDataPoint { - attributes: vec![KeyValue { - key: "service".to_string(), - value: Some(AnyValue { - value: Some(any_value::Value::StringValue(service_label.to_string())), - }), - }], - start_time_unix_nano: 0, - time_unix_nano, - sketch: sketch_bytes, - encoding: CountMinSketchEncoding::Msgpack as i32, - flags: 0, - series_id: 0, - }; - ExportMetricsServiceRequest { - resource_metrics: vec![ResourceMetrics { - resource: None, - scope_metrics: vec![ScopeMetrics { - scope: None, - metrics: vec![Metric { - name: metric_name.to_string(), - description: String::new(), - unit: String::new(), - metadata: Vec::new(), - data: Some(Data::Countminsketch(CountMinSketch { - data_points: vec![dp], - aggregation_temporality: 0, - rows: 0, - cols: 0, - })), - }], - schema_url: String::new(), - }], - schema_url: String::new(), - }], - } -} - -#[ignore = "broken since proto refactor; PR compile-only fix"] -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn e2e_count_min_sketch_msgpack_modified_otlp_path() { - let agg_id = 47u64; - let metric_name = "http_requests_msgpack"; - let service_label = "auth"; - let window_secs = 1u64; - let rows = 2u32; - let cols = 4u32; - - let otlp_grpc_port = 19551u16; - let otlp_http_port = 19552u16; - - let cms_config = make_count_min_agg_config( - agg_id, - metric_name, - window_secs, - vec!["service"], - rows as usize, - cols as usize, - ); - let mut agg_map = HashMap::new(); - agg_map.insert(agg_id, cms_config); - let streaming_config = Arc::new(StreamingConfig::new(agg_map)); - - let sink = Arc::new(CapturingOutputSink::new()); - let engine = PrecomputeEngine::new( - engine_config(), - data_plane::storage_engines::types::HotReloadStreamingConfig::from_arc(streaming_config), - sink.clone(), - Arc::new(data_plane::drivers::ingest::series_resolver::SeriesIdResolver::new()), - Arc::new(data_plane::storage_engines::sketch_db::index::SketchStore::new()), - ); - let ingest_state = engine.ingest_state(); - - tokio::spawn(async move { - let _ = engine.run().await; - }); - - let otlp_receiver = OtlpReceiver::with_ingest_state( - OtlpReceiverConfig { - grpc_port: otlp_grpc_port, - http_port: otlp_http_port, - }, - ingest_state, - ); - tokio::spawn(async move { - let _ = otlp_receiver.run().await; - }); - - tokio::time::sleep(tokio::time::Duration::from_millis(400)).await; - - // Build a known sketch in sketch-core and serialize with msgpack — this - // is what the Go producer (sketchlib-go) will emit once PR I's matching - // Go-side work lands. - let mut cms = asap_sketchlib::CountMinSketch::new(rows as usize, cols as usize); - cms.update("user_a", 1.0); - cms.update("user_b", 1.0); - cms.update("user_a", 1.0); - let sketch_bytes = cms.to_msgpack().expect("serialize CMS msgpack"); - - let client = reqwest::Client::new(); - let req = build_count_min_msgpack_export_request( - metric_name, - service_label, - 100_000_000, - sketch_bytes, - ); - post_otlp_http(&client, otlp_http_port, req).await; - - // Watermark advance using an empty msgpack sketch. - let empty = asap_sketchlib::CountMinSketch::new(rows as usize, cols as usize); - let watermark_req = build_count_min_msgpack_export_request( - metric_name, - service_label, - 2_000_000_000, - empty.to_msgpack().expect("serialize empty CMS msgpack"), - ); - post_otlp_http(&client, otlp_http_port, watermark_req).await; - - tokio::time::sleep(tokio::time::Duration::from_millis(800)).await; - - let captured = sink.drain(); - assert!(!captured.is_empty(), "expected at least one output"); - - let (window0_output, window0_acc_box) = captured - .iter() - .find(|(out, _)| out.start_timestamp == 0) - .expect("no captured output for window 0"); - - assert_eq!(window0_output.policy_fp.as_u64(), agg_id); - - let cms_acc = window0_acc_box - .as_any() - .downcast_ref::() - .expect("captured accumulator should be CountMinSketchAccumulator"); - - // user_a was updated twice → estimate("user_a") should be ≥ 2. - assert!( - cms_acc.inner.estimate("user_a") >= 2.0, - "CountMinSketch msgpack round-trip should preserve user_a count (got {})", - cms_acc.inner.estimate("user_a") - ); -} diff --git a/data_plane/tests/edge_runtime_consumes_precompute_rs.rs b/data_plane/tests/edge_runtime_consumes_precompute_rs.rs index 3dee9e81..e4623e6b 100644 --- a/data_plane/tests/edge_runtime_consumes_precompute_rs.rs +++ b/data_plane/tests/edge_runtime_consumes_precompute_rs.rs @@ -13,24 +13,14 @@ //! - **DDSketch**: round-trip + structural assertions are live — DDSketch //! is a deterministic histogram, so `snapshot → reconstruct → snapshot` //! is byte-identical (`asap_sketchlib`#40). -//! - **KLL**: the *structural* round-trip is live, but the **byte-parity** -//! round-trip is gated `#[ignore]`. KLL compaction is randomized and -//! lossy; `reconstruct_via_runtime` rebuilds the sketch by replaying the -//! envelope's retained items through `apply_delta`, which re-compacts -//! them (different RNG, double compaction) — so the re-snapshot is a -//! valid KLL summary but not byte-identical to the original. True byte -//! parity needs an `asap_sketchlib` `KLL::from_wire_state` that consumes -//! the on-wire `levels`/`items`/`coin` directly — tracked in -//! `asap_sketchlib`#41. -//! - **HLL + CountSketch + CountMinSketch**: the byte-parity work -//! for these three sketches is tracked under -//! `ProjectASAP/ASAPCollector#243`. Tests are present and gated -//! `#[ignore = "blocked on ASAPCollector#243 HLL/CS/CMS byte parity"]` -//! so the gap is visible without breaking CI. - -use asap_precompute_rs::sketches::{ - CMSWrapper, CountSketchWrapper, DDSketchWrapper, HLLWrapper, KLLWrapper, -}; +//! - **KLL**: structural envelope compatibility is live. Byte identity is +//! not a supported contract because reconstruction replays retained items +//! through randomized, lossy compaction. +//! - **HLL + CountSketch + CountMinSketch**: the shared runtime adapter does +//! not support these families; their production decoders are tested at the +//! backend accumulator boundary instead. + +use asap_precompute_rs::sketches::{DDSketchWrapper, KLLWrapper}; use asap_precompute_rs::Sketch; use data_plane::precompute_engine::operators::edge_runtime_adapter::{ @@ -181,44 +171,6 @@ fn ddsketch_backend_sketch_snapshots_to_canonical_envelope_bytes() { // --- KLL ---------------------------------------------------------- -/// Round-trip: a KLL envelope produced by asap-precompute-rs's -/// `KLLWrapper` is reconstructed by the backend's runtime adapter, -/// which returns a re-snapshot from the runtime. -/// -/// **Byte parity is gated `#[ignore]`** — see below. `reconstruct_via_runtime` -/// rebuilds the KLL by replaying the envelope's retained `items` through -/// `apply_delta` → `update()`. For an input large enough to compact -/// (here, 400 items at k=200 → 2 levels), that re-feed re-compacts -/// lossily and with a different RNG seed than the original, so the -/// re-snapshot — though a valid KLL summary — is not byte-identical. -/// True byte parity needs an `asap_sketchlib` `KLL::from_wire_state` -/// that restores the exact compactor state (`levels`/`items`/`coin`) -/// instead of replaying items; tracked in `asap_sketchlib`#41. The -/// *structural* round-trip is covered live by the test below. -#[test] -#[ignore = "KLL byte parity needs asap_sketchlib KLL::from_wire_state (asap_sketchlib#41); \ - reconstruct_via_runtime replays items, which re-compacts lossily"] -fn kll_envelope_round_trip_through_backend_adapter() { - let mut w = KLLWrapper::new(200, Some(42)); - for i in 1..=400 { - w.update(i as f64); - } - let original = w.snapshot().expect("KLL snapshot"); - assert!(!original.is_empty()); - - let reconstructed = reconstruct_via_runtime(SketchType::KLLSketch, &original) - .expect("runtime adapter reconstruction"); - let snapshot_bytes = match reconstructed { - ReconstructedSketch::Kll { snapshot_bytes } => snapshot_bytes, - _ => panic!("expected KLL reconstruction"), - }; - assert!(!snapshot_bytes.is_empty(), "non-empty re-snapshot"); - assert_eq!( - snapshot_bytes, original, - "KLL envelope round-trip via asap-precompute-rs runtime is byte-identical" - ); -} - /// Structural: KLL envelope unwraps to the expected oneof variant via /// the shared `unwrap_envelope_state` helper. #[test] @@ -241,42 +193,3 @@ fn kll_envelope_structural_assertions() { other => panic!("expected KLL state, got {other:?}"), } } - -// --- Sketches gated on ASAPCollector#243 -------------------------- - -#[test] -#[ignore = "blocked on ASAPCollector#243 HLL/CS/CMS byte parity"] -fn hll_envelope_round_trip_through_backend_adapter() { - use asap_sketchlib::HllVariant; - let mut w = HLLWrapper::new(HllVariant::Regular, 12); - for i in 0..1000u32 { - w.update(&i.to_le_bytes()); - } - let bytes = w.snapshot().expect("HLL snapshot"); - let _ = reconstruct_via_runtime(SketchType::HLLSketch, &bytes) - .expect("HLL reconstruction (gated until #243)"); -} - -#[test] -#[ignore = "blocked on ASAPCollector#243 HLL/CS/CMS byte parity"] -fn countsketch_envelope_round_trip_through_backend_adapter() { - let mut w = CountSketchWrapper::new(4, 1024); - for i in 0..100u32 { - w.update(&format!("k{}", i % 10), 1.0); - } - let bytes = w.snapshot().expect("CountSketch snapshot"); - let _ = reconstruct_via_runtime(SketchType::CountSketch, &bytes) - .expect("CountSketch reconstruction (gated until #243)"); -} - -#[test] -#[ignore = "blocked on ASAPCollector#243 HLL/CS/CMS byte parity"] -fn cms_envelope_round_trip_through_backend_adapter() { - let mut w = CMSWrapper::new(4, 1024); - for i in 0..100u32 { - w.update(&format!("k{}", i % 10), 1.0); - } - let bytes = w.snapshot().expect("CMS snapshot"); - let _ = reconstruct_via_runtime(SketchType::CountMinSketch, &bytes) - .expect("CMS reconstruction (gated until #243)"); -} diff --git a/docs/03-how-to-guides/manual-e2e-tests.md b/docs/03-how-to-guides/manual-e2e-tests.md index 94bafb4f..0d3ee778 100644 --- a/docs/03-how-to-guides/manual-e2e-tests.md +++ b/docs/03-how-to-guides/manual-e2e-tests.md @@ -71,15 +71,19 @@ ASAP_COLLECTOR_DIR=../ASAPCollector ./scripts/e2e.sh system The local `all` target never performs those external operations. -## Ignored regressions +## Test hygiene -Tests marked `#[ignore]` are not counted as passing coverage. List the current -known ignored tests and their reasons with: +Tests marked `#[ignore]` are not counted as passing coverage. The repository +does not retain ignored tests for retired or unsupported contracts; those +belong in issue tracking. List suites and verify that no Rust E2E is hidden +behind `#[ignore]` with: ```bash ./scripts/e2e.sh list ``` -In particular, ignored legacy cases remain visible but do not substitute for -the maintained production-process tests. The final local whole-backend test is -`data_plane/tests/backend_process_e2e.rs`. +The final local whole-backend test is +`data_plane/tests/backend_process_e2e.rs`. The environment-gated VictoriaMetrics +comparison in `gorilla-merger/internal/merger/vmload_test.go` remains an +explicit external integration test and reports a Go skip when `VM_ADDR` is not +provided. diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 38348ce2..d0953667 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -31,7 +31,7 @@ Targets: whole Controller plan -> backend install -> OTLP -> store -> PromQL whole-matrix Run every whole-path sketch/query scenario (diagnostic) system Delegate to ASAPCollector's real multi-node system harness - list Print the suites and known ignored E2E tests + list Print the suites and audit Rust E2E ignore markers Useful environment variables: ASAP_COLLECTOR_DIR Sibling ASAPCollector checkout (system target) @@ -143,7 +143,7 @@ whole_matrix() { list_suites() { usage - printf '\nKnown intentionally ignored E2E tests (not counted as passes):\n' + printf '\nRust E2E tests marked #[ignore] (expected: none):\n' rg -n '^[[:space:]]*#\[ignore' \ "${REPO_DIR}/data_plane/tests" \ "${REPO_DIR}/data_plane/src/tests" \ From ea20b3109f377489711a4174fa05eec59130bc1f Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 12:20:02 -0600 Subject: [PATCH 6/8] test: add PromQL differential process coverage --- data_plane/tests/backend_process_e2e.rs | 39 +- .../tests/promql_differential_process_e2e.rs | 332 ++++++++++++++++++ docs/03-how-to-guides/manual-e2e-tests.md | 32 ++ scripts/e2e.sh | 19 +- 4 files changed, 413 insertions(+), 9 deletions(-) create mode 100644 data_plane/tests/promql_differential_process_e2e.rs diff --git a/data_plane/tests/backend_process_e2e.rs b/data_plane/tests/backend_process_e2e.rs index ddf5493f..23c98eb3 100644 --- a/data_plane/tests/backend_process_e2e.rs +++ b/data_plane/tests/backend_process_e2e.rs @@ -67,7 +67,11 @@ async fn wait_http(client: &reqwest::Client, url: &str, child: &mut Child, name: panic!("{name} did not become ready at {url}"); } -fn ddsketch_export(metric: &str, timestamp_ns: u64, counts: Vec, alpha: f64) -> Vec { +fn ddsketch_export(metric: &str, timestamp_ns: u64, values: &[f64], alpha: f64) -> Vec { + let mut sketch = asap_sketchlib::DdSketch::new(alpha); + for value in values { + sketch.update(*value); + } let point = DdSketchDataPoint { attributes: vec![KeyValue { key: "service".into(), @@ -78,9 +82,9 @@ fn ddsketch_export(metric: &str, timestamp_ns: u64, counts: Vec, alpha: f64 start_time_unix_nano: timestamp_ns.saturating_sub(1_000_000_000), time_unix_nano: timestamp_ns, sketch: DdSketchState { - alpha, - store_counts: counts, - store_offset: -1, + alpha: sketch.wire_alpha(), + store_counts: sketch.store_counts, + store_offset: sketch.store_offset, } .encode_to_vec(), encoding: DdSketchEncoding::DdsketchEncodingProto as i32, @@ -112,6 +116,16 @@ fn ddsketch_export(metric: &str, timestamp_ns: u64, counts: Vec, alpha: f64 .encode_to_vec() } +fn exact_quantile(values: &[f64], quantile: f64) -> f64 { + let mut sorted = values.to_vec(); + sorted.sort_by(f64::total_cmp); + let rank = quantile * (sorted.len() - 1) as f64; + let lower = rank.floor() as usize; + let upper = rank.ceil() as usize; + let fraction = rank - lower as f64; + sorted[lower] + (sorted[upper] - sorted[lower]) * fraction +} + fn first_scalar(response: &serde_json::Value) -> Option { response["data"]["result"] .as_array()? @@ -390,13 +404,15 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { .duration_since(std::time::UNIX_EPOCH) .expect("system clock"); let sample_ns = now.as_nanos() as u64; + let raw_values = (1..=100).map(|value| value as f64).collect::>(); + let reference_p99 = exact_quantile(&raw_values, 0.99); client .post(format!("http://{otlp_http}/v1/metrics")) .header("content-type", "application/x-protobuf") .body(ddsketch_export( "whole_process_e2e_latency_ms", sample_ns, - vec![5, 10, 15, 20], + &raw_values, planned_alpha, )) .send() @@ -421,7 +437,7 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { .body(ddsketch_export( "whole_process_e2e_latency_ms", watermark_ns, - Vec::new(), + &[], planned_alpha, )) .send() @@ -443,9 +459,16 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { .await .expect("decode PromQL response"); if let Some(value) = first_scalar(&response) { + let relative_error = (value - reference_p99).abs() / reference_p99; assert!( - value.is_finite() && value > 0.0, - "invalid quantile: {value}" + value.is_finite() && relative_error <= planned_alpha * 1.05, + "backend p99 {value} differs from raw-value oracle {reference_p99}; \ + relative_error={relative_error}, allowed={}", + planned_alpha * 1.05 + ); + assert_eq!( + response["data"]["result"][0]["metric"]["service"], + "whole-e2e" ); return; } diff --git a/data_plane/tests/promql_differential_process_e2e.rs b/data_plane/tests/promql_differential_process_e2e.rs new file mode 100644 index 00000000..1dd7d282 --- /dev/null +++ b/data_plane/tests/promql_differential_process_e2e.rs @@ -0,0 +1,332 @@ +//! Black-box PromQL differential E2E for the production data-plane process. +//! +//! A deterministic raw-value fixture is summarized into the same modified +//! OTLP DDSketch shape emitted by the collector runtime. The production +//! backend ingests that sketch, and its public instant/range PromQL responses +//! are compared with an independent exact quantile oracle over the raw values. + +use std::io::Write; +use std::net::TcpListener; +use std::process::{Child, Command, Stdio}; +use std::time::Duration; + +use asap_otel_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; +use asap_otel_proto::tonic::common::v1::{any_value, AnyValue, KeyValue}; +use asap_otel_proto::tonic::metrics::v1::{ + metric::Data, DdSketch, DdSketchDataPoint, DdSketchEncoding, Metric, ResourceMetrics, + ScopeMetrics, +}; +use asap_sketchlib::proto::sketchlib::DdSketchState; +use prost::Message; +use serde_json::Value; + +const METRIC: &str = "differential_e2e_latency_ms"; +const SERVICE: &str = "checkout"; +const ALPHA: f64 = 0.01; +// Median avoids conflating DDSketch's discrete rank selection with +// Prometheus's interpolation between adjacent values. Small-sample p99 +// interpolation is covered by the broader diagnostic scenario matrix. +const QUANTILE: f64 = 0.5; + +struct ChildGuard(Child); + +impl Drop for ChildGuard { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +fn unused_port() -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").expect("reserve loopback port"); + listener.local_addr().expect("read loopback address").port() +} + +fn exact_quantile(values: &[f64], quantile: f64) -> f64 { + assert!(!values.is_empty()); + assert!((0.0..=1.0).contains(&quantile)); + let mut sorted = values.to_vec(); + sorted.sort_by(f64::total_cmp); + let rank = quantile * (sorted.len() - 1) as f64; + let lower = rank.floor() as usize; + let upper = rank.ceil() as usize; + if lower == upper { + sorted[lower] + } else { + let fraction = rank - lower as f64; + sorted[lower] + (sorted[upper] - sorted[lower]) * fraction + } +} + +fn ddsketch_export(metric: &str, timestamp_ns: u64, values: &[f64]) -> Vec { + let mut sketch = asap_sketchlib::DdSketch::new(ALPHA); + for value in values { + sketch.update(*value); + } + let state = DdSketchState { + alpha: sketch.wire_alpha(), + store_counts: sketch.store_counts, + store_offset: sketch.store_offset, + }; + let point = DdSketchDataPoint { + attributes: vec![KeyValue { + key: "service".into(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue(SERVICE.into())), + }), + }], + start_time_unix_nano: timestamp_ns.saturating_sub(1_000_000_000), + time_unix_nano: timestamp_ns, + sketch: state.encode_to_vec(), + encoding: DdSketchEncoding::DdsketchEncodingProto as i32, + exemplars: Vec::new(), + flags: 0, + series_id: 0, + }; + ExportMetricsServiceRequest { + resource_metrics: vec![ResourceMetrics { + resource: None, + scope_metrics: vec![ScopeMetrics { + scope: None, + metrics: vec![Metric { + name: metric.into(), + description: String::new(), + unit: String::new(), + metadata: Vec::new(), + data: Some(Data::Ddsketch(DdSketch { + data_points: vec![point], + aggregation_temporality: 0, + relative_accuracy: ALPHA, + })), + }], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + } + .encode_to_vec() +} + +fn first_instant(response: &Value) -> Option<(&Value, f64, f64)> { + let series = response["data"]["result"].as_array()?.first()?; + let sample = series["value"].as_array()?; + Some(( + &series["metric"], + sample.first()?.as_f64()?, + sample.get(1)?.as_str()?.parse().ok()?, + )) +} + +fn first_range_values(response: &Value) -> Option<(&Value, Vec<(f64, f64)>)> { + let series = response["data"]["result"].as_array()?.first()?; + let values = series["values"] + .as_array()? + .iter() + .map(|sample| { + let pair = sample.as_array()?; + Some(( + pair.first()?.as_f64()?, + pair.get(1)?.as_str()?.parse().ok()?, + )) + }) + .collect::>>()?; + Some((&series["metric"], values)) +} + +fn assert_approx(reference: f64, actual: f64, context: &str) { + let relative_error = (actual - reference).abs() / reference.abs().max(f64::EPSILON); + assert!( + relative_error <= ALPHA * 1.05, + "{context}: reference={reference}, actual={actual}, relative_error={relative_error}, \ + allowed={}", + ALPHA * 1.05 + ); +} + +async fn wait_until_ready(client: &reqwest::Client, url: &str, child: &mut Child) { + for _ in 0..100 { + if let Some(status) = child.try_wait().expect("inspect data-plane process") { + panic!("data-plane exited before readiness: {status}"); + } + if client + .get(url) + .send() + .await + .is_ok_and(|response| response.status().is_success()) + { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("data-plane did not become ready at {url}"); +} + +async fn get_json(client: &reqwest::Client, url: &str, params: &[(&str, String)]) -> Value { + client + .get(url) + .query(params) + .send() + .await + .expect("send PromQL request") + .json() + .await + .expect("decode PromQL response") +} + +#[tokio::test] +async fn production_backend_matches_raw_oracle_for_instant_and_range_promql() { + let query_port = unused_port(); + let otlp_http_port = unused_port(); + let otlp_grpc_port = unused_port(); + let output_dir = tempfile::tempdir().expect("create log directory"); + let mut config = tempfile::NamedTempFile::new().expect("create streaming config"); + write!( + config, + r#"aggregations: + - aggregationType: DDSketch + aggregationSubType: '' + labels: + grouping: [service] + rollup: [] + aggregated: [] + metric: differential_e2e_latency_ms + parameters: + relativeAccuracy: 0.01 + windowSize: 1 + windowType: tumbling + spatialFilter: '' +"# + ) + .expect("write streaming config"); + + let child = Command::new(env!("CARGO_BIN_EXE_data_plane")) + .arg("--streaming-config") + .arg(config.path()) + .arg("--http-port") + .arg(query_port.to_string()) + .arg("--output-dir") + .arg(output_dir.path()) + .arg("--enable-otel-ingest") + .arg("--otel-http-port") + .arg(otlp_http_port.to_string()) + .arg("--otel-grpc-port") + .arg(otlp_grpc_port.to_string()) + .arg("--precompute-allowed-lateness-ms") + .arg("0") + .arg("--precompute-flush-interval-ms") + .arg("100") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("start production data-plane binary"); + let mut child = ChildGuard(child); + + let client = reqwest::Client::new(); + let query_base = format!("http://127.0.0.1:{query_port}"); + wait_until_ready( + &client, + &format!("{query_base}/api/v1/health"), + &mut child.0, + ) + .await; + + let raw_values = [10.0, 12.0, 15.0, 20.0, 30.0, 45.0, 60.0, 80.0, 100.0]; + let reference = exact_quantile(&raw_values, QUANTILE); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock"); + let sample_ns = now.as_nanos() as u64; + for body in [ + ddsketch_export(METRIC, sample_ns, &raw_values), + ddsketch_export(METRIC, sample_ns + 2_000_000_000, &[]), + ] { + client + .post(format!("http://127.0.0.1:{otlp_http_port}/v1/metrics")) + .header("content-type", "application/x-protobuf") + .body(body) + .send() + .await + .expect("POST modified OTLP") + .error_for_status() + .expect("backend accepted modified OTLP"); + } + + let query = format!("quantile_over_time({QUANTILE}, {METRIC}[10s])"); + let instant_url = format!("{query_base}/api/v1/query"); + let mut instant = Value::Null; + for _ in 0..50 { + instant = get_json(&client, &instant_url, &[("query", query.clone())]).await; + if first_instant(&instant).is_some() { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + assert_eq!(instant["status"], "success", "instant response: {instant}"); + assert_eq!(instant["data"]["resultType"], "vector"); + let (instant_labels, instant_timestamp, instant_value) = + first_instant(&instant).unwrap_or_else(|| panic!("empty instant response: {instant}")); + assert_eq!(instant_labels["service"], SERVICE); + assert_approx(reference, instant_value, "instant query versus raw oracle"); + assert!( + instant["infos"] + .as_array() + .is_some_and(|infos| infos.iter().any(|info| info + .as_str() + .is_some_and(|s| s.contains("data_source: asap_query")))), + "query was not proven to come from ASAPQuery: {instant}" + ); + + let range_url = format!("{query_base}/api/v1/query_range"); + let range = get_json( + &client, + &range_url, + &[ + ("query", query.clone()), + ("start", (instant_timestamp - 10.0).to_string()), + ("end", instant_timestamp.to_string()), + ("step", "1".into()), + ], + ) + .await; + assert_eq!(range["status"], "success", "range response: {range}"); + assert_eq!(range["data"]["resultType"], "matrix"); + let (range_labels, range_values) = + first_range_values(&range).unwrap_or_else(|| panic!("empty range response: {range}")); + assert_eq!(range_labels, instant_labels); + assert!( + !range_values.is_empty(), + "range response had no values: {range}" + ); + for (_, value) in &range_values { + assert_approx(reference, *value, "range query versus raw oracle"); + } + assert_approx( + instant_value, + range_values.last().expect("last range value").1, + "instant/range parity", + ); + + for (start, end, step, expected_error) in [ + ("10", "10", "1", "start must be before end"), + ("10", "11", "0", "step must be positive"), + ] { + let invalid = get_json( + &client, + &range_url, + &[ + ("query", query.clone()), + ("start", start.into()), + ("end", end.into()), + ("step", step.into()), + ], + ) + .await; + assert_eq!(invalid["status"], "error", "invalid range: {invalid}"); + assert!( + invalid["error"] + .as_str() + .is_some_and(|error| error.contains(expected_error)), + "unexpected validation response: {invalid}" + ); + } +} diff --git a/docs/03-how-to-guides/manual-e2e-tests.md b/docs/03-how-to-guides/manual-e2e-tests.md index 0d3ee778..8ce77cc4 100644 --- a/docs/03-how-to-guides/manual-e2e-tests.md +++ b/docs/03-how-to-guides/manual-e2e-tests.md @@ -35,11 +35,43 @@ The component suites can also be run separately: ./scripts/e2e.sh contracts ./scripts/e2e.sh control-plane ./scripts/e2e.sh data-plane +./scripts/e2e.sh differential ./scripts/e2e.sh monitor ./scripts/e2e.sh gorilla-merger ./scripts/e2e.sh whole ``` +`differential` starts the production data-plane process, derives a modified +OTLP DDSketch from a deterministic raw-value fixture, and compares public +instant and range PromQL results with an independent exact quantile oracle. +It also verifies labels, ASAP-local execution, instant/range parity, and range +parameter validation. It does not require Docker or a Prometheus process. + +The complete sketch/query differential inventory is exercised with: + +```bash +./scripts/e2e.sh differential-all +``` + +It first runs the stable production-process raw-oracle comparison and then the +whole-path scenario matrix. The matrix covers DDSketch and KLL quantiles, HLL +cardinality, CountSketch and Count-Min count queries, heap-backed top-k, an +instant/range query pair, grouping, delta/sub-window ingest, and shadow/live +serving. It exits non-zero for every real product regression; no scenario is +ignored or converted into an expected pass. + +| Sketch / path | Public query shape | Oracle / invariant | +| --- | --- | --- | +| DDSketch | `quantile_over_time(0.5, ...[10s])`, instant + range | exact raw-value median and instant/range parity | +| DDSketch | `quantile_over_time(0.99, ...[30s])` | exact raw-value p99 within planned alpha | +| DDSketch delta | `quantile_over_time(0.99, ...[3m])` | reconstructed multi-window distribution | +| KLL | `quantile_over_time(0.5, ...[10s])` | non-empty approximate quantile | +| HLL | `count(metric)` | cardinality result, including multi-SID merge | +| CountSketch | `count_over_time(...[10s])` | frequency result | +| Count-Min Sketch | instant and range `count_over_time` | frequency result and matrix wire shape | +| Heap-backed CountSketch / CMS | `topk(3, metric)` | bounded result count, item labels, and deterministic leader | +| PromQL range validation | equal start/end and zero step | Prometheus-compatible explicit errors | + `whole` is the stable representative DDSketch path. To exercise every currently checked-in sketch/query combination, including scenarios tracking known product regressions, run: diff --git a/scripts/e2e.sh b/scripts/e2e.sh index d0953667..73793b5c 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -26,10 +26,12 @@ Targets: contracts Shared Rust type/protobuf wire contracts control-plane Planner HTTP, OpAMP, publication, and runtime feedback data-plane Query, routing, storage, ingest adapter, and lifecycle tests + differential Production backend PromQL vs deterministic raw-value oracle monitor Real monitor gRPC transport tests gorilla-merger Gorilla HTTP/WAL/block/StoreAPI/compaction/shipper tests whole Controller plan -> backend install -> OTLP -> store -> PromQL - whole-matrix Run every whole-path sketch/query scenario (diagnostic) + whole-matrix All sketch families and query shapes (diagnostic) + differential-all Raw oracle test plus the all-sketch/query matrix system Delegate to ASAPCollector's real multi-node system harness list Print the suites and audit Rust E2E ignore markers @@ -100,6 +102,14 @@ data_plane() { CURRENT_STAGE="data-plane/production-process" say "data-plane: production binary -> modified OTLP -> SketchStore -> PromQL" rust_test data_plane --test component_process_e2e + + differential +} + +differential() { + CURRENT_STAGE="data-plane/promql-differential" + say "data-plane: production backend PromQL -> raw-value oracle + instant/range parity" + rust_test data_plane --test promql_differential_process_e2e } monitor() { @@ -141,6 +151,11 @@ whole_matrix() { rust_test data_plane --test e2e_controller_plans_and_backend_serves } +differential_all() { + differential + whole_matrix +} + list_suites() { usage printf '\nRust E2E tests marked #[ignore] (expected: none):\n' @@ -178,10 +193,12 @@ main() { contracts) need cargo; contracts ;; control-plane) need cargo; control_plane ;; data-plane) need cargo; data_plane ;; + differential) need cargo; differential ;; monitor) need cargo; monitor ;; gorilla-merger) gorilla_merger ;; whole) need cargo; whole ;; whole-matrix) need cargo; whole_matrix ;; + differential-all) need cargo; differential_all ;; system) system_e2e ;; list) list_suites; exit 0 ;; -h|--help|help) usage; exit 0 ;; From 2c7a5ed1aacaa60abfe8abf483382c951fa11941 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 13:24:32 -0600 Subject: [PATCH 7/8] test: add production oracles for every sketch --- .../tests/all_sketches_process_oracle_e2e.rs | 477 ++++++++++++++++++ .../tests/promql_differential_process_e2e.rs | 18 +- docs/03-how-to-guides/manual-e2e-tests.md | 40 +- scripts/e2e.sh | 28 +- 4 files changed, 536 insertions(+), 27 deletions(-) create mode 100644 data_plane/tests/all_sketches_process_oracle_e2e.rs diff --git a/data_plane/tests/all_sketches_process_oracle_e2e.rs b/data_plane/tests/all_sketches_process_oracle_e2e.rs new file mode 100644 index 00000000..ce054a47 --- /dev/null +++ b/data_plane/tests/all_sketches_process_oracle_e2e.rs @@ -0,0 +1,477 @@ +//! Production-process E2E oracle matrix for every supported sketch family. +//! +//! Each test starts the real `data_plane` binary, installs a streaming policy, +//! posts modified OTLP over HTTP, and queries the public Prometheus endpoint. +//! Sketch implementations are used only to encode raw fixtures. Expected +//! answers are independently computed from those raw fixtures. + +use std::collections::{HashMap, HashSet}; +use std::io::Write; +use std::net::TcpListener; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use asap_otel_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; +use asap_otel_proto::tonic::common::v1::{any_value, AnyValue, KeyValue}; +use asap_otel_proto::tonic::metrics::v1::{ + metric::Data, CountMinSketch as OtelCountMinSketch, CountMinSketchDataPoint, + CountMinSketchEncoding, CountSketch as OtelCountSketch, CountSketchDataPoint, + CountSketchEncoding, HllSketch as OtelHllSketch, HllSketchDataPoint, HllSketchEncoding, + KllSketch as OtelKllSketch, KllSketchDataPoint, KllSketchEncoding, Metric, ResourceMetrics, + ScopeMetrics, +}; +use asap_sketchlib::proto::sketchlib::{HllVariant as ProtoHllVariant, HyperLogLogState, KllState}; +use asap_sketchlib::{ + CountMinSketchWithHeap, CountSketchWithHeap, HllSketch, HllVariant, MessagePackCodec, +}; +use prost::Message; +use serde_json::Value; + +const SERVICE: &str = "oracle-e2e"; +const K: u32 = 200; +const HLL_PRECISION: u32 = 10; +const ROWS: usize = 5; +const COLS: usize = 2048; +const HEAP_SIZE: usize = 16; + +struct ChildGuard(Child); + +impl Drop for ChildGuard { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +struct Backend { + _child: ChildGuard, + client: reqwest::Client, + query_base: String, + otlp_url: String, + _config: tempfile::NamedTempFile, + _output_dir: tempfile::TempDir, +} + +fn unused_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .expect("reserve loopback port") + .local_addr() + .expect("read loopback address") + .port() +} + +fn now_ns() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock before UNIX epoch") + .as_nanos() as u64 +} + +fn labels() -> Vec { + vec![KeyValue { + key: "service".into(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue(SERVICE.into())), + }), + }] +} + +fn envelope(metric: &str, data: Data) -> ExportMetricsServiceRequest { + ExportMetricsServiceRequest { + resource_metrics: vec![ResourceMetrics { + resource: None, + scope_metrics: vec![ScopeMetrics { + scope: None, + metrics: vec![Metric { + name: metric.into(), + description: String::new(), + unit: String::new(), + metadata: Vec::new(), + data: Some(data), + }], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + } +} + +async fn start_backend(config_yaml: &str) -> Backend { + let query_port = unused_port(); + let otlp_http_port = unused_port(); + let otlp_grpc_port = unused_port(); + let output_dir = tempfile::tempdir().expect("create data-plane output directory"); + let mut config = tempfile::NamedTempFile::new().expect("create streaming config"); + config + .write_all(config_yaml.as_bytes()) + .expect("write streaming config"); + config.flush().expect("flush streaming config"); + + let child = Command::new(env!("CARGO_BIN_EXE_data_plane")) + .arg("--streaming-config") + .arg(config.path()) + .arg("--http-port") + .arg(query_port.to_string()) + .arg("--output-dir") + .arg(output_dir.path()) + .arg("--enable-otel-ingest") + .arg("--otel-http-port") + .arg(otlp_http_port.to_string()) + .arg("--otel-grpc-port") + .arg(otlp_grpc_port.to_string()) + .arg("--precompute-allowed-lateness-ms") + .arg("0") + .arg("--precompute-flush-interval-ms") + .arg("100") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("start production data-plane binary"); + let mut child = ChildGuard(child); + let client = reqwest::Client::new(); + let query_base = format!("http://127.0.0.1:{query_port}"); + let health = format!("{query_base}/api/v1/health"); + for _ in 0..100 { + if let Some(status) = child.0.try_wait().expect("inspect data-plane process") { + panic!("data-plane exited before readiness: {status}"); + } + if client + .get(&health) + .send() + .await + .is_ok_and(|response| response.status().is_success()) + { + return Backend { + _child: child, + client, + query_base, + otlp_url: format!("http://127.0.0.1:{otlp_http_port}/v1/metrics"), + _config: config, + _output_dir: output_dir, + }; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("data-plane did not become ready at {health}"); +} + +async fn post(backend: &Backend, request: ExportMetricsServiceRequest) { + backend + .client + .post(&backend.otlp_url) + .header("content-type", "application/x-protobuf") + .body(request.encode_to_vec()) + .send() + .await + .expect("post modified OTLP") + .error_for_status() + .expect("production backend accepted modified OTLP"); +} + +async fn query(backend: &Backend, promql: &str) -> Value { + let mut response = Value::Null; + for _ in 0..50 { + response = backend + .client + .get(format!("{}/api/v1/query", backend.query_base)) + .query(&[("query", promql)]) + .send() + .await + .expect("query production backend") + .json() + .await + .expect("decode Prometheus JSON"); + if response["data"]["result"] + .as_array() + .is_some_and(|result| !result.is_empty()) + { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + assert!( + response["infos"] + .as_array() + .is_some_and(|infos| infos.iter().any(|info| info + .as_str() + .is_some_and(|text| text.contains("data_source: asap_query")))), + "query was not proven to execute in ASAPQuery: {response}" + ); + assert_eq!(response["status"], "success", "PromQL response: {response}"); + response +} + +fn scalar_values(response: &Value) -> Vec<(HashMap, f64)> { + response["data"]["result"] + .as_array() + .expect("Prometheus result array") + .iter() + .map(|series| { + let labels = series["metric"] + .as_object() + .expect("metric labels") + .iter() + .map(|(key, value)| { + ( + key.clone(), + value.as_str().expect("string label").to_string(), + ) + }) + .collect(); + let value = series["value"][1] + .as_str() + .expect("string sample value") + .parse() + .expect("numeric sample value"); + (labels, value) + }) + .collect() +} + +fn config(metric: &str, kind: &str, parameters: &str) -> String { + format!( + "aggregations:\n - aggregationType: {kind}\n aggregationSubType: ''\n labels:\n grouping: [service]\n rollup: []\n aggregated: []\n metric: {metric}\n parameters:\n{parameters}\n windowSize: 1\n windowType: tumbling\n spatialFilter: ''\n" + ) +} + +fn kll_export(metric: &str, timestamp_ns: u64, raw: &[f64]) -> ExportMetricsServiceRequest { + let state = KllState { + k: K, + m: 8, + num_levels: 0, + levels: Vec::new(), + items: raw.to_vec(), + coin: None, + offset: 0.0, + value_scale: 0, + residuals: Vec::new(), + }; + envelope( + metric, + Data::Kllsketch(OtelKllSketch { + data_points: vec![KllSketchDataPoint { + attributes: labels(), + start_time_unix_nano: timestamp_ns.saturating_sub(1_000_000_000), + time_unix_nano: timestamp_ns, + sketch: state.encode_to_vec(), + encoding: KllSketchEncoding::Proto as i32, + flags: 0, + series_id: 0, + }], + aggregation_temporality: 0, + k: K, + }), + ) +} + +fn hll_export(metric: &str, timestamp_ns: u64, raw: &[&str]) -> ExportMetricsServiceRequest { + let mut sketch = HllSketch::new(HllVariant::Regular, HLL_PRECISION); + for value in raw { + sketch.update(value.as_bytes()); + } + let state = HyperLogLogState { + variant: ProtoHllVariant::Regular as i32, + precision: HLL_PRECISION, + registers: sketch.registers, + hip_kxq0: 0.0, + hip_kxq1: 0.0, + hip_est: 0.0, + registers_sparse: None, + }; + envelope( + metric, + Data::Hllsketch(OtelHllSketch { + data_points: vec![HllSketchDataPoint { + attributes: labels(), + start_time_unix_nano: timestamp_ns.saturating_sub(1_000_000_000), + time_unix_nano: timestamp_ns, + sketch: state.encode_to_vec(), + encoding: HllSketchEncoding::Proto as i32, + flags: 0, + series_id: 0, + }], + aggregation_temporality: 0, + precision: HLL_PRECISION, + }), + ) +} + +fn cms_export(metric: &str, timestamp_ns: u64, raw: &[&str]) -> ExportMetricsServiceRequest { + let mut sketch = CountMinSketchWithHeap::new(ROWS, COLS, HEAP_SIZE); + for key in raw { + sketch.update(key, 1.0); + } + envelope( + metric, + Data::Countminsketch(OtelCountMinSketch { + data_points: vec![CountMinSketchDataPoint { + attributes: labels(), + start_time_unix_nano: timestamp_ns.saturating_sub(1_000_000_000), + time_unix_nano: timestamp_ns, + sketch: sketch.to_msgpack().expect("encode CMS-with-heap"), + encoding: CountMinSketchEncoding::Msgpack as i32, + flags: 0, + series_id: 0, + }], + aggregation_temporality: 0, + rows: ROWS as i32, + cols: COLS as i32, + }), + ) +} + +fn count_sketch_export( + metric: &str, + timestamp_ns: u64, + raw: &[&str], +) -> ExportMetricsServiceRequest { + let mut sketch = CountSketchWithHeap::new(ROWS, COLS, HEAP_SIZE); + for key in raw { + sketch.update(key, 1.0); + } + envelope( + metric, + Data::Countsketch(OtelCountSketch { + data_points: vec![CountSketchDataPoint { + attributes: labels(), + start_time_unix_nano: timestamp_ns.saturating_sub(1_000_000_000), + time_unix_nano: timestamp_ns, + sketch: sketch.to_msgpack().expect("encode CountSketch-with-heap"), + encoding: CountSketchEncoding::Msgpack as i32, + flags: 0, + series_id: 0, + }], + aggregation_temporality: 0, + rows: ROWS as i32, + cols: COLS as i32, + }), + ) +} + +fn exact_quantile(raw: &[f64], q: f64) -> f64 { + let mut sorted = raw.to_vec(); + sorted.sort_by(f64::total_cmp); + let rank = q * (sorted.len() - 1) as f64; + let low = rank.floor() as usize; + let high = rank.ceil() as usize; + sorted[low] + (sorted[high] - sorted[low]) * (rank - low as f64) +} + +fn raw_frequencies(raw: &[&str]) -> HashMap { + let mut counts = HashMap::new(); + for key in raw { + *counts.entry((*key).to_string()).or_insert(0) += 1; + } + counts +} + +fn assert_topk_oracle(response: &Value, raw: &[&str], k: usize) { + let mut expected: Vec<_> = raw_frequencies(raw).into_iter().collect(); + expected.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + expected.truncate(k); + let expected: HashMap<_, _> = expected.into_iter().collect(); + + let actual: HashMap = scalar_values(response) + .into_iter() + .map(|(labels, value)| { + assert_eq!(labels.get("service").map(String::as_str), Some(SERVICE)); + ( + labels.get("item").expect("topk item label").clone(), + value.round() as u64, + ) + }) + .collect(); + assert_eq!(actual, expected, "top-k differs from raw frequency oracle"); +} + +#[tokio::test] +async fn production_kll_matches_raw_quantile_oracle() { + let metric = "oracle_kll_latency"; + let backend = start_backend(&config(metric, "DatasketchesKLL", " K: 200")).await; + let raw: Vec = (1..=101).map(f64::from).collect(); + let timestamp = now_ns().saturating_sub(2_000_000_000); + post(&backend, kll_export(metric, timestamp, &raw)).await; + post( + &backend, + kll_export(metric, timestamp + 1_000_000_000, &raw), + ) + .await; + let response = query(&backend, &format!("quantile_over_time(0.5, {metric}[10s])")).await; + let samples = scalar_values(&response); + assert_eq!( + samples[0].0.get("service").map(String::as_str), + Some(SERVICE) + ); + let actual = samples[0].1; + assert_eq!(actual, exact_quantile(&raw, 0.5)); +} + +#[tokio::test] +async fn production_hll_matches_raw_distinct_oracle() { + let metric = "oracle_hll_users"; + let backend = start_backend(&config(metric, "HLL", " precision: 10")).await; + let owned: Vec = (0..2_000).map(|i| format!("user-{i}")).collect(); + let mut raw: Vec<&str> = owned.iter().map(String::as_str).collect(); + raw.extend(owned.iter().take(500).map(String::as_str)); + let exact = raw.iter().copied().collect::>().len() as f64; + let timestamp = now_ns().saturating_sub(2_000_000_000); + post(&backend, hll_export(metric, timestamp, &raw)).await; + post( + &backend, + hll_export(metric, timestamp + 1_000_000_000, &raw), + ) + .await; + let response = query(&backend, &format!("count({metric})")).await; + let actual = scalar_values(&response)[0].1; + let relative_error = (actual - exact).abs() / exact; + assert!( + relative_error <= 0.10, + "HLL differs from exact distinct oracle: exact={exact}, actual={actual}, error={relative_error}" + ); +} + +fn frequency_fixture() -> Vec<&'static str> { + let mut raw = Vec::new(); + for (key, count) in [("alpha", 100), ("beta", 50), ("gamma", 200), ("delta", 75)] { + raw.extend(std::iter::repeat_n(key, count)); + } + raw +} + +#[tokio::test] +async fn production_cms_matches_raw_topk_oracle() { + let metric = "oracle_cms_frequency"; + let params = format!( + " w: {COLS}\n d: {ROWS}\n heap_size: {HEAP_SIZE}\n with_heap: true" + ); + let backend = start_backend(&config(metric, "CountMinSketchWithHeap", ¶ms)).await; + let raw = frequency_fixture(); + let timestamp = now_ns().saturating_sub(2_000_000_000); + post(&backend, cms_export(metric, timestamp, &raw)).await; + post( + &backend, + cms_export(metric, timestamp + 1_000_000_000, &raw), + ) + .await; + let response = query(&backend, &format!("topk(3, {metric})")).await; + assert_topk_oracle(&response, &raw, 3); +} + +#[tokio::test] +async fn production_count_sketch_matches_raw_topk_oracle() { + let metric = "oracle_count_sketch_frequency"; + let params = format!( + " w: {COLS}\n d: {ROWS}\n heap_size: {HEAP_SIZE}\n with_heap: true" + ); + let backend = start_backend(&config(metric, "CountSketchWithHeap", ¶ms)).await; + let raw = frequency_fixture(); + let timestamp = now_ns().saturating_sub(2_000_000_000); + post(&backend, count_sketch_export(metric, timestamp, &raw)).await; + post( + &backend, + count_sketch_export(metric, timestamp + 1_000_000_000, &raw), + ) + .await; + let response = query(&backend, &format!("topk(3, {metric})")).await; + assert_topk_oracle(&response, &raw, 3); +} diff --git a/data_plane/tests/promql_differential_process_e2e.rs b/data_plane/tests/promql_differential_process_e2e.rs index 1dd7d282..350315db 100644 --- a/data_plane/tests/promql_differential_process_e2e.rs +++ b/data_plane/tests/promql_differential_process_e2e.rs @@ -23,9 +23,9 @@ use serde_json::Value; const METRIC: &str = "differential_e2e_latency_ms"; const SERVICE: &str = "checkout"; const ALPHA: f64 = 0.01; -// Median avoids conflating DDSketch's discrete rank selection with -// Prometheus's interpolation between adjacent values. Small-sample p99 -// interpolation is covered by the broader diagnostic scenario matrix. +// Median keeps this stable DDSketch oracle focused on the sketch's documented +// relative-error contract. Small-sample p99 semantics are a separate product +// gap tracked by issue #492; this test does not claim to cover them. const QUANTILE: f64 = 0.5; struct ChildGuard(Child); @@ -174,7 +174,7 @@ async fn get_json(client: &reqwest::Client, url: &str, params: &[(&str, String)] } #[tokio::test] -async fn production_backend_matches_raw_oracle_for_instant_and_range_promql() { +async fn production_backend_matches_raw_oracle_and_range_endpoint() { let query_port = unused_port(); let otlp_http_port = unused_port(); let otlp_grpc_port = unused_port(); @@ -235,10 +235,14 @@ async fn production_backend_matches_raw_oracle_for_instant_and_range_promql() { let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .expect("system clock"); - let sample_ns = now.as_nanos() as u64; + // Both observations are genuinely in the past. The second identical + // fixture advances the event-time watermark and closes the first window + // without fabricating a future timestamp or introducing a zero-valued + // range sample that has no counterpart in the raw oracle. + let sample_ns = (now - Duration::from_secs(2)).as_nanos() as u64; for body in [ ddsketch_export(METRIC, sample_ns, &raw_values), - ddsketch_export(METRIC, sample_ns + 2_000_000_000, &[]), + ddsketch_export(METRIC, sample_ns + 1_000_000_000, &raw_values), ] { client .post(format!("http://127.0.0.1:{otlp_http_port}/v1/metrics")) @@ -303,7 +307,7 @@ async fn production_backend_matches_raw_oracle_for_instant_and_range_promql() { assert_approx( instant_value, range_values.last().expect("last range value").1, - "instant/range parity", + "instant/range endpoint consistency", ); for (start, end, step, expected_error) in [ diff --git a/docs/03-how-to-guides/manual-e2e-tests.md b/docs/03-how-to-guides/manual-e2e-tests.md index 8ce77cc4..7e63da01 100644 --- a/docs/03-how-to-guides/manual-e2e-tests.md +++ b/docs/03-how-to-guides/manual-e2e-tests.md @@ -36,6 +36,7 @@ The component suites can also be run separately: ./scripts/e2e.sh control-plane ./scripts/e2e.sh data-plane ./scripts/e2e.sh differential +./scripts/e2e.sh sketch-oracles ./scripts/e2e.sh monitor ./scripts/e2e.sh gorilla-merger ./scripts/e2e.sh whole @@ -44,8 +45,26 @@ The component suites can also be run separately: `differential` starts the production data-plane process, derives a modified OTLP DDSketch from a deterministic raw-value fixture, and compares public instant and range PromQL results with an independent exact quantile oracle. -It also verifies labels, ASAP-local execution, instant/range parity, and range -parameter validation. It does not require Docker or a Prometheus process. +It also verifies labels, ASAP-local execution, instant/range endpoint +consistency, and range parameter validation. It does not claim Prometheus +range-step resampling semantics, which are tracked by issue #487. It does not +require Docker or a Prometheus process. + +To run every sketch family through a production data-plane child process and +compare its public result with an oracle computed independently from the raw +fixture, run: + +```bash +./scripts/e2e.sh sketch-oracles +``` + +This covers DDSketch and KLL against exact raw quantiles, HLL against an exact +raw distinct set, and CountSketch/CMS top-k against exact raw frequency maps. +The sketch library is used only to encode modified-OTLP fixtures, never to +compute expected answers. Each scenario is a real, non-ignored test. The +command currently exits non-zero on the product gaps tracked by backend issues +#489 and #491 and planner issue #340; that failure is intentional evidence, +not an expected-pass or smoke assertion. The complete sketch/query differential inventory is exercised with: @@ -53,8 +72,8 @@ The complete sketch/query differential inventory is exercised with: ./scripts/e2e.sh differential-all ``` -It first runs the stable production-process raw-oracle comparison and then the -whole-path scenario matrix. The matrix covers DDSketch and KLL quantiles, HLL +It first runs all production-process raw-oracle comparisons and then the +whole-path in-process scenario matrix. The matrix covers DDSketch and KLL quantiles, HLL cardinality, CountSketch and Count-Min count queries, heap-backed top-k, an instant/range query pair, grouping, delta/sub-window ingest, and shadow/live serving. It exits non-zero for every real product regression; no scenario is @@ -62,14 +81,13 @@ ignored or converted into an expected pass. | Sketch / path | Public query shape | Oracle / invariant | | --- | --- | --- | -| DDSketch | `quantile_over_time(0.5, ...[10s])`, instant + range | exact raw-value median and instant/range parity | -| DDSketch | `quantile_over_time(0.99, ...[30s])` | exact raw-value p99 within planned alpha | +| DDSketch | `quantile_over_time(0.5, ...[10s])`, instant + range | exact raw-value median and endpoint consistency | +| DDSketch | `quantile_over_time(0.99, ...[30s])` | strict small-sample p99 regression tracked by #492 | | DDSketch delta | `quantile_over_time(0.99, ...[3m])` | reconstructed multi-window distribution | -| KLL | `quantile_over_time(0.5, ...[10s])` | non-empty approximate quantile | -| HLL | `count(metric)` | cardinality result, including multi-SID merge | -| CountSketch | `count_over_time(...[10s])` | frequency result | -| Count-Min Sketch | instant and range `count_over_time` | frequency result and matrix wire shape | -| Heap-backed CountSketch / CMS | `topk(3, metric)` | bounded result count, item labels, and deterministic leader | +| KLL | `quantile_over_time(0.5, ...[10s])` | exact raw-value median (fixture retained below K) | +| HLL | `count(metric)` | exact raw distinct set within a declared 10% bound | +| CountSketch | `topk(3, metric)` | exact raw frequency map and item identities | +| Count-Min Sketch | `topk(3, metric)` | exact raw frequency map and item identities | | PromQL range validation | equal start/end and zero step | Prometheus-compatible explicit errors | `whole` is the stable representative DDSketch path. To exercise every diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 73793b5c..b6dd23bf 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -26,12 +26,13 @@ Targets: contracts Shared Rust type/protobuf wire contracts control-plane Planner HTTP, OpAMP, publication, and runtime feedback data-plane Query, routing, storage, ingest adapter, and lifecycle tests - differential Production backend PromQL vs deterministic raw-value oracle + differential Production DDSketch PromQL vs deterministic raw-value oracle + sketch-oracles Every sketch via production binary + independent raw oracle monitor Real monitor gRPC transport tests gorilla-merger Gorilla HTTP/WAL/block/StoreAPI/compaction/shipper tests whole Controller plan -> backend install -> OTLP -> store -> PromQL whole-matrix All sketch families and query shapes (diagnostic) - differential-all Raw oracle test plus the all-sketch/query matrix + differential-all Production sketch oracles plus the in-process query matrix system Delegate to ASAPCollector's real multi-node system harness list Print the suites and audit Rust E2E ignore markers @@ -108,10 +109,17 @@ data_plane() { differential() { CURRENT_STAGE="data-plane/promql-differential" - say "data-plane: production backend PromQL -> raw-value oracle + instant/range parity" + say "data-plane: production DDSketch PromQL -> raw oracle + range endpoint consistency" rust_test data_plane --test promql_differential_process_e2e } +sketch_oracles() { + differential + CURRENT_STAGE="data-plane/all-sketch-production-oracles" + say "data-plane: production KLL/HLL/CountSketch/CMS -> independent raw-data oracles" + rust_test data_plane --test all_sketches_process_oracle_e2e +} + monitor() { CURRENT_STAGE="monitor-grpc" say "monitor: real bidirectional gRPC server/client" @@ -152,18 +160,19 @@ whole_matrix() { } differential_all() { - differential + sketch_oracles whole_matrix } list_suites() { usage printf '\nRust E2E tests marked #[ignore] (expected: none):\n' - rg -n '^[[:space:]]*#\[ignore' \ - "${REPO_DIR}/data_plane/tests" \ - "${REPO_DIR}/data_plane/src/tests" \ - "${REPO_DIR}/crates" \ - -g '*.rs' || true + local ignored + if ignored="$(git -C "${REPO_DIR}" grep -n -E '^[[:space:]]*#[[:space:]]*\[[[:space:]]*ignore' -- '*.rs')"; then + printf '%s\n' "${ignored}" + die "ignored Rust tests found; convert them to executable E2E/unit tests or remove stale coverage" + fi + printf 'none\n' } system_e2e() { @@ -194,6 +203,7 @@ main() { control-plane) need cargo; control_plane ;; data-plane) need cargo; data_plane ;; differential) need cargo; differential ;; + sketch-oracles) need cargo; sketch_oracles ;; monitor) need cargo; monitor ;; gorilla-merger) gorilla_merger ;; whole) need cargo; whole ;; From 2a36b954672ab20e1731080c502188e6433cddcb Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 13:25:02 -0600 Subject: [PATCH 8/8] test: run every diagnostic after failures --- docs/03-how-to-guides/manual-e2e-tests.md | 5 +++-- scripts/e2e.sh | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/03-how-to-guides/manual-e2e-tests.md b/docs/03-how-to-guides/manual-e2e-tests.md index 7e63da01..d3f2c804 100644 --- a/docs/03-how-to-guides/manual-e2e-tests.md +++ b/docs/03-how-to-guides/manual-e2e-tests.md @@ -72,8 +72,9 @@ The complete sketch/query differential inventory is exercised with: ./scripts/e2e.sh differential-all ``` -It first runs all production-process raw-oracle comparisons and then the -whole-path in-process scenario matrix. The matrix covers DDSketch and KLL quantiles, HLL +It runs all production-process raw-oracle comparisons and the whole-path +in-process scenario matrix even when the oracle suite finds a regression, then +returns non-zero if either suite failed. The matrix covers DDSketch and KLL quantiles, HLL cardinality, CountSketch and Count-Min count queries, heap-backed top-k, an instant/range query pair, grouping, delta/sub-window ingest, and shadow/live serving. It exits non-zero for every real product regression; no scenario is diff --git a/scripts/e2e.sh b/scripts/e2e.sh index b6dd23bf..4417953e 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -160,8 +160,10 @@ whole_matrix() { } differential_all() { - sketch_oracles - whole_matrix + local status=0 + sketch_oracles || status=$? + whole_matrix || status=$? + return "${status}" } list_suites() {