diff --git a/Cargo.lock b/Cargo.lock index 4ec2c5938..4edd38bd0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -461,6 +461,7 @@ name = "asap_types" version = "0.1.0" dependencies = [ "anyhow", + "asap-aware-mapping", "asap-types", "base64 0.21.7", "clap 4.6.6", diff --git a/control_plane/src/accuracy.rs b/control_plane/src/accuracy.rs index 5c7c79d69..4f52b0d9d 100644 --- a/control_plane/src/accuracy.rs +++ b/control_plane/src/accuracy.rs @@ -1,93 +1,43 @@ -//! Theoretical accuracy profile of a chosen `SketchParams`. -//! -//! Mirrors `ASAPQuery-backend/src/stores/sketch_db/accuracy.rs` — -//! both sides compute the same (ε, δ) from the same sketch type -//! + parameters, so a plan the control plane validates here meets -//! the same bound the backend will later surface on query -//! responses. The two files must be kept in lockstep; a golden -//! test at the bottom of this module pins the numeric parity. -//! -//! ## Why the control plane needs this -//! -//! The planner picks `SketchParams` (width/depth/K/α/precision) -//! to meet a user-supplied `accuracy_sla`. Today the cost model -//! approximates the bound inline in a few places -//! (`cost_model.rs:156-158`). Centralising the derivation here: -//! -//! * lets `DeploymentPlanCompiler` / `DeploymentCostPlanner` / any future -//! planner compute the post-hoc ε of the chosen plan and -//! verify it actually meets the SLA. -//! * surfaces the bound to downstream systems (backend via -//! `/api/v1/plan` response; dashboards via -//! `asap_otel_processor_accuracy_epsilon` gauge) without the -//! caller re-deriving from scratch. -//! -//! ## Bounds we encode -//! -//! | Sketch | `kind` | ε formula | δ formula | -//! |-------------------|-----------------------|----------------------|--------------------| -//! | CountMinSketch | `AdditiveFrequency` | e / w | 1 / 2^d | -//! | CountSketch(ε, δ) | `AdditiveFrequency` | ε (user-supplied) | δ (user-supplied) | -//! | HLL(p) | `RelativeCardinality` | 1.04 / √(2^p) | — (Gaussian σ) | -//! | KLL(k) | `RankQuantile` | 2.296 / √k | 0.01 (fixed) | -//! | DDSketch(α) | `RelativeQuantile` | α | 0 (deterministic) | -//! -//! Citations (verbatim from the backend): -//! * CMS — Cormode & Muthukrishnan, *J. Algorithms* 55(1) 2005 -//! * CountSketch — Charikar, Chen, Farach-Colton, ICALP 2002 -//! * HLL — Flajolet et al., DMTCS 2007 -//! * KLL — Karnin, Lang, Liberty, FOCS 2016 -//! * DDSketch — Masson, Rim, Lee, VLDB 2019 +//! Adapt planner sketch parameters to ASAPPlanner accuracy guarantees. use crate::types::SketchParams; -pub use asap_types::{AccuracyKind, AccuracyProfile}; +pub use asap_types::accuracy::{AccuracyKind, AccuracyProfile}; +use planner_types::post_asap::SketchParams as PlannerParams; -/// Planner-specific derivation over Planner-owned sketch parameters. +pub fn derive(params: &SketchParams) -> AccuracyProfile { + let normalized = match params { + SketchParams::CountMinSketch { rows, cols, .. } => PlannerParams::Cms { + depth: (*rows).max(1), + width: (*cols).max(1), + }, + SketchParams::CountSketch { epsilon, delta } => { + return AccuracyProfile { + epsilon: *epsilon, + delta: Some(*delta), + kind: AccuracyKind::AdditiveFrequency, + } + } + SketchParams::HLL { precision } => PlannerParams::Hll { + precision: u8::try_from(*precision).unwrap_or(14), + }, + SketchParams::KLL { k, .. } => PlannerParams::Kll { k: (*k).max(1) }, + SketchParams::DDSketch { + relative_accuracy, .. + } => PlannerParams::DDSketch { + alpha: *relative_accuracy, + }, + }; + AccuracyProfile::from_sketch_params(&normalized) + .expect("shared family has a numeric planner bound") +} + +/// Source adapter for planner configuration parameters. pub trait PlannerAccuracyProfile { fn derive(params: &SketchParams) -> Self; } - impl PlannerAccuracyProfile for AccuracyProfile { fn derive(params: &SketchParams) -> Self { - match params { - SketchParams::CountMinSketch { rows, cols, .. } => { - let cols = (*cols).max(1) as f64; - let rows = (*rows).max(1) as i32; - Self { - epsilon: std::f64::consts::E / cols, - delta: 0.5_f64.powi(rows), - kind: AccuracyKind::AdditiveFrequency, - } - } - SketchParams::CountSketch { epsilon, delta } => Self { - epsilon: *epsilon, - delta: *delta, - kind: AccuracyKind::AdditiveFrequency, - }, - SketchParams::HLL { precision } => { - let m = (1u64 << precision) as f64; - Self { - epsilon: 1.04 / m.sqrt(), - delta: 0.0, - kind: AccuracyKind::RelativeCardinality, - } - } - SketchParams::KLL { k, .. } => { - let k = (*k).max(1) as f64; - Self { - epsilon: 2.296 / k.sqrt(), - delta: 0.01, - kind: AccuracyKind::RankQuantile, - } - } - SketchParams::DDSketch { - relative_accuracy, .. - } => Self { - epsilon: *relative_accuracy, - delta: 0.0, - kind: AccuracyKind::RelativeQuantile, - }, - } + derive(params) } } @@ -97,30 +47,30 @@ mod tests { #[test] fn cms_bound_is_e_over_w() { - let p = AccuracyProfile::derive(&SketchParams::CountMinSketch { + let p = derive(&SketchParams::CountMinSketch { rows: 3, cols: 1000, metric_name: "m".into(), }); assert_eq!(p.kind, AccuracyKind::AdditiveFrequency); assert!((p.epsilon - std::f64::consts::E / 1000.0).abs() < 1e-12); - assert!((p.delta - 0.125).abs() < 1e-12); + assert!((p.delta.unwrap() - (-3.0_f64).exp()).abs() < 1e-12); } #[test] fn countsketch_passes_through_user_supplied_bounds() { - let p = AccuracyProfile::derive(&SketchParams::CountSketch { + let p = derive(&SketchParams::CountSketch { epsilon: 0.01, delta: 0.001, }); assert_eq!(p.kind, AccuracyKind::AdditiveFrequency); assert_eq!(p.epsilon, 0.01); - assert_eq!(p.delta, 0.001); + assert_eq!(p.delta, Some(0.001)); } #[test] fn hll_p14_matches_flajolet_bound() { - let p = AccuracyProfile::derive(&SketchParams::HLL { precision: 14 }); + let p = derive(&SketchParams::HLL { precision: 14 }); assert_eq!(p.kind, AccuracyKind::RelativeCardinality); // 1.04 / √16384 = 0.008125 assert!((p.epsilon - 0.008125).abs() < 1e-9); @@ -128,56 +78,51 @@ mod tests { #[test] fn kll_k200_matches_karnin_lang_liberty_bound() { - let p = AccuracyProfile::derive(&SketchParams::KLL { + let p = derive(&SketchParams::KLL { k: 200, quantiles: vec![0.5, 0.95, 0.99], }); assert_eq!(p.kind, AccuracyKind::RankQuantile); - assert!((p.epsilon - 2.296 / 200.0_f64.sqrt()).abs() < 1e-12); - assert!((p.delta - 0.01).abs() < 1e-12); + assert!((p.epsilon - 2.296 / 200.0_f64.powf(0.9723)).abs() < 1e-12); + assert!((p.delta.unwrap() - 0.01).abs() < 1e-12); } #[test] fn ddsketch_alpha_passes_through_verbatim() { for alpha in [0.005, 0.01, 0.02, 0.05] { - let p = AccuracyProfile::derive(&SketchParams::DDSketch { + let p = derive(&SketchParams::DDSketch { relative_accuracy: alpha, quantiles: vec![0.5, 0.99], }); assert_eq!(p.kind, AccuracyKind::RelativeQuantile); assert_eq!(p.epsilon, alpha); - assert_eq!(p.delta, 0.0); + assert_eq!(p.delta, Some(0.0)); } } - /// **Parity contract** with the backend. Pinned numeric - /// values are identical to what `ASAPQuery-backend`'s - /// `AccuracyProfile::derive` produces for the matching - /// `AggregationConfig`. Any change that drifts these tests - /// likely needs a matching change on the backend side — - /// and vice versa. + /// Keep the user-facing summary explicit about unknown HLL confidence. #[test] fn golden_parity_with_backend() { // HLL precision=14: ε = 0.008125, kind = relative_cardinality - let p = AccuracyProfile::derive(&SketchParams::HLL { precision: 14 }); + let p = derive(&SketchParams::HLL { precision: 14 }); assert_eq!( p.summary(), - "accuracy: ε=0.008125, δ=0, kind=relative_cardinality" + "accuracy: ε=0.008125, δ=unknown, kind=relative_cardinality" ); - // KLL k=200: ε = 2.296 / √200, kind = rank_quantile, δ = 0.01 - let p = AccuracyProfile::derive(&SketchParams::KLL { + // KLL k=200: ε = 2.296 / 200^0.9723, kind = rank_quantile, δ = 0.01 + let p = derive(&SketchParams::KLL { k: 200, quantiles: vec![], }); - let expected_eps = 2.296_f64 / 200.0_f64.sqrt(); + let expected_eps = 2.296_f64 / 200.0_f64.powf(0.9723); assert_eq!( p.summary(), format!("accuracy: ε={}, δ=0.01, kind=rank_quantile", expected_eps) ); // DDSketch α=0.02: passes verbatim, δ=0, kind=relative_quantile - let p = AccuracyProfile::derive(&SketchParams::DDSketch { + let p = derive(&SketchParams::DDSketch { relative_accuracy: 0.02, quantiles: vec![], }); @@ -186,7 +131,7 @@ mod tests { #[test] fn kind_serialises_to_snake_case() { - let p = AccuracyProfile::derive(&SketchParams::HLL { precision: 14 }); + let p = derive(&SketchParams::HLL { precision: 14 }); let json = serde_json::to_string(&p).unwrap(); assert!(json.contains("\"kind\":\"relative_cardinality\"")); } @@ -195,7 +140,7 @@ mod tests { fn round_trip_through_serde() { let src = AccuracyProfile { epsilon: 0.008125, - delta: 0.0, + delta: None, kind: AccuracyKind::RelativeCardinality, }; let json = serde_json::to_string(&src).unwrap(); diff --git a/control_plane/src/lib.rs b/control_plane/src/lib.rs index c6a7812fd..c61639e5c 100644 --- a/control_plane/src/lib.rs +++ b/control_plane/src/lib.rs @@ -1,15 +1,8 @@ -//! Control plane crate — library surface for the ASAPQuery-backend host. -//! -//! Refactor-2026-05 (Phase 9 / `refactor/controller-layered-cleanup`): -//! the controller previously ran as a standalone binary with its own -//! OpAMP server and HTTP API. After the controller crate moved into -//! ASAPQuery-backend, the same modules are exposed as a Rust library so -//! `asap-query-engine` can call them in-process — capability mapping, -//! plan emission, OpAMP push from the backend host. This `lib.rs` -//! declares the public module surface; the existing `main.rs` continues -//! to provide the standalone binary entrypoint for any deployments that -//! still want to run the control plane out-of-process. +//! Control-plane planning, configuration emission, and deployment services. //! +//! These modules support in-process integration and the standalone control-plane +//! binary. Internal module paths are not a wire-protocol compatibility contract. + #![allow( clippy::collapsible_match, clippy::doc_lazy_continuation, @@ -23,52 +16,6 @@ clippy::vec_init_then_push )] -//! ## 2026-05 layered-cleanup refactor — old → new module mapping -//! -//! The internal module layout was restructured to mirror -//! `control_plane/docs/design.md` §5 target layout without splitting into -//! multiple crates: -//! -//! | Old path | New path | -//! |---|---| -//! | `controller/src/algebra/expr.rs` | `controller/src/intent_algebra/relational.rs` | -//! | `controller/src/algebra/lower.rs` | `controller/src/intent_algebra/legacy_lower.rs` | -//! | `controller/src/algebra/directory.rs` | `controller/src/physical/sketch_catalog.rs` | -//! | `controller/src/algebra/physical.rs` | `controller/src/physical/planner.rs` | -//! | `controller/src/algebra/allocator.rs` | `controller/src/physical/allocator.rs` | -//! | `controller/src/algebra/plan.rs` | `controller/src/physical/plan.rs` | -//! | `controller/src/algebra/optimizer.rs` | `controller/src/optimizer/engine.rs` | -//! | `controller/src/planner/cost_model.rs` | `controller/src/optimizer/cost/mod.rs` | -//! | `controller/src/planner/{delta,online}_cost_model.rs` | `controller/src/optimizer/cost/{delta,online}.rs` | -//! | `controller/src/planner/{pareto,tco,wire_cost}.rs` | `controller/src/optimizer/cost/{pareto,tco,wire}.rs` | -//! | `controller/src/planner/rules.rs` | `controller/src/optimizer/rules/mod.rs` | -//! | `controller/src/planner/baseline_planner.rs` | `controller/src/optimizer/baseline.rs` | -//! | `controller/src/planner/stage_split.rs` | `controller/src/physical/stage_split.rs` | -//! | `controller/src/analyzer.rs` | `controller/src/pipeline.rs` | -//! | `controller/src/stage_split/` | `controller/src/physical/colored_dag/` | -//! | `controller/src/query_language/` | `controller/src/query_parser/language/` | -//! | `controller/src/config/workloads.rs` | `controller/src/workload.rs` | -//! | `controller/src/config/{stage_config*,agent,backend,asapquery_backend,precompute}.rs` | `controller/src/emit/{...}.rs` | -//! -//! Public modules to consume from `asap-query-engine`: -//! - `intent_algebra` — `AggIntent` + `QueryExpr` DAG (canonical L3 IR; -//! `relational` carries the L2 relational IR the parsers emit and -//! `lower` lowers it to the canonical L3 types). -//! - `query_parser` — L1 entry point: `parse_query_expr_canonical`/`parse_query` -//! call `asap_frontend_promql::lower_promql` directly (no local parser -//! since design-target-architecture.md Part B; SQL not yet adopted). -//! - `physical` — L5 framework (allocator, planner, plan, sketch_catalog, -//! colored_dag, stage_split, topology). -//! - `optimizer` — L4 rule engine + cost model traits/impls + baseline -//! planner. -//! - `opamp` — OpAMP server (will be invoked from the backend's -//! service startup once Phase 4 wires the in-process integration). -//! - `types`, `types_v2` — control-plane-internal data model. -//! -//! NOT intended for public consumption from outside the workspace — -//! these modules expose the control plane's L1–L5 internals and are not -//! part of any wire/protocol contract. - pub mod accuracy; pub mod backend_client; pub mod clickhouse; @@ -92,14 +39,6 @@ pub mod types; pub mod types_v2; pub mod workload; -// 2026-05 layered-cleanup follow-up: the back-compat shims previously -// defined here (`pub use emit as config`, `pub use pipeline as analyzer`, -// `pub mod algebra { … }`, `pub use physical::colored_dag as stage_split`, -// `pub mod planner { … }`) have been removed. `main.rs` and other -// consumers now reference the canonical module names directly -// (`emit`, `pipeline`, `intent_algebra::relational`, `optimizer`, -// `physical`, `physical::colored_dag`, etc.) per the layered-cleanup -// follow-up task. /// PromQL → ASAPPlanner's canonical post-ASAP plan via /// `asap_aware_mapping::bind::implement_tree`. pub mod asap_tier_implement; diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index fc568a382..0ad1a2f8a 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -36,7 +36,6 @@ use emit::{emit_for_runtime, AgentRuntime}; use emit::{generate_agent_collector_config, post_typed_backend_for_role}; use monitor::{Endpoint, ScrapedData, Scraper, Thresholds, Violation}; use opamp::{AgentRole, OpampServer, RemoteConfig}; -use physical::allocator::SketchAllocator; use physical::colored_dag::emitter::BackendStageConfig; use physical::deployment_cost::online as online_cost_model; use physical::deployment_cost::online::{init_store as init_online_store, OnlineMetricsStore}; @@ -49,7 +48,6 @@ use query_parser::parse_query_expr_canonical; use replan::Replanner; use store::{PlanStore, WorkloadStore}; use types::AgentCollectorConfig; -use types::StageResourceBudgets; use workload::AggRole; use workload::WorkloadRegistry; @@ -1110,28 +1108,10 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> let plan = st.planner.plan(&workload, Some(&wc)); - // ── L1→L5: parse → optimise → bind → stage. One algebra pipeline. ─────── - // When the spec carries a `query_string`, this is the single place the - // algebra runs: parse to canonical L3, optimise, bind to the L4 - // `PhysicalExpr`, and derive the L5 artifacts from that one tree — - // * `bound_physical` → the typed L5 stage-split (below) — the L5; - // * `plan_summary` → the cost summary in the JSON response. - // The SP-3 flat assignment in `plan` remains the fallback when there - // is no `query_string`. - let raw_bps = plan.transmission_cost_summary.raw_bytes_per_sec; - let budgets = StageResourceBudgets::from_workload_chars(&wc); - // Everything that touches `physical::post_asap::PhysicalExpr` (which - // carries `Rc` since Step B of the - // plan-shaped-serving migration adopted ASAPController's own - // `Rc`-based DAG sharing) is scoped to this block and resolved down - // to Send-safe outputs (`Option`, - // `Option>`) *before* any `.await` - // below — an `Rc` alive in this `async fn`'s generator state at a - // yield point would make its `Future` `!Send`, breaking - // `axum::Handler`. - let (plan_summary, stage_configs) = { + // Derive deployment configs from the bound query. Keep its Rc-backed DAG + // scoped before any await so the handler future remains Send. + let stage_configs = { let mut bound_physical: Option = None; - let mut plan_summary = None; if let Some(ref qs) = query_string { match parse_query_expr_canonical(qs, workload.accuracy.clone()) { Err(e) => { @@ -1145,9 +1125,6 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> workload.accuracy.clone(), ) .ok(); - // Cost summary for the JSON response. - let plan_node = SketchAllocator::new(budgets.clone(), raw_bps).allocate(qe); - plan_summary = Some(plan_node.summarise(raw_bps)); } } } @@ -1165,7 +1142,7 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> None }; - (plan_summary, stage_configs) + stage_configs }; // B2 (metric, role): derive the role from the request's @@ -1379,8 +1356,6 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> .await; } - // `plan_summary` was computed in the single algebra pipeline above. - let agents = st.opamp.connected_agents().await; let cost = &plan.transmission_cost_summary; ( @@ -1402,7 +1377,6 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> "estimated_fill_rate": cost.estimated_fill_rate, "flush_rate_hz": cost.flush_rate_hz, }, - "plan_summary": plan_summary, })), ) .into_response() @@ -2469,6 +2443,8 @@ mod api_tests { assert_eq!(body["metric"], "latency"); assert!(body["sketch_type"].as_str().is_some()); assert!(body["valid_until"].as_str().is_some()); + assert!(body.get("plan_summary").is_none()); + assert!(body["transmission_costs"].is_object()); } /// HTTP and stored replan inputs share the resolved typed target, including delta. diff --git a/control_plane/src/physical/allocator.rs b/control_plane/src/physical/allocator.rs deleted file mode 100644 index 9b6be02a4..000000000 --- a/control_plane/src/physical/allocator.rs +++ /dev/null @@ -1,1063 +0,0 @@ -//! Sketch allocator — converts a [`QueryExpr`] tree into an annotated -//! [`PlanNode`] tree, assigning every operator to a [`PipelineStage`] and -//! choosing between sketch and exact execution. -//! -//! # Algorithm -//! -//! 1. Walk the tree bottom-up (children before parents). -//! 2. For each node, determine the *preferred* stage using the rules below. -//! 3. If the preferred stage exceeds its resource budget, demote to the next -//! stage in the chain: `Agent → Backend → Precompute → Db`. -//! 4. Annotate the node with the chosen sketch type, delta-encoding flag, -//! and a human-readable rationale. -//! -//! ## Stage assignment rules -//! -//! | Node type | Default stage | Condition | -//! |-----------|---------------|-----------| -//! | Scan, Ref, Filter, Window, Partition, Distinct | Agent | Always | -//! | Aggregate (single sketch intent, mergeable) | Agent | budget OK | -//! | Aggregate (single sketch intent, mergeable) | Backend | agent budget exceeded | -//! | Aggregate (single intent: Avg) | Db | always (not mergeable) | -//! | Aggregate (single exact intent: Sum/Count/Min/Max) | Backend | mergeable | -//! | Aggregate (single TopK intent) | Precompute | always | -//! | Aggregate (multi-intent or HAVING) | Db | always (general exact) | -//! | Merge | Backend | always | -//! | Project, Sort, Limit, Join, SetOp | Db | always | -//! | Subquery, BinaryOp | Precompute | has sketch children | -//! | LetBinding | same as body | propagated | -//! -//! Step γ7: the canonical IR folds the legacy `SketchAgg` / `WindowedAgg` -//! / `TopK` variants all into `Aggregate`, so the single `Aggregate` arm -//! dispatches on shape. `histogram_quantile(φ, …)` was already -//! substituted at the parser level into a plain `Aggregate{Quantile(φ)}` -//! (Step γ5). A `Window` over a single-intent `Aggregate` is the -//! canonical fold of the legacy `WindowedAgg`; for *stage* allocation -//! the window is informational (the `Window` arm is a passthrough and -//! the inner `Aggregate` arm does the sketch placement). The -//! window-defines-sketch-lifecycle fusion that *does* matter is a -//! `physical::planner` concern — see `physical::window_fusion`. - -use std::rc::Rc; - -use super::plan::{CostEstimate, ExecutionMode, NodeAnnotation, PipelineStage, PlanNode}; -use crate::types::{SketchType, StageResourceBudgets}; -use planner_types::pre_asap::agg_is_exact; -use planner_types::pre_asap::AggIntent; -use planner_types::pre_asap::{QueryExpr, Reduction}; - -// ── Resource budget tracker ─────────────────────────────────────────────────── - -/// Mutable budget state, consumed during allocation. -#[derive(Debug, Clone)] -struct BudgetState { - agent_memory_remaining_bytes: f64, - backend_memory_remaining_bytes: f64, -} - -impl BudgetState { - fn from_budgets(b: &StageResourceBudgets) -> Self { - Self { - agent_memory_remaining_bytes: b - .agent_memory_bytes - .map(|v| v as f64) - .unwrap_or(f64::INFINITY), - backend_memory_remaining_bytes: b - .backend_memory_bytes - .map(|v| v as f64) - .unwrap_or(f64::INFINITY), - } - } - - fn fits_agent(&self, bytes: f64) -> bool { - bytes <= self.agent_memory_remaining_bytes - } - - fn fits_backend(&self, bytes: f64) -> bool { - bytes <= self.backend_memory_remaining_bytes - } - - fn consume_agent(&mut self, bytes: f64) { - self.agent_memory_remaining_bytes = (self.agent_memory_remaining_bytes - bytes).max(0.0); - } - - fn consume_backend(&mut self, bytes: f64) { - self.backend_memory_remaining_bytes = - (self.backend_memory_remaining_bytes - bytes).max(0.0); - } -} - -// ── Public allocator ────────────────────────────────────────────────────────── - -/// Converts a (pre-optimised) canonical [`QueryExpr`] tree into an -/// annotated [`PlanNode`] tree. -pub struct SketchAllocator { - budgets: StageResourceBudgets, - raw_bytes_per_sec: f64, -} - -impl SketchAllocator { - /// Create an allocator. - /// - /// * `budgets` — per-stage memory caps (from [`StageResourceBudgets`]). - /// * `raw_bytes_per_sec` — baseline bandwidth of the raw OTLP stream, - /// used to estimate compression ratios. - pub fn new(budgets: StageResourceBudgets, raw_bytes_per_sec: f64) -> Self { - Self { - budgets, - raw_bytes_per_sec, - } - } - - /// Allocate stages for the entire expression tree. - /// - /// The canonical `Aggregate.by` is already positional (`Vec`), - /// so — unlike the legacy allocator — no inherited `Schema` needs to be - /// threaded for column resolution; stage assignment is purely structural. - pub fn allocate(&self, expr: QueryExpr) -> PlanNode { - let mut budget = BudgetState::from_budgets(&self.budgets); - self.alloc_node(expr, &mut budget) - } - - // ── Recursive allocation ────────────────────────────────────────────────── - - fn alloc_node(&self, expr: QueryExpr, budget: &mut BudgetState) -> PlanNode { - match expr { - // ── Leaves ─────────────────────────────────────────────────────── - QueryExpr::Scan { .. } => { - PlanNode::leaf(expr, PipelineStage::Agent, ExecutionMode::Passthrough) - } - - // ── Structural / filter nodes — always Agent ────────────────── - QueryExpr::Filter { pred, child } => { - let child = self.alloc_node((*child).clone(), budget); - PlanNode { - expr: QueryExpr::Filter { - pred, - child: Rc::new(child.expr.clone()), - }, - stage: PipelineStage::Agent, - mode: ExecutionMode::Passthrough, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec * 0.5, - ..Default::default() - }, - annotation: NodeAnnotation { - rationale: "Filter pushed to Agent to reduce data volume early".into(), - ..Default::default() - }, - children: vec![child], - } - } - - // A `TimeRange` over a single-intent `Aggregate` is the - // canonical fold of the legacy `WindowedAgg` (was `Window` - // before the ASAPPlanner pin migration); for *stage* - // allocation the range is informational — this arm is a - // passthrough and the inner `Aggregate` arm does the sketch - // placement. - QueryExpr::TimeRange { range, child } => { - let child = self.alloc_node((*child).clone(), budget); - PlanNode { - expr: QueryExpr::TimeRange { - range, - child: Rc::new(child.expr.clone()), - }, - stage: PipelineStage::Agent, - mode: ExecutionMode::Passthrough, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale: "Time window computed at Agent".into(), - ..Default::default() - }, - children: vec![child], - } - } - - QueryExpr::Dedup { cols, child } => { - let child = self.alloc_node((*child).clone(), budget); - PlanNode { - expr: QueryExpr::Dedup { - cols, - child: Rc::new(child.expr.clone()), - }, - stage: PipelineStage::Agent, - mode: ExecutionMode::Passthrough, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale: "Distinct at Agent before sketch build".into(), - ..Default::default() - }, - children: vec![child], - } - } - - // ── Aggregate — the canonical IR folds legacy SketchAgg / - // WindowedAgg / TopK all into Aggregate, so dispatch on shape: - // * single TopK intent, no HAVING → heavy-hitter at Precompute - // * single other intent, no HAVING → budget-driven sketch - // * multi-intent or HAVING → general exact Aggregate at Db - QueryExpr::Aggregate { - reduction, - measures: aggs, - output_names, - having, - child, - } => { - if aggs.len() == 1 && having.is_none() { - if let AggIntent::TopK { k, .. } = &aggs[0] { - let k = *k; - let child = self.alloc_node((*child).clone(), budget); - return PlanNode { - expr: QueryExpr::Aggregate { - reduction, - measures: aggs, - output_names, - having, - child: Rc::new(child.expr.clone()), - }, - stage: PipelineStage::Precompute, - mode: ExecutionMode::Sketch, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec * 0.05, - memory_bytes: (k as f64) * 64.0, - ..Default::default() - }, - annotation: NodeAnnotation { - sketch_type: Some(SketchType::CountSketch), - rationale: format!( - "TopK(k={k}) assigned to Precompute engine (CountSketch)" - ), - ..Default::default() - }, - children: vec![child], - }; - } - // Single non-TopK intent → budget-driven sketch agg. - let child = self.alloc_node((*child).clone(), budget); - return self.alloc_sketch_agg(reduction, aggs, output_names, child, budget); - } - // General multi-intent / HAVING aggregate → Db (exact). - let child = self.alloc_node((*child).clone(), budget); - let kinds: Vec<&'static str> = aggs.iter().map(canonical_intent_kind_str).collect(); - PlanNode { - expr: QueryExpr::Aggregate { - reduction, - measures: aggs, - output_names, - having, - child: Rc::new(child.expr.clone()), - }, - stage: PipelineStage::Db, - mode: ExecutionMode::Exact, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec, - ..Default::default() - }, - annotation: NodeAnnotation { - rationale: format!("General Aggregate at Db (exact); intents: {kinds:?}"), - ..Default::default() - }, - children: vec![child], - } - } - - // ── Merge — Backend ─────────────────────────────────────────── - QueryExpr::Concat { - children: inputs, - discriminator_unique_key, - } => { - let children: Vec = inputs - .into_iter() - .map(|inp| self.alloc_node(inp, budget)) - .collect(); - let mem: f64 = children.iter().map(|c| c.cost.memory_bytes).sum(); - PlanNode { - expr: QueryExpr::Concat { - children: children.iter().map(|c| c.expr.clone()).collect(), - discriminator_unique_key, - }, - stage: PipelineStage::Backend, - mode: ExecutionMode::Passthrough, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec * 0.1, - memory_bytes: mem, - ..Default::default() - }, - annotation: NodeAnnotation { - rationale: "Sketch merge at Backend".into(), - ..Default::default() - }, - children, - } - } - - // ── Exact / relational — Db ─────────────────────────────────── - QueryExpr::Project { - cols, - qualifier, - child, - } => { - let child = self.alloc_node((*child).clone(), budget); - PlanNode { - expr: QueryExpr::Project { - cols, - qualifier, - child: Rc::new(child.expr.clone()), - }, - stage: PipelineStage::Db, - mode: ExecutionMode::Exact, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale: "Project at Db".into(), - ..Default::default() - }, - children: vec![child], - } - } - - QueryExpr::Sort { - keys, - partition_by, - child, - } => { - let child = self.alloc_node((*child).clone(), budget); - PlanNode { - expr: QueryExpr::Sort { - keys, - partition_by, - child: Rc::new(child.expr.clone()), - }, - stage: PipelineStage::Db, - mode: ExecutionMode::Exact, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale: "Sort at Db".into(), - ..Default::default() - }, - children: vec![child], - } - } - - QueryExpr::Limit { n, offset, child } => { - let child = self.alloc_node((*child).clone(), budget); - PlanNode { - expr: QueryExpr::Limit { - n, - offset, - child: Rc::new(child.expr.clone()), - }, - stage: PipelineStage::Db, - mode: ExecutionMode::Exact, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale: "Limit at Db".into(), - ..Default::default() - }, - children: vec![child], - } - } - - QueryExpr::Join { - kind, - pred, - left, - right, - } => { - let left_node = self.alloc_node((*left).clone(), budget); - let right_node = self.alloc_node((*right).clone(), budget); - PlanNode { - expr: QueryExpr::Join { - kind, - pred, - left: Rc::new(left_node.expr.clone()), - right: Rc::new(right_node.expr.clone()), - }, - stage: PipelineStage::Db, - mode: ExecutionMode::Exact, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec, - ..Default::default() - }, - annotation: NodeAnnotation { - rationale: "Join at Db (exact)".into(), - ..Default::default() - }, - children: vec![left_node, right_node], - } - } - - QueryExpr::SetOp { - kind, - all, - left, - right, - } => { - let left_node = self.alloc_node((*left).clone(), budget); - let right_node = self.alloc_node((*right).clone(), budget); - PlanNode { - expr: QueryExpr::SetOp { - kind, - all, - left: Rc::new(left_node.expr.clone()), - right: Rc::new(right_node.expr.clone()), - }, - stage: PipelineStage::Db, - mode: ExecutionMode::Exact, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale: "SetOp at Db".into(), - ..Default::default() - }, - children: vec![left_node, right_node], - } - } - - // ── PromQL sub-query ────────────────────────────────────────── - QueryExpr::PromqlSubquery { - range, - resolution, - child, - } => { - let child = self.alloc_node((*child).clone(), budget); - let stage = if child.mode == ExecutionMode::Sketch { - PipelineStage::Precompute - } else { - PipelineStage::Db - }; - let rationale = format!("PromQL subquery at {stage}"); - PlanNode { - expr: QueryExpr::PromqlSubquery { - range, - resolution, - child: Rc::new(child.expr.clone()), - }, - stage, - mode: child.mode.clone(), - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale, - ..Default::default() - }, - children: vec![child], - } - } - - QueryExpr::BinaryOp { - op, - lhs, - rhs, - vector_match, - } => { - let left_node = self.alloc_node((*lhs).clone(), budget); - let right_node = self.alloc_node((*rhs).clone(), budget); - let has_sketch = left_node.mode == ExecutionMode::Sketch - || right_node.mode == ExecutionMode::Sketch; - let stage = if has_sketch { - PipelineStage::Precompute - } else { - PipelineStage::Db - }; - let rationale = format!("BinaryOp at {stage}"); - PlanNode { - expr: QueryExpr::BinaryOp { - op, - vector_match, - lhs: Rc::new(left_node.expr.clone()), - rhs: Rc::new(right_node.expr.clone()), - }, - stage, - mode: if has_sketch { - ExecutionMode::Sketch - } else { - ExecutionMode::Exact - }, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale, - ..Default::default() - }, - children: vec![left_node, right_node], - } - } - - // `LetBinding`/`Ref` don't exist in the canonical `QueryExpr` - // anymore; canonical rewrite ownership lives in ASAPPlanner. - - // `asap_ir`'s PromQL-surface superset (Scalar / EvalTime / - // VectorFromScalar / ScalarFromVector / Relabel / InfoJoin / - // Sample / TimeRange / TimeShift / WindowFunc) — not yet - // targeted by a dedicated allocation rule in this deployment. - // Leaves stay informational Agent leaves; every other new - // variant wraps exactly one child, recursed into and staged - // as an Agent passthrough (mirroring `Filter`/`Window` above) - // until a real rule is written for them. - QueryExpr::PromqlScalarBridge(_) | QueryExpr::EvalTimestamp => { - PlanNode::leaf(expr, PipelineStage::Agent, ExecutionMode::Passthrough) - } - QueryExpr::PromqlVectorFromScalar(child) => { - self.alloc_passthrough_child(child, "VectorFromScalar", budget, |c| { - QueryExpr::PromqlVectorFromScalar(Rc::new(c)) - }) - } - QueryExpr::PromqlScalarFromVector(child) => { - self.alloc_passthrough_child(child, "ScalarFromVector", budget, |c| { - QueryExpr::PromqlScalarFromVector(Rc::new(c)) - }) - } - QueryExpr::PromqlRelabel { dst, value, child } => { - self.alloc_passthrough_child(child, "Relabel", budget, |c| { - QueryExpr::PromqlRelabel { - dst, - value, - child: Rc::new(c), - } - }) - } - QueryExpr::PromqlInfoEnrich { selector, child } => { - self.alloc_passthrough_child(child, "InfoJoin", budget, |c| { - QueryExpr::PromqlInfoEnrich { - selector, - child: Rc::new(c), - } - }) - } - QueryExpr::PromqlSeriesSample { by, kind, child } => { - self.alloc_passthrough_child(child, "Sample", budget, |c| { - QueryExpr::PromqlSeriesSample { - by, - kind, - child: Rc::new(c), - } - }) - } - QueryExpr::TimeShift { shift, child } => { - self.alloc_passthrough_child(child, "TimeShift", budget, |c| QueryExpr::TimeShift { - shift, - child: Rc::new(c), - }) - } - QueryExpr::SQLWindowFunc { - func, - args, - partition_by, - order_by, - output_name, - frame, - child, - } => self.alloc_passthrough_child(child, "WindowFunc", budget, |c| { - QueryExpr::SQLWindowFunc { - func, - args, - partition_by, - order_by, - output_name, - frame, - child: Rc::new(c), - } - }), - // Scalar-expression node (Column/Literal/Compare/BoolAnd/ - // BoolOr/Not/IsNull/IsNotNull/Cast/InList/FunctionCall/Arith/ - // Case) — these live inside `Predicate`/`ProjectItem` scalar - // positions this deployment's front end builds, never as a - // bare top-level relational tree node reaching this - // allocator directly (folded into `QueryExpr` itself by - // ASAPPlanner#205/#214, see - // control_plane/docs/design-asapplanner-pin-migration.md). - // Defensive leaf fallback, matching `Scan`'s treatment, - // rather than a panic, in case that assumption ever breaks. - _ => PlanNode::leaf(expr, PipelineStage::Agent, ExecutionMode::Passthrough), - } - } - - /// Shared body for the single-child PromQL-surface passthrough arms: - /// recurse into `child`, rebuild the node via `rebuild`, and stage it - /// as an informational Agent passthrough carrying the child's cost. - fn alloc_passthrough_child( - &self, - child: Rc, - label: &'static str, - budget: &mut BudgetState, - rebuild: impl FnOnce(QueryExpr) -> QueryExpr, - ) -> PlanNode { - let child_node = self.alloc_node((*child).clone(), budget); - PlanNode { - expr: rebuild(child_node.expr.clone()), - stage: PipelineStage::Agent, - mode: ExecutionMode::Passthrough, - cost: child_node.cost.clone(), - annotation: NodeAnnotation { - rationale: format!("{label} at Agent (informational passthrough)"), - ..Default::default() - }, - children: vec![child_node], - } - } - - // ── Single-intent Aggregate allocation (budget-driven demotion) ─────────── - - /// Allocate a single-intent, no-HAVING `Aggregate` — the canonical - /// shape the legacy `SketchAgg` / `WindowedAgg`-inner-agg folded into. - /// The caller (the `Aggregate` arm of [`Self::alloc_node`]) guarantees - /// `aggs.len() == 1` and that the single intent is not `TopK`. - fn alloc_sketch_agg( - &self, - reduction: Reduction, - aggs: Vec, - output_names: Vec, - child: PlanNode, - budget: &mut BudgetState, - ) -> PlanNode { - let intent = aggs[0].clone(); - - // Exact non-mergeable (Avg) → always Db. - if matches!(intent, AggIntent::Avg { .. }) { - return PlanNode { - expr: QueryExpr::Aggregate { - reduction, - measures: aggs, - output_names, - having: None, - child: Rc::new(child.expr.clone()), - }, - stage: PipelineStage::Db, - mode: ExecutionMode::Exact, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec, - ..Default::default() - }, - annotation: NodeAnnotation { - rationale: "Avg is not mergeable — must run at Db".into(), - ..Default::default() - }, - children: vec![child], - }; - } - - // Exact mergeable (Sum, Count, Min, Max) → Backend. - if agg_is_exact(&intent) { - return PlanNode { - expr: QueryExpr::Aggregate { - reduction, - measures: aggs, - output_names, - having: None, - child: Rc::new(child.expr.clone()), - }, - stage: PipelineStage::Backend, - mode: ExecutionMode::Exact, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec * 0.8, - ..Default::default() - }, - annotation: NodeAnnotation { - rationale: "Exact(Sum/Count/Min/Max) merged at Backend".into(), - ..Default::default() - }, - children: vec![child], - }; - } - - // Sketch operators: resolve to physical, then try - // Agent → Backend → Precompute. - let physical = super::planner::resolve(&intent); - let mem = estimated_sketch_memory(&intent); - let (sketch_type, params) = (physical.sketch_type, physical.sketch_params); - - if budget.fits_agent(mem) { - budget.consume_agent(mem); - return PlanNode { - expr: QueryExpr::Aggregate { - reduction, - measures: aggs, - output_names, - having: None, - child: Rc::new(child.expr.clone()), - }, - stage: PipelineStage::Agent, - mode: ExecutionMode::Sketch, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec * 0.05, - memory_bytes: mem, - compression_ratio: 20.0, - ..Default::default() - }, - annotation: NodeAnnotation { - sketch_type: Some(sketch_type), - sketch_params: Some(params), - rationale: "Sketch at Agent (within budget)".into(), - ..Default::default() - }, - children: vec![child], - }; - } - - if budget.fits_backend(mem) { - budget.consume_backend(mem); - return PlanNode { - expr: QueryExpr::Aggregate { - reduction, - measures: aggs, - output_names, - having: None, - child: Rc::new(child.expr.clone()), - }, - stage: PipelineStage::Backend, - mode: ExecutionMode::Sketch, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec * 0.1, - memory_bytes: mem, - compression_ratio: 10.0, - ..Default::default() - }, - annotation: NodeAnnotation { - sketch_type: Some(sketch_type), - sketch_params: Some(params), - rationale: "Sketch demoted to Backend (Agent budget exceeded)".into(), - budget_demotion: true, - ..Default::default() - }, - children: vec![child], - }; - } - - // Both Agent and Backend budgets exceeded → Precompute. - PlanNode { - expr: QueryExpr::Aggregate { - reduction, - measures: aggs, - output_names, - having: None, - child: Rc::new(child.expr.clone()), - }, - stage: PipelineStage::Precompute, - mode: ExecutionMode::Sketch, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec * 0.2, - memory_bytes: mem, - compression_ratio: 5.0, - ..Default::default() - }, - annotation: NodeAnnotation { - sketch_type: Some(sketch_type), - sketch_params: Some(params), - rationale: "Sketch demoted to Precompute (Agent+Backend budgets exceeded)".into(), - budget_demotion: true, - ..Default::default() - }, - children: vec![child], - } - } -} - -// ── Helpers ─────────────────────────────────────────────────────────────────── - -/// Estimate the memory footprint of a sketch in bytes. -fn estimated_sketch_memory(op: &AggIntent) -> f64 { - super::sketch_catalog::estimated_sketch_memory_bytes(op) as f64 -} - -/// Map a canonical [`AggIntent`] to a short stable kind string for -/// annotation rationale text. -fn canonical_intent_kind_str(intent: &AggIntent) -> &'static str { - if crate::planner_selection::as_frequency(intent).is_some() { - return "frequency"; - } - match intent { - AggIntent::Count { .. } => "count", - AggIntent::Sum { .. } => "sum", - AggIntent::Min { .. } => "min", - AggIntent::Max { .. } => "max", - AggIntent::Avg { .. } => "avg", - AggIntent::StdDev { .. } => "stddev", - AggIntent::Variance { .. } => "variance", - AggIntent::Quantile { .. } => "quantile", - AggIntent::TopK { .. } => "topk", - AggIntent::Cardinality { .. } => "cardinality", - AggIntent::FrequencyL2 { .. } => "frequency_l2", - AggIntent::FrequencyEntropy { .. } => "frequency_entropy", - AggIntent::Rate => "rate", - AggIntent::IRate => "irate", - AggIntent::Increase => "increase", - AggIntent::Absent => "absent", - AggIntent::AbsentOverTime => "absent_over_time", - AggIntent::PresentOverTime => "present_over_time", - AggIntent::Delta => "delta", - AggIntent::Deriv => "deriv", - AggIntent::PredictLinear { .. } => "predict_linear", - AggIntent::DoubleExpSmoothing { .. } => "double_exponential_smoothing", - AggIntent::IDelta => "idelta", - AggIntent::Resets => "resets", - AggIntent::Changes => "changes", - AggIntent::HistogramCount => "histogram_count", - AggIntent::HistogramSum => "histogram_sum", - AggIntent::HistogramAvg => "histogram_avg", - AggIntent::HistogramStdDev => "histogram_stddev", - AggIntent::HistogramStdVar => "histogram_stdvar", - AggIntent::HistogramFraction { .. } => "histogram_fraction", - AggIntent::HistogramQuantile { .. } => "histogram_quantile", - AggIntent::Math(_) => "math", - AggIntent::TimeFn(_) => "time_fn", - AggIntent::Group => "group", - AggIntent::CountValues { .. } => "count_values", - AggIntent::LastOverTime => "last_over_time", - AggIntent::FirstOverTime => "first_over_time", - AggIntent::MadOverTime => "mad_over_time", - AggIntent::TsOfMinOverTime => "ts_of_min_over_time", - AggIntent::TsOfMaxOverTime => "ts_of_max_over_time", - AggIntent::TsOfFirstOverTime => "ts_of_first_over_time", - AggIntent::TsOfLastOverTime => "ts_of_last_over_time", - // Unrecognized Extension (not the Frequency one, guarded above). - AggIntent::Extension { .. } => "extension", - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::physical::plan::{ExecutionMode, PipelineStage}; - use crate::planner_selection::default_frequency; - use crate::types::{SketchType, StageResourceBudgets}; - use crate::types_v2::AccuracyTarget; - use planner_types::pre_asap::{default_cardinality, default_quantile}; - use planner_types::pre_asap::{JoinKind, Predicate, QueryExpr, ScalarValue, Schema, Source}; - - /// Canonical `Scan` leaf — the L3 counterpart of the legacy - /// `QueryExpr::Source(SourceSpec { .. })`. - fn scan(name: &str) -> QueryExpr { - QueryExpr::Scan { - source: Source::TimeSeries { - metric: name.into(), - }, - predicates: vec![], - schema: Schema::default(), - } - } - - /// Single-intent, global (`by: []`), no-HAVING `Aggregate` over a - /// `Scan` — the canonical shape the legacy `SketchAgg` folded into. - fn agg(intent: AggIntent) -> QueryExpr { - QueryExpr::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![intent], - output_names: Vec::new(), - having: None, - child: Rc::new(scan("m")), - } - } - - fn alloc(budgets: StageResourceBudgets, expr: QueryExpr) -> PlanNode { - SketchAllocator::new(budgets, 100_000.0).allocate(expr) - } - - fn unlimited() -> StageResourceBudgets { - StageResourceBudgets::default() - } - - fn tight_agent() -> StageResourceBudgets { - StageResourceBudgets { - agent_memory_bytes: Some(1), // 1 byte — too small for any sketch - ..Default::default() - } - } - - fn tight_all() -> StageResourceBudgets { - StageResourceBudgets { - agent_memory_bytes: Some(1), - backend_memory_bytes: Some(1), - ..Default::default() - } - } - - // ── Scan / leaf ─────────────────────────────────────────────────────────── - - #[test] - fn scan_goes_to_agent() { - let node = alloc(unlimited(), scan("cpu")); - assert_eq!(node.stage, PipelineStage::Agent); - assert_eq!(node.mode, ExecutionMode::Passthrough); - } - - // ── Filter ──────────────────────────────────────────────────────────────── - - #[test] - fn filter_at_agent() { - let expr = QueryExpr::Filter { - pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Boolean(true)))), - child: Rc::new(scan("m")), - }; - let node = alloc(unlimited(), expr); - assert_eq!(node.stage, PipelineStage::Agent); - } - - // ── DDSketch within budget → Agent ──────────────────────────────────────── - - #[test] - fn ddsketch_within_budget_goes_to_agent() { - let node = alloc(unlimited(), agg(default_quantile(0.99))); - assert_eq!(node.stage, PipelineStage::Agent); - assert_eq!(node.mode, ExecutionMode::Sketch); - assert_eq!(node.annotation.sketch_type, Some(SketchType::DDSketch)); - } - - // ── DDSketch tight agent budget → Backend ───────────────────────────────── - - #[test] - fn ddsketch_agent_budget_exceeded_goes_to_backend() { - let node = alloc(tight_agent(), agg(default_quantile(0.99))); - assert_eq!(node.stage, PipelineStage::Backend); - assert!(node.annotation.budget_demotion); - } - - // ── DDSketch tight agent+backend → Precompute ───────────────────────────── - - #[test] - fn ddsketch_all_budgets_exceeded_goes_to_precompute() { - let node = alloc(tight_all(), agg(default_quantile(0.99))); - assert_eq!(node.stage, PipelineStage::Precompute); - assert!(node.annotation.budget_demotion); - } - - // ── Exact(Avg) → Db ─────────────────────────────────────────────────────── - - #[test] - fn exact_avg_goes_to_db() { - let node = alloc(unlimited(), agg(AggIntent::Avg { col: None })); - assert_eq!(node.stage, PipelineStage::Db); - assert_eq!(node.mode, ExecutionMode::Exact); - } - - // ── Exact(Sum) → Backend ────────────────────────────────────────────────── - - #[test] - fn exact_sum_goes_to_backend() { - let node = alloc(unlimited(), agg(AggIntent::Sum { col: None })); - assert_eq!(node.stage, PipelineStage::Backend); - assert_eq!(node.mode, ExecutionMode::Exact); - } - - // ── TopK → Precompute ───────────────────────────────────────────────────── - - #[test] - fn topk_goes_to_precompute() { - let node = alloc( - unlimited(), - agg(AggIntent::TopK { - k: 10, - accuracy: AccuracyTarget::Epsilon(0.05), - }), - ); - assert_eq!(node.stage, PipelineStage::Precompute); - assert_eq!(node.annotation.sketch_type, Some(SketchType::CountSketch)); - } - - // ── Merge → Backend ─────────────────────────────────────────────────────── - - #[test] - fn merge_goes_to_backend() { - let expr = QueryExpr::Concat { - children: vec![scan("a"), scan("b")], - discriminator_unique_key: None, - }; - let node = alloc(unlimited(), expr); - assert_eq!(node.stage, PipelineStage::Backend); - } - - // ── HLL → Agent ─────────────────────────────────────────────────────────── - - #[test] - fn hll_within_budget_at_agent() { - let node = alloc(unlimited(), agg(default_cardinality())); - assert_eq!(node.stage, PipelineStage::Agent); - assert_eq!(node.annotation.sketch_type, Some(SketchType::HLL)); - } - - // ── Frequency → Agent ───────────────────────────────────────────────────── - - #[test] - fn frequency_within_budget_at_agent() { - let node = alloc(unlimited(), agg(default_frequency())); - assert_eq!(node.stage, PipelineStage::Agent); - assert_eq!(node.annotation.sketch_type, Some(SketchType::CountSketch)); - } - - // ── Join → Db ───────────────────────────────────────────────────────────── - - #[test] - fn join_goes_to_db() { - let expr = QueryExpr::Join { - kind: JoinKind::Inner, - pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Boolean(true)))), - left: Rc::new(scan("orders")), - right: Rc::new(scan("items")), - }; - let node = alloc(unlimited(), expr); - assert_eq!(node.stage, PipelineStage::Db); - } - - // ── Multi-intent Aggregate → Db (exact) ─────────────────────────────────── - - #[test] - fn multi_intent_aggregate_goes_to_db() { - let expr = QueryExpr::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::Sum { col: None }, AggIntent::Min { col: None }], - output_names: Vec::new(), - having: None, - child: Rc::new(scan("m")), - }; - let node = alloc(unlimited(), expr); - assert_eq!(node.stage, PipelineStage::Db); - assert_eq!(node.mode, ExecutionMode::Exact); - } - - // ── TimeRange over a single-intent Aggregate (the WindowedAgg fold) ─────── - - #[test] - fn window_over_aggregate_window_passthrough_agg_sketches() { - // Canonical fold of legacy `WindowedAgg`: TimeRange passthrough at - // Agent, inner Aggregate does the sketch placement. - let expr = QueryExpr::TimeRange { - range: std::time::Duration::from_secs(300), - child: Rc::new(agg(default_quantile(0.5))), - }; - let node = alloc(unlimited(), expr); - assert_eq!(node.stage, PipelineStage::Agent); - assert_eq!(node.mode, ExecutionMode::Passthrough); - assert_eq!(node.children.len(), 1); - assert_eq!(node.children[0].stage, PipelineStage::Agent); - assert_eq!(node.children[0].mode, ExecutionMode::Sketch); - } - - // `LetBinding` doesn't exist in the canonical `QueryExpr` anymore - // (canonical rewrite ownership lives in ASAPPlanner) -- the - // `let_binding_inherits_body_stage` test that used to exercise it - // is gone with it. - - // ── Memory estimate helpers ─────────────────────────────────────────────── - - #[test] - fn cardinality_memory_estimate() { - let mem = estimated_sketch_memory(&default_cardinality()); - assert!(mem > 0.0); - } - - #[test] - fn frequency_memory_estimate() { - let mem = estimated_sketch_memory(&default_frequency()); - assert!(mem > 0.0); - } - - // ── PlanSummary from allocated tree ────────────────────────────────────── - - #[test] - fn plan_summary_shows_bandwidth_saved() { - let node = alloc(unlimited(), agg(default_quantile(0.99))); - let summary = node.summarise(100_000.0); - // sketch reduces to ~5% → saved ~95 000 B/s - assert!(summary.bandwidth_saved_bytes_per_sec > 50_000.0); - assert!(summary.agent_memory_bytes > 0.0); - } -} diff --git a/control_plane/src/physical/deployment.rs b/control_plane/src/physical/deployment.rs deleted file mode 100644 index 85de54c57..000000000 --- a/control_plane/src/physical/deployment.rs +++ /dev/null @@ -1,67 +0,0 @@ -//! Physical deployment constraints used by CTSA stage placement. - -use crate::physical::deployment_cost::sketch_capability::{ - default_capability_table, SketchCapability, -}; - -/// Built-in physical resource profile for a sketch implementation. -pub fn sketch_capability(st: &crate::types::SketchType) -> SketchCapability { - use planner_types::post_asap::SketchAlgorithm; - let kind: SketchAlgorithm = st.clone().into(); - default_capability_table() - .remove(&kind) - .expect("the physical capability table covers every sketch algorithm") -} - -/// Resource budget for a single physical pipeline stage. -#[derive(Debug, Clone, Default)] -pub struct StageBudget { - pub memory_bytes: Option, - pub cpu_micros_per_sample: Option, - pub disk_bytes: Option, - pub bandwidth_bytes_per_sec: Option, -} - -impl StageBudget { - pub fn fits(&self, capability: &SketchCapability) -> bool { - self.memory_bytes - .is_none_or(|limit| capability.memory_bytes_per_series <= limit) - && self - .cpu_micros_per_sample - .is_none_or(|limit| capability.cpu_micros_per_insert <= limit) - && self - .bandwidth_bytes_per_sec - .is_none_or(|limit| capability.transmission_bytes as f64 <= limit) - } -} - -/// Resource constraints for physical CTSA placement targets. -#[derive(Debug, Clone, Default)] -pub struct DeploymentConstraints { - pub agent: StageBudget, - pub backend_collector: StageBudget, - pub backend_db: StageBudget, - pub original_db: StageBudget, - pub object_store: StageBudget, -} - -impl DeploymentConstraints { - pub fn from_budgets(budgets: &crate::types::StageResourceBudgets) -> Self { - Self { - agent: StageBudget { - memory_bytes: budgets.agent_memory_bytes, - cpu_micros_per_sample: budgets.agent_cpu_micros_per_sample, - ..Default::default() - }, - backend_collector: StageBudget { - memory_bytes: budgets.backend_memory_bytes, - ..Default::default() - }, - backend_db: StageBudget { - memory_bytes: budgets.precompute_memory_bytes, - ..Default::default() - }, - ..Default::default() - } - } -} diff --git a/control_plane/src/physical/mod.rs b/control_plane/src/physical/mod.rs index f26665fb9..3bd2411b8 100644 --- a/control_plane/src/physical/mod.rs +++ b/control_plane/src/physical/mod.rs @@ -1,36 +1,12 @@ -//! L5 — **physical execution plan** framework. -//! -//! Per `control_plane/docs/design.md` §3/§5/§6 `core::physical`: this is -//! the stage allocator + sketch catalogue + physical planner surface. -//! Refactor 2026-05 (`refactor/controller-layered-cleanup`) absorbed -//! the former `controller/src/algebra/{allocator,physical,plan}.rs` -//! + `controller/src/algebra/directory.rs` (renamed to -//! [`sketch_catalog`]) + the L5 colored-DAG framework formerly at -//! `controller/src/stage_split/` ([`colored_dag`] subdir) here. -//! -//! Module layout: -//! -//! | File | Role | -//! |---|---| -//! | [`allocator`] | `SketchAllocator` — assigns physical ops to pipeline stages over the legacy [`planner_types::pre_asap::QueryExpr`] IR | -//! | [`planner`] | `PhysicalPlanner` — `QueryExpr` → staged sub-plans (DC three-stage emission) | -//! | [`plan`] | `PlanNode` / `PlanSummary` — annotated plan tree with cost estimates + stage colouring | -//! | [`sketch_catalog`] | Candidate sketch types per `AggIntent`, default `SketchParams`, memory estimation | -//! | [`stage_split`] | AST-aware hierarchical stage assignment (legacy `QueryExpr` splitter) | -//! | [`colored_dag`] | L5 typed colouring framework over the canonical [`crate::physical::post_asap::PhysicalExpr`] DAG (`StageId` + `Topology` + per-stage emitter) | -//! | [`topology`] | Re-exports `StageId` + `Topology` from [`colored_dag::stage_id`] — design.md §5 entry point for deployment-topology descriptors | +//! Physical compilation, deployment costs, sketch capabilities, and stage emission. -pub mod allocator; pub mod colored_dag; pub mod compiler; -pub mod deployment; pub mod deployment_cost; pub mod erp; pub mod executable_binding; mod pane_reuse; -pub mod plan; pub mod plan_cache; -pub mod planner; pub mod post_asap; pub(crate) mod realization; pub mod runtime_capability; @@ -39,13 +15,7 @@ pub mod stage_split; pub mod summary_catalog; pub mod summary_reconcile; pub mod topology; -pub mod window_fusion; pub mod workload_cost; pub mod workload_planner; -// Convenience re-exports — preserve the surface that consumers of the -// former `algebra` module relied on. -pub use allocator::SketchAllocator; -pub use plan::{CostEstimate, ExecutionMode, PipelineStage, PlanNode, PlanSummary}; - pub mod publication; diff --git a/control_plane/src/physical/plan.rs b/control_plane/src/physical/plan.rs deleted file mode 100644 index f17f5eea4..000000000 --- a/control_plane/src/physical/plan.rs +++ /dev/null @@ -1,447 +0,0 @@ -//! Annotated plan nodes — the output of the [`super::allocator::SketchAllocator`]. -//! -//! After the optimizer rewrites a [`QueryExpr`](planner_types::pre_asap::QueryExpr) tree, -//! the allocator wraps every node in a [`PlanNode`] that carries: -//! -//! * **`stage`** — which pipeline component executes this operator. -//! * **`mode`** — whether the operator uses sketch approximation or exact -//! computation. -//! * **`cost`** — estimated memory and bandwidth cost at this node. -//! * **`annotation`** — additional hints for the code-generator (e.g. which -//! sketch type to use, whether delta encoding is enabled). -//! -//! The annotated plan tree is serialisable to JSON so it can be included in -//! the `/api/v1/plan` response for observability. - -use serde::{Deserialize, Serialize}; - -use planner_types::pre_asap::QueryExpr; - -// ── Pipeline stages ─────────────────────────────────────────────────────────── - -/// Which component in the data pipeline executes an operator. -/// -/// The ordering `Agent < Backend < Precompute < Db` mirrors the data-flow -/// direction: data originates at the Agent and flows toward the Db. -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum PipelineStage { - /// SDK-side OTel Collector — highest bandwidth savings, lowest latency. - Agent, - /// Central merge collector — aggregates partial sketches from many agents. - Backend, - /// ASAPQuery pre-computation engine — materialises recurring queries. - Precompute, - /// Exact OLAP / time-series database — last resort for non-sketchable ops. - Db, -} - -impl std::fmt::Display for PipelineStage { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let s = match self { - PipelineStage::Agent => "agent", - PipelineStage::Backend => "backend", - PipelineStage::Precompute => "precompute", - PipelineStage::Db => "db", - }; - write!(f, "{s}") - } -} - -// ── Execution mode ──────────────────────────────────────────────────────────── - -/// Whether an operator uses sketch approximation or runs exactly. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ExecutionMode { - /// The operator produces an approximate result via a data sketch. - Sketch, - /// The operator computes an exact result (no error bounds). - Exact, - /// The operator is a structural / routing node (merge, partition, …) - /// that does not itself aggregate — its mode is determined by its children. - Passthrough, -} - -// ── Cost estimate ───────────────────────────────────────────────────────────── - -/// Estimated resource cost of a single plan node. -/// -/// The allocator fills this in using the same cost model as the legacy -/// [`crate::planner::cost_model`]. All fields default to `0.0` for nodes -/// whose cost is negligible or unknown. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct CostEstimate { - /// Outbound bytes per second produced by this node. - pub bytes_per_sec: f64, - /// Memory footprint of the sketch or intermediate state (bytes). - pub memory_bytes: f64, - /// CPU overhead per input sample (µs). - pub cpu_micros_per_sample: f64, - /// Compression ratio relative to the raw OTLP baseline (≥ 1.0 is better). - pub compression_ratio: f64, -} - -// ── Node annotation ─────────────────────────────────────────────────────────── - -/// Extra hints attached to a plan node by the allocator. -/// -/// Not all fields are relevant to all node types; unused fields are `None`. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct NodeAnnotation { - /// Sketch type selected by the allocator (only for sketch nodes). - pub sketch_type: Option, - /// Sketch parameters (width/depth/registers/epsilon). - pub sketch_params: Option, - /// Whether delta encoding should be used at this node. - pub delta_enabled: bool, - /// Minimum cell-change threshold for delta encoding (T). - pub delta_threshold: f64, - /// Human-readable explanation of why this stage/mode was chosen. - pub rationale: String, - /// Whether this node was demoted to a later stage due to budget overflow. - pub budget_demotion: bool, -} - -// ── Plan node ───────────────────────────────────────────────────────────────── - -/// An annotated node in the physical execution plan. -/// -/// The `expr` field holds the logical operator; the surrounding fields -/// describe where and how it runs. -#[derive(Debug, Clone)] -pub struct PlanNode { - /// The logical operator at this node. - pub expr: QueryExpr, - /// Which pipeline stage executes this operator. - pub stage: PipelineStage, - /// Sketch vs. exact vs. passthrough. - pub mode: ExecutionMode, - /// Estimated resource cost. - pub cost: CostEstimate, - /// Allocator hints for code-generation. - pub annotation: NodeAnnotation, - /// Child plan nodes (mirrors `expr`'s children after annotation). - pub children: Vec, -} - -impl PlanNode { - /// Create a leaf `PlanNode` (no children) with default cost/annotation. - pub fn leaf(expr: QueryExpr, stage: PipelineStage, mode: ExecutionMode) -> Self { - Self { - expr, - stage, - mode, - cost: CostEstimate::default(), - annotation: NodeAnnotation::default(), - children: vec![], - } - } - - /// Recursively collect all nodes at a given stage, depth-first. - pub fn nodes_at_stage(&self, target: &PipelineStage) -> Vec<&PlanNode> { - let mut out = vec![]; - if &self.stage == target { - out.push(self); - } - for c in &self.children { - out.extend(c.nodes_at_stage(target)); - } - out - } - - /// Recursively collect all sketch nodes (mode == Sketch). - pub fn sketch_nodes(&self) -> Vec<&PlanNode> { - let mut out = vec![]; - if self.mode == ExecutionMode::Sketch { - out.push(self); - } - for c in &self.children { - out.extend(c.sketch_nodes()); - } - out - } - - /// Total estimated bandwidth of all nodes at `stage` (bytes/sec). - pub fn stage_bandwidth(&self, stage: &PipelineStage) -> f64 { - self.nodes_at_stage(stage) - .iter() - .map(|n| n.cost.bytes_per_sec) - .sum() - } - - /// Total estimated memory of all nodes at `stage` (bytes). - pub fn stage_memory(&self, stage: &PipelineStage) -> f64 { - self.nodes_at_stage(stage) - .iter() - .map(|n| n.cost.memory_bytes) - .sum() - } - - /// Returns a flat, depth-first list of `(depth, node)` pairs for display. - pub fn flatten(&self) -> Vec<(usize, &PlanNode)> { - let mut out = vec![]; - self.flatten_inner(0, &mut out); - out - } - - fn flatten_inner<'a>(&'a self, depth: usize, out: &mut Vec<(usize, &'a PlanNode)>) { - out.push((depth, self)); - for c in &self.children { - c.flatten_inner(depth + 1, out); - } - } -} - -// ── Plan summary (serialisable) ─────────────────────────────────────────────── - -/// A serialisable summary of the full annotated plan, suitable for inclusion -/// in the `/api/v1/plan` JSON response. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct PlanSummary { - /// Total estimated bandwidth saved vs raw OTLP (bytes/sec). - pub bandwidth_saved_bytes_per_sec: f64, - /// Total estimated agent memory for all sketch nodes (bytes). - pub agent_memory_bytes: f64, - /// Total estimated backend memory for all sketch nodes (bytes). - pub backend_memory_bytes: f64, - /// Whether any node was demoted due to budget overflow. - pub has_budget_demotion: bool, - /// List of per-node stage + mode + rationale entries. - pub node_annotations: Vec, -} - -/// One row in the [`PlanSummary::node_annotations`] table. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct NodeSummaryEntry { - pub node_kind: String, - pub stage: PipelineStage, - pub mode: ExecutionMode, - pub rationale: String, - pub memory_bytes: f64, - pub bytes_per_sec: f64, -} - -impl PlanNode { - /// Build a [`PlanSummary`] from this root node. - pub fn summarise(&self, raw_bytes_per_sec: f64) -> PlanSummary { - let flat = self.flatten(); - let agent_mem: f64 = flat - .iter() - .filter(|(_, n)| n.stage == PipelineStage::Agent) - .map(|(_, n)| n.cost.memory_bytes) - .sum(); - let backend_mem: f64 = flat - .iter() - .filter(|(_, n)| n.stage == PipelineStage::Backend) - .map(|(_, n)| n.cost.memory_bytes) - .sum(); - let plan_bw: f64 = flat - .iter() - .filter(|(_, n)| matches!(n.stage, PipelineStage::Agent | PipelineStage::Backend)) - .map(|(_, n)| n.cost.bytes_per_sec) - .fold(f64::INFINITY, f64::min); // min of outbound paths - let saved = if raw_bytes_per_sec > plan_bw { - raw_bytes_per_sec - plan_bw - } else { - 0.0 - }; - let has_demotion = flat.iter().any(|(_, n)| n.annotation.budget_demotion); - let entries = flat - .iter() - .map(|(_, n)| NodeSummaryEntry { - node_kind: format!("{:?}", n.expr) - .split_whitespace() - .next() - .unwrap_or("?") - .to_string(), - stage: n.stage.clone(), - mode: n.mode.clone(), - rationale: n.annotation.rationale.clone(), - memory_bytes: n.cost.memory_bytes, - bytes_per_sec: n.cost.bytes_per_sec, - }) - .collect(); - PlanSummary { - bandwidth_saved_bytes_per_sec: saved, - agent_memory_bytes: agent_mem, - backend_memory_bytes: backend_mem, - has_budget_demotion: has_demotion, - node_annotations: entries, - } - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use planner_types::pre_asap::{QueryExpr, Schema, Source}; - - /// Canonical `Scan` leaf — the L3 counterpart of the legacy - /// `QueryExpr::Source(SourceSpec { .. })`. - fn scan(name: &str) -> QueryExpr { - QueryExpr::Scan { - source: Source::TimeSeries { - metric: name.into(), - }, - predicates: vec![], - schema: Schema::default(), - } - } - - fn source_node(name: &str, stage: PipelineStage) -> PlanNode { - PlanNode::leaf(scan(name), stage, ExecutionMode::Passthrough) - } - - // ── PipelineStage ordering ──────────────────────────────────────────────── - - #[test] - fn stage_ordering_agent_lt_db() { - assert!(PipelineStage::Agent < PipelineStage::Db); - assert!(PipelineStage::Agent < PipelineStage::Backend); - assert!(PipelineStage::Backend < PipelineStage::Precompute); - assert!(PipelineStage::Precompute < PipelineStage::Db); - } - - // ── nodes_at_stage ──────────────────────────────────────────────────────── - - #[test] - fn nodes_at_stage_collects_correctly() { - let root = PlanNode { - expr: scan("root"), - stage: PipelineStage::Agent, - mode: ExecutionMode::Sketch, - cost: CostEstimate { - memory_bytes: 100.0, - ..Default::default() - }, - annotation: NodeAnnotation::default(), - children: vec![ - source_node("child_agent", PipelineStage::Agent), - source_node("child_backend", PipelineStage::Backend), - ], - }; - let agent_nodes = root.nodes_at_stage(&PipelineStage::Agent); - assert_eq!(agent_nodes.len(), 2); // root + child_agent - let backend_nodes = root.nodes_at_stage(&PipelineStage::Backend); - assert_eq!(backend_nodes.len(), 1); - } - - // ── sketch_nodes ───────────────────────────────────────────────────────── - - #[test] - fn sketch_nodes_only_returns_sketch_mode() { - let root = PlanNode { - expr: scan("r"), - stage: PipelineStage::Agent, - mode: ExecutionMode::Sketch, - cost: CostEstimate::default(), - annotation: NodeAnnotation::default(), - children: vec![ - PlanNode::leaf(scan("exact_child"), PipelineStage::Db, ExecutionMode::Exact), - PlanNode::leaf( - scan("sketch_child"), - PipelineStage::Backend, - ExecutionMode::Sketch, - ), - ], - }; - let sn = root.sketch_nodes(); - assert_eq!(sn.len(), 2); // root (Sketch) + sketch_child - } - - // ── stage_bandwidth / stage_memory ──────────────────────────────────────── - - #[test] - fn stage_bandwidth_sums_nodes_at_stage() { - let root = PlanNode { - expr: scan("r"), - stage: PipelineStage::Agent, - mode: ExecutionMode::Sketch, - cost: CostEstimate { - bytes_per_sec: 500.0, - ..Default::default() - }, - annotation: NodeAnnotation::default(), - children: vec![PlanNode { - expr: scan("c"), - stage: PipelineStage::Agent, - mode: ExecutionMode::Passthrough, - cost: CostEstimate { - bytes_per_sec: 200.0, - ..Default::default() - }, - annotation: NodeAnnotation::default(), - children: vec![], - }], - }; - assert!((root.stage_bandwidth(&PipelineStage::Agent) - 700.0).abs() < 1e-6); - } - - // ── flatten ─────────────────────────────────────────────────────────────── - - #[test] - fn flatten_returns_depth_zero_for_root() { - let root = source_node("r", PipelineStage::Agent); - let flat = root.flatten(); - assert_eq!(flat.len(), 1); - assert_eq!(flat[0].0, 0); // depth = 0 - } - - #[test] - fn flatten_depth_increments_per_level() { - let root = PlanNode { - expr: scan("r"), - stage: PipelineStage::Agent, - mode: ExecutionMode::Passthrough, - cost: CostEstimate::default(), - annotation: NodeAnnotation::default(), - children: vec![source_node("c1", PipelineStage::Backend)], - }; - let flat = root.flatten(); - assert_eq!(flat[0].0, 0); - assert_eq!(flat[1].0, 1); - } - - // ── PlanSummary ─────────────────────────────────────────────────────────── - - #[test] - fn summarise_reports_bandwidth_saved() { - let root = PlanNode { - expr: scan("r"), - stage: PipelineStage::Agent, - mode: ExecutionMode::Sketch, - cost: CostEstimate { - bytes_per_sec: 1_000.0, - memory_bytes: 256.0, - ..Default::default() - }, - annotation: NodeAnnotation::default(), - children: vec![], - }; - // Raw baseline is 10 000 B/s; plan reduces to 1 000 B/s → saved = 9 000. - let summary = root.summarise(10_000.0); - assert!((summary.bandwidth_saved_bytes_per_sec - 9_000.0).abs() < 1.0); - assert!((summary.agent_memory_bytes - 256.0).abs() < 1.0); - assert!(!summary.has_budget_demotion); - } - - #[test] - fn summarise_detects_budget_demotion() { - let root = PlanNode { - expr: scan("r"), - stage: PipelineStage::Backend, - mode: ExecutionMode::Sketch, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - budget_demotion: true, - ..Default::default() - }, - children: vec![], - }; - let summary = root.summarise(0.0); - assert!(summary.has_budget_demotion); - } -} diff --git a/control_plane/src/physical/planner.rs b/control_plane/src/physical/planner.rs deleted file mode 100644 index 9cd263bd1..000000000 --- a/control_plane/src/physical/planner.rs +++ /dev/null @@ -1,831 +0,0 @@ -//! Layer 5 — Physical plan IR. -//! -//! Maps the implementation-independent [`AggIntent`] (Layer 3) to concrete -//! sketch implementations and pipeline stages. -//! -//! # Key types -//! -//! - [`PhysicalAggOp`] — resolved AggIntent → concrete SketchType + SketchParams -//! - [`PhysicalOp`] — a physical operator (sketch build, merge, exchange, eval, etc.) -//! - [`PhysicalNode`] — a node in the physical plan tree (operator + placement + cost) -//! - [`Placement`] — where a physical operator runs (Agent, Backend, PromSketch, QueryEngine) -//! -//! Step γ7: the planner consumes the canonical `query_expr::QueryExpr`. -//! The legacy `SketchAgg` / `WindowedAgg` / `TopK` variants are gone — they -//! fold into canonical `Aggregate` / `Window { Aggregate }`. The single -//! `Aggregate` arm dispatches on shape, and the `Window` arm calls -//! [`crate::physical::window_fusion::recognize_windowed_sketch`] to detect -//! the canonical `Window { child: Aggregate }` fold of a legacy -//! `WindowedAgg` and reconstruct the fused `OtelSketchBuild { window }` -//! placement — the window-defines-sketch-lifecycle invariant, now a -//! planner peephole rather than an IR-shape property. - -use std::time::Duration; - -use crate::physical::sketch_catalog; -use crate::physical::window_fusion::{fused_sketch_decision, recognize_windowed_sketch}; -use crate::types::{SketchParams, SketchType}; -use planner_types::pre_asap::AggIntent; -use planner_types::pre_asap::ColumnId; -use planner_types::pre_asap::QueryExpr; - -// ── PhysicalAggOp (resolved sketch intent) ────────────────────────────────── - -/// A resolved physical aggregation operation. -/// -/// This is the output of `resolve()`: concrete sketch implementation -/// chosen for a logical [`AggIntent`]. -#[derive(Debug, Clone)] -pub struct PhysicalAggOp { - /// The logical intent this was derived from. - pub intent: AggIntent, - /// Concrete sketch type. - pub sketch_type: SketchType, - /// Concrete sketch parameters. - pub sketch_params: SketchParams, - /// Estimated memory footprint per series (bytes). - pub estimated_memory_bytes: u64, -} - -/// Resolve an [`AggIntent`] into a [`PhysicalAggOp`] using default mapping. -/// -/// This is the Layer 3 → Layer 5 boundary. -pub fn resolve(intent: &AggIntent) -> PhysicalAggOp { - PhysicalAggOp { - intent: intent.clone(), - sketch_type: sketch_catalog::sketch_type_for_op(intent), - sketch_params: sketch_catalog::sketch_params_for_op(intent), - estimated_memory_bytes: sketch_catalog::estimated_sketch_memory_bytes(intent), - } -} - -// ── Physical operators ────────────────────────────────────────────────────── - -/// A physical operator — concrete implementation of a logical operator. -#[derive(Debug, Clone)] -pub enum PhysicalOp { - // ── Scan / ingest ───────────────────────────────────────────── - /// Read raw OTLP metrics from an SDK or scrape target. - OtlpScan { - endpoint: String, - label_matchers: Vec, - }, - - /// Read from an existing PromSketch store. - PromSketchScan { - store_addr: String, - series_selector: String, - }, - - // ── Sketch build ────────────────────────────────────────────── - /// Build sketch via OTel Collector processor (tumbling window flush). - OtelSketchBuild { - sketch_type: SketchType, - sketch_params: SketchParams, - window: PhysicalWindow, - delta_encoding: bool, - }, - - /// Build sketch via PromSketch's ExponentialHistogram layer. - PromSketchBuild { - sketch_type: SketchType, - eh_k: usize, - time_window: Duration, - }, - - // ── Sketch merge ────────────────────────────────────────────── - /// Merge sketches from N upstream nodes. - SketchMerge { - sketch_type: SketchType, - group_by: Vec, - }, - - // ── Sketch query ────────────────────────────────────────────── - /// Extract result from a sketch (quantile, cardinality, frequency). - SketchEval { - sketch_type: SketchType, - func: EvalFunc, - }, - - // ── Data exchange ───────────────────────────────────────────── - /// Data transfer between pipeline stages. - Exchange { format: ExchangeFormat }, - - // ── Relational / passthrough ────────────────────────────────── - /// Filter rows. - Filter { pred: String }, - /// Top-K ranking. - TopK { k: u64 }, - /// Hash-partitioned aggregation. - HashAggregate { keys: Vec }, - /// Passthrough — no transformation. - Passthrough, -} - -/// Physical window implementation. -#[derive(Debug, Clone)] -pub enum PhysicalWindow { - /// OTel Collector: `time.NewTicker` flush + sketch reset. - OtelTumblingFlush { duration: Duration }, - /// PromSketch ExponentialHistogram: time-decaying buckets. - PromSketchEH { eh_k: usize, time_window: Duration }, - /// No windowing (unbounded / landmark). - None, -} - -/// What to extract from a sketch at query time. -#[derive(Debug, Clone)] -pub enum EvalFunc { - Quantile(Vec), - Cardinality, - Frequency { key: String }, - TopK { k: u64 }, - Extrema { min: bool, max: bool }, -} - -/// Data format for Exchange operators. -#[derive(Debug, Clone)] -pub enum ExchangeFormat { - /// OTLP gRPC / HTTP. - Otlp, - /// Sketch-specific binary (merged sketch bytes). - SketchBinary, -} - -/// Where a physical operator runs. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Placement { - /// Agent OTel Collector (co-located with SDK). - AgentCollector, - /// Backend OTel Collector (merge tier). - BackendCollector, - /// PromSketch store (ASAPQuery). - PromSketchStore, - /// General query engine (ASAPQuery). - QueryEngine, -} - -// ── Physical plan tree ────────────────────────────────────────────────────── - -/// A node in the physical plan tree. -#[derive(Debug, Clone)] -pub struct PhysicalNode { - /// The physical operator at this node. - pub op: PhysicalOp, - /// Where this operator runs. - pub placement: Placement, - /// Estimated cost. - pub cost: PhysicalCost, - /// Child nodes (ordered: left, right, or input list). - pub children: Vec, -} - -/// Cost estimate for a physical operator. -#[derive(Debug, Clone, Default)] -pub struct PhysicalCost { - /// Estimated output bandwidth (bytes/sec). - pub bytes_per_sec: f64, - /// Estimated memory usage (bytes). - pub memory_bytes: f64, - /// Estimated CPU cost (microseconds per sample). - pub cpu_per_sample: f64, -} - -// ── Physical planner ──────────────────────────────────────────────────────── - -use crate::physical::deployment::DeploymentConstraints; -use crate::types::StageResourceBudgets; - -/// Physical planner configuration. -#[derive(Debug, Clone)] -pub struct PhysicalPlannerConfig { - pub budgets: StageResourceBudgets, - pub constraints: DeploymentConstraints, -} - -/// Build a physical plan from an optimized canonical [`QueryExpr`]. -/// -/// Walks the logical tree bottom-up, assigning each node to a pipeline stage -/// (`Placement`), resolving sketch intents to concrete implementations, and -/// inserting `Exchange` nodes at stage boundaries. -/// -/// The canonical `Aggregate.by` is already positional, so — unlike the -/// legacy planner — no inherited `Schema` is threaded; placement is -/// purely structural. -pub fn plan(expr: &QueryExpr, config: &PhysicalPlannerConfig) -> PhysicalNode { - plan_node(expr, config) -} - -fn plan_node(expr: &QueryExpr, config: &PhysicalPlannerConfig) -> PhysicalNode { - match expr { - // ── Leaf: scan at Agent ───────────────────────────────────── - QueryExpr::Scan { .. } => PhysicalNode { - op: PhysicalOp::OtlpScan { - endpoint: String::new(), - label_matchers: vec![], - }, - placement: Placement::AgentCollector, - cost: PhysicalCost::default(), - children: vec![], - }, - - // ── Filter: same placement as child ───────────────────────── - QueryExpr::Filter { pred, child } => { - let child = plan_node(child, config); - PhysicalNode { - placement: child.placement.clone(), - op: PhysicalOp::Filter { - pred: format!("{pred:?}"), - }, - cost: PhysicalCost::default(), - children: vec![child], - } - } - - // ── TimeRange: a TimeRange over a single-intent Aggregate is the - // canonical fold of the legacy WindowedAgg (was `Window` before - // the ASAPPlanner pin migration) — recognize it and reconstruct - // the fused OtelSketchBuild { window } placement. Any other - // TimeRange is a plain passthrough inheriting the child's - // placement. - QueryExpr::TimeRange { - child: window_child, - .. - } => { - if let Some(fused) = recognize_windowed_sketch(expr) { - let child = plan_node(fused.inner_child, config); - let resolved = resolve(fused.agg); - let (op, placement) = fused_sketch_decision(&fused, config); - let mut node = PhysicalNode { - op, - placement, - cost: PhysicalCost { - memory_bytes: resolved.estimated_memory_bytes as f64, - ..Default::default() - }, - children: vec![child], - }; - insert_exchange_if_needed(&mut node); - node - } else { - let child = plan_node(window_child, config); - PhysicalNode { - op: PhysicalOp::Passthrough, - placement: child.placement.clone(), - cost: PhysicalCost::default(), - children: vec![child], - } - } - } - - // ── Aggregate: the canonical IR folds legacy SketchAgg / - // WindowedAgg-inner / TopK all into Aggregate, so dispatch on shape: - // * single TopK intent, no HAVING → TopK at QueryEngine - // * single other intent, no HAVING → sketch build, budget-placed - // * multi-intent or HAVING → exact HashAggregate at QueryEngine - QueryExpr::Aggregate { - reduction, - measures: aggs, - having, - child, - .. - } => { - if aggs.len() == 1 && having.is_none() { - if let AggIntent::TopK { k, .. } = &aggs[0] { - let k = *k as u64; - let child = plan_node(child, config); - let mut node = PhysicalNode { - op: PhysicalOp::TopK { k }, - placement: Placement::QueryEngine, - cost: PhysicalCost::default(), - children: vec![child], - }; - insert_exchange_if_needed(&mut node); - return node; - } - // Single non-TopK intent → sketch build (no window — a - // windowed sketch arrives as `Window { Aggregate }` and is - // handled by the `Window` arm above). - let child = plan_node(child, config); - let resolved = resolve(&aggs[0]); - let placement = decide_sketch_placement(&resolved, config); - let mut node = PhysicalNode { - op: PhysicalOp::OtelSketchBuild { - sketch_type: resolved.sketch_type.clone(), - sketch_params: resolved.sketch_params.clone(), - window: PhysicalWindow::None, - delta_encoding: false, - }, - placement, - cost: PhysicalCost { - memory_bytes: resolved.estimated_memory_bytes as f64, - ..Default::default() - }, - children: vec![child], - }; - insert_exchange_if_needed(&mut node); - return node; - } - // Multi-intent / HAVING aggregate → no single sketch can serve - // it; fall back to an exact hash aggregation at the query engine. - // Always a genuine reduction (never per-entity -- see - // `intent_algebra::lower`'s single-intent-only per-entity rule). - let by = reduction.expect_reduce(); - let child = plan_node(child, config); - let mut node = PhysicalNode { - op: PhysicalOp::HashAggregate { - keys: by.iter().map(|id| format!("{id:?}")).collect(), - }, - placement: Placement::QueryEngine, - cost: PhysicalCost::default(), - children: vec![child], - }; - insert_exchange_if_needed(&mut node); - node - } - - // ── Merge: Backend stage ───────────────────────────────────── - // - // `Partition` no longer exists in the canonical IR — its keys - // fold into `Aggregate.by` at construction time - // (`intent_algebra::lower`), so the `HashAggregate { keys }` this - // arm used to build now comes straight out of the `Aggregate` - // arm above. - QueryExpr::Concat { children, .. } => { - let children: Vec = - children.iter().map(|c| plan_node(c, config)).collect(); - let sketch_type = children - .first() - .and_then(|c| match &c.op { - PhysicalOp::OtelSketchBuild { sketch_type, .. } => Some(sketch_type.clone()), - _ => None, - }) - .unwrap_or(SketchType::DDSketch); - PhysicalNode { - op: PhysicalOp::SketchMerge { - sketch_type, - group_by: vec![], - }, - placement: Placement::BackendCollector, - cost: PhysicalCost::default(), - children, - } - } - - QueryExpr::Dedup { cols, child } => { - let child = plan_node(child, config); - let pred = format!("distinct({})", display_distinct_cols(cols)); - let mut node = PhysicalNode { - op: PhysicalOp::Filter { pred }, - placement: Placement::BackendCollector, - cost: PhysicalCost::default(), - children: vec![child], - }; - insert_exchange_if_needed(&mut node); - node - } - - // ── BinaryOp / Subquery: QueryEngine stage ────────────────── - QueryExpr::BinaryOp { lhs, rhs, .. } => { - let left = plan_node(lhs, config); - let right = plan_node(rhs, config); - PhysicalNode { - op: PhysicalOp::Passthrough, - placement: Placement::QueryEngine, - cost: PhysicalCost::default(), - children: vec![left, right], - } - } - - QueryExpr::PromqlSubquery { child, .. } => { - let child = plan_node(child, config); - let mut node = PhysicalNode { - op: PhysicalOp::Passthrough, - placement: Placement::QueryEngine, - cost: PhysicalCost::default(), - children: vec![child], - }; - insert_exchange_if_needed(&mut node); - node - } - - // ── Sort / Limit / Project: inherit child placement ───────── - QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::Project { child, .. } => { - let child = plan_node(child, config); - PhysicalNode { - op: PhysicalOp::Passthrough, - placement: child.placement.clone(), - cost: PhysicalCost::default(), - children: vec![child], - } - } - - // ── Join / SetOp: both children, QueryEngine placement ────── - QueryExpr::Join { left, right, .. } | QueryExpr::SetOp { left, right, .. } => { - let l = plan_node(left, config); - let r = plan_node(right, config); - PhysicalNode { - op: PhysicalOp::Passthrough, - placement: Placement::QueryEngine, - cost: PhysicalCost::default(), - children: vec![l, r], - } - } - - // The PromQL-surface superset (Scalar/EvalTime/VectorFromScalar/ - // ScalarFromVector/Relabel/InfoJoin/Sample/TimeShift/WindowFunc) - // isn't constructed by this parser today (`TimeRange` has its own - // arm above -- it *is* constructed). `Scalar` / `EvalTime` are - // leaves; every other new variant wraps exactly one child — - // inherit its placement, mirroring the Sort/Limit/Project arm - // above, until a dedicated physical op is written. - QueryExpr::PromqlScalarBridge(_) | QueryExpr::EvalTimestamp => PhysicalNode { - op: PhysicalOp::Passthrough, - placement: Placement::QueryEngine, - cost: PhysicalCost::default(), - children: vec![], - }, - QueryExpr::PromqlVectorFromScalar(child) - | QueryExpr::PromqlScalarFromVector(child) - | QueryExpr::PromqlRelabel { child, .. } - | QueryExpr::PromqlInfoEnrich { child, .. } - | QueryExpr::PromqlSeriesSample { child, .. } - | QueryExpr::TimeShift { child, .. } - | QueryExpr::SQLWindowFunc { child, .. } => { - let child = plan_node(child, config); - PhysicalNode { - op: PhysicalOp::Passthrough, - placement: child.placement.clone(), - cost: PhysicalCost::default(), - children: vec![child], - } - } - // Scalar-expression node (Column/Literal/Compare/BoolAnd/BoolOr/ - // Not/IsNull/IsNotNull/Cast/InList/FunctionCall/Arith/Case) -- - // see `physical::allocator`'s matching catch-all for why this - // never reaches here as a bare top-level node in practice. - _ => PhysicalNode { - op: PhysicalOp::Passthrough, - placement: Placement::QueryEngine, - cost: PhysicalCost::default(), - children: vec![], - }, - } -} - -/// Decide where a sketch runs based on deployment constraints and sketch capability. -/// -/// Uses `StageBudget::fits(SketchCapability)` to check each stage in order: -/// Agent → BackendCollector → QueryEngine. -pub(crate) fn decide_sketch_placement( - resolved: &PhysicalAggOp, - config: &PhysicalPlannerConfig, -) -> Placement { - use crate::physical::deployment::sketch_capability; - - let cap = sketch_capability(&resolved.sketch_type); - - // Try Agent first. - if config.constraints.agent.fits(&cap) { - return Placement::AgentCollector; - } - - // Agent budget exceeded — try Backend. - if config.constraints.backend_collector.fits(&cap) { - return Placement::BackendCollector; - } - - // Both exceeded — defer to QueryEngine. - Placement::QueryEngine -} - -/// Render a `Distinct { cols }` column tuple into a human-readable display -/// string for the `Filter { pred }` rationale. `Distinct { cols: [] }` is -/// whole-row SQL DISTINCT and prints as `*`. Canonical `cols` are -/// positional `ColumnId`s (no name at this layer — `plan_node` has no -/// schema in scope to resolve one), so each renders as `col#`. -fn display_distinct_cols(cols: &[ColumnId]) -> String { - if cols.is_empty() { - return "*".into(); - } - cols.iter() - .map(|id| format!("col#{id}")) - .collect::>() - .join(", ") -} - -/// If a node's child is at a different stage, insert an Exchange node between them. -fn insert_exchange_if_needed(node: &mut PhysicalNode) { - let parent_placement = node.placement.clone(); - for child in &mut node.children { - if child.placement != parent_placement { - let format = match (&child.placement, &parent_placement) { - (Placement::AgentCollector, Placement::BackendCollector) => ExchangeFormat::Otlp, - (Placement::AgentCollector, Placement::QueryEngine) => ExchangeFormat::Otlp, - (Placement::BackendCollector, Placement::QueryEngine) => { - ExchangeFormat::SketchBinary - } - _ => ExchangeFormat::Otlp, - }; - // Wrap the child in an Exchange node - let original_child = std::mem::replace( - child, - PhysicalNode { - op: PhysicalOp::Passthrough, - placement: parent_placement.clone(), - cost: PhysicalCost::default(), - children: vec![], - }, - ); - *child = PhysicalNode { - op: PhysicalOp::Exchange { format }, - placement: parent_placement.clone(), - cost: PhysicalCost::default(), - children: vec![original_child], - }; - } - } -} - -impl PhysicalNode { - /// Count total nodes in the tree. - pub fn node_count(&self) -> usize { - 1 + self.children.iter().map(|c| c.node_count()).sum::() - } - - /// Collect all distinct placements in the tree. - pub fn placements(&self) -> Vec { - let mut out = vec![self.placement.clone()]; - for child in &self.children { - for p in child.placements() { - if !out.contains(&p) { - out.push(p); - } - } - } - out - } - - /// Count Exchange nodes (= stage boundary crossings). - pub fn exchange_count(&self) -> usize { - let self_count = if matches!(self.op, PhysicalOp::Exchange { .. }) { - 1 - } else { - 0 - }; - self_count - + self - .children - .iter() - .map(|c| c.exchange_count()) - .sum::() - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use std::rc::Rc; - - use super::*; - use crate::planner_selection::default_frequency; - use crate::types_v2::AccuracyTarget; - use planner_types::pre_asap::{default_cardinality, default_quantile}; - use planner_types::pre_asap::{Reduction, Schema, Source}; - - fn default_config() -> PhysicalPlannerConfig { - PhysicalPlannerConfig { - budgets: StageResourceBudgets::default(), - constraints: DeploymentConstraints::default(), - } - } - - /// Canonical `Scan` leaf. - fn scan(name: &str) -> QueryExpr { - QueryExpr::Scan { - source: Source::TimeSeries { - metric: name.into(), - }, - predicates: vec![], - schema: Schema::default(), - } - } - - /// Single-intent, global, no-HAVING `Aggregate` over a `Scan` — the - /// canonical fold of the legacy `SketchAgg`. - fn sketch_agg(intent: AggIntent, metric: &str) -> QueryExpr { - QueryExpr::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![intent], - output_names: Vec::new(), - having: None, - child: Rc::new(scan(metric)), - } - } - - /// `TimeRange { Aggregate }` — the canonical fold of the legacy - /// `WindowedAgg`. - fn windowed_agg(intent: AggIntent, size_secs: u64, metric: &str) -> QueryExpr { - QueryExpr::TimeRange { - range: Duration::from_secs(size_secs), - child: Rc::new(sketch_agg(intent, metric)), - } - } - - #[test] - fn resolve_quantile() { - let p = resolve(&default_quantile(0.99)); - assert_eq!(p.sketch_type, SketchType::DDSketch); - assert!(matches!(p.sketch_params, SketchParams::DDSketch { .. })); - assert!(p.estimated_memory_bytes > 0); - } - - #[test] - fn resolve_cardinality() { - let p = resolve(&default_cardinality()); - assert_eq!(p.sketch_type, SketchType::HLL); - assert!(matches!(p.sketch_params, SketchParams::HLL { .. })); - } - - #[test] - fn resolve_frequency() { - let p = resolve(&default_frequency()); - assert_eq!(p.sketch_type, SketchType::CountSketch); - assert!(matches!(p.sketch_params, SketchParams::CountSketch { .. })); - } - - #[test] - fn resolve_preserves_intent() { - let intent = AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.005), - }; - let p = resolve(&intent); - assert_eq!(p.intent, intent); - } - - // ── Physical planner tests ────────────────────────────────────────── - - #[test] - fn plan_simple_sketch_at_agent() { - // Aggregate { Quantile } over Scan → Agent placement - let expr = sketch_agg(default_quantile(0.99), "m"); - let node = plan(&expr, &default_config()); - assert_eq!(node.placement, Placement::AgentCollector); - assert!(matches!(node.op, PhysicalOp::OtelSketchBuild { .. })); - assert_eq!(node.children.len(), 1); // Scan child - } - - #[test] - fn plan_windowed_agg_has_window() { - // Window { Aggregate { Quantile } } → fused OtelSketchBuild with a - // resolved tumbling window (the WindowedAgg fold; PR-5 recognizer). - let expr = windowed_agg(default_quantile(0.5), 300, "m"); - let node = plan(&expr, &default_config()); - assert_eq!(node.placement, Placement::AgentCollector); - match &node.op { - PhysicalOp::OtelSketchBuild { window, .. } => { - assert!(matches!(window, PhysicalWindow::OtelTumblingFlush { .. })); - } - other => panic!("expected OtelSketchBuild, got {other:?}"), - } - } - - #[test] - fn plan_topk_at_query_engine() { - let expr = QueryExpr::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::TopK { - k: 10, - accuracy: AccuracyTarget::Epsilon(0.05), - }], - output_names: Vec::new(), - having: None, - child: Rc::new(sketch_agg(default_frequency(), "m")), - }; - let node = plan(&expr, &default_config()); - assert_eq!(node.placement, Placement::QueryEngine); - assert!(matches!(node.op, PhysicalOp::TopK { k: 10 })); - } - - #[test] - fn plan_topk_inserts_exchange() { - // TopK(QueryEngine) wrapping a sketch Aggregate(Agent) → Exchange - // between them. - let expr = QueryExpr::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::TopK { - k: 5, - accuracy: AccuracyTarget::Epsilon(0.05), - }], - output_names: Vec::new(), - having: None, - child: Rc::new(sketch_agg(default_frequency(), "m")), - }; - let node = plan(&expr, &default_config()); - assert!( - node.exchange_count() > 0, - "expected Exchange between Agent and QueryEngine" - ); - } - - // `plan_partition_at_backend` (a standalone `Partition` node always - // placing at `BackendCollector`) is removed: `Partition` no longer - // exists in the canonical IR — its keys fold into `Aggregate.by` at - // construction time (`intent_algebra::lower`), and a grouped - // single-intent `Aggregate` goes through the same budget-driven - // Agent→Backend→Precompute path as an ungrouped one (`by` isn't a - // placement input), so there is no direct equivalent assertion to - // make here. - - #[test] - fn plan_multi_intent_aggregate_at_query_engine() { - // Multi-intent Aggregate → exact HashAggregate at QueryEngine - // (no single sketch serves multiple intents). - let expr = QueryExpr::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::Sum { col: None }, AggIntent::Min { col: None }], - output_names: Vec::new(), - having: None, - child: Rc::new(scan("trades")), - }; - let node = plan(&expr, &default_config()); - assert_eq!(node.placement, Placement::QueryEngine); - assert!(matches!(node.op, PhysicalOp::HashAggregate { .. })); - } - - #[test] - fn plan_full_pipeline_has_multiple_stages() { - // TopK(Merge(Window(Aggregate(Scan)))) — `windowed_agg` fuses to - // Agent, `Merge` (the `Frequency` intent's sketch fan-in) sits at - // Backend, and the outer `TopK` intent runs at QueryEngine. - // Should span: Agent → Backend → QueryEngine - let expr = QueryExpr::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::TopK { - k: 10, - accuracy: AccuracyTarget::Epsilon(0.05), - }], - output_names: Vec::new(), - having: None, - child: QueryExpr::Concat { - children: vec![windowed_agg(default_frequency(), 60, "requests")], - discriminator_unique_key: None, - } - .into(), - }; - let node = plan(&expr, &default_config()); - let placements = node.placements(); - assert!( - placements.contains(&Placement::AgentCollector), - "should have Agent: {placements:?}" - ); - assert!( - placements.contains(&Placement::BackendCollector), - "should have Backend: {placements:?}" - ); - assert!( - placements.contains(&Placement::QueryEngine), - "should have QueryEngine: {placements:?}" - ); - // Was `>= 2` when `Merge`'s child used to be a `Partition` node - // (which called `insert_exchange_if_needed` itself, contributing - // a second exchange on top of the outer TopK Aggregate's own). - // `Merge` doesn't call `insert_exchange_if_needed` on its - // children — it accepts heterogeneously-placed children by - // design (see its arm above) — so with `Partition` gone (its - // keys fold into `Aggregate.by` at construction time) this shape - // now has exactly one exchange-inserting boundary: the outer - // TopK Aggregate against its `Merge` child. The 3-stage span - // above is still the real invariant this test protects. - assert!( - node.exchange_count() >= 1, - "should have >=1 exchange: {}", - node.exchange_count() - ); - } - - #[test] - fn plan_budget_deferral() { - // With tiny agent budget, sketch should defer to Backend. - let budgets = StageResourceBudgets { - agent_memory_bytes: Some(1), // 1 byte = too small - ..Default::default() - }; - let config = PhysicalPlannerConfig { - constraints: DeploymentConstraints::from_budgets(&budgets), - budgets, - }; - let expr = sketch_agg(default_quantile(0.99), "m"); - let node = plan(&expr, &config); - assert_eq!( - node.placement, - Placement::BackendCollector, - "sketch should be deferred to Backend when agent budget is tiny" - ); - } -} diff --git a/control_plane/src/physical/window_fusion.rs b/control_plane/src/physical/window_fusion.rs deleted file mode 100644 index 3fd326f05..000000000 --- a/control_plane/src/physical/window_fusion.rs +++ /dev/null @@ -1,342 +0,0 @@ -//! The `Window { child: Aggregate }` fusion recognizer. -//! -//! ## The invariant this protects -//! -//! In sketch systems the window defines the sketch's lifecycle (when to -//! flush / reset), so a windowed sketch aggregation must be planned as a -//! *single* fused physical node — `PhysicalOp::OtelSketchBuild { window }` -//! — where the window and the sketch build are resolved together. -//! -//! The canonical L3 IR keeps "one canonical form per plan" (design.md -//! §6): it has no fused windowed-aggregate variant — a windowed sketch is -//! the stacked `Window { child: Aggregate { measures: [one], .. } }` shape. -//! `lower` produces exactly that shape when it folds a -//! single-statistic sketchable `Aggregate` sitting over a `Window`. This -//! module is the planner-side **peephole recognizer** that puts the -//! window-defines-sketch-lifecycle invariant back: it matches the stacked -//! shape and reconstructs the fused physical node. -//! -//! [`recognize_windowed_sketch`] matches the stacked canonical shape and -//! returns a [`FusedWindowSketch`] view; [`fused_sketch_decision`] -//! reconstructs the `(PhysicalOp, Placement)` for the fused sketch build. -//! It is on the canonical planner's hot path; the `#[cfg(test)]` -//! equivalence harness below pins it end-to-end against `plan`. - -#![allow(dead_code)] - -use std::time::Duration; - -use crate::physical::planner::{ - decide_sketch_placement, resolve, PhysicalOp, PhysicalPlannerConfig, PhysicalWindow, Placement, -}; -use asap_types::enums::WindowKind; -use planner_types::pre_asap::AggIntent; -use planner_types::pre_asap::ColumnId; -use planner_types::pre_asap::QueryExpr; - -/// Canonical-shape view of a fused window-over-sketch — the canonical -/// counterpart of a legacy `WindowedAgg` node, recognized out of the -/// stacked `Window { child: Aggregate { .. } }` form. -#[derive(Debug, Clone)] -pub struct FusedWindowSketch<'a> { - /// The single aggregation intent the sketch computes. - pub agg: &'a AggIntent, - /// Outer `Window`'s kind. - pub window_kind: WindowKind, - /// Outer `Window`'s `size`. - pub window_size: Duration, - /// Outer `Window`'s `slide` (`Some` only for `Sliding`). - pub window_slide: Option, - /// The inner `Aggregate`'s grouping columns -- empty both when it's a - /// genuine empty-`by` reduction and when it's per-entity (no grouping - /// concept at all, ASAPController#163/#165); this field doesn't - /// distinguish the two, since nothing downstream currently needs to. - pub by: &'a [ColumnId], - /// The subtree below the inner `Aggregate` — the sketch's input. - pub inner_child: &'a QueryExpr, -} - -/// Recognize the canonical `Window { child: Aggregate { measures: [one], -/// having: None, .. } }` shape — the stacked form `lower` -/// folds a legacy `WindowedAgg` into — and return a [`FusedWindowSketch`] -/// view of it. Returns `None` for any other shape (a multi-intent -/// `Aggregate`, an `Aggregate` with a `having` clause, a `Window` over a -/// non-`Aggregate` child, etc.). -/// -/// A bare canonical `Aggregate` with no enclosing `Window` is *not* -/// recognized here — that is the unfused `SketchAgg` case, which the -/// legacy planner already builds with `PhysicalWindow::None`. This -/// recognizer is specifically the window-defines-lifecycle peephole. -pub fn recognize_windowed_sketch(expr: &QueryExpr) -> Option> { - // ASAPPlanner has no canonical `Window` node (ASAPPlanner#193 -- - // "nothing ever constructs canonical QueryExpr::Window", confirmed by - // upstream's own corpus walk, true even at the previously-pinned - // rev): `asap_frontend_promql::lower_promql` -- the real production - // entry point since #428 -- has only ever emitted `TimeRange { range, - // child }` for a range-vector selector. This recognizer used to match - // `Window` regardless, which means it could never actually fire for - // real `lower_promql`-produced trees; matching `TimeRange` here is a - // fix, not just a rename. `TimeRange` carries no kind/slide -- real - // PromQL has no syntax to request anything but the implicit tumbling - // form, so `window_kind`/`window_slide` default to that (matching - // every other real-traffic call site in this codebase; see - // control_plane/docs/design-asapplanner-pin-migration.md). - let QueryExpr::TimeRange { range, child } = expr else { - return None; - }; - let QueryExpr::Aggregate { - reduction, - measures, - having, - child: inner_child, - .. - } = child.as_ref() - else { - return None; - }; - // The legacy `WindowedAgg` always carried exactly one intent and - // never a HAVING clause — only that shape is the fused sketch. - if measures.len() != 1 || having.is_some() { - return None; - } - let by: &[ColumnId] = reduction - .group_keys() - .map(|keys| keys.keys()) - .unwrap_or(&[]); - Some(FusedWindowSketch { - agg: &measures[0], - window_kind: WindowKind::Tumbling, - window_size: *range, - window_slide: None, - by, - inner_child, - }) -} - -/// Resolve a canonical `(WindowKind, size, slide)` to a -/// [`PhysicalWindow`] for a given placement. -/// -/// The canonical-IR twin of the legacy `WindowSpec`-based window -/// resolution. Every canonical `WindowKind` variant maps — the legacy -/// `WindowKind` is now `Tumbling` / `Sliding` / `Session` only (the -/// catalogue-less `Unbounded` / `Landmark` were retired in PR 12), so -/// the legacy → canonical `WindowKind` mapping is total. -pub fn resolve_window_canonical( - kind: &WindowKind, - size: Duration, - _slide: Option, - placement: &Placement, -) -> PhysicalWindow { - match (kind, placement) { - (WindowKind::Tumbling, Placement::AgentCollector) => { - PhysicalWindow::OtelTumblingFlush { duration: size } - } - (WindowKind::Tumbling, Placement::PromSketchStore) => PhysicalWindow::PromSketchEH { - eh_k: 50, - time_window: size, - }, - (WindowKind::Sliding, Placement::PromSketchStore) => PhysicalWindow::PromSketchEH { - eh_k: 50, - time_window: size, - }, - // Fallback: tumbling-at-size for any other (kind, placement) - // combo — mirrors the legacy `resolve_window` fallback arm. - _ => PhysicalWindow::OtelTumblingFlush { duration: size }, - } -} - -/// Reconstruct the `(PhysicalOp, Placement)` the legacy -/// `physical::planner::plan_node` `WindowedAgg` arm produces, but from -/// the canonical [`FusedWindowSketch`] view. -/// -/// This is the piece PR 8 calls on the hot path once the canonical -/// `plan_node` lands; PR 5 only exercises it from the equivalence -/// harness. It deliberately produces just the node's own decision (op + -/// placement) — the child subtree is the canonical planner's job, which -/// does not exist until PR 8. -pub fn fused_sketch_decision( - fused: &FusedWindowSketch<'_>, - config: &PhysicalPlannerConfig, -) -> (PhysicalOp, Placement) { - let resolved = resolve(fused.agg); - let placement = decide_sketch_placement(&resolved, config); - let window = resolve_window_canonical( - &fused.window_kind, - fused.window_size, - fused.window_slide, - &placement, - ); - let op = PhysicalOp::OtelSketchBuild { - sketch_type: resolved.sketch_type.clone(), - sketch_params: resolved.sketch_params.clone(), - window, - delta_encoding: false, - }; - (op, placement) -} - -// ── Equivalence harness ────────────────────────────────────────────────────── -// -// Pins `recognize_windowed_sketch` + `fused_sketch_decision` against the -// canonical planner: a `Window { Aggregate }` fused sketch — the shape -// `lower` folds a single-statistic sketchable `Aggregate` -// over a `Window` into — produces a resolved (non-`None`) physical window, -// and the planner's own decision matches `fused_sketch_decision`. - -#[cfg(test)] -mod tests { - use std::rc::Rc; - - use super::*; - use crate::physical::deployment::DeploymentConstraints; - use crate::physical::planner::{plan, PhysicalOp}; - use crate::planner_selection::default_frequency; - use crate::types::StageResourceBudgets; - use planner_types::pre_asap::Schema; - use planner_types::pre_asap::Source; - use planner_types::pre_asap::{default_cardinality, default_quantile}; - - fn config() -> PhysicalPlannerConfig { - PhysicalPlannerConfig { - budgets: StageResourceBudgets::default(), - constraints: DeploymentConstraints::default(), - } - } - - /// A canonical `Scan` leaf, built directly — the pre-ASAPPlanner-pin - /// version of this helper built it through `convert_root` - /// (`intent_algebra::lower`), which was deleted alongside the L2 tree - /// it converted from (see `intent_algebra::relational`'s module doc); - /// `Scan` is simple enough to construct directly instead. - fn canonical_scan(metric: &str) -> QueryExpr { - QueryExpr::Scan { - source: Source::TimeSeries { - metric: metric.to_string(), - }, - predicates: vec![], - schema: Schema::with_time_index(vec![], 0, vec![]), - } - } - - /// The canonical `TimeRange { Aggregate { by: [], measures: [agg] } }` - /// shape — what `asap_frontend_promql::lower_promql` (the real - /// production entry point) emits for a windowed single-sketch - /// aggregate. Real PromQL has no syntax for anything but the implicit - /// tumbling form (see `recognize_windowed_sketch`'s doc), so unlike - /// the pre-migration version of this helper, there is no `kind`/ - /// `slide` parameter to take anymore. - fn windowed_sketch(agg: AggIntent, size: Duration) -> QueryExpr { - QueryExpr::TimeRange { - range: size, - child: Rc::new(QueryExpr::Aggregate { - reduction: planner_types::pre_asap::Reduction::by(vec![]), - measures: vec![agg], - output_names: Vec::new(), - having: None, - child: Rc::new(canonical_scan("m")), - }), - } - } - - /// End-to-end fusion assertion: the canonical `TimeRange { Aggregate }` - /// shape, run through the recognizer + `fused_sketch_decision` and - /// through the canonical planner, produces a *fused* `OtelSketchBuild` - /// carrying a resolved (non-`None`) physical window — the - /// window-defines-sketch-lifecycle invariant as a planner peephole. - fn assert_equivalent(agg: AggIntent, size: Duration) { - let cfg = config(); - let canonical = windowed_sketch(agg, size); - - let fused = recognize_windowed_sketch(&canonical) - .expect("canonical TimeRange{Aggregate} should be recognized as a fused sketch"); - let (decided_op, _placement) = fused_sketch_decision(&fused, &cfg); - let PhysicalOp::OtelSketchBuild { - window: decided_window, - .. - } = &decided_op - else { - panic!("fused_sketch_decision did not produce OtelSketchBuild: {decided_op:?}"); - }; - assert!( - !matches!(decided_window, PhysicalWindow::None), - "fused windowed sketch must carry a resolved physical window, got None" - ); - - // End-to-end: the canonical planner produces the same fused - // `OtelSketchBuild` — `recognize_windowed_sketch` is on its hot path. - let node = plan(&canonical, &cfg); - let PhysicalOp::OtelSketchBuild { - window: planned_window, - .. - } = &node.op - else { - panic!( - "canonical planner did not fuse TimeRange{{Aggregate}}: {:?}", - node.op - ); - }; - // PhysicalWindow has no PartialEq — compare its Debug form. - assert_eq!( - format!("{decided_window:?}"), - format!("{planned_window:?}"), - "planner's fused window diverged from fused_sketch_decision" - ); - } - - #[test] - fn equivalence_quantile() { - assert_equivalent(default_quantile(0.99), Duration::from_secs(300)); - } - - #[test] - fn equivalence_cardinality() { - assert_equivalent(default_cardinality(), Duration::from_secs(60)); - } - - #[test] - fn equivalence_frequency() { - assert_equivalent(default_frequency(), Duration::from_secs(120)); - } - - #[test] - fn equivalence_sum_intent() { - assert_equivalent(AggIntent::Sum { col: None }, Duration::from_secs(300)); - } - - #[test] - fn recognizer_rejects_bare_aggregate() { - // A canonical `Aggregate` with no enclosing `TimeRange` is the - // unfused sketch case — not a windowed sketch. - let canonical = QueryExpr::Aggregate { - reduction: planner_types::pre_asap::Reduction::by(vec![]), - measures: vec![AggIntent::Sum { col: None }], - output_names: Vec::new(), - having: None, - child: Rc::new(canonical_scan("m")), - }; - assert!(recognize_windowed_sketch(&canonical).is_none()); - } - - #[test] - fn recognizer_rejects_window_over_non_aggregate() { - // `TimeRange` directly over a `Scan` — no inner `Aggregate` to fuse. - let canonical = QueryExpr::TimeRange { - range: Duration::from_secs(60), - child: Rc::new(canonical_scan("m")), - }; - assert!(recognize_windowed_sketch(&canonical).is_none()); - } - - #[test] - fn recognizer_accepts_windowed_agg() { - // The `TimeRange { Aggregate { Quantile } }` shape - // `lower_promql` emits for `quantile_over_time(...)` — exactly - // the shape the recognizer matches. - let canonical = windowed_sketch(default_quantile(0.99), Duration::from_secs(300)); - let fused = recognize_windowed_sketch(&canonical).expect("should recognize"); - assert_eq!(fused.window_kind, WindowKind::Tumbling); - assert_eq!(fused.window_size, Duration::from_secs(300)); - assert!(fused.window_slide.is_none()); - assert!(matches!(fused.agg, AggIntent::Quantile { .. })); - } -} diff --git a/crates/asap_types/Cargo.toml b/crates/asap_types/Cargo.toml index 1bc83ba7c..2ee4b61fc 100644 --- a/crates/asap_types/Cargo.toml +++ b/crates/asap_types/Cargo.toml @@ -18,3 +18,6 @@ sha2 = "0.10" # Use the same Planner IR as the control plane at compilation boundaries. planner-types.workspace = true + +# Accuracy metadata projects the authoritative Planner guarantees. +asap-aware-mapping.workspace = true diff --git a/crates/asap_types/src/accuracy.rs b/crates/asap_types/src/accuracy.rs index 33c47b5b8..723fa5cd8 100644 --- a/crates/asap_types/src/accuracy.rs +++ b/crates/asap_types/src/accuracy.rs @@ -1,7 +1,9 @@ -//! Shared wire contract for theoretical accuracy metadata. +//! Accuracy wire metadata projected from ASAPPlanner guarantees. +//! Formula ownership stays in ASAPPlanner; unknown confidence remains explicit. use serde::{Deserialize, Serialize}; +/// Interpretation of the reported epsilon; serialized identically across planners and serving. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum AccuracyKind { @@ -28,30 +30,76 @@ impl AccuracyKind { } } +/// Accuracy metadata shared by planning and serving. #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] pub struct AccuracyProfile { pub epsilon: f64, - pub delta: f64, + /// Failure probability; None means the planner cannot establish it. + pub delta: Option, pub kind: AccuracyKind, } impl AccuracyProfile { + /// Exact (ε = δ = 0). pub fn exact() -> Self { Self { epsilon: 0.0, - delta: 0.0, + delta: Some(0.0), kind: AccuracyKind::Exact, } } + /// One-line summary for dashboards and logs. pub fn summary(&self) -> String { format!( "accuracy: ε={}, δ={}, kind={}", self.epsilon, - self.delta, + self.delta + .map(|value| value.to_string()) + .unwrap_or_else(|| "unknown".into()), self.kind.as_str() ) } + + /// Project the four shared family contracts without reimplementing their + /// error formulas. Other families keep their backend-specific adapters. + pub fn from_sketch_params(params: &planner_types::post_asap::SketchParams) -> Option { + use asap_aware_mapping::accuracy::DefaultAccuracyModel; + use planner_types::post_asap::{SketchAlgorithm, SketchParams, SketchQuery}; + use planner_types::pre_asap::ColumnRef; + let (algorithm, query, kind) = match params { + SketchParams::Cms { .. } => ( + SketchAlgorithm::Cms, + SketchQuery::PointCount { + key: ColumnRef::SampleValue, + value: None, + }, + AccuracyKind::AdditiveFrequency, + ), + SketchParams::Hll { .. } => ( + SketchAlgorithm::Hll, + SketchQuery::Cardinality, + AccuracyKind::RelativeCardinality, + ), + SketchParams::Kll { .. } => ( + SketchAlgorithm::Kll, + SketchQuery::Quantile { q: 0.5 }, + AccuracyKind::RankQuantile, + ), + SketchParams::DDSketch { .. } => ( + SketchAlgorithm::DDSketch, + SketchQuery::Quantile { q: 0.5 }, + AccuracyKind::RelativeQuantile, + ), + _ => return None, + }; + let guarantee = DefaultAccuracyModel::sketch_guarantee(&algorithm, params, &query)?; + Some(Self { + epsilon: guarantee.bound.evaluate()?, + delta: guarantee.failure_probability.evaluate(), + kind, + }) + } } #[cfg(test)] @@ -62,7 +110,7 @@ mod tests { fn wire_contract_round_trips_with_stable_kind_name() { let profile = AccuracyProfile { epsilon: 0.01, - delta: 0.001, + delta: Some(0.001), kind: AccuracyKind::AdditiveFrequency, }; let json = serde_json::to_string(&profile).unwrap(); diff --git a/data_plane/src/storage_engines/sketch_db/accuracy.rs b/data_plane/src/storage_engines/sketch_db/accuracy.rs index 6134e7695..30019bedf 100644 --- a/data_plane/src/storage_engines/sketch_db/accuracy.rs +++ b/data_plane/src/storage_engines/sketch_db/accuracy.rs @@ -12,7 +12,7 @@ //! //! ## Scope of this module //! -//! Pure derivation: `AccuracyProfile::derive(&AggregationConfig)` +//! Pure derivation: `derive(&AggregationConfig)` //! looks at `aggregation_type` and the relevant entries in //! `config.parameters` and returns an `AccuracyProfile`. No //! runtime measurement, no sampling — just the textbook bound. @@ -24,240 +24,193 @@ //! ("how far off might this answer be?") the theoretical bound //! is the honest upper envelope. //! -//! ## Bounds we encode -//! -//! | Sketch | `kind` | ε formula | δ formula | -//! |---|---|---|---| -//! | Sum / Min / Max / Increase | `Exact` | 0 | 0 | -//! | CountMinSketch(w, d) | `AdditiveFrequency` | e / w | 1 / 2^d | -//! | CountMinSketchWithHeap(w, d, k) | `TopK` | max(e/w, 1/k) | 1 / 2^d | -//! | CountSketch(w, d) | `AdditiveFrequency` | 1 / √w | 1 / 2^d | -//! | HLL(p) | `RelativeCardinality` | 1.04 / √(2^p) | — (Gaussian std-dev) | -//! | KLL(k) | `RankQuantile` | ≈ 2.296 / √k (worst-case constant) | 1 / 100 (fixed) | -//! | DDSketch(α) | `RelativeQuantile` | α | 0 (deterministic α guarantee) | -//! -//! Constants are chosen to match the tighter published bounds -//! rather than loose textbook versions; sources are cited inline -//! in each branch of [`AccuracyProfile::derive`]. -//! -// See `docs/design_docs/summary-storage.md` for backend storage guarantees. +//! CMS, HLL, KLL, and DDSketch profiles come from ASAPPlanner's +//! `DefaultAccuracyModel`. HLL reports relative standard error with unknown +//! failure probability (`delta: null`), rather than a deterministic guarantee. +//! CountSketch, heap-retention extensions, and GOS augmentation remain local. + +use serde::{Deserialize, Serialize}; use asap_types::aggregation_config::AggregationConfig; use asap_types::AggregationType; -pub use asap_types::{AccuracyKind, AccuracyProfile}; -use serde::{Deserialize, Serialize}; -/// Backend-specific derivation over the installed aggregation config. +pub use asap_types::accuracy::{AccuracyKind, AccuracyProfile}; +use planner_types::post_asap::SketchParams as PlannerParams; + +/// Derive an [`AccuracyProfile`] from a pinned +/// [`AggregationConfig`]. Reads `aggregation_type` and any +/// necessary entries in `parameters`; falls back to exact for +/// unknown / legacy variants (harmless — the caller just gets +/// "0 error" rather than a panic). +/// +/// GOS continuous-query envelope (design-gos-unified-edge-telemetry.md §4, +/// Theorem 1): when the edge gates delta transmission by the GOS relative +/// threshold (`parameters["gos_delta_epsilon"] = ε_st > 0`), the warm +/// sketch answered from delta-applied state carries an extra DETERMINISTIC +/// staleness term of at most `ε_st` (relative) at any query time — it adds +/// linearly to the sketch's own probabilistic bound (`ε_total = ε_sk + +/// ε_st`; the random parts compose in quadrature but the staleness part is +/// adversarial, so linear addition is the honest envelope). δ is +/// unchanged (staleness is not probabilistic). +pub fn derive(config: &AggregationConfig) -> AccuracyProfile { + let mut profile = derive_sketch_only(config); + let eps_st = config + .parameters + .get("gos_delta_epsilon") + .and_then(|v| v.as_f64()) + .unwrap_or(0.0); + if eps_st > 0.0 && profile.kind != AccuracyKind::Exact { + profile.epsilon += eps_st; + } + profile +} + +/// Source adapter for installed aggregation configs. pub trait BackendAccuracyProfile { fn derive(config: &AggregationConfig) -> Self; fn derive_sketch_only(config: &AggregationConfig) -> Self; } - impl BackendAccuracyProfile for AccuracyProfile { - /// Derive an [`AccuracyProfile`] from a pinned - /// [`AggregationConfig`]. Reads `aggregation_type` and any - /// necessary entries in `parameters`; falls back to exact for - /// unknown / legacy variants (harmless — the caller just gets - /// "0 error" rather than a panic). - /// - /// GOS continuous-query envelope (design-gos-unified-edge-telemetry.md §4, - /// Theorem 1): when the edge gates delta transmission by the GOS relative - /// threshold (`parameters["gos_delta_epsilon"] = ε_st > 0`), the warm - /// sketch answered from delta-applied state carries an extra DETERMINISTIC - /// staleness term of at most `ε_st` (relative) at any query time — it adds - /// linearly to the sketch's own probabilistic bound (`ε_total = ε_sk + - /// ε_st`; the random parts compose in quadrature but the staleness part is - /// adversarial, so linear addition is the honest envelope). δ is - /// unchanged (staleness is not probabilistic). fn derive(config: &AggregationConfig) -> Self { - let mut profile = Self::derive_sketch_only(config); - let eps_st = config - .parameters - .get("gos_delta_epsilon") - .and_then(|v| v.as_f64()) - .unwrap_or(0.0); - if eps_st > 0.0 && profile.kind != AccuracyKind::Exact { - profile.epsilon += eps_st; - } - profile + derive(config) } - - /// The sketch's own theoretical bound, without the GOS staleness term. fn derive_sketch_only(config: &AggregationConfig) -> Self { - match config.aggregation_type { - AggregationType::UnivMon => Self { - epsilon: f64::MAX, - delta: 1.0, - kind: AccuracyKind::Uncalibrated, - }, - // Exact aggregates. (The `SetAggregator` / - // `DeltaSetAggregator` exact-set-membership family lived - // here too before its retirement.) - AggregationType::Sum - | AggregationType::Increase - | AggregationType::MinMax - | AggregationType::MultipleSum - | AggregationType::MultipleIncrease - | AggregationType::MultipleMinMax => Self::exact(), - - // CountMinSketch: classic Cormode-Muthukrishnan bound. - // ε = e/w, δ = 1/2^d with w = width, d = depth. We - // pull w from `parameters["w"]` and d from - // `parameters["d"]` — the canonical keys the controller - // emits and `accumulator_factory::cms_params` reads. - // Defaults to (rows=4, cols=1000) when absent. - AggregationType::CountMinSketch => { - let (rows, cols) = cms_params(config); - // Using natural e ≈ 2.71828 for tighter bound. - // Source: Cormode & Muthukrishnan, "An improved - // data stream summary: the count-min sketch and - // its applications," J. Algorithms 55(1) 2005. - let epsilon = std::f64::consts::E / (cols as f64).max(1.0); - let delta = 0.5_f64.powi(rows as i32); - Self { - epsilon, - delta, - kind: AccuracyKind::AdditiveFrequency, - } - } - - // CountMinSketchWithHeap: CMS frequency estimator - // coupled with a heap of the top-`k` heaviest items - // (Metwally et al.'s SpaceSaving-style retention). - // Two bounds apply: - // * per-item point-lookup: ε_point = e/w - // (inherited from the CMS part) - // * top-K retention: any item with true frequency - // ≥ N/heap_size is guaranteed to be in the top-K - // output; each retained count is within N/heap_size - // of the true value (SpaceSaving guarantee). - // We report the **tighter** of the two as the - // user-facing ε — typically the heap bound - // `1/heap_size` dominates when heap_size ≪ w, and the - // CMS bound `e/w` dominates when the heap is generously - // sized. `kind = TopK` signals that ε is the - // combined frequency + retention guarantee. - // δ stays `1/2^d` from the CMS half; retention itself - // is deterministic given an adversarial-free stream, - // but the count estimate remains probabilistic at - // depth d. - // Sources: - // - Cormode & Muthukrishnan 2005 (CMS bound) - // - Metwally, Agrawal, El Abbadi. "Efficient - // computation of frequent and top-k elements in - // data streams." ICDT 2005. (top-K retention) - AggregationType::CountMinSketchWithHeap => { - let (rows, cols) = cms_params(config); - let heap = cms_heap_size(config); - let cms_epsilon = std::f64::consts::E / (cols as f64).max(1.0); - let heap_epsilon = 1.0 / (heap as f64).max(1.0); - // Worst of the two — a user should expect errors - // no bigger than `ε · N`. - let epsilon = cms_epsilon.max(heap_epsilon); - let delta = 0.5_f64.powi(rows as i32); - Self { - epsilon, - delta, - kind: AccuracyKind::TopK, - } - } - - // CountSketch: ε = 1/√w, δ = 1/2^d (Charikar-Chen- - // Farach-Colton). Signed counters → tighter epsilon - // than CMS but same confidence ramp with depth. - AggregationType::CountSketch => { - let (rows, cols) = cms_params(config); - let epsilon = 1.0 / (cols as f64).max(1.0).sqrt(); - let delta = 0.5_f64.powi(rows as i32); - Self { - epsilon, - delta, - kind: AccuracyKind::AdditiveFrequency, - } - } + derive_sketch_only(config) + } +} - // CountSketchWithHeap: CountSketch frequency estimator - // paired with a top-k heap. Mirrors the - // CountMinSketchWithHeap branch above — the CountSketch - // half gives ε_point = 1/√w; the heap half gives - // ε_heap = 1/heap_size for retention. Report the - // tighter (max) of the two. - AggregationType::CountSketchWithHeap => { - let (rows, cols) = cms_params(config); - let heap = cms_heap_size(config); - let cs_epsilon = 1.0 / (cols as f64).max(1.0).sqrt(); - let heap_epsilon = 1.0 / (heap as f64).max(1.0); - let epsilon = cs_epsilon.max(heap_epsilon); - let delta = 0.5_f64.powi(rows as i32); - Self { - epsilon, - delta, - kind: AccuracyKind::TopK, - } - } +/// The sketch's own theoretical bound, without the GOS staleness term. +fn derive_sketch_only(config: &AggregationConfig) -> AccuracyProfile { + match config.aggregation_type { + AggregationType::UnivMon => AccuracyProfile { + epsilon: f64::MAX, + delta: Some(1.0), + kind: AccuracyKind::Uncalibrated, + }, + // Exact aggregates. (The `SetAggregator` / + // `DeltaSetAggregator` exact-set-membership family lived + // here too before its retirement.) + AggregationType::Sum + | AggregationType::Increase + | AggregationType::MinMax + | AggregationType::MultipleSum + | AggregationType::MultipleIncrease + | AggregationType::MultipleMinMax => AccuracyProfile::exact(), + + AggregationType::CountMinSketch => { + let (rows, cols) = cms_params(config); + shared_profile(PlannerParams::Cms { + depth: u32::try_from(rows).unwrap_or(u32::MAX).max(1), + width: u32::try_from(cols).unwrap_or(u32::MAX).max(1), + }) + } - // HLL: std-dev ≈ 1.04/√m, m = 2^precision. Report - // this as relative error ε; δ is the Gaussian - // std-dev convention (stored as 0 because our δ - // field is "confidence parameter" not "variance"; - // future AccuracyKind::RelativeCardinality variant - // could carry the Gaussian flavor explicitly). - // Source: Flajolet et al., "HyperLogLog: the analysis - // of a near-optimal cardinality estimation algorithm," - // DMTCS 2007. - AggregationType::HLL => { - let p = hll_precision(config); - let m = (1u64 << p) as f64; - Self { - epsilon: 1.04 / m.sqrt(), - delta: 0.0, - kind: AccuracyKind::RelativeCardinality, - } + // CountMinSketchWithHeap: CMS frequency estimator + // coupled with a heap of the top-`k` heaviest items + // (Metwally et al.'s SpaceSaving-style retention). + // Two bounds apply: + // * per-item point-lookup: ε_point = e/w + // (inherited from the CMS part) + // * top-K retention: any item with true frequency + // ≥ N/heap_size is guaranteed to be in the top-K + // output; each retained count is within N/heap_size + // of the true value (SpaceSaving guarantee). + // We report the **tighter** of the two as the + // user-facing ε — typically the heap bound + // `1/heap_size` dominates when heap_size ≪ w, and the + // CMS bound `e/w` dominates when the heap is generously + // sized. `kind = TopK` signals that ε is the + // combined frequency + retention guarantee. + // δ stays `exp(-d)` from the CMS half; retention itself + // is deterministic given an adversarial-free stream, + // but the count estimate remains probabilistic at + // depth d. + // Sources: + // - Cormode & Muthukrishnan 2005 (CMS bound) + // - Metwally, Agrawal, El Abbadi. "Efficient + // computation of frequent and top-k elements in + // data streams." ICDT 2005. (top-K retention) + AggregationType::CountMinSketchWithHeap => { + let (rows, cols) = cms_params(config); + let heap = cms_heap_size(config); + let cms = shared_profile(PlannerParams::Cms { + depth: u32::try_from(rows).unwrap_or(u32::MAX).max(1), + width: u32::try_from(cols).unwrap_or(u32::MAX).max(1), + }); + let heap_epsilon = 1.0 / (heap as f64).max(1.0); + // Worst of the two — a user should expect errors + // no bigger than `ε · N`. + let epsilon = cms.epsilon.max(heap_epsilon); + let delta = cms.delta; + AccuracyProfile { + epsilon, + delta, + kind: AccuracyKind::TopK, } + } - // KLL: rank error ε = C/√k with δ ≤ 0.01 (fixed - // confidence; KLL's theoretical guarantee). Empirical - // C ≈ 2.296 for the standard floating-point KLL - // variant implemented here. - // Source: Karnin, Lang, Liberty. "Optimal quantile - // approximation in streams," FOCS 2016. - AggregationType::DatasketchesKLL | AggregationType::HydraKLL => { - let k = kll_k(config); - Self { - epsilon: 2.296 / (k as f64).max(1.0).sqrt(), - delta: 0.01, - kind: AccuracyKind::RankQuantile, - } + // CountSketch: ε = 1/√w, δ = 1/2^d (Charikar-Chen- + // Farach-Colton). Signed counters → tighter epsilon + // than CMS but same confidence ramp with depth. + AggregationType::CountSketch => { + let (rows, cols) = cms_params(config); + let epsilon = 1.0 / (cols as f64).max(1.0).sqrt(); + let delta = Some(0.5_f64.powi(rows as i32)); + AccuracyProfile { + epsilon, + delta, + kind: AccuracyKind::AdditiveFrequency, } + } - // DDSketch: α is the relative quantile error directly - // — it's a design parameter of the sketch, not a - // probabilistic bound. δ = 0 (deterministic). - // Source: Masson, Rim, Lee. "DDSketch: a fast and - // fully-mergeable quantile sketch with relative-error - // guarantees," VLDB 2019. - AggregationType::DDSketch => { - let alpha = ddsketch_alpha(config); - Self { - epsilon: alpha, - delta: 0.0, - kind: AccuracyKind::RelativeQuantile, - } + // CountSketchWithHeap: CountSketch frequency estimator + // paired with a top-k heap. Mirrors the + // CountMinSketchWithHeap branch above — the CountSketch + // half gives ε_point = 1/√w; the heap half gives + // ε_heap = 1/heap_size for retention. Report the + // tighter (max) of the two. + AggregationType::CountSketchWithHeap => { + let (rows, cols) = cms_params(config); + let heap = cms_heap_size(config); + let cs_epsilon = 1.0 / (cols as f64).max(1.0).sqrt(); + let heap_epsilon = 1.0 / (heap as f64).max(1.0); + let epsilon = cs_epsilon.max(heap_epsilon); + let delta = Some(0.5_f64.powi(rows as i32)); + AccuracyProfile { + epsilon, + delta, + kind: AccuracyKind::TopK, } + } - // Legacy / wrapper variants. Return exact — they are - // config-shape placeholders that dispatch to concrete - // aggregator types elsewhere; their accuracy profile - // depends on the sub_type, which the factory resolves - // at updater-construction time. Phase 6.4 v2 can walk - // sub_type to give a tighter answer. - AggregationType::SingleSubpopulation | AggregationType::MultipleSubpopulation => { - Self::exact() - } + AggregationType::HLL => shared_profile(PlannerParams::Hll { + precision: u8::try_from(hll_precision(config)).unwrap_or(14), + }), + AggregationType::DatasketchesKLL | AggregationType::HydraKLL => { + shared_profile(PlannerParams::Kll { + k: kll_k(config).max(1), + }) + } + AggregationType::DDSketch => shared_profile(PlannerParams::DDSketch { + alpha: ddsketch_alpha(config), + }), + + // Legacy / wrapper variants. Return exact — they are + // config-shape placeholders that dispatch to concrete + // aggregator types elsewhere; their accuracy profile + // depends on the sub_type, which the factory resolves + // at updater-construction time. Phase 6.4 v2 can walk + // sub_type to give a tighter answer. + AggregationType::SingleSubpopulation | AggregationType::MultipleSubpopulation => { + AccuracyProfile::exact() } } } +fn shared_profile(params: PlannerParams) -> AccuracyProfile { + AccuracyProfile::from_sketch_params(¶ms).expect("shared family has a numeric planner bound") +} + // Parameter extraction helpers. Kept file-local (not pub) because // they duplicate tiny bits of `precompute_engine::accumulator_factory` // and the backfill-vs-live separation rule (see that module's doc) @@ -369,7 +322,7 @@ impl AccuracyEnvelope { return None; } let mut epsilon = 0.0_f64; - let mut delta = 0.0_f64; + let mut delta = Some(0.0_f64); // Pick the "most lossy" kind: any non-Exact wins over // Exact; if mixed non-Exact kinds span segments we pick // the first non-Exact and trust the per-segment data for @@ -379,9 +332,11 @@ impl AccuracyEnvelope { if s.profile.epsilon > epsilon { epsilon = s.profile.epsilon; } - if s.profile.delta > delta { - delta = s.profile.delta; - } + // A known bound from another segment cannot fill unknown confidence. + delta = match (delta, s.profile.delta) { + (Some(a), Some(b)) => Some(a.max(b)), + _ => None, + }; if matches!(kind, AccuracyKind::Exact) && !matches!(s.profile.kind, AccuracyKind::Exact) { kind = s.profile.kind; @@ -439,18 +394,149 @@ mod tests { ) } + /// Both source adapters must agree for the same normalized family parameters. + #[test] + fn planner_and_backend_shared_family_parity() { + use control_plane::types::SketchParams; + let cases = vec![ + ( + SketchParams::CountMinSketch { + rows: 4, + cols: 1000, + metric_name: "m".into(), + }, + AggregationType::CountMinSketch, + json!({"d": 4, "w": 1000}), + ), + ( + SketchParams::CountMinSketch { + rows: 7, + cols: 4096, + metric_name: "m".into(), + }, + AggregationType::CountMinSketch, + json!({"d": 7, "w": 4096}), + ), + ( + SketchParams::HLL { precision: 14 }, + AggregationType::HLL, + json!({"precision": 14}), + ), + ( + SketchParams::HLL { precision: 10 }, + AggregationType::HLL, + json!({"p": 10}), + ), + ( + SketchParams::KLL { + k: 200, + quantiles: vec![], + }, + AggregationType::DatasketchesKLL, + json!({"K": 200}), + ), + ( + SketchParams::KLL { + k: 512, + quantiles: vec![], + }, + AggregationType::HydraKLL, + json!({"k": 512}), + ), + ( + SketchParams::DDSketch { + relative_accuracy: 0.01, + quantiles: vec![], + }, + AggregationType::DDSketch, + json!({"alpha": 0.01}), + ), + ( + SketchParams::DDSketch { + relative_accuracy: 0.05, + quantiles: vec![], + }, + AggregationType::DDSketch, + json!({"alpha": 0.05}), + ), + ]; + for (params, family, wire) in cases { + let config = base_config(family, serde_json::from_value(wire).unwrap()); + let planner = control_plane::accuracy::derive(¶ms); + let backend = derive(&config); + assert_eq!(planner, backend, "parameters: {params:?}"); + assert_eq!( + serde_json::to_value(planner).unwrap(), + serde_json::to_value(backend).unwrap() + ); + } + } + + /// GOS widens epsilon without inventing confidence for HLL. + #[test] + fn shared_family_gos_preserves_planner_confidence() { + for family in [ + AggregationType::CountMinSketch, + AggregationType::HLL, + AggregationType::DatasketchesKLL, + AggregationType::DDSketch, + ] { + let base = derive(&base_config(family, HashMap::new())); + let widened = derive(&base_config( + family, + HashMap::from([("gos_delta_epsilon".into(), json!(0.05))]), + )); + assert_eq!(widened.epsilon, base.epsilon + 0.05); + assert_eq!(widened.delta, base.delta); + assert_eq!(widened.kind, base.kind); + } + } + + /// Unknown confidence stays unknown regardless of segment order. + #[test] + fn envelope_preserves_unknown_failure_probability() { + let hll = derive(&base_config(AggregationType::HLL, HashMap::new())); + assert_eq!(hll.delta, None); + for profiles in [ + [hll, AccuracyProfile::exact()], + [AccuracyProfile::exact(), hll], + ] { + let segments = profiles + .into_iter() + .enumerate() + .map(|(i, profile)| PerSegmentAccuracy { + agg_id: i as u64, + range_ms: [i as i64, i as i64 + 1], + profile, + }) + .collect(); + let envelope = AccuracyEnvelope::from_segments(segments).unwrap(); + assert_eq!(envelope.profile.delta, None); + assert!(serde_json::to_value(envelope).unwrap()["delta"].is_null()); + } + } + + /// Backend-only UnivMon must not acquire a calibrated shared-family bound. + #[test] + fn univmon_remains_uncalibrated() { + let profile = derive(&base_config(AggregationType::UnivMon, HashMap::new())); + assert_eq!(profile.kind, AccuracyKind::Uncalibrated); + assert_eq!(profile.epsilon, f64::MAX); + assert_eq!(profile.delta, Some(1.0)); + } + #[test] fn sum_is_exact() { - let p = AccuracyProfile::derive(&base_config(AggregationType::Sum, HashMap::new())); + let p = derive(&base_config(AggregationType::Sum, HashMap::new())); assert_eq!(p.kind, AccuracyKind::Exact); assert_eq!(p.epsilon, 0.0); - assert_eq!(p.delta, 0.0); + assert_eq!(p.delta, Some(0.0)); } #[test] fn min_max_increase_are_exact() { for t in [AggregationType::MinMax, AggregationType::Increase] { - let p = AccuracyProfile::derive(&base_config(t, HashMap::new())); + let p = derive(&base_config(t, HashMap::new())); assert_eq!(p.kind, AccuracyKind::Exact); } } @@ -463,10 +549,9 @@ mod tests { let mut params = HashMap::new(); params.insert("d".to_string(), json!(5)); params.insert("w".to_string(), json!(256)); - let base = - AccuracyProfile::derive(&base_config(AggregationType::CountSketch, params.clone())); + let base = derive(&base_config(AggregationType::CountSketch, params.clone())); params.insert("gos_delta_epsilon".to_string(), json!(0.05)); - let widened = AccuracyProfile::derive(&base_config(AggregationType::CountSketch, params)); + let widened = derive(&base_config(AggregationType::CountSketch, params)); assert!((widened.epsilon - (base.epsilon + 0.05)).abs() < 1e-12); assert_eq!(widened.delta, base.delta); assert_eq!(widened.kind, base.kind); @@ -478,7 +563,7 @@ mod tests { // stray parameter must not fabricate an ε>0 "exact" answer. let mut params = HashMap::new(); params.insert("gos_delta_epsilon".to_string(), json!(0.05)); - let p = AccuracyProfile::derive(&base_config(AggregationType::Sum, params)); + let p = derive(&base_config(AggregationType::Sum, params)); assert_eq!(p.epsilon, 0.0); assert_eq!(p.kind, AccuracyKind::Exact); } @@ -488,23 +573,23 @@ mod tests { let mut params = HashMap::new(); params.insert("d".to_string(), json!(5)); params.insert("w".to_string(), json!(2718)); - let p = AccuracyProfile::derive(&base_config(AggregationType::CountMinSketch, params)); + let p = derive(&base_config(AggregationType::CountMinSketch, params)); // e / 2718 ≈ 0.0010001 — very close to 0.001. assert_eq!(p.kind, AccuracyKind::AdditiveFrequency); assert!((p.epsilon - std::f64::consts::E / 2718.0).abs() < 1e-12); // δ = 1/2^5 = 0.03125 - assert!((p.delta - 0.03125).abs() < 1e-12); + assert!((p.delta.unwrap() - (-5.0_f64).exp()).abs() < 1e-12); } #[test] fn cms_uses_defaults_when_params_absent() { - let p = AccuracyProfile::derive(&base_config( + let p = derive(&base_config( AggregationType::CountMinSketch, HashMap::new(), )); // Defaults rows=4, cols=1000 per accumulator_factory. assert!((p.epsilon - std::f64::consts::E / 1000.0).abs() < 1e-12); - assert!((p.delta - 0.0625).abs() < 1e-12); // 1/16 + assert!((p.delta.unwrap() - (-4.0_f64).exp()).abs() < 1e-12); // 1/16 } #[test] @@ -515,13 +600,13 @@ mod tests { params.insert("d".to_string(), json!(5)); params.insert("w".to_string(), json!(1_000_000)); params.insert("heap_size".to_string(), json!(10_000)); - let p = AccuracyProfile::derive(&base_config( + let p = derive(&base_config( AggregationType::CountMinSketchWithHeap, params, )); assert_eq!(p.kind, AccuracyKind::TopK); assert!((p.epsilon - 1.0 / 10_000.0).abs() < 1e-12); - assert!((p.delta - 1.0 / 32.0).abs() < 1e-12); // 1/2^5 + assert!((p.delta.unwrap() - (-5.0_f64).exp()).abs() < 1e-12); // 1/2^5 } #[test] @@ -532,7 +617,7 @@ mod tests { params.insert("d".to_string(), json!(4)); params.insert("w".to_string(), json!(100)); params.insert("heap_size".to_string(), json!(1_000_000)); - let p = AccuracyProfile::derive(&base_config( + let p = derive(&base_config( AggregationType::CountMinSketchWithHeap, params, )); @@ -547,7 +632,7 @@ mod tests { params.insert("w".to_string(), json!(1000)); // heap_size absent → default 100 → 1/100 = 0.01 dominates // e/1000 ≈ 0.00272. - let p = AccuracyProfile::derive(&base_config( + let p = derive(&base_config( AggregationType::CountMinSketchWithHeap, params, )); @@ -564,7 +649,7 @@ mod tests { params.insert("d".to_string(), json!(4)); params.insert("w".to_string(), json!(1_000_000)); params.insert(alias.to_string(), json!(500)); - let p = AccuracyProfile::derive(&base_config( + let p = derive(&base_config( AggregationType::CountMinSketchWithHeap, params, )); @@ -581,17 +666,17 @@ mod tests { let mut params = HashMap::new(); params.insert("d".to_string(), json!(4)); params.insert("w".to_string(), json!(100)); - let p = AccuracyProfile::derive(&base_config(AggregationType::CountSketch, params)); + let p = derive(&base_config(AggregationType::CountSketch, params)); assert_eq!(p.kind, AccuracyKind::AdditiveFrequency); assert!((p.epsilon - 0.1).abs() < 1e-9); // 1/√100 = 0.1 - assert!((p.delta - 0.0625).abs() < 1e-12); // 1/2^4 + assert!((p.delta.unwrap() - 0.0625).abs() < 1e-12); // Backend-only CountSketch convention. } #[test] fn hll_epsilon_matches_flajolet_bound() { let mut params = HashMap::new(); params.insert("precision".to_string(), json!(14)); - let p = AccuracyProfile::derive(&base_config(AggregationType::HLL, params)); + let p = derive(&base_config(AggregationType::HLL, params)); assert_eq!(p.kind, AccuracyKind::RelativeCardinality); // 1.04 / √16384 = 1.04 / 128 = 0.008125 assert!((p.epsilon - 0.008125).abs() < 1e-9); @@ -599,7 +684,7 @@ mod tests { #[test] fn hll_uses_default_precision_14() { - let p = AccuracyProfile::derive(&base_config(AggregationType::HLL, HashMap::new())); + let p = derive(&base_config(AggregationType::HLL, HashMap::new())); assert!((p.epsilon - 0.008125).abs() < 1e-9); } @@ -607,35 +692,35 @@ mod tests { fn kll_epsilon_matches_karnin_lang_liberty_bound() { let mut params = HashMap::new(); params.insert("K".to_string(), json!(200)); - let p = AccuracyProfile::derive(&base_config(AggregationType::DatasketchesKLL, params)); + let p = derive(&base_config(AggregationType::DatasketchesKLL, params)); assert_eq!(p.kind, AccuracyKind::RankQuantile); - // 2.296 / √200 ≈ 0.16235 - assert!((p.epsilon - 2.296 / 200.0_f64.sqrt()).abs() < 1e-12); - assert!((p.delta - 0.01).abs() < 1e-12); + // Planner uses the DataSketches empirical 99th-percentile rank-error fit. + assert!((p.epsilon - 2.296 / 200.0_f64.powf(0.9723)).abs() < 1e-12); + assert!((p.delta.unwrap() - 0.01).abs() < 1e-12); } #[test] fn hydra_kll_follows_the_same_kll_bound() { let mut params = HashMap::new(); params.insert("k".to_string(), json!(400)); - let p = AccuracyProfile::derive(&base_config(AggregationType::HydraKLL, params)); - // 2.296 / √400 = 2.296 / 20 = 0.1148 - assert!((p.epsilon - 0.1148).abs() < 1e-9); + let p = derive(&base_config(AggregationType::HydraKLL, params)); + // Planner uses the DataSketches empirical 99th-percentile rank-error fit. + assert!((p.epsilon - 2.296 / 400.0_f64.powf(0.9723)).abs() < 1e-12); } #[test] fn ddsketch_epsilon_is_alpha_directly() { let mut params = HashMap::new(); params.insert("alpha".to_string(), json!(0.02)); - let p = AccuracyProfile::derive(&base_config(AggregationType::DDSketch, params)); + let p = derive(&base_config(AggregationType::DDSketch, params)); assert_eq!(p.kind, AccuracyKind::RelativeQuantile); assert_eq!(p.epsilon, 0.02); - assert_eq!(p.delta, 0.0); + assert_eq!(p.delta, Some(0.0)); } #[test] fn ddsketch_uses_default_alpha_0_01() { - let p = AccuracyProfile::derive(&base_config(AggregationType::DDSketch, HashMap::new())); + let p = derive(&base_config(AggregationType::DDSketch, HashMap::new())); assert_eq!(p.epsilon, 0.01); } @@ -653,7 +738,7 @@ mod tests { AggregationType::SingleSubpopulation, AggregationType::MultipleSubpopulation, ] { - let p = AccuracyProfile::derive(&base_config(t, HashMap::new())); + let p = derive(&base_config(t, HashMap::new())); assert_eq!(p.kind, AccuracyKind::Exact); } } @@ -662,7 +747,7 @@ mod tests { fn accuracy_profile_roundtrips_through_serde() { let input = AccuracyProfile { epsilon: 0.008125, - delta: 0.0, + delta: None, kind: AccuracyKind::RelativeCardinality, }; let json = serde_json::to_string(&input).unwrap(); @@ -680,11 +765,11 @@ mod tests { let mut params = HashMap::new(); params.insert("d".to_string(), json!(4)); params.insert("w".to_string(), json!(10000)); - let cms = AccuracyProfile::derive(&base_config( + let cms = derive(&base_config( AggregationType::CountMinSketch, params.clone(), )); - let cs = AccuracyProfile::derive(&base_config(AggregationType::CountSketch, params)); + let cs = derive(&base_config(AggregationType::CountSketch, params)); // cms.epsilon = e/10000 ≈ 2.718e-4 // cs.epsilon = 1/√10000 = 0.01 = 1e-2 // So cms < cs (for w=10000). They cross at w = e. diff --git a/data_plane/src/tests/accuracy_empirical_validation_tests.rs b/data_plane/src/tests/accuracy_empirical_validation_tests.rs index ce33e2874..ed88c51d9 100644 --- a/data_plane/src/tests/accuracy_empirical_validation_tests.rs +++ b/data_plane/src/tests/accuracy_empirical_validation_tests.rs @@ -20,7 +20,7 @@ //! test pulls in a lot of version-coupling we don't want. //! Empirical sweeps are produced by `sketchlib bench //! --metrics accuracy` runs in sketch-bench and compared -//! against `AccuracyProfile::derive` externally; the test +//! against `derive` externally; the test //! below pins the *theoretical* side that those comparisons //! are made against. @@ -33,9 +33,7 @@ use asap_types::AggregationType; use asap_types::KeyByLabelNames; use serde_json::{json, Value}; -use crate::storage_engines::sketch_db::accuracy::{ - AccuracyKind, AccuracyProfile, BackendAccuracyProfile, -}; +use crate::storage_engines::sketch_db::accuracy::{derive, AccuracyKind}; fn cfg(agg_type: AggregationType, params: HashMap) -> AggregationConfig { AggregationConfig::new( @@ -65,22 +63,22 @@ fn hll_p14_matches_flajolet_1_04_over_sqrt_m() { // For p = 14 → ε = 1.04 / 128 = 0.008125. let mut params = HashMap::new(); params.insert("precision".to_string(), json!(14u64)); - let p = AccuracyProfile::derive(&cfg(AggregationType::HLL, params)); + let p = derive(&cfg(AggregationType::HLL, params)); assert_eq!(p.kind, AccuracyKind::RelativeCardinality); assert!((p.epsilon - 0.008125).abs() < 1e-9); } #[test] fn cms_3x1000_matches_cormode_muthukrishnan_e_over_w() { - // Cormode-Muthukrishnan 2005: ε = e/w, δ = 1/2^d. - // For (w=1000, d=3): ε = e/1000 ≈ 2.71828e-3, δ = 0.125. + // Cormode-Muthukrishnan 2005: ε = e/w, δ = exp(-d). + // For (w=1000, d=3): ε = e/1000 ≈ 2.71828e-3, δ = exp(-3). let mut params = HashMap::new(); params.insert("d".to_string(), json!(3u64)); params.insert("w".to_string(), json!(1000u64)); - let p = AccuracyProfile::derive(&cfg(AggregationType::CountMinSketch, params)); + let p = derive(&cfg(AggregationType::CountMinSketch, params)); assert_eq!(p.kind, AccuracyKind::AdditiveFrequency); assert!((p.epsilon - std::f64::consts::E / 1000.0).abs() < 1e-12); - assert!((p.delta - 0.125).abs() < 1e-12); + assert!((p.delta.unwrap() - (-3.0_f64).exp()).abs() < 1e-12); } #[test] @@ -90,20 +88,20 @@ fn count_sketch_4x10000_matches_charikar_one_over_sqrt_w() { let mut params = HashMap::new(); params.insert("d".to_string(), json!(4u64)); params.insert("w".to_string(), json!(10_000u64)); - let p = AccuracyProfile::derive(&cfg(AggregationType::CountSketch, params)); + let p = derive(&cfg(AggregationType::CountSketch, params)); assert_eq!(p.kind, AccuracyKind::AdditiveFrequency); assert!((p.epsilon - 0.01).abs() < 1e-12); } #[test] fn kll_k200_matches_karnin_lang_liberty_2_296() { - // Karnin-Lang-Liberty FOCS 2016: rank err ≈ 2.296 / √k. + // Planner uses the DataSketches empirical 99th-percentile KLL rank-error fit. let mut params = HashMap::new(); params.insert("K".to_string(), json!(200u64)); - let p = AccuracyProfile::derive(&cfg(AggregationType::DatasketchesKLL, params)); + let p = derive(&cfg(AggregationType::DatasketchesKLL, params)); assert_eq!(p.kind, AccuracyKind::RankQuantile); - assert!((p.epsilon - 2.296 / 200.0_f64.sqrt()).abs() < 1e-12); - assert!((p.delta - 0.01).abs() < 1e-12); + assert!((p.epsilon - 2.296 / 200.0_f64.powf(0.9723)).abs() < 1e-12); + assert!((p.delta.unwrap() - 0.01).abs() < 1e-12); } #[test] @@ -113,10 +111,10 @@ fn ddsketch_alpha_passes_through_verbatim() { for alpha in [0.005_f64, 0.01, 0.02, 0.05] { let mut params = HashMap::new(); params.insert("alpha".to_string(), json!(alpha)); - let p = AccuracyProfile::derive(&cfg(AggregationType::DDSketch, params)); + let p = derive(&cfg(AggregationType::DDSketch, params)); assert_eq!(p.kind, AccuracyKind::RelativeQuantile); assert_eq!(p.epsilon, alpha); - assert_eq!(p.delta, 0.0); + assert_eq!(p.delta, Some(0.0)); } } @@ -128,10 +126,10 @@ fn cms_with_heap_top_k_combines_cms_and_retention_bounds() { params.insert("d".to_string(), json!(3u64)); params.insert("w".to_string(), json!(1000u64)); params.insert("heap_size".to_string(), json!(50u64)); - let p = AccuracyProfile::derive(&cfg(AggregationType::CountMinSketchWithHeap, params)); + let p = derive(&cfg(AggregationType::CountMinSketchWithHeap, params)); assert_eq!(p.kind, AccuracyKind::TopK); assert!((p.epsilon - 0.02).abs() < 1e-12); - assert!((p.delta - 0.125).abs() < 1e-12); + assert!((p.delta.unwrap() - (-3.0_f64).exp()).abs() < 1e-12); } // -------- monotonicity ---------- @@ -142,7 +140,7 @@ fn hll_epsilon_shrinks_monotonically_with_precision() { for p in [8u64, 10, 12, 14, 16] { let mut params = HashMap::new(); params.insert("precision".to_string(), json!(p)); - let eps = AccuracyProfile::derive(&cfg(AggregationType::HLL, params)).epsilon; + let eps = derive(&cfg(AggregationType::HLL, params)).epsilon; assert!( eps < last, "HLL ε should tighten as precision grows: p={p} ε={eps} previous={last}" @@ -158,7 +156,7 @@ fn cms_epsilon_shrinks_monotonically_with_width() { let mut params = HashMap::new(); params.insert("d".to_string(), json!(4u64)); params.insert("w".to_string(), json!(w)); - let eps = AccuracyProfile::derive(&cfg(AggregationType::CountMinSketch, params)).epsilon; + let eps = derive(&cfg(AggregationType::CountMinSketch, params)).epsilon; assert!(eps < last, "CMS ε at w={w}: {eps} should be < {last}"); last = eps; } @@ -171,7 +169,9 @@ fn cms_delta_shrinks_monotonically_with_depth() { let mut params = HashMap::new(); params.insert("d".to_string(), json!(d)); params.insert("w".to_string(), json!(1000u64)); - let delta = AccuracyProfile::derive(&cfg(AggregationType::CountMinSketch, params)).delta; + let delta = derive(&cfg(AggregationType::CountMinSketch, params)) + .delta + .unwrap(); assert!(delta < last, "CMS δ at d={d}: {delta} should be < {last}"); last = delta; } @@ -184,7 +184,7 @@ fn countsketch_epsilon_shrinks_monotonically_with_width() { let mut params = HashMap::new(); params.insert("d".to_string(), json!(4u64)); params.insert("w".to_string(), json!(w)); - let eps = AccuracyProfile::derive(&cfg(AggregationType::CountSketch, params)).epsilon; + let eps = derive(&cfg(AggregationType::CountSketch, params)).epsilon; assert!( eps < last, "CountSketch ε at w={w}: {eps} should be < {last}" @@ -199,7 +199,7 @@ fn kll_epsilon_shrinks_monotonically_with_k() { for k in [50u64, 100, 200, 500, 1000] { let mut params = HashMap::new(); params.insert("K".to_string(), json!(k)); - let eps = AccuracyProfile::derive(&cfg(AggregationType::DatasketchesKLL, params)).epsilon; + let eps = derive(&cfg(AggregationType::DatasketchesKLL, params)).epsilon; assert!(eps < last, "KLL ε at k={k}: {eps} should be < {last}"); last = eps; } @@ -212,7 +212,7 @@ fn ddsketch_epsilon_tracks_alpha_monotonically() { for alpha in [0.05f64, 0.02, 0.01, 0.005, 0.001] { let mut params = HashMap::new(); params.insert("alpha".to_string(), json!(alpha)); - let eps = AccuracyProfile::derive(&cfg(AggregationType::DDSketch, params)).epsilon; + let eps = derive(&cfg(AggregationType::DDSketch, params)).epsilon; assert!( eps < last, "DDSketch ε at α={alpha}: {eps} should be < {last}" @@ -234,8 +234,7 @@ fn cms_with_heap_epsilon_shrinks_monotonically_with_heap_size_when_heap_dominate params.insert("d".to_string(), json!(5u64)); params.insert("w".to_string(), json!(1_000_000u64)); params.insert("heap_size".to_string(), json!(heap)); - let eps = - AccuracyProfile::derive(&cfg(AggregationType::CountMinSketchWithHeap, params)).epsilon; + let eps = derive(&cfg(AggregationType::CountMinSketchWithHeap, params)).epsilon; assert!( eps < last, "CMS-with-heap ε at heap={heap}: {eps} should be < {last}" @@ -256,8 +255,8 @@ fn relative_ordering_of_bounds_matches_published_intuition() { let mut params = HashMap::new(); params.insert("d".to_string(), json!(4u64)); params.insert("w".to_string(), json!(10_000u64)); - let cms = AccuracyProfile::derive(&cfg(AggregationType::CountMinSketch, params.clone())); - let cs = AccuracyProfile::derive(&cfg(AggregationType::CountSketch, params)); + let cms = derive(&cfg(AggregationType::CountMinSketch, params.clone())); + let cs = derive(&cfg(AggregationType::CountSketch, params)); // CMS at w=10_000: e/10_000 ≈ 0.000272 // CountSketch at w=10_000: 1/√10_000 = 0.01 // So cms.epsilon < cs.epsilon at this width. diff --git a/data_plane/src/tests/accuracy_in_promql_response_tests.rs b/data_plane/src/tests/accuracy_in_promql_response_tests.rs index b53a969df..b14320c56 100644 --- a/data_plane/src/tests/accuracy_in_promql_response_tests.rs +++ b/data_plane/src/tests/accuracy_in_promql_response_tests.rs @@ -19,7 +19,7 @@ use serde_json::Value; fn hll_profile() -> AccuracyProfile { AccuracyProfile { epsilon: 0.008125, - delta: 0.0, + delta: None, kind: AccuracyKind::RelativeCardinality, } } @@ -39,7 +39,7 @@ fn prometheus_response_carries_accuracy_top_level_and_infos_mirror() { let accuracy = &json["accuracy"]; assert!(!accuracy.is_null(), "accuracy field should be present"); assert_eq!(accuracy["epsilon"], 0.008125); - assert_eq!(accuracy["delta"], 0.0); + assert!(accuracy["delta"].is_null()); assert_eq!(accuracy["kind"], "relative_cardinality"); assert!( accuracy @@ -55,7 +55,7 @@ fn prometheus_response_carries_accuracy_top_level_and_infos_mirror() { let line = infos[0].as_str().unwrap(); assert!( line.contains("ε=0.008125") - && line.contains("δ=0") + && line.contains("δ=unknown") && line.contains("relative_cardinality"), "infos summary must include ε, δ, kind: got {line}" ); @@ -86,7 +86,7 @@ fn prometheus_response_per_segment_contains_all_segments_with_worst_case_top() { range_ms: [1_000, 2_000], profile: AccuracyProfile { epsilon: 0.01, - delta: 0.0, + delta: Some(0.0), kind: AccuracyKind::RelativeQuantile, }, }, @@ -95,7 +95,7 @@ fn prometheus_response_per_segment_contains_all_segments_with_worst_case_top() { range_ms: [2_000, 3_000], profile: AccuracyProfile { epsilon: 0.05, - delta: 0.01, + delta: Some(0.01), kind: AccuracyKind::RankQuantile, }, }, @@ -104,7 +104,7 @@ fn prometheus_response_per_segment_contains_all_segments_with_worst_case_top() { // Worst-case envelope = (max ε=0.05, max δ=0.01, first // non-Exact kind wins for `kind`). assert_eq!(envelope.profile.epsilon, 0.05); - assert_eq!(envelope.profile.delta, 0.01); + assert_eq!(envelope.profile.delta, Some(0.01)); assert_eq!(envelope.profile.kind, AccuracyKind::RelativeQuantile); let resp = PrometheusResponse::success(serde_json::json!({ diff --git a/docs/developer_docs/control-plane/accuracy-metadata.md b/docs/developer_docs/control-plane/accuracy-metadata.md new file mode 100644 index 000000000..aaa3b58de --- /dev/null +++ b/docs/developer_docs/control-plane/accuracy-metadata.md @@ -0,0 +1,31 @@ +# Accuracy metadata + +The control plane and data plane adapt sketch parameters to ASAPPlanner's +`DefaultAccuracyModel::sketch_guarantee`. The backend workspace's +`asap_types::accuracy` module only projects that guarantee into the existing +`epsilon`, `delta`, and `kind` response fields; it does not own family formulas. + +CMS, HLL, KLL (including HydraKLL), and DDSketch use this shared adapter. +Parameter aliases and defaults remain at the source boundary. The installed +`AggregationConfig` and its fingerprinted wire fields are unchanged. +`AccumulatorSpec` is not used for accuracy extraction because its construction +normalization differs (for example, KLL's integer range and DDSketch's accepted +aliases and validation). + +Compared with the previous metadata: + +- CMS failure probability now follows Planner's `exp(-depth)` convention. +- KLL uses Planner's DataSketches empirical 99th-percentile rank-error fit. +- HLL's epsilon is relative standard error, and its failure probability is + unknown: JSON contains `"delta": null`, and the text summary says `δ=unknown`. + Clients must not interpret null as zero. Numeric delta values remain numbers. + +GOS still adds its staleness allowance to approximate epsilon without changing +failure probability. Exact aggregates are unaffected. CountSketch and heap +retention remain backend-specific; CMS-with-heap uses the shared CMS guarantee +before applying the retention allowance. An accuracy envelope containing a +segment with unknown confidence also reports unknown confidence. + +The legacy `/api/v1/plan` response no longer includes the disconnected +`plan_summary` estimate. Its `transmission_costs` remain available. Removing the +summary planner does not implement resource-budget-aware placement in v2.