diff --git a/Cargo.lock b/Cargo.lock index 93db6508..c340f589 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -341,43 +341,22 @@ dependencies = [ ] [[package]] -name = "asap-frontend-promql" -version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPController?rev=cc18c9872bbaf0cadf43566892c3974ec9948eba#cc18c9872bbaf0cadf43566892c3974ec9948eba" -dependencies = [ - "asap-ir", - "asap-l2", - "promql-parser 0.9.0 (git+https://github.com/ProjectASAP/promql-parser?branch=asap)", -] - -[[package]] -name = "asap-ir" +name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPController?rev=cc18c9872bbaf0cadf43566892c3974ec9948eba#cc18c9872bbaf0cadf43566892c3974ec9948eba" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=cb70086b4c4a7ba89baf2516be81d0b192137a3a#cb70086b4c4a7ba89baf2516be81d0b192137a3a" dependencies = [ - "serde", + "asap-types", "serde_json", "thiserror 2.0.18", ] [[package]] -name = "asap-l2" -version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPController?rev=cc18c9872bbaf0cadf43566892c3974ec9948eba#cc18c9872bbaf0cadf43566892c3974ec9948eba" -dependencies = [ - "asap-ir", - "thiserror 2.0.18", -] - -[[package]] -name = "asap-plan" +name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPController?rev=cc18c9872bbaf0cadf43566892c3974ec9948eba#cc18c9872bbaf0cadf43566892c3974ec9948eba" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=cb70086b4c4a7ba89baf2516be81d0b192137a3a#cb70086b4c4a7ba89baf2516be81d0b192137a3a" dependencies = [ - "asap-ir", - "asap-sketch", - "serde_json", - "thiserror 2.0.18", + "asap-types", + "promql-parser 0.10.0", ] [[package]] @@ -391,11 +370,13 @@ dependencies = [ ] [[package]] -name = "asap-sketch" +name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPController?rev=cc18c9872bbaf0cadf43566892c3974ec9948eba#cc18c9872bbaf0cadf43566892c3974ec9948eba" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=cb70086b4c4a7ba89baf2516be81d0b192137a3a#cb70086b4c4a7ba89baf2516be81d0b192137a3a" dependencies = [ - "asap-ir", + "serde", + "serde_json", + "thiserror 2.0.18", ] [[package]] @@ -430,8 +411,7 @@ name = "asap_types" version = "0.1.0" dependencies = [ "anyhow", - "asap-ir", - "asap-sketch", + "asap-types", "clap 4.6.1", "serde", "serde_json", @@ -801,11 +781,9 @@ name = "control_plane" version = "0.1.0" dependencies = [ "anyhow", + "asap-aware-mapping", "asap-frontend-promql", - "asap-ir", - "asap-l2", - "asap-plan", - "asap-sketch", + "asap-types", "asap_types", "axum", "bytes", @@ -814,7 +792,7 @@ dependencies = [ "http-body-util", "parking_lot", "prometheus", - "promql-parser 0.9.0 (git+https://github.com/ProjectASAP/promql-parser?rev=c51beafb361af4cc95ed62ae377862c660ceb757)", + "promql-parser 0.9.0", "prost", "prost-build", "reqwest 0.12.28", @@ -1000,9 +978,8 @@ dependencies = [ "anyhow", "arc-swap", "arrow", - "asap-ir", "asap-precompute-rs", - "asap-sketch", + "asap-types", "asap_otel_proto", "asap_sketchlib", "asap_types", @@ -2494,7 +2471,7 @@ dependencies = [ [[package]] name = "promql-parser" version = "0.9.0" -source = "git+https://github.com/ProjectASAP/promql-parser?branch=asap#c51beafb361af4cc95ed62ae377862c660ceb757" +source = "git+https://github.com/ProjectASAP/promql-parser?rev=c51beafb361af4cc95ed62ae377862c660ceb757#c51beafb361af4cc95ed62ae377862c660ceb757" dependencies = [ "cfgrammar", "chrono", @@ -2506,8 +2483,9 @@ dependencies = [ [[package]] name = "promql-parser" -version = "0.9.0" -source = "git+https://github.com/ProjectASAP/promql-parser?rev=c51beafb361af4cc95ed62ae377862c660ceb757#c51beafb361af4cc95ed62ae377862c660ceb757" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e8c90c956be4237a93be6f2298302b56974ef862ab11e0f98869d1f27c69f88" dependencies = [ "cfgrammar", "chrono", diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index d5e4e234..b8b2d997 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -25,23 +25,20 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } chrono = { version = "0.4", features = ["serde"] } # Private mirror of GreptimeTeam/promql-parser (Apache-2.0), matching -# ASAPController's frontend-promql -- carries local patches (limitk / -# limit_ratio, a batch of experimental Prometheus functions) upstream -# crates.io 0.8 doesn't have. Pinned to a commit, not the `asap` branch, -# for the same reproducibility reason as the asap-ir pin below. +# ASAPPlanner's frontend-promql historically -- carries local patches +# (limitk / limit_ratio, a batch of experimental Prometheus functions) +# upstream crates.io 0.8 doesn't have. Pinned to a commit, not the `asap` +# branch, for the same reproducibility reason as the planner-types pin +# below. # -# Known duplication: `asap-frontend-promql` (below) references this same -# repo via a floating `branch = "asap"` rather than `rev =`, so cargo -# resolves two distinct package instances today (verified: `cargo tree -i -# promql-parser` lists both). They happen to be the same commit right -# now. Tried unifying via a workspace `[patch]` -- cargo rejects patching -# a source with a different ref of itself ("patches must point to -# different sources"), and no local checkout exists to patch to path= -# instead (unlike the asap_sketchlib precedent in the root Cargo.toml). -# Accepted as a known, low-severity tradeoff (extra compile time, a -# latent divergence risk if either pin moves) rather than forcing a -# fragile workaround -- revisit if `asap-frontend-promql` ever pins via -# `rev` itself. +# ASAPPlanner's own `asap-frontend-promql` has since switched to the +# public crates.io `promql-parser = "0.10"` release (ASAPPlanner#209) -- +# 0.10 upstreamed most of what this fork carries as patches (the +# duplication note that used to live here is stale now that upstream +# isn't on a floating branch at all), except the legacy `holt_winters` +# alias this repo doesn't reference. Dropping this private-fork pin in +# favor of the public 0.10 release is a reasonable follow-up but is out +# of scope for the ASAPController -> ASAPPlanner pin migration. promql-parser = { git = "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/ProjectASAP/promql-parser", rev = "c51beafb361af4cc95ed62ae377862c660ceb757" } prost = "0.13" bytes = "1" @@ -52,49 +49,60 @@ tonic = { version = "0.12", features = ["gzip"] } tokio-stream = { version = "0.1", features = ["net"] } asap_types.workspace = true -# Transitional pin (docs/migration-plan-backend-plan.md Phase 1b): locked to a -# specific commit on ASAPController's main, not a tag -- ASAPController has no -# tagged releases yet. Re-pin as ASAPController's IR evolves; move to a tag -# once one exists. +# Transitional pin (docs/migration-plan-backend-plan.md Phase 1b; see also +# control_plane/docs/design-asapplanner-pin-migration.md for the migration +# this bump landed): locked to a specific commit on ASAPPlanner's main, +# not a tag -- ASAPPlanner has no tagged releases yet. Re-pin as +# ASAPPlanner's IR evolves; move to a tag once one exists. # -# Bumped to fc09c3a (main tip, "feat(sketch): SummaryExecutor -- -# serving-time execution model for L4Node", #155) to pick up -# `asap_sketch::exec::{SummaryExecutor, execute}` -- the serving-time -# counterpart to `asap_plan::bind`'s planning-time L4Node construction, -# needed for data_plane/docs/l4node-plan-executor-design.md (Step C). -# fc09c3a is a strict descendant of 7fcaf91 (the previous pin, "add -# serving-time traits to WindowKind" / Phase 2 #395), so nothing this -# repo already consumes from that revision moves. +# Bumped to cb70086 (main tip, 2026-08-22) from cc18c987. In between, +# ASAPPlanner consolidated its crates (ASAPPlanner#198): `asap-ir` and +# `asap-sketch` merged into one crate -- Cargo package `asap-types`, with +# `pre_asap`/`post_asap` modules replacing the old L3/L4 split -- and +# `asap-l2` was deleted outright (folded into `asap-frontend-promql` + +# `asap-types::pre_asap`). Depended on here under the `planner-types` key +# (via `package = "asap-types"`), not `asap-types` directly, because this +# repo already has its own crate literally named `asap_types` +# (crates/asap_types) -- Cargo canonicalizes `-`/`_` for the extern-crate +# identifier, so the two would collide on the same Rust path under the +# upstream package's own name. `package =` renames the *local* identifier +# only; nothing upstream changes. # # asap-plan owns the single authoritative AggIntent -> sketch-vs-exact # implementation decision (`boundary::implementation_for`, Cascades # "implementation rule" terminology -- see that crate's doc); capability_for -# delegates to it instead of maintaining its own parallel judgment. asap-plan -# depends only on asap-ir (per its own crate doc's layering invariant), so -# this doesn't pull in datafusion or any front-end weight. asap-sketch is -# asap-plan's own dependency (SummaryKind/SummaryParams), needed here only -# to translate Implementation into this repo's own Capability vocabulary. +# delegates to it instead of maintaining its own parallel judgment. It is +# now its own crate, `asap-aware-mapping` (same role, same public API +# shape -- `implement_tree`/`implement_tree_with`, +# `CostModel::{rank_candidates, size_params, realize_extension, +# readout_extension}`, `boundary::Matcher` -- just relocated and renamed +# alongside the rest of the restructure). It still depends only on +# `asap-types` (per its own crate doc's layering invariant), so this +# doesn't pull in datafusion or any front-end weight. # -# Bumped to 64df20d (main tip, "feat(plan): pluggable AggIntent::Extension -# realization + readout hook", #162) to pick up # `CostModel::{realize_extension, readout_extension}` -- the pluggable # `AggIntent::Extension` hook this repo's `Extension{"frequency"}` intent -# needs (ASAPController#150). 64df20d is a strict descendant of d4c1756 -# (the previous pin), so nothing this repo already consumes moves. -asap-ir = { git = "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/ProjectASAP/ASAPController", rev = "cc18c9872bbaf0cadf43566892c3974ec9948eba" } -asap-l2 = { git = "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/ProjectASAP/ASAPController", rev = "cc18c9872bbaf0cadf43566892c3974ec9948eba" } -asap-sketch = { git = "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/ProjectASAP/ASAPController", rev = "cc18c9872bbaf0cadf43566892c3974ec9948eba" } -asap-plan = { git = "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/ProjectASAP/ASAPController", rev = "cc18c9872bbaf0cadf43566892c3974ec9948eba" } +# needs (ASAPPlanner#150) -- kept the same shape across the restructure +# (verified by reading `asap-aware-mapping/src/cost_model.rs` on `main`). +# +# `asap_sketch::exec` (`SummaryExecutor`/`execute`) -- the serving-time +# counterpart to `asap-aware-mapping`'s planning-time `SummaryNode` +# construction, needed for data_plane/docs/l4node-plan-executor-design.md +# (Step C) -- was deleted upstream (ASAPPlanner#190/#197): upstream's own +# repo search found no implementor and concluded the module was dead +# scaffolding, unaware that `data_plane`'s `summary_executor.rs` in *this* +# repo is a real one. Vendored locally instead of chased upstream -- see +# `data_plane/src/query_engines/asap_query_engine/summary_exec.rs`. +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "cb70086b4c4a7ba89baf2516be81d0b192137a3a" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "cb70086b4c4a7ba89baf2516be81d0b192137a3a" } # L1 adoption (design-target-architecture.md Part B): the PromQL front # end itself, replacing control_plane's own query_parser/promql.rs. -# Pinned via `rev`, not ASAPController's own `branch = "asap"` reference -# on its internal `promql-parser` dependency -- keeps this crate's own -# `promql-parser` pin (above) as the single source of truth for that -# transitive dependency's exact commit, avoiding two git-sourced copies -# of the same crate diverging over time. Same rev as the other -# ASAPController crates above -- no separate re-pin history yet. -asap-frontend-promql = { git = "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/ProjectASAP/ASAPController", rev = "cc18c9872bbaf0cadf43566892c3974ec9948eba" } +# Pinned via `rev`, not a floating branch reference. Same rev as +# `planner-types`/`asap-aware-mapping` above -- these three MUST move +# together (two revs of the same upstream repo's types in one workspace +# resolve to distinct Rust types that won't unify). +asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "cb70086b4c4a7ba89baf2516be81d0b192137a3a" } [dev-dependencies] tokio = { version = "1", features = ["full", "test-util"] } diff --git a/control_plane/src/asap_tier_analysis.rs b/control_plane/src/asap_tier_analysis.rs index 78688a94..1424eb3f 100644 --- a/control_plane/src/asap_tier_analysis.rs +++ b/control_plane/src/asap_tier_analysis.rs @@ -15,7 +15,7 @@ //! ↓ query_parser::parse_query (the control plane's PromQL → ParsedQuery) //! ParsedQuery //! ↓ intent_algebra::lower::lower_parsed_query -//! QueryExpr (intent_algebra) — Scan / Window / Aggregate{ aggs: Vec } +//! QueryExpr (intent_algebra) — Scan / Window / Aggregate{ measures: Vec } //! ↓ walk and call capability_for(&AggIntent) //! Vec //! ``` @@ -339,16 +339,16 @@ fn render_spatial_filter(label_filters: &std::collections::HashMap) { match expr { - QueryExpr::Aggregate { aggs, child, .. } => { + QueryExpr::Aggregate { + measures: aggs, + child, + .. + } => { out.extend(aggs.iter().cloned()); collect_agg_intents(child, out); } - QueryExpr::Window { child, .. } => collect_agg_intents(child, out), - QueryExpr::LetBinding { expr, child, .. } => { - collect_agg_intents(expr, out); - collect_agg_intents(child, out); - } - QueryExpr::Scan { .. } | QueryExpr::Ref { .. } => {} + QueryExpr::TimeRange { child, .. } => collect_agg_intents(child, out), + QueryExpr::Scan { .. } => {} // A-variants lifted in Batch 2 of the relational migration. They // carry no AggIntent themselves — recurse into their children to // find Aggregates further down the tree. `Partition` no longer @@ -1976,7 +1976,8 @@ mod tests { #[test] fn empty_registry_yields_empty_matches() { - let registry = RoutingIndex::build(PolicyRegistry::from_configs(Vec::::new())); + let registry = + RoutingIndex::build(PolicyRegistry::from_configs(Vec::::new())); let cand = candidate( "http_lat", &[], diff --git a/control_plane/src/asap_tier_implement.rs b/control_plane/src/asap_tier_implement.rs index 08ab54f1..2f79a67c 100644 --- a/control_plane/src/asap_tier_implement.rs +++ b/control_plane/src/asap_tier_implement.rs @@ -1,4 +1,4 @@ -//! PromQL → compositional L4 plan (`asap_plan::bind::implement_tree` +//! PromQL → compositional L4 plan (`asap_aware_mapping::bind::implement_tree` //! adoption, Step A of the plan-shaped-serving scoping). //! //! ## Terminology: "implementation", not "bind" @@ -17,9 +17,9 @@ //! logical) so it doesn't become a fifth colliding sense. Following that //! lead here: [`collect_aggregate_roots`] *finds* realizable `Aggregate` //! subtrees, [`implement_promql_for_asap_tier`] *implements* (realizes) -//! them via `asap_plan::bind::implement_tree_in_with` — "bind" is used +//! them via `asap_aware_mapping::bind::implement_tree_in_with` — "bind" is used //! below only where citing `asap-plan`'s own module/function names -//! (`asap_plan::bind`, `implement_tree_in_with`) or the specific +//! (`asap_aware_mapping::bind`, `implement_tree_in_with`) or the specific //! deployment-side "Bind #2" placement decision that table names. //! //! ## The gap this closes @@ -31,8 +31,8 @@ //! composes multiple accumulators into a multi-stage plan (`Avg = Sum / //! Count` stays archive-only for exactly this reason). //! -//! `asap_plan::bind::implement_tree_in_with` already builds a genuinely -//! compositional plan (`Rc` / `SummaryExpr` — `SummaryAgg`, +//! `asap_aware_mapping::bind::implement_tree_in_with` already builds a genuinely +//! compositional plan (`Rc` / `SummaryExpr` — `SummaryAgg`, //! `SummaryEstimate`, `SummaryMerge`, ...) from an L3 `QueryExpr` — but //! it is deliberately conservative about *where* it looks for a //! realizable `Aggregate`: hitting any non-`Aggregate` node (`Filter`, @@ -65,18 +65,18 @@ //! path. `implement_frequency_as_agg_test` below pins this as a known, //! tracked gap (not a silent trap) — closing it needs either an //! `asap-plan` extension hook or a hand-rolled `SummaryAgg`/ -//! `SummaryEstimate` construction here (the L4Node-building helpers in -//! `asap_plan::bind` are private, so today there's no way to do the +//! `SummaryEstimate` construction here (the SummaryNode-building helpers in +//! `asap_aware_mapping::bind` are private, so today there's no way to do the //! latter without reimplementing them). Not yet wired into the live //! serving path for this reason — see module doc for the broader //! plan-shaped-serving scope this is Step A of. use std::rc::Rc; -use asap_plan::{implement_tree_in_with, DefaultCostModel, ImplementError}; -use asap_sketch::L4Node; +use asap_aware_mapping::{implement_tree_with, DefaultCostModel, ImplementError}; +use planner_types::post_asap::SummaryNode; -use crate::intent_algebra::query_expr::{BindingScope, QueryExpr}; +use crate::intent_algebra::query_expr::QueryExpr; use crate::query_parser::parse_query_expr_canonical; use crate::types_v2::AccuracyTarget; @@ -110,7 +110,7 @@ const IMPLEMENT_PROMQL_ACCURACY: AccuracyTarget = AccuracyTarget::Epsilon(0.01); fn collect_aggregate_roots<'a>(expr: &'a QueryExpr, out: &mut Vec<&'a QueryExpr>) { match expr { QueryExpr::Aggregate { - aggs, + measures: aggs, having, child, .. @@ -121,12 +121,8 @@ fn collect_aggregate_roots<'a>(expr: &'a QueryExpr, out: &mut Vec<&'a QueryExpr> collect_aggregate_roots(child, out); } } - QueryExpr::Window { child, .. } => collect_aggregate_roots(child, out), - QueryExpr::LetBinding { expr, child, .. } => { - collect_aggregate_roots(expr, out); - collect_aggregate_roots(child, out); - } - QueryExpr::Scan { .. } | QueryExpr::Ref { .. } => {} + QueryExpr::TimeRange { child, .. } => collect_aggregate_roots(child, out), + QueryExpr::Scan { .. } => {} // A-variants lifted in Batch 2 of the relational migration. They // carry no AggIntent themselves — recurse into their children to // find Aggregates further down the tree. `Partition` no longer @@ -168,15 +164,15 @@ pub enum ImplementPromqlError { /// PromQL failed to parse / lower to canonical `QueryExpr`. UnparseableMetricsql(String), /// L3→L4 implementation failed for a found `Aggregate` root (schema - /// derivation error — see `asap_plan::bind::ImplementError`). + /// derivation error — see `asap_aware_mapping::bind::ImplementError`). Implement(ImplementError), } /// Parse `metricsql`, find every independently-realizable `Aggregate` /// subtree ([`collect_aggregate_roots`]), and implement each into a -/// compositional L4 plan via `asap_plan::bind::implement_tree_in_with`. +/// compositional L4 plan via `asap_aware_mapping::bind::implement_tree_in_with`. /// -/// Returns one `Rc` per root found, in tree order. Empty (not an +/// Returns one `Rc` per root found, in tree order. Empty (not an /// error) when the query has no realizable `Aggregate` at all (a bare /// selector, or a window-bound exact-aggregation the lowerer didn't /// emit an `Aggregate` for) — same "no candidates" contract @@ -188,7 +184,7 @@ pub enum ImplementPromqlError { /// path. pub fn implement_promql_for_asap_tier( metricsql: &str, -) -> Result>, ImplementPromqlError> { +) -> Result>, ImplementPromqlError> { let expr = parse_query_expr_canonical(metricsql, IMPLEMENT_PROMQL_ACCURACY) .map_err(|e| ImplementPromqlError::UnparseableMetricsql(e.to_string()))?; @@ -198,8 +194,7 @@ pub fn implement_promql_for_asap_tier( roots .into_iter() .map(|root| { - implement_tree_in_with(root, &BindingScope::default(), &DefaultCostModel) - .map_err(ImplementPromqlError::Implement) + implement_tree_with(root, &DefaultCostModel).map_err(ImplementPromqlError::Implement) }) .collect() } @@ -207,7 +202,7 @@ pub fn implement_promql_for_asap_tier( #[cfg(test)] mod tests { use super::*; - use asap_sketch::SummaryExpr; + use planner_types::post_asap::SummaryExpr; fn root_count(metricsql: &str) -> usize { implement_promql_for_asap_tier(metricsql) diff --git a/control_plane/src/backend_client.rs b/control_plane/src/backend_client.rs index d2bd0e75..932b836e 100644 --- a/control_plane/src/backend_client.rs +++ b/control_plane/src/backend_client.rs @@ -572,17 +572,16 @@ mod tests { async fn backend_plan_post_round_trips_bytes_via_url_rewrite() { let hits: StdArc>>> = StdArc::new(Mutex::new(Vec::new())); let hits_for_route = hits.clone(); - let app = Router::new() - .route( - "/api/v1/backend-plan", - post(move |body: axum::body::Bytes| { - let hits = hits_for_route.clone(); - async move { - hits.lock().unwrap().push(body.to_vec()); - axum::http::StatusCode::OK - } - }), - ); + let app = Router::new().route( + "/api/v1/backend-plan", + post(move |body: axum::body::Bytes| { + let hits = hits_for_route.clone(); + async move { + hits.lock().unwrap().push(body.to_vec()); + axum::http::StatusCode::OK + } + }), + ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); tokio::spawn(async move { diff --git a/control_plane/src/backend_plan/from_stage_config.rs b/control_plane/src/backend_plan/from_stage_config.rs index d632f417..7249eb6a 100644 --- a/control_plane/src/backend_plan/from_stage_config.rs +++ b/control_plane/src/backend_plan/from_stage_config.rs @@ -20,7 +20,7 @@ use std::collections::HashMap; use anyhow::{Context, Result}; -use asap_sketch::SummaryKind; +use asap_types::SummaryKind; use asap_types::{AggregationConfig, MonitorSpec, PolicyFingerprint, QueryLanguage}; use crate::emit::monitor::{agg_id_for_metric, MonitorIntent}; @@ -147,8 +147,10 @@ fn policy_fingerprint_for_aggregation(agg: &BackendAggregation) -> Result Result<(SummaryKind, asap_sketch::SummaryParams)> { - use asap_sketch::SummaryParams; +fn exact_kind_params_for_override( + exact_type: &str, +) -> Result<(SummaryKind, asap_types::SummaryParams)> { + use asap_types::SummaryParams; match exact_type { "Sum" => Ok((SummaryKind::Sum, SummaryParams::Sum)), "Count" => Ok((SummaryKind::Count, SummaryParams::Count)), @@ -166,9 +168,12 @@ fn exact_kind_params_for_override(exact_type: &str) -> Result<(SummaryKind, asap /// the sketch family + readout op, matching /// `sketch_algebra::capability::Capability`'s variant-per-query-shape /// design. -fn capability_for_readout(agg: &BackendAggregation, op: &asap_sketch::SketchQuery) -> Result { - use asap_sketch::SketchQuery; +fn capability_for_readout( + agg: &BackendAggregation, + op: &planner_types::post_asap::SketchQuery, +) -> Result { use asap_types::AggregationType; + use planner_types::post_asap::SketchQuery; if let Some(exact_type) = &agg.agg_type_override { let agg_type: AggregationType = exact_type @@ -207,8 +212,10 @@ fn sketch_kind_handle(kind: &SummaryKind) -> Result { mod tests { use super::*; use crate::physical::colored_dag::emitter::{AggregationInput, BackendReadout}; - use asap_sketch::{SketchQuery, SummaryParams}; - use asap_types::{AggregationType, KeyByLabelNames, WindowKind as AsapWindowKind}; + use asap_types::{ + AggregationType, KeyByLabelNames, SummaryParams, WindowKind as AsapWindowKind, + }; + use planner_types::post_asap::SketchQuery; use std::collections::HashMap as StdHashMap; fn agg( @@ -327,10 +334,9 @@ mod tests { for agg in &cfg.aggregations { let expected = hand_built_config(agg).policy_fingerprint(); - let m = plan - .materializations - .get(&expected) - .unwrap_or_else(|| panic!("no materialization for expected fingerprint of {agg:?}")); + let m = plan.materializations.get(&expected).unwrap_or_else(|| { + panic!("no materialization for expected fingerprint of {agg:?}") + }); assert_eq!(m.fingerprint, expected); } } @@ -374,7 +380,10 @@ mod tests { .iter() .find(|r| r.materialization == ddsketch_fp) .expect("routing entry for ddsketch"); - assert_eq!(quantile_entry.satisfies, Capability::QuantileApprox(SketchKindHandle::DDSketch)); + assert_eq!( + quantile_entry.satisfies, + Capability::QuantileApprox(SketchKindHandle::DDSketch) + ); let hll_fp = plan .materializations @@ -431,17 +440,24 @@ mod tests { ], }; let plan = from_stage_config(&cfg, &[], 1, 0).expect("build plan"); - let topk_entry = plan - .routing - .iter() - .find(|r| r.satisfies == Capability::FrequencyTopk(SketchKindHandle::CountSketchWithHeap)); - assert!(topk_entry.is_some(), "expected a FrequencyTopk routing entry: {:?}", plan.routing); + let topk_entry = plan.routing.iter().find(|r| { + r.satisfies == Capability::FrequencyTopk(SketchKindHandle::CountSketchWithHeap) + }); + assert!( + topk_entry.is_some(), + "expected a FrequencyTopk routing entry: {:?}", + plan.routing + ); let freq_entry = plan .routing .iter() .find(|r| r.satisfies == Capability::FrequencyEstimate(SketchKindHandle::CountMin)); - assert!(freq_entry.is_some(), "expected a FrequencyEstimate routing entry: {:?}", plan.routing); + assert!( + freq_entry.is_some(), + "expected a FrequencyEstimate routing entry: {:?}", + plan.routing + ); } #[test] @@ -462,7 +478,11 @@ mod tests { }], }; let plan = from_stage_config(&cfg, &[], 1, 0).expect("build plan"); - let (_, m) = plan.materializations.iter().next().expect("one materialization"); + let (_, m) = plan + .materializations + .iter() + .next() + .expect("one materialization"); assert!(m.kind.is_exact()); assert_eq!(m.kind, SummaryKind::Sum); diff --git a/control_plane/src/backend_plan/mod.rs b/control_plane/src/backend_plan/mod.rs index ffcba029..b045051c 100644 --- a/control_plane/src/backend_plan/mod.rs +++ b/control_plane/src/backend_plan/mod.rs @@ -7,7 +7,7 @@ //! `proto/backend_plan.proto`), and the conversions between them. //! //! Deliberately reuses this deployment's existing canonical vocabulary -//! rather than re-encoding it: `asap_sketch::{SummaryKind, SummaryParams}` +//! rather than re-encoding it: `planner_types::post_asap::{SummaryKind, SummaryParams}` //! for the materialization payload (no separate `ExactAggregate` variant — //! see the design doc §3 for why), `asap_ir`/`crate::intent_algebra`'s //! `Source`/`ColumnRef`/`WindowKind` for the L3 IR fragments, @@ -21,7 +21,10 @@ pub mod proto { #![allow(clippy::all)] - include!(concat!(env!("OUT_DIR"), "/control_plane.backend_plan.v1.rs")); + include!(concat!( + env!("OUT_DIR"), + "/control_plane.backend_plan.v1.rs" + )); } mod from_stage_config; @@ -29,8 +32,8 @@ pub use from_stage_config::from_stage_config; use std::collections::HashMap; -use asap_sketch::{SummaryKind, SummaryParams}; use asap_types::{AggregationType, MonitorSpec, PolicyFingerprint}; +use asap_types::{SummaryKind, SummaryParams}; use prost::Message as _; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -79,10 +82,11 @@ impl From<&WindowSpec> for proto::WindowSpec { impl TryFrom for WindowSpec { type Error = DecodeError; fn try_from(w: proto::WindowSpec) -> Result { - let kind = proto::WindowKind::try_from(w.kind).map_err(|_| DecodeError::UnknownEnumValue { - field: "WindowSpec.kind", - value: w.kind, - })?; + let kind = + proto::WindowKind::try_from(w.kind).map_err(|_| DecodeError::UnknownEnumValue { + field: "WindowSpec.kind", + value: w.kind, + })?; Ok(WindowSpec { kind: kind.try_into()?, size_ms: w.size_ms, @@ -200,7 +204,9 @@ impl From<&SummaryParams> for proto::SummaryParams { SummaryParams::Hll { precision } => Wire::Hll(proto::HllParams { precision: *precision as u32, }), - SummaryParams::DDSketch { alpha } => Wire::Ddsketch(proto::DdSketchParams { alpha: *alpha }), + SummaryParams::DDSketch { alpha } => { + Wire::Ddsketch(proto::DdSketchParams { alpha: *alpha }) + } SummaryParams::CmsWithHeap { width, depth, @@ -238,7 +244,9 @@ impl From<&SummaryParams> for proto::SummaryParams { /// pair. Not a `TryFrom` impl because the single wire message decodes to /// TWO domain values (`SummaryKind` is implied, not carried separately on /// the wire) — see this module's doc. -pub fn decode_summary_params(p: proto::SummaryParams) -> Result<(SummaryKind, SummaryParams), DecodeError> { +pub fn decode_summary_params( + p: proto::SummaryParams, +) -> Result<(SummaryKind, SummaryParams), DecodeError> { use proto::summary_params::Params as Wire; let params = p.params.ok_or(DecodeError::MissingOneof("SummaryParams"))?; Ok(match params { @@ -261,7 +269,10 @@ pub fn decode_summary_params(p: proto::SummaryParams) -> Result<(SummaryKind, Su precision: h.precision as u8, }, ), - Wire::Ddsketch(d) => (SummaryKind::DDSketch, SummaryParams::DDSketch { alpha: d.alpha }), + Wire::Ddsketch(d) => ( + SummaryKind::DDSketch, + SummaryParams::DDSketch { alpha: d.alpha }, + ), Wire::CmsWithHeap(c) => ( SummaryKind::CmsWithHeap, SummaryParams::CmsWithHeap { @@ -317,7 +328,9 @@ impl TryFrom for SketchKindHandle { proto::SketchKindHandle::CountSketch => Ok(SketchKindHandle::CountSketch), proto::SketchKindHandle::CountMin => Ok(SketchKindHandle::CountMin), proto::SketchKindHandle::CmsWithHeap => Ok(SketchKindHandle::CmsWithHeap), - proto::SketchKindHandle::CountSketchWithHeap => Ok(SketchKindHandle::CountSketchWithHeap), + proto::SketchKindHandle::CountSketchWithHeap => { + Ok(SketchKindHandle::CountSketchWithHeap) + } proto::SketchKindHandle::Any => Ok(SketchKindHandle::Any), proto::SketchKindHandle::Unspecified => Err(DecodeError::UnknownEnumValue { field: "SketchKindHandle", @@ -339,7 +352,9 @@ impl From for proto::AggregationType { AggregationType::MultipleMinMax => proto::AggregationType::MultipleMinMax, AggregationType::HydraKLL => proto::AggregationType::HydraKll, AggregationType::CountMinSketch => proto::AggregationType::CountMinSketch, - AggregationType::CountMinSketchWithHeap => proto::AggregationType::CountMinSketchWithHeap, + AggregationType::CountMinSketchWithHeap => { + proto::AggregationType::CountMinSketchWithHeap + } AggregationType::CountSketch => proto::AggregationType::CountSketch, AggregationType::CountSketchWithHeap => proto::AggregationType::CountSketchWithHeap, AggregationType::HLL => proto::AggregationType::Hll, @@ -408,12 +423,16 @@ impl TryFrom for Capability { type Error = DecodeError; fn try_from(c: proto::Capability) -> Result { use proto::capability::Capability as Wire; - let decode_handle = |v: i32, field: &'static str| -> Result { - proto::SketchKindHandle::try_from(v) - .map_err(|_| DecodeError::UnknownEnumValue { field, value: v })? - .try_into() - }; - match c.capability.ok_or(DecodeError::MissingOneof("Capability"))? { + let decode_handle = + |v: i32, field: &'static str| -> Result { + proto::SketchKindHandle::try_from(v) + .map_err(|_| DecodeError::UnknownEnumValue { field, value: v })? + .try_into() + }; + match c + .capability + .ok_or(DecodeError::MissingOneof("Capability"))? + { Wire::QuantileApprox(v) => Ok(Capability::QuantileApprox(decode_handle( v, "Capability.quantile_approx", @@ -428,9 +447,11 @@ impl TryFrom for Capability { "Capability.frequency_topk", )?)), Wire::ExactAgg(v) => { - let agg = proto::AggregationType::try_from(v).map_err(|_| DecodeError::UnknownEnumValue { - field: "Capability.exact_agg", - value: v, + let agg = proto::AggregationType::try_from(v).map_err(|_| { + DecodeError::UnknownEnumValue { + field: "Capability.exact_agg", + value: v, + } })?; Ok(Capability::ExactAgg(agg.try_into()?)) } @@ -575,7 +596,8 @@ impl TryFrom for Materialization { type Error = DecodeError; fn try_from(m: proto::Materialization) -> Result { let (kind, params) = decode_summary_params( - m.params.ok_or(DecodeError::MissingOneof("Materialization.params"))?, + m.params + .ok_or(DecodeError::MissingOneof("Materialization.params"))?, )?; Ok(Materialization { fingerprint: PolicyFingerprint(m.fingerprint), @@ -625,11 +647,12 @@ impl From<&RoutingEntry> for proto::RoutingEntry { impl TryFrom for RoutingEntry { type Error = DecodeError; fn try_from(r: proto::RoutingEntry) -> Result { - let storage_backend = - proto::StorageBackend::try_from(r.storage_backend).map_err(|_| DecodeError::UnknownEnumValue { + let storage_backend = proto::StorageBackend::try_from(r.storage_backend).map_err(|_| { + DecodeError::UnknownEnumValue { field: "RoutingEntry.storage_backend", value: r.storage_backend, - })?; + } + })?; Ok(RoutingEntry { satisfies: r .satisfies @@ -714,7 +737,11 @@ mod tests { use super::*; use std::time::Duration; - fn sample_materialization(fingerprint: u64, kind: SummaryKind, params: SummaryParams) -> Materialization { + fn sample_materialization( + fingerprint: u64, + kind: SummaryKind, + params: SummaryParams, + ) -> Materialization { Materialization { fingerprint: PolicyFingerprint(fingerprint), source: Source::TimeSeries { @@ -824,7 +851,10 @@ mod tests { }, ), (SummaryKind::Hll, SummaryParams::Hll { precision: 14 }), - (SummaryKind::DDSketch, SummaryParams::DDSketch { alpha: 0.01 }), + ( + SummaryKind::DDSketch, + SummaryParams::DDSketch { alpha: 0.01 }, + ), ( SummaryKind::CmsWithHeap, SummaryParams::CmsWithHeap { diff --git a/control_plane/src/emit/backend_push.rs b/control_plane/src/emit/backend_push.rs index 9e8f7a2a..5da411e4 100644 --- a/control_plane/src/emit/backend_push.rs +++ b/control_plane/src/emit/backend_push.rs @@ -723,14 +723,14 @@ mod tests { use crate::physical::colored_dag::emitter::{ AggregationInput, BackendAggregation, BackendReadout, }; - use asap_sketch::SketchQuery; - use asap_sketch::{SummaryKind, SummaryParams}; + use planner_types::post_asap::SketchQuery; + use planner_types::post_asap::{SketchKind, SketchParams}; BackendStageConfig { aggregations: vec![BackendAggregation { aggregation_id: agg_id.to_string(), metric_name: metric.to_string(), - sketch_kind: SummaryKind::DDSketch, - sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, + sketch_kind: SketchKind::DDSketch.into(), + sketch_params: SketchParams::DDSketch { alpha: 0.01 }.into(), grouping: vec![], item_label: None, spatial_filter: String::new(), diff --git a/control_plane/src/emit/mod.rs b/control_plane/src/emit/mod.rs index dcad45a4..33203971 100644 --- a/control_plane/src/emit/mod.rs +++ b/control_plane/src/emit/mod.rs @@ -50,7 +50,7 @@ use crate::sketch_algebra::physical_expr::L4Plan; use crate::sketch_algebra::PhysicalExpr; use crate::store::WorkloadStore; use anyhow::Result; -use asap_sketch::{L4Node, SummaryExpr, SummaryKind}; +use planner_types::post_asap::{SketchKind, SummaryExpr, SummaryNode}; use std::rc::Rc; /// Phase ε.1.5 — which edge runtime an agent identifies as. @@ -287,7 +287,7 @@ fn apply_cold_format_from_env(edge_cfg: &mut EdgeStageConfig) { /// (`Logical`-only, unresolved `Ref`, raw Mode-3 archive). These map /// onto the raw-passthrough default pipeline in the routing emitter, /// which is correct. -pub fn extract_root_sketch_kind(expr: &PhysicalExpr) -> Option { +pub fn extract_root_sketch_kind(expr: &PhysicalExpr) -> Option { match expr { PhysicalExpr::Committed(plan) => extract_from_plan(plan), PhysicalExpr::RawAtEdgeSketchAtBackend { family, .. } => Some(family.clone()), @@ -295,7 +295,7 @@ pub fn extract_root_sketch_kind(expr: &PhysicalExpr) -> Option { } } -fn extract_from_plan(plan: &L4Plan) -> Option { +fn extract_from_plan(plan: &L4Plan) -> Option { match plan { L4Plan::Summary(node) => extract_from_node(node), L4Plan::LetBinding { expr, child, .. } => { @@ -305,27 +305,17 @@ fn extract_from_plan(plan: &L4Plan) -> Option { } } -/// Is `kind` an exact accumulator (Sum/Count/MinMax/Increase/Rate) rather -/// than an approximate sketch? Exact accumulators have no sketch family -/// for the 5-sketch routing connector to route on — same as the old, -/// now-retired `PhysicalExpr::ExactAgg` variant, which this function -/// treated as `None`. -fn is_exact_accumulator(kind: &SummaryKind) -> bool { - matches!( - kind, - SummaryKind::Sum - | SummaryKind::Count - | SummaryKind::MinMax - | SummaryKind::Increase - | SummaryKind::Rate - ) -} - -fn extract_from_node(node: &Rc) -> Option { +fn extract_from_node(node: &Rc) -> Option { match &node.expr { - SummaryExpr::SummaryAgg { summary, .. } if !is_exact_accumulator(summary) => { - Some(summary.clone()) - } + // `SummaryAgg`'s `kind`/`params` collapsed into one `family: + // SummaryFamilyType` field (ASAPPlanner#218 -- see + // control_plane/docs/design-asapplanner-pin-migration.md); the + // exact-vs-sketch check this used to need `is_exact_accumulator` + // for is now which enum variant `family` is. + SummaryExpr::SummaryAgg { + family: planner_types::post_asap::SummaryFamilyType::Sketch(kind, _), + .. + } => Some(kind.clone()), // An exact accumulator has no sketch family beneath it (its own // child is always a plain `Logical` leaf) — same as the old // `ExactAgg` case. @@ -352,7 +342,7 @@ fn extract_from_node(node: &Rc) -> Option { /// (`quantile_over_time` → DDSketch, `count`-distinct → HLL, `topk` → /// CountSketch, …). We therefore collect the UNION of every workload /// entry's committed sketch family per metric into a -/// `BTreeSet` (deterministic order). The emitter routes the +/// `BTreeSet` (deterministic order). The emitter routes the /// metric to EACH family in its set and prunes pipelines/processors to /// the union of all sets — eliminating the prior all-5 fan-out that /// shipped sketch state through every family regardless of need. @@ -372,8 +362,8 @@ fn extract_from_node(node: &Rc) -> Option { pub fn collect_metric_to_family( registry: &WorkloadRegistry, workload_store: &WorkloadStore, -) -> std::collections::HashMap> { - let mut out: std::collections::HashMap> = +) -> std::collections::HashMap> { + let mut out: std::collections::HashMap> = std::collections::HashMap::new(); for entry in registry.entries() { // B2 (metric, role) restructure: walk EVERY role registered for @@ -775,7 +765,7 @@ mod runtime_tests { let _env = crate::test_support::env_lock(); use crate::physical::colored_dag::emitter::{EdgeSketchProcessor, ExportTarget}; use crate::physical::colored_dag::stage_id::StageId; - use asap_sketch::SummaryParams; + use planner_types::post_asap::SketchParams; let cfg = EdgeStageConfig { source_metric: Some("m".to_string()), @@ -783,8 +773,8 @@ mod runtime_tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "ddsketch".to_string(), - sketch_kind: SummaryKind::DDSketch, - sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, + sketch_kind: SketchKind::DDSketch, + sketch_params: SketchParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".to_string(), }], exporter_target: ExportTarget::Stage(StageId::Gateway), @@ -960,7 +950,7 @@ mod runtime_tests { #[test] fn collect_metric_to_family_binds_all_six_contract_metrics_from_live_yaml() { - use asap_sketch::SummaryKind; + use planner_types::post_asap::SketchKind; // The 6 contract metrics reproduced inline (mirrors // deploy/configs/mvp-workload.yaml entries 1, 5, 6, 7, 8 plus the @@ -1014,23 +1004,23 @@ mod runtime_tests { // metric needs. For THIS workload every sketched metric is // queried by exactly one capability, so each set has size 1. use std::collections::BTreeSet; - let expected: Vec<(&str, Option>)> = vec![ + let expected: Vec<(&str, Option>)> = vec![ ( "http_latency_ms", - Some(BTreeSet::from([SummaryKind::DDSketch])), + Some(BTreeSet::from([SketchKind::DDSketch])), ), ("http_requests_total", None), // raw passthrough ( "request_size_bytes", - Some(BTreeSet::from([SummaryKind::Kll])), + Some(BTreeSet::from([SketchKind::Kll])), ), ( "unique_users_per_min", - Some(BTreeSet::from([SummaryKind::Hll])), + Some(BTreeSet::from([SketchKind::Hll])), ), ( "top_endpoint_qps", - Some(BTreeSet::from([SummaryKind::CountSketchWithHeap])), + Some(BTreeSet::from([SketchKind::CountSketchWithHeap])), ), // `CountMinSketch` override re-derives statistic to // `Frequency`, `AggIntent::Extension`-shaped — now binds via @@ -1038,7 +1028,7 @@ mod runtime_tests { // see `optimizer::rules::tests::typed_binding_endpoint_request_freq_binds_cms`). ( "endpoint_request_freq", - Some(BTreeSet::from([SummaryKind::Cms])), + Some(BTreeSet::from([SketchKind::Cms])), ), ]; for (metric, want) in &expected { @@ -1170,7 +1160,7 @@ mod runtime_tests { fn collect_metric_to_family_unions_multiple_capabilities_per_metric() { use crate::types::{AggType, QueryWorkload, SketchType, WorkloadCharacteristics}; use crate::workload::AggRole; - use asap_sketch::SummaryKind; + use planner_types::post_asap::SketchKind; use std::collections::BTreeSet; use std::time::Duration; @@ -1227,7 +1217,7 @@ mod runtime_tests { // TopK → CountSketch-with-heap. Not plain `Frequency, None` // (capability-matched CMS default) any more — `Frequency`'s // capability-matched default is `AggIntent::Extension`-shaped, - // which `asap_plan::boundary::implementation_for` maps to + // which `asap_aware_mapping::boundary::implementation_for` maps to // `PassThrough` unconditionally (ASAPController#150), so it no // longer contributes a family to the union at all. Use a // `CountSketch` override instead — it re-derives the statistic to @@ -1253,9 +1243,9 @@ mod runtime_tests { assert_eq!( got, BTreeSet::from([ - SummaryKind::DDSketch, - SummaryKind::Hll, - SummaryKind::CountSketchWithHeap + SketchKind::DDSketch, + SketchKind::Hll, + SketchKind::CountSketchWithHeap ]), "a metric queried by 3 capabilities must accumulate 3 families (UNION, not first-wins)\nmap: {map:?}" ); @@ -1323,7 +1313,7 @@ mod runtime_tests { let _env = crate::test_support::env_lock(); use crate::physical::colored_dag::emitter::{EdgeStageConfig, ExportTarget}; use crate::physical::colored_dag::stage_id::StageId; - use asap_sketch::SummaryKind; + use planner_types::post_asap::SketchKind; let yaml = r#" - metric_name: http_requests_total_latency_ms @@ -1349,7 +1339,7 @@ mod runtime_tests { warm_passthrough_metrics: Vec::new(), metric_to_family: std::collections::HashMap::from([( "http_requests_total_latency_ms".to_string(), - std::collections::BTreeSet::from([SummaryKind::DDSketch]), + std::collections::BTreeSet::from([SketchKind::DDSketch]), )]), metric_to_grouping_labels: std::collections::HashMap::new(), cumulative_counter_metrics: Vec::new(), diff --git a/control_plane/src/emit/otap.rs b/control_plane/src/emit/otap.rs index 5d6b2863..21f26fbf 100644 --- a/control_plane/src/emit/otap.rs +++ b/control_plane/src/emit/otap.rs @@ -42,7 +42,7 @@ use std::collections::BTreeMap; use crate::physical::colored_dag::emitter::{EdgeSketchProcessor, EdgeStageConfig, ExportTarget}; use crate::physical::colored_dag::stage_id::StageId; -use asap_sketch::{SummaryKind, SummaryParams}; +use planner_types::post_asap::{SketchKind, SketchParams}; /// Default URL for Prometheus's native OTLP HTTP receiver. /// Matches `super::stage_config::emit_edge_yaml`'s placeholder so the @@ -298,66 +298,54 @@ fn build_asap_sketches_config(sp: &EdgeSketchProcessor, window_secs: Option Value::String(sketch_kind_tag(&sp.sketch_kind).into()), ); match &sp.sketch_params { - SummaryParams::Kll { k } => { + SketchParams::Kll { k } => { m.insert("k".into(), Value::Number((*k as u64).into())); } - SummaryParams::DDSketch { alpha } => { + SketchParams::DDSketch { alpha } => { m.insert("relative_accuracy".into(), Value::Number((*alpha).into())); m.insert("delta_transmission".into(), Value::Bool(true)); } - SummaryParams::Hll { .. } => { + SketchParams::Hll { .. } => { m.insert("delta_transmission".into(), Value::Bool(true)); } // Heap-bearing width/depth extraction is identical to the bare // kind — this path never distinguished `with_heap` even before - // `SummaryKind` split it into its own variant (heap_size wasn't + // `SketchKind` split it into its own variant (heap_size wasn't // emitted here either way). - SummaryParams::Cms { width, depth } | SummaryParams::CmsWithHeap { width, depth, .. } => { + SketchParams::Cms { width, depth } | SketchParams::CmsWithHeap { width, depth, .. } => { m.insert("rows".into(), Value::Number((*depth as u64).into())); m.insert("columns".into(), Value::Number((*width as u64).into())); m.insert("delta_transmission".into(), Value::Bool(true)); } - SummaryParams::CountSketch { width, depth } - | SummaryParams::CountSketchWithHeap { width, depth, .. } => { + SketchParams::CountSketch { width, depth } + | SketchParams::CountSketchWithHeap { width, depth, .. } => { let epsilon = std::f64::consts::E / (*width as f64); let delta = 2f64.powi(-(*depth as i32)); m.insert("epsilon".into(), Value::Number(epsilon.into())); m.insert("delta".into(), Value::Number(delta.into())); m.insert("delta_transmission".into(), Value::Bool(true)); } - SummaryParams::Sum - | SummaryParams::Count - | SummaryParams::MinMax - | SummaryParams::Increase - | SummaryParams::Rate - | SummaryParams::Kmv { .. } - | SummaryParams::Theta { .. } => { + SketchParams::Kmv { .. } | SketchParams::Theta { .. } => { unreachable!( "edge sketch processor config requested for a non-sketch or unsupported \ - SummaryKind; no Bind* rule in this repo produces one" + SketchKind; no Bind* rule in this repo produces one" ) } } Value::Mapping(m) } -fn sketch_kind_tag(kind: &SummaryKind) -> &'static str { +fn sketch_kind_tag(kind: &SketchKind) -> &'static str { match kind { - SummaryKind::Kll => "kll", - SummaryKind::DDSketch => "ddsketch", - SummaryKind::Hll => "hll", - SummaryKind::Cms | SummaryKind::CmsWithHeap => "cms", - SummaryKind::CountSketch | SummaryKind::CountSketchWithHeap => "count_sketch", - SummaryKind::Sum - | SummaryKind::Count - | SummaryKind::MinMax - | SummaryKind::Increase - | SummaryKind::Rate - | SummaryKind::Kmv - | SummaryKind::Theta => { + SketchKind::Kll => "kll", + SketchKind::DDSketch => "ddsketch", + SketchKind::Hll => "hll", + SketchKind::Cms | SketchKind::CmsWithHeap => "cms", + SketchKind::CountSketch | SketchKind::CountSketchWithHeap => "count_sketch", + SketchKind::Kmv | SketchKind::Theta => { unreachable!( "edge sketch processor config requested for a non-sketch or unsupported \ - SummaryKind; no Bind* rule in this repo produces one" + SketchKind; no Bind* rule in this repo produces one" ) } } @@ -369,7 +357,7 @@ fn sketch_kind_tag(kind: &SummaryKind) -> &'static str { mod tests { use super::*; use crate::physical::colored_dag::emitter::{EdgeSketchProcessor, PrometheusArchiveMetric}; - use asap_sketch::{SummaryKind, SummaryParams}; + use planner_types::post_asap::{SketchKind, SketchParams}; /// Minimal struct-stub used to validate the emitted DAG parses as the /// otap-dataflow schema. We don't pull in the otap-df-config crate @@ -416,8 +404,8 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "ddsketch".to_string(), - sketch_kind: SummaryKind::DDSketch, - sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, + sketch_kind: SketchKind::DDSketch, + sketch_params: SketchParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".to_string(), }], exporter_target: ExportTarget::Stage(StageId::Gateway), diff --git a/control_plane/src/emit/stage_config.rs b/control_plane/src/emit/stage_config.rs index ade65c60..70bbd3bc 100644 --- a/control_plane/src/emit/stage_config.rs +++ b/control_plane/src/emit/stage_config.rs @@ -64,7 +64,11 @@ use crate::physical::colored_dag::emitter::{ // test module's `use super::*` instead (P2-5). use crate::intent_algebra::ColumnRef; use crate::physical::colored_dag::stage_id::StageId; -use asap_sketch::{SketchQuery, SummaryKind, SummaryParams}; +use planner_types::post_asap::{SketchKind, SketchParams, SketchQuery}; +// `BackendAggregation.sketch_kind`/`.sketch_params` span both exact +// accumulators and approximate sketches -- see +// `physical::colored_dag::emitter`'s `use asap_types::{...}` note. +use asap_types::{SummaryKind, SummaryParams}; // ── YAML structural types ───────────────────────────────────────────────────── // @@ -223,7 +227,7 @@ pub fn emit_edge_yaml( // ── MVP §46: 5-sketch routing-connector dispatch ─────────────────────── // // When the planner has populated `cfg.metric_to_family` (the per-metric - // → SummaryKind table sourced from the workload spec), we switch to the + // → SketchKind table sourced from the workload spec), we switch to the // canonical 5-sketch routing-connector wire shape: all referenced // sketch processors live at the top level, the OTel `routing` // *connector* (NOT the deprecated routing processor) lives under @@ -592,11 +596,11 @@ pub fn emit_gateway_yaml( // Processors — one merge processor per merge entry. Naming // convention matches the patched contrib build: - // * SummaryKind::DDSketch → `ddsketchmerge` - // * SummaryKind::Kll → `kllmerge` - // * SummaryKind::Hll → `hllmerge` - // * SummaryKind::Cms → `countminsketchmerge` - // * SummaryKind::CountSketch → `countsketchmerge` + // * SketchKind::DDSketch → `ddsketchmerge` + // * SketchKind::Kll → `kllmerge` + // * SketchKind::Hll → `hllmerge` + // * SketchKind::Cms → `countminsketchmerge` + // * SketchKind::CountSketch → `countsketchmerge` // // We honour `GatewayMergeProcessor::processor_name` if non-empty // (the typed emitter today populates it as `"sketchmergeprocessor"` @@ -895,7 +899,7 @@ fn build_routing_entry(metric_name: &str, cfg: &BackendStageConfig) -> JsonValue warm_shapes.push("count"); } // Heap-bearing kinds count too — `SummaryKind` (unlike the retired - // `sketch_algebra::SketchKind`) promotes `with_heap` to a distinct + // `sketch_algebra::SummaryKind`) promotes `with_heap` to a distinct // identity variant, but a topk-bound Count-Sketch/CMS aggregation // still needs to register here exactly as it did before the split. let has_count_sketch = kinds.iter().any(|k| { @@ -1044,7 +1048,7 @@ fn emit_edge_yaml_5sketch_routing( opamp_endpoint: &str, agent_id: &str, ) -> Result { - use asap_sketch::SummaryKind; + use planner_types::post_asap::SketchKind; let otlp_receiver: Value = serde_yaml::from_str( "protocols:\n grpc:\n endpoint: \"0.0.0.0:4317\"\n max_recv_msg_size_mib: 64\n http:\n endpoint: \"0.0.0.0:4318\"\n", @@ -1063,14 +1067,14 @@ fn emit_edge_yaml_5sketch_routing( // The canonical 5-family order below is the iteration order for // every emit (processors, pipelines, hints) so the YAML is stable // across controller runs regardless of HashMap iteration order. - const FAMILY_ORDER: [SummaryKind; 5] = [ - SummaryKind::DDSketch, - SummaryKind::Kll, - SummaryKind::Hll, - SummaryKind::CountSketch, - SummaryKind::Cms, + const FAMILY_ORDER: [SketchKind; 5] = [ + SketchKind::DDSketch, + SketchKind::Kll, + SketchKind::Hll, + SketchKind::CountSketch, + SketchKind::Cms, ]; - let mut needed_families: std::collections::BTreeSet = + let mut needed_families: std::collections::BTreeSet = std::collections::BTreeSet::new(); for families in cfg.metric_to_family.values() { for kind in families { @@ -1094,10 +1098,10 @@ fn emit_edge_yaml_5sketch_routing( // (so the params flow through), otherwise we synthesise a // default-param block. Keyed by `base_family` — `FAMILY_ORDER` is a // fixed 5-bare-family list with no heap-bearing entries, exactly - // matching pre-`SummaryKind`-split behavior (heap-bearing-ness was + // matching pre-`SketchKind`-split behavior (heap-bearing-ness was // never visible to this bare-kind lookup even when it lived as a // `with_heap` params flag). - let mut family_to_proc: HashMap = HashMap::new(); + let mut family_to_proc: HashMap = HashMap::new(); for sp in &cfg.sketch_processors { family_to_proc.insert(base_family(&sp.sketch_kind), sp); } @@ -1276,7 +1280,7 @@ fn emit_edge_yaml_5sketch_routing( // connector fans its samples into both pipelines. Family order // within each metric's pipeline list follows the canonical // `FAMILY_ORDER` so the YAML is stable. - let mut metric_family_pairs: Vec<(&String, &std::collections::BTreeSet)> = + let mut metric_family_pairs: Vec<(&String, &std::collections::BTreeSet)> = cfg.metric_to_family.iter().collect(); metric_family_pairs.sort_by(|a, b| a.0.cmp(b.0)); @@ -1306,7 +1310,7 @@ fn emit_edge_yaml_5sketch_routing( // added to EACH of its families' pipelines (the `where metric.name // == ""` guard makes it a no-op on the family's other // metrics). - let mut family_to_keep_processors: HashMap> = HashMap::new(); + let mut family_to_keep_processors: HashMap> = HashMap::new(); for (metric, families) in &metric_family_pairs { let Some(labels) = cfg.metric_to_grouping_labels.get(*metric) else { continue; @@ -1618,19 +1622,19 @@ fn emit_edge_yaml_5sketch_routing( serde_yaml::to_string(&doc).context("serialize edge stage config (5-sketch)") } -/// Map a `SummaryKind` to the `family:` token the fused `asap_edge` +/// Map a `SketchKind` to the `family:` token the fused `asap_edge` /// processor's `metrics[]` list expects. These differ from the OTel /// component-id processor names (`KLL`, `countmin`, …) used by the /// routing-connector path — the fused processor takes a lower-case /// family discriminant per entry, matching the hand-written contract in /// `asap-otel-agent-b6-asap-single-sketch.yaml`. -fn sketch_kind_to_asap_edge_family(kind: &SummaryKind) -> &'static str { +fn sketch_kind_to_asap_edge_family(kind: &SketchKind) -> &'static str { match kind { - SummaryKind::DDSketch => "ddsketch", - SummaryKind::Kll => "kll", - SummaryKind::Hll => "hll", - SummaryKind::CountSketch => "countsketch", - SummaryKind::Cms => "countminsketch", + SketchKind::DDSketch => "ddsketch", + SketchKind::Kll => "kll", + SketchKind::Hll => "hll", + SketchKind::CountSketch => "countsketch", + SketchKind::Cms => "countminsketch", // Every caller iterates the fixed 5-bare-family `FAMILY_ORDER` // list (heap-bearing kinds normalize through `base_family` // before reaching here), and no Bind* rule in this repo @@ -1701,7 +1705,7 @@ fn emit_edge_yaml_asap_edge( _opamp_endpoint: &str, _agent_id: &str, ) -> Result { - use asap_sketch::SummaryKind; + use planner_types::post_asap::SketchKind; // ── Receivers ────────────────────────────────────────────────────────── // OTLP gRPC on 4317 + HTTP on 4318 — same as every other edge emit. @@ -1880,18 +1884,18 @@ fn emit_edge_yaml_asap_edge( // (keyed by family) so the per-metric param block mirrors the // routing path; fall back to catalog defaults when the planner // mapped a family with no enumerated processor. - let mut family_to_proc: HashMap = HashMap::new(); + let mut family_to_proc: HashMap = HashMap::new(); for sp in &cfg.sketch_processors { family_to_proc.insert(base_family(&sp.sketch_kind), sp); } - const FAMILY_ORDER: [SummaryKind; 5] = [ - SummaryKind::DDSketch, - SummaryKind::Kll, - SummaryKind::Hll, - SummaryKind::CountSketch, - SummaryKind::Cms, + const FAMILY_ORDER: [SketchKind; 5] = [ + SketchKind::DDSketch, + SketchKind::Kll, + SketchKind::Hll, + SketchKind::CountSketch, + SketchKind::Cms, ]; - let mut metric_family_pairs: Vec<(&String, &std::collections::BTreeSet)> = + let mut metric_family_pairs: Vec<(&String, &std::collections::BTreeSet)> = cfg.metric_to_family.iter().collect(); metric_family_pairs.sort_by(|a, b| a.0.cmp(b.0)); for (metric, families) in &metric_family_pairs { @@ -1899,7 +1903,7 @@ fn emit_edge_yaml_asap_edge( // `FAMILY_ORDER` list — same reasoning as `family_to_proc` above: // a committed heap-bearing kind (`CmsWithHeap`/`CountSketchWithHeap`) // must still match its bare `FAMILY_ORDER` entry. - let bare_families: std::collections::BTreeSet = + let bare_families: std::collections::BTreeSet = families.iter().map(base_family).collect(); for kind in FAMILY_ORDER.iter().filter(|k| bare_families.contains(*k)) { let mut e = Mapping::new(); @@ -1994,7 +1998,7 @@ fn emit_edge_yaml_asap_edge( let whole_stream = effective_by.is_empty() && matches!( kind, - SummaryKind::Hll | SummaryKind::Cms | SummaryKind::CountSketch + SketchKind::Hll | SketchKind::Cms | SketchKind::CountSketch ); if whole_stream { e.insert("mode".into(), Value::String("whole_stream".to_string())); @@ -2041,7 +2045,7 @@ fn emit_edge_yaml_asap_edge( // lossless and serializes byte-identically to dense). We emit the // flag ONLY for the HLL family; non-HLL families carry no // `hll_sparse` key. - if matches!(kind, SummaryKind::Hll) { + if matches!(kind, SketchKind::Hll) { let hll_sparse = if whole_stream { false } else { @@ -2069,49 +2073,41 @@ fn emit_edge_yaml_asap_edge( // its params. let mut countsketch_with_heap = family_to_proc .get(kind) - .is_some_and(|sp| matches!(sp.sketch_kind, SummaryKind::CountSketchWithHeap)); + .is_some_and(|sp| matches!(sp.sketch_kind, SketchKind::CountSketchWithHeap)); match family_to_proc.get(kind).map(|sp| &sp.sketch_params) { - Some(SummaryParams::DDSketch { alpha }) => { + Some(SketchParams::DDSketch { alpha }) => { e.insert("relative_accuracy".into(), Value::Number((*alpha).into())); } - Some(SummaryParams::Kll { k }) => { + Some(SketchParams::Kll { k }) => { e.insert("k".into(), Value::Number((*k as u64).into())); } - Some(SummaryParams::Hll { .. }) => { /* HLL takes no per-entry knob */ } - Some(SummaryParams::CountSketch { width, depth }) - | Some(SummaryParams::CountSketchWithHeap { width, depth, .. }) => { + Some(SketchParams::Hll { .. }) => { /* HLL takes no per-entry knob */ } + Some(SketchParams::CountSketch { width, depth }) + | Some(SketchParams::CountSketchWithHeap { width, depth, .. }) => { e.insert("rows".into(), Value::Number((*depth as u64).into())); e.insert("cols".into(), Value::Number((*width as u64).into())); } - Some(SummaryParams::Cms { width, depth }) - | Some(SummaryParams::CmsWithHeap { width, depth, .. }) => { + Some(SketchParams::Cms { width, depth }) + | Some(SketchParams::CmsWithHeap { width, depth, .. }) => { e.insert("rows".into(), Value::Number((*depth as u64).into())); e.insert("cols".into(), Value::Number((*width as u64).into())); } - Some( - SummaryParams::Sum - | SummaryParams::Count - | SummaryParams::MinMax - | SummaryParams::Increase - | SummaryParams::Rate - | SummaryParams::Kmv { .. } - | SummaryParams::Theta { .. }, - ) => unreachable!( - "5-sketch routing: non-sketch or unsupported SummaryParams; \ + Some(SketchParams::Kmv { .. } | SketchParams::Theta { .. }) => unreachable!( + "5-sketch routing: non-sketch or unsupported SketchParams; \ no Bind* rule in this repo produces one" ), None => { // Family with no enumerated processor — emit catalog // defaults so the entry is still well-formed. match kind { - SummaryKind::DDSketch => { + SketchKind::DDSketch => { e.insert("relative_accuracy".into(), Value::Number(0.01.into())); } - SummaryKind::Kll => { + SketchKind::Kll => { e.insert("k".into(), Value::Number(200u64.into())); } - SummaryKind::Hll => {} - SummaryKind::CountSketch => { + SketchKind::Hll => {} + SketchKind::CountSketch => { e.insert("rows".into(), Value::Number(5u64.into())); e.insert("cols".into(), Value::Number(2048u64.into())); // P1-4: NO enumerated EdgeSketchProcessor for this @@ -2135,7 +2131,7 @@ fn emit_edge_yaml_asap_edge( // step with the backend `with_heap` registration. countsketch_with_heap = cfg.metric_to_item_label.contains_key(*metric); } - SummaryKind::Cms => { + SketchKind::Cms => { e.insert("rows".into(), Value::Number(5u64.into())); e.insert("cols".into(), Value::Number(2048u64.into())); } @@ -2151,7 +2147,7 @@ fn emit_edge_yaml_asap_edge( // families (CMS / HLL) only when `p < 1.0`. Mirrors // `build_edge_processor_block`'s guarded emit so an unset / // 1.0 probability keeps the fused entry byte-identical. - if matches!(kind, SummaryKind::Cms | SummaryKind::Hll) { + if matches!(kind, SketchKind::Cms | SketchKind::Hll) { insert_sample_p(&mut e, cfg.metric_to_sample_p.get(*metric).copied()); } @@ -2172,10 +2168,7 @@ fn emit_edge_yaml_asap_edge( // depending on that default. if matches!( kind, - SummaryKind::DDSketch - | SummaryKind::Hll - | SummaryKind::CountSketch - | SummaryKind::Cms + SketchKind::DDSketch | SketchKind::Hll | SketchKind::CountSketch | SketchKind::Cms ) { e.insert("delta_transmission".into(), Value::Bool(true)); } @@ -2206,7 +2199,7 @@ fn emit_edge_yaml_asap_edge( // that lands — `mapstructure` ignores unknown keys by default — // but the warm-topk behaviour only activates once the asapedge // build carries the fields. See the report's cross-repo note. - if matches!(kind, SummaryKind::CountSketch) && countsketch_with_heap { + if matches!(kind, SketchKind::CountSketch) && countsketch_with_heap { e.insert("emit_heap".into(), Value::Bool(true)); e.insert("heap_size".into(), Value::Number(100u64.into())); // Prefer the workload-declared inner dimension @@ -2250,7 +2243,7 @@ fn emit_edge_yaml_asap_edge( // pure YAML text here (`mapstructure` ignores unknown keys), so // emitting it is safe even before that lands; the corrected // keying only activates once the asapedge build carries it. - if matches!(kind, SummaryKind::Hll | SummaryKind::Cms) { + if matches!(kind, SketchKind::Hll | SketchKind::Cms) { if let Some(item_label) = cfg.metric_to_item_label.get(*metric) { if !item_label.is_empty() { e.insert("item_label".into(), Value::String(item_label.clone())); @@ -2457,30 +2450,30 @@ tsdb_block_duration: {window_secs}s\n", ) } -/// Map a `SummaryKind` to the OTel processor name registered by the +/// Map a `SketchKind` to the OTel processor name registered by the /// patched contrib build's factory. Keep in sync with /// `crate::physical::colored_dag::emitter::edge_processor_name`. -fn sketch_kind_to_processor_name(kind: &SummaryKind) -> &'static str { +fn sketch_kind_to_processor_name(kind: &SketchKind) -> &'static str { match kind { - SummaryKind::DDSketch => "ddsketch", - SummaryKind::Kll => "KLL", - SummaryKind::Hll => "HLL", - SummaryKind::CountSketch => "countsketch", - SummaryKind::Cms => "countmin", + SketchKind::DDSketch => "ddsketch", + SketchKind::Kll => "KLL", + SketchKind::Hll => "HLL", + SketchKind::CountSketch => "countsketch", + SketchKind::Cms => "countmin", // Callers only ever pass a bare `FAMILY_ORDER` entry. other => unreachable!("sketch_kind_to_processor_name: unexpected kind {other:?}"), } } -/// Map a `SummaryKind` to its per-family pipeline name in the routing +/// Map a `SketchKind` to its per-family pipeline name in the routing /// connector layout. -fn sketch_kind_to_pipeline_name(kind: &SummaryKind) -> &'static str { +fn sketch_kind_to_pipeline_name(kind: &SketchKind) -> &'static str { match kind { - SummaryKind::DDSketch => "metrics/ddsketch_path", - SummaryKind::Kll => "metrics/kll_path", - SummaryKind::Hll => "metrics/hll_path", - SummaryKind::CountSketch => "metrics/countsketch_path", - SummaryKind::Cms => "metrics/countminsketch_path", + SketchKind::DDSketch => "metrics/ddsketch_path", + SketchKind::Kll => "metrics/kll_path", + SketchKind::Hll => "metrics/hll_path", + SketchKind::CountSketch => "metrics/countsketch_path", + SketchKind::Cms => "metrics/countminsketch_path", // Callers only ever pass a bare `FAMILY_ORDER` entry. other => unreachable!("sketch_kind_to_pipeline_name: unexpected kind {other:?}"), } @@ -2618,14 +2611,14 @@ fn build_transform_keep_processor_block(metric: &str, labels: &[String]) -> Valu serde_yaml::from_str(&yaml).expect("transform/keep_for_* yaml is well-formed by construction") } -/// Build a default-parameter processor block for a `SummaryKind` when +/// Build a default-parameter processor block for a `SketchKind` when /// the planner's `metric_to_family` references a family that /// `cfg.sketch_processors` didn't enumerate. Defaults match the catalog /// values used by the planner's L4 rules so the wire shape is what the /// rest of the system expects when a metric is later re-routed onto /// this family. fn build_default_edge_processor_block( - kind: &SummaryKind, + kind: &SketchKind, window_secs: Option, metric_name_hint: Option<&str>, sample_p: Option, @@ -2635,27 +2628,24 @@ fn build_default_edge_processor_block( // the tag/processor-name lookups below, which are keyed on the bare // family. `stored_kind`/`params` are what actually land on the // synthesized processor; `CountSketch`'s default stays heap-bearing - // (matching this function's pre-`SummaryKind`-split default of + // (matching this function's pre-`SketchKind`-split default of // `with_heap: true` — `Cms`'s default was `with_heap: false` and // stays bare). let (stored_kind, params) = match kind { - SummaryKind::DDSketch => ( - SummaryKind::DDSketch, - SummaryParams::DDSketch { alpha: 0.01 }, - ), - SummaryKind::Kll => (SummaryKind::Kll, SummaryParams::Kll { k: 200 }), - SummaryKind::Hll => (SummaryKind::Hll, SummaryParams::Hll { precision: 14 }), - SummaryKind::CountSketch => ( - SummaryKind::CountSketchWithHeap, - SummaryParams::CountSketchWithHeap { + SketchKind::DDSketch => (SketchKind::DDSketch, SketchParams::DDSketch { alpha: 0.01 }), + SketchKind::Kll => (SketchKind::Kll, SketchParams::Kll { k: 200 }), + SketchKind::Hll => (SketchKind::Hll, SketchParams::Hll { precision: 14 }), + SketchKind::CountSketch => ( + SketchKind::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { width: 2048, depth: 5, heap_size: 10, }, ), - SummaryKind::Cms => ( - SummaryKind::Cms, - SummaryParams::Cms { + SketchKind::Cms => ( + SketchKind::Cms, + SketchParams::Cms { width: 4096, depth: 4, }, @@ -2798,15 +2788,15 @@ fn build_edge_processor_block( // it explicitly so the wire YAML doesn't depend on a factory // default that could regress to full-state in a future build. match &sp.sketch_params { - SummaryParams::Kll { k } => { + SketchParams::Kll { k } => { m.insert("k".into(), Value::Number((*k as u64).into())); // No delta_transmission for KLL: see comment above. } - SummaryParams::DDSketch { alpha } => { + SketchParams::DDSketch { alpha } => { m.insert("relative_accuracy".into(), Value::Number((*alpha).into())); m.insert("delta_transmission".into(), Value::Bool(true)); } - SummaryParams::Hll { .. } => { + SketchParams::Hll { .. } => { // HLL takes no precision knob in its Config (the // patched build hard-codes p=14); nothing further to set. m.insert("encoding".into(), Value::String("msgpack".into())); @@ -2815,7 +2805,7 @@ fn build_edge_processor_block( // (hash-threshold element sampling in sketchlib-go). insert_sample_p(&mut m, sample_p); } - SummaryParams::Cms { width, depth } | SummaryParams::CmsWithHeap { width, depth, .. } => { + SketchParams::Cms { width, depth } | SketchParams::CmsWithHeap { width, depth, .. } => { m.insert( "metric_name".into(), Value::String( @@ -2832,8 +2822,8 @@ fn build_edge_processor_block( // (geometric admission sampling in sketchlib-go). insert_sample_p(&mut m, sample_p); } - SummaryParams::CountSketch { width, depth } - | SummaryParams::CountSketchWithHeap { width, depth, .. } => { + SketchParams::CountSketch { width, depth } + | SketchParams::CountSketchWithHeap { width, depth, .. } => { // P1-3: the standalone `countsketchprocessor` Config exposes ONLY // `epsilon` / `delta` (no `rows` / `cols` mapstructure keys), and // it RE-DERIVES the sketch dimensions internally via @@ -2860,15 +2850,9 @@ fn build_edge_processor_block( m.insert("encoding".into(), Value::String("msgpack".into())); m.insert("delta_transmission".into(), Value::Bool(true)); } - SummaryParams::Sum - | SummaryParams::Count - | SummaryParams::MinMax - | SummaryParams::Increase - | SummaryParams::Rate - | SummaryParams::Kmv { .. } - | SummaryParams::Theta { .. } => unreachable!( + SketchParams::Kmv { .. } | SketchParams::Theta { .. } => unreachable!( "edge sketch processor config requested for a non-sketch or unsupported \ - SummaryKind; no Bind* rule in this repo produces one" + SketchKind; no Bind* rule in this repo produces one" ), } @@ -2902,21 +2886,13 @@ fn insert_sample_p(m: &mut Mapping, sample_p: Option) { /// here so the emitted YAML round-trips through the patched build. fn gateway_merge_processor_name(mp: &GatewayMergeProcessor) -> String { match mp.sketch_kind { - SummaryKind::Kll => "kllmerge".to_string(), - SummaryKind::DDSketch => "ddsketchmerge".to_string(), - SummaryKind::Hll => "hllmerge".to_string(), - SummaryKind::Cms | SummaryKind::CmsWithHeap => "countminsketchmerge".to_string(), - SummaryKind::CountSketch | SummaryKind::CountSketchWithHeap => { - "countsketchmerge".to_string() - } - SummaryKind::Sum - | SummaryKind::Count - | SummaryKind::MinMax - | SummaryKind::Increase - | SummaryKind::Rate - | SummaryKind::Kmv - | SummaryKind::Theta => unreachable!( - "gateway_merge_processor_name: non-sketch or unsupported SummaryKind; \ + SketchKind::Kll => "kllmerge".to_string(), + SketchKind::DDSketch => "ddsketchmerge".to_string(), + SketchKind::Hll => "hllmerge".to_string(), + SketchKind::Cms | SketchKind::CmsWithHeap => "countminsketchmerge".to_string(), + SketchKind::CountSketch | SketchKind::CountSketchWithHeap => "countsketchmerge".to_string(), + SketchKind::Kmv | SketchKind::Theta => unreachable!( + "gateway_merge_processor_name: non-sketch or unsupported SketchKind; \ no Bind* rule in this repo produces one" ), } @@ -3066,7 +3042,7 @@ fn column_ref_to_wire_key(col: &ColumnRef) -> String { } } -/// Collapse a heap-bearing `SummaryKind` to its bare counterpart. +/// Collapse a heap-bearing `SketchKind` to its bare counterpart. /// Identity for every other kind. /// /// The 5-sketch routing-connector edge YAML path (`emit_edge_yaml`'s @@ -3079,21 +3055,21 @@ fn column_ref_to_wire_key(col: &ColumnRef) -> String { /// from a topk binding) needs to normalize through this before it's /// used as a key or set member in that path, or it silently fails to /// match its bare `FAMILY_ORDER` entry. -fn base_family(kind: &SummaryKind) -> SummaryKind { +fn base_family(kind: &SketchKind) -> SketchKind { match kind { - SummaryKind::CmsWithHeap => SummaryKind::Cms, - SummaryKind::CountSketchWithHeap => SummaryKind::CountSketch, + SketchKind::CmsWithHeap => SketchKind::Cms, + SketchKind::CountSketchWithHeap => SketchKind::CountSketch, other => other.clone(), } } -/// Map a `SummaryKind` to the backend's `AggregationType::Display` +/// Map a `SketchKind` to the backend's `AggregationType::Display` /// string — the same mapping /// [`crate::config::asapquery_backend::map_sketch_type_to_agg_type`] uses /// (the strings must match `AggregationType::FromStr` in the backend's /// `promql_utilities::query_logics::enums`). /// -/// Heap-bearing is now identity, not a params flag (`SummaryKind::CmsWithHeap` +/// Heap-bearing is now identity, not a params flag (`SketchKind::CmsWithHeap` /// / `CountSketchWithHeap`, set by `BindCountSketchOnTopK` — see /// `sketch_algebra::rules::bind_cms_topk`), so this maps on `kind` alone; /// `params` is unused but kept for call-site stability. This is what @@ -3109,6 +3085,9 @@ fn sketch_kind_to_backend_type(kind: &SummaryKind, _params: &SummaryParams) -> & SummaryKind::CountSketch => "CountSketch", SummaryKind::CmsWithHeap => "CountMinSketchWithHeap", SummaryKind::Cms => "CountMinSketch", + // Exact accumulators never reach here -- the caller takes the + // `agg_type_override` branch for them instead (see the call + // site's comment). SummaryKind::Sum | SummaryKind::Count | SummaryKind::MinMax @@ -3122,26 +3101,20 @@ fn sketch_kind_to_backend_type(kind: &SummaryKind, _params: &SummaryParams) -> & } } -/// Stable lowercase tag for a `SummaryKind` — used as a passthrough +/// Stable lowercase tag for a `SketchKind` — used as a passthrough /// `sketch_kind` field in YAML so downstream consumers can dispatch /// without round-tripping through serde. Heap-bearing kinds reuse their /// bare counterpart's tag — this field never distinguished `with_heap` -/// even before `SummaryKind` split it into its own variant. -fn sketch_kind_tag(kind: &SummaryKind) -> &'static str { +/// even before `SketchKind` split it into its own variant. +fn sketch_kind_tag(kind: &SketchKind) -> &'static str { match kind { - SummaryKind::Kll => "kll", - SummaryKind::DDSketch => "ddsketch", - SummaryKind::Hll => "hll", - SummaryKind::Cms | SummaryKind::CmsWithHeap => "cms", - SummaryKind::CountSketch | SummaryKind::CountSketchWithHeap => "count_sketch", - SummaryKind::Sum - | SummaryKind::Count - | SummaryKind::MinMax - | SummaryKind::Increase - | SummaryKind::Rate - | SummaryKind::Kmv - | SummaryKind::Theta => unreachable!( - "sketch_kind_tag: non-sketch or unsupported SummaryKind; \ + SketchKind::Kll => "kll", + SketchKind::DDSketch => "ddsketch", + SketchKind::Hll => "hll", + SketchKind::Cms | SketchKind::CmsWithHeap => "cms", + SketchKind::CountSketch | SketchKind::CountSketchWithHeap => "count_sketch", + SketchKind::Kmv | SketchKind::Theta => unreachable!( + "sketch_kind_tag: non-sketch or unsupported SketchKind; \ no Bind* rule in this repo produces one" ), } @@ -3173,6 +3146,8 @@ fn sketch_params_to_json(p: &SummaryParams) -> JsonValue { SummaryParams::CountSketchWithHeap { width, depth, .. } => { json!({ "w": width, "d": depth, "with_heap": true }) } + // Exact accumulators never reach here -- see + // `sketch_kind_to_backend_type`'s doc. SummaryParams::Sum | SummaryParams::Count | SummaryParams::MinMax @@ -3202,8 +3177,8 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "ddsketch".to_string(), - sketch_kind: SummaryKind::DDSketch, - sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, + sketch_kind: SketchKind::DDSketch, + sketch_params: SketchParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".to_string(), }], exporter_target: ExportTarget::Stage(StageId::Gateway), @@ -3287,8 +3262,8 @@ mod tests { let mut cfg = ddsketch_edge_cfg(); cfg.sketch_processors[0] = EdgeSketchProcessor { processor_name: "KLL".to_string(), - sketch_kind: SummaryKind::Kll, - sketch_params: SummaryParams::Kll { k: 200 }, + sketch_kind: SketchKind::Kll, + sketch_params: SketchParams::Kll { k: 200 }, aggregation_id: "agg7".to_string(), }; let yaml = emit_edge_yaml(&cfg, "ws://c/", "test-agent").expect("emit ok"); @@ -3322,28 +3297,24 @@ mod tests { // flag (see `edge_yaml_kll_uses_k_param`). for (kind, processor_name, params) in [ ( - SummaryKind::DDSketch, + SketchKind::DDSketch, "ddsketch", - SummaryParams::DDSketch { alpha: 0.01 }, + SketchParams::DDSketch { alpha: 0.01 }, ), + (SketchKind::Hll, "HLL", SketchParams::Hll { precision: 14 }), ( - SummaryKind::Hll, - "HLL", - SummaryParams::Hll { precision: 14 }, - ), - ( - SummaryKind::CountSketch, + SketchKind::CountSketch, "countsketch", - SummaryParams::CountSketchWithHeap { + SketchParams::CountSketchWithHeap { width: 2048, depth: 5, heap_size: 10, }, ), ( - SummaryKind::Cms, + SketchKind::Cms, "countmin", - SummaryParams::Cms { + SketchParams::Cms { width: 4096, depth: 4, }, @@ -3371,8 +3342,8 @@ mod tests { cfg.source_metric = Some("endpoint_request_freq".to_string()); cfg.sketch_processors[0] = EdgeSketchProcessor { processor_name: "countmin".to_string(), - sketch_kind: SummaryKind::Cms, - sketch_params: SummaryParams::Cms { + sketch_kind: SketchKind::Cms, + sketch_params: SketchParams::Cms { width: 4096, depth: 4, }, @@ -3404,7 +3375,7 @@ mod tests { otlp_receiver_port: 4317, merge_processors: vec![GatewayMergeProcessor { processor_name: "sketchmergeprocessor".to_string(), - sketch_kind: SummaryKind::DDSketch, + sketch_kind: SketchKind::DDSketch, aggregation_id: "agg0".to_string(), }], exporter_target: ExportTarget::Stage(StageId::Backend), @@ -3448,12 +3419,12 @@ mod tests { merge_processors: vec![ GatewayMergeProcessor { processor_name: "x".into(), - sketch_kind: SummaryKind::Kll, + sketch_kind: SketchKind::Kll, aggregation_id: "agg0".into(), }, GatewayMergeProcessor { processor_name: "x".into(), - sketch_kind: SummaryKind::Hll, + sketch_kind: SketchKind::Hll, aggregation_id: "agg1".into(), }, ], @@ -3653,27 +3624,24 @@ mod tests { /// requested kind. `aggregation_id` is hard-coded — the routing /// emitter doesn't care about it. Accepts the 5 canonical bare /// families callers actually pass; `CountSketch` stores as the - /// heap-bearing variant (matching this fixture's pre-`SummaryKind`-split + /// heap-bearing variant (matching this fixture's pre-`SketchKind`-split /// behavior, when `with_heap: true` was a `CountSketchParams` field /// rather than a distinct kind). - fn backend_cfg_with_kind(kind: SummaryKind) -> BackendStageConfig { + fn backend_cfg_with_kind(kind: SketchKind) -> BackendStageConfig { let (stored_kind, params) = match kind { - SummaryKind::DDSketch => ( - SummaryKind::DDSketch, - SummaryParams::DDSketch { alpha: 0.01 }, - ), - SummaryKind::Kll => (SummaryKind::Kll, SummaryParams::Kll { k: 200 }), - SummaryKind::Hll => (SummaryKind::Hll, SummaryParams::Hll { precision: 14 }), - SummaryKind::Cms => ( - SummaryKind::Cms, - SummaryParams::Cms { + SketchKind::DDSketch => (SketchKind::DDSketch, SketchParams::DDSketch { alpha: 0.01 }), + SketchKind::Kll => (SketchKind::Kll, SketchParams::Kll { k: 200 }), + SketchKind::Hll => (SketchKind::Hll, SketchParams::Hll { precision: 14 }), + SketchKind::Cms => ( + SketchKind::Cms, + SketchParams::Cms { width: 4096, depth: 4, }, ), - SummaryKind::CountSketch => ( - SummaryKind::CountSketchWithHeap, - SummaryParams::CountSketchWithHeap { + SketchKind::CountSketch => ( + SketchKind::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { width: 2048, depth: 5, heap_size: 10, @@ -3686,8 +3654,8 @@ mod tests { item_label: None, aggregation_id: "agg0".into(), metric_name: "test_metric".into(), - sketch_kind: stored_kind, - sketch_params: params, + sketch_kind: stored_kind.into(), + sketch_params: params.into(), window_secs: 60, spatial_filter: String::new(), grouping: Vec::new(), @@ -3697,10 +3665,10 @@ mod tests { readouts: vec![BackendReadout { aggregation_id: "agg0".into(), op: match kind { - SummaryKind::DDSketch | SummaryKind::Kll => SketchQuery::Quantile { q: 0.99 }, - SummaryKind::Hll => SketchQuery::Cardinality, - SummaryKind::CountSketch => SketchQuery::TopK { k: 10 }, - SummaryKind::Cms => SketchQuery::PointCount { + SketchKind::DDSketch | SketchKind::Kll => SketchQuery::Quantile { q: 0.99 }, + SketchKind::Hll => SketchQuery::Cardinality, + SketchKind::CountSketch => SketchQuery::TopK { k: 10 }, + SketchKind::Cms => SketchQuery::PointCount { key: ColumnRef::Named("user_42".into()), value: None, }, @@ -3714,7 +3682,7 @@ mod tests { #[test] fn storage_routing_emits_default_engine_and_metrics_array() { - let ddsketch = backend_cfg_with_kind(SummaryKind::DDSketch); + let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); let plans: Vec<(String, &BackendStageConfig)> = vec![("http_request_duration_seconds".to_string(), &ddsketch)]; let v = emit_backend_storage_routing(&plans).expect("emit ok"); @@ -3732,7 +3700,7 @@ mod tests { /// byte-compatible (modulo the new `tenant` field appearing). #[test] fn storage_routing_default_tenant_for_single_tenant_emit() { - let ddsketch = backend_cfg_with_kind(SummaryKind::DDSketch); + let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); let v = emit_backend_storage_routing(&[("latency".into(), &ddsketch)]).expect("emit ok"); assert_eq!(v["tenant"], DEFAULT_TENANT); } @@ -3744,7 +3712,7 @@ mod tests { /// body-tenant precedence rule. #[test] fn storage_routing_for_tenant_emits_explicit_tenant_field() { - let ddsketch = backend_cfg_with_kind(SummaryKind::DDSketch); + let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); let v = emit_backend_storage_routing_for_tenant("tenant-a", &[("latency".into(), &ddsketch)]) .expect("emit ok"); @@ -3771,7 +3739,7 @@ mod tests { #[test] fn storage_routing_ddasap_query_serves_quantile_archive_serves_others() { - let ddsketch = backend_cfg_with_kind(SummaryKind::DDSketch); + let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); let v = emit_backend_storage_routing(&[("latency".into(), &ddsketch)]).expect("emit ok"); let metric = &v["metrics"][0]; let targets = metric["targets"].as_array().expect("targets array"); @@ -3813,7 +3781,7 @@ mod tests { #[test] fn storage_routing_count_sketch_pulls_topk_off_archive() { - let cs = backend_cfg_with_kind(SummaryKind::CountSketch); + let cs = backend_cfg_with_kind(SketchKind::CountSketch); let v = emit_backend_storage_routing(&[("requests".into(), &cs)]).expect("emit ok"); let archive_shapes: Vec = v["metrics"][0]["targets"][1]["applies_to_query_shape"] .as_array() @@ -3833,7 +3801,7 @@ mod tests { #[test] fn storage_routing_hll_pulls_count_off_archive() { - let hll = backend_cfg_with_kind(SummaryKind::Hll); + let hll = backend_cfg_with_kind(SketchKind::Hll); let v = emit_backend_storage_routing(&[("active_users".into(), &hll)]).expect("emit ok"); let archive_shapes: Vec = v["metrics"][0]["targets"][1]["applies_to_query_shape"] .as_array() @@ -3858,9 +3826,9 @@ mod tests { // runs (HashMap iteration order can drift, but our impl // stages everything through a Vec so order matches input // order). - let ddsketch = backend_cfg_with_kind(SummaryKind::DDSketch); - let hll = backend_cfg_with_kind(SummaryKind::Hll); - let cs = backend_cfg_with_kind(SummaryKind::CountSketch); + let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); + let hll = backend_cfg_with_kind(SketchKind::Hll); + let cs = backend_cfg_with_kind(SketchKind::CountSketch); let plans: Vec<(String, &BackendStageConfig)> = vec![ ("http_requests_total".into(), &cs), ("active_users".into(), &hll), @@ -4025,33 +3993,29 @@ mod tests { /// silently break the backend. #[test] fn phase_b_backend_agg_type_strings_for_every_sketch_kind() { - let cases: Vec<(SummaryKind, SummaryParams, &str)> = vec![ + let cases: Vec<(SketchKind, SketchParams, &str)> = vec![ ( - SummaryKind::Kll, - SummaryParams::Kll { k: 200 }, + SketchKind::Kll, + SketchParams::Kll { k: 200 }, "DatasketchesKLL", ), ( - SummaryKind::DDSketch, - SummaryParams::DDSketch { alpha: 0.01 }, + SketchKind::DDSketch, + SketchParams::DDSketch { alpha: 0.01 }, "DDSketch", ), + (SketchKind::Hll, SketchParams::Hll { precision: 14 }, "HLL"), ( - SummaryKind::Hll, - SummaryParams::Hll { precision: 14 }, - "HLL", - ), - ( - SummaryKind::Cms, - SummaryParams::Cms { + SketchKind::Cms, + SketchParams::Cms { width: 4096, depth: 4, }, "CountMinSketch", ), ( - SummaryKind::CmsWithHeap, - SummaryParams::CmsWithHeap { + SketchKind::CmsWithHeap, + SketchParams::CmsWithHeap { width: 4096, depth: 4, heap_size: 10, @@ -4059,16 +4023,16 @@ mod tests { "CountMinSketchWithHeap", ), ( - SummaryKind::CountSketch, - SummaryParams::CountSketch { + SketchKind::CountSketch, + SketchParams::CountSketch { width: 2048, depth: 5, }, "CountSketch", ), ( - SummaryKind::CountSketchWithHeap, - SummaryParams::CountSketchWithHeap { + SketchKind::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { width: 2048, depth: 5, heap_size: 10, @@ -4077,6 +4041,8 @@ mod tests { ), ]; for (kind, params, expected) in cases { + let kind: SummaryKind = kind.into(); + let params: SummaryParams = params.into(); assert_eq!( sketch_kind_to_backend_type(&kind, ¶ms), expected, @@ -4134,7 +4100,7 @@ mod tests { /// producing aggregation. Pins the sort order + key names. Phase β /// uses this as the wire-format anchor for the wider intent set — /// the JSON shape is intent-orthogonal, so adding new intents to L3 - /// can't drift this off so long as they bind through SummaryKind / + /// can't drift this off so long as they bind through SketchKind / /// SketchParams. #[test] fn phase_b_backend_json_aggregation_readout_alias_snapshot() { @@ -4257,7 +4223,7 @@ mod tests { /// Prometheus-archive metric coexist in one routing JSON. #[test] fn phase_eps1_mixed_mode1_and_mode3_share_one_routing_table() { - let ddsketch = backend_cfg_with_kind(SummaryKind::DDSketch); + let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); let plans: Vec<(String, &BackendStageConfig)> = vec![("latency_seconds".into(), &ddsketch)]; let mode3 = vec!["http_requests_total".to_string()]; let v = emit_backend_storage_routing_with_prometheus(&plans, &mode3).expect("emit ok"); @@ -4623,7 +4589,7 @@ mod tests { /// (ASAPCollector#400). Most fixtures map each metric to exactly one /// family — this keeps them concise while exercising the SET-shaped /// `metric_to_family`. - fn one(kind: SummaryKind) -> std::collections::BTreeSet { + fn one(kind: SketchKind) -> std::collections::BTreeSet { std::collections::BTreeSet::from([kind]) } @@ -4632,16 +4598,16 @@ mod tests { /// metric maps to a single-family set (this workload's per-metric set /// size is 1; see `mvp46_multi_family_metric_*` for the size>1 case). fn five_sketch_edge_cfg() -> EdgeStageConfig { - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); metric_to_family.insert( "http_requests_total_latency_ms".into(), - one(SummaryKind::DDSketch), + one(SketchKind::DDSketch), ); - metric_to_family.insert("request_size_bytes".into(), one(SummaryKind::Kll)); - metric_to_family.insert("unique_users_per_min".into(), one(SummaryKind::Hll)); - metric_to_family.insert("top_endpoint_qps".into(), one(SummaryKind::CountSketch)); - metric_to_family.insert("endpoint_request_freq".into(), one(SummaryKind::Cms)); + metric_to_family.insert("request_size_bytes".into(), one(SketchKind::Kll)); + metric_to_family.insert("unique_users_per_min".into(), one(SketchKind::Hll)); + metric_to_family.insert("top_endpoint_qps".into(), one(SketchKind::CountSketch)); + metric_to_family.insert("endpoint_request_freq".into(), one(SketchKind::Cms)); // `http_requests_total` is intentionally NOT in this map — it // falls through to the `metrics/raw_passthrough` default. EdgeStageConfig { @@ -4996,7 +4962,7 @@ mod tests { /// Helper: build an `EdgeStageConfig` whose `metric_to_family` is the /// given metric→set map, with sensible defaults for the other fields. fn edge_cfg_with_families( - metric_to_family: HashMap>, + metric_to_family: HashMap>, ) -> EdgeStageConfig { EdgeStageConfig { source_metric: None, @@ -5028,7 +4994,7 @@ mod tests { // families — this is the core bandwidth fix. let cfg = edge_cfg_with_families(HashMap::from([( "latency_ms".to_string(), - one(SummaryKind::DDSketch), + one(SketchKind::DDSketch), )])); let yaml = emit_edge_yaml(&cfg, "ws://c/", "test-agent").expect("emit ok"); @@ -5069,8 +5035,8 @@ mod tests { // HLL). Exactly those two pipelines/processors must be emitted; // KLL/CountSketch/CMS pruned. let cfg = edge_cfg_with_families(HashMap::from([ - ("latency_ms".to_string(), one(SummaryKind::DDSketch)), - ("uniques".to_string(), one(SummaryKind::Hll)), + ("latency_ms".to_string(), one(SketchKind::DDSketch)), + ("uniques".to_string(), one(SketchKind::Hll)), ])); let yaml = emit_edge_yaml(&cfg, "ws://c/", "test-agent").expect("emit ok"); @@ -5108,7 +5074,7 @@ mod tests { // them). let cfg = edge_cfg_with_families(HashMap::from([( "http_requests".to_string(), - std::collections::BTreeSet::from([SummaryKind::DDSketch, SummaryKind::Hll]), + std::collections::BTreeSet::from([SketchKind::DDSketch, SketchKind::Hll]), )])); let yaml = emit_edge_yaml(&cfg, "ws://c/", "test-agent").expect("emit ok"); @@ -5920,7 +5886,7 @@ mod tests { use crate::physical::colored_dag::emitter::{ AggregationInput, BackendAggregation, BackendReadout, BackendStageConfig, }; - use asap_sketch::SketchQuery; + use planner_types::post_asap::SketchQuery; let cfg = BackendStageConfig { aggregations: vec![BackendAggregation { @@ -6112,41 +6078,41 @@ mod tests { /// tier (cold), and the counter-shaped sketch inputs in the /// cumulativetodelta list. fn fused_asap_edge_cfg() -> EdgeStageConfig { - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); metric_to_family.insert( "http_requests_total_latency_ms".into(), - one(SummaryKind::DDSketch), + one(SketchKind::DDSketch), ); - metric_to_family.insert("request_size_bytes".into(), one(SummaryKind::Kll)); - metric_to_family.insert("unique_users_per_min".into(), one(SummaryKind::Hll)); - metric_to_family.insert("top_endpoint_qps".into(), one(SummaryKind::CountSketch)); - metric_to_family.insert("endpoint_request_freq".into(), one(SummaryKind::Cms)); + metric_to_family.insert("request_size_bytes".into(), one(SketchKind::Kll)); + metric_to_family.insert("unique_users_per_min".into(), one(SketchKind::Hll)); + metric_to_family.insert("top_endpoint_qps".into(), one(SketchKind::CountSketch)); + metric_to_family.insert("endpoint_request_freq".into(), one(SketchKind::Cms)); // Per-family params, mirroring the target config's per-entry knobs. let sketch_processors = vec![ EdgeSketchProcessor { processor_name: "ddsketch".into(), - sketch_kind: SummaryKind::DDSketch, - sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, + sketch_kind: SketchKind::DDSketch, + sketch_params: SketchParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".into(), }, EdgeSketchProcessor { processor_name: "KLL".into(), - sketch_kind: SummaryKind::Kll, - sketch_params: SummaryParams::Kll { k: 200 }, + sketch_kind: SketchKind::Kll, + sketch_params: SketchParams::Kll { k: 200 }, aggregation_id: "agg1".into(), }, EdgeSketchProcessor { processor_name: "HLL".into(), - sketch_kind: SummaryKind::Hll, - sketch_params: SummaryParams::Hll { precision: 14 }, + sketch_kind: SketchKind::Hll, + sketch_params: SketchParams::Hll { precision: 14 }, aggregation_id: "agg2".into(), }, EdgeSketchProcessor { processor_name: "countsketch".into(), - sketch_kind: SummaryKind::CountSketchWithHeap, - sketch_params: SummaryParams::CountSketchWithHeap { + sketch_kind: SketchKind::CountSketchWithHeap, + sketch_params: SketchParams::CountSketchWithHeap { width: 2048, depth: 5, heap_size: 10, @@ -6155,8 +6121,8 @@ mod tests { }, EdgeSketchProcessor { processor_name: "countmin".into(), - sketch_kind: SummaryKind::Cms, - sketch_params: SummaryParams::Cms { + sketch_kind: SketchKind::Cms, + sketch_params: SketchParams::Cms { width: 2048, depth: 5, }, @@ -6586,9 +6552,9 @@ mod tests { fn fused_asap_edge_per_group_hll_is_per_series_and_sparse() { let _env = crate::test_support::EnvVarGuard::set("ASAP_EDGE_FUSED", "1"); - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); - metric_to_family.insert("distinct_users_by_region".into(), one(SummaryKind::Hll)); + metric_to_family.insert("distinct_users_by_region".into(), one(SketchKind::Hll)); let mut metric_to_grouping_labels: HashMap> = HashMap::new(); metric_to_grouping_labels.insert("distinct_users_by_region".into(), vec!["region".into()]); @@ -6607,8 +6573,8 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "HLL".into(), - sketch_kind: SummaryKind::Hll, - sketch_params: SummaryParams::Hll { precision: 14 }, + sketch_kind: SketchKind::Hll, + sketch_params: SketchParams::Hll { precision: 14 }, aggregation_id: "agg0".into(), }], exporter_target: ExportTarget::Endpoint("data-plane:4317".into()), @@ -6684,9 +6650,9 @@ mod tests { fn fused_asap_edge_per_series_hll_high_cardinality_hint_is_dense() { let _env = crate::test_support::EnvVarGuard::set("ASAP_EDGE_FUSED", "1"); - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); - metric_to_family.insert("distinct_users_by_region".into(), one(SummaryKind::Hll)); + metric_to_family.insert("distinct_users_by_region".into(), one(SketchKind::Hll)); let mut metric_to_grouping_labels: HashMap> = HashMap::new(); metric_to_grouping_labels.insert("distinct_users_by_region".into(), vec!["region".into()]); @@ -6704,8 +6670,8 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "HLL".into(), - sketch_kind: SummaryKind::Hll, - sketch_params: SummaryParams::Hll { precision: 14 }, + sketch_kind: SketchKind::Hll, + sketch_params: SketchParams::Hll { precision: 14 }, aggregation_id: "agg0".into(), }], exporter_target: ExportTarget::Endpoint("data-plane:4317".into()), @@ -6759,9 +6725,9 @@ mod tests { fn fused_asap_edge_whole_stream_hll_is_dense_regardless_of_hint() { let _env = crate::test_support::EnvVarGuard::set("ASAP_EDGE_FUSED", "1"); - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); - metric_to_family.insert("distinct_users_global".into(), one(SummaryKind::Hll)); + metric_to_family.insert("distinct_users_global".into(), one(SketchKind::Hll)); // No grouping label + an item_label ⇒ effective aggregate_by empty ⇒ // whole-stream HLL. @@ -6778,8 +6744,8 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "HLL".into(), - sketch_kind: SummaryKind::Hll, - sketch_params: SummaryParams::Hll { precision: 14 }, + sketch_kind: SketchKind::Hll, + sketch_params: SketchParams::Hll { precision: 14 }, aggregation_id: "agg0".into(), }], exporter_target: ExportTarget::Endpoint("data-plane:4317".into()), @@ -6833,10 +6799,10 @@ mod tests { fn fused_asap_edge_quantile_only_omits_mode_and_sparse() { let _env = crate::test_support::EnvVarGuard::set("ASAP_EDGE_FUSED", "1"); - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); - metric_to_family.insert("latency_ms".into(), one(SummaryKind::DDSketch)); - metric_to_family.insert("payload_bytes".into(), one(SummaryKind::Kll)); + metric_to_family.insert("latency_ms".into(), one(SketchKind::DDSketch)); + metric_to_family.insert("payload_bytes".into(), one(SketchKind::Kll)); let cfg = EdgeStageConfig { source_metric: None, @@ -6845,14 +6811,14 @@ mod tests { sketch_processors: vec![ EdgeSketchProcessor { processor_name: "ddsketch".into(), - sketch_kind: SummaryKind::DDSketch, - sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, + sketch_kind: SketchKind::DDSketch, + sketch_params: SketchParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".into(), }, EdgeSketchProcessor { processor_name: "KLL".into(), - sketch_kind: SummaryKind::Kll, - sketch_params: SummaryParams::Kll { k: 200 }, + sketch_kind: SketchKind::Kll, + sketch_params: SketchParams::Kll { k: 200 }, aggregation_id: "agg1".into(), }, ], @@ -7006,10 +6972,10 @@ mod tests { // Two metrics: a sketch-only one (warm) and one that is BOTH // sketched AND archived (both). The archive set is the precise // plan signal — only `archived_metric` is in it. - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); - metric_to_family.insert("sketch_only_metric".into(), [SummaryKind::Kll].into()); - metric_to_family.insert("archived_metric".into(), [SummaryKind::DDSketch].into()); + metric_to_family.insert("sketch_only_metric".into(), [SketchKind::Kll].into()); + metric_to_family.insert("archived_metric".into(), [SketchKind::DDSketch].into()); let cfg = EdgeStageConfig { source_metric: None, @@ -7085,7 +7051,7 @@ mod tests { let mut cfg = fused_asap_edge_cfg(); cfg.metric_to_family.insert( "http_requests_total_latency_ms".into(), - one(SummaryKind::Kll), + one(SketchKind::Kll), ); let yaml = emit_edge_yaml(&cfg, "ws://c/", "agent-1").expect("emit ok"); @@ -7333,8 +7299,8 @@ mod tests { for &(w, d) in &[(2048u32, 5u32), (1024, 4), (4096, 6), (2, 1), (256, 3)] { let sp = EdgeSketchProcessor { processor_name: "countsketch".into(), - sketch_kind: SummaryKind::CountSketch, - sketch_params: SummaryParams::CountSketch { width: w, depth: d }, + sketch_kind: SketchKind::CountSketch, + sketch_params: SketchParams::CountSketch { width: w, depth: d }, aggregation_id: "agg-cs".into(), }; let block = @@ -7379,14 +7345,11 @@ mod tests { /// `metric_to_family` but provides NO matching `EdgeSketchProcessor`, /// driving the fused emit into the catalog-default (`None`) arm. fn fused_cfg_countsketch_no_processor() -> EdgeStageConfig { - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); // CountSketch family declared, but `sketch_processors` is EMPTY for // it — the `family_to_proc.get(kind)` lookup returns None. - metric_to_family.insert( - "endpoint_request_freq".into(), - one(SummaryKind::CountSketch), - ); + metric_to_family.insert("endpoint_request_freq".into(), one(SketchKind::CountSketch)); EdgeStageConfig { source_metric: None, @@ -7444,11 +7407,11 @@ mod tests { let mut cfg = fused_cfg_countsketch_no_processor(); cfg.metric_to_family.clear(); cfg.metric_to_family - .insert("top_endpoint_qps".into(), one(SummaryKind::CountSketch)); + .insert("top_endpoint_qps".into(), one(SketchKind::CountSketch)); cfg.sketch_processors = vec![EdgeSketchProcessor { processor_name: "countsketch".into(), - sketch_kind: SummaryKind::CountSketchWithHeap, - sketch_params: SummaryParams::CountSketchWithHeap { + sketch_kind: SketchKind::CountSketchWithHeap, + sketch_params: SketchParams::CountSketchWithHeap { width: 2048, depth: 5, heap_size: 10, diff --git a/control_plane/src/emit/telegraf.rs b/control_plane/src/emit/telegraf.rs index 70014299..9e9be6d7 100644 --- a/control_plane/src/emit/telegraf.rs +++ b/control_plane/src/emit/telegraf.rs @@ -41,7 +41,7 @@ use anyhow::{Context, Result}; use crate::physical::colored_dag::emitter::{EdgeSketchProcessor, EdgeStageConfig, ExportTarget}; use crate::physical::colored_dag::stage_id::StageId; -use asap_sketch::{SummaryKind, SummaryParams}; +use planner_types::post_asap::{SketchKind, SketchParams}; /// Default Prometheus remote-write URL for Mode 3 — Telegraf doesn't /// support OTLP-HTTP egress, so we land in the same Prometheus archive @@ -168,65 +168,56 @@ fn emit_processors_allsketches( sketch_kind_tag(&sp.sketch_kind) )); match &sp.sketch_params { - SummaryParams::Kll { k } => { + SketchParams::Kll { k } => { out.push_str(&format!(" k = {k}\n")); } - SummaryParams::DDSketch { alpha } => { + SketchParams::DDSketch { alpha } => { out.push_str(&format!(" relative_accuracy = {alpha}\n")); out.push_str(" delta_transmission = true\n"); } - SummaryParams::Hll { .. } => { + SketchParams::Hll { .. } => { out.push_str(" delta_transmission = true\n"); } // Heap-bearing width/depth extraction is identical to the bare // kind — this path never distinguished `with_heap` even before - // `SummaryKind` split it into its own variant. - SummaryParams::Cms { width, depth } | SummaryParams::CmsWithHeap { width, depth, .. } => { + // `SketchKind` split it into its own variant. + SketchParams::Cms { width, depth } | SketchParams::CmsWithHeap { width, depth, .. } => { out.push_str(&format!(" rows = {depth}\n")); out.push_str(&format!(" columns = {width}\n")); out.push_str(" delta_transmission = true\n"); } - SummaryParams::CountSketch { width, depth } - | SummaryParams::CountSketchWithHeap { width, depth, .. } => { + SketchParams::CountSketch { width, depth } + | SketchParams::CountSketchWithHeap { width, depth, .. } => { let epsilon = std::f64::consts::E / (*width as f64); let delta = 2f64.powi(-(*depth as i32)); out.push_str(&format!(" epsilon = {epsilon}\n")); out.push_str(&format!(" delta = {delta}\n")); out.push_str(" delta_transmission = true\n"); } - SummaryParams::Sum - | SummaryParams::Count - | SummaryParams::MinMax - | SummaryParams::Increase - | SummaryParams::Rate - | SummaryParams::Kmv { .. } - | SummaryParams::Theta { .. } => { + // Exact-accumulator kinds (Sum/Count/MinMax/Increase/Rate) aren't + // representable in `SketchParams` at all anymore -- they're + // `ExactParams`, a distinct type post ASAPPlanner#218's split. + SketchParams::Kmv { .. } | SketchParams::Theta { .. } => { unreachable!( - "edge sketch processor config requested for a non-sketch or unsupported \ - SummaryKind; no Bind* rule in this repo produces one" + "edge sketch processor config requested for an unsupported SketchKind; \ + no Bind* rule in this repo produces one" ) } } out.push('\n'); } -fn sketch_kind_tag(kind: &SummaryKind) -> &'static str { +fn sketch_kind_tag(kind: &SketchKind) -> &'static str { match kind { - SummaryKind::Kll => "kll", - SummaryKind::DDSketch => "ddsketch", - SummaryKind::Hll => "hll", - SummaryKind::Cms | SummaryKind::CmsWithHeap => "cms", - SummaryKind::CountSketch | SummaryKind::CountSketchWithHeap => "count_sketch", - SummaryKind::Sum - | SummaryKind::Count - | SummaryKind::MinMax - | SummaryKind::Increase - | SummaryKind::Rate - | SummaryKind::Kmv - | SummaryKind::Theta => { + SketchKind::Kll => "kll", + SketchKind::DDSketch => "ddsketch", + SketchKind::Hll => "hll", + SketchKind::Cms | SketchKind::CmsWithHeap => "cms", + SketchKind::CountSketch | SketchKind::CountSketchWithHeap => "count_sketch", + SketchKind::Kmv | SketchKind::Theta => { unreachable!( - "edge sketch processor config requested for a non-sketch or unsupported \ - SummaryKind; no Bind* rule in this repo produces one" + "edge sketch processor config requested for an unsupported \ + SketchKind; no Bind* rule in this repo produces one" ) } } @@ -327,7 +318,7 @@ mod toml_minimal { mod tests { use super::*; use crate::physical::colored_dag::emitter::{EdgeSketchProcessor, PrometheusArchiveMetric}; - use asap_sketch::{SummaryKind, SummaryParams}; + use planner_types::post_asap::{SketchKind, SketchParams}; fn ddsketch_edge_cfg_mode1() -> EdgeStageConfig { EdgeStageConfig { @@ -336,8 +327,8 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "ddsketch".to_string(), - sketch_kind: SummaryKind::DDSketch, - sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, + sketch_kind: SketchKind::DDSketch, + sketch_params: SketchParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".to_string(), }], exporter_target: ExportTarget::Stage(StageId::Gateway), @@ -545,8 +536,8 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "KLL".to_string(), - sketch_kind: SummaryKind::Kll, - sketch_params: SummaryParams::Kll { k: 200 }, + sketch_kind: SketchKind::Kll, + sketch_params: SketchParams::Kll { k: 200 }, aggregation_id: "agg0".to_string(), }], exporter_target: ExportTarget::Stage(StageId::Gateway), diff --git a/control_plane/src/epsilon_alloc.rs b/control_plane/src/epsilon_alloc.rs index 1b0131dd..c6395b68 100644 --- a/control_plane/src/epsilon_alloc.rs +++ b/control_plane/src/epsilon_alloc.rs @@ -125,8 +125,8 @@ pub fn allocate_knobs( .iter() .map(|m| { let sample_p = derive_sample_p(epsilon, rate_for(&m.metric_name)); - let delta_threshold = monitor_for(&m.metric_name) - .map(|(tau, k)| derive_delta_threshold(epsilon, tau, k)); + let delta_threshold = + monitor_for(&m.metric_name).map(|(tau, k)| derive_delta_threshold(epsilon, tau, k)); AllocatedMetric { metric_name: m.metric_name.clone(), sketches: m.sketches.clone(), @@ -190,9 +190,7 @@ pub fn metric_rate_from_telemetry( continue; } if let Some(rec) = store.latest(&key) { - if let Some(tp) = - rec.payload["bench"]["throughput_items_per_sec"]["mean"].as_f64() - { + if let Some(tp) = rec.payload["bench"]["throughput_items_per_sec"]["mean"].as_f64() { total += tp; matched = true; } @@ -332,22 +330,30 @@ mod tests { #[test] fn allocate_knobs_attaches_p_and_optional_delta() { - let plan = plan_sketches_for_queries([ - "quantile_over_time(0.99, latency_ms[5m])", - ]); + let plan = plan_sketches_for_queries(["quantile_over_time(0.99, latency_ms[5m])"]); // latency_ms has a monitored threshold τ=7000 over k=4 sites; rate 2000/win let allocated = allocate_knobs( 0.05, &plan, |_m| 2000.0, - |m| if m == "latency_ms" { Some((7000.0, 4)) } else { None }, + |m| { + if m == "latency_ms" { + Some((7000.0, 4)) + } else { + None + } + }, ); let a = allocated .iter() .find(|a| a.metric_name == "latency_ms") .expect("metric present"); // p = 1/(1+0.0025*2000) = 1/6 = 0.1667… - assert!((a.knobs.sample_p - 1.0 / 6.0).abs() < 1e-9, "p={}", a.knobs.sample_p); + assert!( + (a.knobs.sample_p - 1.0 / 6.0).abs() < 1e-9, + "p={}", + a.knobs.sample_p + ); // δ = 0.05*7000/4 = 87.5 assert_eq!(a.knobs.delta_threshold, Some(87.5)); // the sketch set survived from slice 2 @@ -383,7 +389,7 @@ mod tests { ); assert!((m.sample_p - 1.0 / 6.0).abs() < 1e-9); // 1/(1+0.0025*2000) assert_eq!(m.delta_threshold, Some(87.5)); // 0.05*7000/4 - // the unparseable query falls through to cold + // the unparseable query falls through to cold assert_eq!(resp.cold_only.len(), 1); // and the response serializes to JSON cleanly (the handler's job) let v = serde_json::to_value(&resp).expect("serializes"); @@ -392,9 +398,7 @@ mod tests { #[test] fn metric_without_monitor_gets_no_delta() { - let plan = plan_sketches_for_queries([ - "quantile_over_time(0.99, latency_ms[5m])", - ]); + let plan = plan_sketches_for_queries(["quantile_over_time(0.99, latency_ms[5m])"]); let allocated = allocate_knobs(0.05, &plan, |_m| 500.0, |_m| None); let a = &allocated[0]; assert_eq!(a.knobs.delta_threshold, None); @@ -449,7 +453,11 @@ mod tests { |_m, sks| metric_rate_from_telemetry(&store, sks), &monitors, ); - let m = resp.metrics.iter().find(|m| m.metric == "latency_ms").unwrap(); + let m = resp + .metrics + .iter() + .find(|m| m.metric == "latency_ms") + .unwrap(); // p = 1/(1+0.0025*2000) = 1/6 (telemetry), NOT 1/(1+0.0025*1) (default) assert!((m.sample_p - 1.0 / 6.0).abs() < 1e-9, "p={}", m.sample_p); } diff --git a/control_plane/src/intent_algebra/agg_intent.rs b/control_plane/src/intent_algebra/agg_intent.rs index c2e2081f..f7b06ccb 100644 --- a/control_plane/src/intent_algebra/agg_intent.rs +++ b/control_plane/src/intent_algebra/agg_intent.rs @@ -34,7 +34,7 @@ //! //! ## What moved, what didn't //! -//! Per Phase 0's tie-break rule, `asap_ir::AggIntent`'s shape wins +//! Per Phase 0's tie-break rule, `planner_types::pre_asap::AggIntent`'s shape wins //! wherever it differs from this repo's pre-merge version. One //! consequence: `Rate` / `Increase` / `Changes` / `Delta` / `IDelta` / //! `Deriv` / `Resets` no longer carry `window: Duration` — the window @@ -48,8 +48,8 @@ //! the window off the enclosing `QueryExpr::Window` node it already has //! in scope, not off the intent. -use asap_ir::intent_algebra::agg_accuracy as asap_agg_accuracy; -pub use asap_ir::intent_algebra::{ +use planner_types::pre_asap::agg_accuracy as asap_agg_accuracy; +pub use planner_types::pre_asap::{ agg_is_exact, agg_is_mergeable, default_cardinality, default_quantile, is_frequency_heavy_hitter, ranking_measure, AggIntent, MathFunc, RankingMeasure, TimeFunc, }; @@ -106,7 +106,7 @@ pub fn as_frequency(intent: &AggIntent) -> Option { } /// Accuracy parameter as a fractional ε (`0.0` for exact ops). Wraps -/// `asap_ir::agg_accuracy`, which returns `0.0` for `Frequency` (it's +/// `planner_types::pre_asap::agg_accuracy`, which returns `0.0` for `Frequency` (it's /// opaque `Extension` payload to core) — special-cased here so callers /// don't need to know `Frequency` isn't a first-class shared variant. pub fn agg_accuracy(intent: &AggIntent) -> f64 { @@ -123,7 +123,7 @@ pub fn agg_accuracy(intent: &AggIntent) -> f64 { /// today. `false` means a `Bind*` rule may match. `true` means the L5 /// emitter routes the intent to the cold-store / archive tier. /// -/// Every intent `asap_ir::AggIntent` carries that this repo didn't have +/// Every intent `planner_types::pre_asap::AggIntent` carries that this repo didn't have /// before the merge (histogram accessors, math/trig, time/calendar /// accessors, presence functions, `Group`/`CountValues`, the extended /// range-vector reducers) is archive-only — none has a `Bind*` rule yet. @@ -169,7 +169,7 @@ pub fn archive_only(intent: &AggIntent) -> bool { } /// Output column name + type produced by this intent when applied to -/// `input`. Delegates to `asap_ir::AggIntent::output_column` (the +/// `input`. Delegates to `planner_types::pre_asap::AggIntent::output_column` (the /// inherent method on the shared type) for everything except /// control_plane's `Frequency` extension, which core's generic /// `Extension` handling can't name/type correctly (core has no idea diff --git a/control_plane/src/intent_algebra/binder.rs b/control_plane/src/intent_algebra/binder.rs index aa97b74a..d922a44d 100644 --- a/control_plane/src/intent_algebra/binder.rs +++ b/control_plane/src/intent_algebra/binder.rs @@ -13,4 +13,4 @@ //! [`Binder::bind_with_inherited`] for the `BinaryOp`-side-rebinding case //! (issue #52) — a capability this repo's Binder never had. -pub use asap_l2::binder::{Binder, SchemaCatalog, UsageDerivedCatalog}; +pub use planner_types::pre_asap::binder::{Binder, SchemaCatalog, UsageDerivedCatalog}; diff --git a/control_plane/src/intent_algebra/column_resolution.rs b/control_plane/src/intent_algebra/column_resolution.rs index fbb1459f..058e0dcf 100644 --- a/control_plane/src/intent_algebra/column_resolution.rs +++ b/control_plane/src/intent_algebra/column_resolution.rs @@ -23,7 +23,11 @@ //! (`Schema::closed == false`, per `Binder`'s PromQL-only usage-derived //! catalog), so this degrades to the old strict behavior in practice — //! ready for when a closed (SQL) schema starts flowing through. -pub use asap_l2::column_resolution::{ - infer_schema_for_root, infer_source_schema, output_schema_for_aggregate, resolve_column_ref, - resolve_column_refs, resolve_expr, resolve_group_keys_promql, ResolveError, +// `infer_schema_for_root`/`infer_source_schema` no longer exist upstream +// (ASAPPlanner's crate consolidation, ASAPPlanner#198) and had zero real +// callers in this repo beyond this re-export -- dropped rather than +// vendored. See control_plane/docs/design-asapplanner-pin-migration.md. +pub use planner_types::pre_asap::column_resolution::{ + output_schema_for_aggregate, resolve_column_ref, resolve_column_refs, resolve_expr, + resolve_group_keys_promql, ResolveError, }; diff --git a/control_plane/src/intent_algebra/expr_ir.rs b/control_plane/src/intent_algebra/expr_ir.rs index 41736d68..15a06f14 100644 --- a/control_plane/src/intent_algebra/expr_ir.rs +++ b/control_plane/src/intent_algebra/expr_ir.rs @@ -2,35 +2,41 @@ //! //! ## Phase 2 step 2 (docs/migration-plan-backend-plan.md) //! -//! New file, re-exported from ASAPController's `asap-ir` crate. This is -//! the D2 decision from `ASAPController/docs/intent-algebra-reconciliation.md` -//! (already validated by ASAPController's own evolution — control_plane -//! had no equivalent generic-over-column-type scalar IR before this): -//! `Predicate` and `HavingPredicate` in `query_expr.rs` are currently ad -//! hoc, control_plane-only types; `relational.rs`'s `ScalarExpr` is a -//! separate, L2-only scalar type. Once `query_expr.rs`/`relational.rs` -//! are merged (next Phase 2 step), both collapse onto this one -//! `Expr` — `L2Expr = Expr` for the front-end-emitted L2 -//! tree, `L3Expr = Expr` for the canonical positional L3 tree. -//! Nothing in this repo constructs `Expr` yet — that lands with the -//! `query_expr.rs`/`relational.rs` merge, not here. This step only makes -//! the type available. +//! Originally re-exported from ASAPController's `asap-ir` crate as a +//! standalone generic `Expr` type (`L2Expr = Expr`, `L3Expr +//! = Expr`), separate from the relational `QueryExpr` tree. +//! ASAPPlanner has since folded the scalar-expression shapes directly +//! into `QueryExpr` itself (ASAPPlanner#205, landed via #214) — one +//! recursive `QueryExpr` now carries both relational operators +//! (`Scan`/`Filter`/`Aggregate`/…) and scalar expression nodes +//! (`Column`/`Literal`/`Compare`/`BoolAnd`/`Not`/…). There is no +//! standalone `Expr`/`L2Expr`/`L3Expr` type anymore — see +//! `control_plane/docs/design-asapplanner-pin-migration.md`. //! -//! One generic [`Expr`] spans the lowering boundary; the two layers are -//! aliases that differ only in the column-reference type `C`: +//! What this repo called `L3Expr` (the canonical, positional scalar tree) +//! is therefore just `QueryExpr` itself (`QueryExpr` defaults to the positional form, matching how `L3Expr` was +//! always used here). What it called `L2Expr` (the front-end-emitted, +//! name-based tree) is `planner_types::pre_asap::UnresolvedQueryExpr` +//! (`= QueryExpr`). `L3Scalar` is renamed `ScalarValue` +//! upstream (ASAPPlanner#215), same five cases (`Int64`/`Float64`/`Utf8`/ +//! `Boolean`/`Null`). `Predicate` also changed shape alongside this — +//! it now wraps `Box>` (was a bare, unboxed inner value). //! -//! - [`L2Expr`] = `Expr` — name-based. The per-language front ends -//! emit it (PromQL label matchers, SQL `WHERE` / projection / sort-key -//! expressions) on the Layer-2 `relational` tree. -//! - [`L3Expr`] = `Expr` — **positional**. The canonical L3 -//! `query_expr` tree carries it; the converter resolves every `ColumnRef` -//! against the in-scope schema to produce it, so L3 column identity is -//! unambiguous (no name collisions across a join). -//! -//! `Expr` shares the scalar/operator vocabulary -//! ([`L3Scalar`], [`CompareOp`], [`ArithOp`]) — the **union** of what the two -//! front ends need: PromQL contributes `Regex` / `NotRegex` (`=~` / `!~`); SQL -//! contributes arithmetic, `CASE`, `IN`, `CAST`, `IS [NOT] NULL`, scalar -//! function calls, and the `LIKE` / `ILIKE` comparison family. +//! Type aliases below preserve this repo's own `L2Expr`/`L3Expr`/ +//! `L3Scalar` names as the call-site-facing surface (~135 combined call +//! sites), so this is the only file that needs to know the upstream +//! names changed. + +pub use planner_types::pre_asap::{ArithOp, ColumnRef, CompareOp, ScalarValue as L3Scalar}; +use planner_types::pre_asap::{QueryExpr, UnresolvedQueryExpr}; + +/// The front-end-emitted, name-based scalar/relational tree — was a +/// standalone `Expr`, now `QueryExpr` itself (see +/// module doc). +pub type L2Expr = UnresolvedQueryExpr; -pub use asap_ir::intent_algebra::{ArithOp, ColumnRef, CompareOp, Expr, L2Expr, L3Expr, L3Scalar}; +/// The canonical, positional scalar/relational tree — was a standalone +/// `Expr`, now plain `QueryExpr` (defaults its generic param to +/// `ColumnId`) itself (see module doc). +pub type L3Expr = QueryExpr; diff --git a/control_plane/src/intent_algebra/lower.rs b/control_plane/src/intent_algebra/lower.rs deleted file mode 100644 index b3a5a3bf..00000000 --- a/control_plane/src/intent_algebra/lower.rs +++ /dev/null @@ -1,1054 +0,0 @@ -//! The Layer-2 → canonical L3 IR converter. -//! -//! Recursively converts a *whole* `relational::QueryExpr` tree (the raw -//! Layer-2 relational IR the `query_parser` front ends emit) into a -//! *whole* canonical `query_expr::QueryExpr` tree. This is the single -//! entry the parse path routes through — [`convert_root`]. -//! -//! ## Phase 2 step 4 (docs/migration-plan-backend-plan.md) -//! -//! Now that `relational.rs` (L2) itself merged onto `asap_l2` (see that -//! file's module doc), this converter is control_plane's *own* — not a -//! re-export of `asap_l2::lower::convert_root` — because the -//! `Aggregate` arm's multi-agg fusion and `frequency_trigger` heuristic -//! (see the `Frequency` preservation section below) are control_plane's -//! own dispatch design, with no `asap_l2` equivalent (`asap_l2` always -//! threads a single workload-level `AccuracyTarget` through -//! `agg_func_to_intent` with no grouped/windowed heuristic of its own). -//! Every other *structural* piece below (scalar resolution, schema -//! threading, the `GroupKeys` shape, and now also the full `AggFunc`→ -//! `AggIntent` mapping table itself) is unchanged from `asap_l2`'s own -//! converter — `avg_over_time` used to be the one deliberate mapping -//! divergence (a p50 quantile-sketch approximation in place of -//! `asap_l2`'s literal, exact `AggIntent::Avg`) but that's gone too: -//! `AggFunc::Avg` now maps onto the literal `AggIntent::Avg` exactly -//! like `asap_l2::lower` does, since `capability_for(&AggIntent::Avg)` -//! already correctly returns `None` (no ASAP-tier sketch substitute — -//! `avg` needs a cross-policy Sum+Count join, tracked as a follow-up) -//! and ASAPController's own `crates/plan/src/bind.rs` treats -//! `AggIntent::Avg` the same way (`pass_through_intents_stay_logical` -//! keeps it a whole logical, unsketched subtree). `avg` now routes -//! through the same exact/archive path as `Sum`/`Count`/every -//! archive-only intent — `QeCollector::collect_op`'s existing catch-all -//! `exact_required = true` arm already handles it with no dedicated -//! `Avg` arm needed. -//! -//! **PromQL-frontend semantic-retarget step (topk/rate precision fix).** -//! `Rate`/`Increase`/`Changes`/`Delta`/`IDelta`/`Deriv`/`PredictLinear`/ -//! `DoubleExpSmoothing`/`Resets` used to all collapse onto -//! `AggIntent::Sum` or `Count` here — a crude placeholder bucketing -//! predating `asap_l2`'s own per-function `AggIntent` vocabulary. They -//! now map onto their own dedicated intents, matching `asap_l2`'s -//! mapping exactly (this repo no longer diverges from it for these). -//! `Rate`/`Increase` activate a real, previously-dormant -//! `capability_for` arm (`Rate | Increase => ExactAgg(Increase)`) for -//! the first time via the PromQL path; the rest are archive-only -//! either way, so the fix is `capability_for` now correctly returning -//! `None` instead of a wrong `Some(...)` for functions the ASAP tier -//! was never actually able to answer that way. `asap_tier_analysis`'s -//! `outer_fn` field is unaffected by any of this — it's computed -//! independently off the raw PromQL function name, not the lowered -//! `AggIntent`. -//! -//! `query_parser::promql`'s `topk`/`bottomk` handling was the other half -//! of this step: it used to force every `topk(...)` argument into a -//! `Count`-shaped inner regardless of what was actually being ranked -//! (silently wrong for `topk(k, avg_over_time(...))`). It now only takes -//! the heavy-hitter `TopK` path when ranking descending by -//! `count_over_time(...)` specifically (`RankingMeasure::Frequency`, -//! the one realised heavy-hitter measure) — everything else, including -//! every `bottomk`, becomes a generic `Sort + Limit`, matching -//! ASAPController's `frontend-promql` design. -//! -//! Two consequences of adopting `asap_l2::relational::QueryExpr`: -//! -//! - **No more hand-rolled scalar conversion.** `ScalarExpr` is gone — -//! every scalar position carries the shared `L2Expr` directly (same -//! generic `Expr` the canonical tree's `L3Expr` is), so -//! `column_resolution::resolve_expr` does the whole -//! name→position resolution in one generic pass. `between()`, -//! `binary_scalar_op`, and `literal_from_legacy` (all present before -//! this step) are gone with it — nothing left for them to do. -//! - **No more `Partition`, at L2 or L3.** `Aggregate` carries -//! `without: bool` directly (`asap_l2`'s own step-3-equivalent design -//! choice) — `fold_partition_keys` and the standalone -//! `LQueryExpr::Partition` arm (both present before this step) are -//! gone with it. -//! -//! `ScalarSubquery` no longer exists as a concept at all — `asap_l2`'s -//! `relational::QueryExpr` has no such variant (its `Ref`/`LetBinding` -//! are "Reserved: no front end emits yet", same as this repo's own -//! pre-merge state) — so `ConvertError::UnsupportedScalarSubquery` (this -//! step's predecessor) has nothing left to reject; removed. -//! -//! ## `Frequency` preservation (see `relational.rs`'s module doc) -//! -//! `count_over_time(...)` (and the PromQL `topk` bridge's synthetic -//! inner count) must still route through a CMS/CountSketch-family -//! `AggIntent::Extension` rather than an exact `AggIntent::Count`, but -//! `AggFunc::Frequency` no longer exists as a distinct variant to key -//! off of — the frontend now constructs plain `AggFunc::Count` for both -//! cases (matching `asap_l2`'s own frontend-promql, which leaves the -//! sketch-vs-exact choice to L4). The trigger `agg_func_to_intents` uses -//! instead: **`Count` is a `Frequency` candidate when its `Aggregate` is -//! grouped (`by`/`without` non-trivial) OR its input is an `_over_time` -//! `Window`** — precisely the two shapes `query_parser::promql` -//! constructs a windowed `Count` from (the topk bridge is grouped by the -//! topk's own `by` keys; a bare `count_over_time(...)` is windowed but -//! typically ungrouped). An un-windowed, ungrouped `Count` (SQL -//! `COUNT(*)`, not in scope for this PromQL-only step) stays exact. -//! -//! ## Schema threading -//! -//! [`convert`] takes a `&Schema` — the schema in scope at the node — and -//! threads it unchanged to every child. The [`Binder`]-built schema is -//! complete and self-contained (`(ts, value)` plus every referenced -//! name), so threading the root schema down is correct except for the -//! nested-schema-transform case (an `Aggregate` below another -//! `Aggregate`), which the L2→L3 lowering also doesn't handle — proper -//! bottom-up schema flow lands with the canonical `output_schema_in` -//! wiring downstream. - -#![allow(dead_code)] - -use asap_ir::intent_algebra::query_expr::InfoMatcher; -use asap_ir::intent_algebra::BindingName; - -use crate::intent_algebra::agg_intent::AggIntent; -use crate::intent_algebra::binder::Binder; -use crate::intent_algebra::column_resolution::{ - resolve_column_refs, resolve_expr, resolve_group_keys_promql, ResolveError, -}; -use crate::intent_algebra::query_expr::{ - GroupKeys, L3Scalar, Predicate, ProjectItem as CProjectItem, QueryExpr as CQueryExpr, - QueryExprError, Reduction, SortKey as CSortKey, Source, WindowKind as CWindowKind, -}; -use crate::intent_algebra::relational::{AggFunc, QueryExpr as LQueryExpr}; -use crate::intent_algebra::schema::Schema; -use crate::intent_algebra::L3Expr; -use crate::types_v2::AccuracyTarget; - -/// Errors produced while converting a legacy `QueryExpr` to canonical. -#[derive(Debug, thiserror::Error)] -pub enum ConvertError { - /// A column reference (`Aggregate` key, `Distinct` column, scalar - /// `Column` leaf) did not resolve against the inherited schema. - #[error("column resolution failed: {0}")] - Resolve(#[from] ResolveError), - /// An `AggItem.func` has no canonical `AggIntent` equivalent. No - /// `AggFunc` variant reaches this today (every one maps to - /// something) — kept as the escape hatch's shape for a future - /// extension point, matching `asap_l2`'s own error surface. - #[error("AggItem `{alias:?}` uses non-canonical func ({func_dbg}) — no AggIntent equivalent")] - NoCanonicalIntent { - alias: Option, - func_dbg: String, - }, - /// Schema derivation over a converted subtree failed (surfaced by - /// `column_resolution::output_schema_for_aggregate` callers, not by - /// `convert` itself today — kept for API-shape parity with - /// `asap_l2::lower::ConvertError`). - #[error("schema derivation failed: {0}")] - Schema(#[from] QueryExprError), -} - -/// Lower a legacy Layer-2 `QueryExpr` tree to the canonical L3 IR. -pub fn convert_root(legacy: &LQueryExpr) -> Result { - let schema = Binder::new().bind(legacy); - convert(legacy, &schema) -} - -/// Convert a legacy `QueryExpr` tree to canonical against an explicit -/// inherited `schema`. The schema is threaded unchanged to every child — -/// see the module doc on schema threading. -pub fn convert(legacy: &LQueryExpr, schema: &Schema) -> Result { - Ok(match legacy { - LQueryExpr::Source(spec) => { - let scan = match &spec.schema { - Some(sql_schema) => CQueryExpr::Scan { - source: Source::Table { - table_ref: spec.name.clone(), - }, - predicates: Vec::new(), - schema: sql_schema.clone(), - }, - None => CQueryExpr::Scan { - source: Source::TimeSeries { - metric: spec.name.clone(), - }, - predicates: Vec::new(), - // Carry the Binder's complete schema — the same - // self-contained scope every `ColumnId` in this tree - // resolves against. - schema: schema.clone(), - }, - }; - if spec.shift.is_identity() { - scan - } else { - CQueryExpr::TimeShift { - shift: spec.shift, - child: Box::new(scan), - } - } - } - - LQueryExpr::Scalar(v) => CQueryExpr::Scalar(*v), - LQueryExpr::EvalTime => CQueryExpr::EvalTime, - LQueryExpr::VectorFromScalar(input) => { - CQueryExpr::VectorFromScalar(Box::new(convert(input, schema)?)) - } - LQueryExpr::ScalarFromVector(input) => { - CQueryExpr::ScalarFromVector(Box::new(convert(input, schema)?)) - } - LQueryExpr::Relabel { dst, value, input } => CQueryExpr::Relabel { - dst: dst.clone(), - value: resolve_expr(value, schema)?, - child: Box::new(convert(input, schema)?), - }, - LQueryExpr::Sample { keys, kind, input } => CQueryExpr::Sample { - by: resolve_column_refs(keys, schema)?.into(), - kind: *kind, - child: Box::new(convert(input, schema)?), - }, - LQueryExpr::InfoJoin { selector, input } => CQueryExpr::InfoJoin { - selector: selector.clone(), - child: Box::new(convert(input, schema)?), - }, - - LQueryExpr::Ref(name) => CQueryExpr::Ref { - name: BindingName::new(name.clone()), - }, - - LQueryExpr::Filter { pred, input } => CQueryExpr::Filter { - pred: Predicate(resolve_expr(pred, schema)?), - child: Box::new(convert(input, schema)?), - }, - - LQueryExpr::Project { - cols, - qualifier, - input, - } => CQueryExpr::Project { - cols: cols - .iter() - .map(|item| { - Ok(CProjectItem { - alias: item.alias.clone(), - expr: resolve_expr(&item.expr, schema)?, - }) - }) - .collect::, ConvertError>>()?, - qualifier: qualifier.clone(), - child: Box::new(convert(input, schema)?), - }, - - LQueryExpr::Aggregate { - keys, - without, - aggs, - having, - input, - } => { - let by: GroupKeys = if *without { - GroupKeys::without(resolve_column_refs(keys, schema)?) - } else { - resolve_group_keys_promql(keys, schema)?.into() - }; - // See the module doc's "Frequency preservation" section — a - // grouped-or-windowed `Count` is a `Frequency` candidate. - let windowed = matches!(input.as_ref(), LQueryExpr::Window { .. }); - let frequency_trigger = !by.is_empty() || windowed; - - // A single-statistic aggregate *fuses* — done directly in - // canonical terms rather than via an intermediate legacy L3 - // node. Empty `intents` is structurally unreachable (every - // `AggFunc` maps to something), so it always falls through - // to the plain path below rather than being special-cased - // inline. - if aggs.len() == 1 && having.is_none() { - let item = &aggs[0]; - let intents = agg_func_to_intents(&item.func, frequency_trigger); - if !intents.is_empty() { - // Per-entity (issue ASAPController#163/#165): a single - // intent, no `by`/`without` at all, whose intent is - // inherently per-series or whose child is a range - // window (`windowed`, computed above -- this repo's L3 - // shape keeps `Window` *above* the `Aggregate` rather - // than a `TimeRange` child inside it, unlike - // ASAPController's current canonical shape, but it's - // the same structural marker for "this is a bare range - // reduction with no grouping syntax to begin with"). - // `by.is_without()` is never true here -- `without` PromQL - // grouping always resolves via the `without` bool above, - // which only ever produces `GroupKeys::without` with a - // concrete (possibly empty) exclusion list, and that - // path already implies a genuine reduction regardless. - let per_entity_base = by.is_empty() && !by.is_without(); - let nodes: Vec = match input.as_ref() { - LQueryExpr::Window { - duration, - slide, - input: win_input, - } => { - let kind = if slide.is_some() { - CWindowKind::Sliding - } else { - CWindowKind::Tumbling - }; - let win_child = convert(win_input, schema)?; - intents - .into_iter() - .map(|intent| { - let reduction = if per_entity_base - && (intent.is_per_series() || windowed) - { - Reduction::PerEntity - } else { - Reduction::Reduce(by.clone()) - }; - CQueryExpr::Window { - kind: kind.clone(), - size: *duration, - slide: *slide, - child: Box::new(CQueryExpr::Aggregate { - reduction, - aggs: vec![intent], - output_names: Vec::new(), - having: None, - child: Box::new(win_child.clone()), - }), - } - }) - .collect() - } - other => { - let child = convert(other, schema)?; - intents - .into_iter() - .map(|intent| { - let reduction = if per_entity_base - && (intent.is_per_series() || windowed) - { - Reduction::PerEntity - } else { - Reduction::Reduce(by.clone()) - }; - CQueryExpr::Aggregate { - reduction, - aggs: vec![intent], - output_names: Vec::new(), - having: None, - child: Box::new(child.clone()), - } - }) - .collect() - } - }; - return Ok(if nodes.len() == 1 { - nodes.into_iter().next().unwrap() - } else { - CQueryExpr::Merge { children: nodes } - }); - } - } - - // Plain canonical `Aggregate`: multi-agg, `HAVING`, or a - // (structurally unreachable) unmapped `AggFunc`. - let mut intents: Vec = Vec::with_capacity(aggs.len()); - for item in aggs { - let mapped = agg_func_to_intents(&item.func, frequency_trigger); - if mapped.is_empty() { - return Err(ConvertError::NoCanonicalIntent { - alias: item.alias.clone(), - func_dbg: format!("{:?}", item.func), - }); - } - intents.extend(mapped); - } - let having = having - .as_ref() - .map(|h| resolve_expr(h, schema).map(Predicate)) - .transpose()?; - CQueryExpr::Aggregate { - // Always a genuine reduction: this branch is multi-intent, - // HAVING-bearing, or a (structurally unreachable) unmapped - // AggFunc -- per-entity reductions are always the single, - // HAVING-less intent handled above. - reduction: Reduction::Reduce(by), - aggs: intents, - output_names: Vec::new(), - having, - child: Box::new(convert(input, schema)?), - } - } - - LQueryExpr::Window { - duration, - slide, - input, - } => CQueryExpr::Window { - kind: if slide.is_some() { - CWindowKind::Sliding - } else { - CWindowKind::Tumbling - }, - size: *duration, - slide: *slide, - child: Box::new(convert(input, schema)?), - }, - - LQueryExpr::Distinct { cols, input } => CQueryExpr::Distinct { - cols: resolve_column_refs(cols, schema)?, - child: Box::new(convert(input, schema)?), - }, - - LQueryExpr::TopK { k, by, input } => { - // γ4 classification default: heavy-hitter intent. The generic - // `Sort + Limit` case never reaches a legacy `TopK` node — - // see `topk_bridge` module doc. - let by: GroupKeys = resolve_group_keys_promql(by, schema)?.into(); - CQueryExpr::Aggregate { - // A ranking always reduces (empty by ranks the whole - // input, never per-entity). - reduction: Reduction::Reduce(by), - aggs: vec![AggIntent::TopK { - k: *k as usize, - accuracy: AccuracyTarget::Epsilon(0.05), - }], - output_names: Vec::new(), - having: None, - child: Box::new(convert(input, schema)?), - } - } - - LQueryExpr::Merge { inputs } => CQueryExpr::Merge { - children: inputs - .iter() - .map(|i| convert(i, schema)) - .collect::, _>>()?, - }, - - LQueryExpr::Join { - kind, - pred, - left, - right, - } => CQueryExpr::Join { - kind: kind.clone(), - pred: match pred { - Some(p) => Predicate(resolve_expr(p, schema)?), - // Canonical `Join` requires a predicate; a legacy `None` - // pred is a CROSS JOIN — model it as the tautology `true`. - None => Predicate(L3Expr::Literal(L3Scalar::Boolean(true))), - }, - left: Box::new(convert(left, schema)?), - right: Box::new(convert(right, schema)?), - }, - - LQueryExpr::SetOp { - kind, - all, - left, - right, - } => CQueryExpr::SetOp { - kind: kind.clone(), - all: *all, - left: Box::new(convert(left, schema)?), - right: Box::new(convert(right, schema)?), - }, - - LQueryExpr::Sort { - keys, - partition_by, - input, - } => CQueryExpr::Sort { - keys: keys - .iter() - .map(|k| { - Ok(CSortKey { - expr: resolve_expr(&k.expr, schema)?, - ascending: k.ascending, - nulls_first: k.nulls_first, - }) - }) - .collect::, ConvertError>>()?, - partition_by: resolve_column_refs(partition_by, schema)?.into(), - child: Box::new(convert(input, schema)?), - }, - - LQueryExpr::Limit { n, offset, input } => CQueryExpr::Limit { - n: *n as usize, - offset: *offset as usize, - child: Box::new(convert(input, schema)?), - }, - - LQueryExpr::LetBinding { name, expr, body } => CQueryExpr::LetBinding { - name: BindingName::new(name.clone()), - expr: Box::new(convert(expr, schema)?), - // legacy `body` is the canonical `child`. - child: Box::new(convert(body, schema)?), - }, - - LQueryExpr::PromQLSubquery { - range, - resolution, - input, - } => CQueryExpr::Subquery { - range: *range, - resolution: *resolution, - child: Box::new(convert(input, schema)?), - }, - - LQueryExpr::WindowFunc { - func, - args, - partition_by, - order_by, - output_name, - input, - } => CQueryExpr::WindowFunc { - func: func.clone(), - args: args - .iter() - .map(|a| resolve_expr(a, schema)) - .collect::, ResolveError>>()?, - partition_by: resolve_column_refs(partition_by, schema)?.into(), - order_by: order_by - .iter() - .map(|k| { - Ok(CSortKey { - expr: resolve_expr(&k.expr, schema)?, - ascending: k.ascending, - nulls_first: k.nulls_first, - }) - }) - .collect::, ConvertError>>()?, - output_name: output_name.clone(), - child: Box::new(convert(input, schema)?), - }, - - LQueryExpr::BinaryOp { - op, - lhs, - rhs, - vector_match, - } => CQueryExpr::BinaryOp { - op: op.clone(), - lhs: Box::new(convert(lhs, schema)?), - rhs: Box::new(convert(rhs, schema)?), - vector_match: vector_match.clone(), - }, - }) -} - -// ── AggFunc → AggIntent sketch mapping ─────────────────────────────────────── - -/// Map an [`AggFunc`] to the canonical [`AggIntent`]s the `convert` -/// `Aggregate` arm fuses on. `frequency_trigger` is `true` when the -/// enclosing `Aggregate` is grouped or windowed — see the module doc's -/// "Frequency preservation" section; it only affects the `Count` arm. -/// One intent for every function today — the former `StdDev`/`Variance` -/// two-quantile IQR-proxy fan-out is gone (see the `Avg`/`StdDev`/ -/// `Variance` comment below), so the `Merge`-of-siblings path this -/// `Vec` return type still supports is currently unexercised. -fn agg_func_to_intents(func: &AggFunc, frequency_trigger: bool) -> Vec { - let q = |q: f64| AggIntent::Quantile { - col: None, - q, - accuracy: AccuracyTarget::Epsilon(0.01), - }; - match func { - AggFunc::Count if frequency_trigger => vec![crate::intent_algebra::default_frequency()], - AggFunc::Count => vec![AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }], - AggFunc::Sum => vec![AggIntent::Sum { col: None }], - // `Avg` maps onto the literal, exact, non-mergeable - // `AggIntent::Avg` — matching `asap_l2::lower`'s own mapping - // exactly. `capability_for(&AggIntent::Avg)` already returns - // `None` (no ASAP-tier sketch substitute; needs a cross-policy - // Sum+Count join, tracked as a follow-up), matching - // ASAPController's own stance: `crates/plan/src/bind.rs`'s - // `pass_through_intents_stay_logical` test keeps `AggIntent::Avg` - // as a whole logical subtree with no sketch binding. So `avg` - // routes through the exact/archive path, same as `Sum`/`Count`/ - // `TopK`/every archive-only intent — `QeCollector::collect_op` - // already handles this correctly via its catch-all - // `exact_required = true` arm, no dedicated `Avg` arm needed. - AggFunc::Avg => vec![AggIntent::Avg { col: None }], - AggFunc::Min => vec![AggIntent::Min { col: None }], - AggFunc::Max => vec![AggIntent::Max { col: None }], - // `StdDev`/`Variance` map onto their literal `AggIntent`s, - // matching `asap_l2::lower` exactly — same fix as `Avg` above, - // same reasoning: `capability_for` already declares both - // archive-only (`Avg { .. } | StdDev { .. } | Variance { .. } - // => None`), so the former `vec![q(0.25), q(0.75)]` IQR proxy - // (interquartile range as a stand-in for stddev) was silently - // claiming ASAP-tier `QuantileApprox` support neither this - // repo's own capability table nor ASAPController's `asap-plan` - // (`pass_through_intents_stay_logical`) actually backs with a - // real bind rule. - AggFunc::StdDev { population } => vec![AggIntent::StdDev { - col: None, - population: *population, - }], - AggFunc::Variance { population } => vec![AggIntent::Variance { - col: None, - population: *population, - }], - AggFunc::Quantile(phi) => vec![q(*phi)], - AggFunc::CountDistinct => vec![crate::intent_algebra::default_cardinality()], - AggFunc::HeavyHitters { .. } => vec![crate::intent_algebra::default_frequency()], - // `Rate` / `Increase` now map onto their own dedicated - // `AggIntent`s (PromQL-frontend semantic-retarget step) -- - // `capability_for` already has a real, tested - // `Rate | Increase => ExactAgg(Increase)` arm (`sketch_algebra:: - // capability`); this activates it via the PromQL path for the - // first time. `asap_tier_analysis`'s `outer_fn` field is - // unaffected -- it's computed independently, straight off the - // raw PromQL function name (`trace_from_promql`'s - // `set_counter_fn`), not off the lowered `AggIntent`, so it - // still tells the engine's reducer *how* to interpret the - // accumulated value (rate needs a divide-by-range step, increase - // doesn't) regardless of what capability got matched. - AggFunc::Rate { .. } => vec![AggIntent::Rate], - AggFunc::Increase { .. } => vec![AggIntent::Increase], - // `Changes` / `Delta` / `IDelta` / `Deriv` / `PredictLinear` / - // `DoubleExpSmoothing` / `Resets` likewise now map onto their own - // dedicated `AggIntent`s instead of collapsing onto `Count`/`Sum` - // -- all are archive-only (`agg_intent::archive_only`; no - // `Bind*` rule exists for any of them, same as before this fix), - // so the real effect is `capability_for` now correctly returning - // `None` (route to archive) instead of the wrong - // `Some(ExactAgg(Sum))` / `Some(CardinalityApprox)` the old - // Sum/Count collapse produced -- these functions were never - // actually answerable from the ASAP tier that way. - AggFunc::Changes => vec![AggIntent::Changes], - AggFunc::Resets => vec![AggIntent::Resets], - AggFunc::Delta => vec![AggIntent::Delta], - AggFunc::IDelta => vec![AggIntent::IDelta], - AggFunc::Deriv => vec![AggIntent::Deriv], - AggFunc::PredictLinear { seconds } => vec![AggIntent::PredictLinear { seconds: *seconds }], - AggFunc::DoubleExpSmoothing { smoothing, trend } => { - vec![AggIntent::DoubleExpSmoothing { - smoothing: *smoothing, - trend: *trend, - }] - } - // Every remaining `AggFunc` (native-histogram accessors, - // math/trig, time/calendar, `Group`/`CountValues`, the extended - // range-vector reducers) has no pre-`asap_l2`-merge equivalent - // in this repo's PromQL surface at all -- promql.rs doesn't - // construct any of them today (`walk_call_to_op`'s exhaustive - // function-name table has no arm reaching them), so there's no - // existing behavior to preserve. Map each directly onto its - // like-named `AggIntent` (all archive-only per - // `agg_intent::archive_only`, so this is inert until a real - // caller constructs one). `Absent` / `AbsentOverTime` / - // `PresentOverTime` / `LastOverTime` *are* constructed by - // promql.rs today (`absent_over_time` / `present_over_time` / - // `last_over_time`) -- listed here rather than above only - // because they were already correctly mapped before this fix - // (never went through the Sum collapse). - AggFunc::HistogramCount => vec![AggIntent::HistogramCount], - AggFunc::HistogramSum => vec![AggIntent::HistogramSum], - AggFunc::HistogramAvg => vec![AggIntent::HistogramAvg], - AggFunc::HistogramStdDev => vec![AggIntent::HistogramStdDev], - AggFunc::HistogramStdVar => vec![AggIntent::HistogramStdVar], - AggFunc::HistogramFraction { lower, upper } => vec![AggIntent::HistogramFraction { - lower: *lower, - upper: *upper, - }], - AggFunc::HistogramQuantile(phi) => vec![AggIntent::HistogramQuantile { q: *phi }], - AggFunc::Math(f) => vec![AggIntent::Math(f.clone())], - AggFunc::Absent => vec![AggIntent::Absent], - AggFunc::AbsentOverTime => vec![AggIntent::AbsentOverTime], - AggFunc::PresentOverTime => vec![AggIntent::PresentOverTime], - AggFunc::TimeFn(f) => vec![AggIntent::TimeFn(f.clone())], - AggFunc::Group => vec![AggIntent::Group], - AggFunc::CountValues { label } => vec![AggIntent::CountValues { - label: label.clone(), - }], - AggFunc::LastOverTime => vec![AggIntent::LastOverTime], - AggFunc::FirstOverTime => vec![AggIntent::FirstOverTime], - AggFunc::MadOverTime => vec![AggIntent::MadOverTime], - AggFunc::TsOfMinOverTime => vec![AggIntent::TsOfMinOverTime], - AggFunc::TsOfMaxOverTime => vec![AggIntent::TsOfMaxOverTime], - AggFunc::TsOfFirstOverTime => vec![AggIntent::TsOfFirstOverTime], - AggFunc::TsOfLastOverTime => vec![AggIntent::TsOfLastOverTime], - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::intent_algebra::relational::{AggItem, ColumnRef as LColumnRef, SourceSpec}; - use std::time::Duration; - - fn src(name: &str) -> LQueryExpr { - LQueryExpr::Source(SourceSpec::new(name)) - } - - fn agg_item(alias: &str, func: AggFunc) -> AggItem { - AggItem { - alias: Some(alias.into()), - func, - col: LColumnRef::SampleValue, - } - } - - fn agg( - keys: Vec, - without: bool, - aggs: Vec, - input: LQueryExpr, - ) -> LQueryExpr { - LQueryExpr::Aggregate { - keys, - without, - aggs, - having: None, - input: Box::new(input), - } - } - - #[test] - fn source_becomes_scan_with_synthesized_schema() { - let c = convert_root(&src("http_requests_total")).unwrap(); - match c { - CQueryExpr::Scan { - source, - predicates, - schema, - } => { - assert!( - matches!(source, Source::TimeSeries { metric } if metric == "http_requests_total") - ); - assert!(predicates.is_empty()); - assert_eq!(schema.columns.len(), 2); // ts, value - } - other => panic!("expected Scan, got {other:?}"), - } - } - - #[test] - fn ref_and_let_binding_round_trip_names() { - let legacy = LQueryExpr::LetBinding { - name: "cte".into(), - expr: Box::new(src("m")), - body: Box::new(LQueryExpr::Ref("cte".into())), - }; - match convert_root(&legacy).unwrap() { - CQueryExpr::LetBinding { name, child, .. } => { - assert_eq!(name.as_str(), "cte"); - assert!(matches!(*child, CQueryExpr::Ref { name } if name.as_str() == "cte")); - } - other => panic!("expected LetBinding, got {other:?}"), - } - } - - #[test] - fn window_over_aggregate_full_tree() { - let legacy = LQueryExpr::Window { - duration: Duration::from_secs(300), - slide: None, - input: Box::new(agg( - vec![], - false, - vec![agg_item("s", AggFunc::Sum)], - src("m"), - )), - }; - match convert_root(&legacy).unwrap() { - CQueryExpr::Window { - kind, size, child, .. - } => { - assert_eq!(kind, CWindowKind::Tumbling); - assert_eq!(size, Duration::from_secs(300)); - match *child { - CQueryExpr::Aggregate { aggs, child, .. } => { - assert!(matches!(aggs.as_slice(), [AggIntent::Sum { col: None }])); - assert!(matches!(*child, CQueryExpr::Scan { .. })); - } - other => panic!("expected Aggregate, got {other:?}"), - } - } - other => panic!("expected Window, got {other:?}"), - } - } - - #[test] - fn single_aggregate_folds_to_canonical_aggregate() { - let legacy = agg(vec![], false, vec![agg_item("s", AggFunc::Sum)], src("m")); - match convert_root(&legacy).unwrap() { - CQueryExpr::Aggregate { - reduction, aggs, .. - } => { - assert!( - matches!(&reduction, Reduction::Reduce(k) if k.is_empty()), - "no GROUP BY, unwindowed, non-per-series intent → global reduce: {reduction:?}" - ); - assert!(matches!(aggs.as_slice(), [AggIntent::Sum { col: None }])); - } - other => panic!("expected Aggregate, got {other:?}"), - } - } - - #[test] - fn ungrouped_unwindowed_count_is_exact() { - let legacy = agg(vec![], false, vec![agg_item("n", AggFunc::Count)], src("m")); - match convert_root(&legacy).unwrap() { - CQueryExpr::Aggregate { aggs, .. } => assert!(matches!( - aggs.as_slice(), - [AggIntent::Count { - accuracy: AccuracyTarget::Exact - }] - )), - other => panic!("expected Aggregate, got {other:?}"), - } - } - - #[test] - fn windowed_count_is_frequency() { - // Mirrors `count_over_time(m[5m])`: no GROUP BY, but windowed. - let legacy = agg( - vec![], - false, - vec![agg_item("n", AggFunc::Count)], - LQueryExpr::Window { - duration: Duration::from_secs(300), - slide: None, - input: Box::new(src("m")), - }, - ); - match convert_root(&legacy).unwrap() { - CQueryExpr::Window { child, .. } => match *child { - CQueryExpr::Aggregate { aggs, .. } => { - assert!(matches!(aggs.as_slice(), [AggIntent::Extension { .. }])); - assert!(crate::intent_algebra::as_frequency(&aggs[0]).is_some()); - } - other => panic!("expected Aggregate, got {other:?}"), - }, - other => panic!("expected Window, got {other:?}"), - } - } - - #[test] - fn grouped_unwindowed_count_is_frequency() { - // Mirrors the PromQL `topk` bridge's synthetic grouped Count. - let legacy = agg( - vec![LColumnRef::Named("symbol".into())], - false, - vec![agg_item("n", AggFunc::Count)], - src("m"), - ); - match convert_root(&legacy).unwrap() { - CQueryExpr::Aggregate { aggs, .. } => { - assert!(crate::intent_algebra::as_frequency(&aggs[0]).is_some()); - } - other => panic!("expected Aggregate, got {other:?}"), - } - } - - #[test] - fn aggregate_target_column_is_not_a_group_by_key() { - let legacy = agg( - vec![], - false, - vec![AggItem { - alias: Some("s".into()), - func: AggFunc::Sum, - col: LColumnRef::Named("price".into()), - }], - src("trades"), - ); - match convert_root(&legacy).unwrap() { - CQueryExpr::Aggregate { reduction, .. } => { - assert!(matches!(&reduction, Reduction::Reduce(k) if k.is_empty())) - } - other => panic!("expected Aggregate, got {other:?}"), - } - } - - #[test] - fn single_aggregate_over_window_folds_to_window_over_aggregate() { - let legacy = agg( - vec![], - false, - vec![agg_item("q", AggFunc::Quantile(0.99))], - LQueryExpr::Window { - duration: Duration::from_secs(300), - slide: None, - input: Box::new(src("m")), - }, - ); - match convert_root(&legacy).unwrap() { - CQueryExpr::Window { kind, child, .. } => { - assert_eq!(kind, CWindowKind::Tumbling); - assert!(matches!( - *child, - CQueryExpr::Aggregate { ref reduction, ref aggs, .. } - if matches!(reduction, Reduction::PerEntity) - && matches!(aggs.as_slice(), [AggIntent::Quantile { .. }]) - )); - } - other => panic!("expected Window, got {other:?}"), - } - } - - #[test] - fn stddev_maps_to_literal_exact_intent() { - // Matches `asap_l2::lower`'s own mapping: no more two-quantile - // IQR-proxy `Merge` fan-out (that claimed a `QuantileApprox` - // ASAP-tier capability `capability_for` never actually backed - // for `StdDev`/`Variance` — same bug class as the old - // `avg → p50` approximation). - let legacy = agg( - vec![], - false, - vec![agg_item("sd", AggFunc::StdDev { population: false })], - src("m"), - ); - match convert_root(&legacy).unwrap() { - CQueryExpr::Aggregate { aggs, .. } => { - assert!(matches!( - aggs.as_slice(), - [AggIntent::StdDev { - population: false, - .. - }] - )); - } - other => panic!("expected a plain Aggregate, got {other:?}"), - } - } - - #[test] - fn rate_and_increase_map_to_dedicated_intents() { - let cases = [ - ( - AggFunc::Rate { - window: Duration::from_secs(300), - }, - AggIntent::Rate, - ), - ( - AggFunc::Increase { - window: Duration::from_secs(300), - }, - AggIntent::Increase, - ), - ]; - for (func, expected) in cases { - let legacy = agg(vec![], false, vec![agg_item("r", func)], src("m")); - match convert_root(&legacy).unwrap() { - CQueryExpr::Aggregate { aggs, .. } => { - assert_eq!(aggs.as_slice(), [expected]); - } - other => panic!("expected Aggregate, got {other:?}"), - } - } - } - - #[test] - fn changes_resets_delta_family_map_to_dedicated_intents() { - let cases = [ - (AggFunc::Changes, AggIntent::Changes), - (AggFunc::Resets, AggIntent::Resets), - (AggFunc::Delta, AggIntent::Delta), - (AggFunc::IDelta, AggIntent::IDelta), - (AggFunc::Deriv, AggIntent::Deriv), - ( - AggFunc::PredictLinear { seconds: 60.0 }, - AggIntent::PredictLinear { seconds: 60.0 }, - ), - ]; - for (func, expected) in cases { - let legacy = agg(vec![], false, vec![agg_item("x", func)], src("m")); - match convert_root(&legacy).unwrap() { - CQueryExpr::Aggregate { aggs, .. } => { - assert_eq!(aggs.as_slice(), [expected]); - } - other => panic!("expected Aggregate, got {other:?}"), - } - } - } - - #[test] - fn topk_folds_into_aggregate_with_topk_intent() { - let legacy = LQueryExpr::TopK { - k: 5, - by: vec![], - input: Box::new(src("m")), - }; - match convert_root(&legacy).unwrap() { - CQueryExpr::Aggregate { - reduction, aggs, .. - } => { - assert!(matches!(&reduction, Reduction::Reduce(k) if k.is_empty())); - assert!(matches!(aggs.as_slice(), [AggIntent::TopK { k: 5, .. }])); - } - other => panic!("expected Aggregate, got {other:?}"), - } - } - - #[test] - fn project_translates_each_item_expr() { - use crate::intent_algebra::relational::L2ProjectItem; - use crate::intent_algebra::L2Expr; - - let legacy = LQueryExpr::Project { - cols: vec![L2ProjectItem { - alias: Some("v".into()), - expr: L2Expr::Column(LColumnRef::SampleValue), - }], - qualifier: None, - input: Box::new(src("m")), - }; - match convert_root(&legacy).unwrap() { - CQueryExpr::Project { cols, .. } => { - assert_eq!(cols.len(), 1); - assert_eq!(cols[0].alias.as_deref(), Some("v")); - assert!(matches!(cols[0].expr, L3Expr::Column(_))); - } - other => panic!("expected Project, got {other:?}"), - } - } - - #[test] - fn cross_join_none_pred_becomes_true_literal() { - let legacy = LQueryExpr::Join { - kind: crate::intent_algebra::relational::JoinKind::Cross, - pred: None, - left: Box::new(src("a")), - right: Box::new(src("b")), - }; - match convert_root(&legacy).unwrap() { - CQueryExpr::Join { pred, .. } => { - assert!(matches!(pred.0, L3Expr::Literal(L3Scalar::Boolean(true)))); - } - other => panic!("expected Join, got {other:?}"), - } - } - - #[test] - fn promql_subquery_becomes_canonical_subquery() { - let legacy = LQueryExpr::PromQLSubquery { - range: Duration::from_secs(3600), - resolution: Some(Duration::from_secs(60)), - input: Box::new(src("m")), - }; - match convert_root(&legacy).unwrap() { - CQueryExpr::Subquery { - range, resolution, .. - } => { - assert_eq!(range, Duration::from_secs(3600)); - assert_eq!(resolution, Some(Duration::from_secs(60))); - } - other => panic!("expected Subquery, got {other:?}"), - } - } -} diff --git a/control_plane/src/intent_algebra/mod.rs b/control_plane/src/intent_algebra/mod.rs index b96f8e49..5d875375 100644 --- a/control_plane/src/intent_algebra/mod.rs +++ b/control_plane/src/intent_algebra/mod.rs @@ -88,12 +88,12 @@ pub mod schema; pub mod column_resolution; pub mod relational; -// The L2 → canonical-L3 lowering. The `query_parser` entry points emit a -// raw `relational::QueryExpr` tree and route it through `convert_root`, -// which lowers it (single-statistic sketchable `Aggregate` fusion folded -// in) onto the canonical IR. -pub mod lower; -pub use lower::{convert, convert_root, ConvertError}; +// `lower` (the L2 -> canonical-L3 conversion, `convert`/`convert_root`) +// was deleted in the ASAPPlanner pin migration -- see +// `relational.rs`'s module doc. `query_parser` entry points have called +// `asap_frontend_promql::lower_promql` directly since #428; this module +// was already dead in production before the migration and depended on +// the now-deleted `asap_l2` crate's L2 tree type. // Step γ7: the L3 Binder — name resolution as an explicit pass. Produces // the complete self-contained `Schema` every `ColumnId` indexes into; @@ -110,14 +110,14 @@ pub use agg_intent::{ default_frequency, default_quantile, frequency, is_frequency_heavy_hitter, output_column, ranking_measure, AggIntent, MathFunc, RankingMeasure, TimeFunc, }; -pub use expr_ir::{ArithOp, ColumnRef, CompareOp, Expr, L2Expr, L3Expr, L3Scalar}; +pub use expr_ir::{ArithOp, ColumnRef, CompareOp, L2Expr, L3Expr, L3Scalar}; pub use query_expr::{ aggregate_output_schema, between, conjoin, label_filter_to_predicate, AtModifier, BinaryOpKind, - BindingScope, DataModel, GroupKeys, GroupSide, InfoMatcher, JoinKind, LabelFilter, Predicate, - ProjectItem, QueryExpr, QueryExprError, Reduction, SampleKind, SetOpKind, SortKey, Source, - TimeShift, VectorGrouping, VectorMatch, VectorMatchKind, WindowFuncKind, WindowKind, + DataModel, GroupKeys, GroupSide, InfoMatcher, JoinKind, LabelFilter, Predicate, ProjectItem, + QueryExpr, QueryExprError, Reduction, SampleKind, SetOpKind, SortKey, Source, TimeShift, + VectorGrouping, VectorMatch, VectorMatchKind, WindowFuncKind, WindowKind, }; -pub use schema::{cse_reuse_is_legal, Column, ColumnId, CseError, DataType, Schema}; +pub use schema::{Column, ColumnId, DataType, Schema}; // Schema-driven column-resolution helpers used by the planning stack // (`optimizer/engine.rs`, `physical/{allocator,planner,stage_split}.rs`, @@ -125,6 +125,6 @@ pub use schema::{cse_reuse_is_legal, Column, ColumnId, CseError, DataType, Schem // `relational::QueryExpr` traversal. Consumers call `resolve_column_ref` // at the point where they need a positional `ColumnId`. pub use column_resolution::{ - infer_schema_for_root, infer_source_schema, output_schema_for_aggregate, resolve_column_ref, - resolve_column_refs, resolve_expr, resolve_group_keys_promql, ResolveError, + output_schema_for_aggregate, resolve_column_ref, resolve_column_refs, resolve_expr, + resolve_group_keys_promql, ResolveError, }; diff --git a/control_plane/src/intent_algebra/query_expr.rs b/control_plane/src/intent_algebra/query_expr.rs index 0ba5e459..a0f0dde5 100644 --- a/control_plane/src/intent_algebra/query_expr.rs +++ b/control_plane/src/intent_algebra/query_expr.rs @@ -4,7 +4,7 @@ //! ## Phase 2 step 3 (docs/migration-plan-backend-plan.md) //! //! `QueryExpr` and its supporting types are no longer defined in this -//! repo — re-exported from `asap_ir::intent_algebra`. `asap_ir`'s version +//! repo — re-exported from `planner_types::pre_asap::intent_algebra`. `asap_ir`'s version //! is a real superset (~22 variants vs. this repo's pre-merge 16: //! `EvalTime`, `VectorFromScalar`/`ScalarFromVector`, `Relabel`, //! `InfoJoin`, `Sample`, `TimeRange`, `TimeShift`, `WindowFunc` are new, @@ -57,17 +57,23 @@ //! `label_filter_to_predicate` below (`asap_ir`'s `Scan` takes typed //! `Predicate`s, not a separate label-filter list). -use asap_ir::intent_algebra::schema::ColumnId; -pub use asap_ir::intent_algebra::{ - aggregate_output_schema, AtModifier, BinaryOpKind, BindingScope, DataModel, GroupKeys, - GroupSide, InfoMatcher, JoinKind, Predicate, ProjectItem, QueryExpr, QueryExprError, - Reduction, SampleKind, SetOpKind, SortKey, Source, TimeShift, VectorGrouping, VectorMatch, - VectorMatchKind, WindowFuncKind, WindowKind, +use planner_types::pre_asap::schema::ColumnId; +pub use planner_types::pre_asap::{ + aggregate_output_schema, AtModifier, BinaryOpKind, DataModel, GroupKeys, GroupSide, + InfoMatcher, JoinKind, Predicate, ProjectItem, QueryExpr, QueryExprError, Reduction, + SampleKind, SetOpKind, SortKey, Source, TimeShift, VectorGrouping, VectorMatch, + VectorMatchKind, WindowFuncKind, }; -pub use asap_ir::intent_algebra::{ArithOp, ColumnRef, CompareOp, Expr, L3Scalar}; +pub use planner_types::pre_asap::{ArithOp, ColumnRef, CompareOp}; +// `L3Scalar`/`L3Expr`/`L2Expr` and `WindowKind`: see `expr_ir.rs`'s and +// `crates/asap_types/src/enums.rs`'s module docs respectively -- +// ASAPPlanner deleted its `WindowKind` (no `QueryExpr::Window` producer +// left to need it) and folded the old standalone `Expr` into +// `QueryExpr` itself, renaming its scalar-literal type `ScalarValue`. +pub use crate::intent_algebra::expr_ir::{L2Expr, L3Expr, L3Scalar}; +pub use asap_types::enums::WindowKind; use crate::intent_algebra::schema::Schema; -use crate::intent_algebra::L3Expr; /// Equality label filter on a `Scan`. PromQL `{service="api"}` — kept as /// ergonomic sugar for the parser; converted to a typed `Predicate` via @@ -86,11 +92,11 @@ pub struct LabelFilter { /// to the schema; this is a defensive fallback, not the primary path). pub fn label_filter_to_predicate(lf: &LabelFilter, schema: &Schema) -> Option { let id = schema.column_id(&lf.label)?; - Some(Predicate(L3Expr::Compare { + Some(Predicate(Box::new(L3Expr::Compare { left: Box::new(L3Expr::Column(id)), op: CompareOp::Eq, right: Box::new(L3Expr::Literal(L3Scalar::Utf8(lf.equals.clone()))), - })) + }))) } /// Conjoin `predicates` into a single `Predicate` (`BoolAnd`), or `None` @@ -98,11 +104,11 @@ pub fn label_filter_to_predicate(lf: &LabelFilter, schema: &Schema) -> Option) -> Option { - let mut exprs: Vec = predicates.into_iter().map(|p| p.0).collect(); + let mut exprs: Vec = predicates.into_iter().map(|p| *p.0).collect(); match exprs.len() { 0 => None, - 1 => Some(Predicate(exprs.remove(0))), - _ => Some(Predicate(L3Expr::BoolAnd(exprs))), + 1 => Some(Predicate(Box::new(exprs.remove(0)))), + _ => Some(Predicate(Box::new(L3Expr::BoolAnd(exprs)))), } } diff --git a/control_plane/src/intent_algebra/relational.rs b/control_plane/src/intent_algebra/relational.rs index 9c1d0fdc..29d352a8 100644 --- a/control_plane/src/intent_algebra/relational.rs +++ b/control_plane/src/intent_algebra/relational.rs @@ -1,65 +1,43 @@ -//! The **Layer-2 relational IR** — the per-language query algebra the -//! `query_parser` front ends emit, before lowering to the canonical L3 -//! `intent_algebra::query_expr` types. +//! Historically the **Layer-2 relational IR** — the per-language query +//! algebra the `query_parser` front ends emitted, before lowering to the +//! canonical L3 `intent_algebra::query_expr` types via +//! `intent_algebra::lower::convert_root`. //! -//! ## Phase 2 step 4 (docs/migration-plan-backend-plan.md) +//! ## L1 adoption superseded this (design-target-architecture.md Part B, +//! ASAPPlanner pin migration) //! -//! `QueryExpr`, `AggFunc`, `AggItem`, `SourceSpec`, `L2ProjectItem`, -//! `L2SortKey` are no longer defined in this repo — re-exported from -//! `asap_l2::relational` (the `asap-l2` crate, same git dependency / -//! pinned commit as `asap-ir`). `asap_l2`'s L2 is a real superset: every -//! node has a positional-name-based (`L2Expr` / `ColumnRef`) twin of its -//! canonical L3 counterpart (`Scalar`, `EvalTime`, `VectorFromScalar`, -//! `ScalarFromVector`, `Relabel`, `Sample`, `InfoJoin`, `WindowFunc` are -//! new), and `ScalarExpr` (this repo's own 8-variant scalar enum) is gone -//! — every scalar position (`Filter.pred`, `Aggregate.having`, -//! `Join.pred`, `Project` items) now carries the shared `L2Expr` -//! (`expr_ir::Expr`, merged in Phase 2 step 2) directly, same -//! as the canonical L3 tree carries `L3Expr`. +//! `query_parser::parse_query_expr_canonical` has called +//! `asap_frontend_promql::lower_promql` directly since #428 — full L1 +//! parse -> L2 relational -> L3 canonical conversion in one upstream +//! call, with no local L2 tree in the loop at all. This module's own +//! `QueryExpr`/`AggFunc`/`AggItem`/`L2ProjectItem`/`L2SortKey`/ +//! `SourceSpec` (re-exported from the now-deleted `asap_l2` crate — see +//! control_plane/docs/design-asapplanner-pin-migration.md) and +//! `intent_algebra::lower::{convert, convert_root}` were already dead in +//! production before the ASAPPlanner pin bump — grep-confirmed: every +//! real call site is `#[cfg(test)]`-only (`lower.rs`'s own tests, +//! `physical::window_fusion`/`planner`/`allocator`'s test fixtures). +//! `asap_l2` no longer exists upstream to re-export from regardless +//! (ASAPPlanner#213, "delete crates/l2 entirely -- both front ends emit +//! canonical QueryExpr directly"), so rather than vendor a ~20-variant L2 +//! tree type solely to keep already-dead test scaffolding compiling, +//! `lower.rs` and this module's `asap_l2`-sourced re-export were deleted +//! (matching upstream's own precedent for exactly this situation -- +//! ASAPPlanner#181/#192/#197 all deleted comparably dead scaffolding +//! rather than migrate it). The handful of dependent test functions in +//! `physical::window_fusion`/`planner`/`allocator` were removed with it; +//! see those files' own comments at the removal sites. //! -//! `Partition` doesn't exist at L2 either — `Aggregate` carries -//! `without: bool` directly (PromQL `without(...)` vs `by(...)`), matching -//! how L3's `GroupKeys` already modeled this after the `query_expr.rs` -//! merge (step 3). `PartitionKeys` is gone as a result. -//! -//! `AggFunc::Frequency` and `AggFunc::Custom(String)` — this repo's own -//! extensions, with no `asap_l2::relational::AggFunc` equivalent — are -//! **not** ported forward as enum variants (`AggFunc` is a foreign type; -//! Rust's orphan rules forbid adding variants to it from this crate). -//! Per the tie-break rule (adopt ASAPController's version; genuinely -//! control_plane-only functionality gets ported onto it, not used to -//! keep this repo's copy), both are preserved as *behavior* in -//! `lower.rs` instead of as *vocabulary*: -//! -//! - **`Frequency`** (control_plane's per-series sample-count -//! estimation, routing to CMS/CountSketch): `query_parser::promql` now -//! constructs `AggFunc::Count` for `count_over_time(...)` (matching -//! `asap_l2`'s own frontend-promql, which also uses bare `Count` and -//! leaves the sketch-vs-exact choice to L4's `Bind*` rules for the -//! `TopK`-over-`Count` heavy-hitter shape). `lower.rs`'s -//! `agg_func_to_intents` recovers control_plane's original "windowed -//! Count → Frequency sketch, non-windowed Count → exact" split — the -//! `_over_time` window wrapper is exactly the same discriminator -//! `promql.rs`'s own comment documented for the pre-merge -//! `AggFunc::Frequency` construction, so this is a vocabulary -//! substitution, not a behavior change. See `lower.rs`'s module doc for -//! the "grouped OR windowed" precise trigger. -//! - **`Custom(String)`** (arbitrary named aggregate / UDA extension -//! point): zero real construction sites in this repo (grep-verified — -//! `AggFunc::Custom` appeared only in `lower.rs`'s own dead-code -//! handling and one unit test). Dropped outright rather than ported; -//! `lower.rs`'s `agg_func_to_intents` still returns -//! `ConvertError::NoCanonicalIntent` for any `AggFunc` variant it can't -//! map (structurally unreachable today since every `asap_l2::AggFunc` -//! variant maps to something), preserving the escape hatch's shape for -//! when a real extension need arises. +//! What's left here — `AggIntent` forwarding and `PerPartitionWrap` — has +//! real, live consumers (`physical::sketch_catalog`) and doesn't touch +//! the deleted L2 tree at all. -pub use asap_ir::intent_algebra::expr_ir::{ColumnRef, L2Expr}; -pub use asap_ir::intent_algebra::query_expr::{ +pub use crate::intent_algebra::expr_ir::L2Expr; +pub use planner_types::pre_asap::expr_ir::ColumnRef; +pub use planner_types::pre_asap::query_expr::{ BinaryOpKind, GroupSide, JoinKind, SampleKind, SetOpKind, VectorGrouping, VectorMatch, VectorMatchKind, WindowFuncKind, }; -pub use asap_l2::relational::{AggFunc, AggItem, L2ProjectItem, L2SortKey, QueryExpr, SourceSpec}; /// Canonical L3 aggregation intent. Re-exported here so existing /// `relational::AggIntent` references keep working — the type is the @@ -73,10 +51,10 @@ pub use crate::intent_algebra::agg_intent::{ /// Per-partition wrapper that historically lived on the legacy `AggIntent` /// enum as a `PerPartition { inner, keys }` variant. Canonical L3 -/// represents this shape via `QueryExpr::Aggregate { by: keys, aggs: +/// represents this shape via `QueryExpr::Aggregate { by: keys, measures: /// [inner] }`; this wrapper survives only as the input type for the -/// `physical::sketch_catalog` per-partition sizing helpers. No -/// `asap_l2` equivalent — control_plane-only. +/// `physical::sketch_catalog` per-partition sizing helpers. No upstream +/// equivalent — control_plane-only. #[derive(Debug, Clone, PartialEq)] pub struct PerPartitionWrap { pub inner: AggIntent, @@ -85,9 +63,10 @@ pub struct PerPartitionWrap { /// A single filter predicate pushed down to the collector — used by /// `query_parser::promql::apply_qe_filters` to build an `L2Expr` tree -/// from a flat label-matcher list. Control_plane-only: no `asap_l2` +/// from a flat label-matcher list. Control_plane-only: no upstream /// equivalent (its front end builds `L2Expr` directly, never through an -/// intermediate flat predicate list). +/// intermediate flat predicate list). No live constructor today (see +/// module doc) — kept as a self-contained, dependency-free type. #[derive(Debug, Clone)] pub struct Predicate { pub col: String, diff --git a/control_plane/src/intent_algebra/schema.rs b/control_plane/src/intent_algebra/schema.rs index 75d2d989..b5b0332a 100644 --- a/control_plane/src/intent_algebra/schema.rs +++ b/control_plane/src/intent_algebra/schema.rs @@ -4,7 +4,7 @@ //! //! `Column` / `ColumnId` / `DataType` / `Schema` / `CseError` / //! `cse_reuse_is_legal` are no longer defined in this repo — re-exported -//! from `asap_ir::intent_algebra::schema`. Unlike `AggIntent`'s merge +//! from `planner_types::pre_asap::schema`. Unlike `AggIntent`'s merge //! (Phase 1b), this one needed no boundary-conversion layer: asap_ir's //! version is a pure additive superset of control_plane's pre-merge //! version — @@ -35,6 +35,9 @@ //! `Schema::new(..)` / `Schema::with_time_index(..)`, which default the //! new fields the same way the pre-merge constructors did. -pub use asap_ir::intent_algebra::schema::{ - cse_reuse_is_legal, Column, ColumnId, CseError, DataType, Schema, -}; +// `cse_reuse_is_legal`/`CseError` were deleted upstream alongside +// `QueryExpr::Ref`/`LetBinding` (ASAPPlanner#181/#192) -- their only +// consumer in this repo was the now-deleted `optimizer::cse` (see +// control_plane/docs/design-asapplanner-pin-migration.md), so the +// re-export is dropped with it rather than stubbed. +pub use planner_types::pre_asap::schema::{Column, ColumnId, DataType, Schema}; diff --git a/control_plane/src/lib.rs b/control_plane/src/lib.rs index 11832aca..82abf505 100644 --- a/control_plane/src/lib.rs +++ b/control_plane/src/lib.rs @@ -96,7 +96,7 @@ pub mod workload; /// the module docs for the full PromQL shape coverage matrix. pub mod asap_tier_analysis; -/// PromQL → compositional L4 plan via `asap_plan::bind::implement_tree`. +/// PromQL → compositional L4 plan via `asap_aware_mapping::bind::implement_tree`. /// Step A of the plan-shaped-serving scoping (see module doc) -- not yet /// wired into the live serving path. pub mod asap_tier_implement; diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index e328b448..6d9e0c1d 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -1,6 +1,7 @@ use control_plane::accuracy; use control_plane::backend_client; use control_plane::emit; +use control_plane::epsilon_alloc; use control_plane::intent_algebra; use control_plane::metrics_exposer; use control_plane::monitor; @@ -8,7 +9,6 @@ use control_plane::opamp; use control_plane::optimizer; use control_plane::physical; use control_plane::pipeline; -use control_plane::epsilon_alloc; use control_plane::query_parser; use control_plane::replan; use control_plane::runtime_samples; @@ -583,9 +583,8 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> // for `parse_query_expr_canonical`; this is the one call site in the // pipeline with an actual per-query accuracy value available, so // thread it through rather than a flat deployment-wide default. - let accuracy = control_plane::types_v2::accuracy_target_from_legacy_accuracy_sla( - spec.accuracy_sla, - ); + let accuracy = + control_plane::types_v2::accuracy_target_from_legacy_accuracy_sla(spec.accuracy_sla); let workload = match st.analyzer.analyze(spec) { Ok(w) => w, Err(e) => return (StatusCode::UNPROCESSABLE_ENTITY, e.to_string()).into_response(), @@ -604,7 +603,7 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> let raw_bps = plan.transmission_cost_summary.raw_bytes_per_sec; let budgets = StageResourceBudgets::from_workload_chars(&wc); // Everything that touches `sketch_algebra::PhysicalExpr` (which - // carries `Rc` since Step B of the + // carries `Rc` since Step B of the // plan-shaped-serving migration adopted ASAPController's own // `Rc`-based DAG sharing) is scoped to this block and resolved down // to Send-safe outputs (`Option`, @@ -621,7 +620,8 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> warn!(query = %qs, error = %e, "parse_query_expr_canonical failed; skipping algebra pipeline") } Ok(qe) => { - let constraints = optimizer::engine::DeploymentConstraints::from_budgets(&budgets); + let constraints = + optimizer::engine::DeploymentConstraints::from_budgets(&budgets); let (opt_qe, _) = QueryOptimizer::with_constraints(raw_bps, constraints).optimize(qe); // L4 sketch binding: lower the optimised L3 tree to the @@ -629,7 +629,9 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> let accuracy = if workload.accuracy_sla >= 1.0 { control_plane::types_v2::AccuracyTarget::Exact } else { - control_plane::types_v2::AccuracyTarget::Epsilon(1.0 - workload.accuracy_sla) + control_plane::types_v2::AccuracyTarget::Epsilon( + 1.0 - workload.accuracy_sla, + ) }; bound_physical = control_plane::sketch_algebra::bind_query_expr(&opt_qe, accuracy).ok(); @@ -714,140 +716,140 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> // `bind_workload_typed`, which lowers the flat `QueryWorkload` // summary to a `PhysicalExpr` directly. if let Some(configs) = stage_configs { - for (stage_id, stage_cfg) in configs { - match stage_cfg { - crate::physical::colored_dag::StageConfig::Edge(mut edge) => { - // MVP blocker B3 — patch per-metric grouping - // labels onto the edge cfg so the 5-sketch - // routing emitter prepends a - // `transform/keep_for_*` OTTL processor in - // front of each sketch pipeline. The typed - // L5 emitter leaves - // `metric_to_grouping_labels` empty by - // design (same rationale as the - // `agg.grouping = workload.group_by_labels` - // patch on the Backend stage below). - edge.metric_to_grouping_labels.insert( - workload.metric_name.clone(), - workload.group_by_labels.clone(), + for (stage_id, stage_cfg) in configs { + match stage_cfg { + crate::physical::colored_dag::StageConfig::Edge(mut edge) => { + // MVP blocker B3 — patch per-metric grouping + // labels onto the edge cfg so the 5-sketch + // routing emitter prepends a + // `transform/keep_for_*` OTTL processor in + // front of each sketch pipeline. The typed + // L5 emitter leaves + // `metric_to_grouping_labels` empty by + // design (same rationale as the + // `agg.grouping = workload.group_by_labels` + // patch on the Backend stage below). + edge.metric_to_grouping_labels.insert( + workload.metric_name.clone(), + workload.group_by_labels.clone(), + ); + // Issue #2: broadcast push — no single agent id + // in scope, so emit `$AGENT_ID` placeholder and + // rely on the agent container's env to expand it + // at boot. Per-agent re-pushes (push_config_to_agent + // / replan_metric inner loop) get the real id. + match emit::emit_edge_yaml(&edge, &st.opamp_endpoint, "$AGENT_ID") { + Ok(yaml) => { + let hash = short_hash(&yaml); + info!( + stage = "edge", + bytes = yaml.len(), + "[USE_TYPED_STAGE_SPLIT] pushing typed edge YAML" ); - // Issue #2: broadcast push — no single agent id - // in scope, so emit `$AGENT_ID` placeholder and - // rely on the agent container's env to expand it - // at boot. Per-agent re-pushes (push_config_to_agent - // / replan_metric inner loop) get the real id. - match emit::emit_edge_yaml(&edge, &st.opamp_endpoint, "$AGENT_ID") { - Ok(yaml) => { - let hash = short_hash(&yaml); - info!( - stage = "edge", - bytes = yaml.len(), - "[USE_TYPED_STAGE_SPLIT] pushing typed edge YAML" - ); - st.opamp - .push_to_role( - AgentRole::Agent, - RemoteConfig { - config_hash: hash, - yaml, - }, - ) - .await; - } - Err(e) => warn!(error = %e, "emit_edge_yaml failed"), - } + st.opamp + .push_to_role( + AgentRole::Agent, + RemoteConfig { + config_hash: hash, + yaml, + }, + ) + .await; } - crate::physical::colored_dag::StageConfig::Gateway(gw) => { - // Phase C: AgentRole::Gateway is now wired - // through the OpAMP role-routing path, so - // the gateway YAML is pushed to gateway-role - // collectors the same way the edge YAML is - // pushed to agent-role collectors above. - // Issue #2: gateway broadcast — `$AGENT_ID` placeholder. - match emit::emit_gateway_yaml(&gw, &st.opamp_endpoint, "$AGENT_ID") { - Ok(yaml) => { - let hash = short_hash(&yaml); - info!( - stage = "gateway", - bytes = yaml.len(), - "[USE_TYPED_STAGE_SPLIT] pushing typed gateway YAML" - ); - st.opamp - .push_to_role( - AgentRole::Gateway, - RemoteConfig { - config_hash: hash, - yaml, - }, - ) - .await; - } - Err(e) => warn!(error = %e, "emit_gateway_yaml failed"), - } - } - crate::physical::colored_dag::StageConfig::Backend(mut be) => { - // Patch metric_name + grouping from the - // workload spec. The typed L5 emitter: - // * sets `metric_name` from - // `edge.source_metric`, which is - // populated by `extract_edge_facts` - // walking the `Logical(Scan{...})` - // chain. The path-recovery isn't - // guaranteed across every binder - // output shape, so we belt-and-brace - // it with `workload.metric_name`. - // * leaves `grouping` empty because the - // canonical L3 `QueryExpr::Aggregate.by` - // is positional `ColumnId`s against a - // synthesized schema with no label - // columns (open-set label naming is - // a Step γ TODO in - // `intent_algebra::column_resolution`). - // `QueryWorkload` carries both unambiguously, - // and every aggregation under one workload - // shares them — so the patch is uniform. - let item_labels = emit::collect_metric_to_item_label( - &st.workload_registry, - &st.workload_store, + Err(e) => warn!(error = %e, "emit_edge_yaml failed"), + } + } + crate::physical::colored_dag::StageConfig::Gateway(gw) => { + // Phase C: AgentRole::Gateway is now wired + // through the OpAMP role-routing path, so + // the gateway YAML is pushed to gateway-role + // collectors the same way the edge YAML is + // pushed to agent-role collectors above. + // Issue #2: gateway broadcast — `$AGENT_ID` placeholder. + match emit::emit_gateway_yaml(&gw, &st.opamp_endpoint, "$AGENT_ID") { + Ok(yaml) => { + let hash = short_hash(&yaml); + info!( + stage = "gateway", + bytes = yaml.len(), + "[USE_TYPED_STAGE_SPLIT] pushing typed gateway YAML" ); - for agg in &mut be.aggregations { - if agg.metric_name.is_empty() { - agg.metric_name = workload.metric_name.clone(); - } - if agg.window_secs == 0 { - agg.window_secs = workload.time_window.as_secs(); - } - agg.grouping = workload.group_by_labels.clone(); - agg.item_label = item_labels.get(&agg.metric_name).cloned(); - } - // Option B unification: every typed cumulative - // emit (handle_plan here, Replanner triggers - // below, startup pre-pop tick, OpAMP - // on-connect tick) flows through the same - // helper. See [`post_typed_backend_for_role`] - // doc for the swap-semantics rationale + - // cumulative-cache contract. - // CDM monitor specs from the workload registry - // (global; coordinator_url unused for the backend's - // agg_id/τ/window-only entries). - let monitors = st.workload_registry.monitor_intents(""); - post_typed_backend_for_role( - st.backend_client.as_ref(), - &st.backend_routing_cache, - &workload.metric_name, - role, - be, - &monitors, - ) - .await; - - // Mention stage_id so `match` arms aren't - // collapsed into untagged log lines if the - // tracing filter drops the per-arm event. - let _ = stage_id; + st.opamp + .push_to_role( + AgentRole::Gateway, + RemoteConfig { + config_hash: hash, + yaml, + }, + ) + .await; + } + Err(e) => warn!(error = %e, "emit_gateway_yaml failed"), + } + } + crate::physical::colored_dag::StageConfig::Backend(mut be) => { + // Patch metric_name + grouping from the + // workload spec. The typed L5 emitter: + // * sets `metric_name` from + // `edge.source_metric`, which is + // populated by `extract_edge_facts` + // walking the `Logical(Scan{...})` + // chain. The path-recovery isn't + // guaranteed across every binder + // output shape, so we belt-and-brace + // it with `workload.metric_name`. + // * leaves `grouping` empty because the + // canonical L3 `QueryExpr::Aggregate.by` + // is positional `ColumnId`s against a + // synthesized schema with no label + // columns (open-set label naming is + // a Step γ TODO in + // `intent_algebra::column_resolution`). + // `QueryWorkload` carries both unambiguously, + // and every aggregation under one workload + // shares them — so the patch is uniform. + let item_labels = emit::collect_metric_to_item_label( + &st.workload_registry, + &st.workload_store, + ); + for agg in &mut be.aggregations { + if agg.metric_name.is_empty() { + agg.metric_name = workload.metric_name.clone(); + } + if agg.window_secs == 0 { + agg.window_secs = workload.time_window.as_secs(); } + agg.grouping = workload.group_by_labels.clone(); + agg.item_label = item_labels.get(&agg.metric_name).cloned(); } + // Option B unification: every typed cumulative + // emit (handle_plan here, Replanner triggers + // below, startup pre-pop tick, OpAMP + // on-connect tick) flows through the same + // helper. See [`post_typed_backend_for_role`] + // doc for the swap-semantics rationale + + // cumulative-cache contract. + // CDM monitor specs from the workload registry + // (global; coordinator_url unused for the backend's + // agg_id/τ/window-only entries). + let monitors = st.workload_registry.monitor_intents(""); + post_typed_backend_for_role( + st.backend_client.as_ref(), + &st.backend_routing_cache, + &workload.metric_name, + role, + be, + &monitors, + ) + .await; + + // Mention stage_id so `match` arms aren't + // collapsed into untagged log lines if the + // tracing filter drops the per-arm event. + let _ = stage_id; } + } + } } else if physical::stage_split::typed_stage_split_enabled() { warn!( metric = %workload.metric_name, @@ -1002,7 +1004,9 @@ async fn handle_plan_auto( epsilon: req.epsilon, // Auto-derive the CDM window to the edge epoch unless pinned — // a mismatch silently no-ops the grant (alignment guard). - window_secs: am.window_secs.unwrap_or_else(autonomous_monitor_window_secs), + window_secs: am + .window_secs + .unwrap_or_else(autonomous_monitor_window_secs), }); let entry = WorkloadEntry { metric_name: metric.clone(), diff --git a/control_plane/src/optimizer/cost/sketch_capability.rs b/control_plane/src/optimizer/cost/sketch_capability.rs index 370fba82..ce535c67 100644 --- a/control_plane/src/optimizer/cost/sketch_capability.rs +++ b/control_plane/src/optimizer/cost/sketch_capability.rs @@ -7,7 +7,7 @@ //! size, which intents each sketch family serves), read by the optimizer //! for cost-based plan rewriting and by the physical planner to check //! whether a sketch fits within a stage's budget — it was never L4 IR, just -//! filed alongside it because both modules touched `SummaryKind`. +//! filed alongside it because both modules touched `SketchKind`. //! //! Distinct from [`crate::sketch_algebra::schema::SketchStateMetadata`] — //! that struct carries the **L4 type-system flags** (`mergeable` / @@ -22,7 +22,7 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; -use asap_sketch::SummaryKind; +use planner_types::post_asap::SketchKind; /// Performance and capability profile for a single sketch family. /// @@ -118,15 +118,15 @@ struct SketchCapabilitiesFile { count_min_sketch: SketchCapabilityYaml, } -/// Compiled-in capability defaults — one entry per [`SummaryKind`]. +/// Compiled-in capability defaults — one entry per [`SketchKind`]. /// Replaces the per-variant `sketch_capability(SketchType)` function /// that previously lived in `algebra/optimizer.rs`. Numerical values /// are mirrored from the YAML so the in-process defaults match the /// reference deployment file. -pub fn default_capability_table() -> HashMap { +pub fn default_capability_table() -> HashMap { let mut map = HashMap::new(); map.insert( - SummaryKind::DDSketch, + SketchKind::DDSketch, SketchCapability { insert_throughput: 10_000_000.0, query_throughput: 50_000_000.0, @@ -140,7 +140,7 @@ pub fn default_capability_table() -> HashMap { }, ); map.insert( - SummaryKind::Kll, + SketchKind::Kll, SketchCapability { insert_throughput: 5_000_000.0, query_throughput: 20_000_000.0, @@ -154,7 +154,7 @@ pub fn default_capability_table() -> HashMap { }, ); map.insert( - SummaryKind::Hll, + SketchKind::Hll, SketchCapability { insert_throughput: 20_000_000.0, query_throughput: 100_000_000.0, @@ -168,7 +168,7 @@ pub fn default_capability_table() -> HashMap { }, ); map.insert( - SummaryKind::CountSketch, + SketchKind::CountSketch, SketchCapability { insert_throughput: 8_000_000.0, query_throughput: 10_000_000.0, @@ -182,7 +182,7 @@ pub fn default_capability_table() -> HashMap { }, ); map.insert( - SummaryKind::Cms, + SketchKind::Cms, SketchCapability { insert_throughput: 8_000_000.0, query_throughput: 10_000_000.0, @@ -203,15 +203,15 @@ pub fn default_capability_table() -> HashMap { /// Replaces `algebra::optimizer::load_sketch_capabilities`. /// /// Env var: `CONTROLLER_SKETCH_CAPABILITIES=path/to/this/file.yml`. -pub fn load_capability_overrides(path: &str) -> HashMap { +pub fn load_capability_overrides(path: &str) -> HashMap { if let Ok(contents) = std::fs::read_to_string(path) { if let Ok(file) = serde_yaml::from_str::(&contents) { let mut map = HashMap::new(); - map.insert(SummaryKind::DDSketch, file.ddsketch.to_capability()); - map.insert(SummaryKind::Kll, file.kll.to_capability()); - map.insert(SummaryKind::Hll, file.hll.to_capability()); - map.insert(SummaryKind::CountSketch, file.count_sketch.to_capability()); - map.insert(SummaryKind::Cms, file.count_min_sketch.to_capability()); + map.insert(SketchKind::DDSketch, file.ddsketch.to_capability()); + map.insert(SketchKind::Kll, file.kll.to_capability()); + map.insert(SketchKind::Hll, file.hll.to_capability()); + map.insert(SketchKind::CountSketch, file.count_sketch.to_capability()); + map.insert(SketchKind::Cms, file.count_min_sketch.to_capability()); return map; } } @@ -225,17 +225,17 @@ mod tests { #[test] fn default_table_carries_all_five_sketch_kinds() { let t = default_capability_table(); - assert!(t.contains_key(&SummaryKind::DDSketch)); - assert!(t.contains_key(&SummaryKind::Kll)); - assert!(t.contains_key(&SummaryKind::Hll)); - assert!(t.contains_key(&SummaryKind::Cms)); - assert!(t.contains_key(&SummaryKind::CountSketch)); + assert!(t.contains_key(&SketchKind::DDSketch)); + assert!(t.contains_key(&SketchKind::Kll)); + assert!(t.contains_key(&SketchKind::Hll)); + assert!(t.contains_key(&SketchKind::Cms)); + assert!(t.contains_key(&SketchKind::CountSketch)); } #[test] fn default_table_ddsketch_serves_quantile_intent() { let t = default_capability_table(); - let cap = t.get(&SummaryKind::DDSketch).unwrap(); + let cap = t.get(&SketchKind::DDSketch).unwrap(); assert!(cap.supported_intents.contains(&SupportedIntent::Quantile)); assert!(cap.mergeable); } @@ -243,7 +243,7 @@ mod tests { #[test] fn default_table_hll_serves_cardinality_intent() { let t = default_capability_table(); - let cap = t.get(&SummaryKind::Hll).unwrap(); + let cap = t.get(&SketchKind::Hll).unwrap(); assert!(cap .supported_intents .contains(&SupportedIntent::Cardinality)); diff --git a/control_plane/src/optimizer/cost/wire.rs b/control_plane/src/optimizer/cost/wire.rs index 20422033..6fcdfc35 100644 --- a/control_plane/src/optimizer/cost/wire.rs +++ b/control_plane/src/optimizer/cost/wire.rs @@ -31,7 +31,7 @@ //! agent telemetry once the OnlineMetricsStore feeds back into the //! planner. -use asap_sketch::SummaryKind; +use planner_types::post_asap::SketchKind; // ── Wire-cost table ────────────────────────────────────────────────────────── @@ -105,7 +105,7 @@ impl WireCostTable { /// Lookup the per-flush cost for a sketch family. /// - /// `SummaryKind` (unlike the retired `sketch_algebra::SketchKind`) + /// `SketchKind` (unlike the retired `sketch_algebra::SketchKind`) /// distinguishes heap-bearing from bare frequency sketches at the /// kind level rather than via a `with_heap` param flag. This table /// never modeled the heap's extra bytes separately (the old @@ -117,23 +117,22 @@ impl WireCostTable { /// list stays `Hll`-only, see `capability.rs`); they reuse `hll_delta` /// as a same-order-of-magnitude placeholder pending real numbers if /// this repo ever adopts them. - pub const fn for_kind(&self, kind: &SummaryKind) -> SketchWireCost { + // Exhaustive over `SketchKind` alone now (ASAPPlanner#218 split the + // old flat `SummaryKind` into `SketchKind`/`ExactKind` -- the exact- + // accumulator arm this match used to need, and its + // "exact accumulators have no sketch wire-state cost" panic, are + // unreachable by construction now instead of at runtime; see + // control_plane/docs/design-asapplanner-pin-migration.md). + pub const fn for_kind(&self, kind: &SketchKind) -> SketchWireCost { match kind { - SummaryKind::DDSketch => self.ddsketch_delta, - SummaryKind::Kll => self.kll_full, - SummaryKind::Hll => self.hll_delta, - SummaryKind::Kmv | SummaryKind::Theta => self.hll_delta, - SummaryKind::Cms => self.count_min_delta, - SummaryKind::CmsWithHeap => self.count_min_delta, - SummaryKind::CountSketch => self.count_sketch_delta, - SummaryKind::CountSketchWithHeap => self.count_sketch_delta, - SummaryKind::Sum - | SummaryKind::Count - | SummaryKind::MinMax - | SummaryKind::Increase - | SummaryKind::Rate => { - panic!("WireCostTable::for_kind: exact accumulators have no sketch wire-state cost") - } + SketchKind::DDSketch => self.ddsketch_delta, + SketchKind::Kll => self.kll_full, + SketchKind::Hll => self.hll_delta, + SketchKind::Kmv | SketchKind::Theta => self.hll_delta, + SketchKind::Cms => self.count_min_delta, + SketchKind::CmsWithHeap => self.count_min_delta, + SketchKind::CountSketch => self.count_sketch_delta, + SketchKind::CountSketchWithHeap => self.count_sketch_delta, } } } diff --git a/control_plane/src/optimizer/cse.rs b/control_plane/src/optimizer/cse.rs deleted file mode 100644 index 063f722a..00000000 --- a/control_plane/src/optimizer/cse.rs +++ /dev/null @@ -1,430 +0,0 @@ -//! Workload-level Common Sub-Expression Elimination. -//! -//! ## Phase 2 step 5 (docs/migration-plan-backend-plan.md) -//! -//! Relocated from `intent_algebra::cse` to `optimizer::cse` per the -//! Phase 0 decision: ASAPController places this pass in `crates/plan` -//! ("the cost-aware optimizer layer (L4 decisions) over the L3 intent -//! algebra"), not alongside the L3 IR type definitions — this repo's -//! `optimizer` module is that layer (R1-R12 in `engine.rs`). The -//! intended consumer, `optimizer::cost::workload_cost`, was removed as -//! dead code (never wired to a caller) in the 2026-07 retirement pass — -//! this pass has no in-tree consumer today. The -//! algorithm is otherwise identical to `asap_plan::cse` (ASAPController's -//! `crates/plan/src/cse.rs`) — adopted directly per the tie-break rule, -//! including its structural-equality candidate scan (`QueryExpr: -//! PartialEq` on a `Vec`, not a `Debug`-string-keyed `HashMap` — `{:?}` -//! is not a guaranteed-injective, stable identity contract, this repo's -//! pre-merge implementation's own shortcut). -//! -//! [`CseWorkloadPlan`] now carries `asap_ir`'s own `BindingName`/`QueryId` -//! directly (`asap_ir::intent_algebra::{BindingName, QueryId}`), not -//! `types_v2`'s separate wrapper types — matching `asap_plan::cse` and -//! removing the boundary conversion the pre-merge version needed at -//! every `QueryExpr::Ref` construction site (`asap_ir`'s `BindingName` -//! was always the *only* type that could actually name a `Ref`/ -//! `LetBinding`; the `types_v2` copy was this pass's own bookkeeping -//! type, not a real second identity). -//! `types_v2::BindingName` remains the right type everywhere else it's -//! used today (`sketch_algebra::PhysicalExpr`'s own, unrelated L4 -//! binding-name field; `pipeline.rs`'s `QueryId`) — this change is scoped -//! to the CSE↔cost-model boundary only. -//! -//! ## Regression note (R8 `CommonSubexprElim`) -//! -//! `optimizer::engine`'s R8 rule and this pass are *not* redundant, -//! despite both doing "common subexpression elimination": R8 dedupes -//! `Scan` leaves across the **branches of one `Merge` node inside a -//! single query tree**; this pass dedupes `Aggregate`-child subtrees -//! **across the root queries of a multi-query workload**. Intra-tree vs. -//! inter-tree — disjoint inputs, so relocating this pass doesn't change -//! what R8 fires on or when, and both stay. -//! -//! Per `control_plane/docs/design.md` §6 batched-queries example (line -//! ~1256 through ~1320). Multi-root planning hoists shared sub-DAGs into -//! `LetBinding`s so the cost model can credit the producer once. -//! -//! Legality is gated by [`cse_reuse_is_legal`](crate::intent_algebra::cse_reuse_is_legal): -//! a candidate sub-DAG only becomes a `LetBinding` when its output schema -//! has at least one `unique_keys` set (§6 line ~1356 — the field is -//! load-bearing for this pass). - -#![allow(dead_code)] - -use asap_ir::intent_algebra::{BindingName, QueryId}; - -use crate::intent_algebra::cse_reuse_is_legal; -use crate::intent_algebra::query_expr::QueryExpr; - -/// Multi-root container produced by the CSE pass — mirrors the shape of -/// `types_v2::WorkloadPlan` (§6 batched-queries example) but uses the -/// real `intent_algebra::QueryExpr` rather than the JSON wire-shape -/// `QueryExprPlaceholder` string. -/// -/// When `types_v2::WorkloadPlan` swaps the placeholder for the live -/// `QueryExpr`, this type collapses into that one without an API break. -#[derive(Debug, Clone, PartialEq)] -pub struct CseWorkloadPlan { - /// Named shared producers, hoisted by `dedupe_subtrees`. Each is - /// referenced by ≥2 roots via `QueryExpr::Ref`. - pub bindings: Vec<(BindingName, QueryExpr)>, - /// One root per input query, in input order. - pub roots: Vec<(QueryId, QueryExpr)>, -} - -/// Hoist sub-expressions that are *structurally identical* across ≥2 -/// roots into shared `LetBinding`s, leaving each root with `Ref` sites -/// where the duplicate sub-tree used to live. Per design.md §6 line -/// ~1272 ("a workload-level CSE pass `core::lower::workload::dedupe_subtrees`"). -/// -/// **Scope.** Implements the basic case: identifies sub-trees that -/// appear verbatim (structural equality via `PartialEq`) in ≥2 root -/// inputs and hoists them. Schema-equivalent-but-not-identical -/// sub-trees, alpha-equivalence over inner `LetBinding`s, and recursive -/// nested CSE are deferred — they are the optimisation half of the pass -/// and live downstream of this PR. -/// -/// **Legality.** A candidate sub-tree is hoisted only when -/// `cse_reuse_is_legal(&candidate.output_schema(), consumer_count)` -/// returns `Ok(())`. Sub-trees whose output schema lacks `unique_keys` -/// are left in place per design.md §6 line ~1356 (the deduper must be -/// conservative when it can't prove row identity). -pub fn dedupe_subtrees(roots: Vec<(QueryId, QueryExpr)>) -> CseWorkloadPlan { - // Empty / single-root cases: no reuse possible. Return the inputs - // verbatim with no bindings. - if roots.len() < 2 { - return CseWorkloadPlan { - bindings: vec![], - roots, - }; - } - - // Identify candidate sub-trees that appear as the immediate child of - // an `Aggregate` in ≥2 roots. The batched-queries example shape — - // multiple `Aggregate`s sharing one `Window`-child producer — is the - // case this lights up; richer detection is downstream. Grouped by - // structural equality (`QueryExpr: PartialEq`), not `Debug` output — - // `{:?}` is not a guaranteed-injective, stable identity contract. The - // candidate set is one entry per distinct root child, so this linear - // scan is bounded by the number of distinct queries. - let mut candidate_counts: Vec<(QueryExpr, usize)> = Vec::new(); - for (_, root) in &roots { - if let QueryExpr::Aggregate { child, .. } = root { - // Skip already-aliased children (a `Ref` is not a candidate - // for hoisting; it's already pointing at a binding). - if matches!(**child, QueryExpr::Ref { .. }) { - continue; - } - match candidate_counts - .iter_mut() - .find(|(e, _)| e == child.as_ref()) - { - Some(entry) => entry.1 += 1, - None => candidate_counts.push(((**child).clone(), 1)), - } - } - } - - // Pick the most-shared legal candidate. This pass hoists at most one - // binding per call; the "hoist all eligible candidates" - // generalisation is a follow-up. Choosing the most-shared first - // matches the design's priority — biggest reuse first. - let mut chosen: Option<(QueryExpr, usize)> = None; - for (expr, count) in candidate_counts.into_iter() { - if count < 2 { - continue; - } - // Legality gate: producer schema must have `unique_keys` for - // ≥2 consumers to share it (design.md §6 line ~1356). - let Ok(out_schema) = expr.output_schema() else { - continue; - }; - if cse_reuse_is_legal(&out_schema, count).is_err() { - continue; - } - // Bigger fan-in wins; ties broken by input order. - match &chosen { - Some((_, best_count)) if *best_count >= count => {} - _ => chosen = Some((expr, count)), - } - } - - let Some((shared_expr, _count)) = chosen else { - // No eligible candidate — leave roots untouched. - return CseWorkloadPlan { - bindings: vec![], - roots, - }; - }; - - // Rewrite each root: where the Aggregate's child equals - // `shared_expr`, replace with `Ref { name: "shared_0" }`. - let binding_name = BindingName::new("shared_0"); - let mut rewritten: Vec<(QueryId, QueryExpr)> = Vec::with_capacity(roots.len()); - for (qid, root) in roots { - let new_root = match root { - QueryExpr::Aggregate { - reduction, - aggs, - output_names, - having, - child, - } if *child == shared_expr => QueryExpr::Aggregate { - reduction, - aggs, - output_names, - having, - child: Box::new(QueryExpr::Ref { - name: binding_name.clone(), - }), - }, - other => other, - }; - rewritten.push((qid, new_root)); - } - - CseWorkloadPlan { - bindings: vec![(binding_name, shared_expr)], - roots: rewritten, - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::intent_algebra::agg_intent::AggIntent; - use crate::intent_algebra::query_expr::{LabelFilter, Reduction, Source, WindowKind}; - use crate::intent_algebra::schema::{Column, DataType, Schema}; - use crate::types_v2::AccuracyTarget; - use std::time::Duration; - - fn col(name: &str, dtype: DataType) -> Column { - Column { - name: name.into(), - dtype, - nullable: false, - table: None, - } - } - - fn ts_scan() -> QueryExpr { - let schema = Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("service", DataType::Utf8), - col("value", DataType::Float64), - ], - 0, - vec![vec![0, 1]], - ); - let lf = LabelFilter { - label: "service".into(), - equals: "api".into(), - }; - let pred = crate::intent_algebra::label_filter_to_predicate(&lf, &schema) - .expect("service column present in schema"); - QueryExpr::Scan { - source: Source::TimeSeries { - metric: "http_request_duration_seconds".into(), - }, - predicates: vec![pred], - schema, - } - } - - fn windowed_scan() -> QueryExpr { - QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, - child: Box::new(ts_scan()), - } - } - - /// Empty input → empty output (no bindings, no roots). - #[test] - fn dedupe_subtrees_empty_input() { - let out = dedupe_subtrees(vec![]); - assert!(out.bindings.is_empty()); - assert!(out.roots.is_empty()); - } - - /// Single-root input → no reuse possible, returned verbatim. - #[test] - fn dedupe_subtrees_single_root_passthrough() { - let q = QueryExpr::Aggregate { - reduction: Reduction::by(vec![1]), - aggs: vec![AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.01), - }], - output_names: Vec::new(), - having: None, - child: Box::new(windowed_scan()), - }; - let out = dedupe_subtrees(vec![(QueryId::new("q1"), q.clone())]); - assert!(out.bindings.is_empty()); - assert_eq!(out.roots.len(), 1); - assert_eq!(out.roots[0].1, q); - } - - /// Issue #115 (ASAPController): CSE dedupes on `AggIntent` equality. - /// Before `Quantile` carried its input column, `median(a)` and - /// `median(b)` compared equal, so two aggregates over *different* - /// columns collapsed into one — a wrong answer, not just a missed - /// optimisation. The merged `AggIntent::Quantile { col: Option, .. }` - /// already carries the column, so this is a regression guard, not new - /// behavior this repo needed to add. - #[test] - fn quantiles_over_different_columns_do_not_dedupe() { - let mk = |col: usize| QueryExpr::Aggregate { - reduction: Reduction::by(vec![1]), - aggs: vec![AggIntent::Quantile { - col: Some(col), - q: 0.5, - accuracy: AccuracyTarget::Epsilon(0.01), - }], - output_names: Vec::new(), - having: None, - child: Box::new(windowed_scan()), - }; - let (a, b) = (mk(2), mk(3)); - assert_ne!(a, b, "distinct-column quantiles must not compare equal"); - - let out = dedupe_subtrees(vec![(QueryId::new("q1"), a), (QueryId::new("q2"), b)]); - assert_ne!( - out.roots[0].1, out.roots[1].1, - "aggregates over different columns must not collapse" - ); - } - - /// design.md §6 batched-queries example basic case: two queries with - /// identical `Window` sub-trees — the deduper hoists the shared - /// producer into a binding and rewrites each root to reference it. - #[test] - fn dedupe_subtrees_basic() { - let q1 = QueryExpr::Aggregate { - reduction: Reduction::by(vec![1]), - aggs: vec![AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.01), - }], - output_names: Vec::new(), - having: None, - child: Box::new(windowed_scan()), - }; - let q2 = QueryExpr::Aggregate { - reduction: Reduction::by(vec![1]), - aggs: vec![AggIntent::Quantile { - col: None, - q: 0.95, - accuracy: AccuracyTarget::Epsilon(0.01), - }], - output_names: Vec::new(), - having: None, - child: Box::new(windowed_scan()), - }; - - let out = dedupe_subtrees(vec![(QueryId::new("q1"), q1), (QueryId::new("q2"), q2)]); - - // One binding hoisted, two roots rewritten to `Ref { name: "shared_0" }`. - assert_eq!(out.bindings.len(), 1); - assert_eq!(out.bindings[0].0, BindingName::new("shared_0")); - assert_eq!(out.bindings[0].1, windowed_scan()); - - for (_, root) in &out.roots { - match root { - QueryExpr::Aggregate { child, .. } => assert_eq!( - **child, - QueryExpr::Ref { - name: BindingName::new("shared_0"), - }, - "Aggregate child should be a Ref to the hoisted binding" - ), - other => panic!("expected Aggregate root, got {other:?}"), - } - } - } - - /// Two roots with *different* sub-expressions — no shared producer, - /// no binding hoisted, roots returned untouched. - #[test] - fn dedupe_subtrees_no_shared_subexpr() { - let q1 = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - aggs: vec![AggIntent::Sum { col: None }], - output_names: Vec::new(), - having: None, - child: Box::new(windowed_scan()), - }; - // q2 uses a different scan (different metric) → structurally - // distinct → no hoisting. - let other_scan = QueryExpr::Scan { - source: Source::TimeSeries { - metric: "different_metric".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("service", DataType::Utf8), - col("value", DataType::Float64), - ], - 0, - vec![vec![0, 1]], - ), - }; - let q2 = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - aggs: vec![AggIntent::Max { col: None }], - output_names: Vec::new(), - having: None, - child: Box::new(QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, - child: Box::new(other_scan), - }), - }; - - let out = dedupe_subtrees(vec![ - (QueryId::new("q1"), q1.clone()), - (QueryId::new("q2"), q2.clone()), - ]); - assert!(out.bindings.is_empty(), "no shared subexpr → no binding"); - assert_eq!(out.roots[0].1, q1); - assert_eq!(out.roots[1].1, q2); - } - - /// Schema without `unique_keys` → CSE refuses to share even if - /// structurally identical (ASAPController's own regression case for - /// the legality gate, ported alongside the algorithm). - #[test] - fn dedupe_subtrees_no_shared_subexpr_when_unique_keys_absent() { - let scan_no_uk = QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("value", DataType::Float64), - ], - 0, - vec![], - ), - }; - let mk = || QueryExpr::Aggregate { - reduction: Reduction::by(vec![]), - aggs: vec![AggIntent::Sum { col: None }], - output_names: Vec::new(), - having: None, - child: Box::new(scan_no_uk.clone()), - }; - let out = dedupe_subtrees(vec![(QueryId::new("q1"), mk()), (QueryId::new("q2"), mk())]); - assert!(out.bindings.is_empty(), "no unique_keys → no hoisting"); - } -} diff --git a/control_plane/src/optimizer/engine.rs b/control_plane/src/optimizer/engine.rs index 69fc4f52..41f79f5d 100644 --- a/control_plane/src/optimizer/engine.rs +++ b/control_plane/src/optimizer/engine.rs @@ -28,10 +28,6 @@ //! `WindowedAgg` into `Window { Aggregate }`, and `TopK` into an //! `Aggregate` carrying an `AggIntent::TopK`. -use std::collections::HashMap; - -use asap_ir::intent_algebra::BindingName; - use crate::intent_algebra::agg_intent::AggIntent; use crate::intent_algebra::query_expr::{QueryExpr, Reduction, SetOpKind, Source}; use crate::intent_algebra::relational::{agg_is_exact, agg_is_mergeable}; @@ -94,11 +90,11 @@ pub fn load_sketch_capabilities( /// Built-in capability profile for a known sketch type. Thin shim — /// the real defaults live in `optimizer::cost::sketch_capability::default_capability_table`. pub fn sketch_capability(st: &crate::types::SketchType) -> SketchCapability { - use asap_sketch::SummaryKind; - let kind: SummaryKind = st.clone().into(); + use planner_types::post_asap::SketchKind; + let kind: SketchKind = st.clone().into(); default_capability_table() .remove(&kind) - .expect("default_capability_table covers every SummaryKind variant") + .expect("default_capability_table covers every SketchKind variant") } // ── Stage budgets ─────────────────────────────────────────────────────────── @@ -195,9 +191,11 @@ pub struct DefaultCostModel { /// shape (multi-intent / HAVING aggregate, or a non-`Aggregate` node). fn single_sketch_intent(expr: &QueryExpr) -> Option<&AggIntent> { match expr { - QueryExpr::Aggregate { aggs, having, .. } if aggs.len() == 1 && having.is_none() => { - Some(&aggs[0]) - } + QueryExpr::Aggregate { + measures: aggs, + having, + .. + } if aggs.len() == 1 && having.is_none() => Some(&aggs[0]), _ => None, } } @@ -213,16 +211,18 @@ impl CostModel for DefaultCostModel { // into `Aggregate`, so the intent-driven factor reads the single // intent of a single-intent `Aggregate`. let factor = match expr { - QueryExpr::Aggregate { aggs, having, .. } if aggs.len() == 1 && having.is_none() => { - match &aggs[0] { - AggIntent::Quantile { .. } => 0.05, - AggIntent::Cardinality { .. } => 0.02, - op if crate::intent_algebra::as_frequency(op).is_some() => 0.03, - AggIntent::TopK { k, .. } => (*k as f64).recip().min(0.1), - op if agg_is_exact(op) => 1.0, - _ => 0.1, - } - } + QueryExpr::Aggregate { + measures: aggs, + having, + .. + } if aggs.len() == 1 && having.is_none() => match &aggs[0] { + AggIntent::Quantile { .. } => 0.05, + AggIntent::Cardinality { .. } => 0.02, + op if crate::intent_algebra::as_frequency(op).is_some() => 0.03, + AggIntent::TopK { k, .. } => (*k as f64).recip().min(0.1), + op if agg_is_exact(op) => 1.0, + _ => 0.1, + }, QueryExpr::Merge { children } => 1.0 / (children.len().max(1) as f64), QueryExpr::Filter { .. } => 0.5, QueryExpr::Distinct { .. } => 0.9, @@ -248,12 +248,14 @@ impl CostModel for DefaultCostModel { if let Some(dc) = &self.deployment { // Map each expression to its default stage budget. let stage_budget = match expr { - QueryExpr::Scan { .. } | QueryExpr::Filter { .. } | QueryExpr::Window { .. } => { + QueryExpr::Scan { .. } | QueryExpr::Filter { .. } | QueryExpr::TimeRange { .. } => { &dc.agent } - QueryExpr::Aggregate { aggs, having, .. } - if aggs.len() == 1 && having.is_none() => - { + QueryExpr::Aggregate { + measures: aggs, + having, + .. + } if aggs.len() == 1 && having.is_none() => { match &aggs[0] { // TopK-intent aggregate runs at the precompute engine. AggIntent::TopK { .. } => &dc.backend_db, @@ -345,16 +347,13 @@ impl RewriteRule for PredicatePushDown { fn try_rewrite(&self, expr: QueryExpr, _model: &dyn CostModel) -> Option { match expr { QueryExpr::Filter { pred, child } => match *child { - // Filter below Window - QueryExpr::Window { - kind, - size, - slide, + // Filter below TimeRange (the canonical range-vector-selector + // node -- see control_plane/docs/design-asapplanner-pin-migration.md). + QueryExpr::TimeRange { + range, child: inner, - } => Some(QueryExpr::Window { - kind, - size, - slide, + } => Some(QueryExpr::TimeRange { + range, child: Box::new(QueryExpr::Filter { pred, child: inner }), }), // Filter below Sort (safe when pred references input columns only) @@ -397,7 +396,7 @@ impl RewriteRule for MergeLifting { match expr { QueryExpr::Aggregate { ref reduction, - ref aggs, + measures: ref aggs, ref having, ref child, .. @@ -407,7 +406,7 @@ impl RewriteRule for MergeLifting { .iter() .map(|branch| QueryExpr::Aggregate { reduction: reduction.clone(), - aggs: aggs.clone(), + measures: aggs.clone(), output_names: Vec::new(), having: None, child: Box::new(branch.clone()), @@ -441,7 +440,7 @@ impl RewriteRule for HLLDedupElim { match expr { QueryExpr::Aggregate { reduction, - aggs, + measures: aggs, output_names, having: None, child, @@ -449,7 +448,7 @@ impl RewriteRule for HLLDedupElim { if let QueryExpr::Distinct { child: inner, .. } = *child { return Some(QueryExpr::Aggregate { reduction, - aggs, + measures: aggs, output_names, having: None, child: inner, @@ -476,17 +475,13 @@ impl RewriteRule for FilterWindowSwap { fn try_rewrite(&self, expr: QueryExpr, _model: &dyn CostModel) -> Option { match expr { QueryExpr::Filter { pred, child } => { - if let QueryExpr::Window { - kind, - size, - slide, + if let QueryExpr::TimeRange { + range, child: inner, } = *child { - return Some(QueryExpr::Window { - kind, - size, - slide, + return Some(QueryExpr::TimeRange { + range, child: Box::new(QueryExpr::Filter { pred, child: inner }), }); } @@ -524,7 +519,7 @@ impl RewriteRule for TopKFusion { if !keys.is_empty() && keys.iter().all(|k| !k.ascending) { return Some(QueryExpr::Aggregate { reduction: Reduction::by(vec![]), - aggs: vec![AggIntent::TopK { + measures: vec![AggIntent::TopK { k: n, accuracy: AccuracyTarget::Epsilon(0.05), }], @@ -560,6 +555,20 @@ impl RewriteRule for TopKFusion { /// Identify identical `Scan` sub-trees that appear in multiple branches of /// a `Merge` node and hoist them into a `LetBinding`. +/// +/// **Disabled** (ASAPPlanner pin migration): the canonical `QueryExpr` has +/// no `Ref`/`LetBinding` variant anymore (ASAPPlanner#181/#192 deleted +/// them upstream as dead scaffolding; the canonical tree type genuinely +/// has no room for them — Rust doesn't allow adding variants to a foreign +/// enum). Unlike the workload-level `optimizer::cse` this repo deleted +/// outright (zero callers, confirmed dead), this rule is a real, +/// registered `RewriteRule` — but with the hoisting *representation* +/// gone, `try_rewrite` can no longer construct its result at all. +/// Always returns `None` (no rewrite) rather than invent a +/// representation upstream doesn't support; the query still computes the +/// correct answer without this optimization, just re-scanning a +/// duplicated metric across `Merge` branches instead of sharing one scan. +/// See control_plane/docs/design-asapplanner-pin-migration.md. pub struct CommonSubexprElim; /// The metric name of a `Scan { TimeSeries }` leaf, if `qe` is one. @@ -578,54 +587,9 @@ impl RewriteRule for CommonSubexprElim { "CommonSubexprElim" } - fn try_rewrite(&self, expr: QueryExpr, _model: &dyn CostModel) -> Option { - match expr { - QueryExpr::Merge { ref children } => { - // Count occurrences of each scanned metric name. - let mut counts: HashMap = HashMap::new(); - for c in children { - if let Some(m) = scan_metric(c) { - *counts.entry(m.to_string()).or_insert(0) += 1; - } - } - let repeated: Vec = counts - .into_iter() - .filter(|(_, c)| *c > 1) - .map(|(n, _)| n) - .collect(); - if repeated.is_empty() { - return None; - } - // Hoist the first repeated scan into a LetBinding. - let metric = repeated.into_iter().next()?; - let binding_name = format!("__cse_{metric}"); - let hoisted = children - .iter() - .find(|c| scan_metric(c) == Some(metric.as_str()))? - .clone(); - let new_children: Vec = children - .iter() - .cloned() - .map(|c| { - if scan_metric(&c) == Some(metric.as_str()) { - QueryExpr::Ref { - name: BindingName::new(binding_name.clone()), - } - } else { - c - } - }) - .collect(); - Some(QueryExpr::LetBinding { - name: BindingName::new(binding_name), - expr: Box::new(hoisted), - child: Box::new(QueryExpr::Merge { - children: new_children, - }), - }) - } - _ => None, - } + fn try_rewrite(&self, _expr: QueryExpr, _model: &dyn CostModel) -> Option { + // See the struct doc: no `LetBinding`/`Ref` to hoist into anymore. + None } } @@ -664,24 +628,20 @@ impl RewriteRule for WindowMerge { fn try_rewrite(&self, expr: QueryExpr, _model: &dyn CostModel) -> Option { match expr { - QueryExpr::Window { - kind, - size, - slide, - child, - } => { - if let QueryExpr::Window { - kind: inner_kind, - size: inner_size, - slide: inner_slide, + // `TimeRange` is the canonical range-vector-selector node (see + // control_plane/docs/design-asapplanner-pin-migration.md) -- + // was `Window { kind, size, slide, child }`; the identity-merge + // rule collapses to comparing `range` alone now that there's no + // separate kind/slide to also agree on. + QueryExpr::TimeRange { range, child } => { + if let QueryExpr::TimeRange { + range: inner_range, child: inner_child, } = *child { - if kind == inner_kind && size == inner_size && slide == inner_slide { - return Some(QueryExpr::Window { - kind, - size, - slide, + if range == inner_range { + return Some(QueryExpr::TimeRange { + range, child: inner_child, }); } @@ -833,7 +793,10 @@ impl QueryOptimizer { }}; } match expr { - QueryExpr::Scan { .. } | QueryExpr::Ref { .. } => (expr, false), + // `Ref` doesn't exist in the canonical `QueryExpr` anymore + // (see `CommonSubexprElim`'s doc) -- `Scan` is the only leaf + // that still matches here unchanged. + QueryExpr::Scan { .. } => (expr, false), QueryExpr::Filter { pred, child } => { let (new_child, c) = recurse!(child); @@ -862,7 +825,7 @@ impl QueryOptimizer { } QueryExpr::Aggregate { reduction, - aggs, + measures: aggs, output_names, having, child, @@ -871,7 +834,7 @@ impl QueryOptimizer { ( QueryExpr::Aggregate { reduction, - aggs, + measures: aggs, output_names, having, child: new_child, @@ -879,18 +842,11 @@ impl QueryOptimizer { c, ) } - QueryExpr::Window { - kind, - size, - slide, - child, - } => { + QueryExpr::TimeRange { range, child } => { let (new_child, c) = recurse!(child); ( - QueryExpr::Window { - kind, - size, - slide, + QueryExpr::TimeRange { + range, child: new_child, }, c, @@ -1011,18 +967,8 @@ impl QueryOptimizer { cl || cr, ) } - QueryExpr::LetBinding { name, expr, child } => { - let (new_expr, ce) = recurse!(expr); - let (new_child, cb) = recurse!(child); - ( - QueryExpr::LetBinding { - name, - expr: new_expr, - child: new_child, - }, - ce || cb, - ) - } + // `LetBinding` doesn't exist in the canonical `QueryExpr` + // anymore -- see `CommonSubexprElim`'s doc. // `asap_ir`'s `QueryExpr` superset (Scalar / EvalTime / // VectorFromScalar / ScalarFromVector / Relabel / InfoJoin / // Sample / TimeRange / TimeShift / WindowFunc) — PromQL @@ -1085,7 +1031,7 @@ mod tests { fn sketch_agg(intent: AggIntent, child: QueryExpr) -> QueryExpr { QueryExpr::Aggregate { reduction: Reduction::by(vec![]), - aggs: vec![intent], + measures: vec![intent], output_names: Vec::new(), having: None, child: Box::new(child), @@ -1093,7 +1039,7 @@ mod tests { } fn true_pred() -> Predicate { - Predicate(L3Expr::Literal(L3Scalar::Boolean(true))) + Predicate(Box::new(L3Expr::Literal(L3Scalar::Boolean(true)))) } fn opt() -> QueryOptimizer { @@ -1112,7 +1058,7 @@ mod tests { QueryExpr::Filter { child, .. } | QueryExpr::Project { child, .. } | QueryExpr::Aggregate { child, .. } - | QueryExpr::Window { child, .. } + | QueryExpr::TimeRange { child, .. } | QueryExpr::Sort { child, .. } | QueryExpr::Limit { child, .. } | QueryExpr::Subquery { child, .. } => contains_distinct(child), @@ -1124,9 +1070,6 @@ mod tests { rhs: right, .. } => contains_distinct(left) || contains_distinct(right), - QueryExpr::LetBinding { expr, child, .. } => { - contains_distinct(expr) || contains_distinct(child) - } _ => false, } } @@ -1137,16 +1080,14 @@ mod tests { fn r1_pushes_filter_below_window() { let expr = QueryExpr::Filter { pred: true_pred(), - child: Box::new(QueryExpr::Window { - kind: WindowKind::Tumbling, - size: Duration::from_secs(60), - slide: None, + child: Box::new(QueryExpr::TimeRange { + range: Duration::from_secs(60), child: Box::new(scan("m")), }), }; let (result, _) = opt().optimize(expr); assert!( - matches!(&result, QueryExpr::Window { child, .. } + matches!(&result, QueryExpr::TimeRange { child, .. } if matches!(child.as_ref(), QueryExpr::Filter { .. })), "filter should be inside window: {result:?}" ); @@ -1208,7 +1149,7 @@ mod tests { }; let (result, _) = opt().optimize(expr); assert!( - matches!(&result, QueryExpr::Aggregate { aggs, .. } + matches!(&result, QueryExpr::Aggregate { measures: aggs, .. } if matches!(aggs.as_slice(), [AggIntent::TopK { k: 10, .. }])), "expected Aggregate[TopK(10)], got {result:?}" ); @@ -1231,7 +1172,7 @@ mod tests { }; let (result, _) = opt().optimize(expr); assert!( - !matches!(&result, QueryExpr::Aggregate { aggs, .. } + !matches!(&result, QueryExpr::Aggregate { measures: aggs, .. } if matches!(aggs.as_slice(), [AggIntent::TopK { .. }])), "ascending sort should not become a TopK aggregate" ); @@ -1241,21 +1182,17 @@ mod tests { #[test] fn r10_merges_duplicate_windows() { - let expr = QueryExpr::Window { - kind: WindowKind::Tumbling, - size: Duration::from_secs(300), - slide: None, - child: Box::new(QueryExpr::Window { - kind: WindowKind::Tumbling, - size: Duration::from_secs(300), - slide: None, + let expr = QueryExpr::TimeRange { + range: Duration::from_secs(300), + child: Box::new(QueryExpr::TimeRange { + range: Duration::from_secs(300), child: Box::new(scan("m")), }), }; let (result, _) = opt().optimize(expr); assert!( - matches!(&result, QueryExpr::Window { child, .. } - if !matches!(child.as_ref(), QueryExpr::Window { .. })), + matches!(&result, QueryExpr::TimeRange { child, .. } + if !matches!(child.as_ref(), QueryExpr::TimeRange { .. })), "duplicate window should be merged: {result:?}" ); } @@ -1276,10 +1213,8 @@ mod tests { // R3: Window(Filter(Aggregate[Cardinality](Scan))) let expr = QueryExpr::Filter { pred: true_pred(), - child: Box::new(QueryExpr::Window { - kind: WindowKind::Tumbling, - size: Duration::from_secs(60), - slide: None, + child: Box::new(QueryExpr::TimeRange { + range: Duration::from_secs(60), child: Box::new(sketch_agg( default_cardinality(), QueryExpr::Distinct { @@ -1335,30 +1270,20 @@ mod tests { } } - // ── R8: CommonSubexprElim ──────────────────────────────────────────────── + // ── R8: CommonSubexprElim (disabled -- see the rule's own doc) ─────────── #[test] - fn r8_hoists_repeated_scan() { + fn r8_no_longer_hoists_repeated_scan() { + // `CommonSubexprElim` always returns `None` now (no `LetBinding`/ + // `Ref` to hoist into) -- the tree comes back unchanged rather + // than with duplicate scans hoisted into a shared binding. let expr = QueryExpr::Merge { children: vec![scan("dup"), scan("dup"), scan("other")], }; let (result, _) = opt().optimize(expr); match result { - QueryExpr::LetBinding { name, child, .. } => { - assert!(name.as_str().starts_with("__cse_")); - // The two "dup" scans become Refs; "other" stays a Scan. - match *child { - QueryExpr::Merge { children } => { - let refs = children - .iter() - .filter(|c| matches!(c, QueryExpr::Ref { .. })) - .count(); - assert_eq!(refs, 2, "both dup scans should become Refs"); - } - other => panic!("expected Merge under LetBinding, got {other:?}"), - } - } - other => panic!("expected LetBinding, got {other:?}"), + QueryExpr::Merge { children } => assert_eq!(children.len(), 3), + other => panic!("expected Merge(3), got {other:?}"), } } diff --git a/control_plane/src/optimizer/mod.rs b/control_plane/src/optimizer/mod.rs index eaa6af8e..b8fa66e4 100644 --- a/control_plane/src/optimizer/mod.rs +++ b/control_plane/src/optimizer/mod.rs @@ -26,11 +26,22 @@ pub mod baseline; pub mod cost; -pub mod cse; pub mod engine; pub mod rules; +// `cse` (workload-level CSE via `QueryExpr::Ref`/`LetBinding`) was deleted +// as part of the ASAPPlanner pin migration (see +// control_plane/docs/design-asapplanner-pin-migration.md): ASAPPlanner +// deleted the identical `Ref`/`LetBinding` scaffolding upstream +// (ASAPPlanner#181/#192) on the same grounds this file's own module doc +// already stated -- "no in-tree consumer today" -- and the canonical +// `QueryExpr` this crate depends on no longer has room for those variants +// at all (Rust doesn't allow adding variants to a foreign enum). Nothing +// in the live query path called `dedupe_subtrees`; re-introducing +// workload-level CSE needs a representation that doesn't ride on +// `QueryExpr::Ref`/`LetBinding`, tracked as a follow-up rather than +// invented here. + // Re-exports — preserve the surface that `crate::algebra::QueryOptimizer` // and `crate::planner::*` consumers historically relied on. -pub use cse::{dedupe_subtrees, CseWorkloadPlan}; pub use engine::{DeploymentConstraints, QueryOptimizer}; diff --git a/control_plane/src/optimizer/rules/mod.rs b/control_plane/src/optimizer/rules/mod.rs index 3d78e577..7ba4f6b7 100644 --- a/control_plane/src/optimizer/rules/mod.rs +++ b/control_plane/src/optimizer/rules/mod.rs @@ -65,7 +65,7 @@ pub fn bind_workload_typed_with_item_filter( }; use crate::sketch_algebra::cost_model::ForcedFamilyCostModel; use crate::types_v2::AccuracyTarget; - use asap_sketch::SummaryKind; + use planner_types::post_asap::SketchKind; // Contract-row metrics (`classify_demo_metric` returns `Some`) and // operator-supplied overrides both signal "this metric must be @@ -112,20 +112,20 @@ pub fn bind_workload_typed_with_item_filter( // the statistic from the override whenever the derived one is // incompatible, so the override drives both family and statistic. if let Some(st) = w.sketch_type_override.as_ref() { - let ov = SummaryKind::from(st.clone()); + let ov = SketchKind::from(st.clone()); if !is_valid_pair(ov.clone(), statistic) { let (s, ap) = match ov { - SummaryKind::DDSketch | SummaryKind::Kll => { + SketchKind::DDSketch | SketchKind::Kll => { (StatisticClass::Quantile, AccuracyPreference::RelativeError) } - SummaryKind::Hll => (StatisticClass::Cardinality, AccuracyPreference::default()), - SummaryKind::Cms => (StatisticClass::Frequency, AccuracyPreference::default()), - SummaryKind::CountSketch => (StatisticClass::TopK, AccuracyPreference::default()), - // `ov` always comes from `SummaryKind::from(SketchType)` + SketchKind::Hll => (StatisticClass::Cardinality, AccuracyPreference::default()), + SketchKind::Cms => (StatisticClass::Frequency, AccuracyPreference::default()), + SketchKind::CountSketch => (StatisticClass::TopK, AccuracyPreference::default()), + // `ov` always comes from `SketchKind::from(SketchType)` // (`w.sketch_type_override` is the legacy 5-family enum), // so only these 5 canonical families are ever reachable. other => unreachable!( - "sketch_type_override resolved to an unsupported SummaryKind {other:?}" + "sketch_type_override resolved to an unsupported SketchKind {other:?}" ), }; statistic = s; @@ -139,7 +139,7 @@ pub fn bind_workload_typed_with_item_filter( return None; } - // ── Resolve the SummaryKind (override > capability-matched default) ─ + // ── Resolve the SketchKind (override > capability-matched default) ─ // // The workload-spec's `sketch_type_override` (= the spec's // `sketch_family_override` per orchestrator contract) wins over the @@ -148,10 +148,10 @@ pub fn bind_workload_typed_with_item_filter( // workload) is silently dropped — the catalog-default family runs // instead so the binding never produces a nonsense (sketch, stat) // pair. - let override_kind: Option = w + let override_kind: Option = w .sketch_type_override .as_ref() - .map(|st| SummaryKind::from(st.clone())); + .map(|st| SketchKind::from(st.clone())); let kind = match override_kind { Some(k) if is_valid_pair(k.clone(), statistic) => k, _ => pick_family(statistic, accuracy_pref)?, @@ -224,10 +224,8 @@ pub fn bind_workload_typed_with_item_filter( vec![vec![0]], ), }; - let windowed = QueryExpr::Window { - kind: WindowKind::Sliding, - size: w.time_window, - slide: None, + let windowed = QueryExpr::TimeRange { + range: w.time_window, child: Box::new(scan), }; let aggregate = QueryExpr::Aggregate { @@ -237,7 +235,7 @@ pub fn bind_workload_typed_with_item_filter( // `PerEntity` is the representative choice for a windowed shape // with no `by` (ASAPController#163/#165). reduction: crate::intent_algebra::Reduction::PerEntity, - aggs: vec![intent], + measures: vec![intent], output_names: Vec::new(), having: None, child: Box::new(windowed), @@ -252,7 +250,7 @@ pub fn bind_workload_typed_with_item_filter( // dispatcher's tie-break (DDSketch p=6 vs KLL p=5) cannot accidentally // flip `request_size_bytes`'s KLL pick to DDSketch. // - // CMS+TopK note: when the picker selected `SummaryKind::Cms` for a + // CMS+TopK note: when the picker selected `SketchKind::Cms` for a // TopK statistic (only reachable today via a `sketch_family_override: // CountMinSketch` on a TopK metric), this forces the CMS-with-heap // variant. The CMS-Heap pattern (Cormode & Muthukrishnan 2005) gives @@ -263,33 +261,28 @@ pub fn bind_workload_typed_with_item_filter( // // `StatisticClass::Frequency` (the `endpoint_request_freq` contract // row) is `AggIntent::Extension`-shaped. This used to always decline - // (return `None`) here, because `asap_plan::boundary::implementation_for` + // (return `None`) here, because `asap_aware_mapping::boundary::implementation_for` // mapped every `Extension` to `PassThrough` unconditionally (a core- // vs-deployment-specific-shape gap, ASAPController#150). Now that // `ForcedFamilyCostModel::realize_extension`/`readout_extension` // delegate to `ControlPlaneCostModel`'s own `"frequency"` handling, // this contract row commits like any other. let forced = match kind { - SummaryKind::CountSketch => SummaryKind::CountSketchWithHeap, - SummaryKind::Cms if statistic == StatisticClass::TopK => SummaryKind::CmsWithHeap, + SketchKind::CountSketch => SketchKind::CountSketchWithHeap, + SketchKind::Cms if statistic == StatisticClass::TopK => SketchKind::CmsWithHeap, other => other, }; let cost_model = ForcedFamilyCostModel::new(accuracy.clone(), forced); - let node = asap_plan::bind::implement_tree_in_with( - &aggregate, - &crate::intent_algebra::BindingScope::default(), - &cost_model, - ) - .ok()?; - // `implement_tree_in_with` never *errors* on "nothing bound" — an + let node = asap_aware_mapping::bind::implement_tree_with(&aggregate, &cost_model).ok()?; + // `implement_tree_with` never *errors* on "nothing bound" — an // intent `boundary::implementation_for`/`CostModel::realize_extension` // can't realize (e.g. `TopK { accuracy: Exact }`, ASAPController#151, - // still open) still returns `Ok(Rc)`, just wrapping the input + // still open) still returns `Ok(Rc)`, just wrapping the input // as `SummaryExpr::Logical` unchanged. `bind_workload_typed`'s own // contract is `None` for "typed path doesn't support this shape yet" // — translate the two by checking whether anything actually got // committed. - if matches!(node.expr, asap_sketch::SummaryExpr::Logical(_)) { + if matches!(node.expr, planner_types::post_asap::SummaryExpr::Logical(_)) { return None; } Some(crate::sketch_algebra::physical_expr::PhysicalExpr::committed(node)) @@ -482,18 +475,18 @@ mod tests { #[test] fn override_pins_nonquantile_family_over_misclassified_query() { use crate::emit::extract_root_sketch_kind; - use asap_sketch::SummaryKind; + use planner_types::post_asap::SketchKind; for (ov, expect) in [ - (SketchType::DDSketch, SummaryKind::DDSketch), - (SketchType::KLL, SummaryKind::Kll), - (SketchType::HLL, SummaryKind::Hll), + (SketchType::DDSketch, SketchKind::DDSketch), + (SketchType::KLL, SketchKind::Kll), + (SketchType::HLL, SketchKind::Hll), // CountSketch override re-derives statistic as TopK (see the // override arm below), and `bind_cms_topk` always binds the // heap-bearing kind for a top-k intent — matches this - // fixture's pre-`SummaryKind`-split expectation, when + // fixture's pre-`SketchKind`-split expectation, when // `with_heap: true` was a params flag rather than kind // identity. - (SketchType::CountSketch, SummaryKind::CountSketchWithHeap), + (SketchType::CountSketch, SketchKind::CountSketchWithHeap), ] { // Query classifies as Quantile (the mis-derived case observed // live for count()/topk()/count_over_time()); the override must win. @@ -512,7 +505,7 @@ mod tests { // `AggIntent::Extension`-shaped (this deployment's point-frequency // query). `ControlPlaneCostModel::realize_extension`/ // `readout_extension` (ASAPController#150) now realize it as - // `SummaryKind::Cms` — matching `capability_matching::pick_family`'s + // `SketchKind::Cms` — matching `capability_matching::pick_family`'s // own `Frequency -> Cms` mapping — so this override binds like any // other now. let mut w = workload(vec![AggType::Quantile]); @@ -521,7 +514,7 @@ mod tests { .unwrap_or_else(|| panic!("CountMinSketch override should bind (ASAPController#150)")); assert_eq!( extract_root_sketch_kind(&pe), - Some(SummaryKind::Cms), + Some(SketchKind::Cms), "CountMinSketch override should pin Cms (Frequency's family), not decline", ); } @@ -646,14 +639,14 @@ mod tests { // | `endpoint_request_freq` | CMS | use crate::sketch_algebra::physical_expr::PhysicalExpr; - use asap_ir::intent_algebra::expr_ir::ColumnRef; - use asap_sketch::SummaryKind; + use planner_types::post_asap::SketchKind; + use planner_types::pre_asap::expr_ir::ColumnRef; /// Walk the L4 binding output and pull out the approximate sketch /// family. Returns `None` if no sketch node is present (raw / pure /// logical pass-through, or an exact accumulator — see /// `emit::extract_root_sketch_kind`, whose logic this mirrors). - fn extract_family(expr: &PhysicalExpr) -> Option { + fn extract_family(expr: &PhysicalExpr) -> Option { crate::emit::extract_root_sketch_kind(expr) } @@ -661,14 +654,16 @@ mod tests { /// `SummaryEstimate` -- unlike `extract_family`, this needs the /// readout itself (to check `PointCount`'s `key`/`value`), not just /// the sketch family underneath it. - fn extract_query(expr: &PhysicalExpr) -> Option { + fn extract_query(expr: &PhysicalExpr) -> Option { let PhysicalExpr::Committed(crate::sketch_algebra::physical_expr::L4Plan::Summary(node)) = expr else { return None; }; match &node.expr { - asap_sketch::SummaryExpr::SummaryEstimate { query, .. } => Some(query.clone()), + planner_types::post_asap::SummaryExpr::SummaryEstimate { query, .. } => { + Some(query.clone()) + } _ => None, } } @@ -715,7 +710,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("http_latency_ms must bind"); assert_eq!( extract_family(&bound), - Some(SummaryKind::DDSketch), + Some(SketchKind::DDSketch), "http_latency_ms should bind to DDSketch (Quantile, rel-err)", ); } @@ -730,7 +725,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("request_size_bytes must bind"); assert_eq!( extract_family(&bound), - Some(SummaryKind::Kll), + Some(SketchKind::Kll), "request_size_bytes should bind to KLL (Quantile, rank-err)", ); } @@ -742,7 +737,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("unique_users_per_min must bind"); assert_eq!( extract_family(&bound), - Some(SummaryKind::Hll), + Some(SketchKind::Hll), "unique_users_per_min should bind to HLL (Cardinality)", ); } @@ -757,7 +752,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("top_endpoint_qps must bind"); assert_eq!( extract_family(&bound), - Some(SummaryKind::CountSketchWithHeap), + Some(SketchKind::CountSketchWithHeap), "top_endpoint_qps should bind to CountSketch-with-heap (TopK)", ); } @@ -766,7 +761,7 @@ mod tests { fn typed_binding_endpoint_request_freq_binds_cms() { // Contract: `endpoint_request_freq` → CMS (Frequency). `Frequency` // is `AggIntent::Extension`-shaped; this used to decline the typed - // path entirely (`asap_plan::boundary::implementation_for` mapped + // path entirely (`asap_aware_mapping::boundary::implementation_for` mapped // every `Extension` to `PassThrough` unconditionally — core has no // realization opinion for a deployment-specific shape it doesn't // know, ASAPController#150). Now that @@ -776,7 +771,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("endpoint_request_freq must bind"); assert_eq!( extract_family(&bound), - Some(SummaryKind::Cms), + Some(SketchKind::Cms), "endpoint_request_freq should bind to Cms (Frequency)", ); } @@ -790,7 +785,7 @@ mod tests { assert!( matches!( extract_query(&bound), - Some(asap_sketch::SketchQuery::PointCount { + Some(planner_types::post_asap::SketchQuery::PointCount { key: ColumnRef::SampleValue, value: None }) @@ -805,7 +800,7 @@ mod tests { bind_workload_typed_with_item_filter(&w, Some(("endpoint", "checkout"))) .expect("must bind"); match extract_query(&bound_filtered) { - Some(asap_sketch::SketchQuery::PointCount { + Some(planner_types::post_asap::SketchQuery::PointCount { key: ColumnRef::Named(label), value: Some(value), }) => { @@ -829,7 +824,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("override should still bind"); assert_eq!( extract_family(&bound), - Some(SummaryKind::Kll), + Some(SketchKind::Kll), "sketch_type_override=KLL should pin KLL despite the contract's DDSketch default", ); } @@ -843,7 +838,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("override should still bind"); assert_eq!( extract_family(&bound), - Some(SummaryKind::DDSketch), + Some(SketchKind::DDSketch), "sketch_type_override=DDSketch should pin DDSketch despite the contract's KLL default", ); } @@ -861,7 +856,7 @@ mod tests { bind_workload_typed(&w).expect("CountMin override on a TopK metric should still bind"); assert_eq!( extract_family(&bound), - Some(SummaryKind::CmsWithHeap), + Some(SketchKind::CmsWithHeap), "sketch_type_override=CountMinSketch on a TopK metric should pin CMS-with-heap", ); } @@ -875,7 +870,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("top_endpoint_qps must bind"); assert_eq!( extract_family(&bound), - Some(SummaryKind::CountSketchWithHeap), + Some(SketchKind::CountSketchWithHeap), "default TopK pick must remain CountSketch-with-heap (unbiased estimator)", ); } @@ -890,7 +885,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("fallback should bind"); assert_eq!( extract_family(&bound), - Some(SummaryKind::DDSketch), + Some(SketchKind::DDSketch), "invalid (HLL, Quantile) override should be rejected; planner falls back to DDSketch", ); } @@ -902,27 +897,27 @@ mod tests { // Single test that drives the full contract row set through // `bind_workload_typed` — this is the per-task acceptance test // ("verify each produces the expected `PhysicalExpr` family"). - let cases: Vec<(&str, AggType, Option)> = vec![ + let cases: Vec<(&str, AggType, Option)> = vec![ ("http_requests_total", AggType::Frequency, None), ( "http_latency_ms", AggType::Quantile, - Some(SummaryKind::DDSketch), + Some(SketchKind::DDSketch), ), ( "request_size_bytes", AggType::Quantile, - Some(SummaryKind::Kll), + Some(SketchKind::Kll), ), ( "unique_users_per_min", AggType::Cardinality, - Some(SummaryKind::Hll), + Some(SketchKind::Hll), ), ( "top_endpoint_qps", AggType::Frequency, - Some(SummaryKind::CountSketchWithHeap), + Some(SketchKind::CountSketchWithHeap), ), // `Extension`/Frequency now binds via `ControlPlaneCostModel`'s // `realize_extension` (ASAPController#150) — see @@ -930,7 +925,7 @@ mod tests { ( "endpoint_request_freq", AggType::Frequency, - Some(SummaryKind::Cms), + Some(SketchKind::Cms), ), ]; for (metric, agg, expected) in cases { diff --git a/control_plane/src/physical/allocator.rs b/control_plane/src/physical/allocator.rs index 85a09236..1b9256cd 100644 --- a/control_plane/src/physical/allocator.rs +++ b/control_plane/src/physical/allocator.rs @@ -122,7 +122,7 @@ impl SketchAllocator { fn alloc_node(&self, expr: QueryExpr, budget: &mut BudgetState) -> PlanNode { match expr { // ── Leaves ─────────────────────────────────────────────────────── - QueryExpr::Scan { .. } | QueryExpr::Ref { .. } => { + QueryExpr::Scan { .. } => { PlanNode::leaf(expr, PipelineStage::Agent, ExecutionMode::Passthrough) } @@ -148,22 +148,17 @@ impl SketchAllocator { } } - // A `Window` over a single-intent `Aggregate` is the canonical - // fold of the legacy `WindowedAgg`; for *stage* allocation the - // window is informational — this arm is a passthrough and the - // inner `Aggregate` arm does the sketch placement. - QueryExpr::Window { - kind, - size, - slide, - child, - } => { + // A `TimeRange` over a single-intent `Aggregate` is the + // canonical fold of the legacy `WindowedAgg` (was `Window` + // before the ASAPPlanner pin migration); for *stage* + // allocation the range is informational — this arm is a + // passthrough and the inner `Aggregate` arm does the sketch + // placement. + QueryExpr::TimeRange { range, child } => { let child = self.alloc_node(*child, budget); PlanNode { - expr: QueryExpr::Window { - kind, - size, - slide, + expr: QueryExpr::TimeRange { + range, child: Box::new(child.expr.clone()), }, stage: PipelineStage::Agent, @@ -202,7 +197,7 @@ impl SketchAllocator { // * multi-intent or HAVING → general exact Aggregate at Db QueryExpr::Aggregate { reduction, - aggs, + measures: aggs, output_names, having, child, @@ -214,7 +209,7 @@ impl SketchAllocator { return PlanNode { expr: QueryExpr::Aggregate { reduction, - aggs, + measures: aggs, output_names, having, child: Box::new(child.expr.clone()), @@ -246,7 +241,7 @@ impl SketchAllocator { PlanNode { expr: QueryExpr::Aggregate { reduction, - aggs, + measures: aggs, output_names, having, child: Box::new(child.expr.clone()), @@ -480,28 +475,8 @@ impl SketchAllocator { } } - // ── Scoping constructs — propagate body's stage ─────────────── - QueryExpr::LetBinding { name, expr, child } => { - let expr_node = self.alloc_node(*expr, budget); - let body_node = self.alloc_node(*child, budget); - let stage = body_node.stage.clone(); - let mode = body_node.mode.clone(); - PlanNode { - expr: QueryExpr::LetBinding { - name, - expr: Box::new(expr_node.expr.clone()), - child: Box::new(body_node.expr.clone()), - }, - stage, - mode, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale: "LetBinding: stage = body stage".into(), - ..Default::default() - }, - children: vec![expr_node, body_node], - } - } + // `LetBinding`/`Ref` don't exist in the canonical `QueryExpr` + // anymore (see `optimizer::engine::CommonSubexprElim`'s doc). // `asap_ir`'s PromQL-surface superset (Scalar / EvalTime / // VectorFromScalar / ScalarFromVector / Relabel / InfoJoin / @@ -544,12 +519,6 @@ impl SketchAllocator { child: Box::new(c), }) } - QueryExpr::TimeRange { range, child } => { - self.alloc_passthrough_child(child, "TimeRange", budget, |c| QueryExpr::TimeRange { - range, - child: Box::new(c), - }) - } QueryExpr::TimeShift { shift, child } => { self.alloc_passthrough_child(child, "TimeShift", budget, |c| QueryExpr::TimeShift { shift, @@ -573,6 +542,17 @@ impl SketchAllocator { child: Box::new(c), } }), + // Scalar-expression node (Column/Literal/Compare/BoolAnd/ + // BoolOr/Not/IsNull/IsNotNull/Cast/InList/FunctionCall/Arith/ + // Case) — these live inside `Predicate`/`ProjectItem` scalar + // positions this deployment's front end builds, never as a + // bare top-level relational tree node reaching this + // allocator directly (folded into `QueryExpr` itself by + // ASAPPlanner#205/#214, see + // control_plane/docs/design-asapplanner-pin-migration.md). + // Defensive leaf fallback, matching `Scan`'s treatment, + // rather than a panic, in case that assumption ever breaks. + _ => PlanNode::leaf(expr, PipelineStage::Agent, ExecutionMode::Passthrough), } } @@ -621,7 +601,7 @@ impl SketchAllocator { return PlanNode { expr: QueryExpr::Aggregate { reduction, - aggs, + measures: aggs, output_names, having: None, child: Box::new(child.expr.clone()), @@ -645,7 +625,7 @@ impl SketchAllocator { return PlanNode { expr: QueryExpr::Aggregate { reduction, - aggs, + measures: aggs, output_names, having: None, child: Box::new(child.expr.clone()), @@ -675,7 +655,7 @@ impl SketchAllocator { return PlanNode { expr: QueryExpr::Aggregate { reduction, - aggs, + measures: aggs, output_names, having: None, child: Box::new(child.expr.clone()), @@ -703,7 +683,7 @@ impl SketchAllocator { return PlanNode { expr: QueryExpr::Aggregate { reduction, - aggs, + measures: aggs, output_names, having: None, child: Box::new(child.expr.clone()), @@ -731,7 +711,7 @@ impl SketchAllocator { PlanNode { expr: QueryExpr::Aggregate { reduction, - aggs, + measures: aggs, output_names, having: None, child: Box::new(child.expr.clone()), @@ -845,7 +825,7 @@ mod tests { fn agg(intent: AggIntent) -> QueryExpr { QueryExpr::Aggregate { reduction: Reduction::by(vec![]), - aggs: vec![intent], + measures: vec![intent], output_names: Vec::new(), having: None, child: Box::new(scan("m")), @@ -889,7 +869,7 @@ mod tests { #[test] fn filter_at_agent() { let expr = QueryExpr::Filter { - pred: Predicate(L3Expr::Literal(L3Scalar::Boolean(true))), + pred: Predicate(Box::new(L3Expr::Literal(L3Scalar::Boolean(true)))), child: Box::new(scan("m")), }; let node = alloc(unlimited(), expr); @@ -992,7 +972,7 @@ mod tests { fn join_goes_to_db() { let expr = QueryExpr::Join { kind: JoinKind::Inner, - pred: Predicate(L3Expr::Literal(L3Scalar::Boolean(true))), + pred: Predicate(Box::new(L3Expr::Literal(L3Scalar::Boolean(true)))), left: Box::new(scan("orders")), right: Box::new(scan("items")), }; @@ -1006,7 +986,7 @@ mod tests { fn multi_intent_aggregate_goes_to_db() { let expr = QueryExpr::Aggregate { reduction: Reduction::by(vec![]), - aggs: vec![AggIntent::Sum { col: None }, AggIntent::Min { col: None }], + measures: vec![AggIntent::Sum { col: None }, AggIntent::Min { col: None }], output_names: Vec::new(), having: None, child: Box::new(scan("m")), @@ -1016,16 +996,14 @@ mod tests { assert_eq!(node.mode, ExecutionMode::Exact); } - // ── Window over a single-intent Aggregate (the WindowedAgg fold) ────────── + // ── TimeRange over a single-intent Aggregate (the WindowedAgg fold) ─────── #[test] fn window_over_aggregate_window_passthrough_agg_sketches() { - // Canonical fold of legacy `WindowedAgg`: Window passthrough at + // Canonical fold of legacy `WindowedAgg`: TimeRange passthrough at // Agent, inner Aggregate does the sketch placement. - let expr = QueryExpr::Window { - kind: crate::intent_algebra::WindowKind::Tumbling, - size: std::time::Duration::from_secs(300), - slide: None, + let expr = QueryExpr::TimeRange { + range: std::time::Duration::from_secs(300), child: Box::new(agg(default_quantile(0.5))), }; let node = alloc(unlimited(), expr); @@ -1036,21 +1014,10 @@ mod tests { assert_eq!(node.children[0].mode, ExecutionMode::Sketch); } - // ── LetBinding inherits body stage ──────────────────────────────────────── - - #[test] - fn let_binding_inherits_body_stage() { - let expr = QueryExpr::LetBinding { - name: asap_ir::intent_algebra::BindingName::new("base"), - expr: Box::new(scan("cpu")), - child: Box::new(agg(AggIntent::TopK { - k: 5, - accuracy: AccuracyTarget::Epsilon(0.05), - })), - }; - let node = alloc(unlimited(), expr); - assert_eq!(node.stage, PipelineStage::Precompute); - } + // `LetBinding` doesn't exist in the canonical `QueryExpr` anymore + // (see `optimizer::engine::CommonSubexprElim`'s doc) -- the + // `let_binding_inherits_body_stage` test that used to exercise it + // is gone with it. // ── Memory estimate helpers ─────────────────────────────────────────────── diff --git a/control_plane/src/physical/colored_dag/allocator.rs b/control_plane/src/physical/colored_dag/allocator.rs index 3eb6c1db..872308ba 100644 --- a/control_plane/src/physical/colored_dag/allocator.rs +++ b/control_plane/src/physical/colored_dag/allocator.rs @@ -39,7 +39,7 @@ use std::collections::HashMap; use std::rc::Rc; -use asap_sketch::{L4Node, SummaryExpr}; +use planner_types::post_asap::{SummaryExpr, SummaryNode}; use crate::physical::colored_dag::dag::{ColoredDag, ColoredNode, NodeId}; use crate::physical::colored_dag::stage_id::{StageId, Topology}; @@ -126,7 +126,7 @@ impl ThreeStageWalker { /// Recursively visit an [`L4Plan`] — the "what to compute" layer. /// [`L4Plan::Summary`] delegates the actual per-node granularity to - /// [`Self::visit_l4node`] (walking `asap_sketch::L4Node`'s own DAG + /// [`Self::visit_l4node`] (walking `planner_types::post_asap::SummaryNode`'s own DAG /// shape); [`L4Plan::LetBinding`] / [`L4Plan::Ref`] are this crate's /// own named-binding sharing mechanism, unchanged from before Step B. fn visit_plan(&mut self, plan: &L4Plan) -> Result<(NodeId, StageId), AllocateError> { @@ -159,7 +159,7 @@ impl ThreeStageWalker { } } - /// Recursively visit one `asap_sketch::L4Node` — the sketch algebra + /// Recursively visit one `planner_types::post_asap::SummaryNode` — the sketch algebra /// itself, owned upstream. Every semantic node gets its own /// [`ColoredNode`] (matching the granularity the old, locally-defined /// `PhysicalExpr::{SketchAgg,SketchEstimate,SketchMerge}` had), @@ -167,7 +167,7 @@ impl ThreeStageWalker { /// wrapping just that sub-node, so downstream consumers /// (`colored_dag::emitter`, `emit::mod`) keep pattern-matching /// against the same `PhysicalExpr` shape. - fn visit_l4node(&mut self, node: &Rc) -> Result<(NodeId, StageId), AllocateError> { + fn visit_l4node(&mut self, node: &Rc) -> Result<(NodeId, StageId), AllocateError> { let id = self.reserve_node(PhysicalExpr::committed(Rc::clone(node))); let stage = match &node.expr { @@ -279,7 +279,7 @@ impl ThreeStageWalker { use crate::intent_algebra::QueryExpr as QE; match qe { QE::Scan { .. } => Ok(StageId::Edge), - QE::Window { .. } => Ok(StageId::Edge), + QE::TimeRange { .. } => Ok(StageId::Edge), // Aggregate at L3-in-L4: the design.md L5 table says // `Aggregate{exact}` (e.g. `Max`) → Edge, and the *root of // q3* (the same Aggregate after a SketchMerge / Merge) → @@ -291,18 +291,8 @@ impl ThreeStageWalker { // sibling structure) — Phase G+ adds an explicit // `Logical(Merge)` PhysicalExpr variant for the gateway hop. QE::Aggregate { .. } => Ok(StageId::Edge), - // Lexical scope for L3 LetBinding / Ref — mirrors the - // PhysicalExpr-level handling. - QE::LetBinding { name, expr, child } => { - let expr_stage = self.colour_logical(expr)?; - self.scope.insert(name.as_str().to_string(), expr_stage); - self.colour_logical(child) - } - QE::Ref { name } => self - .scope - .get(name.as_str()) - .copied() - .ok_or_else(|| AllocateError::UnresolvedRef(name.as_str().to_string())), + // `LetBinding`/`Ref` don't exist in the canonical `QueryExpr` + // anymore -- see `optimizer::engine::CommonSubexprElim`'s doc. // A-variants lifted in Batch 2 of the relational migration. // No colored-DAG consumer constructs them today; conservatively // route to the Edge stage (matches the per-row Scan/Window @@ -330,9 +320,7 @@ impl ThreeStageWalker { // stage lookup keyed by binding name. pub(crate) fn binding_stage(dag: &ColoredDag, name: &BindingName) -> Option { dag.nodes.iter().find_map(|n| match &n.expr { - PhysicalExpr::Committed(L4Plan::LetBinding { name: n2, .. }) if n2 == name => { - Some(n.stage) - } + PhysicalExpr::Committed(L4Plan::LetBinding { name: n2, .. }) if n2 == name => Some(n.stage), _ => None, }) } @@ -343,7 +331,7 @@ pub(crate) fn binding_stage(dag: &ColoredDag, name: &BindingName) -> Option QueryExpr { @@ -374,19 +362,15 @@ mod tests { } fn windowed_scan() -> QueryExpr { - QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, + QueryExpr::TimeRange { + range: Duration::from_secs(300), child: Box::new(ts_scan()), } } #[test] fn allocate_unsupported_topology_errors() { - let leaf = PhysicalExpr::committed( - asap_plan::bind::logical(&ts_scan(), &BindingScope::default()).unwrap(), - ); + let leaf = PhysicalExpr::committed(asap_aware_mapping::bind::logical(&ts_scan()).unwrap()); let err = StageAllocator .allocate(&leaf, Topology::SingleStage) .unwrap_err(); @@ -400,7 +384,7 @@ mod tests { fn three_stage_quantile_dag_basic() { let q = QueryExpr::Aggregate { reduction: crate::intent_algebra::Reduction::PerEntity, - aggs: vec![crate::intent_algebra::AggIntent::Quantile { + measures: vec![crate::intent_algebra::AggIntent::Quantile { col: None, q: 0.99, accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), @@ -409,7 +393,7 @@ mod tests { having: None, child: Box::new(windowed_scan()), }; - let node = asap_plan::bind::implement_tree(&q).unwrap(); + let node = asap_aware_mapping::bind::implement_tree(&q).unwrap(); let expr = PhysicalExpr::committed(node); let dag = StageAllocator .allocate(&expr, Topology::ThreeStage) diff --git a/control_plane/src/physical/colored_dag/dag.rs b/control_plane/src/physical/colored_dag/dag.rs index ae268f9c..192afc45 100644 --- a/control_plane/src/physical/colored_dag/dag.rs +++ b/control_plane/src/physical/colored_dag/dag.rs @@ -145,7 +145,7 @@ impl ColoredDag { mod tests { use super::*; use crate::intent_algebra::schema::{Column, DataType}; - use crate::intent_algebra::{BindingScope, QueryExpr, Schema, Source}; + use crate::intent_algebra::{QueryExpr, Schema, Source}; use crate::sketch_algebra::PhysicalExpr; // These three dummies only need to be *structurally valid* and @@ -179,26 +179,24 @@ mod tests { } fn dummy_logical() -> PhysicalExpr { - PhysicalExpr::committed( - asap_plan::bind::logical(&dummy_scan(), &BindingScope::default()).unwrap(), - ) + PhysicalExpr::committed(asap_aware_mapping::bind::logical(&dummy_scan()).unwrap()) } fn dummy_agg() -> PhysicalExpr { let q = QueryExpr::Aggregate { reduction: crate::intent_algebra::Reduction::by(vec![]), - aggs: vec![crate::intent_algebra::AggIntent::Sum { col: None }], + measures: vec![crate::intent_algebra::AggIntent::Sum { col: None }], output_names: Vec::new(), having: None, child: Box::new(dummy_scan()), }; - PhysicalExpr::committed(asap_plan::bind::implement_tree(&q).unwrap()) + PhysicalExpr::committed(asap_aware_mapping::bind::implement_tree(&q).unwrap()) } fn dummy_estimate() -> PhysicalExpr { let q = QueryExpr::Aggregate { reduction: crate::intent_algebra::Reduction::by(vec![]), - aggs: vec![crate::intent_algebra::AggIntent::Quantile { + measures: vec![crate::intent_algebra::AggIntent::Quantile { col: None, q: 0.99, accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), @@ -207,7 +205,7 @@ mod tests { having: None, child: Box::new(dummy_scan()), }; - PhysicalExpr::committed(asap_plan::bind::implement_tree(&q).unwrap()) + PhysicalExpr::committed(asap_aware_mapping::bind::implement_tree(&q).unwrap()) } #[test] diff --git a/control_plane/src/physical/colored_dag/emitter.rs b/control_plane/src/physical/colored_dag/emitter.rs index 60eb02d8..65d8a9d9 100644 --- a/control_plane/src/physical/colored_dag/emitter.rs +++ b/control_plane/src/physical/colored_dag/emitter.rs @@ -35,13 +35,25 @@ use crate::physical::colored_dag::dag::ColoredDag; use crate::physical::colored_dag::stage_id::{StageId, Topology}; use crate::sketch_algebra::physical_expr::{L4Plan, PhysicalExpr}; use crate::types_v2::BindingName; -use asap_sketch::{SketchQuery, SummaryExpr, SummaryKind, SummaryParams}; +use planner_types::post_asap::{SketchKind, SketchParams, SketchQuery, SummaryExpr}; +// `BackendAggregation.sketch_kind`/`.sketch_params` (below) span BOTH +// exact accumulators (Sum/Count/MinMax/Increase/Rate, via +// `agg_type_override`) and approximate sketches -- unlike +// `EdgeSketchProcessor`/`GatewayMergeProcessor`, which are always real +// materialized sketches. They use the vendored flat +// `asap_types::{SummaryKind, SummaryParams}` (same 14-variant shape as +// ASAPController's pre-ASAPPlanner#218 flat `SummaryKind` -- see +// `crates/asap_types/src/accumulator_spec.rs`'s module doc and +// control_plane/docs/design-asapplanner-pin-migration.md), aliased here +// so this file's existing `BackendAggregation`-adjacent code needs no +// further changes. +use asap_types::{SummaryKind as BackendSketchKind, SummaryParams as BackendSketchParams}; /// Flattened view of one [`ColoredNode`](crate::physical::colored_dag::dag::ColoredNode)'s /// `PhysicalExpr`, for the tuple-style `(&node.expr, node.stage)` matching /// this file uses throughout. Mirrors the shape the old, locally-defined /// flat `PhysicalExpr` had before Step B folded most of its variants into -/// `asap_sketch::SummaryExpr` — see `sketch_algebra::physical_expr`'s +/// `planner_types::post_asap::SummaryExpr` — see `sketch_algebra::physical_expr`'s /// module docs. enum NodeKind<'a> { Logical(&'a crate::intent_algebra::QueryExpr), @@ -49,21 +61,27 @@ enum NodeKind<'a> { /// (Sum/Count/MinMax/Increase/Rate) are classified as [`Self::ExactAgg`] /// instead, matching the old `PhysicalExpr::ExactAgg`'s separate shape. SketchAgg { - sketch_type: &'a SummaryKind, - params: &'a SummaryParams, + sketch_type: &'a SketchKind, + params: &'a SketchParams, }, /// An exact accumulator — the old `PhysicalExpr::ExactAgg`. This /// emitter has never had a match arm for it (falls through to the /// catch-all below, same as before Step B — a pre-existing gap, not /// introduced by this migration). ExactAgg, - SketchEstimate { query: &'a SketchQuery }, + SketchEstimate { + query: &'a SketchQuery, + }, SketchMerge, - LetBinding { name: &'a BindingName }, - Ref { name: &'a BindingName }, + LetBinding { + name: &'a BindingName, + }, + Ref { + name: &'a BindingName, + }, RawAtEdgeSketchAtBackend { - family: &'a SummaryKind, - params: &'a SummaryParams, + family: &'a SketchKind, + params: &'a SketchParams, }, RawAtEdgePrometheusArchive { metric: &'a str, @@ -75,31 +93,33 @@ enum NodeKind<'a> { Other, } -/// Is `kind` an exact accumulator (Sum/Count/MinMax/Increase/Rate) rather -/// than an approximate sketch? -fn is_exact_accumulator(kind: &SummaryKind) -> bool { - matches!( - kind, - SummaryKind::Sum - | SummaryKind::Count - | SummaryKind::MinMax - | SummaryKind::Increase - | SummaryKind::Rate - ) -} - fn classify(expr: &PhysicalExpr) -> NodeKind<'_> { match expr { PhysicalExpr::Committed(L4Plan::Summary(node)) => match &node.expr { SummaryExpr::Logical(qe) => NodeKind::Logical(qe), - SummaryExpr::SummaryAgg { summary, params, .. } if is_exact_accumulator(summary) => { - let _ = params; - NodeKind::ExactAgg - } - SummaryExpr::SummaryAgg { summary, params, .. } => NodeKind::SketchAgg { - sketch_type: summary, + // `SummaryAgg`'s `kind`/`params` fields collapsed into one + // `family: SummaryFamilyType` field (ASAPPlanner#218 -- + // see control_plane/docs/design-asapplanner-pin-migration.md); + // the exact-vs-sketch classification this used to need + // `is_exact_accumulator` for is now which enum variant + // `family` is, not a value-level check on `kind`. + SummaryExpr::SummaryAgg { + family: planner_types::post_asap::SummaryFamilyType::ExactAggregate(..), + .. + } => NodeKind::ExactAgg, + SummaryExpr::SummaryAgg { + family: planner_types::post_asap::SummaryFamilyType::Sketch(kind, params), + .. + } => NodeKind::SketchAgg { + sketch_type: kind, params, }, + // `Plain`/`Sample`/`Wavelet`/`StatModel` never occur on a real + // `SummaryAgg` (never `Plain` by construction; `Sample`/ + // `Wavelet`/`StatModel` are unreachable via this deployment's + // own `CostModel` -- see `sketch_algebra::capability`'s + // `implementation_to_capability` doc for the same reasoning). + SummaryExpr::SummaryAgg { .. } => NodeKind::Other, SummaryExpr::SummaryEstimate { query, .. } => NodeKind::SketchEstimate { query }, SummaryExpr::SummaryMerge { .. } => NodeKind::SketchMerge, SummaryExpr::SummaryJoin { .. } @@ -132,10 +152,10 @@ pub enum EmitError { #[error("unsupported topology for this emitter: {0:?} (expected {1:?})")] UnsupportedTopology(Topology, Topology), /// A sketch processor name could not be derived for the supplied - /// `SummaryKind`. Should not occur with the catalog ranges shipped + /// `SketchKind`. Should not occur with the catalog ranges shipped /// in Phase C — kept as a defensive error for future kinds. #[error("no edge processor known for sketch kind {0:?}")] - NoEdgeProcessor(SummaryKind), + NoEdgeProcessor(SketchKind), /// Backend would emit an empty StreamingConfig because no sketch /// state ever reaches it (e.g. a colouring with only `Logical` /// nodes). Surfaced as a clean error so callers can fall back to @@ -267,7 +287,7 @@ pub struct EdgeStageConfig { /// because different planned queries on the same metric require /// different capabilities (e.g. `quantile_over_time` → DDSketch, /// `count`-distinct → HLL, `topk` → CountSketch all on one metric). - /// The value type is therefore a `BTreeSet` (the UNION + /// The value type is therefore a `BTreeSet` (the UNION /// of capabilities across all of that metric's workload entries), /// not a single family. A metric in two families produces two /// routing-connector OTTL conditions → its samples fan into both @@ -280,11 +300,11 @@ pub struct EdgeStageConfig { /// This eliminates the prior multi-family fan-out (every metric /// shipped sketch state through all 5 families regardless of need). /// - /// `SketchFamily` is a control-plane-side alias for `asap_sketch::SummaryKind`. + /// `SketchFamily` is a control-plane-side alias for `planner_types::post_asap::SketchKind`. /// Empty map ⇒ legacy single-pipeline / /// Mode-3 / warm-passthrough wire shapes are emitted unchanged /// (backward-compat). - pub metric_to_family: HashMap>, + pub metric_to_family: HashMap>, /// MVP blocker B3 — per-metric attribute allowlist the agent must /// reduce wire attrs to BEFORE the sketch processor sees them. /// Maps each metric to its grouping-label list; the 5-sketch routing @@ -551,18 +571,18 @@ pub struct PrometheusArchiveMetric { /// One sketch processor configured at an edge agent. /// /// Not `Serialize`/`Deserialize` (see `PhysicalExpr`'s doc for why — -/// `SummaryKind`/`SummaryParams` have no serde impl, and nothing on the +/// `SketchKind`/`SketchParams` have no serde impl, and nothing on the /// real emit path ever whole-struct-serialized this type; every actual /// YAML/JSON payload goes through a hand-written builder). #[derive(Debug, Clone, PartialEq)] pub struct EdgeSketchProcessor { /// OTel processor component id — `KLL`, `ddsketch`, `HLL`, - /// `countmin`, etc. Maps 1:1 from `SummaryKind`. + /// `countmin`, etc. Maps 1:1 from `SketchKind`. pub processor_name: String, /// Sketch family (mirror of the `SketchAgg::sketch_type` field). - pub sketch_kind: SummaryKind, + pub sketch_kind: SketchKind, /// Sketch parameters (mirror of the `SketchAgg::params` field). - pub sketch_params: SummaryParams, + pub sketch_params: SketchParams, /// Internal emitter plumbing — threads `EdgeSketchProcessor` → /// `GatewayMergeProcessor` (which DOES surface it on the wire to /// route merged streams) during the DAG walk. Phase E derives a @@ -603,7 +623,7 @@ pub struct GatewayMergeProcessor { pub processor_name: String, /// Sketch family being merged. All inputs to the merge agree on /// this (L4 type checker enforces it; design.md §6.4). - pub sketch_kind: SummaryKind, + pub sketch_kind: SketchKind, /// Aggregation id — matches the upstream edge's /// `EdgeSketchProcessor::aggregation_id` so the gateway routes /// streams correctly. @@ -620,7 +640,7 @@ pub struct GatewayMergeProcessor { /// Logical content of the backend `StreamingConfig`. /// /// Not `PartialEq` — `readouts: Vec` carries -/// `asap_sketch::SketchQuery`, which (like `SummaryExpr`/`L4Node`) has no +/// `planner_types::post_asap::SketchQuery`, which (like `SummaryExpr`/`SummaryNode`) has no /// `PartialEq` impl upstream. Nothing on the real emit path compares /// whole `BackendStageConfig`/`BackendReadout` values — every actual /// wire payload goes through `build_backend_readout_json`'s hand-written @@ -646,7 +666,7 @@ pub struct BackendStageConfig { /// `sketch_kind`, `sketch_params`, grouping labels, and `spatial_filter`). /// /// Not `Serialize`/`Deserialize` — same reason as `EdgeSketchProcessor`: -/// `SummaryKind`/`SummaryParams` have no serde impl, and the real wire +/// `SketchKind`/`SketchParams` have no serde impl, and the real wire /// payload is built by `emit::stage_config::build_backend_aggregation_json` /// (a hand-written JSON builder reading these fields), never a whole-struct /// serialize. @@ -658,12 +678,14 @@ pub struct BackendAggregation { /// `http_requests_total_latency_ms`). Required by the backend's /// `AggregationConfig` parser. pub metric_name: String, - /// Sketch family. - pub sketch_kind: SummaryKind, - /// Sketch parameters — the backend uses these to build its - /// per-aggregation `Sketch` instance (KLL with the right `k`, - /// DDSketch with the right `alpha`, etc.). - pub sketch_params: SummaryParams, + /// Accumulator family -- exact (Sum/Count/MinMax/Increase/Rate) or + /// approximate sketch (see this file's `use asap_types::{...}` note + /// above for why this is the flat type, not `SketchKind`). + pub sketch_kind: BackendSketchKind, + /// Accumulator parameters — the backend uses these to build its + /// per-aggregation instance (KLL with the right `k`, DDSketch with + /// the right `alpha`, etc.). + pub sketch_params: BackendSketchParams, /// Tumbling window size in seconds. Required by the backend; the /// parser rejects zero-window aggregations. pub window_secs: u64, @@ -910,8 +932,8 @@ impl Emitter for ThreeStageEmitter { item_label: None, aggregation_id, metric_name: edge.source_metric.clone().unwrap_or_default(), - sketch_kind: sketch_type.clone(), - sketch_params: params.clone(), + sketch_kind: sketch_type.clone().into(), + sketch_params: params.clone().into(), window_secs: edge.window_secs.unwrap_or(0), spatial_filter: spatial_filter_from_label_filters(&edge.label_filters), // Populated post-emit by the caller (handle_plan) @@ -994,8 +1016,8 @@ impl Emitter for ThreeStageEmitter { item_label: None, aggregation_id: aid, metric_name: edge.source_metric.clone().unwrap_or_default(), - sketch_kind: family.clone(), - sketch_params: params.clone(), + sketch_kind: family.clone().into(), + sketch_params: params.clone().into(), window_secs: edge.window_secs.unwrap_or(0), spatial_filter: spatial_filter_from_label_filters(&edge.label_filters), grouping: Vec::new(), @@ -1041,7 +1063,7 @@ impl Emitter for ThreeStageEmitter { // ── Helpers ─────────────────────────────────────────────────────────────────── -/// Map a `SummaryKind` to the OTel collector processor name. Mirrors the +/// Map a `SketchKind` to the OTel collector processor name. Mirrors the /// names the existing OpAMP YAML emitter (and the per-sketch processor /// crates in `opentelemetry-collector-contrib`) already use. /// @@ -1054,20 +1076,18 @@ impl Emitter for ThreeStageEmitter { /// have no OTel edge processor — nothing in this repo's binding rules /// constructs a `SketchAgg`/`RawAtEdgeSketchAtBackend` with one of these /// kinds today, but the match must stay exhaustive. -pub(crate) fn edge_processor_name(kind: &SummaryKind) -> Result { +pub(crate) fn edge_processor_name(kind: &SketchKind) -> Result { match kind { - SummaryKind::Kll => Ok("KLL".into()), - SummaryKind::DDSketch => Ok("ddsketch".into()), - SummaryKind::Hll => Ok("HLL".into()), - SummaryKind::Cms | SummaryKind::CmsWithHeap => Ok("countmin".into()), - SummaryKind::CountSketch | SummaryKind::CountSketchWithHeap => Ok("countsketch".into()), - SummaryKind::Sum - | SummaryKind::Count - | SummaryKind::MinMax - | SummaryKind::Increase - | SummaryKind::Rate - | SummaryKind::Kmv - | SummaryKind::Theta => Err(EmitError::NoEdgeProcessor(kind.clone())), + SketchKind::Kll => Ok("KLL".into()), + SketchKind::DDSketch => Ok("ddsketch".into()), + SketchKind::Hll => Ok("HLL".into()), + SketchKind::Cms | SketchKind::CmsWithHeap => Ok("countmin".into()), + SketchKind::CountSketch | SketchKind::CountSketchWithHeap => Ok("countsketch".into()), + // Exact-accumulator kinds (Sum/Count/MinMax/Increase/Rate) aren't + // representable here anymore -- they're `ExactKind`, a distinct + // type post ASAPPlanner#218's split, not a `SketchKind` variant + // this function could even be called with. + SketchKind::Kmv | SketchKind::Theta => Err(EmitError::NoEdgeProcessor(kind.clone())), } } @@ -1099,7 +1119,7 @@ fn extract_edge_facts(qe: &crate::intent_algebra::QueryExpr, edge: &mut EdgeStag left, op: CompareOp::Eq, right, - } = &p.0 + } = p.0.as_ref() { if let (L3Expr::Column(id), L3Expr::Literal(L3Scalar::Utf8(v))) = (left.as_ref(), right.as_ref()) @@ -1114,20 +1134,15 @@ fn extract_edge_facts(qe: &crate::intent_algebra::QueryExpr, edge: &mut EdgeStag } } } - QE::Window { size, child, .. } => { + QE::TimeRange { range, child } => { if edge.window_secs.is_none() { - edge.window_secs = Some(size.as_secs()); + edge.window_secs = Some(range.as_secs()); } extract_edge_facts(child, edge); } QE::Aggregate { child, .. } => { extract_edge_facts(child, edge); } - QE::LetBinding { expr, child, .. } => { - extract_edge_facts(expr, edge); - extract_edge_facts(child, edge); - } - QE::Ref { .. } => {} // A-variants lifted in Batch 2 of the relational migration. No // canonical-side consumer constructs them yet — recurse into // children so we still surface edge facts (source metric, label @@ -1193,7 +1208,7 @@ fn first_sketch_child_via_edges( dag: &ColoredDag, parent: crate::physical::colored_dag::dag::NodeId, sketch_agg_ids: &HashMap, -) -> Option<(SummaryKind, String)> { +) -> Option<(SketchKind, String)> { for cid in children_of(dag, parent) { let cnode = dag.nodes.get(cid.0)?; match classify(&cnode.expr) { @@ -1209,14 +1224,14 @@ fn first_sketch_child_via_edges( } NodeKind::Ref { name } => { // Resolve the ref to its binding's expr id, then recurse. - if let Some(bid) = dag - .nodes - .iter() - .enumerate() - .find_map(|(i, n)| match classify(&n.expr) { - NodeKind::LetBinding { name: n2 } if n2 == name => Some(i), - _ => None, - }) + if let Some(bid) = + dag.nodes + .iter() + .enumerate() + .find_map(|(i, n)| match classify(&n.expr) { + NodeKind::LetBinding { name: n2 } if n2 == name => Some(i), + _ => None, + }) { let bnode_id = crate::physical::colored_dag::dag::NodeId(bid); if let Some(found) = first_sketch_child_via_edges(dag, bnode_id, sketch_agg_ids) diff --git a/control_plane/src/physical/colored_dag/tests.rs b/control_plane/src/physical/colored_dag/tests.rs index a73b0c69..cb46b5e8 100644 --- a/control_plane/src/physical/colored_dag/tests.rs +++ b/control_plane/src/physical/colored_dag/tests.rs @@ -10,12 +10,13 @@ use std::rc::Rc; use std::time::Duration; -use asap_sketch::{L4Node, L4Schema, SketchQuery, SummaryExpr, SummaryKind, SummaryParams}; - -use crate::intent_algebra::{ - BindingScope, ColumnRef, LabelFilter, QueryExpr, Reduction, Schema, Source, WindowKind, +use planner_types::post_asap::{ + SketchKind, SketchParams, SketchQuery, SummaryExpr, SummaryFamilyType, SummaryNode, + SummarySchema, }; + use crate::intent_algebra::schema::{Column, DataType}; +use crate::intent_algebra::{ColumnRef, LabelFilter, QueryExpr, Reduction, Schema, Source}; use crate::physical::colored_dag::allocator::StageAllocator; use crate::physical::colored_dag::emitter::{EmitError, Emitter, StageConfig, ThreeStageEmitter}; use crate::physical::colored_dag::stage_id::{StageId, Topology}; @@ -71,10 +72,8 @@ fn ts_scan(metric: &str, label: Option<(&str, &str)>) -> QueryExpr { } fn windowed_scan() -> QueryExpr { - QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, + QueryExpr::TimeRange { + range: Duration::from_secs(300), child: Box::new(ts_scan( "http_request_duration_seconds", Some(("service", "api")), @@ -82,13 +81,13 @@ fn windowed_scan() -> QueryExpr { } } -/// Empty `L4Schema` — the coloring/emitter logic under test here never +/// Empty `SummarySchema` — the coloring/emitter logic under test here never /// inspects node schemas (only `SummaryExpr` shape + `PhysicalExpr` /// placement), so hand-built L4 nodes below carry a placeholder, same /// spirit as this file's old comment: "these dummies only need to be /// *structurally valid* and distinct `PhysicalExpr` values". -fn dummy_l4_schema() -> L4Schema { - L4Schema { +fn dummy_l4_schema() -> SummarySchema { + SummarySchema { fields: vec![], time_index: None, } @@ -96,20 +95,23 @@ fn dummy_l4_schema() -> L4Schema { /// Wrap `qe` as an unbound `Logical` L4 leaf — mirrors the old /// `PhysicalExpr::Logical(qe)` construction for hand-built fixtures. -fn logical_l4(qe: QueryExpr) -> Rc { - asap_plan::bind::logical(&qe, &BindingScope::default()).unwrap() +fn logical_l4(qe: QueryExpr) -> Rc { + asap_aware_mapping::bind::logical(&qe).unwrap() } /// Hand-build a `SummaryAgg` node — mirrors the old /// `PhysicalExpr::SketchAgg { sketch_type, params, child }` construction, /// for fixtures that need a specific family without going through /// `implement_tree`'s cost-model selection. -fn sketch_agg_l4(summary: SummaryKind, params: SummaryParams, child: Rc) -> Rc { - Rc::new(L4Node { +fn sketch_agg_l4( + kind: SketchKind, + params: SketchParams, + child: Rc, +) -> Rc { + Rc::new(SummaryNode { expr: SummaryExpr::SummaryAgg { child, - summary, - params, + family: SummaryFamilyType::Sketch(kind, params), col: ColumnRef::SampleValue, reduction: Reduction::by(vec![]), }, @@ -119,9 +121,12 @@ fn sketch_agg_l4(summary: SummaryKind, params: SummaryParams, child: Rc) /// Hand-build a `SummaryEstimate` node — mirrors the old /// `PhysicalExpr::SketchEstimate { op, child }`. -fn estimate_l4(query: SketchQuery, summary_input: Rc) -> Rc { - Rc::new(L4Node { - expr: SummaryExpr::SummaryEstimate { summary_input, query }, +fn estimate_l4(query: SketchQuery, summary_input: Rc) -> Rc { + Rc::new(SummaryNode { + expr: SummaryExpr::SummaryEstimate { + summary_input, + query, + }, schema: dummy_l4_schema(), }) } @@ -131,8 +136,8 @@ fn estimate_l4(query: SketchQuery, summary_input: Rc) -> Rc { /// upstream replacement) carries no `algebra` field — `MergeAlgebra` was /// this crate's own addition and doesn't exist upstream (see /// `physical_expr.rs`'s module docs). -fn merge_l4(children: Vec>) -> Rc { - Rc::new(L4Node { +fn merge_l4(children: Vec>) -> Rc { + Rc::new(SummaryNode { expr: SummaryExpr::SummaryMerge { children }, schema: dummy_l4_schema(), }) @@ -165,7 +170,7 @@ fn is_ref(expr: &PhysicalExpr) -> bool { fn quantile_kll_dag() -> PhysicalExpr { let q = QueryExpr::Aggregate { reduction: Reduction::by(vec![]), - aggs: vec![crate::intent_algebra::AggIntent::Quantile { + measures: vec![crate::intent_algebra::AggIntent::Quantile { col: None, q: 0.99, accuracy: AccuracyTarget::Epsilon(0.01), @@ -174,7 +179,7 @@ fn quantile_kll_dag() -> PhysicalExpr { having: None, child: Box::new(windowed_scan()), }; - PhysicalExpr::committed(asap_plan::bind::implement_tree(&q).unwrap()) + PhysicalExpr::committed(asap_aware_mapping::bind::implement_tree(&q).unwrap()) } // ── Allocator: per-rule + edge-case tests ───────────────────────────────────── @@ -216,8 +221,8 @@ fn allocator_sketch_agg_under_scan_pinned_edge() { // Exact design.md §6 invariant: a SummaryAgg whose child is a Scan // (wrapped in Logical) MUST land on Edge. let expr = PhysicalExpr::committed(sketch_agg_l4( - SummaryKind::Hll, - SummaryParams::Hll { precision: 14 }, + SketchKind::Hll, + SketchParams::Hll { precision: 14 }, logical_l4(ts_scan("events", None)), )); let dag = StageAllocator @@ -248,18 +253,18 @@ fn allocator_let_binding_color_propagates() { // // NOTE — shape change forced by the type system, not just syntax: // the old fixture nested `Ref` *inside* a `SketchEstimate`'s child. - // The new `SummaryEstimate::sketch_input` field is `Rc` — + // The new `SummaryEstimate::sketch_input` field is `Rc` — // upstream `asap_sketch`'s own type, which has no `Ref`/`LetBinding` // concept at all — so a `Ref`/`LetBinding` can only appear where an // `L4Plan` is expected (this crate's own named-binding sharing - // mechanism layered *above* `L4Node`, not inside it; see + // mechanism layered *above* `SummaryNode`, not inside it; see // `physical_expr.rs`'s module docs). The property under test — // LetBinding colors by its bound expression's stage — is preserved // with the `child` position held by a bare `Ref` instead of a // `SketchEstimate{child: Ref}`. let inner_agg = sketch_agg_l4( - SummaryKind::Kll, - SummaryParams::Kll { k: 200 }, + SketchKind::Kll, + SketchParams::Kll { k: 200 }, logical_l4(windowed_scan()), ); let bind = PhysicalExpr::Committed(L4Plan::LetBinding { @@ -288,8 +293,8 @@ fn allocator_ref_resolves_to_binding_stage() { // note there); the Ref child of the LetBinding must color Edge (the // binding's stage). let inner_agg = sketch_agg_l4( - SummaryKind::Kll, - SummaryParams::Kll { k: 200 }, + SketchKind::Kll, + SketchParams::Kll { k: 200 }, logical_l4(windowed_scan()), ); let bind = PhysicalExpr::Committed(L4Plan::LetBinding { @@ -315,8 +320,8 @@ fn allocator_sketch_merge_lands_gateway() { // SummaryMerge over edge-built KLL sketches → Gateway. let one_agg = || { sketch_agg_l4( - SummaryKind::Kll, - SummaryParams::Kll { k: 200 }, + SketchKind::Kll, + SketchParams::Kll { k: 200 }, logical_l4(windowed_scan()), ) }; @@ -345,8 +350,8 @@ fn emitter_three_stage_emits_three_configs() { // SummaryMerge over two SummaryAggs. let one_agg = || { sketch_agg_l4( - SummaryKind::Kll, - SummaryParams::Kll { k: 200 }, + SketchKind::Kll, + SketchParams::Kll { k: 200 }, logical_l4(windowed_scan()), ) }; @@ -372,7 +377,7 @@ fn emitter_edge_config_has_correct_processor_kll() { StageConfig::Edge(e) => { assert_eq!(e.sketch_processors.len(), 1); assert_eq!(e.sketch_processors[0].processor_name, "KLL"); - assert_eq!(e.sketch_processors[0].sketch_kind, SummaryKind::Kll); + assert_eq!(e.sketch_processors[0].sketch_kind, SketchKind::Kll); assert_eq!( e.source_metric.as_deref(), Some("http_request_duration_seconds") @@ -388,8 +393,8 @@ fn emitter_edge_config_has_correct_processor_ddsketch() { let expr = PhysicalExpr::committed(estimate_l4( SketchQuery::Quantile { q: 0.99 }, sketch_agg_l4( - SummaryKind::DDSketch, - SummaryParams::DDSketch { alpha: 0.01 }, + SketchKind::DDSketch, + SketchParams::DDSketch { alpha: 0.01 }, logical_l4(windowed_scan()), ), )); @@ -419,7 +424,7 @@ fn emitter_backend_config_routes_aggregation_id() { StageConfig::Backend(b) => { assert_eq!(b.aggregations.len(), 1); assert_eq!(b.aggregations[0].aggregation_id, edge_aid); - assert_eq!(b.aggregations[0].sketch_kind, SummaryKind::Kll); + assert_eq!(b.aggregations[0].sketch_kind, SketchKind::Kll.into()); assert_eq!(b.readouts.len(), 1); assert_eq!(b.readouts[0].aggregation_id, edge_aid); // `SketchQuery` has no `PartialEq` upstream — destructure @@ -470,8 +475,8 @@ fn end_to_end_quantile_workload() { // root). let agg = || { sketch_agg_l4( - SummaryKind::Kll, - SummaryParams::Kll { k: 200 }, + SketchKind::Kll, + SketchParams::Kll { k: 200 }, logical_l4(windowed_scan()), ) }; @@ -502,7 +507,7 @@ fn end_to_end_quantile_workload() { StageConfig::Gateway(g) => { assert!(!g.merge_processors.is_empty()); assert_eq!(g.merge_processors[0].processor_name, "sketchmergeprocessor"); - assert_eq!(g.merge_processors[0].sketch_kind, SummaryKind::Kll); + assert_eq!(g.merge_processors[0].sketch_kind, SketchKind::Kll); } _ => unreachable!(), } diff --git a/control_plane/src/physical/planner.rs b/control_plane/src/physical/planner.rs index bff56df6..f0e8cd90 100644 --- a/control_plane/src/physical/planner.rs +++ b/control_plane/src/physical/planner.rs @@ -229,13 +229,6 @@ fn plan_node(expr: &QueryExpr, config: &PhysicalPlannerConfig) -> PhysicalNode { children: vec![], }, - QueryExpr::Ref { .. } => PhysicalNode { - op: PhysicalOp::Passthrough, - placement: Placement::QueryEngine, - cost: PhysicalCost::default(), - children: vec![], - }, - // ── Filter: same placement as child ───────────────────────── QueryExpr::Filter { pred, child } => { let child = plan_node(child, config); @@ -249,12 +242,13 @@ fn plan_node(expr: &QueryExpr, config: &PhysicalPlannerConfig) -> PhysicalNode { } } - // ── Window: a Window over a single-intent Aggregate is the - // canonical fold of the legacy WindowedAgg — recognize it and - // reconstruct the fused OtelSketchBuild { window } placement. - // Any other Window is a plain passthrough inheriting the child's + // ── TimeRange: a TimeRange over a single-intent Aggregate is the + // canonical fold of the legacy WindowedAgg (was `Window` before + // the ASAPPlanner pin migration) — recognize it and reconstruct + // the fused OtelSketchBuild { window } placement. Any other + // TimeRange is a plain passthrough inheriting the child's // placement. - QueryExpr::Window { + QueryExpr::TimeRange { child: window_child, .. } => { @@ -291,7 +285,7 @@ fn plan_node(expr: &QueryExpr, config: &PhysicalPlannerConfig) -> PhysicalNode { // * multi-intent or HAVING → exact HashAggregate at QueryEngine QueryExpr::Aggregate { reduction, - aggs, + measures: aggs, having, child, .. @@ -440,15 +434,13 @@ fn plan_node(expr: &QueryExpr, config: &PhysicalPlannerConfig) -> PhysicalNode { } } - // ── LetBinding ────────────────────────────────────────────── - QueryExpr::LetBinding { child, .. } => plan_node(child, config), - // The PromQL-surface superset (Scalar/EvalTime/VectorFromScalar/ - // ScalarFromVector/Relabel/InfoJoin/Sample/TimeRange/TimeShift/ - // WindowFunc) isn't constructed by this parser today. `Scalar` / - // `EvalTime` are leaves; every other new variant wraps exactly - // one child — inherit its placement, mirroring the Sort/Limit/ - // Project arm above, until a dedicated physical op is written. + // ScalarFromVector/Relabel/InfoJoin/Sample/TimeShift/WindowFunc) + // isn't constructed by this parser today (`TimeRange` has its own + // arm above -- it *is* constructed). `Scalar` / `EvalTime` are + // leaves; every other new variant wraps exactly one child — + // inherit its placement, mirroring the Sort/Limit/Project arm + // above, until a dedicated physical op is written. QueryExpr::Scalar(_) | QueryExpr::EvalTime => PhysicalNode { op: PhysicalOp::Passthrough, placement: Placement::QueryEngine, @@ -460,7 +452,6 @@ fn plan_node(expr: &QueryExpr, config: &PhysicalPlannerConfig) -> PhysicalNode { | QueryExpr::Relabel { child, .. } | QueryExpr::InfoJoin { child, .. } | QueryExpr::Sample { child, .. } - | QueryExpr::TimeRange { child, .. } | QueryExpr::TimeShift { child, .. } | QueryExpr::WindowFunc { child, .. } => { let child = plan_node(child, config); @@ -471,6 +462,16 @@ fn plan_node(expr: &QueryExpr, config: &PhysicalPlannerConfig) -> PhysicalNode { children: vec![child], } } + // Scalar-expression node (Column/Literal/Compare/BoolAnd/BoolOr/ + // Not/IsNull/IsNotNull/Cast/InList/FunctionCall/Arith/Case) -- + // see `physical::allocator`'s matching catch-all for why this + // never reaches here as a bare top-level node in practice. + _ => PhysicalNode { + op: PhysicalOp::Passthrough, + placement: Placement::QueryEngine, + cost: PhysicalCost::default(), + children: vec![], + }, } } @@ -591,7 +592,7 @@ mod tests { use crate::intent_algebra::relational::{ default_cardinality, default_frequency, default_quantile, }; - use crate::intent_algebra::{Reduction, Schema, Source, WindowKind}; + use crate::intent_algebra::{Reduction, Schema, Source}; use crate::types_v2::AccuracyTarget; fn default_config() -> PhysicalPlannerConfig { @@ -617,20 +618,18 @@ mod tests { fn sketch_agg(intent: AggIntent, metric: &str) -> QueryExpr { QueryExpr::Aggregate { reduction: Reduction::by(vec![]), - aggs: vec![intent], + measures: vec![intent], output_names: Vec::new(), having: None, child: Box::new(scan(metric)), } } - /// `Window { Aggregate }` — the canonical fold of the legacy + /// `TimeRange { Aggregate }` — the canonical fold of the legacy /// `WindowedAgg`. fn windowed_agg(intent: AggIntent, size_secs: u64, metric: &str) -> QueryExpr { - QueryExpr::Window { - kind: WindowKind::Tumbling, - size: Duration::from_secs(size_secs), - slide: None, + QueryExpr::TimeRange { + range: Duration::from_secs(size_secs), child: Box::new(sketch_agg(intent, metric)), } } @@ -699,7 +698,7 @@ mod tests { fn plan_topk_at_query_engine() { let expr = QueryExpr::Aggregate { reduction: Reduction::by(vec![]), - aggs: vec![AggIntent::TopK { + measures: vec![AggIntent::TopK { k: 10, accuracy: AccuracyTarget::Epsilon(0.05), }], @@ -718,7 +717,7 @@ mod tests { // between them. let expr = QueryExpr::Aggregate { reduction: Reduction::by(vec![]), - aggs: vec![AggIntent::TopK { + measures: vec![AggIntent::TopK { k: 5, accuracy: AccuracyTarget::Epsilon(0.05), }], @@ -748,7 +747,7 @@ mod tests { // (no single sketch serves multiple intents). let expr = QueryExpr::Aggregate { reduction: Reduction::by(vec![]), - aggs: vec![AggIntent::Sum { col: None }, AggIntent::Min { col: None }], + measures: vec![AggIntent::Sum { col: None }, AggIntent::Min { col: None }], output_names: Vec::new(), having: None, child: Box::new(scan("trades")), @@ -766,7 +765,7 @@ mod tests { // Should span: Agent → Backend → QueryEngine let expr = QueryExpr::Aggregate { reduction: Reduction::by(vec![]), - aggs: vec![AggIntent::TopK { + measures: vec![AggIntent::TopK { k: 10, accuracy: AccuracyTarget::Epsilon(0.05), }], diff --git a/control_plane/src/physical/window_fusion.rs b/control_plane/src/physical/window_fusion.rs index 99845e1b..ee107317 100644 --- a/control_plane/src/physical/window_fusion.rs +++ b/control_plane/src/physical/window_fusion.rs @@ -9,7 +9,7 @@ //! //! The canonical L3 IR keeps "one canonical form per plan" (design.md //! §6): it has no fused windowed-aggregate variant — a windowed sketch is -//! the stacked `Window { child: Aggregate { aggs: [one], .. } }` shape. +//! the stacked `Window { child: Aggregate { measures: [one], .. } }` shape. //! `lower` produces exactly that shape when it folds a //! single-statistic sketchable `Aggregate` sitting over a `Window`. This //! module is the planner-side **peephole recognizer** that puts the @@ -55,7 +55,7 @@ pub struct FusedWindowSketch<'a> { pub inner_child: &'a QueryExpr, } -/// Recognize the canonical `Window { child: Aggregate { aggs: [one], +/// Recognize the canonical `Window { child: Aggregate { measures: [one], /// having: None, .. } }` shape — the stacked form `lower` /// folds a legacy `WindowedAgg` into — and return a [`FusedWindowSketch`] /// view of it. Returns `None` for any other shape (a multi-intent @@ -67,18 +67,25 @@ pub struct FusedWindowSketch<'a> { /// legacy planner already builds with `PhysicalWindow::None`. This /// recognizer is specifically the window-defines-lifecycle peephole. pub fn recognize_windowed_sketch(expr: &QueryExpr) -> Option> { - let QueryExpr::Window { - kind, - size, - slide, - child, - } = expr - else { + // ASAPPlanner has no canonical `Window` node (ASAPPlanner#193 -- + // "nothing ever constructs canonical QueryExpr::Window", confirmed by + // upstream's own corpus walk, true even at the previously-pinned + // rev): `asap_frontend_promql::lower_promql` -- the real production + // entry point since #428 -- has only ever emitted `TimeRange { range, + // child }` for a range-vector selector. This recognizer used to match + // `Window` regardless, which means it could never actually fire for + // real `lower_promql`-produced trees; matching `TimeRange` here is a + // fix, not just a rename. `TimeRange` carries no kind/slide -- real + // PromQL has no syntax to request anything but the implicit tumbling + // form, so `window_kind`/`window_slide` default to that (matching + // every other real-traffic call site in this codebase; see + // control_plane/docs/design-asapplanner-pin-migration.md). + let QueryExpr::TimeRange { range, child } = expr else { return None; }; let QueryExpr::Aggregate { reduction, - aggs, + measures, having, child: inner_child, .. @@ -88,7 +95,7 @@ pub fn recognize_windowed_sketch(expr: &QueryExpr) -> Option Option QueryExpr { - convert_root(&LQueryExpr::Source(SourceSpec::new(metric))).expect("convert source") + QueryExpr::Scan { + source: Source::TimeSeries { + metric: metric.to_string(), + }, + predicates: vec![], + schema: Schema::with_time_index(vec![], 0, vec![]), + } } - /// The canonical `Window { Aggregate { by: [], aggs: [agg] } }` shape — - /// the fold `lower` produces for a windowed single-sketch - /// aggregate. - fn windowed_sketch( - agg: AggIntent, - kind: WindowKind, - size: Duration, - slide: Option, - ) -> QueryExpr { - QueryExpr::Window { - kind, - size, - slide, + /// The canonical `TimeRange { Aggregate { by: [], measures: [agg] } }` + /// shape — what `asap_frontend_promql::lower_promql` (the real + /// production entry point) emits for a windowed single-sketch + /// aggregate. Real PromQL has no syntax for anything but the implicit + /// tumbling form (see `recognize_windowed_sketch`'s doc), so unlike + /// the pre-migration version of this helper, there is no `kind`/ + /// `slide` parameter to take anymore. + fn windowed_sketch(agg: AggIntent, size: Duration) -> QueryExpr { + QueryExpr::TimeRange { + range: size, child: Box::new(QueryExpr::Aggregate { reduction: crate::intent_algebra::Reduction::by(vec![]), - aggs: vec![agg], + measures: vec![agg], output_names: Vec::new(), having: None, child: Box::new(canonical_scan("m")), @@ -224,22 +234,17 @@ mod tests { } } - /// End-to-end fusion assertion: the canonical `Window { Aggregate }` + /// End-to-end fusion assertion: the canonical `TimeRange { Aggregate }` /// shape, run through the recognizer + `fused_sketch_decision` and /// through the canonical planner, produces a *fused* `OtelSketchBuild` /// carrying a resolved (non-`None`) physical window — the /// window-defines-sketch-lifecycle invariant as a planner peephole. - fn assert_equivalent( - agg: AggIntent, - kind: WindowKind, - size: Duration, - slide: Option, - ) { + fn assert_equivalent(agg: AggIntent, size: Duration) { let cfg = config(); - let canonical = windowed_sketch(agg, kind, size, slide); + let canonical = windowed_sketch(agg, size); let fused = recognize_windowed_sketch(&canonical) - .expect("canonical Window{Aggregate} should be recognized as a fused sketch"); + .expect("canonical TimeRange{Aggregate} should be recognized as a fused sketch"); let (decided_op, _placement) = fused_sketch_decision(&fused, &cfg); let PhysicalOp::OtelSketchBuild { window: decided_window, @@ -262,7 +267,7 @@ mod tests { } = &node.op else { panic!( - "canonical planner did not fuse Window{{Aggregate}}: {:?}", + "canonical planner did not fuse TimeRange{{Aggregate}}: {:?}", node.op ); }; @@ -275,72 +280,32 @@ mod tests { } #[test] - fn equivalence_tumbling_quantile() { - assert_equivalent( - default_quantile(0.99), - WindowKind::Tumbling, - Duration::from_secs(300), - None, - ); - } - - #[test] - fn equivalence_sliding_quantile() { - assert_equivalent( - default_quantile(0.5), - WindowKind::Sliding, - Duration::from_secs(600), - Some(Duration::from_secs(60)), - ); - } - - #[test] - fn equivalence_session_quantile() { - assert_equivalent( - default_quantile(0.95), - WindowKind::Session, - Duration::from_secs(30), - None, - ); + fn equivalence_quantile() { + assert_equivalent(default_quantile(0.99), Duration::from_secs(300)); } #[test] - fn equivalence_tumbling_cardinality() { - assert_equivalent( - default_cardinality(), - WindowKind::Tumbling, - Duration::from_secs(60), - None, - ); + fn equivalence_cardinality() { + assert_equivalent(default_cardinality(), Duration::from_secs(60)); } #[test] - fn equivalence_tumbling_frequency() { - assert_equivalent( - default_frequency(), - WindowKind::Tumbling, - Duration::from_secs(120), - None, - ); + fn equivalence_frequency() { + assert_equivalent(default_frequency(), Duration::from_secs(120)); } #[test] fn equivalence_sum_intent() { - assert_equivalent( - AggIntent::Sum { col: None }, - WindowKind::Tumbling, - Duration::from_secs(300), - None, - ); + assert_equivalent(AggIntent::Sum { col: None }, Duration::from_secs(300)); } #[test] fn recognizer_rejects_bare_aggregate() { - // A canonical `Aggregate` with no enclosing `Window` is the unfused - // sketch case — not a windowed sketch. + // A canonical `Aggregate` with no enclosing `TimeRange` is the + // unfused sketch case — not a windowed sketch. let canonical = QueryExpr::Aggregate { reduction: crate::intent_algebra::Reduction::by(vec![]), - aggs: vec![AggIntent::Sum { col: None }], + measures: vec![AggIntent::Sum { col: None }], output_names: Vec::new(), having: None, child: Box::new(canonical_scan("m")), @@ -350,37 +315,20 @@ mod tests { #[test] fn recognizer_rejects_window_over_non_aggregate() { - // `Window` directly over a `Scan` — no inner `Aggregate` to fuse. - let legacy = LQueryExpr::Window { - duration: Duration::from_secs(60), - slide: None, - input: Box::new(LQueryExpr::Source(SourceSpec::new("m"))), + // `TimeRange` directly over a `Scan` — no inner `Aggregate` to fuse. + let canonical = QueryExpr::TimeRange { + range: Duration::from_secs(60), + child: Box::new(canonical_scan("m")), }; - let canonical = convert_root(&legacy).expect("convert"); assert!(recognize_windowed_sketch(&canonical).is_none()); } #[test] - fn recognizer_accepts_converted_windowed_agg() { - // The L2 `Aggregate { Quantile } over Window` shape the parsers - // emit converts to canonical `Window { Aggregate }` — exactly the - // shape the recognizer matches. - let legacy = LQueryExpr::Aggregate { - keys: vec![], - without: false, - aggs: vec![AggItem { - alias: Some("q".into()), - func: AggFunc::Quantile(0.99), - col: LColumnRef::SampleValue, - }], - having: None, - input: Box::new(LQueryExpr::Window { - duration: Duration::from_secs(300), - slide: None, - input: Box::new(LQueryExpr::Source(SourceSpec::new("m"))), - }), - }; - let canonical = convert_root(&legacy).expect("convert"); + fn recognizer_accepts_windowed_agg() { + // The `TimeRange { Aggregate { Quantile } }` shape + // `lower_promql` emits for `quantile_over_time(...)` — exactly + // the shape the recognizer matches. + let canonical = windowed_sketch(default_quantile(0.99), Duration::from_secs(300)); let fused = recognize_windowed_sketch(&canonical).expect("should recognize"); assert_eq!(fused.window_kind, WindowKind::Tumbling); assert_eq!(fused.window_size, Duration::from_secs(300)); diff --git a/control_plane/src/pipeline.rs b/control_plane/src/pipeline.rs index 760d2c84..aa93fcbe 100644 --- a/control_plane/src/pipeline.rs +++ b/control_plane/src/pipeline.rs @@ -178,7 +178,9 @@ impl Analyzer { .map(|q| { query_parser::parse_query( q, - spec.accuracy.clone().unwrap_or(AccuracyTarget::Epsilon(0.01)), + spec.accuracy + .clone() + .unwrap_or(AccuracyTarget::Epsilon(0.01)), ) }) .transpose() diff --git a/control_plane/src/query_parser/mod.rs b/control_plane/src/query_parser/mod.rs index ceda82a0..667af47f 100644 --- a/control_plane/src/query_parser/mod.rs +++ b/control_plane/src/query_parser/mod.rs @@ -151,8 +151,7 @@ fn root_scan_schema(qe: &QueryExpr) -> Option<&crate::intent_algebra::Schema> { match qe { QueryExpr::Scan { schema, .. } => Some(schema), QueryExpr::Filter { child, .. } - | QueryExpr::Window { child, .. } - // `TimeRange`/`TimeShift` are `asap_l2::lower`'s range-vector-selector + // `TimeRange`/`TimeShift` are `lower_promql`'s range-vector-selector // and offset/@ markers (L1 adoption, design-target-architecture.md // Part B) -- pass-through wrappers over the same `Scan`. | QueryExpr::TimeRange { child, .. } @@ -171,11 +170,7 @@ fn root_scan_schema(qe: &QueryExpr) -> Option<&crate::intent_algebra::Schema> { rhs: right, .. } => root_scan_schema(left).or_else(|| root_scan_schema(right)), - QueryExpr::LetBinding { expr, child, .. } => { - root_scan_schema(expr).or_else(|| root_scan_schema(child)) - } - // `Ref` has no reachable `Scan` without a `LetBinding` scope, and - // the remaining PromQL-surface superset (Scalar/EvalTime/ + // The remaining PromQL-surface superset (Scalar/EvalTime/ // VectorFromScalar/ScalarFromVector/Relabel/InfoJoin/Sample/ // WindowFunc) is real, new capability `lower_promql` adds but this // crate's flat `ParsedQuery` extraction doesn't attempt to unpack @@ -231,13 +226,7 @@ impl QeCollector { collect_filters_from_scalar(pred, schema, &mut self.label_filters); self.visit(child, schema); } - QueryExpr::Window { size, child, .. } => { - if self.time_window.is_none() { - self.time_window = Some(*size); - } - self.visit(child, schema); - } - // `TimeRange` is `asap_l2::lower`'s range-vector-selector marker + // `TimeRange` is `lower_promql`'s range-vector-selector marker // (`m[5m]` in `quantile_over_time(φ, m[5m])`) -- the range-window // duration this collector's `time_window` field wants, same as // `Window::size` above. `TimeShift` (`offset`/`@`) is a pure @@ -251,7 +240,7 @@ impl QeCollector { QueryExpr::TimeShift { child, .. } => self.visit(child, schema), QueryExpr::Aggregate { reduction, - aggs, + measures: aggs, child, .. } => { @@ -302,16 +291,6 @@ impl QeCollector { self.visit(left, schema); self.visit(right, schema); } - QueryExpr::LetBinding { expr, child, .. } => { - self.visit(expr, schema); - self.visit(child, schema); - } - // `Ref` has no reachable `Scan` without a `LetBinding` scope - // to resolve it against, and the PromQL-surface superset - // (Scalar/EvalTime/VectorFromScalar/ScalarFromVector/Relabel/ - // InfoJoin/Sample/TimeRange/TimeShift/WindowFunc) isn't - // constructed by this parser today. - QueryExpr::Ref { .. } => {} _ => {} } } @@ -536,7 +515,8 @@ mod tests { use crate::intent_algebra::query_expr::QueryExpr as CQueryExpr; // A bare `avg_over_time(m[w])` (no `by`) lowers to canonical // `Aggregate { child: TimeRange { child: Scan } }`. - let expr = parse_query_expr_canonical("avg_over_time(cpu_seconds_total[10m])", ACC).unwrap(); + let expr = + parse_query_expr_canonical("avg_over_time(cpu_seconds_total[10m])", ACC).unwrap(); match expr { CQueryExpr::Aggregate { child, .. } => match *child { CQueryExpr::TimeRange { child, .. } => { @@ -576,7 +556,7 @@ mod doc_verify_all { match &expr { QueryExpr::Aggregate { reduction, - aggs, + measures: aggs, child, .. } => { @@ -603,7 +583,11 @@ mod doc_verify_all { // (see this module's doc: no longer necessarily the `Frequency` // extension), the outer `TopK` shape itself is unaffected. match &expr { - QueryExpr::Aggregate { aggs, child, .. } => { + QueryExpr::Aggregate { + measures: aggs, + child, + .. + } => { assert!(matches!(aggs.as_slice(), [AggIntent::TopK { k: 10, .. }])); // Inner reduction the TopK ranks by: `Aggregate { Count, // child: TimeRange { child: Scan } }` -- same `TimeRange` @@ -613,7 +597,7 @@ mod doc_verify_all { // being forced exact -- adopted as-is per this module's doc. match child.as_ref() { QueryExpr::Aggregate { - aggs: inner_aggs, + measures: inner_aggs, child: inner_child, .. } => { diff --git a/control_plane/src/replan.rs b/control_plane/src/replan.rs index 55ca78a1..73627959 100644 --- a/control_plane/src/replan.rs +++ b/control_plane/src/replan.rs @@ -655,7 +655,7 @@ impl Replanner { use crate::physical::colored_dag::emitter::{ AggregationInput, BackendAggregation, BackendStageConfig, }; - use asap_sketch::{SummaryKind, SummaryParams}; + use planner_types::post_asap::{SketchKind, SketchParams}; let window_secs = workload.time_window.as_secs().max(1); Some(BackendStageConfig { aggregations: vec![BackendAggregation { @@ -667,9 +667,9 @@ impl Replanner { // these are not emitted on the wire. DDSketch is the // chosen sentinel because every backend that recognises // `AggregationType::FromStr` also accepts DDSketch (and - // we don't have a `SummaryKind::None` variant today). - sketch_kind: SummaryKind::DDSketch, - sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, + // we don't have a `SketchKind::None` variant today). + sketch_kind: SketchKind::DDSketch.into(), + sketch_params: SketchParams::DDSketch { alpha: 0.01 }.into(), window_secs, spatial_filter: String::new(), grouping: workload.group_by_labels.clone(), @@ -1254,7 +1254,7 @@ mod tests { use crate::physical::colored_dag::emitter::{ AggregationInput, BackendAggregation, BackendStageConfig, }; - use asap_sketch::{SummaryKind, SummaryParams}; + use planner_types::post_asap::{SketchKind, SketchParams}; let (url, hits) = start_repost_mock().await; let client = StdArc::new(BackendClient::new(url)); @@ -1304,8 +1304,8 @@ mod tests { item_label: None, aggregation_id: "exact-http_requests_total-sum".to_string(), metric_name: "http_requests_total".to_string(), - sketch_kind: SummaryKind::DDSketch, - sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, + sketch_kind: SketchKind::DDSketch.into(), + sketch_params: SketchParams::DDSketch { alpha: 0.01 }.into(), grouping: vec!["zone".to_string()], spatial_filter: String::new(), window_secs: 60, diff --git a/control_plane/src/sketch_algebra/capability.rs b/control_plane/src/sketch_algebra/capability.rs index d6bbb6c7..398b6305 100644 --- a/control_plane/src/sketch_algebra/capability.rs +++ b/control_plane/src/sketch_algebra/capability.rs @@ -23,8 +23,8 @@ use crate::intent_algebra::agg_intent::AggIntent; use crate::sketch_algebra::matcher::sketch_family_satisfied; use crate::types_v2::AccuracyTarget; -use asap_sketch::SummaryKind; use asap_types::AggregationType; +use planner_types::post_asap::SketchKind; // ── Query-side capability tag ──────────────────────────────────────────────── @@ -105,7 +105,7 @@ pub enum Capability { /// carry that distinction back (replacing an earlier raw-string /// `query_contains_rate_call` re-parse). `rate`/`increase` now bind /// their own `AggIntent::Rate`/`AggIntent::Increase` (matching -/// `asap_plan::boundary::implementation_for`'s `SummaryKind::Rate`/ +/// `asap_aware_mapping::boundary::implementation_for`'s `SummaryKind::Rate`/ /// `Increase` — ASAPController models Rate as a distinct summary /// family), both mapping to `Capability::ExactAgg(AggregationType::Increase)`; /// only `sum`/`sum by (...)`/bare selectors and `sum_over_time` still @@ -295,7 +295,7 @@ impl OuterAgg { } /// Compact, hashable handle for sketch implementation choice. Mirrors -/// `asap_sketch::SummaryKind` but adds the `Any` query-side wildcard +/// `planner_types::post_asap::SummaryKind` but adds the `Any` query-side wildcard /// (not a sketch family — a dispatch hint). #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum SketchKindHandle { @@ -335,14 +335,14 @@ impl Capability { /// delegate their family-compatibility logic to /// [`sketch_family_satisfied`] (`enum-unification-plan.md` /// §5/§8 Step 4) — via [`resolve_handle`], which picks a concrete - /// per-family stand-in for the `Any` wildcard since `SummaryKind` + /// per-family stand-in for the `Any` wildcard since `SketchKind` /// has no wildcard concept of its own; family-matching subsumes it. /// `ExactAgg` is intentionally NOT routed through this path — see /// [`multi_pop_satisfies_single`]'s doc for why. pub fn is_satisfied_by(&self, indexed: &Capability) -> bool { match (self, indexed) { (Capability::QuantileApprox(req), Capability::QuantileApprox(have)) => { - sketch_kinds_compatible(*req, SummaryKind::Kll, *have, SummaryKind::Kll) + sketch_kinds_compatible(*req, SketchKind::Kll, *have, SketchKind::Kll) } // Cardinality has no inner handle; family match is total. (Capability::CardinalityApprox, Capability::CardinalityApprox) => true, @@ -353,9 +353,9 @@ impl Capability { (Capability::FrequencyTopk(req), Capability::FrequencyTopk(have)) => { sketch_kinds_compatible( *req, - SummaryKind::CmsWithHeap, + SketchKind::CmsWithHeap, *have, - SummaryKind::CmsWithHeap, + SketchKind::CmsWithHeap, ) } // Bare frequency: any frequency-family handle works on the @@ -365,10 +365,10 @@ impl Capability { // the sketch matrix). A heap-bearing `FrequencyTopk` indexed // capability ALSO satisfies a bare-frequency required capability. (Capability::FrequencyEstimate(req), Capability::FrequencyEstimate(have)) => { - sketch_kinds_compatible(*req, SummaryKind::Cms, *have, SummaryKind::Cms) + sketch_kinds_compatible(*req, SketchKind::Cms, *have, SketchKind::Cms) } (Capability::FrequencyEstimate(req), Capability::FrequencyTopk(have)) => { - sketch_kinds_compatible(*req, SummaryKind::Cms, *have, SummaryKind::CmsWithHeap) + sketch_kinds_compatible(*req, SketchKind::Cms, *have, SketchKind::CmsWithHeap) } // Exact-aggregation family: the agg_type must match // exactly OR be the single-pop ⇆ multi-pop equivalent. A @@ -394,19 +394,19 @@ impl Capability { } } -/// Map a concrete [`SketchKindHandle`] to its [`SummaryKind`] +/// Map a concrete [`SketchKindHandle`] to its [`SketchKind`] /// equivalent. `Any` has no single equivalent by design — resolve it to /// a concrete per-family stand-in via [`resolve_handle`] before calling /// this. -fn to_summary_kind(h: SketchKindHandle) -> Option { +fn to_summary_kind(h: SketchKindHandle) -> Option { match h { - SketchKindHandle::DDSketch => Some(SummaryKind::DDSketch), - SketchKindHandle::Kll => Some(SummaryKind::Kll), - SketchKindHandle::Hll => Some(SummaryKind::Hll), - SketchKindHandle::CountSketch => Some(SummaryKind::CountSketch), - SketchKindHandle::CountMin => Some(SummaryKind::Cms), - SketchKindHandle::CmsWithHeap => Some(SummaryKind::CmsWithHeap), - SketchKindHandle::CountSketchWithHeap => Some(SummaryKind::CountSketchWithHeap), + SketchKindHandle::DDSketch => Some(SketchKind::DDSketch), + SketchKindHandle::Kll => Some(SketchKind::Kll), + SketchKindHandle::Hll => Some(SketchKind::Hll), + SketchKindHandle::CountSketch => Some(SketchKind::CountSketch), + SketchKindHandle::CountMin => Some(SketchKind::Cms), + SketchKindHandle::CmsWithHeap => Some(SketchKind::CmsWithHeap), + SketchKindHandle::CountSketchWithHeap => Some(SketchKind::CountSketchWithHeap), // Defensive: `Any` should never reach this function directly — // every call site resolves it via `resolve_handle` first. `None` // here means "does not satisfy anything", the safe default. @@ -414,11 +414,11 @@ fn to_summary_kind(h: SketchKindHandle) -> Option { } } -/// Resolve a [`SketchKindHandle`] to the [`SummaryKind`] fed into +/// Resolve a [`SketchKindHandle`] to the [`SketchKind`] fed into /// [`sketch_family_satisfied`]. `Any` (the query-side "any /// implementation in this family satisfies" wildcard) resolves to /// `any_stand_in` — a concrete per-family placeholder — because -/// `SummaryKind` has no wildcard concept of its own; +/// `SketchKind` has no wildcard concept of its own; /// `sketch_family_satisfied`'s same-family-satisfies rule already /// treats every member of a family as interchangeable, so picking ANY /// concrete family member as the stand-in reproduces the wildcard's @@ -430,9 +430,9 @@ fn to_summary_kind(h: SketchKindHandle) -> Option { /// related by the asymmetric "heap satisfies bare" rule, not equal), so a /// bare stand-in would let a heap-less available sketch wrongly satisfy a /// top-k requirement. Every `FrequencyTopk` call site in this module -/// passes `SummaryKind::CmsWithHeap` as `any_stand_in` for exactly this +/// passes `SketchKind::CmsWithHeap` as `any_stand_in` for exactly this /// reason. -fn resolve_handle(h: SketchKindHandle, any_stand_in: SummaryKind) -> Option { +fn resolve_handle(h: SketchKindHandle, any_stand_in: SketchKind) -> Option { match h { SketchKindHandle::Any => Some(any_stand_in), other => to_summary_kind(other), @@ -445,9 +445,9 @@ fn resolve_handle(h: SketchKindHandle, any_stand_in: SummaryKind) -> Option bool { match ( resolve_handle(required, required_any_stand_in), @@ -458,7 +458,6 @@ fn sketch_kinds_compatible( } } - /// True when `available` is the multi-population equivalent of /// `required`'s single-population variant — i.e. a `MultipleSum` /// policy can serve a `Sum` query (via re-aggregation across keys), @@ -481,7 +480,7 @@ fn multi_pop_satisfies_single(required: AggregationType, available: AggregationT /// `rate()`/`increase()` PromQL both lower to a required /// `Capability::ExactAgg(AggregationType::Increase)` (`capability_for`'s /// `AggIntent::Rate | AggIntent::Increase` case, matching -/// `asap_plan::boundary::implementation_for`'s `SummaryKind::Increase | +/// `asap_aware_mapping::boundary::implementation_for`'s `SummaryKind::Increase | /// SummaryKind::Rate` — ASAPController models Rate as its own summary /// family). But this workspace's data plane has no storage kind distinct /// from Sum for it: `evaluate_exact_agg_rate` (`sketch_reducer.rs`) @@ -535,11 +534,11 @@ fn sum_satisfies_increase(required: AggregationType, available: AggregationType) /// Beyond the `Extension`/`Frequency` special case (deployment-specific, /// see below — `asap-plan` deliberately has no opinion on a shape it /// can't see into), every other `AggIntent` variant's capability is -/// derived from [`asap_plan::boundary::implementation_for`] — the single +/// derived from [`asap_aware_mapping::boundary::implementation_for`] — the single /// upstream authority for "how would this intent be realized" — rather /// than a second, hand-maintained, parallel judgment kept in sync by /// hand. See [`implementation_to_capability`] for the -/// `asap_sketch::SummaryKind` → `Capability` family translation this +/// `planner_types::post_asap::SummaryKind` → `Capability` family translation this /// still requires (the two crates' capability vocabularies aren't the /// same *shape*, even once they agree on substance), and its doc comment /// for the one deliberate override (`Count{Exact}`). @@ -560,37 +559,40 @@ pub fn capability_for(intent: &AggIntent) -> Option { Some(Capability::FrequencyEstimate(SketchKindHandle::Any)) }; } - implementation_to_capability(asap_plan::boundary::implementation_for(intent)) + implementation_to_capability(asap_aware_mapping::boundary::implementation_for(intent)) } -/// Translate `asap-plan`'s per-intent implementation decision into this -/// repo's own [`Capability`] vocabulary. +/// Translate `asap-aware-mapping`'s per-intent implementation decision +/// into this repo's own [`Capability`] vocabulary. /// -/// `Implementation::Summary` carries an `asap_sketch::SummaryKind` for -/// both the approximate-sketch and exact-accumulator cases (told apart -/// via `kind.is_exact()`) — this repo's `Capability` groups those -/// into coarser families (`QuantileApprox`/`CardinalityApprox`/ -/// `FrequencyEstimate`/`FrequencyTopk` for sketches; `ExactAgg(AggregationType)` -/// for accumulators) because that's the granularity the sketch index +/// `Implementation` used to carry one merged `Summary { kind, params }` +/// variant for both the approximate-sketch and exact-accumulator cases +/// (told apart via `kind.is_exact()`, ASAPController#170); ASAPPlanner +/// split them back into distinct `Sketch { kind: SketchKind, .. }` / +/// `ExactAggregate { kind: ExactKind, .. }` variants (ASAPPlanner#218) — +/// see control_plane/docs/design-asapplanner-pin-migration.md. This +/// repo's `Capability` groups those into coarser families +/// (`QuantileApprox`/`CardinalityApprox`/`FrequencyEstimate`/ +/// `FrequencyTopk` for sketches; `ExactAgg(AggregationType)` for +/// accumulators) because that's the granularity the sketch index /// (`is_satisfied_by`) and the wire-shared `sketch_index::Capability` /// actually match on — the required side never pins a *specific* -/// concrete implementation (`Any`), only the family. This function is -/// exhaustive over `SummaryKind` (no wildcard fallthrough), so a new -/// variant there fails to compile here until given an explicit mapping. -fn implementation_to_capability(implementation: asap_plan::Implementation) -> Option { - use asap_plan::Implementation; - use asap_sketch::SummaryKind; +/// concrete implementation (`Any`), only the family. Both match arms are +/// exhaustive over their respective kind enum (no wildcard fallthrough), +/// so a new variant on either side fails to compile here until given an +/// explicit mapping. +fn implementation_to_capability( + implementation: asap_aware_mapping::Implementation, +) -> Option { + use asap_aware_mapping::Implementation; + use planner_types::post_asap::{ExactKind, SketchKind}; match implementation { Implementation::PassThrough => None, - // Exact accumulator half of the merged `Summary` variant - // (ASAPController#170 collapsed `Sketch`/`ExactAccumulator` into - // one `Summary { kind, params }`, recoverable via `kind.is_exact()` - // — see `asap_sketch::SummaryKind::is_exact`'s doc). - Implementation::Summary { kind, .. } if kind.is_exact() => match kind { - SummaryKind::Sum => Some(Capability::ExactAgg(AggregationType::Sum)), - SummaryKind::MinMax => Some(Capability::ExactAgg(AggregationType::MinMax)), - SummaryKind::Increase | SummaryKind::Rate => { + Implementation::ExactAggregate { kind, .. } => match kind { + ExactKind::Sum => Some(Capability::ExactAgg(AggregationType::Sum)), + ExactKind::MinMax => Some(Capability::ExactAgg(AggregationType::MinMax)), + ExactKind::Increase | ExactKind::Rate => { Some(Capability::ExactAgg(AggregationType::Increase)) } // `AggregationType` (this repo's own exact-accumulator-family @@ -599,52 +601,38 @@ fn implementation_to_capability(implementation: asap_plan::Implementation) -> Op // for both `Statistic::Sum` and `Statistic::Count`, so a // `count_over_time` query matched against a `Sum` policy // would silently return sum-of-values, not sample-count). - // `asap_plan::boundary::implementation_for` still reports - // `Count{Exact}` as exact (it assumes a real count + // `asap_aware_mapping::boundary::implementation_for` still + // reports `Count{Exact}` as exact (it assumes a real count // accumulator exists, which is true in ASAPController's own // reference implementation) — deliberately overridden here to // `None` (archive) until a real `SumCountAccumulator` lands. - SummaryKind::Count => None, - SummaryKind::Kll - | SummaryKind::DDSketch - | SummaryKind::Hll - | SummaryKind::Theta - | SummaryKind::Kmv - | SummaryKind::Cms - | SummaryKind::CmsWithHeap - | SummaryKind::CountSketch - | SummaryKind::CountSketchWithHeap => { - unreachable!( - "{kind:?} is a sketch-family SummaryKind, so kind.is_exact() is false — \ - never reached inside the is_exact() guard" - ) - } + ExactKind::Count => None, }, - // Approximate-sketch half. - Implementation::Summary { kind, .. } => match kind { - SummaryKind::Kll | SummaryKind::DDSketch => { + Implementation::Sketch { kind, .. } => match kind { + SketchKind::Kll | SketchKind::DDSketch => { Some(Capability::QuantileApprox(SketchKindHandle::Any)) } - SummaryKind::Hll | SummaryKind::Theta | SummaryKind::Kmv => { + SketchKind::Hll | SketchKind::Theta | SketchKind::Kmv => { Some(Capability::CardinalityApprox) } - SummaryKind::Cms | SummaryKind::CountSketch => { + SketchKind::Cms | SketchKind::CountSketch => { Some(Capability::FrequencyEstimate(SketchKindHandle::Any)) } - SummaryKind::CmsWithHeap | SummaryKind::CountSketchWithHeap => { + SketchKind::CmsWithHeap | SketchKind::CountSketchWithHeap => { Some(Capability::FrequencyTopk(SketchKindHandle::Any)) } - SummaryKind::Sum - | SummaryKind::Count - | SummaryKind::MinMax - | SummaryKind::Increase - | SummaryKind::Rate => { - unreachable!( - "{kind:?} is an exact-accumulator SummaryKind, so kind.is_exact() is \ - true — the arm above already handles it" - ) - } }, + // `Sample`/`Wavelet`/`StatModel`: no core `AggIntent` dispatch + // produces these today (only a deployment's own + // `CostModel::realize_extension` for an `AggIntent::Extension` + // could), and `ControlPlaneCostModel::realize_extension` (this + // repo's own impl, `sketch_algebra/cost_model.rs`) only ever + // returns `Sketch`/`PassThrough` for the `Frequency` extension — + // never reachable via this repo's own dispatch, same status as + // the sibling families had before the ASAPPlanner#218 split. + Implementation::Sample { .. } + | Implementation::Wavelet { .. } + | Implementation::StatModel { .. } => None, } } diff --git a/control_plane/src/sketch_algebra/capability_matching.rs b/control_plane/src/sketch_algebra/capability_matching.rs index 53c5608f..1a533ca8 100644 --- a/control_plane/src/sketch_algebra/capability_matching.rs +++ b/control_plane/src/sketch_algebra/capability_matching.rs @@ -50,7 +50,7 @@ #![allow(dead_code)] -use asap_sketch::SummaryKind; +use planner_types::post_asap::SketchKind; /// Query intent the user is expressing — abstracted away from the L1 /// language (PromQL `quantile_over_time`, SQL `PERCENTILE_CONT`, etc.) and @@ -137,9 +137,9 @@ pub enum AccuracyPreference { /// Muthukrishnan 2005). NB: this is the planner-side capability /// declaration; the backend's "top-K from CountMin state" readout path /// is a separate workstream — see the module-level docs. -pub fn is_valid_pair(sketch: SummaryKind, statistic: StatisticClass) -> bool { +pub fn is_valid_pair(sketch: SketchKind, statistic: StatisticClass) -> bool { + use SketchKind::*; use StatisticClass::*; - use SummaryKind::*; match (sketch, statistic) { (DDSketch, Quantile) | (Kll, Quantile) @@ -167,10 +167,10 @@ pub fn is_valid_pair(sketch: SummaryKind, statistic: StatisticClass) -> bool { /// `sketch_family_override` (treated as `QueryWorkload::sketch_type_override` /// at the planner-rules layer) wins over the capability-matched default — /// see `planner::rules::bind_workload_typed`. -pub fn pick_family(statistic: StatisticClass, accuracy: AccuracyPreference) -> Option { +pub fn pick_family(statistic: StatisticClass, accuracy: AccuracyPreference) -> Option { use AccuracyPreference::*; + use SketchKind::*; use StatisticClass::*; - use SummaryKind::*; let kind = match (statistic, accuracy) { (Quantile, RelativeError) => DDSketch, (Quantile, RankError) => Kll, @@ -220,71 +220,65 @@ mod tests { #[test] fn ddsketch_is_quantile_only() { assert!(is_valid_pair( - SummaryKind::DDSketch, + SketchKind::DDSketch, StatisticClass::Quantile )); assert!(!is_valid_pair( - SummaryKind::DDSketch, + SketchKind::DDSketch, StatisticClass::Cardinality )); - assert!(!is_valid_pair(SummaryKind::DDSketch, StatisticClass::TopK)); + assert!(!is_valid_pair(SketchKind::DDSketch, StatisticClass::TopK)); assert!(!is_valid_pair( - SummaryKind::DDSketch, + SketchKind::DDSketch, StatisticClass::Frequency )); assert!(!is_valid_pair( - SummaryKind::DDSketch, + SketchKind::DDSketch, StatisticClass::SumRateCount )); } #[test] fn kll_is_quantile_only() { - assert!(is_valid_pair(SummaryKind::Kll, StatisticClass::Quantile)); + assert!(is_valid_pair(SketchKind::Kll, StatisticClass::Quantile)); + assert!(!is_valid_pair(SketchKind::Kll, StatisticClass::Cardinality)); + assert!(!is_valid_pair(SketchKind::Kll, StatisticClass::TopK)); + assert!(!is_valid_pair(SketchKind::Kll, StatisticClass::Frequency)); assert!(!is_valid_pair( - SummaryKind::Kll, - StatisticClass::Cardinality - )); - assert!(!is_valid_pair(SummaryKind::Kll, StatisticClass::TopK)); - assert!(!is_valid_pair(SummaryKind::Kll, StatisticClass::Frequency)); - assert!(!is_valid_pair( - SummaryKind::Kll, + SketchKind::Kll, StatisticClass::SumRateCount )); } #[test] fn hll_is_cardinality_only() { - assert!(is_valid_pair(SummaryKind::Hll, StatisticClass::Cardinality)); - assert!(!is_valid_pair(SummaryKind::Hll, StatisticClass::Quantile)); - assert!(!is_valid_pair(SummaryKind::Hll, StatisticClass::TopK)); - assert!(!is_valid_pair(SummaryKind::Hll, StatisticClass::Frequency)); + assert!(is_valid_pair(SketchKind::Hll, StatisticClass::Cardinality)); + assert!(!is_valid_pair(SketchKind::Hll, StatisticClass::Quantile)); + assert!(!is_valid_pair(SketchKind::Hll, StatisticClass::TopK)); + assert!(!is_valid_pair(SketchKind::Hll, StatisticClass::Frequency)); assert!(!is_valid_pair( - SummaryKind::Hll, + SketchKind::Hll, StatisticClass::SumRateCount )); } #[test] fn countsketch_is_topk_only() { - assert!(is_valid_pair( - SummaryKind::CountSketch, - StatisticClass::TopK - )); + assert!(is_valid_pair(SketchKind::CountSketch, StatisticClass::TopK)); assert!(!is_valid_pair( - SummaryKind::CountSketch, + SketchKind::CountSketch, StatisticClass::Quantile )); assert!(!is_valid_pair( - SummaryKind::CountSketch, + SketchKind::CountSketch, StatisticClass::Cardinality )); assert!(!is_valid_pair( - SummaryKind::CountSketch, + SketchKind::CountSketch, StatisticClass::Frequency )); assert!(!is_valid_pair( - SummaryKind::CountSketch, + SketchKind::CountSketch, StatisticClass::SumRateCount )); } @@ -293,15 +287,12 @@ mod tests { fn cms_supports_frequency_and_topk() { // CMS validly answers Frequency (point-frequency, additive bound) // AND TopK via the CMS-Heap pattern (Cormode & Muthukrishnan 2005). - assert!(is_valid_pair(SummaryKind::Cms, StatisticClass::Frequency)); - assert!(is_valid_pair(SummaryKind::Cms, StatisticClass::TopK)); - assert!(!is_valid_pair(SummaryKind::Cms, StatisticClass::Quantile)); - assert!(!is_valid_pair( - SummaryKind::Cms, - StatisticClass::Cardinality - )); + assert!(is_valid_pair(SketchKind::Cms, StatisticClass::Frequency)); + assert!(is_valid_pair(SketchKind::Cms, StatisticClass::TopK)); + assert!(!is_valid_pair(SketchKind::Cms, StatisticClass::Quantile)); + assert!(!is_valid_pair(SketchKind::Cms, StatisticClass::Cardinality)); assert!(!is_valid_pair( - SummaryKind::Cms, + SketchKind::Cms, StatisticClass::SumRateCount )); } @@ -315,7 +306,7 @@ mod tests { // TopK-shaped workload (CMS-Heap pattern, Cormode & // Muthukrishnan 2005). assert!( - is_valid_pair(SummaryKind::Cms, StatisticClass::TopK), + is_valid_pair(SketchKind::Cms, StatisticClass::TopK), "CMS should support TopK via the CMS-Heap pattern", ); } @@ -326,7 +317,7 @@ mod tests { fn pick_family_quantile_relative_picks_ddsketch() { assert_eq!( pick_family(StatisticClass::Quantile, AccuracyPreference::RelativeError), - Some(SummaryKind::DDSketch), + Some(SketchKind::DDSketch), ); } @@ -334,7 +325,7 @@ mod tests { fn pick_family_quantile_rank_picks_kll() { assert_eq!( pick_family(StatisticClass::Quantile, AccuracyPreference::RankError), - Some(SummaryKind::Kll), + Some(SketchKind::Kll), ); } @@ -346,7 +337,7 @@ mod tests { ] { assert_eq!( pick_family(StatisticClass::Cardinality, pref), - Some(SummaryKind::Hll), + Some(SketchKind::Hll), ); } } @@ -355,7 +346,7 @@ mod tests { fn pick_family_topk_picks_countsketch() { assert_eq!( pick_family(StatisticClass::TopK, AccuracyPreference::default()), - Some(SummaryKind::CountSketch), + Some(SketchKind::CountSketch), ); } @@ -363,7 +354,7 @@ mod tests { fn pick_family_frequency_picks_cms() { assert_eq!( pick_family(StatisticClass::Frequency, AccuracyPreference::default()), - Some(SummaryKind::Cms), + Some(SketchKind::Cms), ); } @@ -429,17 +420,17 @@ mod tests { assert_eq!(classify_demo_metric(""), None); } - // ── End-to-end: every contract row maps to its expected SummaryKind ──────── + // ── End-to-end: every contract row maps to its expected SketchKind ──────── #[test] fn every_contract_metric_picks_its_contract_family() { let cases = [ ("http_requests_total", None), - ("http_latency_ms", Some(SummaryKind::DDSketch)), - ("request_size_bytes", Some(SummaryKind::Kll)), - ("unique_users_per_min", Some(SummaryKind::Hll)), - ("top_endpoint_qps", Some(SummaryKind::CountSketch)), - ("endpoint_request_freq", Some(SummaryKind::Cms)), + ("http_latency_ms", Some(SketchKind::DDSketch)), + ("request_size_bytes", Some(SketchKind::Kll)), + ("unique_users_per_min", Some(SketchKind::Hll)), + ("top_endpoint_qps", Some(SketchKind::CountSketch)), + ("endpoint_request_freq", Some(SketchKind::Cms)), ]; for (metric, want_kind) in cases { let (stat, pref) = classify_demo_metric(metric) diff --git a/control_plane/src/sketch_algebra/cost_model.rs b/control_plane/src/sketch_algebra/cost_model.rs index 26258f81..20d7e27b 100644 --- a/control_plane/src/sketch_algebra/cost_model.rs +++ b/control_plane/src/sketch_algebra/cost_model.rs @@ -1,6 +1,6 @@ //! `ControlPlaneCostModel` — plugs control_plane's own sketch-family //! selection + parameter-sizing policy into -//! `asap_plan::bind::implement_tree_in_with` via the two `CostModel` +//! `asap_aware_mapping::bind::implement_tree_in_with` via the two `CostModel` //! extension points (`rank_candidates` for family choice, `size_params` //! for parameter sizing — the latter added by ASAPController PR #146 //! specifically to support this migration). @@ -11,7 +11,7 @@ //! priority order, same recall-tier logic — just re-homed behind the //! `CostModel` trait instead of a bespoke `Rule` dispatcher, so the L3→L4 //! walk itself (schema derivation, `col`/`by` computation, DAG -//! construction) can be `asap_plan::bind`'s rather than a forked copy. +//! construction) can be `asap_aware_mapping::bind`'s rather than a forked copy. //! //! `AggIntent::Extension` (control_plane's `Frequency` point-query) used //! to be one of two shapes `implement_tree_in_with` couldn't realize even @@ -35,10 +35,10 @@ #![allow(dead_code)] -use asap_ir::intent_algebra::expr_ir::ColumnRef; -use asap_plan::boundary::Implementation; -use asap_plan::CostModel; -use asap_sketch::{SketchQuery, SummaryKind, SummaryParams}; +use asap_aware_mapping::boundary::Implementation; +use asap_aware_mapping::CostModel; +use planner_types::post_asap::{SketchKind, SketchParams, SketchQuery}; +use planner_types::pre_asap::expr_ir::ColumnRef; use crate::intent_algebra::agg_intent::FREQUENCY_EXT_KIND; use crate::intent_algebra::AggIntent; @@ -123,16 +123,16 @@ impl ControlPlaneCostModel { /// Verbatim port of `bind_cms_topk.rs`'s `TopkRecallTier` + /// `candidate_families` + `cheapest_family`. Public (within the /// crate) for the same reason as [`Self::topk_eps_delta`]. - pub(crate) fn topk_family_order(&self, intent_accuracy: &AccuracyTarget) -> Vec { + pub(crate) fn topk_family_order(&self, intent_accuracy: &AccuracyTarget) -> Vec { let tight = matches!(self.workload_accuracy, AccuracyTarget::Exact) || matches!(intent_accuracy, AccuracyTarget::Exact); - let allowed: &[SummaryKind] = if tight { - &[SummaryKind::CountSketchWithHeap] + let allowed: &[SketchKind] = if tight { + &[SketchKind::CountSketchWithHeap] } else { - &[SummaryKind::CmsWithHeap, SummaryKind::CountSketchWithHeap] + &[SketchKind::CmsWithHeap, SketchKind::CountSketchWithHeap] }; let table = WireCostTable::default(); - let mut ranked: Vec = allowed.to_vec(); + let mut ranked: Vec = allowed.to_vec(); ranked.sort_by_key(|k| table.for_kind(k).per_flush()); ranked } @@ -164,7 +164,7 @@ fn intent_accuracy(intent: &AggIntent) -> AccuracyTarget { } impl CostModel for ControlPlaneCostModel { - fn rank_candidates(&self, intent: &AggIntent, candidates: &[SummaryKind]) -> Vec { + fn rank_candidates(&self, intent: &AggIntent, candidates: &[SketchKind]) -> Vec { match intent { // bind_ddsketch_quantile (priority 6) always wins the old // dispatcher's tie-break over bind_kll_quantile (priority 5) @@ -174,7 +174,7 @@ impl CostModel for ControlPlaneCostModel { // eps). Pure static reorder: DDSketch before Kll. AggIntent::Quantile { .. } => { let mut v = candidates.to_vec(); - if let Some(pos) = v.iter().position(|k| *k == SummaryKind::DDSketch) { + if let Some(pos) = v.iter().position(|k| *k == SketchKind::DDSketch) { let dd = v.remove(pos); v.insert(0, dd); } @@ -190,11 +190,11 @@ impl CostModel for ControlPlaneCostModel { fn size_params( &self, - kind: SummaryKind, + kind: SketchKind, intent: &AggIntent, eps: f64, delta: f64, - ) -> SummaryParams { + ) -> SketchParams { match intent { AggIntent::TopK { k, .. } => { let (eps, delta) = self.topk_eps_delta(&intent_accuracy(intent)); @@ -206,12 +206,12 @@ impl CostModel for ControlPlaneCostModel { let w = w.next_power_of_two(); let heap_size = *k as u32; match kind { - SummaryKind::CmsWithHeap => SummaryParams::CmsWithHeap { + SketchKind::CmsWithHeap => SketchParams::CmsWithHeap { width: w, depth: d, heap_size, }, - _ => SummaryParams::CountSketchWithHeap { + _ => SketchParams::CountSketchWithHeap { width: w, depth: d, heap_size, @@ -225,30 +225,34 @@ impl CostModel for ControlPlaneCostModel { // reaches `bind_summary_with` upstream — see // `implementation_for_with`'s `Exact => exact_realization` // arm), kept as a safe fallback rather than a panic. - return asap_plan::boundary::default_size_params(kind, intent, eps, delta); + return asap_aware_mapping::boundary::default_size_params( + kind, intent, eps, delta, + ); }; match kind { - SummaryKind::Kll => SummaryParams::Kll { + SketchKind::Kll => SketchParams::Kll { k: kll_k_for_eps(eps), }, - SummaryKind::DDSketch if (0.0..1.0).contains(&eps) => { - SummaryParams::DDSketch { alpha: eps } + SketchKind::DDSketch if (0.0..1.0).contains(&eps) => { + SketchParams::DDSketch { alpha: eps } } - SummaryKind::Hll => SummaryParams::Hll { + SketchKind::Hll => SketchParams::Hll { precision: hll_precision_for_eps(eps), }, - SummaryKind::Cms => { + SketchKind::Cms => { let (w, d) = Self::cms_width_depth(eps, delta); - SummaryParams::Cms { width: w, depth: d } + SketchParams::Cms { width: w, depth: d } + } + other => { + asap_aware_mapping::boundary::default_size_params(other, intent, eps, delta) } - other => asap_plan::boundary::default_size_params(other, intent, eps, delta), } } } } /// Realize control_plane's `Frequency` (`ext_kind: "frequency"`) - /// point-query intent (ASAPController#150). `SummaryKind::Cms` (heap- + /// point-query intent (ASAPController#150). `SketchKind::Cms` (heap- /// less — a point lookup needs no heap, unlike `TopK`) matches /// `capability_matching::pick_family`'s own `Frequency → Cms` mapping /// (and its `is_valid_pair` truth table, which declares `(Cms, @@ -277,9 +281,9 @@ impl CostModel for ControlPlaneCostModel { return Implementation::PassThrough; }; let (width, depth) = Self::cms_width_depth(eps, delta); - Implementation::Summary { - kind: SummaryKind::Cms, - params: SummaryParams::Cms { + Implementation::Sketch { + kind: SketchKind::Cms, + params: SketchParams::Cms { width: width.next_power_of_two(), depth, }, @@ -329,11 +333,11 @@ impl CostModel for ControlPlaneCostModel { /// a fresh selection decision. pub struct ForcedFamilyCostModel { inner: ControlPlaneCostModel, - forced: SummaryKind, + forced: SketchKind, } impl ForcedFamilyCostModel { - pub fn new(workload_accuracy: AccuracyTarget, forced: SummaryKind) -> Self { + pub fn new(workload_accuracy: AccuracyTarget, forced: SketchKind) -> Self { Self { inner: ControlPlaneCostModel::new(workload_accuracy), forced, @@ -342,21 +346,17 @@ impl ForcedFamilyCostModel { } impl CostModel for ForcedFamilyCostModel { - fn rank_candidates( - &self, - _intent: &AggIntent, - _candidates: &[SummaryKind], - ) -> Vec { + fn rank_candidates(&self, _intent: &AggIntent, _candidates: &[SketchKind]) -> Vec { vec![self.forced.clone()] } fn size_params( &self, - kind: SummaryKind, + kind: SketchKind, intent: &AggIntent, eps: f64, delta: f64, - ) -> SummaryParams { + ) -> SketchParams { self.inner.size_params(kind, intent, eps, delta) } @@ -410,13 +410,13 @@ impl CostModel for ForcedFamilyCostModel { /// installed yet, or for metrics a partial/stale plan doesn't cover. pub struct ObservedFamilyCostModel { inner: ControlPlaneCostModel, - observed: Option<(SummaryKind, SummaryParams)>, + observed: Option<(SketchKind, SketchParams)>, } impl ObservedFamilyCostModel { pub fn new( workload_accuracy: AccuracyTarget, - observed: Option<(SummaryKind, SummaryParams)>, + observed: Option<(SketchKind, SketchParams)>, ) -> Self { Self { inner: ControlPlaneCostModel::new(workload_accuracy), @@ -426,7 +426,7 @@ impl ObservedFamilyCostModel { } impl CostModel for ObservedFamilyCostModel { - fn rank_candidates(&self, intent: &AggIntent, candidates: &[SummaryKind]) -> Vec { + fn rank_candidates(&self, intent: &AggIntent, candidates: &[SketchKind]) -> Vec { match &self.observed { Some((kind, _)) if candidates.contains(kind) => vec![kind.clone()], _ => self.inner.rank_candidates(intent, candidates), @@ -435,11 +435,11 @@ impl CostModel for ObservedFamilyCostModel { fn size_params( &self, - kind: SummaryKind, + kind: SketchKind, intent: &AggIntent, eps: f64, delta: f64, - ) -> SummaryParams { + ) -> SketchParams { match &self.observed { Some((okind, oparams)) if *okind == kind => oparams.clone(), _ => self.inner.size_params(kind, intent, eps, delta), @@ -512,16 +512,16 @@ mod tests { let model = ControlPlaneCostModel::new(AccuracyTarget::Epsilon(0.1)); let ranked = model.rank_candidates( &default_quantile(0.99), - &[SummaryKind::Kll, SummaryKind::DDSketch], + &[SketchKind::Kll, SketchKind::DDSketch], ); - assert_eq!(ranked, vec![SummaryKind::DDSketch, SummaryKind::Kll]); + assert_eq!(ranked, vec![SketchKind::DDSketch, SketchKind::Kll]); } #[test] fn kll_k_matches_bind_kll_quantile_rungs() { let model = ControlPlaneCostModel::new(AccuracyTarget::Epsilon(1.0)); - let params = model.size_params(SummaryKind::Kll, &default_quantile(0.99), 0.01, 0.01); - assert_eq!(params, SummaryParams::Kll { k: 200 }); + let params = model.size_params(SketchKind::Kll, &default_quantile(0.99), 0.01, 0.01); + assert_eq!(params, SketchParams::Kll { k: 200 }); let model = ControlPlaneCostModel::new(AccuracyTarget::Epsilon(1.0)); let tight = crate::intent_algebra::AggIntent::Quantile { @@ -529,15 +529,15 @@ mod tests { q: 0.99, accuracy: eps(0.001), }; - let params = model.size_params(SummaryKind::Kll, &tight, 0.01, 0.01); - assert_eq!(params, SummaryParams::Kll { k: 2048 }); + let params = model.size_params(SketchKind::Kll, &tight, 0.01, 0.01); + assert_eq!(params, SketchParams::Kll { k: 2048 }); } #[test] fn hll_precision_matches_bind_hll_cardinality_rungs() { let model = ControlPlaneCostModel::new(AccuracyTarget::Epsilon(1.0)); - let params = model.size_params(SummaryKind::Hll, &default_cardinality(), 0.01, 0.01); - assert_eq!(params, SummaryParams::Hll { precision: 14 }); + let params = model.size_params(SketchKind::Hll, &default_cardinality(), 0.01, 0.01); + assert_eq!(params, SketchParams::Hll { precision: 14 }); } #[test] @@ -549,9 +549,9 @@ mod tests { }; let ranked = model.rank_candidates( &intent, - &[SummaryKind::CmsWithHeap, SummaryKind::CountSketchWithHeap], + &[SketchKind::CmsWithHeap, SketchKind::CountSketchWithHeap], ); - assert_eq!(ranked, vec![SummaryKind::CountSketchWithHeap]); + assert_eq!(ranked, vec![SketchKind::CountSketchWithHeap]); } #[test] @@ -563,9 +563,9 @@ mod tests { }; let ranked = model.rank_candidates( &intent, - &[SummaryKind::CmsWithHeap, SummaryKind::CountSketchWithHeap], + &[SketchKind::CmsWithHeap, SketchKind::CountSketchWithHeap], ); - assert_eq!(ranked[0], SummaryKind::CmsWithHeap); + assert_eq!(ranked[0], SketchKind::CmsWithHeap); } #[test] @@ -575,8 +575,8 @@ mod tests { k: 5, accuracy: eps(0.01), }; - let params = model.size_params(SummaryKind::CmsWithHeap, &intent, 0.0, 0.0); - let SummaryParams::CmsWithHeap { + let params = model.size_params(SketchKind::CmsWithHeap, &intent, 0.0, 0.0); + let SketchParams::CmsWithHeap { width, heap_size, .. } = params else { diff --git a/control_plane/src/sketch_algebra/lower.rs b/control_plane/src/sketch_algebra/lower.rs index fcd1ca32..e7449174 100644 --- a/control_plane/src/sketch_algebra/lower.rs +++ b/control_plane/src/sketch_algebra/lower.rs @@ -1,5 +1,5 @@ //! L3 → L4/L5 lowering — `QueryExpr` walk that adopts -//! `asap_plan::bind::implement_tree_in_with` for the sketch algebra +//! `asap_aware_mapping::bind::implement_tree_in_with` for the sketch algebra //! itself (Step B of the plan-shaped-serving migration), with //! `crate::sketch_algebra::cost_model::ControlPlaneCostModel` plugged in //! for family selection + parameter sizing. @@ -10,7 +10,7 @@ //! `Logical(…)`." //! //! Two node shapes are rewritten *before* delegating to -//! `implement_tree_in_with`, because `asap_plan::boundary::implementation_for` +//! `implement_tree_in_with`, because `asap_aware_mapping::boundary::implementation_for` //! actively binds them to an `Implementation` this deployment's data plane //! doesn't (or, deliberately, shouldn't) serve — not something the //! `CostModel` hook can reach, since the decision of *whether* to call @@ -31,24 +31,22 @@ #![allow(dead_code)] -use std::rc::Rc; - -use asap_plan::bind::implement_tree_in_with; -use asap_plan::cost_model::CostModel; +use asap_aware_mapping::bind::implement_tree_with; +use asap_aware_mapping::cost_model::CostModel; use thiserror::Error; -use crate::intent_algebra::{AggIntent, BindingScope, QueryExpr}; +use crate::intent_algebra::{AggIntent, QueryExpr}; use crate::sketch_algebra::cost_model::ControlPlaneCostModel; use crate::sketch_algebra::physical_expr::{L4Plan, PhysicalExpr}; -use crate::types_v2::{AccuracyTarget, BindingName}; +use crate::types_v2::AccuracyTarget; /// Errors surfaced by the `bind_query_expr` lowering. #[derive(Debug, Error)] pub enum BindingError { - /// L3 schema derivation failed while lifting an edge to `L4Schema` — - /// forwarded from `asap_plan::bind`. + /// L3 schema derivation failed while lifting an edge to `SummarySchema` — + /// forwarded from `asap_aware_mapping::bind`. #[error("L3->L4 implementation failed: {0}")] - Implement(#[from] asap_plan::ImplementError), + Implement(#[from] asap_aware_mapping::ImplementError), } /// Lower an L3 `QueryExpr` to L4/L5 under the supplied workload-level @@ -72,7 +70,7 @@ pub fn bind_query_expr( /// live-serving path (`l4_lowering.rs`) must NOT re-derive a family/params /// choice independently of what was actually planned — it looks up what's /// really registered in the `SketchStore` and hands in a cost model that -/// echoes that back, so the resulting `L4Node` matches reality by +/// echoes that back, so the resulting `SummaryNode` matches reality by /// construction rather than by a coincidental accuracy-target match. See /// `control_plane/docs/design-target-architecture.md`'s "planning vs /// serving" split. @@ -85,32 +83,28 @@ pub fn bind_query_expr_with_cost_model( fn bind_recursive(expr: &QueryExpr, cost_model: &dyn CostModel) -> Result { match expr { - QueryExpr::LetBinding { name, expr, child } => Ok(L4Plan::LetBinding { - name: BindingName::new(name.as_str()), - expr: Rc::new(bind_recursive(expr, cost_model)?), - child: Rc::new(bind_recursive(child, cost_model)?), - }), - QueryExpr::Ref { name } => Ok(L4Plan::Ref { - name: BindingName::new(name.as_str()), - }), + // `QueryExpr::LetBinding`/`::Ref` don't exist in the canonical IR + // anymore (ASAPPlanner#181/#192 -- see + // control_plane/docs/design-asapplanner-pin-migration.md), so + // this walk can never actually receive that shape; the arms that + // used to produce `L4Plan::LetBinding`/`L4Plan::Ref` here are + // gone with it. - // The canonical L3 IR places `Window` *above* a single-statistic - // sketchable `Aggregate` (`lower`'s window-swap). `implement_tree_in_with` + // The canonical L3 IR places `TimeRange` *above* a single-statistic + // sketchable `Aggregate` (`lower_promql`'s window-swap; was + // `Window` before the ASAPPlanner pin migration). `implement_tree_with` // only recurses through the `Aggregate` spine (see its module // docs' "conservative fallbacks" — a logical parent above a - // bindable aggregate subsumes it unbound), so push the window + // bindable aggregate subsumes it unbound), so push the range // down under the aggregate and re-dispatch, exactly as the old - // hand-written walk did — the window then rides along inside the + // hand-written walk did — the range then rides along inside the // bound node's `Logical(...)` child. - QueryExpr::Window { - kind, - size, - slide, - child, - } if matches!(child.as_ref(), QueryExpr::Aggregate { .. }) => { + QueryExpr::TimeRange { range, child } + if matches!(child.as_ref(), QueryExpr::Aggregate { .. }) => + { let QueryExpr::Aggregate { reduction, - aggs, + measures: aggs, output_names, having, child: agg_child, @@ -120,13 +114,11 @@ fn bind_recursive(expr: &QueryExpr, cost_model: &dyn CostModel) -> Result Result Result { - Ok(L4Plan::Summary(asap_plan::bind::logical( - expr, - &BindingScope::default(), - )?)) + Ok(L4Plan::Summary(asap_aware_mapping::bind::logical(expr)?)) } _ => { let rewritten = rewrite_rate_to_increase(expr); - let node = implement_tree_in_with(&rewritten, &BindingScope::default(), cost_model)?; + let node = implement_tree_with(&rewritten, cost_model)?; Ok(L4Plan::Summary(node)) } } @@ -184,13 +175,13 @@ fn rewrite_rate_to_increase(expr: &QueryExpr) -> QueryExpr { match expr { QueryExpr::Aggregate { reduction, - aggs, + measures: aggs, output_names, having, child, } => QueryExpr::Aggregate { reduction: reduction.clone(), - aggs: aggs + measures: aggs .iter() .map(|intent| { if matches!(intent, AggIntent::Rate) { diff --git a/control_plane/src/sketch_algebra/matcher.rs b/control_plane/src/sketch_algebra/matcher.rs index 10c94e72..e13e1d87 100644 --- a/control_plane/src/sketch_algebra/matcher.rs +++ b/control_plane/src/sketch_algebra/matcher.rs @@ -1,6 +1,6 @@ -//! The reference downstream implementation of [`asap_plan::Matcher`]. +//! The reference downstream implementation of [`asap_aware_mapping::Matcher`]. //! -//! `asap_plan::boundary::Matcher` is a trait with no default implementation +//! `asap_aware_mapping::boundary::Matcher` is a trait with no default implementation //! and no shipped instance — deliberately, per its crate doc: which //! `Implementation`s are actually *available* anywhere is entirely a //! downstream deployment's concern, and even the pure sketch-algebra @@ -37,8 +37,8 @@ //! which is actively being built to replace //! `storage_engines/sketch_db/query/sketch_reducer.rs`. Restored in full. -use asap_plan::{Implementation, Matcher}; -use asap_sketch::SummaryKind; +use asap_aware_mapping::{Implementation, Matcher}; +use planner_types::post_asap::SketchKind; /// [`Matcher`] impl covering pure sketch-family compatibility. See the /// module doc for what this deliberately does not cover. @@ -63,31 +63,34 @@ impl Matcher for SummaryFamilyMatcher { /// but not the reverse (a heap-less sketch cannot enumerate top-k /// items it never tracked). fn is_satisfied_by(&self, required: &Implementation, available: &Implementation) -> bool { - // ASAPController#170 merged `Sketch`/`ExactAccumulator` into one - // `Summary { kind, params }` variant, recoverable via - // `kind.is_exact()`. The variant-tag mismatch that used to fall - // through to `_ => false` (comparing a `Sketch` against an - // `ExactAccumulator`) is now an explicit `is_exact()` mismatch - // between the two sides, still falling through the same way. + // ASAPController#170 had merged `Sketch`/`ExactAccumulator` into + // one `Summary { kind, params }` variant, recoverable via + // `kind.is_exact()`; ASAPPlanner#218 split them back into + // distinct `Sketch`/`ExactAggregate` variants (see + // control_plane/docs/design-asapplanner-pin-migration.md). The + // variant-tag mismatch this match falls through to `_ => false` + // for (comparing a `Sketch` against an `ExactAggregate`) is now + // just the natural consequence of them being separate variants + // again, same behavior as the `is_exact()` mismatch this replaced. match (required, available) { (Implementation::PassThrough, _) => true, ( - Implementation::Summary { kind: required, .. }, - Implementation::Summary { kind: have, .. }, - ) if required.is_exact() && have.is_exact() => required == have, + Implementation::ExactAggregate { kind: required, .. }, + Implementation::ExactAggregate { kind: have, .. }, + ) => required == have, ( - Implementation::Summary { kind: required, .. }, - Implementation::Summary { kind: have, .. }, - ) if !required.is_exact() && !have.is_exact() => sketch_family_satisfied(required, have), + Implementation::Sketch { kind: required, .. }, + Implementation::Sketch { kind: have, .. }, + ) => sketch_family_satisfied(required, have), _ => false, } } } -/// Pure `SummaryKind`-to-`SummaryKind` family-compatibility check — the +/// Pure `SketchKind`-to-`SketchKind` family-compatibility check — the /// same rule [`SummaryFamilyMatcher::is_satisfied_by`] applies in its /// `Sketch` arm, exposed directly for callers that only have bare kinds -/// (no [`asap_sketch::SummaryParams`]) to compare. +/// (no [`planner_types::post_asap::SketchParams`]) to compare. /// `control_plane::sketch_algebra::capability::Capability::is_satisfied_by` /// is the first such caller: its `SketchKindHandle` query-side dispatch /// tag never carries params, so constructing a full @@ -95,16 +98,17 @@ impl Matcher for SummaryFamilyMatcher { /// would mean fabricating meaningless param values. See that module's /// doc for why `Capability`/`SketchKindHandle` themselves aren't deleted /// outright (`scratchpad/artifacts/enum-unification-plan.md` §8 Step 4). -pub fn sketch_family_satisfied(required: &SummaryKind, available: &SummaryKind) -> bool { - match (summary_family(required), summary_family(available)) { - (Some(req_family), Some(have_family)) => req_family.satisfied_by(have_family), - _ => false, - } +pub fn sketch_family_satisfied(required: &SketchKind, available: &SketchKind) -> bool { + let req_family = summary_family(required); + let have_family = summary_family(available); + req_family.satisfied_by(have_family) } -/// The family a [`SummaryKind`] belongs to, for [`SummaryFamilyMatcher`]. -/// `None` for the exact-accumulator kinds, which aren't grouped into -/// families (see [`SummaryFamilyMatcher::is_satisfied_by`]'s doc). +/// The family a [`SketchKind`] belongs to, for [`SummaryFamilyMatcher`]. +/// Total now (every `SketchKind` variant is an approximate-sketch family +/// by construction, post ASAPPlanner#218's split — the exact-accumulator +/// kinds this used to also cover live in `ExactKind` now, a distinct type +/// this function never sees). #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum SummaryFamily { Quantile, @@ -126,60 +130,46 @@ impl SummaryFamily { } } -fn summary_family(kind: &SummaryKind) -> Option { +fn summary_family(kind: &SketchKind) -> SummaryFamily { match kind { - SummaryKind::Kll | SummaryKind::DDSketch => Some(SummaryFamily::Quantile), - SummaryKind::Hll | SummaryKind::Theta | SummaryKind::Kmv => { - Some(SummaryFamily::Cardinality) - } - SummaryKind::Cms | SummaryKind::CountSketch => Some(SummaryFamily::Frequency), - SummaryKind::CmsWithHeap | SummaryKind::CountSketchWithHeap => { - Some(SummaryFamily::FrequencyTopk) - } - SummaryKind::Sum - | SummaryKind::Count - | SummaryKind::MinMax - | SummaryKind::Increase - | SummaryKind::Rate => None, + SketchKind::Kll | SketchKind::DDSketch => SummaryFamily::Quantile, + SketchKind::Hll | SketchKind::Theta | SketchKind::Kmv => SummaryFamily::Cardinality, + SketchKind::Cms | SketchKind::CountSketch => SummaryFamily::Frequency, + SketchKind::CmsWithHeap | SketchKind::CountSketchWithHeap => SummaryFamily::FrequencyTopk, } } #[cfg(test)] mod tests { use super::*; - use asap_sketch::SummaryParams; + use planner_types::post_asap::{ExactKind, ExactParams, SketchParams}; - /// A valid `SummaryParams` for `kind` — `is_satisfied_by` only matches + /// A valid `SketchParams` for `kind` — `is_satisfied_by` only matches /// on `kind`, never `params`, but the test values should still be /// real, constructible `(kind, params)` pairs rather than nonsense /// combinations (e.g. `Hll` paired with `Kll`'s params) that could /// never arise from real code. - fn params_for(kind: &SummaryKind) -> SummaryParams { + fn params_for(kind: &SketchKind) -> SketchParams { match kind { - SummaryKind::Sum => SummaryParams::Sum, - SummaryKind::Count => SummaryParams::Count, - SummaryKind::MinMax => SummaryParams::MinMax, - SummaryKind::Increase => SummaryParams::Increase, - SummaryKind::Rate => SummaryParams::Rate, - SummaryKind::Kll => SummaryParams::Kll { k: 200 }, - SummaryKind::Cms => SummaryParams::Cms { + SketchKind::Kll => SketchParams::Kll { k: 200 }, + SketchKind::Cms => SketchParams::Cms { width: 100, depth: 5, }, - SummaryKind::Hll => SummaryParams::Hll { precision: 14 }, - SummaryKind::DDSketch => SummaryParams::DDSketch { alpha: 0.01 }, - SummaryKind::CmsWithHeap => SummaryParams::CmsWithHeap { + SketchKind::Hll => SketchParams::Hll { precision: 14 }, + SketchKind::DDSketch => SketchParams::DDSketch { alpha: 0.01 }, + SketchKind::CmsWithHeap => SketchParams::CmsWithHeap { width: 100, depth: 5, heap_size: 10, }, - SummaryKind::Kmv => SummaryParams::Kmv { k: 1024 }, - SummaryKind::Theta => SummaryParams::Theta { k: 1024 }, - SummaryKind::CountSketch => SummaryParams::CountSketch { + SketchKind::Kmv => SketchParams::Kmv { k: 1024 }, + SketchKind::Theta => SketchParams::Theta { k: 1024 }, + SketchKind::CountSketch => SketchParams::CountSketch { width: 100, depth: 5, }, - SummaryKind::CountSketchWithHeap => SummaryParams::CountSketchWithHeap { + SketchKind::CountSketchWithHeap => SketchParams::CountSketchWithHeap { width: 100, depth: 5, heap_size: 10, @@ -187,39 +177,49 @@ mod tests { } } - fn sketch(kind: SummaryKind) -> Implementation { - let params = params_for(&kind); - Implementation::Summary { kind, params } + fn exact_params_for(kind: &ExactKind) -> ExactParams { + match kind { + ExactKind::Sum => ExactParams::Sum, + ExactKind::Count => ExactParams::Count, + ExactKind::MinMax => ExactParams::MinMax, + ExactKind::Increase => ExactParams::Increase, + ExactKind::Rate => ExactParams::Rate, + } } - fn accumulator(kind: SummaryKind) -> Implementation { + fn sketch(kind: SketchKind) -> Implementation { let params = params_for(&kind); - Implementation::Summary { kind, params } + Implementation::Sketch { kind, params } + } + + fn accumulator(kind: ExactKind) -> Implementation { + let params = exact_params_for(&kind); + Implementation::ExactAggregate { kind, params } } #[test] fn same_sketch_kind_satisfies_itself() { let m = SummaryFamilyMatcher; - assert!(m.is_satisfied_by(&sketch(SummaryKind::Kll), &sketch(SummaryKind::Kll))); - assert!(m.is_satisfied_by(&sketch(SummaryKind::Hll), &sketch(SummaryKind::Hll))); + assert!(m.is_satisfied_by(&sketch(SketchKind::Kll), &sketch(SketchKind::Kll))); + assert!(m.is_satisfied_by(&sketch(SketchKind::Hll), &sketch(SketchKind::Hll))); } #[test] fn same_family_alternate_kind_satisfies() { let m = SummaryFamilyMatcher; // Kll / DDSketch are interchangeable quantile answers. - assert!(m.is_satisfied_by(&sketch(SummaryKind::Kll), &sketch(SummaryKind::DDSketch))); + assert!(m.is_satisfied_by(&sketch(SketchKind::Kll), &sketch(SketchKind::DDSketch))); // Hll / Theta / Kmv are interchangeable cardinality answers. - assert!(m.is_satisfied_by(&sketch(SummaryKind::Hll), &sketch(SummaryKind::Theta))); - assert!(m.is_satisfied_by(&sketch(SummaryKind::Hll), &sketch(SummaryKind::Kmv))); + assert!(m.is_satisfied_by(&sketch(SketchKind::Hll), &sketch(SketchKind::Theta))); + assert!(m.is_satisfied_by(&sketch(SketchKind::Hll), &sketch(SketchKind::Kmv))); } #[test] fn cross_family_never_satisfies() { let m = SummaryFamilyMatcher; - assert!(!m.is_satisfied_by(&sketch(SummaryKind::Kll), &sketch(SummaryKind::Hll))); - assert!(!m.is_satisfied_by(&sketch(SummaryKind::Hll), &sketch(SummaryKind::Kll))); - assert!(!m.is_satisfied_by(&sketch(SummaryKind::Kll), &sketch(SummaryKind::Cms))); + assert!(!m.is_satisfied_by(&sketch(SketchKind::Kll), &sketch(SketchKind::Hll))); + assert!(!m.is_satisfied_by(&sketch(SketchKind::Hll), &sketch(SketchKind::Kll))); + assert!(!m.is_satisfied_by(&sketch(SketchKind::Kll), &sketch(SketchKind::Cms))); } #[test] @@ -227,10 +227,10 @@ mod tests { let m = SummaryFamilyMatcher; // A CmsWithHeap instance already carries the plain CMS matrix, so // it answers a bare frequency point-query too. - assert!(m.is_satisfied_by(&sketch(SummaryKind::Cms), &sketch(SummaryKind::CmsWithHeap))); + assert!(m.is_satisfied_by(&sketch(SketchKind::Cms), &sketch(SketchKind::CmsWithHeap))); assert!(m.is_satisfied_by( - &sketch(SummaryKind::CountSketch), - &sketch(SummaryKind::CountSketchWithHeap) + &sketch(SketchKind::CountSketch), + &sketch(SketchKind::CountSketchWithHeap) )); } @@ -239,60 +239,57 @@ mod tests { let m = SummaryFamilyMatcher; // The reverse does not hold: a heap-less sketch never tracked the // heavy-hitter heap, so it cannot enumerate top-k items. - assert!(!m.is_satisfied_by(&sketch(SummaryKind::CmsWithHeap), &sketch(SummaryKind::Cms))); + assert!(!m.is_satisfied_by(&sketch(SketchKind::CmsWithHeap), &sketch(SketchKind::Cms))); assert!(!m.is_satisfied_by( - &sketch(SummaryKind::CountSketchWithHeap), - &sketch(SummaryKind::CountSketch) + &sketch(SketchKind::CountSketchWithHeap), + &sketch(SketchKind::CountSketch) )); } #[test] fn cms_and_count_sketch_are_the_same_frequency_family() { let m = SummaryFamilyMatcher; - assert!(m.is_satisfied_by(&sketch(SummaryKind::Cms), &sketch(SummaryKind::CountSketch))); + assert!(m.is_satisfied_by(&sketch(SketchKind::Cms), &sketch(SketchKind::CountSketch))); assert!(m.is_satisfied_by( - &sketch(SummaryKind::CmsWithHeap), - &sketch(SummaryKind::CountSketchWithHeap) + &sketch(SketchKind::CmsWithHeap), + &sketch(SketchKind::CountSketchWithHeap) )); } #[test] fn exact_accumulator_requires_the_exact_same_kind() { let m = SummaryFamilyMatcher; - assert!(m.is_satisfied_by( - &accumulator(SummaryKind::Sum), - &accumulator(SummaryKind::Sum) - )); + assert!(m.is_satisfied_by(&accumulator(ExactKind::Sum), &accumulator(ExactKind::Sum))); assert!(!m.is_satisfied_by( - &accumulator(SummaryKind::Sum), - &accumulator(SummaryKind::MinMax) + &accumulator(ExactKind::Sum), + &accumulator(ExactKind::MinMax) )); assert!(!m.is_satisfied_by( - &accumulator(SummaryKind::Increase), - &accumulator(SummaryKind::Rate) + &accumulator(ExactKind::Increase), + &accumulator(ExactKind::Rate) )); } #[test] fn sketch_and_accumulator_never_satisfy_each_other() { let m = SummaryFamilyMatcher; - assert!(!m.is_satisfied_by(&sketch(SummaryKind::Kll), &accumulator(SummaryKind::Sum))); - assert!(!m.is_satisfied_by(&accumulator(SummaryKind::Sum), &sketch(SummaryKind::Kll))); + assert!(!m.is_satisfied_by(&sketch(SketchKind::Kll), &accumulator(ExactKind::Sum))); + assert!(!m.is_satisfied_by(&accumulator(ExactKind::Sum), &sketch(SketchKind::Kll))); } #[test] fn pass_through_required_is_vacuously_satisfied() { let m = SummaryFamilyMatcher; - assert!(m.is_satisfied_by(&Implementation::PassThrough, &sketch(SummaryKind::Kll))); - assert!(m.is_satisfied_by(&Implementation::PassThrough, &accumulator(SummaryKind::Sum))); + assert!(m.is_satisfied_by(&Implementation::PassThrough, &sketch(SketchKind::Kll))); + assert!(m.is_satisfied_by(&Implementation::PassThrough, &accumulator(ExactKind::Sum))); assert!(m.is_satisfied_by(&Implementation::PassThrough, &Implementation::PassThrough)); } #[test] fn pass_through_available_never_satisfies_a_real_requirement() { let m = SummaryFamilyMatcher; - assert!(!m.is_satisfied_by(&sketch(SummaryKind::Kll), &Implementation::PassThrough)); - assert!(!m.is_satisfied_by(&accumulator(SummaryKind::Sum), &Implementation::PassThrough)); + assert!(!m.is_satisfied_by(&sketch(SketchKind::Kll), &Implementation::PassThrough)); + assert!(!m.is_satisfied_by(&accumulator(ExactKind::Sum), &Implementation::PassThrough)); } // ── sketch_family_satisfied (the bare-kind entry point) ────────────── @@ -301,36 +298,26 @@ mod tests { fn sketch_family_satisfied_matches_is_satisfied_by_on_the_sketch_arm() { // The free function is meant to be exactly the logic // `SummaryFamilyMatcher::is_satisfied_by` applies to its `Sketch` - // arm, just without needing `SummaryParams` to call it. + // arm, just without needing `SketchParams` to call it. assert!(sketch_family_satisfied( - &SummaryKind::Kll, - &SummaryKind::DDSketch + &SketchKind::Kll, + &SketchKind::DDSketch )); assert!(sketch_family_satisfied( - &SummaryKind::Cms, - &SummaryKind::CmsWithHeap - )); - assert!(!sketch_family_satisfied( - &SummaryKind::CmsWithHeap, - &SummaryKind::Cms + &SketchKind::Cms, + &SketchKind::CmsWithHeap )); assert!(!sketch_family_satisfied( - &SummaryKind::Kll, - &SummaryKind::Hll + &SketchKind::CmsWithHeap, + &SketchKind::Cms )); + assert!(!sketch_family_satisfied(&SketchKind::Kll, &SketchKind::Hll)); } - #[test] - fn sketch_family_satisfied_rejects_exact_accumulator_kinds() { - // Exact-accumulator kinds have no family (`summary_family` returns - // `None` for them) — never satisfied by anything via this path. - assert!(!sketch_family_satisfied( - &SummaryKind::Sum, - &SummaryKind::Sum - )); - assert!(!sketch_family_satisfied( - &SummaryKind::Kll, - &SummaryKind::Sum - )); - } + // The old `sketch_family_satisfied_rejects_exact_accumulator_kinds` + // test (passing `SummaryKind::Sum` -- an exact-accumulator kind -- to + // this sketch-only function) no longer type-checks at all post + // ASAPPlanner#218's split: `SketchKind` has no exact-accumulator + // variants to construct in the first place, so the property that test + // asserted is now enforced by the type system instead of at runtime. } diff --git a/control_plane/src/sketch_algebra/mod.rs b/control_plane/src/sketch_algebra/mod.rs index 346c8863..1a77b2c5 100644 --- a/control_plane/src/sketch_algebra/mod.rs +++ b/control_plane/src/sketch_algebra/mod.rs @@ -6,13 +6,13 @@ //! deployment-independent). Single-rooted per query; multi-root //! workload-level fan-in lives one layer up in //! `types_v2::WorkloadPlan`. -//! - `asap_sketch::SummaryKind` / `SummaryParams` — typed sketch-family +//! - `planner_types::post_asap::SummaryKind` / `SummaryParams` — typed sketch-family //! selector + parameter payload (moved out of this crate — formerly //! `sketch_params::SketchKind`/`SketchParams` — Stage 3 of the //! sketch-identity unification; see //! `scratchpad/artifacts/enum-unification-plan.md`). //! - [`bind_query_expr`] — the L3→L4 lowering driver: bottom-up walk -//! that delegates to `asap_plan::bind::implement_tree_in_with` (the +//! that delegates to `asap_aware_mapping::bind::implement_tree_in_with` (the //! local `Bind*` rule family it used to fire was retired in Step B). //! //! Scope reduction (per orchestrator spec): the variant set ships the diff --git a/control_plane/src/sketch_algebra/physical_expr.rs b/control_plane/src/sketch_algebra/physical_expr.rs index c8248c62..98a377c7 100644 --- a/control_plane/src/sketch_algebra/physical_expr.rs +++ b/control_plane/src/sketch_algebra/physical_expr.rs @@ -7,18 +7,18 @@ //! Step B of the plan-shaped-serving migration retires this crate's own //! `PhysicalExpr`-as-L4-algebra (the old `Logical` / `SketchAgg` / //! `SketchEstimate` / `SketchMerge` / `ExactAgg` variants) in favor of -//! ASAPController's canonical L4 IR, `asap_sketch::{SummaryExpr, L4Node}` +//! ASAPController's canonical L4 IR, `planner_types::post_asap::{SummaryExpr, SummaryNode}` //! — the same move Step 3 of the enum-unification made for -//! `SketchKind → SummaryKind`, one layer up. `implement_promql_for_asap_tier` -//! (`asap_tier_implement.rs`, Step A) already builds `Rc` trees via -//! `asap_plan::bind::implement_tree_in_with`; this module gives the rest of +//! `SketchKind → SketchKind`, one layer up. `implement_promql_for_asap_tier` +//! (`asap_tier_implement.rs`, Step A) already builds `Rc` trees via +//! `asap_aware_mapping::bind::implement_tree_in_with`; this module gives the rest of //! the crate (optimizer, physical, emit) the same IR shape. //! -//! Two things `asap_sketch::L4Node` genuinely doesn't have, kept here: +//! Two things `planner_types::post_asap::SummaryNode` genuinely doesn't have, kept here: //! //! - **`LetBinding` / `Ref`** — named fan-in sharing (SQL //! `WITH name AS (expr) ...` / a `SketchAgg` shared by two `SketchEstimate` -//! readouts). `L4Node`'s own DAG sharing is structural (multiple `Rc` +//! readouts). `SummaryNode`'s own DAG sharing is structural (multiple `Rc` //! references to the same node), not named — but the rule-firing walk in //! this crate discovers sharing incrementally, per-node, so it still needs //! a name to thread a bound value across sibling calls. This is a @@ -36,21 +36,21 @@ //! already pattern-match on them. //! //! `SketchAgg` / `SketchEstimate` / `SketchMerge` / `Logical` / `ExactAgg` -//! don't get their own variants anymore — `asap_sketch::SummaryExpr` +//! don't get their own variants anymore — `planner_types::post_asap::SummaryExpr` //! already unifies all of them (including "exact accumulator" and "sketch" //! as the same `SummaryAgg` node, with or without a wrapping -//! `SummaryEstimate`) inside a single `L4Plan::Summary(Rc)`. +//! `SummaryEstimate`) inside a single `L4Plan::Summary(Rc)`. #![allow(dead_code)] use std::rc::Rc; use std::time::Duration; -use asap_sketch::{L4Node, SummaryKind, SummaryParams}; +use planner_types::post_asap::{SketchKind, SketchParams, SummaryNode}; use crate::types_v2::BindingName; -/// L4 IR — "what to compute". Wraps `asap_sketch::L4Node` (the sketch +/// L4 IR — "what to compute". Wraps `planner_types::post_asap::SummaryNode` (the sketch /// algebra itself, owned upstream) and adds only the named-binding sharing /// mechanism `asap_sketch` doesn't have. See module docs. #[derive(Debug, Clone)] @@ -58,7 +58,7 @@ pub enum L4Plan { /// A committed L4 sub-tree — `SummaryAgg` / `SummaryEstimate` / /// `SummaryMerge` / `Logical`, whatever `implement_tree_in_with` (or a /// deployment-specific pre-pass) produced. - Summary(Rc), + Summary(Rc), /// SQL `WITH name AS (expr) SELECT ... FROM name` / sketch-state /// fan-in: name a sub-expression so multiple parents can reference it. @@ -100,9 +100,9 @@ pub enum PhysicalExpr { /// this metric. RawAtEdgeSketchAtBackend { /// Sketch family the backend will build at ingest. - family: SummaryKind, + family: SketchKind, /// Sketch parameters (validated by the catalog at bind time). - params: SummaryParams, + params: SketchParams, /// Input sub-tree — typically `Summary(Logical(Window{...}))` or /// `Summary(Logical(Scan{...}))`. child: Box, @@ -138,8 +138,8 @@ pub enum PhysicalExpr { impl PhysicalExpr { /// Convenience constructor for the common case: a committed - /// `L4Node` sketch-built at the edge, no placement wrapper. - pub fn committed(node: Rc) -> Self { + /// `SummaryNode` sketch-built at the edge, no placement wrapper. + pub fn committed(node: Rc) -> Self { PhysicalExpr::Committed(L4Plan::Summary(node)) } } @@ -194,10 +194,8 @@ mod tests { } fn windowed_scan() -> QueryExpr { - QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, + QueryExpr::TimeRange { + range: Duration::from_secs(300), child: Box::new(ts_scan()), } } @@ -206,7 +204,7 @@ mod tests { fn committed_wraps_an_implement_tree_result() { let q = QueryExpr::Aggregate { reduction: crate::intent_algebra::Reduction::by(vec![]), - aggs: vec![crate::intent_algebra::AggIntent::Quantile { + measures: vec![crate::intent_algebra::AggIntent::Quantile { col: None, q: 0.99, accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), @@ -215,19 +213,32 @@ mod tests { having: None, child: Box::new(windowed_scan()), }; - let node = asap_plan::bind::implement_tree(&q).expect("implements"); + let node = asap_aware_mapping::bind::implement_tree(&q).expect("implements"); let e = PhysicalExpr::committed(node); match e { PhysicalExpr::Committed(L4Plan::Summary(node)) => match &node.expr { - asap_sketch::SummaryExpr::SummaryEstimate { query, summary_input } => { - assert!(matches!(query, asap_sketch::SketchQuery::Quantile { q } if *q == 0.99)); + planner_types::post_asap::SummaryExpr::SummaryEstimate { + query, + summary_input, + } => { + assert!( + matches!(query, planner_types::post_asap::SketchQuery::Quantile { q } if *q == 0.99) + ); match &summary_input.expr { - asap_sketch::SummaryExpr::SummaryAgg { - summary, params, child, .. + planner_types::post_asap::SummaryExpr::SummaryAgg { + family, child, .. } => { - assert_eq!(summary, &SummaryKind::Kll); - assert_eq!(params, &SummaryParams::Kll { k: 200 }); - assert!(matches!(child.expr, asap_sketch::SummaryExpr::Logical(_))); + assert_eq!( + family, + &planner_types::post_asap::SummaryFamilyType::Sketch( + SketchKind::Kll, + SketchParams::Kll { k: 200 } + ) + ); + assert!(matches!( + child.expr, + planner_types::post_asap::SummaryExpr::Logical(_) + )); } other => panic!("expected SummaryAgg, got {other:?}"), } diff --git a/control_plane/src/sketch_algebra/tests.rs b/control_plane/src/sketch_algebra/tests.rs index b2a679d7..a9c4442f 100644 --- a/control_plane/src/sketch_algebra/tests.rs +++ b/control_plane/src/sketch_algebra/tests.rs @@ -5,13 +5,14 @@ use std::rc::Rc; use std::time::Duration; -use asap_ir::intent_algebra::expr_ir::ColumnRef; -use asap_sketch::{L4Node, SketchQuery, SummaryExpr, SummaryKind, SummaryParams}; +use planner_types::post_asap::{ + ExactKind, ExactParams, SketchKind, SketchParams, SketchQuery, SummaryExpr, SummaryFamilyType, + SummaryNode, +}; +use planner_types::pre_asap::expr_ir::ColumnRef; use crate::intent_algebra::schema::{Column, DataType}; -use crate::intent_algebra::{ - AggIntent, BindingScope, LabelFilter, QueryExpr, Reduction, Schema, Source, WindowKind, -}; +use crate::intent_algebra::{AggIntent, LabelFilter, QueryExpr, Reduction, Schema, Source}; use crate::sketch_algebra::cost_model::ForcedFamilyCostModel; use crate::sketch_algebra::lower::bind_query_expr; use crate::sketch_algebra::physical_expr::{L4Plan, PhysicalExpr}; @@ -54,10 +55,8 @@ fn ts_scan() -> QueryExpr { } fn windowed_scan() -> QueryExpr { - QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, + QueryExpr::TimeRange { + range: Duration::from_secs(300), child: Box::new(ts_scan()), } } @@ -68,7 +67,7 @@ fn agg_quantile(q: f64, accuracy: AccuracyTarget) -> QueryExpr { // `quantile_over_time(...)` lowers to: per-series, not a // cross-series reduction (see #165's `Reduction`). reduction: Reduction::PerEntity, - aggs: vec![AggIntent::Quantile { + measures: vec![AggIntent::Quantile { col: None, q, accuracy, @@ -100,10 +99,10 @@ fn plan_is_archive(plan: &L4Plan) -> bool { } } -fn node_is_archive(node: &Rc) -> bool { +fn node_is_archive(node: &Rc) -> bool { match &node.expr { SummaryExpr::Logical(qe) => match qe.as_ref() { - QueryExpr::Aggregate { aggs, .. } => { + QueryExpr::Aggregate { measures: aggs, .. } => { aggs.iter().any(crate::intent_algebra::archive_only) } _ => false, @@ -131,10 +130,9 @@ fn bind_kll_quantile_basic() { // tie-break exercised separately by // `bind_picks_ddsketch_over_kll_when_eps_explicit` below. let expr = agg_quantile(0.99, AccuracyTarget::Epsilon(0.01)); - let cost_model = ForcedFamilyCostModel::new(AccuracyTarget::Epsilon(0.01), SummaryKind::Kll); - let node = - asap_plan::bind::implement_tree_in_with(&expr, &BindingScope::default(), &cost_model) - .expect("KLL should bind a Quantile{0.99, ε=0.01}"); + let cost_model = ForcedFamilyCostModel::new(AccuracyTarget::Epsilon(0.01), SketchKind::Kll); + let node = asap_aware_mapping::bind::implement_tree_with(&expr, &cost_model) + .expect("KLL should bind a Quantile{0.99, ε=0.01}"); match &node.expr { SummaryExpr::SummaryEstimate { query, @@ -142,14 +140,11 @@ fn bind_kll_quantile_basic() { } => { assert!(matches!(query, SketchQuery::Quantile { q } if *q == 0.99)); match &summary_input.expr { - SummaryExpr::SummaryAgg { - summary, - params, - child, - .. - } => { - assert_eq!(summary, &SummaryKind::Kll); - assert_eq!(params, &SummaryParams::Kll { k: 200 }); + SummaryExpr::SummaryAgg { family, child, .. } => { + assert_eq!( + family, + &SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) + ); assert!(matches!(child.expr, SummaryExpr::Logical(_))); } other => panic!("expected SummaryAgg, got {other:?}"), @@ -165,10 +160,9 @@ fn bind_ddsketch_quantile_basic() { // of the retired `BindDDSketchOnQuantile` rule struct. let expr = agg_quantile(0.99, AccuracyTarget::Epsilon(0.01)); let cost_model = - ForcedFamilyCostModel::new(AccuracyTarget::Epsilon(0.01), SummaryKind::DDSketch); - let node = - asap_plan::bind::implement_tree_in_with(&expr, &BindingScope::default(), &cost_model) - .expect("DDSketch should bind a Quantile{0.99, ε=0.01}"); + ForcedFamilyCostModel::new(AccuracyTarget::Epsilon(0.01), SketchKind::DDSketch); + let node = asap_aware_mapping::bind::implement_tree_with(&expr, &cost_model) + .expect("DDSketch should bind a Quantile{0.99, ε=0.01}"); match &node.expr { SummaryExpr::SummaryEstimate { query, @@ -176,15 +170,15 @@ fn bind_ddsketch_quantile_basic() { } => { assert!(matches!(query, SketchQuery::Quantile { q } if *q == 0.99)); match &summary_input.expr { - SummaryExpr::SummaryAgg { summary, params, .. } => { - assert_eq!(summary, &SummaryKind::DDSketch); - match params { - SummaryParams::DDSketch { alpha } => { - assert!((alpha - 0.01).abs() < 1e-12) - } - other => panic!("expected DDSketch params, got {other:?}"), + SummaryExpr::SummaryAgg { family, .. } => match family { + SummaryFamilyType::Sketch( + SketchKind::DDSketch, + SketchParams::DDSketch { alpha }, + ) => { + assert!((alpha - 0.01).abs() < 1e-12) } - } + other => panic!("expected DDSketch family, got {other:?}"), + }, other => panic!("expected SummaryAgg, got {other:?}"), } } @@ -204,11 +198,10 @@ fn bind_picks_ddsketch_over_kll_when_eps_explicit() { match bound { PhysicalExpr::Committed(L4Plan::Summary(node)) => match &node.expr { SummaryExpr::SummaryEstimate { summary_input, .. } => match &summary_input.expr { - SummaryExpr::SummaryAgg { summary, .. } => { - assert_eq!( - summary, - &SummaryKind::DDSketch, - "dispatcher should pick DDSketch over KLL on ε-driven Quantile" + SummaryExpr::SummaryAgg { family, .. } => { + assert!( + matches!(family, SummaryFamilyType::Sketch(SketchKind::DDSketch, _)), + "dispatcher should pick DDSketch over KLL on ε-driven Quantile, got {family:?}" ); } other => panic!("expected SummaryAgg, got {other:?}"), @@ -225,19 +218,19 @@ fn agg_topk(k: usize, accuracy: AccuracyTarget) -> QueryExpr { // A ranking always reduces — empty `by` ranks the whole input, // never per-entity (see `lower.rs`'s `LQueryExpr::TopK` handling). reduction: Reduction::by(vec![]), - aggs: vec![AggIntent::TopK { k, accuracy }], + measures: vec![AggIntent::TopK { k, accuracy }], output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), } } -/// Pull the bound `(SummaryKind, w, d)` out of a top-k binding. -/// `SummaryKind` (unlike the retired `sketch_algebra::SketchKind`) -/// promotes `with_heap` to kind identity — the top-k cost model always -/// binds `CmsWithHeap`/`CountSketchWithHeap` for a top-k intent, never -/// the bare kind, so there's no separate heap flag to return anymore. -fn topk_binding_family(bound: &PhysicalExpr) -> (SummaryKind, u32, u32) { +/// Pull the bound `(SketchKind, w, d)` out of a top-k binding. +/// `SketchKind` promotes `with_heap` to kind identity — the top-k cost +/// model always binds `CmsWithHeap`/`CountSketchWithHeap` for a top-k +/// intent, never the bare kind, so there's no separate heap flag to +/// return anymore. +fn topk_binding_family(bound: &PhysicalExpr) -> (SketchKind, u32, u32) { match bound { PhysicalExpr::Committed(L4Plan::Summary(node)) => match &node.expr { SummaryExpr::SummaryEstimate { @@ -246,15 +239,17 @@ fn topk_binding_family(bound: &PhysicalExpr) -> (SummaryKind, u32, u32) { } => { assert!(matches!(query, SketchQuery::TopK { k } if *k == 10)); match &summary_input.expr { - SummaryExpr::SummaryAgg { summary, params, .. } => match params { - SummaryParams::CmsWithHeap { width, depth, .. } => { - (summary.clone(), *width, *depth) - } - SummaryParams::CountSketchWithHeap { width, depth, .. } => { - (summary.clone(), *width, *depth) - } + SummaryExpr::SummaryAgg { family, .. } => match family { + SummaryFamilyType::Sketch( + kind @ SketchKind::CmsWithHeap, + SketchParams::CmsWithHeap { width, depth, .. }, + ) + | SummaryFamilyType::Sketch( + kind @ SketchKind::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { width, depth, .. }, + ) => (kind.clone(), *width, *depth), other => { - panic!("expected CmsWithHeap/CountSketchWithHeap params, got {other:?}") + panic!("expected CmsWithHeap/CountSketchWithHeap family, got {other:?}") } }, other => panic!("expected SummaryAgg, got {other:?}"), @@ -280,7 +275,7 @@ fn bind_cms_topk_loose_recall_picks_cms_heap() { let (kind, w, d) = topk_binding_family(&bound); assert_eq!( kind, - SummaryKind::CmsWithHeap, + SketchKind::CmsWithHeap, "loose-recall top-k must bind the cheap CMS-with-heap, not CountSketch" ); assert!(w >= 2); @@ -294,7 +289,7 @@ fn bind_cms_topk_loose_recall_picks_cms_heap() { /// NOTE — behavior change forced by the new binder, not just a rename: /// the old fixture used `AggIntent::TopK{accuracy: Exact}` (the intent's /// OWN accuracy) to signal "tight/exact-recall". Under -/// `asap_plan::boundary::implementation_for_with`, the per-intent +/// `asap_aware_mapping::boundary::implementation_for_with`, the per-intent /// summary-vs-exact boundary decision checks the intent's own `accuracy` /// field FIRST: `TopK{accuracy: Exact}` now declines to bind at all /// (`SummaryExpr::Logical`) rather than reaching the cost model's @@ -315,7 +310,7 @@ fn bind_cms_topk_tight_recall_picks_countsketch() { let (kind, w, d) = topk_binding_family(&bound); assert_eq!( kind, - SummaryKind::CountSketchWithHeap, + SketchKind::CountSketchWithHeap, "exact-rank top-k must bind the unbiased CountSketch-with-heap" ); assert!(w >= 2); @@ -331,8 +326,8 @@ fn bind_cms_topk_tight_recall_picks_countsketch() { fn bind_cms_topk_picks_cost_min_meeting_sla() { use crate::optimizer::cost::wire::WireCostTable; let table = WireCostTable::default(); - let cms = table.for_kind(&SummaryKind::Cms).per_flush(); - let cs = table.for_kind(&SummaryKind::CountSketch).per_flush(); + let cms = table.for_kind(&SketchKind::Cms).per_flush(); + let cs = table.for_kind(&SketchKind::CountSketch).per_flush(); assert!( cms < cs, "CMS-heap ({cms} B) must be cheaper than CountSketch ({cs} B) on the wire" @@ -354,14 +349,14 @@ fn bind_cms_topk_picks_cost_min_meeting_sla() { cms.min(cs), "must pick the cost-min family that meets the SLA" ); - assert_eq!(kind, SummaryKind::CmsWithHeap); + assert_eq!(kind, SketchKind::CmsWithHeap); } #[test] fn bind_hll_cardinality_basic() { let expr = QueryExpr::Aggregate { reduction: Reduction::PerEntity, - aggs: vec![AggIntent::Cardinality { + measures: vec![AggIntent::Cardinality { col: None, accuracy: AccuracyTarget::Epsilon(0.01), }], @@ -378,18 +373,18 @@ fn bind_hll_cardinality_basic() { } => { assert!(matches!(query, SketchQuery::Cardinality)); match &summary_input.expr { - SummaryExpr::SummaryAgg { summary, params, .. } => { - assert_eq!(summary, &SummaryKind::Hll); - match params { - SummaryParams::Hll { precision } => { - assert!( - *precision >= 12, - "ε=0.01 should land on at least precision 12 (~1.6%) per the rung table" - ); - } - other => panic!("expected HllParams, got {other:?}"), + SummaryExpr::SummaryAgg { family, .. } => match family { + SummaryFamilyType::Sketch( + SketchKind::Hll, + SketchParams::Hll { precision }, + ) => { + assert!( + *precision >= 12, + "ε=0.01 should land on at least precision 12 (~1.6%) per the rung table" + ); } - } + other => panic!("expected Hll family, got {other:?}"), + }, other => panic!("expected SummaryAgg, got {other:?}"), } } @@ -403,16 +398,16 @@ fn bind_hll_cardinality_basic() { fn sum_now_binds_to_exact_agg_after_pr_6_followup() { // `AggIntent::Sum` binds to a bare `SummaryAgg` with `summary: // SummaryKind::Sum` and no `SummaryEstimate` wrapper (the partial - // state *is* the value — see `asap_plan::bind`'s module docs). The + // state *is* the value — see `asap_aware_mapping::bind`'s module docs). The // old locally-defined `PhysicalExpr::ExactAgg { agg_type, .. }` // variant (and `asap_types::AggregationType`) no longer exist at - // the L4 IR level: `asap_sketch::SummaryExpr` unifies exact + // the L4 IR level: `planner_types::post_asap::SummaryExpr` unifies exact // accumulators and approximate sketches into the same `SummaryAgg` // node shape, keyed by `SummaryKind` (see `physical_expr.rs`'s // module docs). let expr = QueryExpr::Aggregate { reduction: Reduction::PerEntity, - aggs: vec![AggIntent::Sum { col: None }], + measures: vec![AggIntent::Sum { col: None }], output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), @@ -420,13 +415,12 @@ fn sum_now_binds_to_exact_agg_after_pr_6_followup() { let bound = bind_query_expr(&expr, AccuracyTarget::Exact).expect("no error"); match bound { PhysicalExpr::Committed(L4Plan::Summary(node)) => match &node.expr { - SummaryExpr::SummaryAgg { summary, params, .. } => { + SummaryExpr::SummaryAgg { family, .. } => { assert_eq!( - summary, - &SummaryKind::Sum, + family, + &SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), "Sum should bind to SummaryAgg(Sum)" ); - assert_eq!(params, &SummaryParams::Sum); } other => panic!("expected bare SummaryAgg(Sum), got {other:?}"), }, @@ -468,7 +462,7 @@ fn bind_exact_accuracy_disables_quantile_binding() { fn phase_b_pattern_only_temporal_quantile_binds_to_sketch() { let expr = QueryExpr::Aggregate { reduction: Reduction::PerEntity, - aggs: vec![AggIntent::Quantile { + measures: vec![AggIntent::Quantile { col: None, q: 0.99, accuracy: AccuracyTarget::Epsilon(0.01), @@ -486,8 +480,11 @@ fn phase_b_pattern_only_temporal_quantile_binds_to_sketch() { } => { assert!(matches!(query, SketchQuery::Quantile { .. })); match &summary_input.expr { - SummaryExpr::SummaryAgg { summary, .. } => { - assert!(matches!(summary, SummaryKind::Kll | SummaryKind::DDSketch)); + SummaryExpr::SummaryAgg { family, .. } => { + assert!(matches!( + family, + SummaryFamilyType::Sketch(SketchKind::Kll | SketchKind::DDSketch, _) + )); } other => panic!("expected SummaryAgg under SummaryEstimate, got {other:?}"), } @@ -509,7 +506,7 @@ fn phase_b_pattern_only_temporal_quantile_binds_to_sketch() { fn phase_b_pattern_only_temporal_sum_binds_to_exact_agg() { let expr = QueryExpr::Aggregate { reduction: Reduction::PerEntity, - aggs: vec![AggIntent::Sum { col: None }], + measures: vec![AggIntent::Sum { col: None }], output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), @@ -517,8 +514,11 @@ fn phase_b_pattern_only_temporal_sum_binds_to_exact_agg() { let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).unwrap(); match bound { PhysicalExpr::Committed(L4Plan::Summary(node)) => match &node.expr { - SummaryExpr::SummaryAgg { summary, .. } => { - assert_eq!(summary, &SummaryKind::Sum); + SummaryExpr::SummaryAgg { family, .. } => { + assert_eq!( + family, + &SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) + ); } other => panic!("expected SummaryAgg(Sum), got {other:?}"), }, @@ -538,7 +538,7 @@ fn phase_b_pattern_only_temporal_sum_binds_to_exact_agg() { fn phase_b_pattern_only_spatial_aggregate_binds_to_multiple_sum() { let expr = QueryExpr::Aggregate { reduction: Reduction::by(vec![1]), // service column - aggs: vec![AggIntent::Sum { col: None }], + measures: vec![AggIntent::Sum { col: None }], output_names: Vec::new(), having: None, child: Box::new(ts_scan()), @@ -547,9 +547,12 @@ fn phase_b_pattern_only_spatial_aggregate_binds_to_multiple_sum() { match bound { PhysicalExpr::Committed(L4Plan::Summary(node)) => match &node.expr { SummaryExpr::SummaryAgg { - summary, reduction, .. + family, reduction, .. } => { - assert_eq!(summary, &SummaryKind::Sum); + assert_eq!( + family, + &SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) + ); assert_eq!( reduction.group_keys().map(|k| k.keys()), Some(&[1][..]), @@ -573,7 +576,7 @@ fn phase_b_pattern_only_spatial_aggregate_binds_to_multiple_sum() { fn phase_b_pattern_temporal_and_spatial_combined_binds_to_multiple_increase() { let expr = QueryExpr::Aggregate { reduction: Reduction::by(vec![1]), - aggs: vec![AggIntent::Rate], + measures: vec![AggIntent::Rate], output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), @@ -582,9 +585,12 @@ fn phase_b_pattern_temporal_and_spatial_combined_binds_to_multiple_increase() { match bound { PhysicalExpr::Committed(L4Plan::Summary(node)) => match &node.expr { SummaryExpr::SummaryAgg { - summary, reduction, .. + family, reduction, .. } => { - assert_eq!(summary, &SummaryKind::Increase); + assert_eq!( + family, + &SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase) + ); assert_eq!(reduction.group_keys().map(|k| k.keys()), Some(&[1][..])); } other => panic!("expected SummaryAgg(Increase, by=[1]), got {other:?}"), @@ -613,7 +619,7 @@ fn phase_b_pattern_archive_only_routes_to_archive() { ); let expr = QueryExpr::Aggregate { reduction: Reduction::PerEntity, - aggs: vec![intent.clone()], + measures: vec![intent.clone()], output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), @@ -624,7 +630,7 @@ fn phase_b_pattern_archive_only_routes_to_archive() { match bound { PhysicalExpr::Committed(L4Plan::Summary(node)) => match &node.expr { SummaryExpr::Logical(qe) => match qe.as_ref() { - QueryExpr::Aggregate { aggs, .. } => { + QueryExpr::Aggregate { measures: aggs, .. } => { assert_eq!(aggs, &vec![intent]); } other => panic!("expected Aggregate, got {other:?}"), @@ -673,7 +679,7 @@ fn phase_b_e2e_quantile_over_time_binds_to_quantile_sketch() { ); let kind = crate::emit::extract_root_sketch_kind(&bound); assert!( - matches!(kind, Some(SummaryKind::Kll) | Some(SummaryKind::DDSketch)), + matches!(kind, Some(SketchKind::Kll) | Some(SketchKind::DDSketch)), "expected quantile summary family, got {kind:?}" ); assert!( @@ -797,7 +803,7 @@ fn phase_b_e2e_archive_only_e2e_binding() { let intent = AggIntent::Absent; let expr = QueryExpr::Aggregate { reduction: Reduction::PerEntity, - aggs: vec![intent.clone()], + measures: vec![intent.clone()], output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), @@ -840,7 +846,7 @@ fn phase_b_archive_only_intents_round_trip_through_binder() { for intent in intents { let expr = QueryExpr::Aggregate { reduction: Reduction::PerEntity, - aggs: vec![intent.clone()], + measures: vec![intent.clone()], output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), @@ -850,7 +856,7 @@ fn phase_b_archive_only_intents_round_trip_through_binder() { match bound { PhysicalExpr::Committed(L4Plan::Summary(node)) => match &node.expr { SummaryExpr::Logical(qe) => match qe.as_ref() { - QueryExpr::Aggregate { aggs, .. } => { + QueryExpr::Aggregate { measures: aggs, .. } => { assert_eq!(aggs.len(), 1); assert!( crate::intent_algebra::archive_only(&aggs[0]), @@ -888,7 +894,7 @@ fn frequency_extension_binds_cms() { let intent = crate::intent_algebra::frequency(AccuracyTarget::Epsilon(0.01), None); let expr = QueryExpr::Aggregate { reduction: Reduction::PerEntity, - aggs: vec![intent], + measures: vec![intent], output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), @@ -907,7 +913,7 @@ fn frequency_extension_binds_cms() { matches!( &summary_input.expr, SummaryExpr::SummaryAgg { - summary: SummaryKind::Cms, + family: SummaryFamilyType::Sketch(SketchKind::Cms, _), .. } ), diff --git a/control_plane/src/threshold_alloc.rs b/control_plane/src/threshold_alloc.rs index ac27723f..03cf9867 100644 --- a/control_plane/src/threshold_alloc.rs +++ b/control_plane/src/threshold_alloc.rs @@ -47,7 +47,10 @@ mod tests { #[test] fn degenerate_dims_return_infinity() { - assert_eq!(f2_isotropic_threshold(0.1, 1000.0, 4, 0, 256), f64::INFINITY); + assert_eq!( + f2_isotropic_threshold(0.1, 1000.0, 4, 0, 256), + f64::INFINITY + ); assert_eq!(f2_isotropic_threshold(0.1, 1000.0, 4, 5, 0), f64::INFINITY); } } diff --git a/control_plane/src/types.rs b/control_plane/src/types.rs index b7cdc325..881b60e2 100644 --- a/control_plane/src/types.rs +++ b/control_plane/src/types.rs @@ -197,37 +197,44 @@ impl std::fmt::Display for SketchType { // Moved from the retired `sketch_algebra::sketch_params` (Stage 3 of the // sketch-identity unification — see // scratchpad/artifacts/enum-unification-plan.md) when `sketch_algebra::SketchKind` -// was replaced by `asap_sketch::SummaryKind`. `SketchType::from(SummaryKind)` -// only covers the 5 canonical families this legacy type has room for — -// callers (`optimizer::engine::load_sketch_capabilities`) only ever feed it -// kinds sourced from `default_capability_table`/`load_capability_overrides`, -// which are exhaustively those 5. -impl From for SketchType { - fn from(k: asap_sketch::SummaryKind) -> Self { +// was replaced by `planner_types::post_asap::SummaryKind` (later `planner_types::post_asap:: +// SketchKind` once ASAPPlanner split the old flat `SummaryKind` per-family — +// see control_plane/docs/design-asapplanner-pin-migration.md). `SketchType:: +// from(SketchKind)` only covers the 5 canonical families this legacy type has +// room for — callers (`optimizer::engine::load_sketch_capabilities`) only +// ever feed it kinds sourced from `default_capability_table`/ +// `load_capability_overrides`, which are exhaustively those 5, all +// approximate-sketch families (no exact-accumulator kind is ever passed +// here, so the post-split `SketchKind` — never `ExactKind` — is the right +// upstream type to convert from/to). +impl From for SketchType { + fn from(k: planner_types::post_asap::SketchKind) -> Self { + use planner_types::post_asap::SketchKind; match k { - asap_sketch::SummaryKind::Kll => SketchType::KLL, - asap_sketch::SummaryKind::DDSketch => SketchType::DDSketch, - asap_sketch::SummaryKind::Hll => SketchType::HLL, - asap_sketch::SummaryKind::Cms => SketchType::CountMinSketch, - asap_sketch::SummaryKind::CountSketch => SketchType::CountSketch, + SketchKind::Kll => SketchType::KLL, + SketchKind::DDSketch => SketchType::DDSketch, + SketchKind::Hll => SketchType::HLL, + SketchKind::Cms => SketchType::CountMinSketch, + SketchKind::CountSketch => SketchType::CountSketch, other => unreachable!( - "SketchType::from(SummaryKind): no legacy equivalent for {other:?} \ + "SketchType::from(SketchKind): no legacy equivalent for {other:?} \ (callers only pass the 5 canonical families)" ), } } } -/// Inverse of `From for SketchType`. Round-trippable for the 5 -/// canonical families: `SummaryKind::from(SketchType::from(k)) == k`. -impl From for asap_sketch::SummaryKind { +/// Inverse of `From for SketchType`. Round-trippable for the 5 +/// canonical families: `SketchKind::from(SketchType::from(k)) == k`. +impl From for planner_types::post_asap::SketchKind { fn from(t: SketchType) -> Self { + use planner_types::post_asap::SketchKind; match t { - SketchType::KLL => asap_sketch::SummaryKind::Kll, - SketchType::DDSketch => asap_sketch::SummaryKind::DDSketch, - SketchType::HLL => asap_sketch::SummaryKind::Hll, - SketchType::CountMinSketch => asap_sketch::SummaryKind::Cms, - SketchType::CountSketch => asap_sketch::SummaryKind::CountSketch, + SketchType::KLL => SketchKind::Kll, + SketchType::DDSketch => SketchKind::DDSketch, + SketchType::HLL => SketchKind::Hll, + SketchType::CountMinSketch => SketchKind::Cms, + SketchType::CountSketch => SketchKind::CountSketch, } } } diff --git a/control_plane/src/types_v2.rs b/control_plane/src/types_v2.rs index 5cad49aa..86b18ae8 100644 --- a/control_plane/src/types_v2.rs +++ b/control_plane/src/types_v2.rs @@ -49,18 +49,20 @@ pub enum QueryLanguage { /// Per-target accuracy SLA, in the typed form `design.md` §6 calls for. /// /// Phase 1b (docs/migration-plan-backend-plan.md): re-exported from -/// `asap_ir::types` rather than defined locally -- `AggIntent`'s -/// `accuracy` fields are typed against ASAPController's `AccuracyTarget`, -/// so keeping a separate local type here would force a conversion at -/// every one of the ~400 `AggIntent` call sites. Two real differences -/// from the pre-merge local type, both confirmed safe to fold on -/// (no external YAML/JSON persists the old wire shape -- only one -/// in-Rust test fixture, `pipeline.rs`, needed updating): +/// `planner_types::types` (formerly `asap_ir::types` -- ASAPPlanner +/// consolidated `asap-ir` into `asap-types`, see +/// control_plane/docs/design-asapplanner-pin-migration.md) rather than +/// defined locally -- `AggIntent`'s `accuracy` fields are typed against +/// ASAPPlanner's `AccuracyTarget`, so keeping a separate local type here +/// would force a conversion at every one of the ~400 `AggIntent` call +/// sites. Two real differences from the pre-merge local type, both +/// confirmed safe to fold on (no external YAML/JSON persists the old wire +/// shape -- only one in-Rust test fixture, `pipeline.rs`, needed updating): /// - Wire shape: was `#[serde(tag = "kind", content = "value")]` /// (`{"kind": "epsilon", "value": 0.02}`); now serde's default /// externally-tagged representation (`{"Epsilon": 0.02}`). /// - `EpsilonDelta`'s second field is `epsilon`, not `eps`. -pub use asap_ir::types::AccuracyTarget; +pub use planner_types::types::AccuracyTarget; /// Translate the legacy `accuracy_sla: f64` field -- a fractional /// "1.0 = exact, 0.0 = anything goes" SLA -- into the typed form. diff --git a/control_plane/src/workload.rs b/control_plane/src/workload.rs index 388e914b..900dd194 100644 --- a/control_plane/src/workload.rs +++ b/control_plane/src/workload.rs @@ -433,7 +433,10 @@ impl WorkloadRegistry { /// `StreamingConfig` (the coordinator then derives the ε-floor `p`). Shared /// across registry clones via the `Arc>` overlay. pub fn insert_runtime(&self, entry: WorkloadEntry) { - let mut rt = self.runtime.write().expect("runtime registry lock poisoned"); + let mut rt = self + .runtime + .write() + .expect("runtime registry lock poisoned"); rt.retain(|e| e.metric_name != entry.metric_name); rt.push(entry); } @@ -445,16 +448,16 @@ impl WorkloadRegistry { Ok(entries) => { info!(path, count = entries.len(), "loaded workload registry"); Self { - entries, - runtime: Default::default(), - } + entries, + runtime: Default::default(), + } } Err(e) => { warn!(path, error = %e, "invalid workloads YAML; using empty registry"); Self { - entries: vec![], - runtime: Default::default(), - } + entries: vec![], + runtime: Default::default(), + } } }, Err(_) => { @@ -641,46 +644,46 @@ mod tests { #[test] fn for_role_filters_correctly() { let reg = WorkloadRegistry::from_entries(vec![ - WorkloadEntry { - metric_name: "a".into(), - query_string: None, - accuracy_sla: 0.01, - assign_to_role: "agent".into(), - sketch_family_override: None, - target_path: None, - grouping_labels: vec![], - sample_p: 1.0, - distinct_keys_per_window: None, - item_label: None, - monitor: None, - }, - WorkloadEntry { - metric_name: "b".into(), - query_string: None, - accuracy_sla: 0.05, - assign_to_role: "backend".into(), - sketch_family_override: None, - target_path: None, - grouping_labels: vec![], - sample_p: 1.0, - distinct_keys_per_window: None, - item_label: None, - monitor: None, - }, - WorkloadEntry { - metric_name: "c".into(), - query_string: None, - accuracy_sla: 0.02, - assign_to_role: "agent".into(), - sketch_family_override: None, - target_path: None, - grouping_labels: vec![], - sample_p: 1.0, - distinct_keys_per_window: None, - item_label: None, - monitor: None, - }, - ]); + WorkloadEntry { + metric_name: "a".into(), + query_string: None, + accuracy_sla: 0.01, + assign_to_role: "agent".into(), + sketch_family_override: None, + target_path: None, + grouping_labels: vec![], + sample_p: 1.0, + distinct_keys_per_window: None, + item_label: None, + monitor: None, + }, + WorkloadEntry { + metric_name: "b".into(), + query_string: None, + accuracy_sla: 0.05, + assign_to_role: "backend".into(), + sketch_family_override: None, + target_path: None, + grouping_labels: vec![], + sample_p: 1.0, + distinct_keys_per_window: None, + item_label: None, + monitor: None, + }, + WorkloadEntry { + metric_name: "c".into(), + query_string: None, + accuracy_sla: 0.02, + assign_to_role: "agent".into(), + sketch_family_override: None, + target_path: None, + grouping_labels: vec![], + sample_p: 1.0, + distinct_keys_per_window: None, + item_label: None, + monitor: None, + }, + ]); assert_eq!(reg.for_role("agent").len(), 2); assert_eq!(reg.for_role("backend").len(), 1); assert_eq!(reg.first_for_role("agent").unwrap().metric_name, "a"); diff --git a/crates/asap_otel_proto/tests/coupling_wire_compat.rs b/crates/asap_otel_proto/tests/coupling_wire_compat.rs index c4266964..d585a34b 100644 --- a/crates/asap_otel_proto/tests/coupling_wire_compat.rs +++ b/crates/asap_otel_proto/tests/coupling_wire_compat.rs @@ -3,13 +3,14 @@ //! (asap-precompute-go/monitor/grpcclient/monitorpb TestCouplingFixture) and //! MUST decode field-for-field here via prost — proving SlackGrant.sample_p //! (coord->edge) and MonitorReport.rate (edge->coord) cross the wire. -use asap_otel_proto::monitor::v1::{ - coord_to_edge, edge_to_coord, CoordToEdge, EdgeToCoord, -}; +use asap_otel_proto::monitor::v1::{coord_to_edge, edge_to_coord, CoordToEdge, EdgeToCoord}; use prost::Message; fn unhex(s: &str) -> Vec { - (0..s.len()).step_by(2).map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()).collect() + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()) + .collect() } #[test] @@ -46,11 +47,18 @@ fn rust_grant_sample_p_emit_for_go() { use asap_otel_proto::monitor::v1::SlackGrant; let env = CoordToEdge { msg: Some(coord_to_edge::Msg::Grant(SlackGrant { - agg_id: 99, round: 4, local_slack: 1.0, window_start_ms: 1_700_000_000_000, - sample_p: 0.3, ..Default::default() + agg_id: 99, + round: 4, + local_slack: 1.0, + window_start_ms: 1_700_000_000_000, + sample_p: 0.3, + ..Default::default() })), }; let mut buf = Vec::new(); env.encode(&mut buf).unwrap(); - println!("RUST_GRANT_HEX={}", buf.iter().map(|b| format!("{:02x}", b)).collect::()); + println!( + "RUST_GRANT_HEX={}", + buf.iter().map(|b| format!("{:02x}", b)).collect::() + ); } diff --git a/crates/asap_types/Cargo.toml b/crates/asap_types/Cargo.toml index e343d8a0..f395bf8d 100644 --- a/crates/asap_types/Cargo.toml +++ b/crates/asap_types/Cargo.toml @@ -12,17 +12,22 @@ anyhow.workspace = true clap.workspace = true xxhash-rust = { version = "0.8", features = ["xxh64"] } -# First cross-crate ASAPController dependency for data_plane (transitively, -# via this crate): WindowType -> asap_ir::intent_algebra::query_expr::WindowKind +# First cross-crate ASAPPlanner dependency for data_plane (transitively, +# via this crate): WindowType -> planner_types::pre_asap::query_expr::WindowKind # unification (scratchpad/artifacts/enum-unification-plan.md). Pin matches -# control_plane's -- see control_plane/Cargo.toml's comment for the rationale. -asap-ir = { git = "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/ProjectASAP/ASAPController", rev = "cc18c9872bbaf0cadf43566892c3974ec9948eba" } -# Step 5 of the sketch-identity unification (see -# scratchpad/artifacts/enum-unification-plan.md): `AccumulatorSpec` +# control_plane's -- see control_plane/Cargo.toml's comment for the +# rationale, including why this is aliased to `planner-types` rather than +# depended on as `asap-types` directly (this crate IS `asap_types` -- +# depending on the upstream package under its own name would collide). +# +# `asap-ir` and `asap-sketch` (this repo's old two separate pins) merged +# into this one upstream crate (ASAPPlanner#198); `pre_asap`/`post_asap` +# modules replace the old L3/L4 split. `AccumulatorSpec` # (accumulator_spec.rs) converges data_plane's identity representation -# onto ASAPController's `SummaryKind`/`SummaryParams`, same as -# control_plane already does (Stage 3, merged). Pin MUST match -# control_plane's pin exactly (`control_plane/Cargo.toml`) -- two -# different revs of the same git dependency in one workspace resolve -# to two distinct Rust types that won't unify. -asap-sketch = { git = "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/ProjectASAP/ASAPController", rev = "cc18c9872bbaf0cadf43566892c3974ec9948eba" } +# onto ASAPPlanner's `SketchKind`/`SketchParams` (formerly +# `SummaryKind`/`SummaryParams`, split per-family in ASAPPlanner#218), +# same as control_plane already does. Pin MUST match control_plane's pin +# exactly (`control_plane/Cargo.toml`) -- two different revs of the same +# git dependency in one workspace resolve to two distinct Rust types that +# won't unify. +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "cb70086b4c4a7ba89baf2516be81d0b192137a3a" } diff --git a/crates/asap_types/src/accumulator_spec.rs b/crates/asap_types/src/accumulator_spec.rs index 4ea86881..2100ccce 100644 --- a/crates/asap_types/src/accumulator_spec.rs +++ b/crates/asap_types/src/accumulator_spec.rs @@ -4,7 +4,7 @@ //! //! **Step 5 of the sketch-identity unification** (see //! `scratchpad/artifacts/enum-unification-plan.md`, §7-8). Converges -//! accumulator *identity* onto ASAPController's `asap_sketch::SummaryKind` +//! accumulator *identity* onto ASAPController's `planner_types::post_asap::SummaryKind` //! / `SummaryParams` — the same representation `control_plane` already //! uses as of Stage 3 (merged) — extended with the one axis that //! representation doesn't have: keyed-vs-unkeyed grouping, which @@ -48,8 +48,24 @@ //! //! ## What doesn't fit `SummaryKind`/`SummaryParams` //! -//! `asap_sketch`'s types are ASAPController's, not ours to extend from -//! this repo, and two data_plane-specific details don't fit them: +//! `SummaryKind`/`SummaryParams` were ASAPController's (now ASAPPlanner's) +//! types, not ours to extend from this repo, until ASAPPlanner split them +//! into a per-family `(ExactKind, SketchKind, SamplingKind, ...)` union +//! (ASAPPlanner#218) with no single flat type spanning both exact and +//! approximate accumulator identity anymore — see +//! `control_plane/docs/design-asapplanner-pin-migration.md`. This +//! module's own `AccumulatorSpec::kind`/`params` dispatch (below, and its +//! ~25 call sites in `data_plane::precompute_engine::accumulator_factory`) +//! never needed that pre-ASAP/post-ASAP distinction in the first place — +//! it's a flat "which concrete Rust accumulator struct to construct" +//! question, entirely internal to this workspace. Rather than thread a +//! two-level `Exact(ExactKind) | Sketch(SketchKind)` wrapper through every +//! one of those call sites for a distinction they don't care about, +//! `SummaryKind`/`SummaryParams` are now vendored here as local types, +//! same 14-variant shape as before the split (mirrors the same call this +//! workspace made for `WindowKind` — see `enums.rs`). +//! +//! Two data_plane-specific details don't fit them: //! //! - **Min/max direction.** `SummaryParams::MinMax` carries no fields — //! upstream doesn't model a direction axis. `accumulator_factory.rs` @@ -74,7 +90,240 @@ use crate::aggregation_config::AggregationConfig; use crate::key_by_label_names::KeyByLabelNames; use crate::AggregationType; -pub use asap_sketch::{SummaryKind, SummaryParams}; +/// Which accumulator family to run — identity only (no keyed/unkeyed +/// axis, no heap-vs-bare ambiguity: heap-bearing sketches are their own +/// variant, e.g. `CmsWithHeap` vs `Cms`). Vendored (see module doc): same +/// 14-variant shape ASAPController's pre-split `asap_sketch::SummaryKind` +/// had. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum SummaryKind { + Sum, + Count, + MinMax, + Increase, + Rate, + Kll, + Cms, + Hll, + DDSketch, + CmsWithHeap, + Kmv, + Theta, + CountSketch, + CountSketchWithHeap, +} + +impl SummaryKind { + /// True for the exact, zero-error mergeable-accumulator kinds (no + /// tuning parameters, upstream's old `ExactKind` set); false for the + /// approximate sketch families (upstream's old `SketchKind` set). + pub fn is_exact(&self) -> bool { + matches!( + self, + SummaryKind::Sum + | SummaryKind::Count + | SummaryKind::MinMax + | SummaryKind::Increase + | SummaryKind::Rate + ) + } +} + +/// Typed tuning parameters matching a [`SummaryKind`]. Vendored +/// alongside it (see module doc) — same shape as ASAPController's +/// pre-split `asap_sketch::SummaryParams`. +#[derive(Debug, Clone, PartialEq)] +pub enum SummaryParams { + Sum, + Count, + MinMax, + Increase, + Rate, + Kll { + k: u32, + }, + Cms { + width: u32, + depth: u32, + }, + Hll { + precision: u8, + }, + DDSketch { + alpha: f64, + }, + CmsWithHeap { + width: u32, + depth: u32, + heap_size: u32, + }, + Kmv { + k: u32, + }, + Theta { + k: u32, + }, + CountSketch { + width: u32, + depth: u32, + }, + CountSketchWithHeap { + width: u32, + depth: u32, + heap_size: u32, + }, +} + +/// Widen an upstream (post-ASAPPlanner#218) `ExactKind` into this crate's +/// flat [`SummaryKind`]. See the module doc for why the flat type exists. +impl From for SummaryKind { + fn from(k: planner_types::post_asap::ExactKind) -> Self { + use planner_types::post_asap::ExactKind as K; + match k { + K::Sum => SummaryKind::Sum, + K::Count => SummaryKind::Count, + K::MinMax => SummaryKind::MinMax, + K::Increase => SummaryKind::Increase, + K::Rate => SummaryKind::Rate, + } + } +} + +impl From for SummaryParams { + fn from(p: planner_types::post_asap::ExactParams) -> Self { + use planner_types::post_asap::ExactParams as P; + match p { + P::Sum => SummaryParams::Sum, + P::Count => SummaryParams::Count, + P::MinMax => SummaryParams::MinMax, + P::Increase => SummaryParams::Increase, + P::Rate => SummaryParams::Rate, + } + } +} + +/// Widen an upstream (post-ASAPPlanner#218) `SketchKind` -- always a real +/// approximate sketch, never exact -- into this crate's flat +/// [`SummaryKind`]. Used at the boundary where a wire/config type needs +/// to represent both exact and approximate accumulators in one field +/// (e.g. `control_plane`'s `BackendAggregation`) but the value in hand is +/// known-sketch. See the module doc for why the flat type exists. +impl From for SummaryKind { + fn from(k: planner_types::post_asap::SketchKind) -> Self { + use planner_types::post_asap::SketchKind as K; + match k { + K::Kll => SummaryKind::Kll, + K::Cms => SummaryKind::Cms, + K::Hll => SummaryKind::Hll, + K::DDSketch => SummaryKind::DDSketch, + K::CmsWithHeap => SummaryKind::CmsWithHeap, + K::Kmv => SummaryKind::Kmv, + K::Theta => SummaryKind::Theta, + K::CountSketch => SummaryKind::CountSketch, + K::CountSketchWithHeap => SummaryKind::CountSketchWithHeap, + } + } +} + +/// Narrow this crate's flat [`SummaryKind`] back down to an upstream +/// `SketchKind`, when it identifies a real approximate sketch. `None` for +/// the exact-accumulator variants (Sum/Count/MinMax/Increase/Rate), +/// which have no `SketchKind` equivalent -- the inverse of the widening +/// [`From`] impl above, fallible because that direction isn't total. +impl SummaryKind { + pub fn as_sketch_kind(&self) -> Option { + use planner_types::post_asap::SketchKind as K; + Some(match self { + SummaryKind::Kll => K::Kll, + SummaryKind::Cms => K::Cms, + SummaryKind::Hll => K::Hll, + SummaryKind::DDSketch => K::DDSketch, + SummaryKind::CmsWithHeap => K::CmsWithHeap, + SummaryKind::Kmv => K::Kmv, + SummaryKind::Theta => K::Theta, + SummaryKind::CountSketch => K::CountSketch, + SummaryKind::CountSketchWithHeap => K::CountSketchWithHeap, + SummaryKind::Sum + | SummaryKind::Count + | SummaryKind::MinMax + | SummaryKind::Increase + | SummaryKind::Rate => return None, + }) + } +} + +/// Same narrowing as [`SummaryKind::as_sketch_kind`], for the paired +/// params. `None` whenever `self` isn't a sketch-family variant. +impl SummaryParams { + pub fn as_sketch_params(&self) -> Option { + use planner_types::post_asap::SketchParams as P; + Some(match self.clone() { + SummaryParams::Kll { k } => P::Kll { k }, + SummaryParams::Cms { width, depth } => P::Cms { width, depth }, + SummaryParams::Hll { precision } => P::Hll { precision }, + SummaryParams::DDSketch { alpha } => P::DDSketch { alpha }, + SummaryParams::CmsWithHeap { + width, + depth, + heap_size, + } => P::CmsWithHeap { + width, + depth, + heap_size, + }, + SummaryParams::Kmv { k } => P::Kmv { k }, + SummaryParams::Theta { k } => P::Theta { k }, + SummaryParams::CountSketch { width, depth } => P::CountSketch { width, depth }, + SummaryParams::CountSketchWithHeap { + width, + depth, + heap_size, + } => P::CountSketchWithHeap { + width, + depth, + heap_size, + }, + SummaryParams::Sum + | SummaryParams::Count + | SummaryParams::MinMax + | SummaryParams::Increase + | SummaryParams::Rate => return None, + }) + } +} + +impl From for SummaryParams { + fn from(p: planner_types::post_asap::SketchParams) -> Self { + use planner_types::post_asap::SketchParams as P; + match p { + P::Kll { k } => SummaryParams::Kll { k }, + P::Cms { width, depth } => SummaryParams::Cms { width, depth }, + P::Hll { precision } => SummaryParams::Hll { precision }, + P::DDSketch { alpha } => SummaryParams::DDSketch { alpha }, + P::CmsWithHeap { + width, + depth, + heap_size, + } => SummaryParams::CmsWithHeap { + width, + depth, + heap_size, + }, + P::Kmv { k } => SummaryParams::Kmv { k }, + P::Theta { k } => SummaryParams::Theta { k }, + P::CountSketch { width, depth } => SummaryParams::CountSketch { width, depth }, + P::CountSketchWithHeap { + width, + depth, + heap_size, + } => SummaryParams::CountSketchWithHeap { + width, + depth, + heap_size, + }, + } + } +} /// Data_plane's typed replacement for /// `(aggregation_type, aggregation_sub_type, parameters)`: which diff --git a/crates/asap_types/src/enums.rs b/crates/asap_types/src/enums.rs index 1fc7d063..e72d4476 100644 --- a/crates/asap_types/src/enums.rs +++ b/crates/asap_types/src/enums.rs @@ -118,13 +118,57 @@ impl FromStr for CleanupPolicy { /// Window lifecycle/flush semantics for streaming aggregations. /// -/// Formerly a local `WindowType` (`Tumbling`/`Sliding`) enum. Retired in -/// favor of `asap_ir::intent_algebra::query_expr::WindowKind` directly — -/// same concept, plus a `Session` variant this workspace didn't have. -/// `Copy`/`Default`/`Hash`/`Display`/`FromStr` and -/// `#[serde(rename_all = "snake_case")]` were added upstream -/// (ASAPController PR #143) specifically so this re-export could replace -/// the old local type without touching any call site's behavior: same -/// `Tumbling` default, same lowercase `Display`/`FromStr` round-trip, same -/// wire format. -pub use asap_ir::intent_algebra::query_expr::WindowKind; +/// Formerly a local `WindowType` (`Tumbling`/`Sliding`) enum, retired in +/// favor of a re-export of `asap_ir::intent_algebra::query_expr::WindowKind` +/// (ASAPController PR #143 added `Copy`/`Default`/`Hash`/`Display`/ +/// `FromStr` and `#[serde(rename_all = "snake_case")]` upstream +/// specifically so that re-export could replace the old local type +/// without touching any call site's behavior). +/// +/// **Vendored back as a local type** (ASAPPlanner pin migration, see +/// `control_plane/docs/design-asapplanner-pin-migration.md`): ASAPPlanner +/// deleted `QueryExpr::Window` outright ("no producer exists" — issue +/// #181/#192) and with it every trace of a window-lifecycle `WindowKind` +/// concept; ASAPPlanner's own scope (a batch query-workload planner, not +/// a streaming execution engine) has no use for tumbling/sliding/session +/// flush semantics. This workspace's streaming aggregation config still +/// does, so the type moves back to being owned here — same shape as the +/// old re-export (`Tumbling` default, lowercase `Display`/`FromStr` +/// round-trip, same wire format), so none of this repo's ~145 call sites +/// needed to change. +#[derive( + Debug, Clone, Copy, Default, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] +#[serde(rename_all = "snake_case")] +pub enum WindowKind { + /// Fixed, non-overlapping windows (e.g. a new 30s bucket every 30s). + #[default] + Tumbling, + /// Overlapping windows that advance by less than their width. + Sliding, + /// Gap-based windows that close after a period of inactivity. + Session, +} + +impl fmt::Display for WindowKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + WindowKind::Tumbling => write!(f, "tumbling"), + WindowKind::Sliding => write!(f, "sliding"), + WindowKind::Session => write!(f, "session"), + } + } +} + +impl FromStr for WindowKind { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "tumbling" => Ok(WindowKind::Tumbling), + "sliding" => Ok(WindowKind::Sliding), + "session" => Ok(WindowKind::Session), + _ => Err(format!("Unknown window kind: '{s}'")), + } + } +} diff --git a/crates/asap_types/src/routing_index.rs b/crates/asap_types/src/routing_index.rs index 2ad2d09d..038d6dc6 100644 --- a/crates/asap_types/src/routing_index.rs +++ b/crates/asap_types/src/routing_index.rs @@ -57,7 +57,10 @@ impl RoutingIndex { for (fp, cfg) in registry.iter() { by_metric.entry(cfg.metric.clone()).or_default().push(*fp); } - Self { registry, by_metric } + Self { + registry, + by_metric, + } } /// Tier 1 — exact fingerprint lookup. Delegates to the underlying @@ -72,10 +75,7 @@ impl RoutingIndex { /// candidates" as a normal, expected outcome (capability miss → /// archive fallback), not a failure to report. pub fn candidates_for_metric(&self, metric: &str) -> &[PolicyFingerprint] { - self.by_metric - .get(metric) - .map(Vec::as_slice) - .unwrap_or(&[]) + self.by_metric.get(metric).map(Vec::as_slice).unwrap_or(&[]) } /// Live policy count (same as the underlying registry's). @@ -156,7 +156,10 @@ mod tests { None, ); let b = cfg("http_lat"); - assert_ne!(PolicyFingerprint::from_config(&a), PolicyFingerprint::from_config(&b)); + assert_ne!( + PolicyFingerprint::from_config(&a), + PolicyFingerprint::from_config(&b) + ); let idx = RoutingIndex::build(PolicyRegistry::from_configs(vec![a, b])); assert_eq!(idx.candidates_for_metric("http_lat").len(), 2); } @@ -172,13 +175,19 @@ mod tests { let a = cfg("http_lat"); let fp = PolicyFingerprint::from_config(&a); let idx = RoutingIndex::build(PolicyRegistry::from_configs(vec![a])); - assert_eq!(idx.get(fp).map(|c| c.metric.clone()), Some("http_lat".to_string())); - assert!(idx.get(PolicyFingerprint::from_config(&cfg("nope"))).is_none()); + assert_eq!( + idx.get(fp).map(|c| c.metric.clone()), + Some("http_lat".to_string()) + ); + assert!(idx + .get(PolicyFingerprint::from_config(&cfg("nope"))) + .is_none()); } #[test] fn len_and_is_empty_match_registry() { - let idx = RoutingIndex::build(PolicyRegistry::from_configs(Vec::::new())); + let idx = + RoutingIndex::build(PolicyRegistry::from_configs(Vec::::new())); assert!(idx.is_empty()); assert_eq!(idx.len(), 0); diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index 87a1882d..380358fe 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -14,27 +14,31 @@ asap_types.workspace = true control_plane = { path = "../control_plane" } # Step C (data_plane/docs/l4node-plan-executor-design.md): data_plane -# implements `asap_sketch::exec::SummaryExecutor`, which means naming -# `asap_sketch::{L4Node, SketchQuery, exec::*}` types directly -- not -# reachable through `control_plane::asap_tier_implement`'s public -# `Vec>` return type alone (Rust requires importing a type's -# defining crate to implement traits on it or match its variants, even -# when a dependency's function already returns that type). Pin MUST -# match control_plane's `asap-sketch` pin exactly -- same rule as +# vendors its own `SummaryExecutor`/`execute` (see +# `query_engines/asap_query_engine/summary_exec.rs` -- upstream deleted +# `asap_sketch::exec` in ASAPPlanner#190/#197; see +# control_plane/docs/design-asapplanner-pin-migration.md), which means +# naming `planner_types::post_asap::{SummaryNode, SketchQuery, ...}` +# types directly -- not reachable through +# `control_plane::asap_tier_implement`'s public `Vec>` +# return type alone (Rust requires importing a type's defining crate to +# implement traits on it or match its variants, even when a dependency's +# function already returns that type). Pin MUST match control_plane's +# `planner-types` pin exactly -- same rule as # `crates/asap_types/Cargo.toml`'s existing pin comment: two revs of the # same git dependency in one workspace resolve to two distinct Rust # types that won't unify. # -# Bumped to 64df20d (main tip, ASAPController#162, merged -- matches -# control_plane's own pin) to pick up -# `SketchQuery::PointCount.value: Option`, needed for the -# named-key PointCount readout. -asap-sketch = { git = "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/ProjectASAP/ASAPController", rev = "cc18c9872bbaf0cadf43566892c3974ec9948eba" } -# `asap_sketch::L4Node`'s own fields (`SummaryExpr::Logical(Box)`, -# `SummaryAgg { col: ColumnRef, reduction: Reduction, .. }`) are `asap-ir` -# types, not re-exported by `asap-sketch` -- `find_candidates` needs to -# walk/match them directly. Same pin-must-match rule as `asap-sketch` above. -asap-ir = { git = "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/ProjectASAP/ASAPController", rev = "cc18c9872bbaf0cadf43566892c3974ec9948eba" } +# Bumped to cb70086 (main tip, 2026-08-22) from cc18c987 -- ASAPPlanner's +# crate consolidation (ASAPPlanner#198) merged `asap-ir` and `asap-sketch` +# into one crate (Cargo package `asap-types`, depended on here under the +# `planner-types` key -- see control_plane/Cargo.toml's comment for why). +# `L4Node` renamed to `SummaryNode` (ASAPPlanner#217); its fields +# (`SummaryExpr::Logical(Box)`, `SummaryAgg { col: ColumnRef, +# reduction: Reduction, .. }`) are `pre_asap` types, in the same crate now +# (not a separate `asap-ir` import) -- `find_candidates` still needs to +# walk/match them directly. +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "cb70086b4c4a7ba89baf2516be81d0b192137a3a" } # Shared external (workspace) serde.workspace = true diff --git a/data_plane/src/bin/monitor_coordinator_harness.rs b/data_plane/src/bin/monitor_coordinator_harness.rs index 35e3963a..5ae094be 100644 --- a/data_plane/src/bin/monitor_coordinator_harness.rs +++ b/data_plane/src/bin/monitor_coordinator_harness.rs @@ -30,7 +30,10 @@ async fn main() -> Result<(), Box> { let port: u16 = args.next().and_then(|s| s.parse().ok()).unwrap_or(4319); let agg_id: u64 = args.next().and_then(|s| s.parse().ok()).unwrap_or(1); let tau: f64 = args.next().and_then(|s| s.parse().ok()).unwrap_or(100.0); - let window_ms: u64 = args.next().and_then(|s| s.parse().ok()).unwrap_or(3_600_000); + let window_ms: u64 = args + .next() + .and_then(|s| s.parse().ok()) + .unwrap_or(3_600_000); let timeout_secs: u64 = args.next().and_then(|s| s.parse().ok()).unwrap_or(30); let key: Vec = args.next().map(|s| s.into_bytes()).unwrap_or_default(); diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index b7d2b284..08ac102d 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -2251,10 +2251,8 @@ mod tests { async fn setup_test_server_with_backend_plan( hot_reload: Option, ) -> u16 { - let adapter_config = AdapterConfig::prometheus_promql( - "http://127.0.0.1:9999".to_string(), - false, - ); + let adapter_config = + AdapterConfig::prometheus_promql("http://127.0.0.1:9999".to_string(), false); let config = HttpServerConfig { port: 0, handle_http_requests: true, @@ -2504,7 +2502,9 @@ aggregations: let client = Client::new(); let initial = client - .get(format!("http://127.0.0.1:{server_port}/api/v1/backend-plan")) + .get(format!( + "http://127.0.0.1:{server_port}/api/v1/backend-plan" + )) .send() .await .expect("GET failed"); @@ -2520,7 +2520,9 @@ aggregations: let bytes = new_plan.encode_to_vec(); let post_resp = client - .post(format!("http://127.0.0.1:{server_port}/api/v1/backend-plan")) + .post(format!( + "http://127.0.0.1:{server_port}/api/v1/backend-plan" + )) .header("content-type", "application/x-protobuf") .body(bytes) .send() @@ -2536,7 +2538,9 @@ aggregations: assert_eq!(post_body["plan_id"], 7); let after = client - .get(format!("http://127.0.0.1:{server_port}/api/v1/backend-plan")) + .get(format!( + "http://127.0.0.1:{server_port}/api/v1/backend-plan" + )) .send() .await .expect("GET after swap failed"); @@ -2553,14 +2557,18 @@ aggregations: let client = Client::new(); let get_resp = client - .get(format!("http://127.0.0.1:{server_port}/api/v1/backend-plan")) + .get(format!( + "http://127.0.0.1:{server_port}/api/v1/backend-plan" + )) .send() .await .unwrap(); assert_eq!(get_resp.status(), reqwest::StatusCode::SERVICE_UNAVAILABLE); let post_resp = client - .post(format!("http://127.0.0.1:{server_port}/api/v1/backend-plan")) + .post(format!( + "http://127.0.0.1:{server_port}/api/v1/backend-plan" + )) .body("anything") .send() .await @@ -2577,7 +2585,9 @@ aggregations: let client = Client::new(); let resp = client - .post(format!("http://127.0.0.1:{server_port}/api/v1/backend-plan")) + .post(format!( + "http://127.0.0.1:{server_port}/api/v1/backend-plan" + )) .body(vec![0xFFu8, 0xFF, 0xFF]) .send() .await diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 18291a1f..9788ce0b 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -612,7 +612,9 @@ async fn main() -> Result<()> { // sample_p grant. Global-threshold alerting is retired (see // data_plane::monitor module docs) — this coordinator never fires one. let monitor_handle = if args.enable_monitor_coordinator { - use data_plane::monitor::{Functional, MonitorConfig, MonitorCoordinator, MonitorServiceImpl}; + use data_plane::monitor::{ + Functional, MonitorConfig, MonitorCoordinator, MonitorServiceImpl, + }; let specs: Vec = streaming_config .monitors() .iter() diff --git a/data_plane/src/monitor/coordinator.rs b/data_plane/src/monitor/coordinator.rs index 2c17a560..091e8a4b 100644 --- a/data_plane/src/monitor/coordinator.rs +++ b/data_plane/src/monitor/coordinator.rs @@ -325,7 +325,10 @@ mod tests { m.on_register("e1", 60_000, 0); let a = m.on_report("e1", 0, 100_000.0); let p = sample_p_in(&a, "e1"); - assert!(p < 1.0, "a lone high-rate edge should still be sampled, got p={p}"); + assert!( + p < 1.0, + "a lone high-rate edge should still be sampled, got p={p}" + ); } #[test] diff --git a/data_plane/src/monitor/server.rs b/data_plane/src/monitor/server.rs index 1fd5a682..f24b7962 100644 --- a/data_plane/src/monitor/server.rs +++ b/data_plane/src/monitor/server.rs @@ -257,7 +257,13 @@ impl MonitorCoordinator { // are no longer consulted — no countdown left to feed them // into; they still ride the wire message for compatibility). if let Some(actions) = self - .apply_report(rep.agg_id, rep.key.clone(), &rep.edge_id, rep.window_start_ms, rep.rate) + .apply_report( + rep.agg_id, + rep.key.clone(), + &rep.edge_id, + rep.window_start_ms, + rep.rate, + ) .await { self.dispatch(rep.agg_id, &rep.key, actions).await; @@ -367,8 +373,7 @@ mod reconfigure_tests { ); // Drop agg 1, keep agg 2 unchanged. - let (added, changed, removed) = - coord.reconfigure(vec![cfg(2, "s0", 5000.0)]).await; + let (added, changed, removed) = coord.reconfigure(vec![cfg(2, "s0", 5000.0)]).await; assert_eq!((added, changed, removed), (0, 0, 1)); assert_eq!(coord.monitor_count(), 1); } @@ -380,7 +385,10 @@ mod reconfigure_tests { let actions = coord .apply_register(7, b"s0".to_vec(), "edge-a", 15_000, 15_000) .await; - assert!(actions.is_some(), "register for a configured monitor succeeds"); + assert!( + actions.is_some(), + "register for a configured monitor succeeds" + ); assert_eq!(coord.monitors.lock().await.len(), 1); // Changing the spec (new τ) must drop the stale live state so the next diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index f33c215d..6edc6a3c 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -471,7 +471,7 @@ impl ASAPQueryEngine { // fix): every `result` now comes from `SummaryExecutor` // (`live_served_result` above), and `bind_query_expr`'s // lowering already realizes the full aggregation (including - // any `by (...)`) into the `L4Node` it executed — re-folding + // any `by (...)`) into the `SummaryNode` it executed — re-folding // here would double-apply it. combined_result = Some(result); } @@ -1185,7 +1185,7 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu // `result` now comes from `SummaryExecutor` above, and // `bind_query_expr`'s lowering already realizes the full // aggregation (including any `by (...)`) into the - // `L4Node` it executed — re-folding here would + // `SummaryNode` it executed — re-folding here would // double-apply it. combined_result = Some(result); } @@ -2939,7 +2939,7 @@ mod asap_tier_classify_tests { // Whichever query has a `rate(...)` call ANYWHERE in its tree // (bare `rate(...)` or composed `sum by (...) (rate(...))`) binds // to `AggIntent::Rate` and now maps to `ExactAgg(Increase)` -- - // matching `asap_plan::boundary::implementation_for`'s + // matching `asap_aware_mapping::boundary::implementation_for`'s // `SummaryKind::Rate` (ASAPController models Rate as its own // summary family; see `capability_for`'s module doc and // `Capability::is_satisfied_by`'s `sum_satisfies_increase` for @@ -3375,7 +3375,7 @@ mod outer_agg_integration_tests { /// -- identity here, since each zone already has one row). There's no /// equivalent in `SummaryExecutor`'s single-tree-bind model: the outer /// `AggIntent::Max` commits unconditionally to its own `MinMax` - /// accumulator (`asap_plan::boundary::implementation_for_with`), which + /// accumulator (`asap_aware_mapping::boundary::implementation_for_with`), which /// requires a real, independently-registered `MinMax` sid that never /// exists for this shape -- so the whole tree fails to realize even /// though the inner quantile would answer fine standalone. This is a @@ -3497,7 +3497,7 @@ mod outer_agg_integration_tests { /// `execute_max_by_zone_over_quantile_over_time_capability_misses_pending_asapcontroller_171` /// above, except `AggIntent::Avg` maps to `Implementation::PassThrough` /// rather than an accumulator commitment - /// (`asap_plan::boundary::implementation_for_with`), so + /// (`asap_aware_mapping::boundary::implementation_for_with`), so /// `implement_tree_in_with`'s conservative fallback wraps the WHOLE /// tree — including the otherwise-realizable inner quantile — as one /// opaque `Logical` blob. Same accepted-gap outcome either way: diff --git a/data_plane/src/query_engines/asap_query_engine/l4_lowering.rs b/data_plane/src/query_engines/asap_query_engine/l4_lowering.rs index 1898fa11..d473eb14 100644 --- a/data_plane/src/query_engines/asap_query_engine/l4_lowering.rs +++ b/data_plane/src/query_engines/asap_query_engine/l4_lowering.rs @@ -1,4 +1,4 @@ -//! PromQL string → `asap_sketch::L4Node` bridge, shared by the actual +//! PromQL string → `planner_types::post_asap::SummaryNode` bridge, shared by the actual //! serving cutover (`live_serve.rs`, via `l4_readout.rs`). See //! `data_plane/docs/l4node-plan-executor-design.md`'s "Rollout" section //! for the general design. @@ -22,7 +22,7 @@ //! accuracy-driven cost model prefers in the abstract (e.g. DDSketch //! over Kll for quantiles, unconditionally), with no guarantee it matches //! what's actually registered — and `SummaryExecutor::find_candidates` -//! requires an exact `(SummaryKind, SummaryParams)` match, by design (see +//! requires an exact `(SketchKind, SketchParams)` match, by design (see //! `summary_executor.rs::summary_params_match`'s doc: this deployment //! chose strict equality over silently serving an answer under a looser //! guarantee than what was planned). @@ -30,7 +30,7 @@ //! So before binding, [`lower_promql_to_l4node`] looks up what's actually //! registered for the query's target metric and constructs an //! [`ObservedFamilyCostModel`] that echoes that back — the resulting -//! `L4Node` matches reality by construction, not by a coincidental +//! `SummaryNode` matches reality by construction, not by a coincidental //! accuracy-target match. When nothing is registered for the metric (or //! this deployment's family/param mapping doesn't recognize the //! registered shape), `observed` is `None` and binding falls back to the @@ -40,7 +40,7 @@ use std::rc::Rc; -use asap_sketch::{L4Node, SummaryExpr, SummaryKind, SummaryParams}; +use planner_types::post_asap::{SketchKind, SketchParams, SummaryExpr, SummaryNode}; use control_plane::sketch_algebra::capability::{OuterFn, SketchKindHandle}; use control_plane::sketch_algebra::cost_model::ObservedFamilyCostModel; @@ -53,10 +53,10 @@ use crate::query_engines::asap_query_engine::summary_executor::find_metric_in_qu use crate::storage_engines::sketch_db::data::{AggKind, SketchConfig}; use crate::storage_engines::sketch_db::index::SketchStore; -/// Why a query couldn't be answered through the `L4Node`/`SummaryExecutor` +/// Why a query couldn't be answered through the `SummaryNode`/`SummaryExecutor` /// path — covers both `lower_promql_to_l4node`'s own failure to produce a /// tree, AND (via `l4_readout.rs`'s `execute_l4_readout`) a failure of -/// `asap_sketch::exec::execute()` on a tree that DID lower successfully. +/// `crate::query_engines::asap_query_engine::summary_exec::execute()` on a tree that DID lower successfully. /// None of these are errors in the alarming sense — every variant is an /// expected, frequent outcome for *some* fraction of live traffic; the /// caller's only obligation is "fall back to the legacy path," never @@ -96,18 +96,18 @@ pub enum LoweringSkip { /// outcome, just detected one step earlier so the caller can skip /// without even constructing a `QueryExecutionContext`. NotRealized, - /// The tree lowered successfully, but `asap_sketch::exec::execute()` + /// The tree lowered successfully, but `crate::query_engines::asap_query_engine::summary_exec::execute()` /// itself returned `Err` (`NoCandidates`, `MergeKindParamsMismatch`, /// a decode/merge failure surfaced from `summary_executor.rs`, ...). /// Always safe to just fall back — this means "can't answer this way /// right now" (e.g. the sid catalog doesn't have an exact - /// `(SummaryKind, SummaryParams)` match), never "answered wrong." + /// `(SketchKind, SketchParams)` match), never "answered wrong." ExecuteFailed(String), } /// Map a registered sid's `(SketchKindHandle, SketchConfig)` — the /// durable record of what planning actually decided for this metric — to -/// the `(SummaryKind, SummaryParams)` pair `ObservedFamilyCostModel` +/// the `(SketchKind, SketchParams)` pair `ObservedFamilyCostModel` /// needs to reproduce that decision exactly. `None` for shapes this /// deployment doesn't map (e.g. `SketchKindHandle::Any`, which is an /// analysis-time wildcard that's never actually registered on a sid). @@ -118,54 +118,59 @@ pub enum LoweringSkip { /// note), so `heap_size` here is a placeholder; `summary_params_match` /// only compares `width`/`depth` for these kinds, so it doesn't affect /// matching. -fn observed_summary_params(kind: SketchKindHandle, config: &SketchConfig) -> Option<(SummaryKind, SummaryParams)> { +fn observed_summary_params( + kind: SketchKindHandle, + config: &SketchConfig, +) -> Option<(SketchKind, SketchParams)> { const PLACEHOLDER_HEAP_SIZE: u32 = 100; match (kind, config) { (SketchKindHandle::DDSketch, SketchConfig::DDSketch { relative_accuracy }) => Some(( - SummaryKind::DDSketch, - SummaryParams::DDSketch { + SketchKind::DDSketch, + SketchParams::DDSketch { alpha: *relative_accuracy, }, )), (SketchKindHandle::Kll, SketchConfig::Kll { k }) => { - Some((SummaryKind::Kll, SummaryParams::Kll { k: *k })) + Some((SketchKind::Kll, SketchParams::Kll { k: *k })) } (SketchKindHandle::Hll, SketchConfig::Hll { precision }) => Some(( - SummaryKind::Hll, - SummaryParams::Hll { + SketchKind::Hll, + SketchParams::Hll { precision: *precision as u8, }, )), (SketchKindHandle::CountMin, SketchConfig::CountMin { rows, cols }) => Some(( - SummaryKind::Cms, - SummaryParams::Cms { + SketchKind::Cms, + SketchParams::Cms { width: *cols as u32, depth: *rows as u32, }, )), (SketchKindHandle::CmsWithHeap, SketchConfig::CountMin { rows, cols }) => Some(( - SummaryKind::CmsWithHeap, - SummaryParams::CmsWithHeap { + SketchKind::CmsWithHeap, + SketchParams::CmsWithHeap { width: *cols as u32, depth: *rows as u32, heap_size: PLACEHOLDER_HEAP_SIZE, }, )), (SketchKindHandle::CountSketch, SketchConfig::CountSketch { rows, cols }) => Some(( - SummaryKind::CountSketch, - SummaryParams::CountSketch { + SketchKind::CountSketch, + SketchParams::CountSketch { width: *cols as u32, depth: *rows as u32, }, )), - (SketchKindHandle::CountSketchWithHeap, SketchConfig::CountSketch { rows, cols }) => Some(( - SummaryKind::CountSketchWithHeap, - SummaryParams::CountSketchWithHeap { - width: *cols as u32, - depth: *rows as u32, - heap_size: PLACEHOLDER_HEAP_SIZE, - }, - )), + (SketchKindHandle::CountSketchWithHeap, SketchConfig::CountSketch { rows, cols }) => { + Some(( + SketchKind::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { + width: *cols as u32, + depth: *rows as u32, + heap_size: PLACEHOLDER_HEAP_SIZE, + }, + )) + } _ => None, } } @@ -179,7 +184,10 @@ fn observed_summary_params(kind: SketchKindHandle, config: &SketchConfig) -> Opt /// `None` when nothing is registered (or only `ExactAgg` sids are — /// those never consult `CostModel` at all, so there's nothing to /// observe for them). -fn observed_family_for_metric(index: &SketchStore, metric: &str) -> Option<(SummaryKind, SummaryParams)> { +fn observed_family_for_metric( + index: &SketchStore, + metric: &str, +) -> Option<(SketchKind, SketchParams)> { for sid in index.instances_matching(metric, &Default::default()) { let found = index.with_instance(sid, |m| match &m.agg_kind { AggKind::Sketch { kind, config, .. } => observed_summary_params(*kind, config), @@ -204,15 +212,23 @@ fn observed_family_for_metric(index: &SketchStore, metric: &str) -> Option<(Summ fn observed_family_for_metric_from_plan( plan: &control_plane::backend_plan::BackendPlan, metric: &str, -) -> Option<(SummaryKind, SummaryParams)> { +) -> Option<(SketchKind, SketchParams)> { plan.materializations.values().find_map(|m| { - matches!(&m.source, control_plane::intent_algebra::Source::TimeSeries { metric: mm } if mm == metric) - .then(|| (m.kind.clone(), m.params.clone())) + if !matches!(&m.source, control_plane::intent_algebra::Source::TimeSeries { metric: mm } if mm == metric) + { + return None; + } + // `Materialization.kind`/`.params` are the flat type (spans + // exact accumulators too) -- narrow to the sketch-only pair + // this function returns, skipping exact-accumulator + // materializations (mirrors `observed_family_for_metric`'s + // "first sketch-typed one found" semantics). + Some((m.kind.as_sketch_kind()?, m.params.as_sketch_params()?)) }) } -/// Lower a raw PromQL query string to the `L4Node` tree -/// `asap_sketch::exec::execute`/`SummaryExecutor` needs — the actual +/// Lower a raw PromQL query string to the `SummaryNode` tree +/// `crate::query_engines::asap_query_engine::summary_exec::execute`/`SummaryExecutor` needs — the actual /// serving cutover (`live_serve.rs`). Returns `Err` for any shape serving /// shouldn't attempt (parse failure, `rate()`, or anything that doesn't /// realize to a concrete sketch/exact-agg binding) — see @@ -222,7 +238,7 @@ pub fn lower_promql_to_l4node( query: &str, accuracy: AccuracyTarget, backend_plan: Option<&control_plane::backend_plan::BackendPlan>, -) -> Result, LoweringSkip> { +) -> Result, LoweringSkip> { // Reuse the SAME candidate analysis `engine.rs` already runs for the // legacy dispatch, rather than re-deriving rate detection via a // second raw-AST walk. `OuterFn::Rate` is the one shape that binds @@ -245,7 +261,7 @@ pub fn lower_promql_to_l4node( // reading it straight off an installed `BackendPlan`'s // materializations when one covers this metric -- // `Materialization.kind`/`.params` already ARE the - // `(SummaryKind, SummaryParams)` pair this needs, no + // `(SketchKind, SketchParams)` pair this needs, no // `AggregationConfig` reconstruction required (design-backend-plan-wire-format.md // §5). Otherwise fall back to the `SketchStore`-reconstruction path // (`observed_family_for_metric`), which is `None` when this metric @@ -294,7 +310,8 @@ mod tests { #[test] fn rate_query_is_skipped_before_binding() { let idx = empty_index(); - let result = lower_promql_to_l4node(&idx, "rate(http_requests_total[5m])", accuracy(), None); + let result = + lower_promql_to_l4node(&idx, "rate(http_requests_total[5m])", accuracy(), None); assert!( matches!(result, Err(LoweringSkip::RateShape)), "expected RateShape, got {result:?}" @@ -304,7 +321,8 @@ mod tests { #[test] fn irate_query_is_skipped_before_binding() { let idx = empty_index(); - let result = lower_promql_to_l4node(&idx, "irate(http_requests_total[5m])", accuracy(), None); + let result = + lower_promql_to_l4node(&idx, "irate(http_requests_total[5m])", accuracy(), None); assert!( matches!(result, Err(LoweringSkip::RateShape)), "expected RateShape, got {result:?}" @@ -355,9 +373,13 @@ mod tests { // falls back to the accuracy-driven default -- same outcome as // before this module started consulting the `SketchStore`. let idx = empty_index(); - let node = - lower_promql_to_l4node(&idx, "count_over_time(http_requests_total[5m])", accuracy(), None) - .expect("Frequency intent must realize via bind_query_expr/ControlPlaneCostModel"); + let node = lower_promql_to_l4node( + &idx, + "count_over_time(http_requests_total[5m])", + accuracy(), + None, + ) + .expect("Frequency intent must realize via bind_query_expr/ControlPlaneCostModel"); assert!( !matches!(node.expr, SummaryExpr::Logical(_)), "expected a real SummaryAgg/SummaryEstimate binding, got Logical (the gap \ @@ -433,8 +455,14 @@ mod tests { }, group_by: Vec::new(), rollup: Vec::new(), - kind: SummaryKind::DDSketch, - params: SummaryParams::DDSketch { alpha: 0.01 }, + // `Materialization.kind`/`.params` span both exact + // accumulators and sketches -- the flat + // `asap_types::SummaryKind`, not this file's own + // `planner_types::post_asap::SketchKind` import (see + // `physical::colored_dag::emitter`'s `use + // asap_types::{...}` note in control_plane). + kind: asap_types::SummaryKind::DDSketch, + params: asap_types::SummaryParams::DDSketch { alpha: 0.01 }, col: ColumnRef::SampleValue, retention: None, }, @@ -448,18 +476,19 @@ mod tests { } } - /// Extract the bound `(SummaryKind, SummaryParams)` from the - /// `SummaryEstimate { summary_input: L4Node { expr: SummaryAgg { + /// Extract the bound `(SketchKind, SketchParams)` from the + /// `SummaryEstimate { summary_input: SummaryNode { expr: SummaryAgg { /// summary, params, .. }, .. }, .. }` shape a bare /// `quantile_over_time` query lowers to (confirmed by inspecting /// the tree directly). - fn bound_family(node: &L4Node) -> (SummaryKind, SummaryParams) { + fn bound_family(node: &SummaryNode) -> (SketchKind, SketchParams) { match &node.expr { SummaryExpr::SummaryEstimate { summary_input, .. } => match &summary_input.expr { - SummaryExpr::SummaryAgg { summary, params, .. } => { - (summary.clone(), params.clone()) - } - other => panic!("expected SummaryAgg, got {other:?}"), + SummaryExpr::SummaryAgg { + family: planner_types::post_asap::SummaryFamilyType::Sketch(kind, params), + .. + } => (kind.clone(), params.clone()), + other => panic!("expected a Sketch SummaryAgg, got {other:?}"), }, other => panic!("expected SummaryEstimate, got {other:?}"), } @@ -471,9 +500,10 @@ mod tests { // SketchStore reconstruction is the only source. let idx = SketchStore::new(); register_kll(&idx, "m"); - let node = lower_promql_to_l4node(&idx, "quantile_over_time(0.99, m[1m])", accuracy(), None) - .expect("should lower"); - assert_eq!(bound_family(&node).0, SummaryKind::Kll); + let node = + lower_promql_to_l4node(&idx, "quantile_over_time(0.99, m[1m])", accuracy(), None) + .expect("should lower"); + assert_eq!(bound_family(&node).0, SketchKind::Kll); } #[test] @@ -496,7 +526,7 @@ mod tests { .expect("should lower"); assert_eq!( bound_family(&node).0, - SummaryKind::DDSketch, + SketchKind::DDSketch, "BackendPlan's materialization must take priority over SketchStore reconstruction" ); } @@ -517,7 +547,7 @@ mod tests { Some(&plan), ) .expect("should lower"); - assert_eq!(bound_family(&node).0, SummaryKind::Kll); + assert_eq!(bound_family(&node).0, SketchKind::Kll); } } } diff --git a/data_plane/src/query_engines/asap_query_engine/l4_readout.rs b/data_plane/src/query_engines/asap_query_engine/l4_readout.rs index 15fb3b1c..cd1a2c04 100644 --- a/data_plane/src/query_engines/asap_query_engine/l4_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/l4_readout.rs @@ -1,12 +1,11 @@ -//! `L4Node` lowering + execution + conversion into `ASAPTierResult`'s +//! `SummaryNode` lowering + execution + conversion into `ASAPTierResult`'s //! `(series, coverage)` shape — the core `live_serve.rs` (the serving //! cutover) calls into. See //! `data_plane/docs/l4node-plan-executor-design.md` for the design. use std::collections::BTreeMap; -use asap_sketch::exec::{execute, ExecOutcome}; -use asap_sketch::{L4Node, SummaryExpr}; +use crate::query_engines::asap_query_engine::summary_exec::{execute, ExecOutcome}; use control_plane::types_v2::AccuracyTarget; use crate::query_engines::asap_query_engine::l4_lowering::{lower_promql_to_l4node, LoweringSkip}; @@ -93,7 +92,7 @@ pub fn execute_l4_readout( Ok(ExecOutcome::State(groups)) => { let mut coverage: Option<(u64, u64)> = None; let mut series = Vec::new(); - for (group_key, state, _kind, _params) in &groups { + for (group_key, state, _family) in &groups { fold_coverage(&mut coverage, state.exact_coverage()); let Some(value) = state.exact_value(&None) else { continue; @@ -278,9 +277,16 @@ mod tests { let idx = SketchStore::new(); register_hll(&idx, 1, "svc-a", &["a", "b", "c"]); register_hll(&idx, 2, "svc-b", &["d", "e", "f"]); - let outcome = - execute_l4_readout(&idx, "count(unique_users)", 1_000, 2_000, true, accuracy(), None) - .expect("should execute"); + let outcome = execute_l4_readout( + &idx, + "count(unique_users)", + 1_000, + 2_000, + true, + accuracy(), + None, + ) + .expect("should execute"); assert_eq!( outcome.series.len(), 1, @@ -324,8 +330,16 @@ mod tests { (1_000, 2_000), Box::new(crate::precompute_engine::operators::SumAccumulator::with_sum(42.0)), ); - let outcome = execute_l4_readout(&idx, "sum(bytes_total)", 1_000, 2_000, true, accuracy(), None) - .expect("should execute"); + let outcome = execute_l4_readout( + &idx, + "sum(bytes_total)", + 1_000, + 2_000, + true, + accuracy(), + None, + ) + .expect("should execute"); // Window-end-only coverage: a single window (1_000, 2_000) is // keyed by its end (2_000) alone, so both bounds equal 2_000 -- // same semantics as `SummaryValue::coverage()`, reconfirmed for diff --git a/data_plane/src/query_engines/asap_query_engine/live_serve.rs b/data_plane/src/query_engines/asap_query_engine/live_serve.rs index 360358d6..b8db37be 100644 --- a/data_plane/src/query_engines/asap_query_engine/live_serve.rs +++ b/data_plane/src/query_engines/asap_query_engine/live_serve.rs @@ -290,9 +290,8 @@ mod tests { register_hll(&idx, 2, "svc-b", &["d", "e", "f"]); let result = try_serve_from_summary_executor(&idx, "count(unique_users)", 1_000, 2_000, true, None); - let result = result.expect( - "global-merge shape is no longer ambiguous -- it must be served, not declined", - ); + let result = result + .expect("global-merge shape is no longer ambiguous -- it must be served, not declined"); assert_eq!( result.series.len(), 1, @@ -311,8 +310,14 @@ mod tests { fn flag_on_unservable_query_falls_back() { let _guard = set_live_env("1"); let idx = SketchStore::new(); - let result = - try_serve_from_summary_executor(&idx, "rate(http_requests_total[5m])", 0, 1000, true, None); + let result = try_serve_from_summary_executor( + &idx, + "rate(http_requests_total[5m])", + 0, + 1000, + true, + None, + ); assert!(result.is_none()); } } diff --git a/data_plane/src/query_engines/asap_query_engine/mod.rs b/data_plane/src/query_engines/asap_query_engine/mod.rs index 583a8923..736abb53 100644 --- a/data_plane/src/query_engines/asap_query_engine/mod.rs +++ b/data_plane/src/query_engines/asap_query_engine/mod.rs @@ -13,6 +13,7 @@ pub mod engine; pub mod l4_lowering; pub mod l4_readout; pub mod live_serve; +pub mod summary_exec; pub mod summary_executor; // Phase-5 reorg: ASAP-tier reducer moved to `sketch_db::query`. The diff --git a/data_plane/src/query_engines/asap_query_engine/summary_exec.rs b/data_plane/src/query_engines/asap_query_engine/summary_exec.rs new file mode 100644 index 00000000..2950b74c --- /dev/null +++ b/data_plane/src/query_engines/asap_query_engine/summary_exec.rs @@ -0,0 +1,698 @@ +//! Serving-time execution model for the post-ASAP IR — the counterpart to +//! `asap_aware_mapping::bind`'s planning-time `QueryExpr -> SummaryNode`. +//! +//! **Vendored, not upstream.** This module used to be +//! `asap_sketch::exec` (`crates/sketch/src/exec.rs`) in what's now +//! ASAPPlanner. It was deleted there in +//! [ASAPPlanner#190](https://github.com/ProjectASAP/ASAPPlanner/issues/190) +//! (folded into #197) on the grounds that "the only implementors anywhere +//! in this workspace are `MockExecutor`/`ReductionSpyExecutor`, both +//! `#[cfg(test)]`-only... no bin, example, or downstream crate in this +//! repo constructs or runs a real executor" — true from ASAPPlanner's own +//! repo, but this module's very own [`SummaryExecutor`] trait *is* +//! implemented for real by +//! [`summary_executor::AsapSummaryExecutor`](crate::query_engines::asap_query_engine::summary_executor), +//! just in this (different) repo, which upstream's search couldn't see. +//! ASAPPlanner's own README now explicitly scopes itself away from +//! execution ("not doing any physical query planning") and lists +//! connecting its output to a real backend as an open question (#1) — see +//! `control_plane/docs/design-asapplanner-pin-migration.md` §3.2 for the +//! full account. Ported here verbatim in spirit (same trait shape, same +//! tree-walking algorithm, same tests) with one adaptation: ASAPPlanner +//! split the old flat `SummaryKind`/`SummaryParams` pair into a +//! per-family `SummaryFamilyType` tagged union +//! (ASAPPlanner#218) before `exec.rs` was deleted, so `SummaryAgg`'s +//! `kind`/`params` fields collapsed into one `family: SummaryFamilyType` +//! field on the real, current `SummaryExpr::SummaryAgg` — this module's +//! `find_candidates`/`ExecOutcome::State` follow that shape rather than +//! the pre-split one the last upstream copy of `exec.rs` had. + +use std::collections::BTreeMap; + +use planner_types::pre_asap::{ColumnRef, QueryExpr, Reduction}; + +use planner_types::post_asap::{SketchQuery, SummaryExpr, SummaryFamilyType, SummaryNode}; + +/// Deployment-supplied backend for executing a [`SummaryNode`] tree +/// against materialized state. +pub trait SummaryExecutor { + /// Reference to one materialized summary instance (e.g. a sid). + type Handle: Clone; + /// Decoded, in-memory state for one instance (or an already-merged + /// group of them). + type State; + /// A query answer. + type Value; + /// Deployment error type. + type Error; + /// Opaque per-group identity for a `SummaryAgg`'s grouping — e.g. a + /// label-value map for a PromQL/time-series deployment. `Ord` so + /// `execute` can fold same-group handles/states deterministically; + /// `Default` for the single-group case (`Reduction::Reduce` with an + /// empty `by`, or a `Logical` leaf, which has no grouping concept at + /// all) — every outcome is always a per-group list, and the + /// single-group case is simply a list of one entry under + /// `GroupKey::default()`. + type GroupKey: Clone + Ord + Default; + + /// Resolve a `SummaryAgg` leaf to matching materialized-instance + /// handles, each tagged with the group it belongs to. Must only + /// return handles whose committed summary family is exactly + /// `family` (same `(kind, params)` pair, whichever family it is — + /// `ExactAggregate`/`Sketch`/`Sample`/`Wavelet`/`StatModel`; never + /// `Plain`, since a `SummaryAgg` always commits a real family — see + /// the field's own doc on `SummaryExpr::SummaryAgg`). + /// + /// `reduction` (the same [`Reduction`] the pre-ASAP `Aggregate` node + /// this was bound from carried) tells you which of two shapes to + /// produce — the two are **not** interchangeable (ASAPPlanner#163): + /// - `Reduction::Reduce(by)`: a genuine cross-series reduction. Tag + /// every returned handle with the `GroupKey` for its `by` values; + /// when `by` is empty there is exactly one group — every handle + /// should share one deployment-chosen `GroupKey` (e.g. + /// `GroupKey::default()`), and they will all be merged together. + /// - `Reduction::PerEntity`: there is no grouping concept for this + /// shape at all. Tag every returned handle with its own, distinct + /// `GroupKey` (its full entity/series identity) — handles must + /// never share a `GroupKey` here, since `execute` merges same-group + /// handles together and merging unrelated entities would silently + /// produce a wrong answer. + #[allow(clippy::type_complexity)] + fn find_candidates( + &self, + family: &SummaryFamilyType, + col: &ColumnRef, + reduction: &Reduction, + child: &SummaryNode, + ) -> Result, Self::Error>; + + /// Fetch/decode one handle's raw state. + fn fetch_state(&self, handle: &Self::Handle) -> Result; + + /// Merge two or more same-family, same-group states. + fn merge_states(&self, states: Vec) -> Result; + + /// Read a query out of one group's built state. + fn readout(&self, state: &Self::State, query: &SketchQuery) + -> Result; + + /// Handle a `SummaryExpr::Logical` node (nothing committed at the + /// post-ASAP level). Ungrouped by construction — `Logical` defers + /// entirely to the deployment, which decides for itself whether/how + /// the underlying `QueryExpr` groups. + fn logical(&self, expr: &QueryExpr) -> Result; +} + +/// Result of executing one [`SummaryNode`] — one summary state per group +/// (tagged with the shared family), or one final value per group. Always +/// a list, even when there's exactly one (ungrouped) group — see +/// [`SummaryExecutor::GroupKey`]. +pub enum ExecOutcome { + State(Vec<(E::GroupKey, E::State, SummaryFamilyType)>), + Value(Vec<(E::GroupKey, E::Value)>), +} + +/// Errors [`execute`] itself can raise, on top of the deployment's own +/// [`SummaryExecutor::Error`]. +#[derive(Debug)] +pub enum ExecError { + /// [`SummaryExecutor::find_candidates`] returned no handles. + NoCandidates, + /// A `SummaryMerge` child produced a `Value`, not a `State`. + MergeChildNotState, + /// `SummaryMerge` children disagreed on their summary family — checked + /// globally across every group from every child, not just within one + /// group, because family agreement is a planning-time property of the + /// `SummaryMerge` node itself (fixed before any group value is known), + /// not something that can validly vary per group. + MergeKindParamsMismatch, + /// A `SummaryMerge` had zero children. + EmptyMerge, + /// `SummaryJoin`/`SummarySubtract`/`SummaryDelete` — unreachable today. + NotYetSupported(&'static str), + /// The deployment's own error. + Executor(Inner), +} + +impl From for ExecError { + fn from(e: Inner) -> Self { + ExecError::Executor(e) + } +} + +/// Recursively execute `node` against `exec`. +pub fn execute( + node: &SummaryNode, + exec: &E, +) -> Result, ExecError> { + match &node.expr { + SummaryExpr::Logical(qe) => Ok(ExecOutcome::Value(vec![( + E::GroupKey::default(), + exec.logical(qe)?, + )])), + + SummaryExpr::SummaryAgg { + child, + family, + col, + reduction, + } => { + let tagged = exec.find_candidates(family, col, reduction, child)?; + if tagged.is_empty() { + return Err(ExecError::NoCandidates); + } + // Group by `GroupKey` first (a `SummaryAgg` may see several + // handles for the same group, e.g. cross-host instances of + // the same metric/group-by value that need folding), then + // fold each group's handles independently — a group's state + // must never be combined with another group's. + let mut by_group: BTreeMap> = BTreeMap::new(); + for (key, handle) in tagged { + by_group.entry(key).or_default().push(handle); + } + let mut out = Vec::with_capacity(by_group.len()); + for (key, handles) in by_group { + let states = handles + .iter() + .map(|h| exec.fetch_state(h)) + .collect::, _>>()?; + let state = fold_states(states, exec)?; + out.push((key, state, family.clone())); + } + Ok(ExecOutcome::State(out)) + } + + SummaryExpr::SummaryEstimate { + summary_input, + query, + } => { + let groups = expect_state(execute(summary_input, exec)?)?; + let mut out = Vec::with_capacity(groups.len()); + for (key, state, _family) in groups { + out.push((key, exec.readout(&state, query)?)); + } + Ok(ExecOutcome::Value(out)) + } + + SummaryExpr::SummaryMerge { children } => { + if children.is_empty() { + return Err(ExecError::EmptyMerge); + } + let mut agreed: Option = None; + // Collect every child's (group -> state) pairs first, so a + // group present in only some children still gets folded from + // whatever states exist for it (mirrors today's per-(group, + // window) ExactAgg merge: fold whatever's present, don't + // require every child to cover every group). + let mut by_group: BTreeMap> = BTreeMap::new(); + for child in children { + let groups = expect_state(execute(child, exec)?)?; + for (key, state, family) in groups { + match &agreed { + None => agreed = Some(family), + Some(f) if *f == family => {} + Some(_) => return Err(ExecError::MergeKindParamsMismatch), + } + by_group.entry(key).or_default().push(state); + } + } + // `children` non-empty => at least one child produced a + // `State` outcome (checked above) => that outcome had at + // least one group (a `SummaryAgg`/`SummaryMerge` never + // produces an empty group list — `SummaryAgg` errors via + // `NoCandidates` first, and `SummaryMerge` recursively bottoms + // out at a `SummaryAgg`) => `agreed` is always `Some` here. + let family = agreed.expect("checked non-empty above"); + let mut out = Vec::with_capacity(by_group.len()); + for (key, states) in by_group { + let merged = fold_states(states, exec)?; + out.push((key, merged, family.clone())); + } + Ok(ExecOutcome::State(out)) + } + + SummaryExpr::SummaryJoin { .. } => Err(ExecError::NotYetSupported("SummaryJoin")), + SummaryExpr::SummarySubtract { .. } => Err(ExecError::NotYetSupported("SummarySubtract")), + SummaryExpr::SummaryDelete { .. } => Err(ExecError::NotYetSupported("SummaryDelete")), + } +} + +/// Fold one-or-more same-family, same-group states into one. +fn fold_states( + mut states: Vec, + exec: &E, +) -> Result> { + if states.len() == 1 { + Ok(states.pop().expect("len == 1")) + } else { + Ok(exec.merge_states(states)?) + } +} + +/// Unwrap an [`ExecOutcome::State`], erroring if it was actually a +/// `Value`. +#[allow(clippy::type_complexity)] +fn expect_state( + outcome: ExecOutcome, +) -> Result, ExecError> { + match outcome { + ExecOutcome::State(groups) => Ok(groups), + ExecOutcome::Value(_) => Err(ExecError::MergeChildNotState), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use planner_types::post_asap::{SummaryField, SummarySchema}; + use planner_types::pre_asap::{Column, DataType, Schema, Source}; + use std::cell::RefCell; + use std::collections::HashMap; + use std::rc::Rc; + + fn scan() -> QueryExpr { + QueryExpr::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Column::new("ts", DataType::Timestamp, false), + Column::new("value", DataType::Float64, false), + ], + 0, + vec![], + ), + } + } + + fn lift(schema_fields: Vec<&str>) -> SummarySchema { + SummarySchema { + fields: schema_fields + .into_iter() + .map(|n| SummaryField { + name: n.into(), + dtype: SummaryFamilyType::Plain(DataType::Float64), + nullable: false, + }) + .collect(), + time_index: None, + } + } + + fn logical_node() -> Rc { + Rc::new(SummaryNode { + expr: SummaryExpr::Logical(Box::new(scan())), + schema: lift(vec!["ts", "value"]), + }) + } + + /// Builds a `SummaryAgg` with `Reduction::Reduce(vec![])` (an ordinary, + /// ungrouped cross-series reduction) — the shape every pre-#163 test in + /// this module exercises. Use [`agg_node_with_reduction`] to exercise + /// `Reduction` itself. + fn agg_node(family: SummaryFamilyType, child: Rc) -> Rc { + agg_node_with_reduction(family, child, Reduction::by(vec![])) + } + + fn agg_node_with_reduction( + family: SummaryFamilyType, + child: Rc, + reduction: Reduction, + ) -> Rc { + Rc::new(SummaryNode { + expr: SummaryExpr::SummaryAgg { + child, + family, + col: ColumnRef::SampleValue, + reduction, + }, + schema: lift(vec!["value"]), + }) + } + + fn estimate_node(summary_input: Rc, query: SketchQuery) -> Rc { + Rc::new(SummaryNode { + expr: SummaryExpr::SummaryEstimate { + summary_input, + query, + }, + schema: lift(vec!["value"]), + }) + } + + fn merge_node(children: Vec>) -> Rc { + Rc::new(SummaryNode { + expr: SummaryExpr::SummaryMerge { children }, + schema: lift(vec!["value"]), + }) + } + + /// Trivial in-memory executor for exercising `execute`'s tree-walking, + /// grouping, and merge-precondition logic — `State`/`Value` are just + /// `f64`s, `GroupKey` is `String` (empty string = the ungrouped + /// default), and `merge_states` is `sum`, so tests can assert on + /// concrete numbers without any real sketch-math dependency. + /// + /// `find_candidates` intentionally ignores `family`/`reduction`/`child` + /// and returns every registered handle tagged with its own registered + /// group — real deployments filter on those; these tests only + /// exercise `execute`'s own tree-walking and grouping/merge logic, not + /// a real candidate search. See + /// `reduction_is_passed_through_to_find_candidates_unmodified` below + /// for a test that *does* inspect `reduction`. + struct MockExecutor { + /// handle -> (group, value). + values: RefCell>, + next_handle: RefCell, + } + + impl MockExecutor { + fn new() -> Self { + Self { + values: RefCell::new(HashMap::new()), + next_handle: RefCell::new(0), + } + } + + /// Register one materialized instance under the default + /// (ungrouped) group, returning its handle. + fn register(&self, value: f64) -> u64 { + self.register_grouped("", value) + } + + /// Register one materialized instance under an explicit group. + fn register_grouped(&self, group: &str, value: f64) -> u64 { + let mut h = self.next_handle.borrow_mut(); + let handle = *h; + *h += 1; + self.values + .borrow_mut() + .insert(handle, (group.to_string(), value)); + handle + } + } + + impl SummaryExecutor for MockExecutor { + type Handle = u64; + type State = f64; + type Value = f64; + type Error = String; + type GroupKey = String; + + fn find_candidates( + &self, + _family: &SummaryFamilyType, + _col: &ColumnRef, + _reduction: &Reduction, + _child: &SummaryNode, + ) -> Result, Self::Error> { + Ok(self + .values + .borrow() + .iter() + .map(|(h, (g, _))| (g.clone(), *h)) + .collect()) + } + + fn fetch_state(&self, handle: &Self::Handle) -> Result { + self.values + .borrow() + .get(handle) + .map(|(_, v)| *v) + .ok_or_else(|| "unknown handle".to_string()) + } + + fn merge_states(&self, states: Vec) -> Result { + Ok(states.into_iter().sum()) + } + + fn readout( + &self, + state: &Self::State, + _query: &SketchQuery, + ) -> Result { + Ok(*state) + } + + fn logical(&self, _expr: &QueryExpr) -> Result { + Ok(-1.0) + } + } + + fn kll() -> SummaryFamilyType { + use planner_types::post_asap::{SketchKind, SketchParams}; + SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) + } + + fn sum() -> SummaryFamilyType { + use planner_types::post_asap::{ExactKind, ExactParams}; + SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) + } + + /// Extract the single (ungrouped) output's value, asserting there's + /// exactly one group -- the shape every pre-grouping test expects. + fn only(v: Vec<(String, T)>) -> T { + assert_eq!(v.len(), 1, "expected exactly one (ungrouped) output group"); + v.into_iter().next().unwrap().1 + } + + #[test] + fn single_agg_reads_out_directly() { + let exec = MockExecutor::new(); + exec.register(7.0); + let tree = estimate_node(agg_node(sum(), logical_node()), SketchQuery::Cardinality); + let ExecOutcome::Value(v) = execute(&tree, &exec).unwrap() else { + panic!("expected a value"); + }; + assert_eq!(only(v), 7.0); + } + + #[test] + fn multiple_candidates_for_one_agg_are_merged() { + let exec = MockExecutor::new(); + exec.register(3.0); + exec.register(4.0); + let tree = estimate_node(agg_node(sum(), logical_node()), SketchQuery::Cardinality); + let ExecOutcome::Value(v) = execute(&tree, &exec).unwrap() else { + panic!("expected a value"); + }; + assert_eq!(only(v), 7.0); // MockExecutor::merge_states sums + } + + #[test] + fn logical_leaf_defers_to_executor() { + let exec = MockExecutor::new(); + let tree = logical_node(); + let ExecOutcome::Value(v) = execute(&tree, &exec).unwrap() else { + panic!("expected a value"); + }; + assert_eq!(only(v), -1.0); + } + + #[test] + fn nested_summary_agg_recurses_through_both_levels() { + // quantile(0.9, sum by (job) (m)) shape: Kll wraps Sum. + let exec = MockExecutor::new(); + exec.register(10.0); + let inner = agg_node(sum(), logical_node()); + let outer = agg_node(kll(), inner); + let tree = estimate_node(outer, SketchQuery::Quantile { q: 0.9 }); + let ExecOutcome::Value(v) = execute(&tree, &exec).unwrap() else { + panic!("expected a value"); + }; + // MockExecutor::find_candidates ignores the tree shape and always + // returns every registered handle -- the point of this test is + // that `execute` actually reaches both levels (no panic/short + // circuit on the nested SummaryAgg), not the specific value. + assert_eq!(only(v), 10.0); + } + + #[test] + fn merge_of_two_aggs_with_matching_kind_params_sums() { + let exec = MockExecutor::new(); + let a = agg_node(sum(), logical_node()); + let b = agg_node(sum(), logical_node()); + // Each SummaryAgg independently pulls every registered handle from + // the shared MockExecutor (2 handles, same default group), so the + // merge sees two already-summed inputs. Register after building + // the tree so each `find_candidates` call sees the same two + // handles. + exec.register(1.0); + exec.register(2.0); + let merged = merge_node(vec![a, b]); + let tree = estimate_node(merged, SketchQuery::Cardinality); + let ExecOutcome::Value(v) = execute(&tree, &exec).unwrap() else { + panic!("expected a value"); + }; + // Each child sums to 3.0 (1.0 + 2.0); the outer merge sums the two + // children's results: 3.0 + 3.0. + assert_eq!(only(v), 6.0); + } + + #[test] + fn merge_of_merges_nests_arbitrarily_deep() { + let exec = MockExecutor::new(); + exec.register(5.0); + let leaf_a = agg_node(sum(), logical_node()); + let leaf_b = agg_node(sum(), logical_node()); + let inner_merge = merge_node(vec![leaf_a, leaf_b]); + let leaf_c = agg_node(sum(), logical_node()); + let outer_merge = merge_node(vec![inner_merge, leaf_c]); + let tree = estimate_node(outer_merge, SketchQuery::Cardinality); + assert!(execute(&tree, &exec).is_ok()); + } + + #[test] + fn merge_rejects_mismatched_kind_params() { + let exec = MockExecutor::new(); + exec.register(1.0); + let a = agg_node(sum(), logical_node()); + let b = agg_node(kll(), logical_node()); + let merged = merge_node(vec![a, b]); + let tree = estimate_node(merged, SketchQuery::Cardinality); + match execute(&tree, &exec) { + Err(ExecError::MergeKindParamsMismatch) => {} + other => panic!( + "expected MergeKindParamsMismatch, got a different outcome: {}", + other.is_ok() + ), + } + } + + #[test] + fn merge_rejects_a_value_producing_child() { + let exec = MockExecutor::new(); + exec.register(1.0); + let state_child = agg_node(sum(), logical_node()); + let value_child = estimate_node(logical_node(), SketchQuery::Cardinality); + let merged = merge_node(vec![state_child, value_child]); + let tree = estimate_node(merged, SketchQuery::Cardinality); + match execute(&tree, &exec) { + Err(ExecError::MergeChildNotState) => {} + other => panic!( + "expected MergeChildNotState, got a different outcome: {}", + other.is_ok() + ), + } + } + + #[test] + fn no_candidates_is_a_distinct_error() { + let exec = MockExecutor::new(); // nothing registered + let tree = estimate_node(agg_node(sum(), logical_node()), SketchQuery::Cardinality); + match execute(&tree, &exec) { + Err(ExecError::NoCandidates) => {} + other => panic!( + "expected NoCandidates, got a different outcome: {}", + other.is_ok() + ), + } + } + + #[test] + fn grouped_summary_agg_produces_one_series_per_group() { + // The gap ASAPPlanner#159 flagged: `quantile by (zone) (m)` must + // produce one output series per zone, not one series merging + // every zone's sketches together. + let exec = MockExecutor::new(); + exec.register_grouped("us-east", 3.0); + exec.register_grouped("us-west", 4.0); + let tree = estimate_node(agg_node(sum(), logical_node()), SketchQuery::Cardinality); + let ExecOutcome::Value(mut v) = execute(&tree, &exec).unwrap() else { + panic!("expected a value"); + }; + v.sort_by(|a, b| a.0.cmp(&b.0)); + assert_eq!( + v, + vec![("us-east".to_string(), 3.0), ("us-west".to_string(), 4.0)] + ); + } + + #[test] + fn grouped_summary_merge_folds_within_group_not_across() { + // Two children (e.g. edge + gateway tiers of the same SummaryAgg + // shape) each see both groups' handles; the merge must fold + // us-east's two values together and us-west's two values + // together, WITHOUT ever mixing a us-east state into us-west's + // output or vice versa. + let exec = MockExecutor::new(); + let a = agg_node(sum(), logical_node()); + let b = agg_node(sum(), logical_node()); + exec.register_grouped("us-east", 1.0); + exec.register_grouped("us-west", 10.0); + let merged = merge_node(vec![a, b]); + let tree = estimate_node(merged, SketchQuery::Cardinality); + let ExecOutcome::Value(mut v) = execute(&tree, &exec).unwrap() else { + panic!("expected a value"); + }; + v.sort_by(|x, y| x.0.cmp(&y.0)); + // us-east: both children see the one us-east handle (1.0) -> each + // child's us-east state is 1.0 -> merge sums the two children: + // 1.0 + 1.0 = 2.0. Same for us-west: 10.0 + 10.0 = 20.0. Neither + // group's value (2.0, 20.0) equals the cross-group sum (11.0) -- + // proves the merge didn't mix groups. + assert_eq!( + v, + vec![("us-east".to_string(), 2.0), ("us-west".to_string(), 20.0)] + ); + } + + /// A wiring test for issue ASAPPlanner#163: `execute` must pass a + /// `SummaryAgg`'s `reduction` through to `find_candidates` completely + /// unmodified — not re-derive it, not silently drop it in favor of a + /// bare `by` list. `MockExecutor` above ignores `reduction` entirely + /// (it isn't testing this), so this test uses its own executor that + /// records exactly what it received. + struct ReductionSpyExecutor { + seen: RefCell>, + } + + impl SummaryExecutor for ReductionSpyExecutor { + type Handle = u64; + type State = f64; + type Value = f64; + type Error = String; + type GroupKey = u32; + + fn find_candidates( + &self, + _family: &SummaryFamilyType, + _col: &ColumnRef, + reduction: &Reduction, + _child: &SummaryNode, + ) -> Result, Self::Error> { + self.seen.borrow_mut().push(reduction.clone()); + Ok(vec![(0, 0)]) + } + + fn fetch_state(&self, _handle: &Self::Handle) -> Result { + Ok(1.0) + } + + fn merge_states(&self, states: Vec) -> Result { + Ok(states.into_iter().sum()) + } + + fn readout( + &self, + state: &Self::State, + _query: &SketchQuery, + ) -> Result { + Ok(*state) + } + + fn logical(&self, _expr: &QueryExpr) -> Result { + Ok(-1.0) + } + } + + #[test] + fn reduction_is_passed_through_to_find_candidates_unmodified() { + let exec = ReductionSpyExecutor { + seen: RefCell::new(vec![]), + }; + let tree = agg_node_with_reduction(sum(), logical_node(), Reduction::PerEntity); + execute(&tree, &exec).unwrap(); + assert_eq!(exec.seen.borrow().as_slice(), &[Reduction::PerEntity]); + } +} diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index d687f255..78346e43 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -1,5 +1,5 @@ -//! `data_plane`'s implementation of `asap_sketch::exec::SummaryExecutor` -//! — the serving-time interface that resolves an `L4Node` plan tree +//! `data_plane`'s implementation of `crate::query_engines::asap_query_engine::summary_exec::SummaryExecutor` +//! — the serving-time interface that resolves an `SummaryNode` plan tree //! against whatever is actually materialized right now. See //! `data_plane/docs/l4node-plan-executor-design.md` for the surrounding //! design. @@ -11,7 +11,7 @@ //! specific item key) and a per-item point lookup (`count(cms_metric //! {item="x"})`, `SketchQuery::PointCount{key: Named(_), value: Some(_)}` //! — `value` is where the filter's actual value lives; see -//! `asap_sketch::SketchQuery::PointCount`'s doc for why `readout` can't +//! `planner_types::post_asap::SketchQuery::PointCount`'s doc for why `readout` can't //! resolve it itself from a `Filter` predicate) — for CMS/CountSketch/ //! CMS-with-heap/CountSketch-with-heap, both cumulative (instant) and //! per-window (matrix/range). All modes do real cross-sid merging via @@ -46,7 +46,7 @@ //! `exact_agg_kind_match`'s doc for why `MinMax`/`Count`/`Rate` aren't //! matched) — one sid is one aggregation, read out directly, with no //! special-casing of exact-vs-approximate at the `find_candidates`/merge -//! level. But `readout`/`SketchQuery` NEVER see these: `asap_plan::bind` +//! level. But `readout`/`SketchQuery` NEVER see these: `asap_aware_mapping::bind` //! never wraps an `ExactAccumulator` implementation in a //! `SummaryEstimate` (`estimate = false` in `bind_summary_agg`), so //! `execute()` on such a tree returns `ExecOutcome::State` at the root @@ -63,9 +63,17 @@ use std::collections::{BTreeMap, BTreeSet}; use std::rc::Rc; use std::sync::Arc; -use asap_ir::intent_algebra::{ColumnId, ColumnRef, QueryExpr, Reduction, Source}; -use asap_sketch::exec::SummaryExecutor; -use asap_sketch::{L4Node, SketchQuery, SummaryExpr, SummaryKind, SummaryParams}; +use crate::query_engines::asap_query_engine::summary_exec::SummaryExecutor; +use planner_types::post_asap::{SketchQuery, SummaryExpr, SummaryFamilyType, SummaryNode}; +use planner_types::pre_asap::{ColumnId, ColumnRef, QueryExpr, Reduction, Source}; +// This file's own flat `(SummaryKind, SummaryParams)` -- spans both exact +// accumulators and approximate sketches in one pair, matching every +// internal matcher below (`summary_params_match`/`exact_agg_kind_match`) +// -- vendored because ASAPPlanner split its old flat `SummaryKind` into +// per-family `SketchKind`/`ExactKind` (ASAPPlanner#218). See +// `crates/asap_types/src/accumulator_spec.rs`'s module doc and +// control_plane/docs/design-asapplanner-pin-migration.md. +use asap_types::{SummaryKind, SummaryParams}; use control_plane::sketch_algebra::capability::SketchKindHandle; @@ -323,12 +331,32 @@ impl<'a> SummaryExecutor for QueryExecutionContext<'a> { fn find_candidates( &self, - sketch: &SummaryKind, - params: &SummaryParams, + family: &SummaryFamilyType, _col: &ColumnRef, reduction: &Reduction, - child: &L4Node, + child: &SummaryNode, ) -> Result, Self::Error> { + // `SummaryAgg`'s `kind`/`params` collapsed into this one `family` + // field (ASAPPlanner#218 -- see this file's `use asap_types::{...}` + // note above); recover the flat `(SummaryKind, SummaryParams)` + // pair every matcher below still expects. `Plain`/`Sample`/ + // `Wavelet`/`StatModel` never occur on a real `SummaryAgg` (never + // `Plain` by construction; the others are unreachable via this + // deployment's own `CostModel` -- see + // `sketch_algebra::capability`'s `implementation_to_capability` + // doc for the same reasoning), so there's no candidate to find. + let (sketch, params): (SummaryKind, SummaryParams) = match family { + SummaryFamilyType::ExactAggregate(kind, params) => { + (kind.clone().into(), params.clone().into()) + } + SummaryFamilyType::Sketch(kind, params) => (kind.clone().into(), params.clone().into()), + SummaryFamilyType::Plain(_) + | SummaryFamilyType::Sample(..) + | SummaryFamilyType::Wavelet(..) + | SummaryFamilyType::StatModel(..) => return Ok(Vec::new()), + }; + let sketch = &sketch; + let params = ¶ms; let metric = find_metric(child).ok_or(SummaryExecutorError::NoMetricFound)?; let by: &[ColumnId] = reduction.group_keys().map(|k| k.keys()).unwrap_or(&[]); @@ -387,7 +415,8 @@ impl<'a> SummaryExecutor for QueryExecutionContext<'a> { let Some(series) = series.into_iter().next() else { continue; }; - let group_key = resolve_group_key(reduction, &by_names, &series.series_label_values); + let group_key = + resolve_group_key(reduction, &by_names, &series.series_label_values); out.push(( group_key, SidHandle::Sketch { @@ -418,7 +447,7 @@ impl<'a> SummaryExecutor for QueryExecutionContext<'a> { } } } - // Empty is NOT an error here -- `asap_sketch::exec::execute()` + // Empty is NOT an error here -- `crate::query_engines::asap_query_engine::summary_exec::execute()` // itself checks `find_candidates`'s result for emptiness and // raises the canonical `ExecError::NoCandidates`; erroring here // too would just wrap that in `ExecError::Executor(..)` instead, @@ -477,7 +506,7 @@ impl<'a> SummaryExecutor for QueryExecutionContext<'a> { query: &SketchQuery, ) -> Result { // `ExactAgg` states never reach here in practice -- see this - // module's doc (`asap_plan::bind` never wraps an ExactAccumulator + // module's doc (`asap_aware_mapping::bind` never wraps an ExactAccumulator // in a `SummaryEstimate`, so `execute()` stops at // `ExecOutcome::State` before ever calling `readout`). Defensive, // not a real path. @@ -625,7 +654,7 @@ fn sketch_query_value(rs: &SummaryState, query: &SketchQuery) -> Result Option Option Option { +fn find_metric(node: &SummaryNode) -> Option { match &node.expr { SummaryExpr::Logical(qe) => find_metric_in_query_expr(qe), SummaryExpr::SummaryAgg { child, .. } => find_metric(child), @@ -931,8 +960,7 @@ pub(crate) fn find_metric_in_query_expr(qe: &QueryExpr) -> Option { source: Source::TimeSeries { metric }, .. } => Some(metric.clone()), - QueryExpr::Window { child, .. } - | QueryExpr::Filter { child, .. } + QueryExpr::Filter { child, .. } | QueryExpr::Project { child, .. } | QueryExpr::Aggregate { child, .. } | QueryExpr::Distinct { child, .. } @@ -942,9 +970,6 @@ pub(crate) fn find_metric_in_query_expr(qe: &QueryExpr) -> Option { | QueryExpr::TimeRange { child, .. } | QueryExpr::TimeShift { child, .. } | QueryExpr::WindowFunc { child, .. } => find_metric_in_query_expr(child), - QueryExpr::LetBinding { expr, child, .. } => { - find_metric_in_query_expr(expr).or_else(|| find_metric_in_query_expr(child)) - } QueryExpr::Merge { children } => children.iter().find_map(find_metric_in_query_expr), QueryExpr::Join { left, .. } | QueryExpr::SetOp { left, .. } => { find_metric_in_query_expr(left) @@ -961,15 +986,15 @@ pub(crate) fn find_metric_in_query_expr(qe: &QueryExpr) -> Option { #[cfg(test)] mod tests { use super::*; + use crate::query_engines::asap_query_engine::summary_exec::{execute, ExecOutcome}; use crate::storage_engines::sketch_db::index::{ AccuracyBound, Capability, SketchInstanceMetadata, SketchSampleState, SketchStore, }; - use asap_ir::intent_algebra::{Column, DataType, Schema}; - use asap_sketch::exec::{execute, ExecOutcome}; - use asap_sketch::schema::{L4DataType, L4Field, L4Schema}; + use planner_types::post_asap::{SummaryField, SummarySchema}; + use planner_types::pre_asap::{Column, DataType, Schema}; use std::rc::Rc; - fn scan_node(metric: &str, group_by_field: Option<&str>) -> Rc { + fn scan_node(metric: &str, group_by_field: Option<&str>) -> Rc { let qe = QueryExpr::Scan { source: Source::TimeSeries { metric: metric.to_string(), @@ -984,70 +1009,74 @@ mod tests { vec![], ), }; - let mut fields = vec![L4Field { + let mut fields = vec![SummaryField { name: "value".into(), - dtype: L4DataType::Primitive(DataType::Float64), + dtype: SummaryFamilyType::Plain(DataType::Float64), nullable: false, }]; if let Some(name) = group_by_field { - fields.push(L4Field { + fields.push(SummaryField { name: name.into(), - dtype: L4DataType::Primitive(DataType::Utf8), + dtype: SummaryFamilyType::Plain(DataType::Utf8), nullable: false, }); } - Rc::new(L4Node { + Rc::new(SummaryNode { expr: SummaryExpr::Logical(Box::new(qe)), - schema: L4Schema { + schema: SummarySchema { fields, time_index: None, }, }) } - fn kll_agg_node(child: Rc, reduction: Reduction) -> Rc { - Rc::new(L4Node { + fn kll_agg_node(child: Rc, reduction: Reduction) -> Rc { + Rc::new(SummaryNode { expr: SummaryExpr::SummaryAgg { child, - summary: SummaryKind::Kll, - params: SummaryParams::Kll { k: 200 }, + family: SummaryFamilyType::Sketch( + planner_types::post_asap::SketchKind::Kll, + planner_types::post_asap::SketchParams::Kll { k: 200 }, + ), col: ColumnRef::SampleValue, reduction, }, - schema: L4Schema { + schema: SummarySchema { fields: vec![], time_index: None, }, }) } - fn hll_agg_node(child: Rc) -> Rc { + fn hll_agg_node(child: Rc) -> Rc { hll_agg_node_with(child, Reduction::by(vec![])) } - fn hll_agg_node_with(child: Rc, reduction: Reduction) -> Rc { - Rc::new(L4Node { + fn hll_agg_node_with(child: Rc, reduction: Reduction) -> Rc { + Rc::new(SummaryNode { expr: SummaryExpr::SummaryAgg { child, - summary: SummaryKind::Hll, - params: SummaryParams::Hll { precision: 10 }, + family: SummaryFamilyType::Sketch( + planner_types::post_asap::SketchKind::Hll, + planner_types::post_asap::SketchParams::Hll { precision: 10 }, + ), col: ColumnRef::SampleValue, reduction, }, - schema: L4Schema { + schema: SummarySchema { fields: vec![], time_index: None, }, }) } - fn estimate_node(summary_input: Rc, query: SketchQuery) -> Rc { - Rc::new(L4Node { + fn estimate_node(summary_input: Rc, query: SketchQuery) -> Rc { + Rc::new(SummaryNode { expr: SummaryExpr::SummaryEstimate { summary_input, query, }, - schema: L4Schema { + schema: SummarySchema { fields: vec![], time_index: None, }, @@ -1167,19 +1196,21 @@ mod tests { sk.to_msgpack().expect("encode CountMinSketch msgpack") } - fn cms_agg_node(child: Rc) -> Rc { - Rc::new(L4Node { + fn cms_agg_node(child: Rc) -> Rc { + Rc::new(SummaryNode { expr: SummaryExpr::SummaryAgg { child, - summary: SummaryKind::Cms, - params: SummaryParams::Cms { - width: 256, - depth: 4, - }, + family: SummaryFamilyType::Sketch( + planner_types::post_asap::SketchKind::Cms, + planner_types::post_asap::SketchParams::Cms { + width: 256, + depth: 4, + }, + ), col: ColumnRef::SampleValue, reduction: Reduction::by(vec![]), }, - schema: L4Schema { + schema: SummarySchema { fields: vec![], time_index: None, }, @@ -1223,20 +1254,22 @@ mod tests { .expect("encode CountMinSketchWithHeap msgpack") } - fn cms_with_heap_agg_node(child: Rc) -> Rc { - Rc::new(L4Node { + fn cms_with_heap_agg_node(child: Rc) -> Rc { + Rc::new(SummaryNode { expr: SummaryExpr::SummaryAgg { child, - summary: SummaryKind::CmsWithHeap, - params: SummaryParams::CmsWithHeap { - width: 256, - depth: 4, - heap_size: 10, - }, + family: SummaryFamilyType::Sketch( + planner_types::post_asap::SketchKind::CmsWithHeap, + planner_types::post_asap::SketchParams::CmsWithHeap { + width: 256, + depth: 4, + heap_size: 10, + }, + ), col: ColumnRef::SampleValue, reduction: Reduction::by(vec![]), }, - schema: L4Schema { + schema: SummarySchema { fields: vec![], time_index: None, }, @@ -1267,19 +1300,21 @@ mod tests { } } - fn sum_agg_node(child: Rc, by: Vec) -> Rc { - Rc::new(L4Node { + fn sum_agg_node(child: Rc, by: Vec) -> Rc { + Rc::new(SummaryNode { expr: SummaryExpr::SummaryAgg { child, - summary: SummaryKind::Sum, - params: SummaryParams::Sum, + family: SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Sum, + planner_types::post_asap::ExactParams::Sum, + ), col: ColumnRef::SampleValue, // Sum is a genuine PromQL aggregation operator -- an empty // `by` always means "reduce fully," never `PerEntity` (see // `resolve_group_key`'s doc). reduction: Reduction::by(by), }, - schema: L4Schema { + schema: SummarySchema { fields: vec![], time_index: None, }, @@ -1873,7 +1908,9 @@ mod tests { let tree = estimate_node(cms_agg_node(child), SketchQuery::TopK { k: 5 }); let exec = ctx(&idx); match execute(&tree, &exec) { - Err(asap_sketch::exec::ExecError::Executor(SummaryExecutorError::Unsupported(_))) => {} + Err(crate::query_engines::asap_query_engine::summary_exec::ExecError::Executor( + SummaryExecutorError::Unsupported(_), + )) => {} other => panic!("expected Unsupported, got {}", other.is_ok()), } } @@ -2033,7 +2070,8 @@ mod tests { ); let exec = ctx(&idx); match execute(&tree, &exec) { - Err(asap_sketch::exec::ExecError::NoCandidates) => {} + Err(crate::query_engines::asap_query_engine::summary_exec::ExecError::NoCandidates) => { + } other => panic!("expected NoCandidates, got {}", other.is_ok()), } } @@ -2054,15 +2092,17 @@ mod tests { }, ); let child = scan_node("latency_ms", None); - let mismatched = Rc::new(L4Node { + let mismatched = Rc::new(SummaryNode { expr: SummaryExpr::SummaryAgg { child, - summary: SummaryKind::Kll, - params: SummaryParams::Kll { k: 500 }, + family: SummaryFamilyType::Sketch( + planner_types::post_asap::SketchKind::Kll, + planner_types::post_asap::SketchParams::Kll { k: 500 }, + ), col: ColumnRef::SampleValue, reduction: Reduction::by(vec![]), }, - schema: L4Schema { + schema: SummarySchema { fields: vec![], time_index: None, }, @@ -2070,7 +2110,8 @@ mod tests { let tree = estimate_node(mismatched, SketchQuery::Quantile { q: 0.5 }); let exec = ctx(&idx); match execute(&tree, &exec) { - Err(asap_sketch::exec::ExecError::NoCandidates) => {} + Err(crate::query_engines::asap_query_engine::summary_exec::ExecError::NoCandidates) => { + } other => panic!( "expected NoCandidates (param mismatch), got {}", other.is_ok() @@ -2376,9 +2417,14 @@ mod tests { 1, "ungrouped query produces exactly one group" ); - let (_key, state, kind, params) = &groups[0]; - assert_eq!(*kind, SummaryKind::Sum); - assert_eq!(*params, SummaryParams::Sum); + let (_key, state, family) = &groups[0]; + assert_eq!( + *family, + SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Sum, + planner_types::post_asap::ExactParams::Sum + ) + ); assert_eq!( state.exact_value(&None), Some(25.0), @@ -2423,8 +2469,10 @@ mod tests { // this tree resolves to a real Sketch Value above). let handles = exec .find_candidates( - &SummaryKind::Kll, - &SummaryParams::Kll { k: 200 }, + &SummaryFamilyType::Sketch( + planner_types::post_asap::SketchKind::Kll, + planner_types::post_asap::SketchParams::Kll { k: 200 }, + ), &ColumnRef::SampleValue, &Reduction::by(vec![]), &scan_node("latency_ms", None), @@ -2503,22 +2551,25 @@ mod tests { ); let child = scan_node("latency_max_ms", None); - let tree = Rc::new(L4Node { + let tree = Rc::new(SummaryNode { expr: SummaryExpr::SummaryAgg { child, - summary: SummaryKind::MinMax, - params: SummaryParams::MinMax, + family: SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::MinMax, + planner_types::post_asap::ExactParams::MinMax, + ), col: ColumnRef::SampleValue, reduction: Reduction::by(vec![]), }, - schema: L4Schema { + schema: SummarySchema { fields: vec![], time_index: None, }, }); let exec = ctx(&idx); match execute(&tree, &exec) { - Err(asap_sketch::exec::ExecError::NoCandidates) => {} + Err(crate::query_engines::asap_query_engine::summary_exec::ExecError::NoCandidates) => { + } other => panic!("expected NoCandidates, got {}", other.is_ok()), } } diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index cf996ca2..eaeb0964 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -743,7 +743,11 @@ impl SketchStore { // reducer uses it as the rolling base. Only splice it // when that window-end has no frames yet (don't // duplicate a base the in-window scan already saw). - let frames = by_label_id.entry(label_id).or_default().entry(w_end).or_default(); + let frames = by_label_id + .entry(label_id) + .or_default() + .entry(w_end) + .or_default(); if frames.is_empty() { frames.push(state); } @@ -983,7 +987,11 @@ impl SketchStore { } } for (label_map, (w_end, state)) in latest_full { - let frames = by_label_map.entry(label_map).or_default().entry(w_end).or_default(); + let frames = by_label_map + .entry(label_map) + .or_default() + .entry(w_end) + .or_default(); if frames.is_empty() { frames.push(state); } diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index b2b8e03e..f222cf26 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -1120,7 +1120,8 @@ async fn controller_really_pushes_backend_plan_and_query_really_serves() { assert_eq!(installed["status"], "success"); assert_eq!(installed["plan_id"], 1); assert_eq!( - installed["materialization_count"], 1, + installed["materialization_count"], + 1, "GET /api/v1/backend-plan must reflect the just-installed plan, not a stale/empty one:\n{}", serde_json::to_string_pretty(&installed).unwrap_or_default() ); diff --git a/data_plane/tests/monitor_grpc.rs b/data_plane/tests/monitor_grpc.rs index 91eabc2c..31b1d565 100644 --- a/data_plane/tests/monitor_grpc.rs +++ b/data_plane/tests/monitor_grpc.rs @@ -82,7 +82,9 @@ fn sum_cfg(epsilon: f64) -> MonitorConfig { async fn connect_edge(url: &str) -> (mpsc::Sender, Arc>>) { use asap_otel_proto::monitor::v1::coord_to_edge; - let mut client = MonitorServiceClient::connect(url.to_string()).await.unwrap(); + let mut client = MonitorServiceClient::connect(url.to_string()) + .await + .unwrap(); let (tx, rx) = mpsc::channel(32); let mut inbound = client .monitor(ReceiverStream::new(rx)) @@ -116,7 +118,10 @@ async fn single_edge_gets_sampled_over_grpc() { tokio::time::sleep(Duration::from_millis(150)).await; let got = *p.lock().unwrap().last().expect("edge received a grant"); - assert!(got > 0.0 && got < 1.0, "expected a real sampling grant, got {got}"); + assert!( + got > 0.0 && got < 1.0, + "expected a real sampling grant, got {got}" + ); } /// Live multi-edge coupling: two edges open real gRPC streams, report SKEWED @@ -135,14 +140,28 @@ async fn two_edges_get_differentiated_sample_p_over_grpc() { // e1 hot (100k items/win), e2 quiet (1k/win). for seq in 1..=4 { - tx_hot.send(report_edge("e1", seq, 100_000.0)).await.unwrap(); - tx_quiet.send(report_edge("e2", seq, 1_000.0)).await.unwrap(); + tx_hot + .send(report_edge("e1", seq, 100_000.0)) + .await + .unwrap(); + tx_quiet + .send(report_edge("e2", seq, 1_000.0)) + .await + .unwrap(); tokio::time::sleep(Duration::from_millis(40)).await; } tokio::time::sleep(Duration::from_millis(250)).await; - let hot = *p_hot.lock().unwrap().last().expect("hot edge received a grant"); - let quiet = *p_quiet.lock().unwrap().last().expect("quiet edge received a grant"); + let hot = *p_hot + .lock() + .unwrap() + .last() + .expect("hot edge received a grant"); + let quiet = *p_quiet + .lock() + .unwrap() + .last() + .expect("quiet edge received a grant"); println!("LIVE coupling over gRPC: hot(e1,rate=100k) sample_p={hot} | quiet(e2,rate=1k) sample_p={quiet}"); assert!( @@ -170,6 +189,12 @@ async fn report_grants_only_the_reporting_edge_over_grpc() { tx_a.send(report_edge("e1", 1, 50_000.0)).await.unwrap(); tokio::time::sleep(Duration::from_millis(150)).await; - assert!(!p_a.lock().unwrap().is_empty(), "e1 should have received its own grant"); - assert!(p_b.lock().unwrap().is_empty(), "e2 must not receive a grant it never reported for"); + assert!( + !p_a.lock().unwrap().is_empty(), + "e1 should have received its own grant" + ); + assert!( + p_b.lock().unwrap().is_empty(), + "e2 must not receive a grant it never reported for" + ); }