diff --git a/crates/asap-aware-mapping/src/bind.rs b/crates/asap-aware-mapping/src/bind.rs index 6417ab22..64e721b5 100644 --- a/crates/asap-aware-mapping/src/bind.rs +++ b/crates/asap-aware-mapping/src/bind.rs @@ -1,20 +1,22 @@ //! The pre-ASAP → post-ASAP binding pass (issue #98). //! -//! Walks a canonical pre-ASAP [`QueryExpr`] and emits the sketch-bound +//! Walks a canonical pre-ASAP [`QueryExpr`] and emits the summary-bound //! post-ASAP IR ([`SummaryExpr`] / [`SummaryNode`] in `asap-sketch`). Per //! node, the [`boundary`](crate::boundary) decision picks the realization: //! -//! - **Sketch** — the `Aggregate` becomes a [`SummaryExpr::SummaryAgg`] -//! carrying the committed `(SummaryKind, SummaryParams)`, wrapped in a -//! [`SummaryExpr::SummaryEstimate`] that reads the answer back out (the -//! `Sketch(…)` column type does not propagate past the estimate); -//! - **Exact accumulator** — a `SummaryAgg` with an exact kind -//! (`Sum`/`Count`/`MinMax`/`Rate`/`Increase`) and no estimate: the partial -//! state *is* the value, so a deployment's later finalization step is the -//! identity; +//! - **Summary family** (sketch / sample / wavelet / statistical model) — +//! the `Aggregate` becomes a [`SummaryExpr::SummaryAgg`] carrying the +//! committed `family: SummaryFamilyType` (that family's own +//! `(kind, params)`), wrapped in a [`SummaryExpr::SummaryEstimate`] that +//! reads the answer back out (the summary-state column type does not +//! propagate past the estimate); +//! - **Exact accumulator** — a `SummaryAgg` with `family: +//! SummaryFamilyType::ExactAggregate` (`Sum`/`Count`/`MinMax`/`Rate`/ +//! `Increase`) and no estimate: the partial state *is* the value, so a +//! deployment's later finalization step is the identity; //! - **Pass-through** — the whole pre-ASAP subtree is wrapped as //! [`SummaryExpr::Logical`], schema lifted with every column -//! `SummaryDataType::Primitive`. +//! `SummaryFamilyType::Plain`. //! //! Binding recurses through the `Aggregate` spine, so nested aggregates each //! get their own decision (`quantile(0.9, sum by (svc) (rate(m[5m]))))` binds @@ -34,8 +36,7 @@ use std::rc::Rc; use asap_types::post_asap::{ - SketchQuery, SummaryDataType, SummaryExpr, SummaryField, SummaryKind, SummaryNode, - SummaryParams, SummarySchema, + SketchQuery, SummaryExpr, SummaryFamilyType, SummaryField, SummaryNode, SummarySchema, }; use asap_types::pre_asap::agg_intent::AggIntent; use asap_types::pre_asap::expr_ir::ColumnRef; @@ -79,30 +80,50 @@ pub fn implement_tree_with( // The bindable shape: exactly one intent, no HAVING. (Multi-intent // nodes and HAVING stay logical — see the module docs.) if let ([intent], None) = (measures.as_slice(), having) { - match implementation_for_with(intent, cost_model) { - Implementation::Summary { kind, params } => { - let estimate = !kind.is_exact(); - return bind_summary_agg( - expr, reduction, intent, child, kind, params, estimate, cost_model, - ); - } - Implementation::PassThrough => {} + let implementation = implementation_for_with(intent, cost_model); + if let Some((family, estimate)) = summary_family(implementation) { + return bind_summary_agg( + expr, reduction, intent, child, family, estimate, cost_model, + ); } } } logical(expr) } +/// Translate an [`Implementation`] into the `(family, needs a +/// SummaryEstimate readout)` pair [`bind_summary_agg`] needs, or `None` for +/// `PassThrough` (the caller falls back to [`logical`]). +/// +/// Every family's partial state needs a readout to recover a value, except +/// `ExactAggregate` — its partial state *is* the value already, so no +/// estimate step follows it. +fn summary_family(implementation: Implementation) -> Option<(SummaryFamilyType, bool)> { + Some(match implementation { + Implementation::ExactAggregate { kind, params } => { + (SummaryFamilyType::ExactAggregate(kind, params), false) + } + Implementation::Sketch { kind, params } => (SummaryFamilyType::Sketch(kind, params), true), + Implementation::Sample { kind, params } => (SummaryFamilyType::Sample(kind, params), true), + Implementation::Wavelet { kind, params } => { + (SummaryFamilyType::Wavelet(kind, params), true) + } + Implementation::StatModel { kind, params } => { + (SummaryFamilyType::StatModel(kind, params), true) + } + Implementation::PassThrough => return None, + }) +} + /// Emit `SummaryAgg` (recursively binding the child), plus the -/// `SummaryEstimate` readout when the realization is a sketch. +/// `SummaryEstimate` readout when `estimate` is set. #[allow(clippy::too_many_arguments)] fn bind_summary_agg( node: &QueryExpr, reduction: &Reduction, intent: &AggIntent, child: &QueryExpr, - kind: SummaryKind, - params: SummaryParams, + family: SummaryFamilyType, estimate: bool, cost_model: &dyn CostModel, ) -> Result, ImplementError> { @@ -123,7 +144,7 @@ fn bind_summary_agg( let mut state_schema = lift(&out_schema); if let Some(field) = state_schema.fields.get_mut(state_idx) { - field.dtype = SummaryDataType::Sketch(kind.clone(), params.clone()); + field.dtype = family.clone(); } // `reduction` is carried onto `SummaryAgg` verbatim — not flattened to a @@ -134,8 +155,7 @@ fn bind_summary_agg( let agg = Rc::new(SummaryNode { expr: SummaryExpr::SummaryAgg { child: implement_tree_with(child, cost_model)?, - summary: kind, - params, + family, col, reduction: reduction.clone(), }, @@ -143,7 +163,8 @@ fn bind_summary_agg( }); match query { // The readout: downstream of the estimate the schema is the plain - // pre-ASAP row shape again (the `Sketch(…)` type does not propagate). + // pre-ASAP row shape again (the summary-state type does not + // propagate). Some(query) => Ok(Rc::new(SummaryNode { expr: SummaryExpr::SummaryEstimate { summary_input: agg, @@ -190,7 +211,7 @@ fn summarised_column(intent: &AggIntent, child_schema: &Schema) -> ColumnRef { } } -/// The `SummaryEstimate` readout for a sketch-bound intent. +/// The `SummaryEstimate` readout for a summary-bound intent. fn readout(intent: &AggIntent, col: &ColumnRef, cost_model: &dyn CostModel) -> SketchQuery { match intent { AggIntent::Quantile { q, .. } => SketchQuery::Quantile { q: *q }, @@ -203,17 +224,19 @@ fn readout(intent: &AggIntent, col: &ColumnRef, cost_model: &dyn CostModel) -> S // Core doesn't know the shape of a deployment-specific `Extension` // intent, so it can't build its readout either — delegate to the // same `CostModel` that decided (via `realize_extension`) this - // intent gets a sketch realization at all. See `readout_extension`'s + // intent gets a summary realization at all. See `readout_extension`'s // doc for the invariant this depends on. AggIntent::Extension { ext_kind, payload } => { cost_model.readout_extension(ext_kind, payload, col) } - other => unreachable!("no sketch realization for {other:?} (boundary::implementation_for)"), + other => { + unreachable!("no summary realization for {other:?} (boundary::implementation_for)") + } } } /// Wrap an unrewritten pre-ASAP subtree, lifting its schema with every column -/// `SummaryDataType::Primitive`. Public so a deployment can force a node it +/// `SummaryFamilyType::Plain`. Public so a deployment can force a node it /// knows `implement_tree_in_with` would otherwise actively (mis)bind — /// e.g. an intent this crate's `boundary::implementation_for` maps to an /// accumulator kind the deployment's runtime doesn't actually implement — @@ -234,7 +257,7 @@ fn lift(schema: &Schema) -> SummarySchema { .iter() .map(|c| SummaryField { name: c.name.clone(), - dtype: SummaryDataType::Primitive(c.dtype.clone()), + dtype: SummaryFamilyType::Plain(c.dtype.clone()), nullable: c.nullable, }) .collect(), @@ -245,6 +268,7 @@ fn lift(schema: &Schema) -> SummarySchema { #[cfg(test)] mod tests { use super::*; + use asap_types::post_asap::{ExactKind, ExactParams, SketchKind, SketchParams}; use asap_types::pre_asap::agg_intent::default_quantile; use asap_types::pre_asap::expr_ir::{CompareOp, ScalarValue}; use asap_types::pre_asap::query_expr::{Predicate, Source}; @@ -314,31 +338,32 @@ mod tests { // Estimate edge: plain row shape — group key + Float64 answer. assert_eq!( field(&root.schema, "quantile_0_99").dtype, - SummaryDataType::Primitive(DataType::Float64) + SummaryFamilyType::Plain(DataType::Float64) ); assert_eq!( field(&root.schema, "job").dtype, - SummaryDataType::Primitive(DataType::Utf8) + SummaryFamilyType::Plain(DataType::Utf8) ); let SummaryExpr::SummaryAgg { child, - summary, - params, + family, col, reduction, } = &summary_input.expr else { panic!("expected SummaryAgg, got {:?}", summary_input.expr); }; - assert_eq!(summary, &SummaryKind::Kll); - assert_eq!(params, &SummaryParams::Kll { k: 200 }); + assert_eq!( + family, + &SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) + ); assert_eq!(col, &ColumnRef::SampleValue); assert_eq!(reduction, &Reduction::by(vec![2])); - // SummaryAgg edge: the state column carries the committed (kind, params). + // SummaryAgg edge: the state column carries the committed family. assert_eq!( field(&summary_input.schema, "quantile_0_99").dtype, - SummaryDataType::Sketch(SummaryKind::Kll, SummaryParams::Kll { k: 200 }) + SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) ); assert!(matches!(child.expr, SummaryExpr::Logical(ref e) if matches!(**e, QueryExpr::Scan { .. }))); @@ -353,10 +378,10 @@ mod tests { fn rank_candidates( &self, _intent: &AggIntent, - candidates: &[SummaryKind], - ) -> Vec { + candidates: &[SketchKind], + ) -> Vec { let mut v = candidates.to_vec(); - if let Some(pos) = v.iter().position(|k| *k == SummaryKind::DDSketch) { + if let Some(pos) = v.iter().position(|k| *k == SketchKind::DDSketch) { let ddsketch = v.remove(pos); v.insert(0, ddsketch); } @@ -373,24 +398,29 @@ mod tests { let SummaryExpr::SummaryEstimate { summary_input, .. } = &default_root.expr else { panic!("expected SummaryEstimate root, got {:?}", default_root.expr); }; - let SummaryExpr::SummaryAgg { summary, .. } = &summary_input.expr else { + let SummaryExpr::SummaryAgg { family, .. } = &summary_input.expr else { panic!("expected SummaryAgg, got {:?}", summary_input.expr); }; - assert_eq!(summary, &SummaryKind::Kll); + assert!(matches!( + family, + SummaryFamilyType::Sketch(SketchKind::Kll, _) + )); // With `PreferDDSketch`: DDSketch instead, same query. let custom_root = implement_tree_with(&q, &PreferDDSketch).unwrap(); let SummaryExpr::SummaryEstimate { summary_input, .. } = &custom_root.expr else { panic!("expected SummaryEstimate root, got {:?}", custom_root.expr); }; - let SummaryExpr::SummaryAgg { - summary, params, .. - } = &summary_input.expr - else { + let SummaryExpr::SummaryAgg { family, .. } = &summary_input.expr else { panic!("expected SummaryAgg, got {:?}", summary_input.expr); }; - assert_eq!(summary, &SummaryKind::DDSketch); - assert_eq!(params, &SummaryParams::DDSketch { alpha: 0.01 }); + assert_eq!( + family, + &SummaryFamilyType::Sketch( + SketchKind::DDSketch, + SketchParams::DDSketch { alpha: 0.01 } + ) + ); } /// A deployment-supplied `CostModel` can realize an `AggIntent::Extension` @@ -404,8 +434,8 @@ mod tests { fn rank_candidates( &self, _intent: &AggIntent, - candidates: &[SummaryKind], - ) -> Vec { + candidates: &[SketchKind], + ) -> Vec { candidates.to_vec() } @@ -415,9 +445,9 @@ mod tests { _payload: &serde_json::Value, ) -> crate::boundary::Implementation { if ext_kind == "frequency" { - crate::boundary::Implementation::Summary { - kind: SummaryKind::CountSketch, - params: SummaryParams::CountSketch { + crate::boundary::Implementation::Sketch { + kind: SketchKind::CountSketch, + params: SketchParams::CountSketch { width: 256, depth: 4, }, @@ -478,19 +508,18 @@ mod tests { if k == "item" && v == "checkout" )); - let SummaryExpr::SummaryAgg { - summary, params, .. - } = &summary_input.expr - else { + let SummaryExpr::SummaryAgg { family, .. } = &summary_input.expr else { panic!("expected SummaryAgg, got {:?}", summary_input.expr); }; - assert_eq!(summary, &SummaryKind::CountSketch); assert_eq!( - params, - &SummaryParams::CountSketch { - width: 256, - depth: 4 - } + family, + &SummaryFamilyType::Sketch( + SketchKind::CountSketch, + SketchParams::CountSketch { + width: 256, + depth: 4 + } + ) ); } @@ -498,20 +527,19 @@ mod tests { fn exact_sum_binds_accumulator_without_estimate() { let q = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); let root = implement_tree(&q).unwrap(); - let SummaryExpr::SummaryAgg { - summary, params, .. - } = &root.expr - else { + let SummaryExpr::SummaryAgg { family, .. } = &root.expr else { panic!( "expected bare SummaryAgg (no estimate), got {:?}", root.expr ); }; - assert_eq!(summary, &SummaryKind::Sum); - assert_eq!(params, &SummaryParams::Sum); + assert_eq!( + family, + &SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) + ); assert_eq!( field(&root.schema, "sum").dtype, - SummaryDataType::Sketch(SummaryKind::Sum, SummaryParams::Sum) + SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) ); } @@ -527,10 +555,13 @@ mod tests { }, ); let root = implement_tree(&q).unwrap(); - let SummaryExpr::SummaryAgg { summary, .. } = &root.expr else { + let SummaryExpr::SummaryAgg { family, .. } = &root.expr else { panic!("expected SummaryAgg, got {:?}", root.expr); }; - assert_eq!(summary, &SummaryKind::Rate); + assert_eq!( + family, + &SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) + ); assert_eq!( root.schema .fields @@ -541,7 +572,7 @@ mod tests { ); assert_eq!( field(&root.schema, "value").dtype, - SummaryDataType::Sketch(SummaryKind::Rate, SummaryParams::Rate) + SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) ); assert_eq!(root.schema.time_index, Some(0)); } @@ -603,19 +634,25 @@ mod tests { let SummaryExpr::SummaryEstimate { summary_input, .. } = &root.expr else { panic!("expected estimate root, got {:?}", root.expr); }; - let SummaryExpr::SummaryAgg { child, summary, .. } = &summary_input.expr else { + let SummaryExpr::SummaryAgg { child, family, .. } = &summary_input.expr else { panic!("expected outer SummaryAgg, got {:?}", summary_input.expr); }; - assert_eq!(summary, &SummaryKind::Kll); + assert!(matches!( + family, + SummaryFamilyType::Sketch(SketchKind::Kll, _) + )); let SummaryExpr::SummaryAgg { - summary: inner_kind, + family: inner_family, child: leaf, .. } = &child.expr else { panic!("expected inner SummaryAgg, got {:?}", child.expr); }; - assert_eq!(inner_kind, &SummaryKind::Sum); + assert_eq!( + inner_family, + &SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) + ); assert!(matches!(leaf.expr, SummaryExpr::Logical(_))); } @@ -741,7 +778,7 @@ mod tests { assert!(matches!( &summary_input.expr, SummaryExpr::SummaryAgg { - summary: SummaryKind::CmsWithHeap, + family: SummaryFamilyType::Sketch(SketchKind::CmsWithHeap, _), .. } )); diff --git a/crates/asap-aware-mapping/src/boundary.rs b/crates/asap-aware-mapping/src/boundary.rs index 8139b93c..55e47963 100644 --- a/crates/asap-aware-mapping/src/boundary.rs +++ b/crates/asap-aware-mapping/src/boundary.rs @@ -1,9 +1,10 @@ //! Sketch-vs-exact boundary — the per-intent accuracy decision (issue #98). //! //! The per-node choice of how an [`AggIntent`] is *realised*: by an -//! approximate sketch, by an exact mergeable accumulator, or by an ordinary -//! exact operator (pass-through). This is a post-ASAP concern: the pre-ASAP -//! IR carries only the intent + accuracy target, never the realization. +//! approximate summary (sketch, sample, wavelet, statistical model, …), by +//! an exact mergeable accumulator, or by an ordinary exact operator +//! (pass-through). This is a post-ASAP concern: the pre-ASAP IR carries only +//! the intent + accuracy target, never the realization. //! //! The decision consumes three inputs: //! @@ -23,8 +24,20 @@ //! new `AggIntent` variant fails to compile until it is given an explicit //! realization — there is no silent fall-through. The [`bind`](crate::bind) //! pass fires it per node over nested trees. - -use asap_types::post_asap::{SummaryKind, SummaryParams}; +//! +//! Today's core dispatch only ever picks [`Implementation::ExactAggregate`], +//! [`Implementation::Sketch`], or [`Implementation::PassThrough`] — no +//! `AggIntent` variant maps to sampling/wavelet/statistical-model yet. +//! [`Implementation::Sample`]/[`Wavelet`](Implementation::Wavelet)/ +//! [`StatModel`](Implementation::StatModel) exist so a deployment's own +//! [`CostModel::realize_extension`](crate::cost_model::CostModel::realize_extension) +//! can choose one of them for an `AggIntent::Extension` node — core has no +//! opinion on when that's the right choice. + +use asap_types::post_asap::{ + ExactKind, ExactParams, SamplingKind, SamplingParams, SketchKind, SketchParams, StatModelKind, + StatModelParams, WaveletKind, WaveletParams, +}; use asap_types::pre_asap::agg_intent::{agg_is_mergeable, AggIntent}; use asap_types::types::AccuracyTarget; @@ -33,17 +46,39 @@ use crate::cost_model::{CostModel, DefaultCostModel}; /// How an [`AggIntent`] is realised at post-ASAP binding time. #[derive(Debug, Clone, PartialEq)] pub enum Implementation { - /// A summary — either an approximate sketch sized to the intent's - /// [`AccuracyTarget`], or an exact **mergeable** accumulator (partial - /// state ≡ the value itself: `Sum` / `Count` / `MinMax` / `Rate` / - /// `Increase`). `kind.is_exact()` tells the two apart; binding needs - /// that fact to decide whether a `SummaryEstimate` readout is needed - /// afterward (approximate) or the built state *is* the answer already - /// (exact — no estimate step). Either way this is still a summary — it - /// pre-aggregates and merges across stages. - Summary { - kind: SummaryKind, - params: SummaryParams, + /// An exact **mergeable** accumulator (partial state ≡ the value + /// itself: `Sum` / `Count` / `MinMax` / `Rate` / `Increase`). The + /// built state *is* the answer already — no `SummaryEstimate` readout + /// step. + ExactAggregate { + kind: ExactKind, + params: ExactParams, + }, + /// An approximate sketch sized to the intent's [`AccuracyTarget`]. + /// Needs a `SummaryEstimate` readout to recover a value. + Sketch { + kind: SketchKind, + params: SketchParams, + }, + /// A sampling-based summary (a retained row subset). Needs a + /// `SummaryEstimate` readout. Not chosen by any core `AggIntent` + /// dispatch today — see the module docs. + Sample { + kind: SamplingKind, + params: SamplingParams, + }, + /// A wavelet-transform summary. Needs a `SummaryEstimate` readout. Not + /// chosen by any core `AggIntent` dispatch today — see the module docs. + Wavelet { + kind: WaveletKind, + params: WaveletParams, + }, + /// A fitted statistical/parametric-model summary. Needs a + /// `SummaryEstimate` readout. Not chosen by any core `AggIntent` + /// dispatch today — see the module docs. + StatModel { + kind: StatModelKind, + params: StatModelParams, }, /// No summary form — the node stays a logical pre-ASAP operator and is /// executed exactly (per-series transforms, non-mergeable reducers, exact @@ -51,10 +86,10 @@ pub enum Implementation { PassThrough, } -/// Does an already-**available** [`Implementation`] — e.g. a sketch +/// Does an already-**available** [`Implementation`] — e.g. a summary /// instance a downstream deployment already materialized somewhere, found /// via whatever inventory/index that deployment keeps — satisfy a -/// **required** `Implementation` (what [`implementation_for`]/ +/// **required** [`Implementation`] (what [`implementation_for`]/ /// [`implementation_for_with`] computed for some [`AggIntent`])? /// /// This is the query-optimization-literature "materialized view matching" @@ -69,7 +104,7 @@ pub enum Implementation { /// crate can settle on its own. Two real, reasonable answers already /// diverge outside this crate: /// -/// - A **pure sketch-algebra** answer would say a `Summary{kind: Kll, ..}` +/// - A **pure sketch-algebra** answer would say a `Sketch{kind: Kll, ..}` /// requirement is satisfied by an available `DDSketch` (both quantile /// sketches), and that a heap-bearing top-k sketch also answers a bare /// frequency point-query (the heap is additional info on the same @@ -77,10 +112,10 @@ pub enum Implementation { /// - A **deployment with its own storage-layout rules** may need more: /// e.g. whether a multi-population accumulator can serve a /// single-population query via re-aggregation is a fact about that -/// deployment's storage layout, not about `SummaryKind` at all — -/// `SummaryKind`'s exact accumulators don't encode grouping (grouping -/// lives on the post-ASAP node's `by` instead), so there is nothing in -/// this crate's own vocabulary to subsume. +/// deployment's storage layout, not about any summary family's kind at +/// all — a family's own kind doesn't encode grouping (grouping lives on +/// the post-ASAP node's `by` instead), so there is nothing in this +/// crate's own vocabulary to subsume. /// /// Implementations are expected to consult `required`/`available`'s /// `kind` (and whatever grouping/placement context the deployment tracks @@ -95,19 +130,19 @@ pub trait Matcher { /// one. `ln(1/0.01) → depth 5`, matching the conventional CMS sizing. pub const DEFAULT_DELTA: f64 = 0.01; -/// The summary families that can serve an intent, most-preferred first. -/// This is the `AggIntent → SummaryKind` map of issue #98; [`implementation_for`] binds +/// The sketch families that can serve an intent, most-preferred first. +/// This is the `AggIntent → SketchKind` map of issue #98; [`implementation_for`] binds /// the head of the list. The tail entries are the alternatives a future cost /// model (#6/#33) may pick instead — listed here so the candidate set has one /// home. -pub fn summary_candidates(intent: &AggIntent) -> &'static [SummaryKind] { +pub fn summary_candidates(intent: &AggIntent) -> &'static [SketchKind] { match intent { - AggIntent::Quantile { .. } => &[SummaryKind::Kll, SummaryKind::DDSketch], - AggIntent::Cardinality { .. } => &[SummaryKind::Hll, SummaryKind::Theta, SummaryKind::Kmv], + AggIntent::Quantile { .. } => &[SketchKind::Kll, SketchKind::DDSketch], + AggIntent::Cardinality { .. } => &[SketchKind::Hll, SketchKind::Theta, SketchKind::Kmv], // Count-Sketch-with-heap is CMS-with-heap's balanced/zero-mean-error // alternative for the same heavy-hitter shape. - AggIntent::TopK { .. } => &[SummaryKind::CmsWithHeap, SummaryKind::CountSketchWithHeap], - AggIntent::Count { .. } => &[SummaryKind::Cms, SummaryKind::CountSketch], + AggIntent::TopK { .. } => &[SketchKind::CmsWithHeap, SketchKind::CountSketchWithHeap], + AggIntent::Count { .. } => &[SketchKind::Cms, SketchKind::CountSketch], _ => &[], } } @@ -138,12 +173,14 @@ pub fn implementation_for_with(intent: &AggIntent, cost_model: &dyn CostModel) - }, // ── Exact mergeable accumulators ───────────────────────────────────── - AggIntent::Sum { .. } => accumulator(intent, SummaryKind::Sum, SummaryParams::Sum), + AggIntent::Sum { .. } => exact_accumulator(intent, ExactKind::Sum, ExactParams::Sum), AggIntent::Min { .. } | AggIntent::Max { .. } => { - accumulator(intent, SummaryKind::MinMax, SummaryParams::MinMax) + exact_accumulator(intent, ExactKind::MinMax, ExactParams::MinMax) + } + AggIntent::Rate => exact_accumulator(intent, ExactKind::Rate, ExactParams::Rate), + AggIntent::Increase => { + exact_accumulator(intent, ExactKind::Increase, ExactParams::Increase) } - AggIntent::Rate => accumulator(intent, SummaryKind::Rate, SummaryParams::Rate), - AggIntent::Increase => accumulator(intent, SummaryKind::Increase, SummaryParams::Increase), // ── Exact, non-mergeable reducers — richer partial state than a // single value (see `agg_is_mergeable`), so no accumulator form. @@ -195,7 +232,9 @@ pub fn implementation_for_with(intent: &AggIntent, cost_model: &dyn CostModel) - // realization opinion for a shape it doesn't know, so it defers // entirely to the `CostModel` (issue #150): `realize_extension` // defaults to `PassThrough`, preserving today's behavior for - // every deployment that doesn't override it. + // every deployment that doesn't override it. This is also the + // only path that can currently produce `Implementation::Sample`/ + // `Wavelet`/`StatModel` — see the module docs. AggIntent::Extension { ext_kind, payload } => { cost_model.realize_extension(ext_kind, payload) } @@ -208,26 +247,22 @@ pub fn implementation_for_with(intent: &AggIntent, cost_model: &dyn CostModel) - /// need the full multiset / heap / set) and pass through. fn exact_realization(intent: &AggIntent) -> Implementation { match intent { - AggIntent::Count { .. } => accumulator(intent, SummaryKind::Count, SummaryParams::Count), + AggIntent::Count { .. } => exact_accumulator(intent, ExactKind::Count, ExactParams::Count), _ => Implementation::PassThrough, } } -fn accumulator(intent: &AggIntent, kind: SummaryKind, params: SummaryParams) -> Implementation { +fn exact_accumulator(intent: &AggIntent, kind: ExactKind, params: ExactParams) -> Implementation { // An exact accumulator is only sound when partial states merge // (`agg(A ∪ B) = combine(agg(A), agg(B))`). debug_assert!( agg_is_mergeable(intent), "accumulator for non-mergeable {intent:?}" ); - debug_assert!( - kind.is_exact(), - "accumulator() called with a non-exact kind {kind:?}" - ); - Implementation::Summary { kind, params } + Implementation::ExactAggregate { kind, params } } -/// Bind the preferred candidate summary, with parameters sized to the +/// Bind the preferred candidate sketch, with parameters sized to the /// target, ranking [`summary_candidates`] via `cost_model` (see /// [`crate::cost_model`]) instead of taking the static-order head /// unconditionally. @@ -249,10 +284,10 @@ fn bind_summary_with( .next() .expect("approximate intent has at least one candidate summary"); let params = cost_model.size_params(kind.clone(), intent, eps, delta); - Implementation::Summary { kind, params } + Implementation::Sketch { kind, params } } -/// `asap-plan`'s built-in `SummaryParams` sizing, keyed off the resolved +/// `asap-plan`'s built-in `SketchParams` sizing, keyed off the resolved /// `(eps, delta)` accuracy budget. [`CostModel::size_params`]'s default /// body — factored out to a free function so a deployment's own /// `CostModel` impl can still delegate to it for the candidates it @@ -263,26 +298,26 @@ fn bind_summary_with( /// range. A non-positive ε saturates to the clamp maximum (tightest /// allowed). pub fn default_size_params( - kind: SummaryKind, + kind: SketchKind, intent: &AggIntent, eps: f64, delta: f64, -) -> SummaryParams { +) -> SketchParams { match kind { - SummaryKind::Kll => SummaryParams::Kll { k: kll_k(eps) }, - SummaryKind::Cms => SummaryParams::Cms { + SketchKind::Kll => SketchParams::Kll { k: kll_k(eps) }, + SketchKind::Cms => SketchParams::Cms { width: cms_width(eps), depth: cms_depth(delta), }, - SummaryKind::Hll => SummaryParams::Hll { + SketchKind::Hll => SketchParams::Hll { precision: hll_precision(eps), }, - SummaryKind::CmsWithHeap => { + SketchKind::CmsWithHeap => { let k = match intent { AggIntent::TopK { k, .. } => *k, _ => unreachable!("CmsWithHeap is only a TopK candidate"), }; - SummaryParams::CmsWithHeap { + SketchParams::CmsWithHeap { width: cms_width(eps), depth: cms_depth(delta), heap_size: k as u32, @@ -291,35 +326,30 @@ pub fn default_size_params( // Non-preferred candidates (DDSketch / Theta / Kmv / CountSketch / // CountSketchWithHeap) are only reachable once a cost model picks // them; sized here so that wiring is local. - SummaryKind::DDSketch => SummaryParams::DDSketch { alpha: eps }, - SummaryKind::Theta => SummaryParams::Theta { k: kmv_k(eps) }, - SummaryKind::Kmv => SummaryParams::Kmv { k: kmv_k(eps) }, + SketchKind::DDSketch => SketchParams::DDSketch { alpha: eps }, + SketchKind::Theta => SketchParams::Theta { k: kmv_k(eps) }, + SketchKind::Kmv => SketchParams::Kmv { k: kmv_k(eps) }, // Count-Sketch is CMS's balanced/zero-mean-error alternative — // same (width, depth) shape, sized the same way for now (a // Count-Sketch-specific bound uses an L2-norm error guarantee // rather than CMS's L1-norm one; this is a placeholder pending // that refinement, same status as the other non-preferred // candidates above). - SummaryKind::CountSketch => SummaryParams::CountSketch { + SketchKind::CountSketch => SketchParams::CountSketch { width: cms_width(eps), depth: cms_depth(delta), }, - SummaryKind::CountSketchWithHeap => { + SketchKind::CountSketchWithHeap => { let k = match intent { AggIntent::TopK { k, .. } => *k, _ => unreachable!("CountSketchWithHeap is only a TopK candidate"), }; - SummaryParams::CountSketchWithHeap { + SketchParams::CountSketchWithHeap { width: cms_width(eps), depth: cms_depth(delta), heap_size: k as u32, } } - SummaryKind::Sum - | SummaryKind::Count - | SummaryKind::MinMax - | SummaryKind::Increase - | SummaryKind::Rate => unreachable!("exact accumulators are not sketch candidates"), } } @@ -380,16 +410,19 @@ mod tests { /// Shorthand for asserting the realization *category*. #[derive(Debug, PartialEq)] enum Cat { - Sketch(SummaryKind), - Acc(SummaryKind), + Sketch(SketchKind), + Acc(ExactKind), Pass, } fn cat(intent: &AggIntent) -> Cat { match implementation_for(intent) { - Implementation::Summary { kind, .. } if kind.is_exact() => Cat::Acc(kind), - Implementation::Summary { kind, .. } => Cat::Sketch(kind), + Implementation::ExactAggregate { kind, .. } => Cat::Acc(kind), + Implementation::Sketch { kind, .. } => Cat::Sketch(kind), Implementation::PassThrough => Cat::Pass, + other => { + panic!("this coverage matrix expects only Exact/Sketch/PassThrough, got {other:?}") + } } } @@ -401,7 +434,8 @@ mod tests { fn agg_intent_to_summary_kind_coverage_matrix() { use AggIntent as A; use Cat::*; - use SummaryKind as K; + use ExactKind as E; + use SketchKind as K; let matrix: Vec<(A, Cat)> = vec![ // approximate-capable, at an ε target → sketch (default_quantile(0.99), Sketch(K::Kll)), @@ -439,7 +473,7 @@ mod tests { A::Count { accuracy: AccuracyTarget::Exact, }, - Acc(K::Count), + Acc(E::Count), ), ( A::TopK { @@ -449,11 +483,11 @@ mod tests { Pass, ), // exact mergeable accumulators - (A::Sum { col: None }, Acc(K::Sum)), - (A::Min { col: None }, Acc(K::MinMax)), - (A::Max { col: None }, Acc(K::MinMax)), - (A::Rate, Acc(K::Rate)), - (A::Increase, Acc(K::Increase)), + (A::Sum { col: None }, Acc(E::Sum)), + (A::Min { col: None }, Acc(E::MinMax)), + (A::Max { col: None }, Acc(E::MinMax)), + (A::Rate, Acc(E::Rate)), + (A::Increase, Acc(E::Increase)), // exact but non-mergeable → pass-through (A::Avg { col: None }, Pass), ( @@ -548,9 +582,9 @@ mod tests { let approx = default_quantile(0.99); // ε = 0.01 assert_eq!( implementation_for(&approx), - Implementation::Summary { - kind: SummaryKind::Kll, - params: SummaryParams::Kll { k: 200 }, // design.md worked example + Implementation::Sketch { + kind: SketchKind::Kll, + params: SketchParams::Kll { k: 200 }, // design.md worked example } ); @@ -561,9 +595,9 @@ mod tests { }; assert_eq!( implementation_for(&looser), - Implementation::Summary { - kind: SummaryKind::Kll, - params: SummaryParams::Kll { k: 40 }, // ⌈2/0.05⌉ + Implementation::Sketch { + kind: SketchKind::Kll, + params: SketchParams::Kll { k: 40 }, // ⌈2/0.05⌉ } ); } @@ -574,9 +608,9 @@ mod tests { // sizing must invert it back exactly. assert_eq!( implementation_for(&default_cardinality()), - Implementation::Summary { - kind: SummaryKind::Hll, - params: SummaryParams::Hll { precision: 14 }, + Implementation::Sketch { + kind: SketchKind::Hll, + params: SketchParams::Hll { precision: 14 }, } ); } @@ -591,9 +625,9 @@ mod tests { }; assert_eq!( implementation_for(&intent), - Implementation::Summary { - kind: SummaryKind::Cms, - params: SummaryParams::Cms { + Implementation::Sketch { + kind: SketchKind::Cms, + params: SketchParams::Cms { width: 2719, depth: 7 }, // ⌈e/0.001⌉, ⌈ln 1000⌉ @@ -605,9 +639,9 @@ mod tests { }; assert_eq!( implementation_for(&intent), - Implementation::Summary { - kind: SummaryKind::Cms, - params: SummaryParams::Cms { + Implementation::Sketch { + kind: SketchKind::Cms, + params: SketchParams::Cms { width: 2719, depth: 5 }, @@ -622,10 +656,10 @@ mod tests { accuracy: eps(0.01), }; match implementation_for(&intent) { - Implementation::Summary { - kind: SummaryKind::CmsWithHeap, + Implementation::Sketch { + kind: SketchKind::CmsWithHeap, params: - SummaryParams::CmsWithHeap { + SketchParams::CmsWithHeap { width, depth, heap_size, @@ -643,24 +677,24 @@ mod tests { fn candidate_lists_match_the_issue_map() { assert_eq!( summary_candidates(&default_quantile(0.5)), - &[SummaryKind::Kll, SummaryKind::DDSketch] + &[SketchKind::Kll, SketchKind::DDSketch] ); assert_eq!( summary_candidates(&default_cardinality()), - &[SummaryKind::Hll, SummaryKind::Theta, SummaryKind::Kmv] + &[SketchKind::Hll, SketchKind::Theta, SketchKind::Kmv] ); assert_eq!( summary_candidates(&AggIntent::TopK { k: 5, accuracy: eps(0.01) }), - &[SummaryKind::CmsWithHeap, SummaryKind::CountSketchWithHeap] + &[SketchKind::CmsWithHeap, SketchKind::CountSketchWithHeap] ); assert_eq!( summary_candidates(&AggIntent::Count { accuracy: eps(0.01) }), - &[SummaryKind::Cms, SummaryKind::CountSketch] + &[SketchKind::Cms, SketchKind::CountSketch] ); assert!(summary_candidates(&AggIntent::Rate).is_empty()); } @@ -674,9 +708,9 @@ mod tests { }; assert_eq!( implementation_for(&intent), - Implementation::Summary { - kind: SummaryKind::Kll, - params: SummaryParams::Kll { k: 65_535 }, + Implementation::Sketch { + kind: SketchKind::Kll, + params: SketchParams::Kll { k: 65_535 }, } ); } diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index 4ac7a0b1..713c79ba 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -11,14 +11,19 @@ //! each downstream (ASAPCollector + ASAPQuery-backend, ASAPFusion, …) to //! fork [`boundary::implementation_for`]. //! -//! "Summary" here is deliberately broader than the classic streaming -//! sketches [`SummaryKind`] enumerates today (Kll/DDSketch/Hll/Cms/…): -//! [`CostModel::rank_candidates`] ranks whatever [`SummaryKind`] variants -//! [`boundary::summary_candidates`] offers for an intent, so it already -//! extends to non-sketch realizations — sampling-based summaries, -//! wavelet-transform summaries, OMP/compressive-sensing summaries, … — -//! the day a variant for one lands in [`asap_sketch`]; nothing in this -//! trait or [`boundary`]'s dispatch is sketch-specific. +//! This trait is scoped to the approximate-**sketch** family specifically +//! ([`CostModel::rank_candidates`]/[`size_params`](CostModel::size_params) +//! take/return [`SketchKind`]/[`SketchParams`]) — `asap_sketch` also has +//! sibling families for sampling-based, wavelet-transform, and fitted +//! statistical-model summaries +//! ([`asap_types::post_asap::SamplingKind`]/…/[`asap_types::post_asap::StatModelKind`]), +//! each with its own `(Kind, Params)` pair, deliberately *not* folded into +//! this trait: no core `AggIntent` picks one of those families today (only +//! [`CostModel::realize_extension`] can, for a deployment-specific +//! `AggIntent::Extension`), so there is no ranking/sizing decision for this +//! trait to own yet. Should a family other than `Sketch` ever need its own +//! `rank_candidates`/`size_params`, it gets its own trait methods rather +//! than overloading these ones across incompatible `Kind`/`Params` types. //! //! Every entry point that doesn't take an explicit `&dyn CostModel` //! ([`implementation_for`](crate::boundary::implementation_for), @@ -27,7 +32,7 @@ //! model keeps today's static-preference-order behavior exactly, byte for //! byte. -use asap_types::post_asap::{SketchQuery, SummaryKind, SummaryParams}; +use asap_types::post_asap::{SketchKind, SketchParams, SketchQuery}; use asap_types::pre_asap::agg_intent::AggIntent; use asap_types::pre_asap::expr_ir::ColumnRef; @@ -49,15 +54,15 @@ pub trait CostModel { /// /// Implementations MAY reorder freely and MAY drop entries that aren't /// available in their deployment, but MUST NOT invent a candidate that - /// wasn't in the input — an unknown [`SummaryKind`] has no - /// [`SummaryParams`](asap_types::post_asap::SummaryParams) sizing logic in + /// wasn't in the input — an unknown [`SketchKind`] has no + /// [`SketchParams`](asap_types::post_asap::SketchParams) sizing logic in /// [`boundary::implementation_for_with`] and binding it will panic. /// Returning an empty `Vec` means "no candidate is acceptable"; /// `implementation_for_with` treats that the same as `candidates` /// having been empty to begin with. - fn rank_candidates(&self, intent: &AggIntent, candidates: &[SummaryKind]) -> Vec; + fn rank_candidates(&self, intent: &AggIntent, candidates: &[SketchKind]) -> Vec; - /// Size [`SummaryParams`] for `kind` (one of the candidates + /// Size [`SketchParams`] for `kind` (one of the candidates /// [`rank_candidates`](Self::rank_candidates) put first) under the /// resolved `(eps, delta)` accuracy budget. /// @@ -71,11 +76,11 @@ pub trait CostModel { /// not resize them, can leave this method unimplemented. fn size_params( &self, - kind: SummaryKind, + kind: SketchKind, intent: &AggIntent, eps: f64, delta: f64, - ) -> SummaryParams { + ) -> SketchParams { crate::boundary::default_size_params(kind, intent, eps, delta) } @@ -120,7 +125,7 @@ pub trait CostModel { pub struct DefaultCostModel; impl CostModel for DefaultCostModel { - fn rank_candidates(&self, _intent: &AggIntent, candidates: &[SummaryKind]) -> Vec { + fn rank_candidates(&self, _intent: &AggIntent, candidates: &[SketchKind]) -> Vec { candidates.to_vec() } } @@ -147,8 +152,8 @@ mod tests { fn rank_candidates( &self, _intent: &AggIntent, - candidates: &[SummaryKind], - ) -> Vec { + candidates: &[SketchKind], + ) -> Vec { let mut v = candidates.to_vec(); v.reverse(); v @@ -170,8 +175,8 @@ mod tests { fn size_params_default_body_matches_default_size_params() { let intent = default_cardinality(); assert_eq!( - AlwaysPreferLast.size_params(SummaryKind::Hll, &intent, 0.01, 0.01), - crate::boundary::default_size_params(SummaryKind::Hll, &intent, 0.01, 0.01), + AlwaysPreferLast.size_params(SketchKind::Hll, &intent, 0.01, 0.01), + crate::boundary::default_size_params(SketchKind::Hll, &intent, 0.01, 0.01), ); } @@ -185,22 +190,22 @@ mod tests { fn rank_candidates( &self, _intent: &AggIntent, - candidates: &[SummaryKind], - ) -> Vec { + candidates: &[SketchKind], + ) -> Vec { candidates.to_vec() } fn size_params( &self, - kind: SummaryKind, + kind: SketchKind, intent: &AggIntent, eps: f64, delta: f64, - ) -> SummaryParams { + ) -> SketchParams { match kind { - SummaryKind::Kll => { + SketchKind::Kll => { let k = if eps >= 0.01 { 200 } else { 2048 }; - SummaryParams::Kll { k } + SketchParams::Kll { k } } other => crate::boundary::default_size_params(other, intent, eps, delta), } @@ -213,13 +218,13 @@ mod tests { let intent = default_quantile(0.99); assert_eq!( - DiscreteKllRungs.size_params(SummaryKind::Kll, &intent, 0.001, 0.01), - SummaryParams::Kll { k: 2048 }, + DiscreteKllRungs.size_params(SketchKind::Kll, &intent, 0.001, 0.01), + SketchParams::Kll { k: 2048 }, ); // Untouched kinds still fall through to the default formula. assert_eq!( - DiscreteKllRungs.size_params(SummaryKind::Hll, &intent, 0.01, 0.01), - crate::boundary::default_size_params(SummaryKind::Hll, &intent, 0.01, 0.01), + DiscreteKllRungs.size_params(SketchKind::Hll, &intent, 0.01, 0.01), + crate::boundary::default_size_params(SketchKind::Hll, &intent, 0.01, 0.01), ); } } diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index d963d81e..77323ec0 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -19,8 +19,9 @@ //! Two real occupants and one stub: //! //! - [`boundary`] — the per-intent sketch-vs-exact (accuracy) decision: -//! `AggIntent → SummaryKind + SummaryParams` sized to the `AccuracyTarget` -//! (issue #98). [`boundary::implementation_for`] is the per-node decision; +//! `AggIntent → Implementation` (a summary family's own `(Kind, Params)`, +//! or an exact accumulator) sized to the `AccuracyTarget` (issue #98). +//! [`boundary::implementation_for`] is the per-node decision; //! [`bind::implement_tree`] drives it over a whole tree — see the //! terminology section below for why these two are named around //! "implementation" rather than "bind". @@ -64,7 +65,7 @@ //! compatibility rules (e.g. a heap-bearing top-k sketch also satisfying a //! bare frequency point-query) turned out to have deployment-specific //! competitors (e.g. single-vs-multi-population re-aggregation) that -//! don't reduce to a fact about `SummaryKind` alone. `control_plane`'s own +//! don't reduce to a fact about a summary family's kind alone. `control_plane`'s own //! `sketch_algebra::capability::Capability`/`is_satisfied_by` is the //! reference downstream implementation. diff --git a/crates/integration-tests/tests/l4_binding.rs b/crates/integration-tests/tests/l4_binding.rs index 907d9011..0eddcf26 100644 --- a/crates/integration-tests/tests/l4_binding.rs +++ b/crates/integration-tests/tests/l4_binding.rs @@ -2,22 +2,23 @@ //! //! Drives the full pipeline — PromQL text → pre-ASAP `QueryExpr` //! (`lower_promql`) → post-ASAP `SummaryExpr` DAG -//! (`asap_aware_mapping::implement_tree`) — and pins the sketch-bound shape -//! node by node, including the `(SummaryKind, SummaryParams)` committed on -//! each edge's schema. This is the design doc's §"L4 — sketch algebra" -//! worked example, running for real. +//! (`asap_aware_mapping::implement_tree`) — and pins the summary-bound shape +//! node by node, including the family `(Kind, Params)` committed on each +//! edge's schema. This is the design doc's §"L4 — sketch algebra" worked +//! example, running for real. use asap_aware_mapping::implement_tree; use asap_frontend_promql::lower_promql; use asap_types::post_asap::{ - SketchQuery, SummaryDataType, SummaryExpr, SummaryKind, SummaryParams, SummarySchema, + ExactKind, ExactParams, SketchKind, SketchParams, SketchQuery, SummaryExpr, SummaryFamilyType, + SummarySchema, }; use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::query_expr::{QueryExpr, Reduction}; use asap_types::pre_asap::schema::DataType; use asap_types::types::AccuracyTarget; -fn dtype<'a>(schema: &'a SummarySchema, name: &str) -> &'a SummaryDataType { +fn dtype<'a>(schema: &'a SummarySchema, name: &str) -> &'a SummaryFamilyType { &schema .fields .iter() @@ -31,7 +32,7 @@ fn dtype<'a>(schema: &'a SummarySchema, name: &str) -> &'a SummaryDataType { /// ```text /// SummaryEstimate { query: Quantile{0.99} } → {quantile_0_99: Float64} /// └─ SummaryAgg { Kll{k:200}, col: SampleValue } → {quantile_0_99: Sketch(Kll, {k:200})} -/// └─ SummaryAgg { Rate, col: SampleValue } → {ts, value: Sketch(Rate), …} +/// └─ SummaryAgg { Rate, col: SampleValue } → {ts, value: ExactAggregate(Rate), …} /// └─ Logical(TimeRange{5m} → Scan) → {ts, value} /// ``` /// @@ -58,8 +59,8 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { assert!(matches!(query, SketchQuery::Quantile { q } if *q == 0.99)); assert_eq!( dtype(&root.schema, "quantile_0_99"), - &SummaryDataType::Primitive(DataType::Float64), - "the Sketch(…) type must not propagate past the estimate" + &SummaryFamilyType::Plain(DataType::Float64), + "the summary-state type must not propagate past the estimate" ); // The quantile: KLL committed, k=200 sized from ε=0.01. `quantile(...)` @@ -69,16 +70,17 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { // same empty `by: []` (issue #163). let SummaryExpr::SummaryAgg { child, - summary, - params, + family, col, reduction, } = &summary_input.expr else { panic!("expected SummaryAgg, got {:?}", summary_input.expr); }; - assert_eq!(summary, &SummaryKind::Kll); - assert_eq!(params, &SummaryParams::Kll { k: 200 }); + assert_eq!( + family, + &SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) + ); assert_eq!(col, &ColumnRef::SampleValue); assert_eq!( reduction, @@ -87,7 +89,7 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { ); assert_eq!( dtype(&summary_input.schema, "quantile_0_99"), - &SummaryDataType::Sketch(SummaryKind::Kll, SummaryParams::Kll { k: 200 }) + &SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) ); // The rate: exact counter-reset-aware accumulator, per-series (labels @@ -95,20 +97,21 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { // grouping concept at all — every entity stays its own summary. let SummaryExpr::SummaryAgg { child: leaf, - summary, - params, + family, reduction, .. } = &child.expr else { panic!("expected inner SummaryAgg for rate, got {:?}", child.expr); }; - assert_eq!(summary, &SummaryKind::Rate); - assert_eq!(params, &SummaryParams::Rate); + assert_eq!( + family, + &SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) + ); assert_eq!(reduction, &Reduction::PerEntity); assert_eq!( dtype(&child.schema, "value"), - &SummaryDataType::Sketch(SummaryKind::Rate, SummaryParams::Rate) + &SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) ); assert_eq!( child.schema.time_index, @@ -129,8 +132,8 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { leaf.schema .fields .iter() - .all(|f| matches!(f.dtype, SummaryDataType::Primitive(_))), - "logical edges carry only primitive columns" + .all(|f| matches!(f.dtype, SummaryFamilyType::Plain(_))), + "logical edges carry only plain columns" ); } @@ -143,16 +146,15 @@ fn promql_exact_workload_binds_accumulators_not_sketches() { .expect("lowering failed"); let root = implement_tree(&l3).expect("binding failed"); let SummaryExpr::SummaryAgg { - summary, - params, - reduction, - .. + family, reduction, .. } = &root.expr else { panic!("expected SummaryAgg, got {:?}", root.expr); }; - assert_eq!(summary, &SummaryKind::Sum); - assert_eq!(params, &SummaryParams::Sum); + assert_eq!( + family, + &SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) + ); assert_eq!( reduction, &Reduction::by(vec![2]), @@ -160,7 +162,7 @@ fn promql_exact_workload_binds_accumulators_not_sketches() { ); assert_eq!( dtype(&root.schema, "job"), - &SummaryDataType::Primitive(DataType::Utf8), + &SummaryFamilyType::Plain(DataType::Utf8), "group keys pass through verbatim" ); diff --git a/crates/types/src/post_asap/expr.rs b/crates/types/src/post_asap/expr.rs index e7c97eff..2bc23afc 100644 --- a/crates/types/src/post_asap/expr.rs +++ b/crates/types/src/post_asap/expr.rs @@ -1,14 +1,15 @@ use std::rc::Rc; -use super::schema::SummarySchema; -use super::sketch::{SketchQuery, SummaryKind, SummaryParams}; +use super::schema::{SummaryFamilyType, SummarySchema}; +use super::sketch::SketchQuery; use crate::pre_asap::{ColumnRef, QueryExpr, Reduction}; // ── Post-ASAP DAG node ─────────────────────────────────────────────────────── /// A node in the post-ASAP DAG: wraps the expression and its derived output /// schema so every edge carries a typed schema. `SummarySchema` may contain -/// `SummaryDataType::Sketch` columns; the pre-ASAP `Schema` cannot. +/// summary-state-typed columns (`SummaryFamilyType`'s non-`Plain` variants); +/// the pre-ASAP `Schema` cannot. #[derive(Debug, Clone)] pub struct SummaryNode { pub expr: SummaryExpr, @@ -21,7 +22,7 @@ pub struct SummaryNode { /// Sketch-bound IR produced by post-ASAP binding rules. Those rules /// selectively replace logical aggregates and joins in the pre-ASAP -/// `QueryExpr` with their sketch-bound counterparts; everything not +/// `QueryExpr` with their summary-bound counterparts; everything not /// rewritten passes through as `Logical(Box)`. /// /// Traversing from the root node yields a DAG; shared sub-expressions appear @@ -30,18 +31,22 @@ pub struct SummaryNode { pub enum SummaryExpr { /// Any pre-ASAP node no binding rule rewrote (e.g. `Filter`, `Project`, /// `Sort`). Output schema is the inner node's schema, lifted to - /// `SummarySchema` with all fields as `SummaryDataType::Primitive`. + /// `SummarySchema` with all fields as `SummaryFamilyType::Plain`. Logical(Box), - /// Summary aggregation. Post-ASAP binding chose `summary` + `params` - /// from the catalog for `AggIntent` under `DeploymentConstraints`. - /// Output schema: grouping columns (verbatim) + one `Sketch(summary, - /// params)` field carrying partial summary state per group. + /// Summary aggregation. Post-ASAP binding chose `family` — which + /// summary family (exact accumulator, sketch, sample, wavelet, or + /// statistical model) and its `(kind, params)` — from the catalog for + /// `AggIntent` under `DeploymentConstraints`. + /// Output schema: grouping columns (verbatim) + one field carrying + /// partial summary state per group, typed `family`. SummaryAgg { child: Rc, - summary: SummaryKind, - params: SummaryParams, - /// The column being summarised (fed into the sketch). + /// Which summary family realizes this aggregation, and that + /// family's own `(kind, params)`. Never `SummaryFamilyType::Plain` + /// — this node always produces summary state, not a plain value. + family: SummaryFamilyType, + /// The column being summarised (fed into the summary). col: ColumnRef, /// How this aggregation's output rows relate to `child`'s — the /// same [`Reduction`] the pre-ASAP `Aggregate` node it was bound @@ -55,36 +60,36 @@ pub enum SummaryExpr { reduction: Reduction, }, - /// Sketch-aware join (KMV / theta for join-cardinality; join-sample for + /// Summary-aware join (KMV / theta for join-cardinality; join-sample for /// sampling). Emitted only when a `Bind*OnJoin` rule fires. - /// Output schema: one `Sketch(sketch, params)` field read by a downstream + /// Output schema: one field typed `family`, read by a downstream /// `SummaryEstimate`. SummaryJoin { outer: Rc, inner: Rc, key: ColumnRef, - summary: SummaryKind, - params: SummaryParams, + /// Never `SummaryFamilyType::Plain` — see [`SummaryAgg::family`](SummaryExpr::SummaryAgg). + family: SummaryFamilyType, }, - /// Subtract one sketch from another. Valid only for sketches with a + /// Subtract one summary from another. Valid only for families with a /// linear-inverse property (CMS, theta, count-based). Catalog flag - /// `subtractable` must be true for the sketch family. - /// Output schema: one `Sketch(s, p)` field (same family + params as inputs). + /// `subtractable` must be true for the family. + /// Output schema: one field (same family + params as inputs). SummarySubtract { left: Rc, right: Rc, }, - /// Delete a key from a sketch (CMS update with −1, deletable Bloom + /// Delete a key from a summary (CMS update with −1, deletable Bloom /// filter). Catalog flag `deletable` must be true. Output schema = - /// input schema unchanged in type (same `Sketch(s, p)` field). + /// input schema unchanged in type (same field type as input). SummaryDelete { summary_input: Rc, key: ColumnRef, }, - /// Read out a query result from a built summary. The `Sketch(…)` field + /// Read out a query result from a built summary. The summary-state field /// type does *not* propagate downstream of an estimate — the output /// schema is a regular row-shaped schema (Float64 for quantile, Int64 /// for count/cardinality, `[(key, count)]` for top-k). @@ -93,11 +98,11 @@ pub enum SummaryExpr { query: SketchQuery, }, - /// ⊕ — union of sketches across stages / shards. Distinct from the - /// pre-ASAP `Merge` because sketch union has type constraints: all - /// inputs must agree on `(sketch, params)` and the catalog flag + /// ⊕ — union of summaries across stages / shards. Distinct from the + /// pre-ASAP `Merge` because summary union has type constraints: all + /// inputs must agree on `family` (kind + params) and the catalog flag /// `mergeable` must be true. Inserted by a deployment's own stage /// allocator (not modeled in this crate) on cut edges. - /// Output schema: one `Sketch(s, p)` field (same family + params as inputs). + /// Output schema: one field (same family + params as inputs). SummaryMerge { children: Vec> }, } diff --git a/crates/types/src/post_asap/mod.rs b/crates/types/src/post_asap/mod.rs index 4097e6b8..788f495a 100644 --- a/crates/types/src/post_asap/mod.rs +++ b/crates/types/src/post_asap/mod.rs @@ -1,16 +1,24 @@ -//! The post-ASAP IR: sketch-bound types, distinct from +//! The post-ASAP IR: summary-bound types, distinct from //! [`crate::pre_asap`]'s pre-ASAP IR. //! //! Where [`crate::pre_asap`] carries *intent* only ("compute a -//! quantile to ε accuracy"), this module is the sketch-bound IR: the sketch -//! kind + parameters are committed ([`sketch::SummaryKind`] / -//! [`sketch::SummaryParams`]), and [`expr::SummaryNode`] / [`expr::SummaryExpr`] -//! describe the summary computation. +//! quantile to ε accuracy"), this module is the summary-bound IR: the +//! summary family, kind, and parameters are committed (one `(Kind, Params)` +//! pair per family — [`sketch::ExactKind`]/[`sketch::ExactParams`], +//! [`sketch::SketchKind`]/[`sketch::SketchParams`], +//! [`sketch::SamplingKind`]/[`sketch::SamplingParams`], +//! [`sketch::WaveletKind`]/[`sketch::WaveletParams`], +//! [`sketch::StatModelKind`]/[`sketch::StatModelParams`]), and +//! [`expr::SummaryNode`] / [`expr::SummaryExpr`] describe the summary +//! computation. pub mod expr; pub mod schema; pub mod sketch; pub use expr::{SummaryExpr, SummaryNode}; -pub use schema::{SummaryDataType, SummaryField, SummarySchema}; -pub use sketch::{SketchQuery, SummaryKind, SummaryParams}; +pub use schema::{SummaryFamilyType, SummaryField, SummarySchema}; +pub use sketch::{ + ExactKind, ExactParams, SamplingKind, SamplingParams, SketchKind, SketchParams, SketchQuery, + StatModelKind, StatModelParams, WaveletKind, WaveletParams, +}; diff --git a/crates/types/src/post_asap/schema.rs b/crates/types/src/post_asap/schema.rs index c4099376..20599137 100644 --- a/crates/types/src/post_asap/schema.rs +++ b/crates/types/src/post_asap/schema.rs @@ -1,25 +1,40 @@ -use super::sketch::{SummaryKind, SummaryParams}; +use super::sketch::{ + ExactKind, ExactParams, SamplingKind, SamplingParams, SketchKind, SketchParams, StatModelKind, + StatModelParams, WaveletKind, WaveletParams, +}; use crate::pre_asap::DataType; // ── Post-ASAP data types ──────────────────────────────────────────────────── /// Column types that may appear on a post-ASAP DAG edge. A strict superset of -/// the pre-ASAP [`DataType`]: adds `Sketch` for edges that carry partial -/// sketch state between a `SummaryAgg` and a downstream `SummaryEstimate` or -/// `SummaryMerge`. +/// the pre-ASAP [`DataType`]: adds one variant per summary *family* for +/// edges that carry partial summary state between a `SummaryAgg` and a +/// downstream `SummaryEstimate` or `SummaryMerge`. /// -/// The `Sketch` variant carries `(kind, params)` so the type system can reject -/// merges of incompatible sketches at plan construction time — a `SummaryMerge` -/// over `Sketch(Kll, …)` and `Sketch(Cms, …)` inputs is a plan-time error. +/// Every non-`Plain` variant carries `(kind, params)` from that family's own +/// pair of types, so the type system can reject merges of incompatible +/// summaries at plan construction time — a `SummaryMerge` over +/// `Sketch(Kll, …)` and `Sketch(Cms, …)` inputs is a plan-time error, and a +/// `Sketch(…)` can never be confused for a `Sample(…)` even though both are +/// "opaque summary state" at a glance. #[derive(Debug, Clone, PartialEq)] -pub enum SummaryDataType { - /// Any base pre-ASAP column type — passed through unchanged from - /// pre-ASAP edges. - Primitive(DataType), - /// Opaque summary state (exact accumulator or approximate sketch). - /// The `(kind, params)` pair is the type identity: two summary columns - /// are compatible only if both match exactly. - Sketch(SummaryKind, SummaryParams), +pub enum SummaryFamilyType { + /// An ordinary, readable value — the same closed vocabulary as the + /// pre-ASAP `DataType` (`Int64`/`Float64`/`Utf8`/`Bool`/`Timestamp`), + /// passed through unchanged from a pre-ASAP edge. + Plain(DataType), + /// Exact, mergeable accumulator state (`Sum`/`Count`/`MinMax`/`Rate`/ + /// `Increase`) — the partial state *is* the value; no readout needed. + ExactAggregate(ExactKind, ExactParams), + /// Approximate sketch state (KLL/CMS/HLL/…), read out via a + /// `SummaryEstimate`. + Sketch(SketchKind, SketchParams), + /// Sampling-based summary state (a retained row subset). + Sample(SamplingKind, SamplingParams), + /// Wavelet-transform summary state (a coefficient vector). + Wavelet(WaveletKind, WaveletParams), + /// Fitted statistical/parametric-model summary state. + StatModel(StatModelKind, StatModelParams), } // ── Post-ASAP schema ───────────────────────────────────────────────────────── @@ -27,14 +42,15 @@ pub enum SummaryDataType { #[derive(Debug, Clone)] pub struct SummaryField { pub name: String, - pub dtype: SummaryDataType, + pub dtype: SummaryFamilyType, pub nullable: bool, } /// Schema carried on every edge of the post-ASAP DAG. Extends the pre-ASAP -/// `Schema` with the ability to express sketch-state columns. The two are +/// `Schema` with the ability to express summary-state columns. The two are /// separate types so a pre-ASAP node structurally cannot carry a -/// `Sketch`-typed column — any attempt to do so is a compile-time type error. +/// summary-state-typed column — any attempt to do so is a compile-time type +/// error. #[derive(Debug, Clone)] pub struct SummarySchema { pub fields: Vec, diff --git a/crates/types/src/post_asap/sketch.rs b/crates/types/src/post_asap/sketch.rs index 555e9825..88d358bc 100644 --- a/crates/types/src/post_asap/sketch.rs +++ b/crates/types/src/post_asap/sketch.rs @@ -1,19 +1,12 @@ use crate::pre_asap::ColumnRef; -// ── Summary kind identifiers ─────────────────────────────────────────────────── - -/// Identifies a precomputed aggregation family — either an exact accumulator -/// or an approximate sketch. Used as a type tag in `SummaryDataType` and as the -/// binding choice recorded in `SummaryExpr` nodes. -/// -/// `PartialOrd`/`Ord` (derived, by variant declaration order below): a -/// downstream deployment (ASAPQuery-backend's `control_plane`) needs a -/// deterministic set of families per metric — e.g. `BTreeSet` -/// — to emit reproducible YAML/JSON config. This crate has no ordering -/// opinion of its own; the derive just makes one available. +// ── Exact accumulators ────────────────────────────────────────────────────── + +/// An exact, mergeable accumulator family — zero approximation error. The +/// partial state built for one of these *is* the answer; no +/// `SummaryEstimate` readout is needed to get a value out of it. #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum SummaryKind { - // ── Exact accumulators (no approximation error) ────────────────────────── +pub enum ExactKind { /// Exact sum accumulator (mergeable by addition). Sum, /// Exact count accumulator (mergeable by addition). @@ -24,7 +17,27 @@ pub enum SummaryKind { Increase, /// Rate accumulator (increase / time window duration). Rate, - // ── Approximate sketches ───────────────────────────────────────────────── +} + +/// Parameters for an [`ExactKind`] accumulator. All exact accumulators have +/// fixed semantics — no tuning parameters — so each variant carries none; +/// kept as a per-kind enum (mirroring [`SketchParams`]) so a mismatched +/// `(kind, params)` pair is still a type error, not a runtime check. +#[derive(Debug, Clone, PartialEq)] +pub enum ExactParams { + Sum, + Count, + MinMax, + Increase, + Rate, +} + +// ── Approximate sketches ───────────────────────────────────────────────────── + +/// An approximate, mergeable sketch family — bounded error, sized by its +/// [`SketchParams`]. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum SketchKind { /// KLL quantile sketch (mergeable, ε-accurate rank queries). Kll, /// Count-Min Sketch (mergeable, (ε,δ)-accurate frequency queries). @@ -48,37 +61,12 @@ pub enum SummaryKind { CountSketchWithHeap, } -impl SummaryKind { - /// Is this family an exact accumulator (zero approximation error) rather - /// than an approximate sketch? The partial state built for an exact kind - /// *is* the answer — no `SummaryEstimate` readout is needed to get a - /// value out of it. Used by `asap-plan::boundary::Implementation::Summary` - /// to recover, from `kind` alone, the same fact the now-collapsed - /// `Sketch`/`ExactAccumulator` variant tags used to carry directly. - pub fn is_exact(&self) -> bool { - matches!( - self, - Self::Sum | Self::Count | Self::MinMax | Self::Increase | Self::Rate - ) - } -} - -// ── Sketch parameters ───────────────────────────────────────────────────────── - -/// Concrete, catalog-validated parameters for a specific summary instance. -/// The variant must correspond to the associated `SummaryKind`; mismatches -/// are caught at post-ASAP bind time, before any later, deployment-specific -/// stage ever sees the plan. -/// Exact-accumulator variants carry no parameters (their semantics are fixed). +/// Concrete, catalog-validated parameters for a specific [`SketchKind`] +/// instance. The variant must correspond to the associated `SketchKind`; +/// mismatches are caught at post-ASAP bind time, before any later, +/// deployment-specific stage ever sees the plan. #[derive(Debug, Clone, PartialEq)] -pub enum SummaryParams { - // Exact accumulators — no tuning parameters - Sum, - Count, - MinMax, - Increase, - Rate, - // Approximate sketches — algorithm-specific tuning parameters +pub enum SketchParams { Kll { k: u32, }, @@ -114,9 +102,77 @@ pub enum SummaryParams { }, } +// ── Sampling summaries ─────────────────────────────────────────────────────── + +/// A sampling-based summary family — retains an actual (weighted) subset of +/// rows rather than a compressed sketch. Minimal starting vocabulary: one +/// family, extend as a deployment needs another (stratified, weighted, …). +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum SamplingKind { + /// Reservoir sampling (mergeable via weighted reservoir merge; + /// uniform-random retained subset of a fixed size). + Reservoir, +} + +/// Parameters for a [`SamplingKind`] instance. +#[derive(Debug, Clone, PartialEq)] +pub enum SamplingParams { + Reservoir { + /// Reservoir capacity — the number of retained rows. + size: u32, + }, +} + +// ── Wavelet summaries ──────────────────────────────────────────────────────── + +/// A wavelet-transform-based summary family — a compressed coefficient +/// vector supporting approximate range-sum / histogram queries. Minimal +/// starting vocabulary: one basis, extend as a deployment needs another. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum WaveletKind { + /// Haar wavelet synopsis — the simplest, most common streaming basis. + Haar, +} + +/// Parameters for a [`WaveletKind`] instance. +#[derive(Debug, Clone, PartialEq)] +pub enum WaveletParams { + Haar { + /// Number of retained coefficients — the compression / accuracy knob. + coefficients: u32, + }, +} + +// ── Statistical-model summaries ────────────────────────────────────────────── + +/// A fitted statistical/parametric-model summary family — e.g. a distribution +/// fit used to answer approximate quantile/density queries without +/// retaining raw samples. Deliberately open-ended: `family` names the model +/// family and is interpreted by whatever deployment builds/reads it, the +/// same "core doesn't enumerate every deployment shape" stance +/// `AggIntent::Extension` already takes for pre-ASAP intents. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum StatModelKind { + /// A parametric model fit to the data (e.g. a fitted distribution or + /// regression). `family` (in [`StatModelParams::Parametric`]) names + /// which one. + Parametric, +} + +/// Parameters for a [`StatModelKind`] instance. +#[derive(Debug, Clone, PartialEq)] +pub enum StatModelParams { + Parametric { + /// Deployment-interpreted model family name (e.g. + /// `"gaussian_mixture"`). Core has no fixed catalog of families — + /// see [`StatModelKind::Parametric`]. + family: String, + }, +} + // ── Sketch read-out queries ─────────────────────────────────────────────────── -/// What to extract from a built sketch. Carried by `SummaryEstimate`. +/// What to extract from a built summary. Carried by `SummaryEstimate`. #[derive(Debug, Clone)] pub enum SketchQuery { /// Extract the value at quantile rank `q` ∈ (0, 1]. @@ -138,40 +194,3 @@ pub enum SketchQuery { /// Top-k most frequent (key, count) pairs. TopK { k: usize }, } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn is_exact_matches_the_enum_declaration_split() { - for kind in [ - SummaryKind::Sum, - SummaryKind::Count, - SummaryKind::MinMax, - SummaryKind::Increase, - SummaryKind::Rate, - ] { - assert!( - kind.is_exact(), - "{kind:?} is declared as an exact accumulator" - ); - } - for kind in [ - SummaryKind::Kll, - SummaryKind::Cms, - SummaryKind::Hll, - SummaryKind::DDSketch, - SummaryKind::CmsWithHeap, - SummaryKind::Kmv, - SummaryKind::Theta, - SummaryKind::CountSketch, - SummaryKind::CountSketchWithHeap, - ] { - assert!( - !kind.is_exact(), - "{kind:?} is declared as an approximate sketch" - ); - } - } -}