diff --git a/crates/asap-aware-mapping/src/bind.rs b/crates/asap-aware-mapping/src/bind.rs deleted file mode 100644 index 60a2508c..00000000 --- a/crates/asap-aware-mapping/src/bind.rs +++ /dev/null @@ -1,943 +0,0 @@ -//! The pre-ASAP → post-ASAP binding pass (issue #98). -//! -//! Walks a canonical pre-ASAP [`QueryExpr`] and emits the summary-bound -//! post-ASAP IR ([`SummaryExpr`] / [`SummaryNode`] in `asap-sketch`). Per -//! node, the [`boundary`](crate::boundary) decision picks the realization: -//! -//! - **Summary family** (sketch / sample / wavelet / statistical model) — -//! the `Aggregate` becomes a [`SummaryExpr::SummaryAgg`] carrying the -//! committed `family: SummaryFamilyType` (that family's own -//! `(kind, params)`), wrapped in a [`SummaryExpr::SummaryEstimate`] that -//! reads the answer back out (the summary-state column type does not -//! propagate past the estimate); -//! - **Exact accumulator** — a `SummaryAgg` with `family: -//! SummaryFamilyType::ExactAggregate` (`Sum`/`Count`/`MinMax`/`Rate`/ -//! `Increase`) and no estimate: the partial state *is* the value, so a -//! deployment's later finalization step is the identity; -//! - **Pass-through** — the whole pre-ASAP subtree is wrapped as -//! [`SummaryExpr::Logical`], schema lifted with every column -//! `SummaryFamilyType::Plain`. -//! -//! Binding recurses through the `Aggregate` spine, so nested aggregates each -//! get their own decision (`quantile(0.9, sum by (svc) (rate(m[5m]))))` binds -//! KLL over an exact `Sum` accumulator over an exact `Rate` accumulator). -//! -//! ## Conservative fallbacks -//! -//! [`SummaryExpr::Logical`] boxes a whole pre-ASAP subtree — it has no -//! post-ASAP children — so a *logical* operator above a bindable aggregate -//! (`Filter`/`BinaryOp`/… over a quantile) subsumes the aggregate into the -//! logical wrapper unbound. Rewriting through logical parents is the -//! post-ASAP rule engine's job (#6/#33), not this pass's. Similarly -//! conservative: multi-intent `Aggregate` nodes (SQL `SELECT SUM(a), AVG(b)`) -//! and aggregates with a `HAVING` predicate (the filter would need the -//! estimate first) stay logical. - -use std::rc::Rc; - -use asap_types::post_asap::{ - SketchQuery, SummaryExpr, SummaryFamilyType, SummaryField, SummaryNode, SummarySchema, -}; -use asap_types::pre_asap::agg_intent::AggIntent; -use asap_types::pre_asap::expr_ir::ColumnRef; -use asap_types::pre_asap::query_expr::{QueryExpr, QueryExprError, Reduction}; -use asap_types::pre_asap::schema::Schema; -use thiserror::Error; - -use crate::boundary::{implementation_for_with, Implementation}; -use crate::cost_model::{CostModel, CseCandidate, DefaultCostModel, ShareDecision}; - -/// Errors from the pre-ASAP → post-ASAP binding pass. -#[derive(Debug, Error)] -pub enum ImplementError { - /// Schema derivation failed while lifting an edge to `SummarySchema`. - #[error("schema derivation failed during pre-ASAP → post-ASAP binding: {0}")] - Schema(#[from] QueryExprError), -} - -/// Bind a single query to the post-ASAP IR. Ranks candidate summaries via -/// [`DefaultCostModel`] (`asap-plan`'s built-in static preference order, -/// unchanged); use [`implement_tree_with`] to plug in a deployment-specific -/// [`CostModel`] instead. -pub fn implement_tree(expr: &QueryExpr) -> Result, ImplementError> { - implement_tree_with(expr, &DefaultCostModel) -} - -/// Like [`implement_tree`], but ranks candidate summaries via `cost_model` (see -/// [`crate::cost_model`]) instead of the built-in static preference order. -pub fn implement_tree_with( - expr: &QueryExpr, - cost_model: &dyn CostModel, -) -> Result, ImplementError> { - if let QueryExpr::Aggregate { - reduction, - measures, - having, - child, - .. - } = expr - { - // The bindable shape: exactly one intent, no HAVING. (Multi-intent - // nodes and HAVING stay logical — see the module docs.) - if let ([intent], None) = (measures.as_slice(), having) { - let implementation = implementation_for_with(intent, cost_model); - if let Some((family, estimate)) = summary_family(implementation) { - return bind_summary_agg( - expr, reduction, intent, child, family, estimate, cost_model, - ); - } - } - } - logical(expr) -} - -/// Bind a whole workload's worth of already-CSE'd roots -/// ([`asap_types::pre_asap::cse::share_common_subtrees`]'s output), reusing -/// one bound [`SummaryNode`] wherever two roots share the same `Rc` *and* -/// `cost_model` decides it's worth it (issue #212, #222, #223 stages 2 and -/// 4, #237). -/// -/// This is a real caller for `share_common_subtrees`, wired up deliberately: -/// the pass's own landing plan calls out that its predecessor -/// (`asap-plan::cse::dedupe_subtrees`) was deleted in #192 for being unwired -/// dead code, and lands this memoization alongside it so that never becomes -/// true again. -/// -/// The memo key is `Rc::as_ptr` — pointer identity, not a second -/// `PartialEq`/structural-equality pass. `share_common_subtrees` already made -/// the (non-negotiable, `PartialEq`-checked) sharing decision; this only -/// needs to recognize when it already bound the exact `Rc` a later root -/// hands back, and is deliberately *not* a general "does an available -/// `Implementation` satisfy this one" lookup — that subsumption question is -/// `asap_aware_mapping::boundary::Matcher`'s documented, deliberately-unfilled -/// job, not this one's. -/// -/// A first pass over `roots` counts each distinct `Rc` pointer's -/// true `consumer_count` across the whole workload, so the -/// [`CseCandidate`]/[`CostModel::cse_share_decision`] cost comparison (see -/// `docs/design_docs/cse-cost-model-decision.md`) sees the real total, not a running -/// count that grows as roots are processed left to right. The decision is -/// made once, the first time a shared pointer is bound, and cached alongside -/// the bound `SummaryNode` so every later occurrence of that same `Rc` -/// applies the same decision consistently — either every consumer reuses one -/// shared `SummaryNode`, or every consumer (including the first) binds -/// independently. -/// -/// Only whole-root sharing is memoized (matching two workload roots that are -/// themselves the same `Rc` after CSE) — [`implement_tree`] is -/// called at most once per distinct root pointer when the decision is -/// `Share`, but it still walks each such tree's own internal structure -/// fresh; a subtree shared only *below* two different roots' top level does -/// not additionally memoize inside that walk. Widening this to sub-root -/// memoization is future work. -pub fn implement_workload( - roots: Vec<(Id, Rc)>, -) -> Vec<(Id, Result, ImplementError>)> { - implement_workload_with(roots, &DefaultCostModel) -} - -/// Like [`implement_workload`], but ranks candidate summaries — and decides -/// CSE sharing — via `cost_model` instead of the built-in defaults (see -/// [`crate::cost_model`]). -pub fn implement_workload_with( - roots: Vec<(Id, Rc)>, - cost_model: &dyn CostModel, -) -> Vec<(Id, Result, ImplementError>)> { - let mut consumer_count: std::collections::HashMap<*const QueryExpr, usize> = - std::collections::HashMap::new(); - for (_, expr) in &roots { - *consumer_count.entry(Rc::as_ptr(expr)).or_insert(0) += 1; - } - - let mut memo: std::collections::HashMap<*const QueryExpr, (Rc, ShareDecision)> = - std::collections::HashMap::new(); - roots - .into_iter() - .map(|(id, expr)| { - let ptr = Rc::as_ptr(&expr); - let result = match memo.get(&ptr) { - Some((cached, ShareDecision::Share)) => Ok(Rc::clone(cached)), - Some((_, ShareDecision::RecomputeIndependently)) => { - implement_tree_with(&expr, cost_model) - } - None => implement_tree_with(&expr, cost_model).inspect(|node| { - let count = consumer_count[&ptr]; - let decision = if count > 1 { - let candidate = CseCandidate { - subtree: &expr, - bound_summary: node, - consumer_count: count, - }; - cost_model.cse_share_decision(&candidate) - } else { - // Only one consumer: nothing to compare against, and - // this branch is never consulted again for `ptr`. - ShareDecision::Share - }; - memo.insert(ptr, (Rc::clone(node), decision)); - }), - }; - (id, result) - }) - .collect() -} - -/// Translate an [`Implementation`] into the `(family, needs a -/// SummaryEstimate readout)` pair [`bind_summary_agg`] needs, or `None` for -/// `PassThrough` (the caller falls back to [`logical`]). -/// -/// Every family's partial state needs a readout to recover a value, except -/// `ExactAggregate` — its partial state *is* the value already, so no -/// estimate step follows it. -fn summary_family(implementation: Implementation) -> Option<(SummaryFamilyType, bool)> { - Some(match implementation { - Implementation::ExactAggregate { kind, params } => { - (SummaryFamilyType::ExactAggregate(kind, params), false) - } - Implementation::Sketch { kind, params } => (SummaryFamilyType::Sketch(kind, params), true), - Implementation::Sample { kind, params } => (SummaryFamilyType::Sample(kind, params), true), - Implementation::Wavelet { kind, params } => { - (SummaryFamilyType::Wavelet(kind, params), true) - } - Implementation::StatModel { kind, params } => { - (SummaryFamilyType::StatModel(kind, params), true) - } - Implementation::PassThrough => return None, - }) -} - -/// Emit `SummaryAgg` (recursively binding the child), plus the -/// `SummaryEstimate` readout when `estimate` is set. -#[allow(clippy::too_many_arguments)] -fn bind_summary_agg( - node: &QueryExpr, - reduction: &Reduction, - intent: &AggIntent, - child: &QueryExpr, - family: SummaryFamilyType, - estimate: bool, - cost_model: &dyn CostModel, -) -> Result, ImplementError> { - let child_schema = child.output_schema()?; - // The single canonical pre-ASAP derivation (per-series vs cross-series, - // name overrides) already computes the row shape; binding only retypes - // the summary state column. - let per_series = matches!(reduction, Reduction::PerEntity); - let by: Vec = reduction - .group_keys() - .map(|g| g.to_vec()) - .unwrap_or_default(); - let out_schema = node.output_schema()?; - let state_idx = summary_col_index(&out_schema, &by, per_series); - - let col = summarised_column(intent, &child_schema); - let query = estimate.then(|| readout(intent, &col, cost_model)); - - let mut state_schema = lift(&out_schema); - if let Some(field) = state_schema.fields.get_mut(state_idx) { - field.dtype = family.clone(); - } - - // `reduction` is carried onto `SummaryAgg` verbatim — not flattened to a - // bare `Vec` — so `SummaryExecutor::find_candidates` can tell - // a genuine empty-`by` reduction apart from a per-entity shape with no - // grouping concept at all (issue #163). `bind_summary_agg` is the single - // place that decides this; nothing downstream re-derives it. - let agg = Rc::new(SummaryNode { - expr: SummaryExpr::SummaryAgg { - child: implement_tree_with(child, cost_model)?, - family, - col, - reduction: reduction.clone(), - }, - schema: state_schema, - }); - match query { - // The readout: downstream of the estimate the schema is the plain - // pre-ASAP row shape again (the summary-state type does not - // propagate). - Some(query) => Ok(Rc::new(SummaryNode { - expr: SummaryExpr::SummaryEstimate { - summary_input: agg, - query, - }, - schema: lift(&out_schema), - })), - None => Ok(agg), - } -} - -/// Index of the summary-state column in the aggregate's output schema: -/// cross-series output is `by ++ [agg]` (the column after the keys); -/// a per-series reduction keeps every label and replaces the sample value -/// (named `value` — mirror `per_series_reduction_schema`'s fallback). -/// `per_series` is the caller's already-read `Reduction` (issue #165) — -/// this never re-derives it, so it can't disagree with the caller. -fn summary_col_index(out_schema: &Schema, by: &[usize], per_series: bool) -> usize { - if per_series { - out_schema - .column_id("value") - .or_else(|| (0..out_schema.columns.len()).find(|&i| Some(i) != out_schema.time_index)) - .unwrap_or(0) - } else { - by.len() - } -} - -/// The column fed into the summary: the intent's positional input column -/// resolved to a name against the child schema, or the PromQL sample value. -fn summarised_column(intent: &AggIntent, child_schema: &Schema) -> ColumnRef { - match intent - .input_col() - .and_then(|id| child_schema.columns.get(id)) - { - Some(c) => match &c.table { - Some(t) => ColumnRef::Qualified { - table: t.clone(), - name: c.name.clone(), - }, - None => ColumnRef::Named(c.name.clone()), - }, - None => ColumnRef::SampleValue, - } -} - -/// The `SummaryEstimate` readout for a summary-bound intent. -fn readout(intent: &AggIntent, col: &ColumnRef, cost_model: &dyn CostModel) -> SketchQuery { - match intent { - AggIntent::Quantile { q, .. } => SketchQuery::Quantile { q: *q }, - AggIntent::Cardinality { .. } => SketchQuery::Cardinality, - AggIntent::TopK { k, .. } => SketchQuery::TopK { k: *k }, - AggIntent::Count { .. } => SketchQuery::PointCount { - key: col.clone(), - value: None, - }, - // Core doesn't know the shape of a deployment-specific `Extension` - // intent, so it can't build its readout either — delegate to the - // same `CostModel` that decided (via `realize_extension`) this - // intent gets a summary realization at all. See `readout_extension`'s - // doc for the invariant this depends on. - AggIntent::Extension { ext_kind, payload } => { - cost_model.readout_extension(ext_kind, payload, col) - } - other => { - unreachable!("no summary realization for {other:?} (boundary::implementation_for)") - } - } -} - -/// Wrap an unrewritten pre-ASAP subtree, lifting its schema with every column -/// `SummaryFamilyType::Plain`. Public so a deployment can force a node it -/// knows `implement_tree_in_with` would otherwise actively (mis)bind — -/// e.g. an intent this crate's `boundary::implementation_for` maps to an -/// accumulator kind the deployment's runtime doesn't actually implement — -/// through the same fallback this crate's own dispatch uses, without -/// duplicating the schema-lift logic. -pub fn logical(expr: &QueryExpr) -> Result, ImplementError> { - let schema = expr.output_schema()?; - Ok(Rc::new(SummaryNode { - expr: SummaryExpr::Logical(Box::new(expr.clone())), - schema: lift(&schema), - })) -} - -fn lift(schema: &Schema) -> SummarySchema { - SummarySchema { - fields: schema - .columns - .iter() - .map(|c| SummaryField { - name: c.name.clone(), - dtype: SummaryFamilyType::Plain(c.dtype.clone()), - nullable: c.nullable, - }) - .collect(), - time_index: schema.time_index, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::post_asap::{ExactKind, ExactParams, SketchKind, SketchParams}; - use asap_types::pre_asap::agg_intent::default_quantile; - use asap_types::pre_asap::expr_ir::{CompareOpKind, ScalarValue}; - use asap_types::pre_asap::query_expr::{Predicate, Source}; - use asap_types::pre_asap::schema::{Column, DataType}; - use asap_types::types::AccuracyTarget; - use std::time::Duration; - - fn metric_scan(labels: &[&str]) -> QueryExpr { - let mut columns = vec![ - Column::new("ts", DataType::Timestamp, false), - Column::new("value", DataType::Float64, false), - ]; - columns.extend(labels.iter().map(|n| Column::new(*n, DataType::Utf8, true))); - QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index(columns, 0, vec![]), - } - } - - /// A cross-series reduction, grouped by `by` (possibly empty — a - /// genuine full reduction, never "no grouping concept"). - fn agg(by: Vec, intent: AggIntent, child: QueryExpr) -> QueryExpr { - QueryExpr::Aggregate { - reduction: Reduction::by(by), - measures: vec![intent], - output_names: vec![], - having: None, - child: Rc::new(child), - } - } - - /// A per-entity reduction: no grouping concept at all (issue #165). - fn agg_per_entity(intent: AggIntent, child: QueryExpr) -> QueryExpr { - QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![intent], - output_names: vec![], - having: None, - child: Rc::new(child), - } - } - - fn field<'a>(schema: &'a SummarySchema, name: &str) -> &'a SummaryField { - schema - .fields - .iter() - .find(|f| f.name == name) - .unwrap_or_else(|| panic!("no field {name:?} in {schema:?}")) - } - - #[test] - fn quantile_binds_kll_wrapped_in_estimate() { - // quantile by (job) (m) at ε=0.01 → Estimate(Quantile) over - // SummaryAgg(Kll{k:200}) over Logical(Scan). job = col 2. - let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); - let root = implement_tree(&q).unwrap(); - - let SummaryExpr::SummaryEstimate { - summary_input, - query, - } = &root.expr - else { - panic!("expected SummaryEstimate root, got {:?}", root.expr); - }; - assert!(matches!(query, SketchQuery::Quantile { q } if *q == 0.99)); - // Estimate edge: plain row shape — group key + Float64 answer. - assert_eq!( - field(&root.schema, "quantile_0_99").dtype, - SummaryFamilyType::Plain(DataType::Float64) - ); - assert_eq!( - field(&root.schema, "job").dtype, - SummaryFamilyType::Plain(DataType::Utf8) - ); - - let SummaryExpr::SummaryAgg { - child, - family, - col, - reduction, - } = &summary_input.expr - else { - panic!("expected SummaryAgg, got {:?}", summary_input.expr); - }; - assert_eq!( - family, - &SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) - ); - assert_eq!(col, &ColumnRef::SampleValue); - assert_eq!(reduction, &Reduction::by(vec![2])); - // SummaryAgg edge: the state column carries the committed family. - assert_eq!( - field(&summary_input.schema, "quantile_0_99").dtype, - SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) - ); - assert!(matches!(child.expr, SummaryExpr::Logical(ref e) - if matches!(**e, QueryExpr::Scan { .. }))); - } - - /// A deployment-supplied [`CostModel`] can override the default KLL - /// choice — `implement_tree_with` must actually consult it, not just accept and - /// ignore it (issue: cost model interface, see `crate::cost_model`). - struct PreferDDSketch; - - impl CostModel for PreferDDSketch { - fn rank_candidates( - &self, - _intent: &AggIntent, - candidates: &[SketchKind], - ) -> Vec { - let mut v = candidates.to_vec(); - if let Some(pos) = v.iter().position(|k| *k == SketchKind::DDSketch) { - let ddsketch = v.remove(pos); - v.insert(0, ddsketch); - } - v - } - } - - #[test] - fn bind_with_custom_cost_model_overrides_default_summary_choice() { - let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); - - // Default: KLL (see `quantile_binds_kll_wrapped_in_estimate` above). - let default_root = implement_tree(&q).unwrap(); - let SummaryExpr::SummaryEstimate { summary_input, .. } = &default_root.expr else { - panic!("expected SummaryEstimate root, got {:?}", default_root.expr); - }; - let SummaryExpr::SummaryAgg { family, .. } = &summary_input.expr else { - panic!("expected SummaryAgg, got {:?}", summary_input.expr); - }; - assert!(matches!( - family, - SummaryFamilyType::Sketch(SketchKind::Kll, _) - )); - - // With `PreferDDSketch`: DDSketch instead, same query. - let custom_root = implement_tree_with(&q, &PreferDDSketch).unwrap(); - let SummaryExpr::SummaryEstimate { summary_input, .. } = &custom_root.expr else { - panic!("expected SummaryEstimate root, got {:?}", custom_root.expr); - }; - let SummaryExpr::SummaryAgg { family, .. } = &summary_input.expr else { - panic!("expected SummaryAgg, got {:?}", summary_input.expr); - }; - assert_eq!( - family, - &SummaryFamilyType::Sketch( - SketchKind::DDSketch, - SketchParams::DDSketch { alpha: 0.01 } - ) - ); - } - - /// A deployment-supplied `CostModel` can realize an `AggIntent::Extension` - /// intent as a real sketch instead of the default `PassThrough` (issue - /// #150) — `implement_tree_with` must consult `realize_extension` for - /// the `Extension` arm, and `readout` must consult `readout_extension` - /// to build its `SketchQuery` without panicking. - struct FrequencyCostModel; - - impl CostModel for FrequencyCostModel { - fn rank_candidates( - &self, - _intent: &AggIntent, - candidates: &[SketchKind], - ) -> Vec { - candidates.to_vec() - } - - fn realize_extension( - &self, - ext_kind: &str, - _payload: &serde_json::Value, - ) -> crate::boundary::Implementation { - if ext_kind == "frequency" { - crate::boundary::Implementation::Sketch { - kind: SketchKind::CountSketch, - params: SketchParams::CountSketch { - width: 256, - depth: 4, - }, - } - } else { - crate::boundary::Implementation::PassThrough - } - } - - fn readout_extension( - &self, - ext_kind: &str, - payload: &serde_json::Value, - _col: &ColumnRef, - ) -> SketchQuery { - assert_eq!(ext_kind, "frequency"); - let value = payload["item"].as_str().map(str::to_string); - SketchQuery::PointCount { - key: ColumnRef::Named("item".into()), - value, - } - } - } - - #[test] - fn extension_intent_stays_logical_by_default() { - // Without a CostModel overriding `realize_extension`, an - // `Extension` intent must stay `PassThrough` -- today's behavior, - // unchanged. - let intent = AggIntent::Extension { - ext_kind: "frequency".to_string(), - payload: serde_json::json!({ "item": "checkout" }), - }; - let q = agg(vec![], intent, metric_scan(&[])); - let root = implement_tree(&q).unwrap(); - assert!(matches!(root.expr, SummaryExpr::Logical(_))); - } - - #[test] - fn extension_intent_binds_via_custom_cost_model() { - let intent = AggIntent::Extension { - ext_kind: "frequency".to_string(), - payload: serde_json::json!({ "item": "checkout" }), - }; - let q = agg(vec![], intent, metric_scan(&[])); - let root = implement_tree_with(&q, &FrequencyCostModel).unwrap(); - - let SummaryExpr::SummaryEstimate { - summary_input, - query, - } = &root.expr - else { - panic!("expected SummaryEstimate root, got {:?}", root.expr); - }; - assert!(matches!( - query, - SketchQuery::PointCount { key: ColumnRef::Named(k), value: Some(v) } - if k == "item" && v == "checkout" - )); - - let SummaryExpr::SummaryAgg { family, .. } = &summary_input.expr else { - panic!("expected SummaryAgg, got {:?}", summary_input.expr); - }; - assert_eq!( - family, - &SummaryFamilyType::Sketch( - SketchKind::CountSketch, - SketchParams::CountSketch { - width: 256, - depth: 4 - } - ) - ); - } - - #[test] - fn exact_sum_binds_accumulator_without_estimate() { - let q = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let root = implement_tree(&q).unwrap(); - let SummaryExpr::SummaryAgg { family, .. } = &root.expr else { - panic!( - "expected bare SummaryAgg (no estimate), got {:?}", - root.expr - ); - }; - assert_eq!( - family, - &SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - ); - assert_eq!( - field(&root.schema, "sum").dtype, - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - ); - } - - #[test] - fn per_series_rate_keeps_labels_and_retypes_value() { - // rate(m[5m]) — per-series: every label survives; the sample value - // column becomes the Rate accumulator state. - let q = agg_per_entity( - AggIntent::Rate, - QueryExpr::TimeRange { - range: Duration::from_secs(300), - child: Rc::new(metric_scan(&["job"])), - }, - ); - let root = implement_tree(&q).unwrap(); - let SummaryExpr::SummaryAgg { family, .. } = &root.expr else { - panic!("expected SummaryAgg, got {:?}", root.expr); - }; - assert_eq!( - family, - &SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) - ); - assert_eq!( - root.schema - .fields - .iter() - .map(|f| f.name.as_str()) - .collect::>(), - vec!["ts", "value", "job"], - ); - assert_eq!( - field(&root.schema, "value").dtype, - SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) - ); - assert_eq!(root.schema.time_index, Some(0)); - } - - /// Issue #163, case 1: a bare per-series range function (e.g. - /// `quantile_over_time(...)`) binds to `SummaryAgg { reduction: - /// PerEntity, .. }` — proving the pre-ASAP `Reduction` this crate - /// already computes (issue #165) is carried onto the post-ASAP node - /// verbatim, not flattened back into an ambiguous bare `Vec`. - #[test] - fn bare_per_series_aggregate_binds_summary_agg_with_per_entity_reduction() { - let q = agg_per_entity( - default_quantile(0.99), - QueryExpr::TimeRange { - range: Duration::from_secs(10), - child: Rc::new(metric_scan(&["job"])), - }, - ); - let root = implement_tree(&q).unwrap(); - let SummaryExpr::SummaryEstimate { summary_input, .. } = &root.expr else { - panic!("expected estimate root, got {:?}", root.expr); - }; - let SummaryExpr::SummaryAgg { reduction, .. } = &summary_input.expr else { - panic!("expected SummaryAgg, got {:?}", summary_input.expr); - }; - assert_eq!(reduction, &Reduction::PerEntity); - } - - /// Issue #163, case 2: an aggregation operator explicitly invoked with - /// no `by(...)` (e.g. `count(hll_metric)`) binds to `SummaryAgg { - /// reduction: Reduce(vec![]), .. }` — byte-identical `by: []` to the - /// previous test at the old `Vec` shape; `reduction` is what - /// tells them apart now. - #[test] - fn explicit_empty_by_aggregate_binds_summary_agg_with_reduce_reduction() { - let intent = AggIntent::Cardinality { - col: None, - accuracy: AccuracyTarget::Epsilon(0.01), - }; - let q = agg(vec![], intent, metric_scan(&["job"])); - let root = implement_tree(&q).unwrap(); - let SummaryExpr::SummaryEstimate { summary_input, .. } = &root.expr else { - panic!("expected estimate root, got {:?}", root.expr); - }; - let SummaryExpr::SummaryAgg { reduction, .. } = &summary_input.expr else { - panic!("expected SummaryAgg, got {:?}", summary_input.expr); - }; - assert_eq!(reduction, &Reduction::by(vec![])); - } - - #[test] - fn nested_aggregates_bind_per_node() { - // quantile(0.9, sum by (job) (m)) — the boundary fires per node over - // the nested tree: KLL over an exact Sum accumulator. - let inner = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let outer = agg(vec![], default_quantile(0.9), inner); - let root = implement_tree(&outer).unwrap(); - - let SummaryExpr::SummaryEstimate { summary_input, .. } = &root.expr else { - panic!("expected estimate root, got {:?}", root.expr); - }; - let SummaryExpr::SummaryAgg { child, family, .. } = &summary_input.expr else { - panic!("expected outer SummaryAgg, got {:?}", summary_input.expr); - }; - assert!(matches!( - family, - SummaryFamilyType::Sketch(SketchKind::Kll, _) - )); - let SummaryExpr::SummaryAgg { - family: inner_family, - child: leaf, - .. - } = &child.expr - else { - panic!("expected inner SummaryAgg, got {:?}", child.expr); - }; - assert_eq!( - inner_family, - &SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - ); - assert!(matches!(leaf.expr, SummaryExpr::Logical(_))); - } - - /// Issue #115: the summary is built over the intent's own input column. - /// Before `Cardinality`/`Quantile` carried `col`, `summarised_column` always - /// fell through to `ColumnRef::SampleValue`, so an HLL was built over the - /// wrong column for every SQL `COUNT(DISTINCT c)`. - #[test] - fn sketch_binds_the_intents_input_column() { - // `metric_scan(&["job"])` → columns [ts=0, value=1, job=2]. - let cases = [ - (Some(2), ColumnRef::Named("job".into())), - (Some(1), ColumnRef::Named("value".into())), - // PromQL convention: no column ⇒ the synthetic sample value. - (None, ColumnRef::SampleValue), - ]; - for (col, want) in cases { - let intent = AggIntent::Cardinality { - col, - accuracy: AccuracyTarget::Epsilon(0.01), - }; - let root = implement_tree(&agg(vec![0], intent, metric_scan(&["job"]))).unwrap(); - let bound = find_summary_col(&root) - .unwrap_or_else(|| panic!("expected a SummaryAgg for col={col:?}")); - assert_eq!(bound, want, "wrong summarised column for col={col:?}"); - } - } - - /// The `col` of the first `SummaryAgg` in the tree. - fn find_summary_col(node: &SummaryNode) -> Option { - match &node.expr { - SummaryExpr::SummaryAgg { col, .. } => Some(col.clone()), - SummaryExpr::SummaryEstimate { summary_input, .. } => find_summary_col(summary_input), - _ => None, - } - } - - #[test] - fn pass_through_intents_stay_logical() { - // avg is exact but non-mergeable; histogram_quantile (classic - // buckets, #79) is never sketchable; exact quantile is exact by - // decree. All three stay whole logical subtrees. - for intent in [ - AggIntent::Avg { col: None }, - AggIntent::HistogramQuantile { q: 0.99 }, - AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::Exact, - }, - ] { - let q = agg(vec![2], intent.clone(), metric_scan(&["job"])); - let root = implement_tree(&q).unwrap(); - assert!( - matches!(root.expr, SummaryExpr::Logical(ref e) if **e == q), - "expected Logical passthrough for {intent:?}" - ); - } - } - - #[test] - fn logical_parent_subsumes_bindable_child() { - // Filter over a bindable quantile: `Logical` has no post-ASAP - // children, so the conservative fallback keeps the whole subtree - // logical. - let q = QueryExpr::Filter { - pred: Predicate(Rc::new(QueryExpr::Compare { - left: Rc::new(QueryExpr::Column(0)), - op: CompareOpKind::Gt, - right: Rc::new(QueryExpr::Literal(ScalarValue::Float64(0.5))), - })), - child: Rc::new(agg(vec![], default_quantile(0.99), metric_scan(&[]))), - }; - let root = implement_tree(&q).unwrap(); - assert!(matches!(root.expr, SummaryExpr::Logical(ref e) if **e == q)); - } - - #[test] - fn having_and_multi_intent_stay_logical() { - let mut q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); - if let QueryExpr::Aggregate { having, .. } = &mut q { - *having = Some(Predicate(Rc::new(QueryExpr::Literal( - ScalarValue::Boolean(true), - )))); - } - assert!(matches!( - implement_tree(&q).unwrap().expr, - SummaryExpr::Logical(_) - )); - - let multi = QueryExpr::Aggregate { - reduction: Reduction::by(vec![2]), - measures: vec![AggIntent::Sum { col: None }, AggIntent::Avg { col: None }], - output_names: vec![], - having: None, - child: Rc::new(metric_scan(&["job"])), - }; - assert!(matches!( - implement_tree(&multi).unwrap().expr, - SummaryExpr::Logical(_) - )); - } - - #[test] - fn topk_binds_cms_with_heap_and_topk_readout() { - let q = agg( - vec![2], - AggIntent::TopK { - k: 5, - accuracy: AccuracyTarget::Epsilon(0.01), - }, - metric_scan(&["job"]), - ); - let root = implement_tree(&q).unwrap(); - let SummaryExpr::SummaryEstimate { - summary_input, - query, - } = &root.expr - else { - panic!("expected estimate root, got {:?}", root.expr); - }; - assert!(matches!(query, SketchQuery::TopK { k: 5 })); - assert!(matches!( - &summary_input.expr, - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::Sketch(SketchKind::CmsWithHeap, _), - .. - } - )); - } - - #[test] - fn sql_reducer_resolves_named_input_column() { - // SUM(bytes) over a tabular scan: `col` resolves positionally to the - // named column, not the PromQL sample value. - let scan = QueryExpr::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - predicates: vec![], - schema: Schema { - columns: vec![ - Column::new("host", DataType::Utf8, false), - Column::new("bytes", DataType::Int64, false), - ], - time_index: None, - unique_keys: vec![], - closed: true, - }, - }; - let q = agg(vec![0], AggIntent::Sum { col: Some(1) }, scan); - let root = implement_tree(&q).unwrap(); - let SummaryExpr::SummaryAgg { col, .. } = &root.expr else { - panic!("expected SummaryAgg, got {:?}", root.expr); - }; - assert_eq!(col, &ColumnRef::Named("bytes".into())); - } - - /// Issue #237, #223 stage 4: a `CostModel` that declines CSE sharing - /// makes `implement_workload_with` bind each occurrence independently, - /// even though the two roots are the exact same `Rc` (as - /// `share_common_subtrees` would hand back for two identical workload - /// entries) — the opposite of `DefaultCostModel`'s unconditional-share - /// behavior pinned by `crates/integration-tests/tests/cse.rs`. - #[test] - fn implement_workload_with_recomputes_independently_when_cost_model_declines_sharing() { - struct NeverShareCse; - impl CostModel for NeverShareCse { - fn rank_candidates( - &self, - _intent: &AggIntent, - candidates: &[SketchKind], - ) -> Vec { - candidates.to_vec() - } - fn cse_share_decision(&self, _candidate: &CseCandidate) -> ShareDecision { - ShareDecision::RecomputeIndependently - } - } - - let shared = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); - let bound = implement_workload_with( - vec![("a", Rc::clone(&shared)), ("b", Rc::clone(&shared))], - &NeverShareCse, - ); - let [(_, ra), (_, rb)] = bound.as_slice() else { - panic!("expected 2 bound results"); - }; - let ra = ra.as_ref().expect("a failed to bind"); - let rb = rb.as_ref().expect("b failed to bind"); - assert!( - !Rc::ptr_eq(ra, rb), - "a CostModel that declines CSE sharing must bind each occurrence \ - independently, even for two roots that are the same Rc" - ); - } -} diff --git a/crates/asap-aware-mapping/src/boundary.rs b/crates/asap-aware-mapping/src/boundary.rs deleted file mode 100644 index ff3f2b39..00000000 --- a/crates/asap-aware-mapping/src/boundary.rs +++ /dev/null @@ -1,992 +0,0 @@ -//! Sketch-vs-exact boundary — the per-intent accuracy decision (issue #98). -//! -//! The per-node choice of how an [`AggIntent`] is *realised*: by an -//! approximate summary (sketch, sample, wavelet, statistical model, …), by -//! an exact mergeable accumulator, or by an ordinary exact operator -//! (pass-through). This is a post-ASAP concern: the pre-ASAP IR carries only -//! the intent + accuracy target, never the realization. -//! -//! The decision consumes three inputs: -//! -//! - the [`AccuracyTarget`] threaded onto the approximate-capable intents -//! (`Quantile` / `Cardinality` / `Count` / `TopK`) — `Exact` forbids a -//! sketch; `Epsilon` / `EpsilonDelta` size the sketch parameters; -//! - the [`agg_is_exact`] / [`agg_is_mergeable`] helpers in `asap-ir` — -//! an exact accumulator exists only for mergeable intents -//! (`Avg`/`StdDev`/`Variance` need richer partial state, so they -//! pass through to an ordinary exact operator); -//! - histogram sketchability (#79): [`AggIntent::HistogramQuantile`] -//! (classic cumulative-`le`-bucket interpolation) is **not** re-sketchable — -//! pre-aggregated bucket counts can't feed a quantile sketch — while the -//! generic `Quantile` path (native histograms / raw samples) is. -//! -//! [`implementation_for`] is a single exhaustive match over the intent vocabulary, so a -//! new `AggIntent` variant fails to compile until it is given an explicit -//! realization — there is no silent fall-through. The [`bind`](crate::bind) -//! pass fires it per node over nested trees. -//! -//! Today's core dispatch only ever picks [`Implementation::ExactAggregate`], -//! [`Implementation::Sketch`], or [`Implementation::PassThrough`] — no -//! `AggIntent` variant maps to sampling/wavelet/statistical-model yet. -//! [`Implementation::Sample`]/[`Wavelet`](Implementation::Wavelet)/ -//! [`StatModel`](Implementation::StatModel) exist so a deployment's own -//! [`CostModel::realize_extension`](crate::cost_model::CostModel::realize_extension) -//! can choose one of them for an `AggIntent::Extension` node — core has no -//! opinion on when that's the right choice. - -use asap_types::post_asap::{ - ExactKind, ExactParams, SamplingKind, SamplingParams, SketchKind, SketchParams, StatModelKind, - StatModelParams, WaveletKind, WaveletParams, -}; -use asap_types::pre_asap::agg_intent::{agg_is_mergeable, AggIntent}; -use asap_types::types::AccuracyTarget; - -use crate::cost_model::{CostModel, DefaultCostModel}; - -/// How an [`AggIntent`] is realised at post-ASAP binding time. -#[derive(Debug, Clone, PartialEq)] -pub enum Implementation { - /// An exact **mergeable** accumulator (partial state ≡ the value - /// itself: `Sum` / `Count` / `MinMax` / `Rate` / `Increase`). The - /// built state *is* the answer already — no `SummaryEstimate` readout - /// step. - ExactAggregate { - kind: ExactKind, - params: ExactParams, - }, - /// An approximate sketch sized to the intent's [`AccuracyTarget`]. - /// Needs a `SummaryEstimate` readout to recover a value. - Sketch { - kind: SketchKind, - params: SketchParams, - }, - /// A sampling-based summary (a retained row subset). Needs a - /// `SummaryEstimate` readout. Not chosen by any core `AggIntent` - /// dispatch today — see the module docs. - Sample { - kind: SamplingKind, - params: SamplingParams, - }, - /// A wavelet-transform summary. Needs a `SummaryEstimate` readout. Not - /// chosen by any core `AggIntent` dispatch today — see the module docs. - Wavelet { - kind: WaveletKind, - params: WaveletParams, - }, - /// A fitted statistical/parametric-model summary. Needs a - /// `SummaryEstimate` readout. Not chosen by any core `AggIntent` - /// dispatch today — see the module docs. - StatModel { - kind: StatModelKind, - params: StatModelParams, - }, - /// No summary form — the node stays a logical pre-ASAP operator and is - /// executed exactly (per-series transforms, non-mergeable reducers, exact - /// quantile/top-k/cardinality, classic-bucket `HistogramQuantile`, …). - PassThrough, -} - -/// Does an already-**available** [`Implementation`] — e.g. a summary -/// instance a downstream deployment already materialized somewhere, found -/// via whatever inventory/index that deployment keeps — satisfy a -/// **required** [`Implementation`] (what [`implementation_for`]/ -/// [`implementation_for_with`] computed for some [`AggIntent`])? -/// -/// This is the query-optimization-literature "materialized view matching" -/// / "answering queries using views" question, narrowed to this crate's -/// summary vocabulary: not "can I build this from scratch" (that's what -/// `implementation_for` answers) but "does something that already exists -/// answer this". -/// -/// `asap-plan` deliberately ships no implementation of this trait and no -/// default method body — unlike [`implementation_for`], which decision an -/// available `Implementation` satisfies a required one is not a fact this -/// crate can settle on its own. Two real, reasonable answers already -/// diverge outside this crate: -/// -/// - A **pure sketch-algebra** answer would say a `Sketch{kind: Kll, ..}` -/// requirement is satisfied by an available `DDSketch` (both quantile -/// sketches), and that a heap-bearing top-k sketch also answers a bare -/// frequency point-query (the heap is additional info on the same -/// underlying matrix) — but not the reverse. -/// - A **deployment with its own storage-layout rules** may need more: -/// e.g. whether a multi-population accumulator can serve a -/// single-population query via re-aggregation is a fact about that -/// deployment's storage layout, not about any summary family's kind at -/// all — a family's own kind doesn't encode grouping (grouping lives on -/// the post-ASAP node's `by` instead), so there is nothing in this -/// crate's own vocabulary to subsume. -/// -/// Implementations are expected to consult `required`/`available`'s -/// `kind` (and whatever grouping/placement context the deployment tracks -/// alongside `Implementation`, which this trait's signature doesn't carry -/// because this crate has no inventory concept to carry it in). -pub trait Matcher { - fn is_satisfied_by(&self, required: &Implementation, available: &Implementation) -> bool; -} - -/// Confidence δ assumed when the target carries only an ε -/// (`AccuracyTarget::Epsilon`): the (ε, δ)-parameterised sketches (CMS) need -/// one. `ln(1/0.01) → depth 5`, matching the conventional CMS sizing. -pub const DEFAULT_DELTA: f64 = 0.01; - -/// The sketch families that can serve an intent, most-preferred first. -/// This is the `AggIntent → SketchKind` map of issue #98; [`implementation_for`] binds -/// the head of the list. The tail entries are the alternatives a future cost -/// model (#6/#33) may pick instead — listed here so the candidate set has one -/// home. -pub fn summary_candidates(intent: &AggIntent) -> &'static [SketchKind] { - match intent { - AggIntent::Quantile { .. } => &[SketchKind::Kll, SketchKind::DDSketch], - AggIntent::Cardinality { .. } => &[SketchKind::Hll, SketchKind::Theta, SketchKind::Kmv], - // Count-Sketch-with-heap is CMS-with-heap's balanced/zero-mean-error - // alternative for the same heavy-hitter shape. - AggIntent::TopK { .. } => &[SketchKind::CmsWithHeap, SketchKind::CountSketchWithHeap], - AggIntent::Count { .. } => &[SketchKind::Cms, SketchKind::CountSketch], - _ => &[], - } -} - -/// The sketch-vs-exact boundary decision for one intent. -/// -/// Exhaustive over the [`AggIntent`] vocabulary — adding a variant without an -/// explicit realization is a compile error, and the coverage-matrix test pins -/// each variant's category. Ranks candidate summaries via -/// [`DefaultCostModel`] (`asap-plan`'s built-in static preference order, -/// unchanged); use [`implementation_for_with`] to plug in a deployment-specific -/// [`CostModel`] instead. -pub fn implementation_for(intent: &AggIntent) -> Implementation { - implementation_for_with(intent, &DefaultCostModel) -} - -/// Like [`implementation_for`], but ranks candidate summaries via `cost_model` (see -/// [`crate::cost_model`]) instead of the built-in static preference order. -pub fn implementation_for_with(intent: &AggIntent, cost_model: &dyn CostModel) -> Implementation { - match intent { - // ── Approximate-capable intents — the AccuracyTarget decides ──────── - AggIntent::Quantile { accuracy, .. } - | AggIntent::Cardinality { accuracy, .. } - | AggIntent::Count { accuracy } - | AggIntent::TopK { accuracy, .. } => match accuracy { - AccuracyTarget::Exact => exact_realization(intent), - _ => bind_summary_with(intent, accuracy, cost_model), - }, - - // ── Exact mergeable accumulators ───────────────────────────────────── - AggIntent::Sum { .. } => exact_accumulator(intent, ExactKind::Sum, ExactParams::Sum), - AggIntent::Min { .. } | AggIntent::Max { .. } => { - exact_accumulator(intent, ExactKind::MinMax, ExactParams::MinMax) - } - AggIntent::Rate => exact_accumulator(intent, ExactKind::Rate, ExactParams::Rate), - AggIntent::Increase => { - exact_accumulator(intent, ExactKind::Increase, ExactParams::Increase) - } - - // ── Exact, non-mergeable reducers — richer partial state than a - // single value (see `agg_is_mergeable`), so no accumulator form. - AggIntent::Avg { .. } | AggIntent::StdDev { .. } | AggIntent::Variance { .. } => { - Implementation::PassThrough - } - - // ── Classic-bucket histogram_quantile (#79): exact `le`-bucket - // interpolation over pre-aggregated counts — NOT re-sketchable. - // (The native/raw form lowers to the generic `Quantile` above.) - AggIntent::HistogramQuantile { .. } => Implementation::PassThrough, - - // ── Per-series transforms and reductions with no sketch realization: - // counter-derivatives (#44), math (#45), time/calendar (#46), - // presence (#47), native-histogram accessors (#43), and the - // `*OverTime` reducers (#51). All exact by construction. - AggIntent::Changes - | AggIntent::Delta - | AggIntent::IDelta - | AggIntent::Deriv - | AggIntent::Resets - | AggIntent::PredictLinear { .. } - | AggIntent::DoubleExpSmoothing { .. } - | AggIntent::HistogramCount - | AggIntent::HistogramSum - | AggIntent::HistogramAvg - | AggIntent::HistogramStdDev - | AggIntent::HistogramStdVar - | AggIntent::HistogramFraction { .. } - | AggIntent::Math(_) - | AggIntent::Absent - | AggIntent::AbsentOverTime - | AggIntent::PresentOverTime - | AggIntent::TimeFn(_) - | AggIntent::LastOverTime - | AggIntent::FirstOverTime - | AggIntent::MadOverTime - | AggIntent::TsOfMinOverTime - | AggIntent::TsOfMaxOverTime - | AggIntent::TsOfFirstOverTime - | AggIntent::TsOfLastOverTime => Implementation::PassThrough, - - // ── Group / count_values (#49): exact per `agg_is_exact`, but their - // output is structural (constant-1 / a synthesized label column), - // not a value a summary accumulator carries. - AggIntent::Group | AggIntent::CountValues { .. } => Implementation::PassThrough, - - // ── Extension (deployment-model-specific, issue #131) — core has no - // realization opinion for a shape it doesn't know, so it defers - // entirely to the `CostModel` (issue #150): `realize_extension` - // defaults to `PassThrough`, preserving today's behavior for - // every deployment that doesn't override it. This is also the - // only path that can currently produce `Implementation::Sample`/ - // `Wavelet`/`StatModel` — see the module docs. - AggIntent::Extension { ext_kind, payload } => { - cost_model.realize_extension(ext_kind, payload) - } - } -} - -/// Exact realization of an approximate-capable intent whose target is -/// `AccuracyTarget::Exact`. `Count` has a mergeable exact accumulator; exact -/// quantile / top-k / cardinality have no single-value summary form (they -/// need the full multiset / heap / set) and pass through. -fn exact_realization(intent: &AggIntent) -> Implementation { - match intent { - AggIntent::Count { .. } => exact_accumulator(intent, ExactKind::Count, ExactParams::Count), - _ => Implementation::PassThrough, - } -} - -fn exact_accumulator(intent: &AggIntent, kind: ExactKind, params: ExactParams) -> Implementation { - // An exact accumulator is only sound when partial states merge - // (`agg(A ∪ B) = combine(agg(A), agg(B))`). - debug_assert!( - agg_is_mergeable(intent), - "accumulator for non-mergeable {intent:?}" - ); - Implementation::ExactAggregate { kind, params } -} - -/// Bind the preferred candidate sketch, with parameters sized to the -/// target, ranking [`summary_candidates`] via `cost_model` (see -/// [`crate::cost_model`]) instead of taking the static-order head -/// unconditionally. -fn bind_summary_with( - intent: &AggIntent, - accuracy: &AccuracyTarget, - cost_model: &dyn CostModel, -) -> Implementation { - let (eps, delta) = match accuracy { - // Unreachable via `implementation_for` (Exact routes to `exact_realization`); - // degrade to the tightest parameters if called directly. - AccuracyTarget::Exact => (f64::MIN_POSITIVE, DEFAULT_DELTA), - AccuracyTarget::Epsilon(e) => (*e, DEFAULT_DELTA), - AccuracyTarget::EpsilonDelta { epsilon, delta } => (*epsilon, *delta), - }; - let ranked = cost_model.rank_candidates(intent, summary_candidates(intent)); - let kind = ranked - .into_iter() - .next() - .expect("approximate intent has at least one candidate summary"); - let params = cost_model.size_params(kind.clone(), intent, eps, delta); - Implementation::Sketch { kind, params } -} - -/// `asap-plan`'s built-in `SketchParams` sizing, keyed off the resolved -/// `(eps, delta)` accuracy budget. [`CostModel::size_params`]'s default -/// body — factored out to a free function so a deployment's own -/// `CostModel` impl can still delegate to it for the candidates it -/// doesn't want to resize itself. -/// -/// Each formula inverts the sketch family's standard error bound to the -/// smallest parameter satisfying the target, clamped to the family's sane -/// range. A non-positive ε saturates to the clamp maximum (tightest -/// allowed). -pub fn default_size_params( - kind: SketchKind, - intent: &AggIntent, - eps: f64, - delta: f64, -) -> SketchParams { - match kind { - SketchKind::Kll => SketchParams::Kll { k: kll_k(eps) }, - SketchKind::Cms => SketchParams::Cms { - width: cms_width(eps), - depth: cms_depth(delta), - }, - SketchKind::Hll => SketchParams::Hll { - precision: hll_precision(eps), - }, - SketchKind::CmsWithHeap => { - let k = match intent { - AggIntent::TopK { k, .. } => *k, - _ => unreachable!("CmsWithHeap is only a TopK candidate"), - }; - SketchParams::CmsWithHeap { - width: cms_width(eps), - depth: cms_depth(delta), - heap_size: k as u32, - } - } - // Non-preferred candidates (DDSketch / Theta / Kmv / CountSketch / - // CountSketchWithHeap) are only reachable once a cost model picks - // them; sized here so that wiring is local. - SketchKind::DDSketch => SketchParams::DDSketch { alpha: eps }, - SketchKind::Theta => SketchParams::Theta { k: kmv_k(eps) }, - SketchKind::Kmv => SketchParams::Kmv { k: kmv_k(eps) }, - // Count-Sketch is CMS's balanced/zero-mean-error alternative — - // same (width, depth) shape, sized the same way for now (a - // Count-Sketch-specific bound uses an L2-norm error guarantee - // rather than CMS's L1-norm one; this is a placeholder pending - // that refinement, same status as the other non-preferred - // candidates above). - SketchKind::CountSketch => SketchParams::CountSketch { - width: cms_width(eps), - depth: cms_depth(delta), - }, - SketchKind::CountSketchWithHeap => { - let k = match intent { - AggIntent::TopK { k, .. } => *k, - _ => unreachable!("CountSketchWithHeap is only a TopK candidate"), - }; - SketchParams::CountSketchWithHeap { - width: cms_width(eps), - depth: cms_depth(delta), - heap_size: k as u32, - } - } - } -} - -/// A deployment's explicit bet about how "typical" (non-adversarial) its -/// workload's collision pattern is expected to be, consumed only by -/// [`posterior_aware_size_params`]. -/// -/// This is **not** derived from Chen et al.'s posterior-error-estimation -/// technique (issue #239, `asap_types::post_asap::query_time::error_estimation`) -/// — that technique computes a tighter bound *at query time* from a -/// sketch's real counter values, and this repo has no sketch runtime yet -/// for a real counter array to size against (see that module's docs, and -/// `asap_types::post_asap::query_time`'s module doc for why it's a -/// deliberately separate folder from this crate's own *plan-time* code). -/// This struct is this crate's own *plan-time* analogue of the same -/// underlying intuition — an expected-case (skewed / non-adversarial) -/// workload needs a smaller sketch than the adversarial worst case — -/// expressed as an explicit, caller-supplied assumption rather than -/// anything observed or proven. Issue #250 tracks actually connecting the -/// two: feeding query-time-observed posterior error back into a future -/// replan's `width_relaxation` instead of a bare caller guess. -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct ExpectedCaseSizing { - /// Fraction, in `(0, 1]`, of the traditional worst-case width - /// ([`cms_width`]) the caller is betting is enough. `1.0` (or any - /// value outside `(0, 1)`) reproduces the worst-case width exactly — - /// no risk taken. A smaller value shrinks the sketch proportionally, - /// at the cost documented on [`posterior_aware_size_params`]. - pub width_relaxation: f64, -} - -/// Opt-in alternative to [`default_size_params`] for the CMS-family kinds -/// (`Cms` / `CmsWithHeap` / `CountSketch` / `CountSketchWithHeap`): sizes -/// width to `assumption.width_relaxation` of the worst-case [`cms_width`], -/// trading the unconditional worst-case `(ε,δ)` guarantee for a smaller -/// sketch under an explicit, caller-stated non-adversarial-workload bet — -/// see [`ExpectedCaseSizing`]. -/// -/// **The tradeoff, spelled out:** [`default_size_params`]'s width guarantees -/// `Pr[error > ε·|F|₁] < δ` for *any* input, including an adversarial one -/// built to maximize collisions (§3.3 of the posterior-error-estimation -/// paper this issue is about — see -/// `asap_types::post_asap::query_time::error_estimation`'s module docs). -/// Shrinking -/// width below that only keeps the same `(ε,δ)` guarantee if the real -/// workload's collision load stays within `width_relaxation` of the -/// worst-case assumption — this function does not check that, cannot check -/// it (no data exists at plan time), and does not change the formal -/// guarantee's statement; it only changes how much hardware is spent -/// chasing it. Callers accept that gap explicitly by choosing -/// `width_relaxation < 1.0`. -/// -/// Depth ([`cms_depth`]) is left unchanged from [`default_size_params`]: -/// depth trades away confidence *exponentially* (`Pr[all r rows bad] = -/// p^r` — each extra row multiplies the failure probability down), a -/// differently-shaped and materially riskier tradeoff than width's linear -/// relaxation. Issue #239 asks for *a* tighter-sizing option under a -/// stated assumption, not a full redesign of the depth/width tradeoff -/// space, so depth relaxation is left as explicit future scope. -/// -/// For every `SketchKind` outside the CMS family, this is identical to -/// [`default_size_params`] — `width_relaxation` only ever touches the -/// [`cms_width`]-sized formulas this issue is about. -/// -/// [`default_size_params`]'s own behavior is completely unchanged by this -/// function's existence — this is a separate, additive entry point, never -/// called from [`default_size_params`] or [`implementation_for`]. -pub fn posterior_aware_size_params( - kind: SketchKind, - intent: &AggIntent, - eps: f64, - delta: f64, - assumption: ExpectedCaseSizing, -) -> SketchParams { - let relaxed_width = |eps: f64| -> u32 { - let base = cms_width(eps); - let f = assumption.width_relaxation; - if !(f.is_finite() && f > 0.0 && f < 1.0) { - return base; // out-of-range bet: no relaxation, fall back to worst case - } - saturating_ceil(base as f64 * f, 2, base) - }; - match kind { - SketchKind::Cms => SketchParams::Cms { - width: relaxed_width(eps), - depth: cms_depth(delta), - }, - SketchKind::CmsWithHeap => { - let k = match intent { - AggIntent::TopK { k, .. } => *k, - _ => unreachable!("CmsWithHeap is only a TopK candidate"), - }; - SketchParams::CmsWithHeap { - width: relaxed_width(eps), - depth: cms_depth(delta), - heap_size: k as u32, - } - } - SketchKind::CountSketch => SketchParams::CountSketch { - width: relaxed_width(eps), - depth: cms_depth(delta), - }, - SketchKind::CountSketchWithHeap => { - let k = match intent { - AggIntent::TopK { k, .. } => *k, - _ => unreachable!("CountSketchWithHeap is only a TopK candidate"), - }; - SketchParams::CountSketchWithHeap { - width: relaxed_width(eps), - depth: cms_depth(delta), - heap_size: k as u32, - } - } - // Every other kind is untouched by this issue's CMS-specific - // relaxation — defer to the existing formula verbatim. Spelled out - // exhaustively, matching `default_size_params`'s own match, rather - // than a wildcard arm: a future `SketchKind` variant then fails to - // compile *here* too, instead of silently inheriting worst-case - // sizing with no signal that this function never considered it. - SketchKind::Kll => default_size_params(kind, intent, eps, delta), - SketchKind::Hll => default_size_params(kind, intent, eps, delta), - SketchKind::DDSketch => default_size_params(kind, intent, eps, delta), - SketchKind::Theta => default_size_params(kind, intent, eps, delta), - SketchKind::Kmv => default_size_params(kind, intent, eps, delta), - } -} - -// ── Parameter sizing ────────────────────────────────────────────────────────── -// -// Each function inverts the sketch family's standard error bound to the -// smallest parameter satisfying the target, clamped to the family's sane -// range. A non-positive ε saturates to the clamp maximum (tightest allowed). - -/// KLL: rank error ε ≈ 2/k ⇒ `k = ⌈2/ε⌉`. ε = 0.01 → k = 200, matching the -/// design doc's worked example (`KLL{k=200}` satisfies ε=0.01). -fn kll_k(eps: f64) -> u32 { - saturating_ceil(2.0 / eps, 8, 65_535) -} - -/// HLL: standard error ≈ 1.04/√(2^p) ⇒ `p = ⌈log2((1.04/ε)²)⌉`. The default -/// `Cardinality` target (`asap-ir::default_cardinality`) inverts to p = 14. -fn hll_precision(eps: f64) -> u8 { - saturating_ceil((1.04 / eps).powi(2).log2(), 4, 18) as u8 -} - -/// CMS: over-count ≤ ε·N with width `w = ⌈e/ε⌉` columns. -fn cms_width(eps: f64) -> u32 { - saturating_ceil(std::f64::consts::E / eps, 2, 1 << 26) -} - -/// CMS: failure probability ≤ δ with depth `d = ⌈ln(1/δ)⌉` rows. -/// δ = 0.01 → depth 5. -fn cms_depth(delta: f64) -> u32 { - saturating_ceil((1.0 / delta).ln(), 1, 32) -} - -/// KMV / theta: relative error ≈ 1/√k ⇒ `k = ⌈1/ε²⌉`. -fn kmv_k(eps: f64) -> u32 { - saturating_ceil(1.0 / (eps * eps), 16, 1 << 26) -} - -/// `⌈x⌉` clamped to `[lo, hi]`; NaN / non-positive x saturate to `hi` -/// (a degenerate ε means "as accurate as this family goes"). -fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { - if !x.is_finite() || x <= 0.0 { - return hi; - } - (x.ceil() as u32).clamp(lo, hi) -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::pre_asap::agg_intent::{ - agg_is_exact, default_cardinality, default_quantile, MathFunc, TimeFunc, - }; - - fn eps(e: f64) -> AccuracyTarget { - AccuracyTarget::Epsilon(e) - } - - /// Shorthand for asserting the realization *category*. - #[derive(Debug, PartialEq)] - enum Cat { - Sketch(SketchKind), - Acc(ExactKind), - Pass, - } - - fn cat(intent: &AggIntent) -> Cat { - match implementation_for(intent) { - Implementation::ExactAggregate { kind, .. } => Cat::Acc(kind), - Implementation::Sketch { kind, .. } => Cat::Sketch(kind), - Implementation::PassThrough => Cat::Pass, - other => { - panic!("this coverage matrix expects only Exact/Sketch/PassThrough, got {other:?}") - } - } - } - - /// The `AggIntent → SummaryKind` coverage matrix (issue #98): every intent - /// variant maps to a sketch, an exact accumulator, or an explicit - /// pass-through. `implementation_for`'s match is exhaustive, so a new variant cannot - /// compile without a decision; this matrix pins what each decision *is*. - #[test] - fn agg_intent_to_summary_kind_coverage_matrix() { - use AggIntent as A; - use Cat::*; - use ExactKind as E; - use SketchKind as K; - let matrix: Vec<(A, Cat)> = vec![ - // approximate-capable, at an ε target → sketch - (default_quantile(0.99), Sketch(K::Kll)), - (default_cardinality(), Sketch(K::Hll)), - ( - A::Count { - accuracy: eps(0.01), - }, - Sketch(K::Cms), - ), - ( - A::TopK { - k: 10, - accuracy: eps(0.01), - }, - Sketch(K::CmsWithHeap), - ), - // the same intents at Exact → exact realization - ( - A::Quantile { - col: None, - q: 0.5, - accuracy: AccuracyTarget::Exact, - }, - Pass, - ), - ( - A::Cardinality { - col: None, - accuracy: AccuracyTarget::Exact, - }, - Pass, - ), - ( - A::Count { - accuracy: AccuracyTarget::Exact, - }, - Acc(E::Count), - ), - ( - A::TopK { - k: 10, - accuracy: AccuracyTarget::Exact, - }, - Pass, - ), - // exact mergeable accumulators - (A::Sum { col: None }, Acc(E::Sum)), - (A::Min { col: None }, Acc(E::MinMax)), - (A::Max { col: None }, Acc(E::MinMax)), - (A::Rate, Acc(E::Rate)), - (A::Increase, Acc(E::Increase)), - // exact but non-mergeable → pass-through - (A::Avg { col: None }, Pass), - ( - A::StdDev { - col: None, - population: false, - }, - Pass, - ), - ( - A::Variance { - col: None, - population: true, - }, - Pass, - ), - // classic-bucket histogram_quantile is not re-sketchable (#79) - (A::HistogramQuantile { q: 0.99 }, Pass), - // counter-derivative / range-vector functions (#44) - (A::Changes, Pass), - (A::Delta, Pass), - (A::IDelta, Pass), - (A::Deriv, Pass), - (A::Resets, Pass), - (A::PredictLinear { seconds: 60.0 }, Pass), - ( - A::DoubleExpSmoothing { - smoothing: 0.5, - trend: 0.5, - }, - Pass, - ), - // native-histogram accessors (#43) - (A::HistogramCount, Pass), - (A::HistogramSum, Pass), - (A::HistogramAvg, Pass), - (A::HistogramStdDev, Pass), - (A::HistogramStdVar, Pass), - ( - A::HistogramFraction { - lower: 0.0, - upper: 1.0, - }, - Pass, - ), - // per-sample transforms (#45, #46) + presence (#47) - (A::Math(MathFunc::Abs), Pass), - (A::TimeFn(TimeFunc::Hour), Pass), - (A::Absent, Pass), - (A::AbsentOverTime, Pass), - (A::PresentOverTime, Pass), - // extended aggregations (#49) - (A::Group, Pass), - (A::CountValues { label: "v".into() }, Pass), - // additional range reducers (#51) - (A::LastOverTime, Pass), - (A::FirstOverTime, Pass), - (A::MadOverTime, Pass), - (A::TsOfMinOverTime, Pass), - (A::TsOfMaxOverTime, Pass), - (A::TsOfFirstOverTime, Pass), - (A::TsOfLastOverTime, Pass), - ]; - for (intent, expected) in &matrix { - assert_eq!(&cat(intent), expected, "realization for {intent:?}"); - } - // Every accumulator pick is mergeable; every sketch pick is on a - // genuinely approximate target (the `agg_is_*` helpers stay truthful). - for (intent, expected) in &matrix { - if let Cat::Acc(_) = expected { - assert!(agg_is_mergeable(intent), "{intent:?}"); - } - if let Cat::Sketch(_) = expected { - assert!( - !agg_is_exact(intent) || matches!(intent, AggIntent::Count { .. }), - "{intent:?} sketches only under an approximate target" - ); - } - } - } - - #[test] - fn accuracy_target_drives_the_boundary() { - // Same intent, three targets → three different decisions. - let exact = AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::Exact, - }; - assert_eq!(implementation_for(&exact), Implementation::PassThrough); - - let approx = default_quantile(0.99); // ε = 0.01 - assert_eq!( - implementation_for(&approx), - Implementation::Sketch { - kind: SketchKind::Kll, - params: SketchParams::Kll { k: 200 }, // design.md worked example - } - ); - - let looser = AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: eps(0.05), - }; - assert_eq!( - implementation_for(&looser), - Implementation::Sketch { - kind: SketchKind::Kll, - params: SketchParams::Kll { k: 40 }, // ⌈2/0.05⌉ - } - ); - } - - #[test] - fn default_cardinality_inverts_to_hll_precision_14() { - // `default_cardinality` encodes HLL's standard error at p=14; the - // sizing must invert it back exactly. - assert_eq!( - implementation_for(&default_cardinality()), - Implementation::Sketch { - kind: SketchKind::Hll, - params: SketchParams::Hll { precision: 14 }, - } - ); - } - - #[test] - fn epsilon_delta_sizes_cms_depth() { - let intent = AggIntent::Count { - accuracy: AccuracyTarget::EpsilonDelta { - epsilon: 0.001, - delta: 0.001, - }, - }; - assert_eq!( - implementation_for(&intent), - Implementation::Sketch { - kind: SketchKind::Cms, - params: SketchParams::Cms { - width: 2719, - depth: 7 - }, // ⌈e/0.001⌉, ⌈ln 1000⌉ - } - ); - // Epsilon-only falls back to DEFAULT_DELTA → depth 5. - let intent = AggIntent::Count { - accuracy: eps(0.001), - }; - assert_eq!( - implementation_for(&intent), - Implementation::Sketch { - kind: SketchKind::Cms, - params: SketchParams::Cms { - width: 2719, - depth: 5 - }, - } - ); - } - - #[test] - fn topk_heap_size_tracks_k() { - let intent = AggIntent::TopK { - k: 25, - accuracy: eps(0.01), - }; - match implementation_for(&intent) { - Implementation::Sketch { - kind: SketchKind::CmsWithHeap, - params: - SketchParams::CmsWithHeap { - width, - depth, - heap_size, - }, - } => { - assert_eq!(heap_size, 25); - assert_eq!(width, 272); // ⌈e/0.01⌉ - assert_eq!(depth, 5); - } - other => panic!("expected CmsWithHeap, got {other:?}"), - } - } - - #[test] - fn candidate_lists_match_the_issue_map() { - assert_eq!( - summary_candidates(&default_quantile(0.5)), - &[SketchKind::Kll, SketchKind::DDSketch] - ); - assert_eq!( - summary_candidates(&default_cardinality()), - &[SketchKind::Hll, SketchKind::Theta, SketchKind::Kmv] - ); - assert_eq!( - summary_candidates(&AggIntent::TopK { - k: 5, - accuracy: eps(0.01) - }), - &[SketchKind::CmsWithHeap, SketchKind::CountSketchWithHeap] - ); - assert_eq!( - summary_candidates(&AggIntent::Count { - accuracy: eps(0.01) - }), - &[SketchKind::Cms, SketchKind::CountSketch] - ); - assert!(summary_candidates(&AggIntent::Rate).is_empty()); - } - - #[test] - fn degenerate_epsilon_saturates_to_tightest_params() { - let intent = AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: eps(0.0), - }; - assert_eq!( - implementation_for(&intent), - Implementation::Sketch { - kind: SketchKind::Kll, - params: SketchParams::Kll { k: 65_535 }, - } - ); - } - - // ── posterior_aware_size_params (issue #239, integration point 2) ────── - - fn count_intent(e: f64) -> AggIntent { - AggIntent::Count { accuracy: eps(e) } - } - - #[test] - fn posterior_aware_sizing_shrinks_width_under_stated_assumption() { - let intent = count_intent(0.01); - let worst_case = default_size_params(SketchKind::Cms, &intent, 0.01, 0.01); - let relaxed = posterior_aware_size_params( - SketchKind::Cms, - &intent, - 0.01, - 0.01, - ExpectedCaseSizing { - width_relaxation: 0.5, - }, - ); - match (worst_case, relaxed) { - ( - SketchParams::Cms { - width: w0, - depth: d0, - }, - SketchParams::Cms { - width: w1, - depth: d1, - }, - ) => { - assert!( - w1 < w0, - "expected relaxed width {w1} to be strictly smaller than worst-case {w0}" - ); - assert_eq!(d0, d1, "depth must be unaffected by width_relaxation"); - } - other => panic!("expected Cms/Cms pair, got {other:?}"), - } - } - - #[test] - fn posterior_aware_sizing_at_full_relaxation_matches_worst_case() { - // width_relaxation = 1.0 must reproduce default_size_params exactly - // — the "no risk taken" boundary. - let intent = count_intent(0.01); - let worst_case = default_size_params(SketchKind::Cms, &intent, 0.01, 0.01); - let relaxed = posterior_aware_size_params( - SketchKind::Cms, - &intent, - 0.01, - 0.01, - ExpectedCaseSizing { - width_relaxation: 1.0, - }, - ); - assert_eq!(worst_case, relaxed); - } - - #[test] - fn posterior_aware_sizing_invalid_relaxation_falls_back_to_worst_case() { - let intent = count_intent(0.01); - let worst_case = default_size_params(SketchKind::Cms, &intent, 0.01, 0.01); - for bad in [0.0, -0.5, 1.5, f64::NAN, f64::INFINITY] { - let relaxed = posterior_aware_size_params( - SketchKind::Cms, - &intent, - 0.01, - 0.01, - ExpectedCaseSizing { - width_relaxation: bad, - }, - ); - assert_eq!( - worst_case, relaxed, - "width_relaxation={bad} should fall back to the worst-case width" - ); - } - } - - #[test] - fn posterior_aware_sizing_applies_to_every_cms_family_kind() { - let cms_heap_intent = AggIntent::TopK { - k: 7, - accuracy: eps(0.01), - }; - let assumption = ExpectedCaseSizing { - width_relaxation: 0.25, - }; - // CountSketch - assert_eq!( - posterior_aware_size_params( - SketchKind::CountSketch, - &count_intent(0.01), - 0.01, - 0.01, - assumption - ), - SketchParams::CountSketch { - width: 68, - depth: 5 - }, // ceil(272 * 0.25) - ); - // CmsWithHeap / CountSketchWithHeap carry k through untouched. - match posterior_aware_size_params( - SketchKind::CmsWithHeap, - &cms_heap_intent, - 0.01, - 0.01, - assumption, - ) { - SketchParams::CmsWithHeap { - width, - depth, - heap_size, - } => { - assert_eq!(width, 68); - assert_eq!(depth, 5); - assert_eq!(heap_size, 7); - } - other => panic!("expected CmsWithHeap, got {other:?}"), - } - } - - #[test] - fn posterior_aware_sizing_leaves_non_cms_kinds_unchanged() { - // Kll/Hll/etc. have no width_relaxation concept — must be byte-for- - // byte identical to default_size_params. - let intent = default_quantile(0.99); - let assumption = ExpectedCaseSizing { - width_relaxation: 0.1, - }; - assert_eq!( - posterior_aware_size_params(SketchKind::Kll, &intent, 0.01, 0.01, assumption), - default_size_params(SketchKind::Kll, &intent, 0.01, 0.01), - ); - } - - #[test] - fn default_size_params_unchanged_by_new_function_existing() { - // Regression pin: default_size_params's own worst-case behavior for - // existing callers must be untouched by adding - // posterior_aware_size_params alongside it. - assert_eq!( - default_size_params(SketchKind::Cms, &count_intent(0.001), 0.001, 0.001), - SketchParams::Cms { - width: 2719, - depth: 7 - }, - ); - } -} diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index 89cddb98..c02adf3a 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -6,14 +6,14 @@ //! needs knowledge this crate doesn't have and shouldn't acquire: the crate //! doc's layering invariant is that `asap-plan` depends only on [`asap_ir`], //! never on a runtime or a deployment model. What it *can* own is the -//! interface every deployment's cost model plugs into, so [`boundary`]'s +//! interface every deployment's cost model plugs into, so [`replacement`]'s //! summary selection has exactly one extension point instead of forcing //! each downstream (ASAPCollector + ASAPQuery-backend, ASAPFusion, …) to -//! fork [`boundary::implementation_for`]. +//! fork `replacement::implementations_for_with`. //! //! This trait is scoped to the approximate-**sketch** family specifically //! ([`CostModel::rank_candidates`]/[`size_params`](CostModel::size_params) -//! take/return [`SketchKind`]/[`SketchParams`]) — `asap_sketch` also has +//! take/return [`SketchAlgorithm`]/[`SketchParams`]) — `asap_sketch` also has //! sibling families for sampling-based, wavelet-transform, and fitted //! statistical-model summaries //! ([`asap_types::post_asap::SamplingKind`]/…/[`asap_types::post_asap::StatModelKind`]), @@ -26,8 +26,8 @@ //! than overloading these ones across incompatible `Kind`/`Params` types. //! //! Every entry point that doesn't take an explicit `&dyn CostModel` -//! ([`implementation_for`](crate::boundary::implementation_for), -//! [`implement_tree`](crate::bind::implement_tree)) runs against +//! ([`SketchAlgorithmStrategy::default_cost_model`](crate::replacement::SketchAlgorithmStrategy::default_cost_model), +//! [`search_workload`](crate::replacement::search_workload)) runs against //! [`DefaultCostModel`], so a deployment that never plugs in its own cost //! model keeps today's static-preference-order behavior exactly, byte for //! byte. @@ -41,22 +41,29 @@ //! cost comparison rather than a fixed rule. See //! `docs/design_docs/cse-cost-model-decision.md` for the full design discussion (why //! cost-based, why not a full plan-search engine, the layering constraint -//! that forces detection to stay cost-agnostic). [`bind::implement_workload_with`](crate::bind::implement_workload_with) -//! is the caller. +//! that forces detection to stay cost-agnostic). +//! [`PlanSpace::cost_sorted`](crate::replacement::PlanSpace::cost_sorted) +//! (via [`crate::replacement`]'s own `cse_preference`) and +//! [`DefaultCostModel::estimate_cost`] are this crate's own callers. + +use std::rc::Rc; use asap_types::post_asap::{ - SketchKind, SketchParams, SketchQuery, SummaryFamilyType, SummaryNode, + SketchAlgorithm, SketchParams, SketchQuery, SummaryFamilyType, SummaryNode, }; use asap_types::pre_asap::agg_intent::AggIntent; use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::query_expr::QueryExpr; -use crate::boundary::Implementation; +use crate::replacement::{ + realize_child, Implementation, Replacement, ReplacementSubDAG, TargetSubDAG, +}; /// A CSE-detected, legality-gated shared subtree with two or more consumers /// — the unit [`CostModel::cse_share_decision`] decides over. Built by -/// [`bind::implement_workload_with`](crate::bind::implement_workload_with) -/// the first time it binds a subtree that +/// [`PlanSpace::cost_sorted`](crate::replacement::PlanSpace::cost_sorted) +/// (via [`crate::replacement`]'s own `cse_preference`) the first time it +/// needs a representative bound node for a subtree that /// [`asap_types::pre_asap::cse::share_common_subtrees`] already collapsed /// onto one `Rc` for two or more workload roots. See /// `docs/design_docs/cse-cost-model-decision.md`. @@ -73,6 +80,42 @@ pub struct CseCandidate<'a> { pub consumer_count: usize, } +/// A cost estimate produced by a [`CostModel`] hook. A newtype around `f64` +/// rather than a bare `f64` return type, so a future cost dimension (e.g. +/// separate CPU/memory/network estimates, once a deployment actually needs +/// to compare along more than one axis) can be added as a field here +/// without changing every hook's signature a second time. Today it's still +/// a single unitless scalar — the same magnitude convention +/// [`default_cse_recompute_cost`]/[`default_cse_shared_maintenance_cost`] +/// already used as bare `f64`s, just wrapped. +#[derive(Debug, Clone, Copy, PartialEq, PartialOrd)] +pub struct Cost(pub f64); + +impl Cost { + /// The cost of an operation that costs nothing at all. + pub const ZERO: Cost = Cost(0.0); +} + +impl std::fmt::Display for Cost { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl std::ops::Add for Cost { + type Output = Cost; + fn add(self, rhs: Cost) -> Cost { + Cost(self.0 + rhs.0) + } +} + +impl std::ops::Mul for Cost { + type Output = Cost; + fn mul(self, rhs: usize) -> Cost { + Cost(self.0 * rhs as f64) + } +} + /// The decision [`CostModel::cse_share_decision`] returns for one /// [`CseCandidate`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -100,8 +143,8 @@ pub enum ShareDecision { /// leaf costs little to recompute, a deep multi-join subtree costs a lot. /// A deployment with real per-row/per-update cost knowledge should /// override [`CostModel::cse_recompute_cost`] instead of relying on this. -pub fn default_cse_recompute_cost(subtree: &QueryExpr) -> f64 { - asap_types::pre_asap::cse::dag_node_count(subtree) as f64 +pub fn default_cse_recompute_cost(subtree: &QueryExpr) -> Cost { + Cost(asap_types::pre_asap::cse::dag_node_count(subtree) as f64) } /// Default [`CostModel::cse_shared_maintenance_cost`]: a small @@ -116,7 +159,7 @@ pub fn default_cse_recompute_cost(subtree: &QueryExpr) -> f64 { /// deployment with real memory/update-cost numbers should override /// [`CostModel::cse_shared_maintenance_cost`] instead of relying on this /// table. -pub fn default_cse_shared_maintenance_cost(family: &SummaryFamilyType) -> f64 { +pub fn default_cse_shared_maintenance_cost(family: &SummaryFamilyType) -> Cost { const UNIT: f64 = 1.0; let weight = match family { SummaryFamilyType::Plain(_) => 1.0, @@ -126,32 +169,36 @@ pub fn default_cse_shared_maintenance_cost(family: &SummaryFamilyType) -> f64 { SummaryFamilyType::Wavelet(..) => 5.0, SummaryFamilyType::StatModel(..) => 6.0, }; - weight * UNIT + Cost(weight * UNIT) } -/// Ranks the candidate summary families for one [`AggIntent`], best choice +/// Ranks the candidate sketch algorithms for one [`AggIntent`], best choice /// first. /// -/// [`boundary::summary_candidates`] returns every family that *can* answer an +/// [`replacement::summary_candidates`] returns every algorithm that *can* answer an /// intent, in an arbitrary static preference order (issue #98's "one home" /// for the candidate set). A `CostModel` re-orders that list under real, /// deployment-specific cost knowledge this crate has no way to know about — -/// [`boundary::implementation_for_with`] implements whichever candidate ends -/// up first after ranking. +/// `replacement::implementations_for_with` constructs every candidate in the +/// resulting order. pub trait CostModel { /// Rank `candidates` (as returned by - /// [`summary_candidates`](crate::boundary::summary_candidates)) for + /// [`summary_candidates`](crate::replacement::summary_candidates)) for /// `intent`, best choice first. /// - /// Implementations MAY reorder freely and MAY drop entries that aren't - /// available in their deployment, but MUST NOT invent a candidate that - /// wasn't in the input — an unknown [`SketchKind`] has no - /// [`SketchParams`](asap_types::post_asap::SketchParams) sizing logic in - /// [`boundary::implementation_for_with`] and binding it will panic. - /// Returning an empty `Vec` means "no candidate is acceptable"; - /// `implementation_for_with` treats that the same as `candidates` - /// having been empty to begin with. - fn rank_candidates(&self, intent: &AggIntent, candidates: &[SketchKind]) -> Vec; + /// Implementations MAY reorder freely, but MUST return exactly the input + /// candidates: no additions, removals, or duplicates. Candidate legality + /// and availability belong to replacement generation, not costing; letting + /// this hook filter would violate [`ReplacementStrategy`]'s exhaustive, + /// never-prune contract. This invariant is checked at every production call + /// site, and a violation panics with a contract error. + /// + /// [`ReplacementStrategy`]: crate::replacement::ReplacementStrategy + fn rank_candidates( + &self, + intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec; /// Size [`SketchParams`] for `kind` (one of the candidates /// [`rank_candidates`](Self::rank_candidates) put first) under the @@ -160,24 +207,24 @@ pub trait CostModel { /// Splitting sizing out from candidate selection lets a deployment own /// its own parameter-sizing math (e.g. an empirically-tuned table, or /// discrete rungs required by a downstream catalog) without forking - /// [`boundary::implementation_for_with`] — the same "one extension + /// `replacement::implementations_for_with` — the same "one extension /// point" rationale as `rank_candidates`, one level deeper. Default: - /// [`boundary::default_size_params`], `asap-plan`'s built-in formulas + /// [`replacement::default_size_params`], `asap-plan`'s built-in formulas /// (unchanged) — a deployment that only needs to reorder candidates, /// not resize them, can leave this method unimplemented. fn size_params( &self, - kind: SketchKind, + kind: SketchAlgorithm, intent: &AggIntent, eps: f64, delta: f64, ) -> SketchParams { - crate::boundary::default_size_params(kind, intent, eps, delta) + crate::replacement::default_size_params(kind, intent, eps, delta) } /// Realize an `AggIntent::Extension { ext_kind, payload }` — a /// deployment-specific intent shape core has no realization opinion - /// for (issue #131). `boundary::implementation_for_with` consults this + /// for (issue #131). `replacement::implementations_for_with` consults this /// for every `Extension` node instead of hardcoding `PassThrough` /// (issue #150). Default: `PassThrough` — preserves today's behavior /// for every deployment that doesn't override this, exactly like @@ -190,7 +237,7 @@ pub trait CostModel { /// same `CostModel` realized as `Implementation::Sketch` via /// [`realize_extension`](Self::realize_extension). Only ever called /// when `realize_extension` returned `Sketch` for the same - /// `(ext_kind, payload)` — `bind::readout` has no other way to build a + /// `(ext_kind, payload)` — `replacement::readout` has no other way to build a /// `SketchQuery` for a shape core doesn't know. A deployment that /// overrides `realize_extension` to return `Sketch` for some /// `ext_kind` MUST also override this for that same `ext_kind`, or @@ -212,7 +259,7 @@ pub trait CostModel { /// independently at a single use site. Default: /// [`default_cse_recompute_cost`] (a structural-size proxy). See /// `docs/design_docs/cse-cost-model-decision.md`. - fn cse_recompute_cost(&self, candidate: &CseCandidate) -> f64 { + fn cse_recompute_cost(&self, candidate: &CseCandidate) -> Cost { default_cse_recompute_cost(candidate.subtree) } @@ -222,9 +269,9 @@ pub trait CostModel { /// weight table), applied to whichever field of /// `candidate.bound_summary`'s output schema actually carries summary /// state (falls back to the cheapest, `Plain`, weight if none does — - /// e.g. `bound_summary` is a passthrough `Logical` node with nothing + /// e.g. `bound_summary` is a passthrough `KeepPreAsap` node with nothing /// summary-shaped to maintain). See `docs/design_docs/cse-cost-model-decision.md`. - fn cse_shared_maintenance_cost(&self, candidate: &CseCandidate) -> f64 { + fn cse_shared_maintenance_cost(&self, candidate: &CseCandidate) -> Cost { let family = candidate .bound_summary .schema @@ -252,7 +299,7 @@ pub trait CostModel { /// (keeping this comparison), or override this method directly for a /// wholly different policy. fn cse_share_decision(&self, candidate: &CseCandidate) -> ShareDecision { - let recompute_total = self.cse_recompute_cost(candidate) * candidate.consumer_count as f64; + let recompute_total = self.cse_recompute_cost(candidate) * candidate.consumer_count; let shared = self.cse_shared_maintenance_cost(candidate); if shared <= recompute_total { ShareDecision::Share @@ -260,24 +307,138 @@ pub trait CostModel { ShareDecision::RecomputeIndependently } } + + /// Estimate a comparable, numeric cost for one already-constructed + /// [`ReplacementSubDAG`] candidate at `target` — a real `f64`, not just a + /// relative rank, meant for a caller that wants to *display* "candidate A + /// costs ≈ X, candidate B costs ≈ Y" (e.g. a DAG-visualization view built + /// on [`PlanSpace::cost_sorted`](crate::replacement::PlanSpace::cost_sorted)), + /// not just order candidates against each other — that ordering job + /// already belongs to [`rank_candidates`](Self::rank_candidates) (for a + /// [`SketchAlgorithmStrategy`](crate::replacement::SketchAlgorithmStrategy) + /// group) and [`cse_share_decision`](Self::cse_share_decision) (for a + /// [`SharedSubtreeStrategy`](crate::replacement::SharedSubtreeStrategy) + /// group). + /// + /// One method covers both candidate shapes this crate ships: + /// `candidate.replacement`'s [`Replacement::Summary`] arm (a + /// `SketchAlgorithmStrategy` candidate — the bound `SummaryNode` is right + /// there, nothing to reconstruct) and its [`Replacement::Rewrite`] arm + /// (a `SharedSubtreeStrategy` share-vs-recompute candidate — no bound + /// `SummaryNode` of its own, since sharing is a decision about a target + /// already bound some other way; a representative binding is recovered + /// from `target` itself). `target` is threaded through explicitly + /// (rather than only ever the target embedded in `candidate` — there + /// isn't one for a `Rewrite`) so both arms have the `consumer_count` + /// context a cost estimate needs to be meaningful. + /// + /// Default: **not a real cost model** — always returns `f64::NAN`. + /// `f64::partial_cmp` against `NAN` is always `None`, so a caller that + /// forgot to check whether its `CostModel` actually overrides this can't + /// silently treat the placeholder as a real comparison. A deployment + /// that wants numeric costs exposed should override this method; + /// [`DefaultCostModel`] does, reusing + /// [`cse_recompute_cost`](Self::cse_recompute_cost)/ + /// [`cse_shared_maintenance_cost`](Self::cse_shared_maintenance_cost) — + /// the same arithmetic that already backs `cse_share_decision` — rather + /// than inventing a second, drifting cost formula. + fn estimate_cost(&self, candidate: &ReplacementSubDAG, target: &TargetSubDAG<'_>) -> f64 { + let _ = (candidate, target); + f64::NAN + } +} + +/// Apply [`CostModel::rank_candidates`] and enforce its permutation-only +/// contract at the boundary where planner code consumes the result. +pub(crate) fn validated_candidate_ranking( + cost_model: &dyn CostModel, + intent: &AggIntent, + candidates: &[SketchAlgorithm], +) -> Vec { + let ranked = cost_model.rank_candidates(intent, candidates); + let mut expected = candidates.to_vec(); + let mut actual = ranked.clone(); + expected.sort(); + actual.sort(); + assert_eq!( + actual, expected, + "CostModel::rank_candidates must return a permutation of its input; candidate generation is exhaustive and cost models may not add, remove, or duplicate candidates" + ); + ranked } /// The default cost model: preserves [`summary_candidates`]'s built-in static -/// order and [`boundary::default_size_params`]'s built-in sizing unchanged. +/// order and [`replacement::default_size_params`]'s built-in sizing unchanged. /// -/// [`summary_candidates`]: crate::boundary::summary_candidates +/// [`summary_candidates`]: crate::replacement::summary_candidates pub struct DefaultCostModel; impl CostModel for DefaultCostModel { - fn rank_candidates(&self, _intent: &AggIntent, candidates: &[SketchKind]) -> Vec { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { candidates.to_vec() } + + /// Real numbers, reusing [`CostModel::cse_recompute_cost`]/ + /// [`CostModel::cse_shared_maintenance_cost`] — the same arithmetic + /// `cse_share_decision`'s default body already composes — rather than a + /// second formula: + /// + /// - [`Replacement::Summary`]: `cse_recompute_cost` (the one-time + /// structural cost of building `target` at all) plus + /// `cse_shared_maintenance_cost` of the candidate's own bound family + /// (a pricier family — a sketch over an exact accumulator, say — + /// costs more here, consistent with the per-family weighting + /// [`default_cse_shared_maintenance_cost`] already orders candidates + /// by). + /// - [`Replacement::Rewrite`]: recovers one representative bound + /// `SummaryNode` for `target` via `realize_child` (the same + /// rank-and-take-first helper `replacement::realize_child` reuses for the + /// identical need), then charges + /// `cse_shared_maintenance_cost` for the candidate that shares + /// `target`'s own `Rc` (`Rc::ptr_eq`), or `cse_recompute_cost * + /// consumer_count` for the one that doesn't — the same two terms + /// `cse_share_decision` already compares against each other. `NaN` + /// only if `target` itself can't be bound at all (schema derivation + /// failed) — never expected for a target that's already part of a + /// legitimate workload tree. + fn estimate_cost(&self, candidate: &ReplacementSubDAG, target: &TargetSubDAG<'_>) -> f64 { + let consumer_count = target.consumer_count.max(1); + match &candidate.replacement { + Replacement::Summary(node) => { + let cse = CseCandidate { + subtree: target.root, + bound_summary: node, + consumer_count, + }; + (self.cse_recompute_cost(&cse) + self.cse_shared_maintenance_cost(&cse)).0 + } + Replacement::Rewrite(rc) => { + let Ok(bound) = realize_child(target.root, self) else { + return f64::NAN; + }; + let cse = CseCandidate { + subtree: target.root, + bound_summary: &bound, + consumer_count, + }; + if Rc::ptr_eq(rc, target.root) { + self.cse_shared_maintenance_cost(&cse).0 + } else { + (self.cse_recompute_cost(&cse) * consumer_count).0 + } + } + } + } } #[cfg(test)] mod tests { use super::*; - use crate::boundary::summary_candidates; + use crate::replacement::summary_candidates; use asap_types::pre_asap::agg_intent::default_cardinality; #[test] @@ -296,8 +457,8 @@ mod tests { fn rank_candidates( &self, _intent: &AggIntent, - candidates: &[SketchKind], - ) -> Vec { + candidates: &[SketchAlgorithm], + ) -> Vec { let mut v = candidates.to_vec(); v.reverse(); v @@ -308,10 +469,52 @@ mod tests { fn custom_cost_model_can_reorder_candidates() { let intent = default_cardinality(); let candidates = summary_candidates(&intent); - let ranked = AlwaysPreferLast.rank_candidates(&intent, candidates); + let ranked = validated_candidate_ranking(&AlwaysPreferLast, &intent, candidates); assert_eq!(ranked.first(), candidates.last()); } + struct DropsLast; + + impl CostModel for DropsLast { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + candidates[..candidates.len() - 1].to_vec() + } + } + + #[test] + #[should_panic(expected = "must return a permutation of its input")] + fn candidate_ranking_rejects_filtering() { + let intent = default_cardinality(); + let candidates = summary_candidates(&intent); + validated_candidate_ranking(&DropsLast, &intent, candidates); + } + + struct DuplicatesFirst; + + impl CostModel for DuplicatesFirst { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + let mut ranked = candidates.to_vec(); + ranked.push(candidates[0].clone()); + ranked + } + } + + #[test] + #[should_panic(expected = "must return a permutation of its input")] + fn candidate_ranking_rejects_additions_and_duplicates() { + let intent = default_cardinality(); + let candidates = summary_candidates(&intent); + validated_candidate_ranking(&DuplicatesFirst, &intent, candidates); + } + /// A deployment that only overrides `rank_candidates` keeps /// `asap-plan`'s built-in sizing via the trait's default `size_params` /// body — the split is opt-in per method, not all-or-nothing. @@ -319,8 +522,8 @@ mod tests { fn size_params_default_body_matches_default_size_params() { let intent = default_cardinality(); assert_eq!( - AlwaysPreferLast.size_params(SketchKind::Hll, &intent, 0.01, 0.01), - crate::boundary::default_size_params(SketchKind::Hll, &intent, 0.01, 0.01), + AlwaysPreferLast.size_params(SketchAlgorithm::Hll, &intent, 0.01, 0.01), + crate::replacement::default_size_params(SketchAlgorithm::Hll, &intent, 0.01, 0.01), ); } @@ -334,24 +537,24 @@ mod tests { fn rank_candidates( &self, _intent: &AggIntent, - candidates: &[SketchKind], - ) -> Vec { + candidates: &[SketchAlgorithm], + ) -> Vec { candidates.to_vec() } fn size_params( &self, - kind: SketchKind, + kind: SketchAlgorithm, intent: &AggIntent, eps: f64, delta: f64, ) -> SketchParams { match kind { - SketchKind::Kll => { + SketchAlgorithm::Kll => { let k = if eps >= 0.01 { 200 } else { 2048 }; SketchParams::Kll { k } } - other => crate::boundary::default_size_params(other, intent, eps, delta), + other => crate::replacement::default_size_params(other, intent, eps, delta), } } } @@ -362,19 +565,21 @@ mod tests { let intent = default_quantile(0.99); assert_eq!( - DiscreteKllRungs.size_params(SketchKind::Kll, &intent, 0.001, 0.01), + DiscreteKllRungs.size_params(SketchAlgorithm::Kll, &intent, 0.001, 0.01), SketchParams::Kll { k: 2048 }, ); // Untouched kinds still fall through to the default formula. assert_eq!( - DiscreteKllRungs.size_params(SketchKind::Hll, &intent, 0.01, 0.01), - crate::boundary::default_size_params(SketchKind::Hll, &intent, 0.01, 0.01), + DiscreteKllRungs.size_params(SketchAlgorithm::Hll, &intent, 0.01, 0.01), + crate::replacement::default_size_params(SketchAlgorithm::Hll, &intent, 0.01, 0.01), ); } // ── CSE sharing (issue #237, #223 stage 4) ────────────────────────── - use asap_types::post_asap::{ExactKind, ExactParams, SummaryExpr, SummaryField, SummarySchema}; + use asap_types::post_asap::{ + ExactKind, ExactParams, SketchKind, SummaryExpr, SummaryField, SummarySchema, + }; use asap_types::pre_asap::query_expr::Source; use asap_types::pre_asap::schema::{Column, DataType, Schema}; @@ -397,7 +602,7 @@ mod tests { SummaryNode { expr: SummaryExpr::SummaryAgg { child: std::rc::Rc::new(SummaryNode { - expr: SummaryExpr::Logical(Box::new(scan())), + expr: SummaryExpr::KeepPreAsap(Box::new(scan())), schema: SummarySchema { fields: vec![], time_index: None, @@ -425,7 +630,7 @@ mod tests { cols: vec![0], child: std::rc::Rc::new(leaf.clone()), }; - assert!(default_cse_recompute_cost(&leaf) > 0.0); + assert!(default_cse_recompute_cost(&leaf) > Cost::ZERO); assert!(default_cse_recompute_cost(&nested) > default_cse_recompute_cost(&leaf)); } @@ -460,12 +665,12 @@ mod tests { }; assert_eq!( default_cse_recompute_cost(&no_sharing), - 3.0, + Cost(3.0), "no sharing: Join + 2 independent Scans = 3 unique nodes" ); assert_eq!( default_cse_recompute_cost(&with_sharing), - 2.0, + Cost(2.0), "internal sharing: Join + 1 shared Scan (referenced twice) = \ 2 unique nodes, not 3 — a tree-shaped size measure would \ wrongly charge for the shared Scan twice" @@ -479,8 +684,7 @@ mod tests { ExactParams::Sum, )); let sketch = default_cse_shared_maintenance_cost(&SummaryFamilyType::Sketch( - SketchKind::Hll, - SketchParams::Hll { precision: 12 }, + SketchKind::new(SketchAlgorithm::Hll, SketchParams::Hll { precision: 12 }), )); assert!( exact < sketch, @@ -536,12 +740,12 @@ mod tests { fn rank_candidates( &self, _intent: &AggIntent, - candidates: &[SketchKind], - ) -> Vec { + candidates: &[SketchAlgorithm], + ) -> Vec { candidates.to_vec() } - fn cse_recompute_cost(&self, _candidate: &CseCandidate) -> f64 { - 1e9 + fn cse_recompute_cost(&self, _candidate: &CseCandidate) -> Cost { + Cost(1e9) } } @@ -564,4 +768,113 @@ mod tests { ShareDecision::Share ); } + + // ── estimate_cost ──────────────────────────────────────────────────── + + /// The trait's default `estimate_cost` body is an explicit placeholder, + /// not a real cost model — a `CostModel` that only overrides + /// `rank_candidates` (the minimum required to implement the trait) must + /// still get `f64::NAN` back, never a value that looks like a real + /// estimate. + #[test] + fn estimate_cost_default_body_is_a_nan_placeholder() { + struct RankOnly; + impl CostModel for RankOnly { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + candidates.to_vec() + } + } + + let root = Rc::new(scan()); + let target = TargetSubDAG::new(&root); + let candidate = ReplacementSubDAG { + replacement: Replacement::Summary(Rc::new(summary_node(SummaryFamilyType::Plain( + asap_types::pre_asap::DataType::Float64, + )))), + rationale: "whatever".into(), + }; + assert!(RankOnly.estimate_cost(&candidate, &target).is_nan()); + } + + /// `DefaultCostModel::estimate_cost` for a [`Replacement::Summary`] + /// candidate reuses [`default_cse_shared_maintenance_cost`]'s own + /// per-family ordering: a candidate bound to a cheap-to-maintain family + /// (an exact accumulator) must cost less than one bound to an + /// expensive-to-maintain family (a fitted statistical model), same + /// target either way — consistent with + /// `default_shared_maintenance_cost_orders_families_cheapest_to_priciest` + /// above. + #[test] + fn estimate_cost_for_summary_orders_candidates_by_family_cheapest_to_priciest() { + let root = Rc::new(scan()); + let target = TargetSubDAG::new(&root); + + let cheap = ReplacementSubDAG { + replacement: Replacement::Summary(Rc::new(summary_node( + SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + ))), + rationale: "exact accumulator".into(), + }; + let pricey = ReplacementSubDAG { + replacement: Replacement::Summary(Rc::new(summary_node(SummaryFamilyType::StatModel( + asap_types::post_asap::StatModelKind::Parametric, + asap_types::post_asap::StatModelParams::Parametric { + family: "gaussian_mixture".into(), + }, + )))), + rationale: "fitted statistical model".into(), + }; + + let cheap_cost = DefaultCostModel.estimate_cost(&cheap, &target); + let pricey_cost = DefaultCostModel.estimate_cost(&pricey, &target); + assert!( + cheap_cost.is_finite() && pricey_cost.is_finite(), + "cheap={cheap_cost}, pricey={pricey_cost}" + ); + assert!( + cheap_cost < pricey_cost, + "an ExactAggregate candidate should cost less than a StatModel one: \ + exact={cheap_cost}, stat_model={pricey_cost}" + ); + } + + /// `DefaultCostModel::estimate_cost` for a [`Replacement::Rewrite`] pair + /// (the `SharedSubtreeStrategy` share-vs-recompute shape) agrees with + /// what `cse_share_decision` would already pick for the same target: with + /// many consumers of a cheap-to-recompute leaf, the "share" candidate + /// (the target's own `Rc`) must cost less than the "recompute + /// independently" one (a fresh `Rc`) — mirrors + /// `cse_share_decision_shares_when_recompute_dominates_maintenance` + /// above, through `estimate_cost` instead of `cse_share_decision` + /// directly. + #[test] + fn estimate_cost_for_rewrite_prefers_sharing_when_recompute_dominates_maintenance() { + let target_root = Rc::new(scan()); + let target = TargetSubDAG::with_consumer_count(&target_root, 20); + + let share = ReplacementSubDAG { + replacement: Replacement::Rewrite(Rc::clone(&target_root)), + rationale: "build once and share".into(), + }; + let recompute = ReplacementSubDAG { + replacement: Replacement::Rewrite(Rc::new((*target_root).clone())), + rationale: "build independently".into(), + }; + + let share_cost = DefaultCostModel.estimate_cost(&share, &target); + let recompute_cost = DefaultCostModel.estimate_cost(&recompute, &target); + assert!( + share_cost.is_finite() && recompute_cost.is_finite(), + "share={share_cost}, recompute={recompute_cost}" + ); + assert!( + share_cost < recompute_cost, + "with 20 consumers of a cheap-to-recompute leaf, sharing should cost less: \ + share={share_cost}, recompute={recompute_cost}" + ); + } } diff --git a/crates/asap-aware-mapping/src/explanation.rs b/crates/asap-aware-mapping/src/explanation.rs new file mode 100644 index 00000000..fa25dd3d --- /dev/null +++ b/crates/asap-aware-mapping/src/explanation.rs @@ -0,0 +1,784 @@ +//! This crate's **explanation of a replacement**: for a `TargetSubDAG` that +//! [`crate::replacement::search_workload`] found something to say about, why +//! does that candidate exist? (issue #33: "Add logic to detect which +//! optimizations are applicable to a query workload"; this module: issue +//! #257.) +//! +//! This module does not answer "is optimization X applicable here, yes or +//! no" — that framing implies a classifier deciding admissibility from +//! scratch. What it actually does is narrower and more mechanical: reuse a +//! matching candidate's own [`crate::replacement::ReplacementSubDAG::rationale`] +//! to explain, in the candidate's own words, why a [`Replacement`] exists at +//! a given target. No new prose is invented here; see "The reframing" below +//! for exactly what's being reused and why. +//! +//! ## The reframing: an explanation *is* "this `TargetSubDAG`'s candidate +//! list is non-trivial" +//! +//! Earlier (PR #247, superseded by this module — see "What this replaces" +//! below), "is optimization X applicable here?" was a yes/no fact each rule +//! re-derived by walking the tree itself. That made sense before there was +//! any other structure to consult. But [`crate::replacement::search_workload`] +//! (issue #252) now *already* computes, for every +//! [`TargetSubDAG`](crate::replacement::TargetSubDAG) in the workload, every +//! semantically valid [`crate::replacement::ReplacementSubDAG`] a registered +//! [`ReplacementStrategy`] can propose — a [`PlanSpace`] of [`MemoGroup`]s. A +//! rule re-deriving the same yes/no fact from scratch would be answering a +//! question the search already answered, via a second, independently +//! maintained traversal that has to keep agreeing with the first one. +//! +//! Once that candidate space exists, "which optimizations are applicable" +//! collapses into a single question this module asks of *that* data instead: +//! **for a given `TargetSubDAG`, does its candidate list contain anything +//! other than the trivial, no-op realization?** A `TargetSubDAG` whose only +//! candidate is "the one thing `SketchAlgorithmStrategy` would have committed +//! to anyway, with no alternative" has no optimization to report — that +//! candidate isn't an *opportunity*, it's just the target's existing shape +//! reflected back. A `TargetSubDAG` with more than one candidate (several +//! sketch families to choose between), or one candidate that is itself a +//! genuine alternative to the status quo (share this already-shared subtree +//! instead of recomputing it at every consumer), *is* an applicability +//! finding — [`explain_replacements`] and +//! [`explain_replacements_with`] just translate [`PlanSpace`]'s +//! [`MemoGroup`]s into that shape: +//! +//! - [`ExplanationKind::SketchApproximation`] — the `TargetSubDAG`'s +//! candidate list contains at least one [`Replacement::Summary`] that +//! actually realizes a sketch family (`SummaryFamilyType::Sketch`), i.e. +//! [`SketchAlgorithmStrategy`] found something to offer beyond whatever +//! exact/pass-through candidate [`crate::replacement`]'s own +//! `implementations_for_with` would have committed to on its own. +//! - [`ExplanationKind::CommonSubexpressionReuse`] — the `TargetSubDAG` +//! has two or more consumers *and* its candidate list contains the +//! [`SharedSubtreeStrategy`] "build once and share" candidate (the one +//! whose `Rc` is the group's own `target`) — i.e. sharing this subtree +//! instead of recomputing it independently is a real, reported choice, not +//! just an accident of how the workload happened to be built. +//! +//! Each finding's `reason` is literally the matching candidate's own +//! [`crate::replacement::ReplacementSubDAG::rationale`] (joined, if more than one candidate +//! qualifies) — this module invents no new prose to explain *why* a +//! candidate is valid; that explanation already exists on the candidate a +//! [`ReplacementStrategy`] produced, and repeating it here (rather than +//! re-describing the same fact in different words) keeps exactly one place +//! that has to be right about "why is this a valid alternative". +//! +//! ## What this replaces, and what carries over unmodified +//! +//! [`ReplacementExplanation`] and [`ExplanationKind`] keep PR #247's original +//! shape and contract — a struct/enum pair meant for a downstream +//! DAG-visualization consumer, `#[non_exhaustive]` discipline (only an +//! optimization backed by a real, registered [`ReplacementStrategy`] gets a +//! variant; see the catalog table below for everything still deliberately +//! unrepresented). So do the two top-level entry points, +//! [`explain_replacements`] and [`explain_replacements_with`] +//! — same "workload roots in, findings out" contract, mirroring +//! [`crate::replacement::search_workload`]/[`crate::replacement::search_workload_with`]'s +//! own signature shape. Only the *data source* changed: this module now +//! calls those two functions and translates the result, rather than running +//! its own rules and their supporting traversal over the tree a second time. +//! All of that old traversal is deleted, not kept alongside the new +//! implementation — see "Two guarantees the old traversal made, re-verified" +//! below for the two properties it's important that deletion didn't quietly +//! lose. +//! +//! ## Why a second, applicability-specific rule trait doesn't exist here +//! +//! PR #247 gave this module its own extension-point trait, `ApplicabilityRule` +//! (`fn optimization(&self) -> ExplanationKind` + `fn evaluate(&self, roots) +//! -> Vec`), the same shape [`crate::cost_model::CostModel`] +//! and [`crate::replacement::Matcher`] use elsewhere in this crate. Once +//! findings are a *view* over [`PlanSpace`] rather than an independent +//! computation, that trait would be a second extension point answering a +//! question [`ReplacementStrategy`] (issue #251) already answers: "does this +//! `TargetSubDAG` have an alternative worth reporting, and why". A caller who +//! wants a new optimization represented as a finding needs a new +//! `impl ReplacementStrategy` wired into +//! [`crate::replacement::search_workload_with`]'s strategy set *regardless* +//! (that's the only way its candidates end up in the [`PlanSpace`] this +//! module reads) — adding an `ApplicabilityRule` too would mean maintaining +//! two extension points for the same new capability, one of which (the rule) +//! would just be re-describing candidates the other (the strategy) already +//! produced. So this module ships no extension-point trait of its own: +//! [`ReplacementStrategy`] already *is* that extension point, one layer +//! down, and [`explain_replacements_with`]'s own `strategies` +//! parameter is where a caller plugs in a custom one (or a custom +//! `CostModel`, via [`crate::replacement::SketchAlgorithmStrategy::new`]) — the identical spot +//! [`crate::replacement::search_workload_with`] itself exposes. +//! +//! ## Two guarantees the old traversal made, re-verified against the new one +//! +//! 1. **A finding is reported at the maximal `TargetSubDAG`, never once more +//! per subsumed descendant.** [`crate::replacement`]'s own +//! `discover_targets` (used by [`crate::replacement::search_workload_with`], +//! and so by this module) walks every workload root's whole DAG but only +//! *recurses into a node's children the first time that node's `Rc` is +//! seen*; every subsequent occurrence still counts towards +//! `consumer_count`, but never triggers a second descent. A node nested +//! under an already-discovered shared ancestor therefore only becomes its +//! own `TargetSubDAG` if something *outside* that ancestor also +//! references it — identical to PR #247's own discovery pass, which +//! reported "the highest point sharing starts," not a finding at every +//! subsumed level below it. Same guarantee, same mechanism, just living in +//! [`crate::replacement`] now instead of here. +//! 2. **A node reachable via more than one path is one finding, not one per +//! path.** [`MemoGroup`]s are keyed by `Rc` pointer identity in +//! [`PlanSpace`]'s internal map — there is exactly one group per distinct +//! `Rc`, full stop, so a shared `Aggregate` reached via two different +//! `BinaryOp` branches (or two different workload roots) is exactly one +//! group, hence at most one [`ExplanationKind::SketchApproximation`] +//! finding, no matter how many paths reach it. +//! [`tests::a_shared_sketchable_aggregate_is_reported_only_once`] pins +//! this directly. +//! +//! ## One thing [`PlanSpace`] doesn't carry that this module still needs: +//! human-readable `location` text +//! +//! [`MemoGroup`]/[`PlanSpace`] deliberately track only `Rc` +//! pointer identity — the currency the search itself needs — not +//! caller-facing prose. [`ReplacementExplanation::location`] is prose (a +//! breadcrumb like `root "dash_a" > lhs`), so this module keeps one small, +//! self-contained walk of its own, [`collect_locations`], whose *only* job +//! is turning "this `Rc`" into "the human-readable place(s) it occurs" for a +//! finding already decided by [`PlanSpace`]. This is not a reincarnation of +//! the deleted rule traversal: it makes no applicability decision (it runs +//! the same regardless of what any strategy found), and duplicating this +//! small, self-contained shape rather than threading location strings +//! through [`crate::replacement`]'s own `discover_targets` matches the same +//! call that module's own docs already make for its (test-only) +//! `count_consumers` counterpart — see [`crate::replacement`]'s "Where +//! `TargetSubDAG` discovery comes from" section. +//! +//! ## Catalog primitives deliberately left as future work +//! +//! The internal catalog +//! (`ProjectASAP/internal-docs/catalog_of_optimizations.md`) lists several +//! primitives with **no [`ReplacementStrategy`] implementation anywhere in +//! this codebase today**. Faking a variant for one of them would report a +//! finding this codebase cannot back with a real candidate, so none of the +//! below get an [`ExplanationKind`] variant yet — each gets one once a real +//! strategy exists and is wired into [`crate::replacement::default_strategies`]: +//! +//! | Catalog entry | Status | Where a future `ExplanationKind` would come from | +//! |---|---|---| +//! | Semantic-equivalent rewriting (e.g. `avg` → `sum`/`count`) | No `ReplacementStrategy` implementation in this codebase yet (tracked separately, issue #253) | Once wired into `default_strategies()`: any `Replacement::Rewrite` candidate that strategy proposes | +//! | Roll-ups (fine-to-coarse group-by reuse) | No `ReplacementStrategy` implementation in this codebase yet (tracked separately, issue #254) | Once wired into `default_strategies()`: any `Replacement::Rewrite` candidate that strategy proposes | +//! | Wavelets/OMP | Params type exists (`WaveletKind`/`WaveletParams`), reachable only via a deployment `CostModel::realize_extension` (no core `AggIntent` dispatch picks it) | A `ReplacementStrategy` that inspects a deployment's own `CostModel`, once some intent shape actually maps to `Implementation::Wavelet` | +//! | Sampling | Same story as Wavelets: `SamplingKind`/`SamplingParams` exist, unreachable from core dispatch | Same hook as Wavelets, for `Implementation::Sample` | +//! | Deep generative compression | No representation at all — no `Implementation`/`SummaryFamilyType` variant | Needs a new summary family added to `asap_types::post_asap` first | +//! | Approximation frameworks for windows | No representation — `TimeRange`/`PromqlSubquery` windows are always evaluated exactly | Would key off those node types once an approximate-window operator exists | +//! | Function decomposition | No representation anywhere | No hook point identified yet | +//! | Continuous distributed monitoring | No representation — `RepeatingEntry`/`RepetitionInterval` in `asap_types::workload` describe *that* a query repeats, not any monitoring-specific decomposition | Would likely key off `RepeatingEntry` once such logic exists | +//! | Incremental computation across time | No representation — nothing carries state across repeated evaluations of a `RepeatingEntry` today | Would key off `RepeatingEntry` + `TimeShift`/`TimeRange` once incremental state-carry exists | +//! | Delta encoding | No representation — `AggIntent::Delta`/`IDelta` are PromQL *value*-difference semantics, not a wire/storage delta-encoding optimization | Would plug into a future deployment-side wire/storage encoding decision (post-ASAP), not this crate's IR-level dispatch | +//! +//! [`ReplacementStrategy`]: crate::replacement::ReplacementStrategy +//! [`ReplacementSubDAG`]: crate::replacement::ReplacementSubDAG +//! [`Replacement`]: crate::replacement::Replacement +//! [`Replacement::Summary`]: crate::replacement::Replacement::Summary +//! [`Replacement::Rewrite`]: crate::replacement::Replacement::Rewrite +//! [`SketchAlgorithmStrategy`]: crate::replacement::SketchAlgorithmStrategy +//! [`SharedSubtreeStrategy`]: crate::replacement::SharedSubtreeStrategy +//! [`PlanSpace`]: crate::replacement::PlanSpace +//! [`MemoGroup`]: crate::replacement::MemoGroup + +use std::collections::HashMap; +use std::fmt::Display; +use std::rc::Rc; + +use asap_types::post_asap::{SummaryExpr, SummaryFamilyType, SummaryNode}; +use asap_types::pre_asap::cse::{structural_hash, HashCache}; +use asap_types::pre_asap::query_expr::QueryExpr; + +use crate::replacement::{self, MemoGroup, PlanSpace, Replacement, ReplacementStrategy}; + +/// Which kind of replacement a [`ReplacementExplanation`] is about. +/// +/// `#[non_exhaustive]`: only optimizations with a real [`ReplacementStrategy`] +/// behind them get a variant (see the module docs' "Catalog primitives +/// deliberately left as future work" table for everything else in the +/// catalog). +/// +/// [`ReplacementStrategy`]: crate::replacement::ReplacementStrategy +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum ExplanationKind { + /// A `TargetSubDAG`'s candidate list contains at least one + /// [`Replacement::Summary`] that realizes a sketch family — + /// [`crate::replacement::SketchAlgorithmStrategy`] found a genuine sketch + /// alternative for this `Aggregate`, beyond whatever exact/pass-through + /// candidate `crate::replacement`'s own `implementations_for_with` would + /// have committed to on its own. + SketchApproximation, + /// A `TargetSubDAG` has two or more consumers *and* its candidate list + /// contains [`crate::replacement::SharedSubtreeStrategy`]'s "build once + /// and share" candidate — the catalog's cross-statistic / cross-metrics / + /// cross-subpopulation reuse entries, all the same underlying structural + /// fact. + CommonSubexpressionReuse, +} + +/// Why a [`Replacement`] of `kind` exists at `location` (a human-readable +/// breadcrumb into the workload — e.g. `root "dashboard_p99"` or +/// `root "ratio" > lhs`): `reason` (human-readable, meant for a report/log, +/// not machine parsing — literally the matching candidate's own +/// [`crate::replacement::ReplacementSubDAG::rationale`]). +/// +/// `node_hash` is [`structural_hash`](asap_types::pre_asap::cse::structural_hash) +/// of the `TargetSubDAG`'s own `target` subtree — the same function, on the +/// same `Rc` shape, that [`asap_types::dag_export::DagNode::hash`] +/// is computed with. A downstream consumer that independently exported the +/// same `QueryExpr` (e.g. via `asap_types::dag_export::export`) can match +/// this explanation to a `DagNode` by first comparing hashes and then +/// confirming structural equality with [`ReplacementExplanation::target`]. +#[derive(Debug, Clone, PartialEq)] +pub struct ReplacementExplanation { + pub kind: ExplanationKind, + pub location: String, + pub reason: String, + pub node_hash: u64, + /// The exact target expression the explanation describes. Reporting + /// integrations use this together with `node_hash`: the hash narrows the + /// search, and structural equality makes the final match collision-safe. + pub target: Rc, +} + +/// Explain every replacement [`crate::replacement::search_workload`] finds +/// across a workload's pre-ASAP query roots, using +/// [`crate::replacement::default_strategies`]. +/// +/// `roots` — like [`crate::replacement::search_workload`]'s own `Id` type +/// parameter — is caller-chosen: a `QueryWorkload` entry's own key, an index, +/// a query name. It only needs [`Display`], since a finding's `location` is +/// prose, not a structured key back to the caller. +/// +/// Internally runs [`crate::replacement::search_workload`] to build the +/// candidate-plan space, then reads findings off it — see the module docs' +/// "The reframing" section for what that translation actually checks. +pub fn explain_replacements( + roots: Vec<(Id, QueryExpr)>, +) -> Vec { + explain_replacements_with(roots, &replacement::default_strategies()) +} + +/// Like [`explain_replacements`], but searches with `strategies` +/// instead of [`crate::replacement::default_strategies`] — the extension +/// point for a deployment-specific [`ReplacementStrategy`], or a custom +/// `CostModel` plugged into +/// [`crate::replacement::SketchAlgorithmStrategy::new`] (e.g. via +/// [`crate::replacement::default_strategies_with`]). +/// +/// [`ReplacementStrategy`]: crate::replacement::ReplacementStrategy +pub fn explain_replacements_with<'s, Id: Display>( + roots: Vec<(Id, QueryExpr)>, + strategies: &[Box], +) -> Vec { + let ided: Vec<(String, Rc)> = roots + .into_iter() + .map(|(id, expr)| (id.to_string(), Rc::new(expr))) + .collect(); + let space = replacement::search_workload_with(ided, strategies); + findings_from_plan_space(&space) +} + +/// Translate every discovered [`MemoGroup`] in `space` into zero, one, or two +/// [`ReplacementExplanation`]s (a `TargetSubDAG` can be both sketch-approximable +/// *and* shared — the two optimizations are independent axes, not mutually +/// exclusive). +/// +/// `space`'s own `Id` is always `String` here: [`explain_replacements_with`] +/// already converted the caller's `Id: Display` into a `String` (via +/// `to_string()`) before calling [`crate::replacement::search_workload_with`], +/// so this function (and [`collect_locations`], which formats `id` with +/// [`std::fmt::Debug`] for the breadcrumb text) doesn't need its own generic +/// `Id` bound. +fn findings_from_plan_space(space: &PlanSpace) -> Vec { + let locations = collect_locations(&space.roots); + // One cache for the whole pass, mirroring `dag_export::export`'s own + // `HashCache` reuse — this is a bottom-up pass over every discovered + // group, so amortizing the cache across groups (rather than resetting it + // per group) is real, not just a micro-optimization. + let mut hash_cache = HashCache::new(); + let mut findings = Vec::new(); + for group in space.groups() { + let location = locations + .get(&Rc::as_ptr(&group.target)) + .map(|locs| locs.join(", ")) + .unwrap_or_default(); + let node_hash = structural_hash(&group.target, &mut hash_cache); + + if let Some(reason) = sketch_finding_reason(group) { + findings.push(ReplacementExplanation { + kind: ExplanationKind::SketchApproximation, + location: location.clone(), + reason, + node_hash, + target: Rc::clone(&group.target), + }); + } + if let Some(reason) = shared_subexpr_finding_reason(group) { + findings.push(ReplacementExplanation { + kind: ExplanationKind::CommonSubexpressionReuse, + location, + reason, + node_hash, + target: Rc::clone(&group.target), + }); + } + } + findings +} + +/// Does `group`'s candidate list contain a genuine sketch-family realization? +/// If so, the finding's `reason` is every such candidate's own `rationale`, +/// joined — this module does not invent new prose to restate why a candidate +/// is valid. +fn sketch_finding_reason(group: &MemoGroup) -> Option { + let reasons: Vec<&str> = group + .candidates + .iter() + .filter( + |c| matches!(&c.replacement, Replacement::Summary(node) if is_sketch_realization(node)), + ) + .map(|c| c.rationale.as_str()) + .collect(); + if reasons.is_empty() { + None + } else { + Some(reasons.join("; ")) + } +} + +/// Does `group` have two or more consumers *and* a "build once and share" +/// candidate (the [`Replacement::Rewrite`] whose `Rc` is the group's own +/// `target`) in its candidate list? If so, the finding's `reason` is that +/// candidate's own `rationale`. +fn shared_subexpr_finding_reason(group: &MemoGroup) -> Option { + if group.consumer_count < 2 { + return None; + } + group + .candidates + .iter() + .find( + |c| matches!(&c.replacement, Replacement::Rewrite(rc) if Rc::ptr_eq(rc, &group.target)), + ) + .map(|c| c.rationale.clone()) +} + +/// Does `node` (unwrapping any `SummaryEstimate` layer, the same shape +/// [`crate::replacement`]'s own private `sketch_kind_of` unwraps) ultimately +/// realize a [`SummaryFamilyType::Sketch`] family? This module only needs the +/// yes/no fact (a candidate's own `rationale` already names the specific +/// `SketchKind`/`SketchAlgorithm` for a finding's `reason` text), so unlike +/// `replacement.rs`'s counterpart this returns `bool`, not the kind itself. +fn is_sketch_realization(node: &SummaryNode) -> bool { + match &node.expr { + SummaryExpr::SummaryEstimate { summary_input, .. } => is_sketch_realization(summary_input), + SummaryExpr::SummaryAgg { family, .. } => matches!(family, SummaryFamilyType::Sketch(..)), + _ => false, + } +} + +// ── location breadcrumbs ───────────────────────────────────────────────── + +/// Build `location` text for every distinct `TargetSubDAG` reachable from +/// `roots` — see the module docs' "One thing `PlanSpace` doesn't carry" +/// section for why this module needs its own small walk for this. Returns +/// every breadcrumb path that reaches a given `Rc`, not just the first: a +/// shared node referenced from two workload roots (or two branches of one +/// root) needs both breadcrumbs in its finding's `location`, not just one. +fn collect_locations(roots: &[(String, Rc)]) -> HashMap<*const QueryExpr, Vec> { + let mut locations: HashMap<*const QueryExpr, Vec> = HashMap::new(); + for (id, root) in roots { + visit(root, format!("root {id:?}"), &mut locations); + } + locations +} + +/// Record `label` as one of `node`'s breadcrumbs, then propagate that path +/// through its children. A shared ancestor is intentionally traversed once +/// per incoming path so every descendant receives every valid breadcrumb. +fn visit( + node: &Rc, + label: String, + locations: &mut HashMap<*const QueryExpr, Vec>, +) { + let ptr = Rc::as_ptr(node); + locations.entry(ptr).or_default().push(label.clone()); + visit_children(node, &label, locations); +} + +/// `node`'s own **relational-skeleton** operator children — the same scope +/// `crate::replacement`'s own target-discovery `walk_children` (and +/// `asap_types::pre_asap::cse::share_common_subtrees`'s `rebuild_children`) +/// use. Exhaustive over every `QueryExpr` variant: a new variant fails to +/// compile here until this match is extended too. +fn visit_children( + node: &QueryExpr, + label: &str, + locations: &mut HashMap<*const QueryExpr, Vec>, +) { + use QueryExpr::*; + match node { + Scan { .. } | PromqlScalarBridge(_) | QueryTimestamp => {} + PromqlVectorFromScalar(c) | PromqlScalarFromVector(c) => { + visit(c, format!("{label} > child"), locations) + } + PromqlRelabel { child, .. } + | PromqlInfoEnrich { child, .. } + | PromqlSeriesSample { child, .. } + | Filter { child, .. } + | Project { child, .. } + | Aggregate { child, .. } + | Dedup { child, .. } + | PromqlSubquery { child, .. } + | TimeRange { child, .. } + | TimeShift { child, .. } + | SQLWindowFunc { child, .. } + | Sort { child, .. } + | Limit { child, .. } => visit(child, format!("{label} > child"), locations), + Concat { children } => { + for (i, c) in children.iter().enumerate() { + visit_children(c, &format!("{label} > concat[{i}]"), locations); + } + } + Join { left, right, .. } | SetOp { left, right, .. } => { + visit(left, format!("{label} > left"), locations); + visit(right, format!("{label} > right"), locations); + } + BinaryOp { lhs, rhs, .. } => { + visit(lhs, format!("{label} > lhs"), locations); + visit(rhs, format!("{label} > rhs"), locations); + } + Column(_) + | Literal(_) + | Compare { .. } + | BoolAnd(_) + | BoolOr(_) + | Not(_) + | IsNull(_) + | IsNotNull(_) + | Cast { .. } + | InList { .. } + | FunctionCall { .. } + | Arithmetic { .. } + | Case { .. } => {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + use asap_types::pre_asap::agg_intent::{default_quantile, AggIntent}; + use asap_types::pre_asap::query_expr::{Reduction, Source}; + use asap_types::pre_asap::schema::{Column, DataType, Schema}; + use asap_types::types::AccuracyTarget; + + fn metric_scan(labels: &[&str]) -> QueryExpr { + let mut columns = vec![ + Column::new("ts", DataType::Timestamp, false), + Column::new("value", DataType::Float64, false), + ]; + columns.extend(labels.iter().map(|n| Column::new(*n, DataType::Utf8, true))); + QueryExpr::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index(columns, 0, vec![]), + } + } + + fn agg(by: Vec, intent: AggIntent, child: QueryExpr) -> QueryExpr { + QueryExpr::Aggregate { + reduction: Reduction::by(by), + measures: vec![intent], + output_names: vec![], + having: None, + child: Rc::new(child), + } + } + + // ── SketchApproximation ────────────────────────────────────────────── + + #[test] + fn approximate_quantile_is_a_sketch_applicability_finding() { + let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + let findings = explain_replacements(vec![("dashboard_p99", q)]); + let sketch: Vec<_> = findings + .iter() + .filter(|f| f.kind == ExplanationKind::SketchApproximation) + .collect(); + assert_eq!( + sketch.len(), + 1, + "expected one sketch finding, got {findings:?}" + ); + assert!(sketch[0].location.contains("dashboard_p99")); + assert!(sketch[0].reason.to_lowercase().contains("kll")); + } + + /// `node_hash` must be the literal `structural_hash` a downstream + /// consumer would compute over the *same* `QueryExpr` subtree via + /// `asap_types::dag_export::export` — the whole point of carrying it is + /// that two independent exports of the same tree agree, with no + /// string-matching against `location` required. + #[test] + fn node_hash_matches_dag_export_hash_for_the_same_subtree() { + let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + let graph = asap_types::dag_export::export(&q); + let expected_hash = graph.nodes[graph.root as usize].hash; + + let findings = explain_replacements(vec![("dashboard_p99", q)]); + let sketch = findings + .iter() + .find(|f| f.kind == ExplanationKind::SketchApproximation) + .expect("expected a sketch finding"); + assert_eq!( + sketch.node_hash, expected_hash, + "ReplacementExplanation::node_hash must match dag_export's DagNode::hash \ + for the same QueryExpr subtree" + ); + } + + #[test] + fn exact_quantile_is_not_a_sketch_applicability_finding() { + let q = agg( + vec![2], + AggIntent::Quantile { + col: None, + q: 0.99, + accuracy: AccuracyTarget::Exact, + }, + metric_scan(&["job"]), + ); + let findings = explain_replacements(vec![("exact_p99", q)]); + assert!( + findings + .iter() + .all(|f| f.kind != ExplanationKind::SketchApproximation), + "an Exact accuracy target must not report sketch-applicability, got {findings:?}" + ); + } + + #[test] + fn nested_aggregate_still_finds_the_inner_sketchable_node() { + // avg(quantile(0.9, sum by (job) (m))) shaped test isn't representable + // (avg is PassThrough, not a wrapper we recurse through structurally + // the way an Aggregate's own child is) — instead nest a sketchable + // quantile under an exact sum, the same nesting replacement.rs's own + // nested_aggregates_bind_per_node test uses. + let inner = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let outer = agg(vec![], default_quantile(0.9), inner); + let findings = explain_replacements(vec![("q", outer)]); + let sketch_count = findings + .iter() + .filter(|f| f.kind == ExplanationKind::SketchApproximation) + .count(); + assert_eq!( + sketch_count, 1, + "expected the outer quantile only, got {findings:?}" + ); + } + + #[test] + fn pass_through_intent_reports_no_sketch_finding() { + let q = agg(vec![2], AggIntent::Avg { col: None }, metric_scan(&["job"])); + let findings = explain_replacements(vec![("avg_latency", q)]); + assert!(findings + .iter() + .all(|f| f.kind != ExplanationKind::SketchApproximation)); + } + + /// A sketch-applicable `Aggregate` reachable via two paths that CSE + /// collapses onto one `Rc` — the same `median(x) == median(x)` shape + /// `pre_asap::cse`'s own `single_query_shares_its_own_repeated_subtree` + /// test uses — must be reported once, not once per path: it is exactly + /// one [`crate::replacement::MemoGroup`], keyed by `Rc` pointer identity, + /// not one per path that reaches it. + #[test] + fn a_shared_sketchable_aggregate_is_reported_only_once() { + let quantile = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + let root = QueryExpr::BinaryOp { + op: asap_types::pre_asap::query_expr::BinaryOpKind::Compare( + asap_types::pre_asap::expr_ir::CompareOpKind::Eq, + ), + lhs: Rc::new(quantile.clone()), + rhs: Rc::new(quantile), + vector_match: None, + }; + let findings = explain_replacements(vec![("ratio", root)]); + let sketch: Vec<_> = findings + .iter() + .filter(|f| f.kind == ExplanationKind::SketchApproximation) + .collect(); + assert_eq!( + sketch.len(), + 1, + "a single shared Aggregate must produce one finding, not one \ + per path that reaches it: got {findings:?}" + ); + } + + // ── CommonSubexpressionReuse ───────────────────────────────────────── + + #[test] + fn two_roots_with_the_same_grouped_aggregate_share_a_reuse_finding() { + // Grouped (`by (job)`), so the shared `Aggregate`'s output schema + // carries a provable unique key — share_common_subtrees's legality + // gate — and identical across both roots, so it is shareable. + let a = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let b = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let findings = explain_replacements(vec![("dash_a", a), ("dash_b", b)]); + let reuse: Vec<_> = findings + .iter() + .filter(|f| f.kind == ExplanationKind::CommonSubexpressionReuse) + .collect(); + assert_eq!( + reuse.len(), + 1, + "expected one reuse finding, got {findings:?}" + ); + assert!(reuse[0].location.contains("dash_a")); + assert!(reuse[0].location.contains("dash_b")); + } + + #[test] + fn descendant_of_a_shared_root_keeps_every_root_breadcrumb() { + let inner = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + let outer = agg(vec![2], AggIntent::Sum { col: None }, inner); + let findings = explain_replacements(vec![("dash_a", outer.clone()), ("dash_b", outer)]); + let inner_sketch = findings + .iter() + .find(|f| { + f.kind == ExplanationKind::SketchApproximation && f.location.contains("child") + }) + .expect("expected the nested sketch explanation"); + assert!(inner_sketch.location.contains("dash_a")); + assert!(inner_sketch.location.contains("dash_b")); + } + + #[test] + fn distinct_queries_report_no_reuse_finding() { + let a = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let b = agg( + vec![2], + AggIntent::Sum { col: None }, + metric_scan(&["route"]), + ); + let findings = explain_replacements(vec![("dash_a", a), ("dash_b", b)]); + assert!( + findings + .iter() + .all(|f| f.kind != ExplanationKind::CommonSubexpressionReuse), + "structurally different queries must not report reuse, got {findings:?}" + ); + } + + #[test] + fn ungrouped_identical_aggregates_are_not_shareable_so_no_finding() { + // Empty `by`: no provable unique key — share_common_subtrees never + // hoists these, so consumer_count stays 1 for each and this module + // must not report a finding either. + let a = agg(vec![], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let b = agg(vec![], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let findings = explain_replacements(vec![("a", a), ("b", b)]); + assert!(findings + .iter() + .all(|f| f.kind != ExplanationKind::CommonSubexpressionReuse)); + } + + #[test] + fn single_query_repeated_subexpression_is_a_reuse_finding() { + // The same shared branch appearing twice within one query (an `a/a` + // shape) — single-query CSE. + let branch = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let q = QueryExpr::BinaryOp { + op: asap_types::pre_asap::query_expr::BinaryOpKind::Arithmetic( + asap_types::pre_asap::expr_ir::ArithmeticOpKind::Div, + ), + lhs: Rc::new(branch.clone()), + rhs: Rc::new(branch), + vector_match: None, + }; + let findings = explain_replacements(vec![("ratio", q)]); + let reuse: Vec<_> = findings + .iter() + .filter(|f| f.kind == ExplanationKind::CommonSubexpressionReuse) + .collect(); + assert_eq!( + reuse.len(), + 1, + "expected one reuse finding, got {findings:?}" + ); + assert!(reuse[0].location.contains("lhs")); + assert!(reuse[0].location.contains("rhs")); + } + + /// A shared node nested three levels under two *different*, unshared + /// `Filter` parents (mirrors `crate::replacement::tests:: + /// nested_shared_subtree_below_an_unshared_parent_is_still_discovered`) + /// must still be exactly one finding — the maximal-`TargetSubDAG` + /// guarantee the module docs describe, now provided by + /// `crate::replacement`'s own target discovery rather than this module's + /// (deleted) traversal. + #[test] + fn a_deeply_shared_subtree_under_different_parents_is_reported_once() { + use asap_types::pre_asap::expr_ir::ScalarValue; + use asap_types::pre_asap::query_expr::Predicate; + + let shared = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let root_a = QueryExpr::Filter { + pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Int64(1)))), + child: Rc::new(shared.clone()), + }; + let root_b = QueryExpr::Filter { + pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Int64(2)))), + child: Rc::new(shared), + }; + let findings = explain_replacements(vec![("a", root_a), ("b", root_b)]); + let reuse: Vec<_> = findings + .iter() + .filter(|f| f.kind == ExplanationKind::CommonSubexpressionReuse) + .collect(); + assert_eq!( + reuse.len(), + 1, + "a node shared under two different parents must be one finding, got {findings:?}" + ); + assert!(reuse[0].location.contains('a')); + assert!(reuse[0].location.contains('b')); + } + + // ── Custom strategy set / cost model plumbing ─────────────────────── + + struct AlwaysDDSketch; + impl crate::cost_model::CostModel for AlwaysDDSketch { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[asap_types::post_asap::SketchAlgorithm], + ) -> Vec { + let mut v = candidates.to_vec(); + if let Some(pos) = v + .iter() + .position(|k| *k == asap_types::post_asap::SketchAlgorithm::DDSketch) + { + let dd = v.remove(pos); + v.insert(0, dd); + } + v + } + } + + #[test] + fn custom_cost_model_changes_the_reported_sketch_kind() { + let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + let custom_model = AlwaysDDSketch; + let strategies: Vec> = vec![Box::new( + crate::replacement::SketchAlgorithmStrategy::new(&custom_model), + )]; + let findings = explain_replacements_with(vec![("q", q)], &strategies); + assert_eq!(findings.len(), 1); + assert!(findings[0].reason.to_lowercase().contains("ddsketch")); + } +} diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index b428c2ed..ac4ce3eb 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -8,12 +8,11 @@ //! **Common sub-expression elimination (CSE) is not this crate's job.** //! Detection is a primary pass over the pre-ASAP `QueryExpr` IR itself //! (`asap_types::pre_asap`, design tracked in issue #223), run before a -//! tree ever reaches [`bind::implement_tree`] — see issue #222 for why -//! (batch query optimization needs to see shared work across a +//! tree ever reaches [`replacement::SketchAlgorithmStrategy`] — see issue #222 +//! for why (batch query optimization needs to see shared work across a //! `QueryWorkload` before summary binding, not after). This crate may //! eventually run a second, narrower CSE pass of its own over an -//! already-[`implement_tree`](bind::implement_tree)'d -//! `SummaryExpr`/`SummaryNode` DAG, recognizing sharing that's invisible +//! already-bound `SummaryExpr`/`SummaryNode` DAG, recognizing sharing that's invisible //! at the pre-ASAP level by construction — e.g. `Quantile(x, 0.99)` and //! `Quantile(x, 0.95)` are structurally distinct `AggIntent`s but can //! still share one built sketch, read out twice. That post-ASAP pass is @@ -30,23 +29,74 @@ //! //! ## Status //! -//! Two real occupants and one stub: +//! This crate has two replacement/search capabilities — and deliberately no +//! third one that commits to a single, final, physically-materialized +//! answer for a whole workload: +//! +//! - [`replacement::ReplacementStrategy`] — per-target, never prunes, +//! exhaustive. The `TargetSubDAG`/`ReplacementSubDAG`/ +//! `ReplacementStrategy` vocabulary `docs/design_docs/asap_aware_mapping.md` stubs out +//! under "Key concepts (not yet implemented)", implemented for real (issue +//! #251, part of #33). [`replacement::SketchAlgorithmStrategy::replacements`] +//! both *decides* what an `AggIntent` may become +//! ([`replacement::implementations_for_with`], exhaustive and ranked via a +//! `CostModel`, sized to the `AccuracyTarget`) and *constructs* each +//! candidate's bound [`SummaryNode`](asap_types::post_asap::SummaryNode) — +//! every candidate comes back, not just one. +//! [`replacement::SharedSubtreeStrategy`] does the analogous job for the +//! build-independently-vs-build-once-and-share choice at a CSE-detected +//! shared subtree. +//! - [`replacement::search_workload`]/[`replacement::PlanSpace::cost_sorted`] +//! — workload-wide, every candidate + cost, never materializes one +//! physical answer. Merged into the same module (issue #252, part of +//! #33): [`replacement::search_workload`]/[`replacement::search_workload_with`] +//! *search* — discover every candidate `TargetSubDAG` across a whole +//! workload (not just one target in isolation) and run every registered +//! strategy against each one, to a fixpoint, without ever materializing a +//! flat `2^N`-sized candidate-plan list: [`replacement::PlanSpace`] holds +//! one Cascades-style [`replacement::MemoGroup`] per distinct +//! `TargetSubDAG`, each carrying every alternative discovered for it. +//! [`replacement::PlanSpace::cost_sorted`] is the final +//! `sorted_by(cost_model)` step, ranking each group's candidates +//! best-first via the same [`CostModel`](cost_model::CostModel) the +//! single-target steps above already consult — see [`replacement`]'s own +//! module docs for the full design (MEMO groups vs. flat plans, dedup +//! discipline, termination, cost-based ranking). +//! +//! **Picking *which* candidate, and materializing one final answer, is a +//! downstream deployment's job, out of this crate's scope.** This crate's +//! output boundary is [`replacement::PlanSpace`]: every candidate +//! replacement plus its cost, meant for a downstream consumer (e.g. a +//! DAG-visualization view, or a deployment's own physical binder). Which +//! sketch to commit to *and* where to place it are a joint decision only a +//! deployment can see the full picture for — picking one in isolation, with +//! no real consumer of that single materialized answer inside this crate, +//! is out of scope. (A prior workload-wide "keep first/cost-preferred +//! candidate per node, memoized by `Rc` identity" entry point — +//! `bind::implement_workload`/`implement_workload_with` — used to live here +//! and was removed for exactly this reason; see the terminology table below +//! for where that "one answer" step now belongs, downstream.) //! -//! - [`boundary`] — the per-intent sketch-vs-exact (accuracy) decision: -//! `AggIntent → Implementation` (a summary family's own `(Kind, Params)`, -//! or an exact accumulator) sized to the `AccuracyTarget` (issue #98). -//! [`boundary::implementation_for`] is the per-node decision; -//! [`bind::implement_tree`] drives it over a whole tree, and -//! [`bind::implement_workload`] drives it over a whole workload's roots — -//! memoized on `Rc` identity so two roots that -//! `asap_types::pre_asap::cse::share_common_subtrees` already collapsed -//! onto one shared subtree bind to one shared `SummaryNode` too (issue -//! #212, #222, #223) — see the terminology section below for why these are -//! named around "implementation" rather than "bind". //! - [`cost_model`] — the [`CostModel`](cost_model::CostModel) trait every //! deployment's cost-based sketch selection plugs into (issues #6, #33). //! `asap-plan` itself only ships [`DefaultCostModel`](cost_model::DefaultCostModel), -//! which preserves [`boundary`]'s built-in static preference order. +//! which preserves [`replacement`]'s built-in static preference order and +//! — via [`CostModel::estimate_cost`](cost_model::CostModel::estimate_cost) +//! — exposes an actual numeric cost per candidate, not just a relative +//! rank, for a caller (e.g. a DAG-visualization view) that wants to show +//! "candidate A costs ≈ X" next to "candidate B costs ≈ Y". +//! - [`explanation`] — this crate's explanation of a replacement: a +//! reporting *view* over [`replacement`]'s candidate-plan space (issue +//! #257, part of #33) that translates every discovered `TargetSubDAG` with +//! a non-trivial candidate list into an +//! [`explanation::ReplacementExplanation`] (why a replacement exists, +//! where, reusing the candidate's own rationale rather than inventing new +//! prose), meant for the same downstream consumer (e.g. a +//! DAG-visualization view) the crate doc's `## Status` section above +//! already names for [`replacement::PlanSpace`] itself. Superseded PR +//! #247's own rule-based traversal, which re-walked the tree once per +//! optimization before [`replacement::search_workload`] existed to read +//! from instead — see that module's docs for the full reframing. //! //! ## Terminology — "bind" already means three different things nearby; //! this crate's own logical→physical step is named "implementation" instead @@ -54,29 +104,30 @@ //! `asap-plan` and its downstream consumers (e.g. `ASAPQuery-backend`'s //! `control_plane`) independently reused the word "bind" for three //! *different*, layer-specific meanings — none of which is what this -//! crate's [`boundary`]/[`bind`] modules do. To avoid becoming a fourth, -//! colliding sense of the same word, this crate names its own logical -//! intent → physical realization step after the term the query-optimization -//! literature already uses for exactly that step: **implementation** -//! (Cascades/Volcano's "implementation rule", logical → physical, as -//! distinct from a *transformation rule*, logical → logical — see Graefe, -//! *The Cascades Framework for Query Optimization*): +//! crate's own [`replacement`] module does. To avoid becoming a +//! fourth, colliding sense of the same word, this crate names its own +//! logical intent → physical realization step after the term the +//! query-optimization literature already uses for exactly that step: +//! **implementation** (Cascades/Volcano's "implementation rule", logical → +//! physical, as distinct from a *transformation rule*, logical → logical — +//! see Graefe, *The Cascades Framework for Query Optimization*). //! //! | Term | Stage | Meaning | Lives in | //! |---|---|---|---| //! | **Parse** | parse | text (PromQL/SQL) → AST | `asap-frontend-promql` / `asap-frontend-sql` | //! | **Bind #1** | name resolution | `ColumnRef` (a name) → `ColumnId` (a concrete schema column) — the classic RDBMS "Parse → **Bind** → Optimize" pipeline sense (e.g. SQL Server's query-processor terminology) | [`asap_types::pre_asap::binder::Binder`](https://docs.rs/asap-types) | -//! | **Implementation** — [`boundary::implementation_for`] | pre-ASAP → post-ASAP, *one node* | choosing a concrete physical realization (a sketch family, an exact accumulator, or pass-through) for one [`AggIntent`](asap_types::pre_asap::agg_intent::AggIntent) | [`boundary`] | -//! | **`implement_tree`** — [`bind::implement_tree`] | pre-ASAP → post-ASAP, *whole tree* | walk a whole `QueryExpr` tree, calling [`boundary::implementation_for`] per node, and emit the complete post-ASAP [`SummaryExpr`](asap_types::post_asap::SummaryExpr)/`SummaryNode` DAG — named after "implementation" too rather than reusing "bind" a second time | [`bind`] | -//! | **Bind #2** (downstream, not in this crate) | post-ASAP → deployment placement | a *deployment's* own physical binder, additionally deciding **placement** (edge vs. backend, wire format, …) — a genuinely different, deployment-specific decision this crate doesn't model at all | e.g. `control_plane::sketch_algebra::rules::bind_*` (as of this writing; expected to fold into that deployment's cost-model layer rather than stay a separate "bind" concept) | +//! | **Implementation** — `replacement::implementations_for_with` | pre-ASAP → post-ASAP, *one node* | enumerating every concrete physical realization (a sketch family, an exact accumulator, or pass-through) for one [`AggIntent`](asap_types::pre_asap::agg_intent::AggIntent) | [`replacement`] | +//! | **Replacement** — [`replacement::SketchAlgorithmStrategy::replacements`] | pre-ASAP → post-ASAP, *one target, every candidate* | wrap each `implementations_for_with` candidate into its own bound [`SummaryNode`](asap_types::post_asap::SummaryNode), ranked — a caller wanting one answer takes the first entry itself | [`replacement`] | +//! | **Search** — [`replacement::search_workload`]/[`replacement::search_workload_with`] | pre-ASAP → post-ASAP, *whole workload, every candidate* | a Cascades/Volcano-style MEMO search: discover every candidate `TargetSubDAG` across a whole workload (not just one target in isolation), run every registered `ReplacementStrategy` against each to a fixpoint, and dedup into a [`replacement::PlanSpace`] — one [`replacement::MemoGroup`] per distinct `TargetSubDAG` holding every alternative discovered for it, never a flat `2^N`-sized list of whole candidate plans | [`replacement`] | +//! | **Bind #2** (downstream, not in this crate) | post-ASAP → deployment placement | a *deployment's* own physical binder, deciding **which** candidate to commit to *and* **placement** (edge vs. backend, wire format, …) for a whole workload — a genuinely different, deployment-specific decision this crate doesn't model at all (this is also where a prior workload-wide "keep first/cost-preferred candidate per node" step, `bind::implement_workload`/`implement_workload_with`, would belong if a deployment still wants that exact behavior — it isn't shipped by this crate) | e.g. `control_plane::sketch_algebra::rules::bind_*` (as of this writing; expected to fold into that deployment's cost-model layer rather than stay a separate "bind" concept) | //! //! A related question (tracked alongside issues #6/#33): whether this //! crate should also own a **matching** predicate — "does an already //! *available* `Implementation` satisfy a *required* one" — the way a //! database's materialized-view matching / "answering queries using //! views" layer does. It owns the *question*, not an *answer*: -//! [`boundary::Matcher`] is a trait with no default implementation and no -//! shipped instance, the same shape as [`cost_model::CostModel`] and for +//! [`replacement::Matcher`] is a trait with no default implementation and +//! no shipped instance, the same shape as [`cost_model::CostModel`] and for //! the same reason — which `Implementation`s are actually *available* //! anywhere is entirely a downstream deployment's concern (an inventory //! this crate has no way to see), and even the pure sketch-algebra @@ -87,15 +138,17 @@ //! `sketch_algebra::capability::Capability`/`is_satisfied_by` is the //! reference downstream implementation. -pub mod bind; -pub mod boundary; pub mod cost_model; +pub mod explanation; +pub mod replacement; -pub use bind::{ - implement_tree, implement_tree_with, implement_workload, implement_workload_with, - ImplementError, +pub use cost_model::{CostModel, DefaultCostModel}; +pub use explanation::{ + explain_replacements, explain_replacements_with, ExplanationKind, ReplacementExplanation, }; -pub use boundary::{ - implementation_for, implementation_for_with, summary_candidates, Implementation, Matcher, +pub use replacement::{ + default_strategies, default_strategies_with, search_workload, search_workload_with, + summary_candidates, ImplementError, Implementation, Matcher, MemoGroup, PlanSpace, RankedGroup, + Replacement, ReplacementStrategy, ReplacementSubDAG, SharedSubtreeStrategy, + SketchAlgorithmStrategy, TargetSubDAG, MAX_SEARCH_ITERATIONS, }; -pub use cost_model::{CostModel, DefaultCostModel}; diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs new file mode 100644 index 00000000..3632b39b --- /dev/null +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -0,0 +1,3973 @@ +//! `TargetSubDAG` / `ReplacementSubDAG` / `ReplacementStrategy` — the +//! candidate-replacement vocabulary `docs/design_docs/asap_aware_mapping.md` stubs out +//! under "Key concepts (not yet implemented)", implemented for real (issue +//! #251, part of #33). +//! +//! ## One step, not two: `SketchAlgorithmStrategy::replacements()` decides *and* builds +//! +//! For a bindable `Aggregate`, `SketchAlgorithmStrategy::replacements()` is the +//! single place this crate both decides what an `AggIntent` may become and +//! turns each of those candidates into a real, executable +//! [`ReplacementSubDAG`]: +//! +//! 1. **Decide**: [`implementations_for_with`] enumerates every valid +//! [`Implementation`] for the target's intent — exhaustive, and ranked +//! most-preferred-first via a [`CostModel`] (candidate sketch family/kind, +//! already sized to the target's own accuracy target: `Implementation::Sketch`'s +//! `params` are the output of inverting that accuracy target through +//! `CostModel::size_params`, not a placeholder filled in later). +//! 2. **Build**: for each candidate in that list, [`construct_summary`] +//! mechanically turns the already-decided `(kind, params)` into a real +//! [`SummaryNode`] — derives the child schema, resolves the summarized +//! column, builds the readout query, recurses into the child (via +//! [`realize_child`], so a nested aggregate gets its own +//! independent enumeration, never the outer target's forced choice), and +//! assembles the `SummaryAgg`/`SummaryEstimate` node. +//! +//! There is no separate decision step and construction step living in +//! different modules bridged by a named "given an `Implementation`, bind it" +//! function — step 2 is *not* a second decision (nothing about which +//! candidate to prefer happens there), it is mechanical construction that +//! has to run regardless of how `(kind, params)` were chosen, so it lives +//! directly inside the one method that needs it. +//! +//! - [`TargetSubDAG`] — a reference to a pre-ASAP [`QueryExpr`] node that is a +//! candidate for replacement, plus how many places in the workload already +//! reference it (its `consumer_count`) — the one piece of cross-node +//! context [`SharedSubtreeStrategy`] needs that a bare node reference alone +//! doesn't carry. +//! - [`ReplacementSubDAG`] — one candidate replacement for a `TargetSubDAG`: +//! either a fully bound [`SummaryNode`] or a pre-ASAP [`QueryExpr`] rewrite +//! (still logical, structurally different from the target but semantically +//! equivalent) — see [`Replacement`] — plus a human-readable `rationale`. +//! - [`ReplacementStrategy`] — `matches` + `replacements`, the same +//! extension-point shape [`CostModel`] and [`Matcher`] already use in this +//! crate: a new replacement source is a new `impl ReplacementStrategy`, not +//! a restructuring of this trait or of any existing strategy. `replacements` +//! is **exhaustive, not ranked, not filtered** — reporting "every valid +//! candidate" is core's job; picking the best one is left to the caller. +//! [`crate::explanation`] (issue #257) is this trait's own downstream +//! consumer, not a second extension point: it explains why a replacement +//! exists as a pure view over the candidates strategies registered here +//! already produced, rather than re-deriving that explanation with a rule +//! of its own. +//! +//! A caller that wants one executable answer takes the first +//! (`cost_model`-preferred) entry off `replacements()` itself +//! (`.into_iter().next()`) — that "keep the head" step lives entirely on the +//! calling side, not behind a second module-level entry point. This +//! module's own [`realize_child`] performs that take-first step +//! internally, but only for one, narrow, single-target purpose: recursing +//! into a node's child while constructing one concrete candidate (see +//! [`construct_summary_agg`]) and, symmetrically, recovering one +//! representative bound node for a [`CostModel::cse_share_decision`] +//! comparison (see [`realize_one`]) — never a workload-wide "commit to one +//! final answer" step. Committing to one physically-materialized answer for +//! a whole workload (previously `bind::implement_workload`/ +//! `implement_workload_with`) is out of this crate's scope — see the crate +//! doc's `## Status` section for why. Every other caller goes through +//! `SketchAlgorithmStrategy::replacements` directly and decides for itself. +//! +//! This means an ordinary single-target bind sizes and fully constructs +//! *every* sketch candidate at every sketch-capable node (not just the one a +//! caller keeps) — a deliberate tradeoff, made so there is exactly one place +//! in this crate that decides what an `AggIntent` may become, at the cost of +//! extra work per bind proportional to each node's own candidate count. +//! +//! ## The two strategies, and why these two +//! +//! - [`SketchAlgorithmStrategy`] wraps [`implementations_for_with`]'s exhaustive, +//! ranked list directly: for the same bindable-`Aggregate` shape this crate +//! binds (single intent, no `HAVING`), every entry becomes its own bound +//! candidate. +//! - [`SharedSubtreeStrategy`] wraps +//! `asap_types::pre_asap::cse::share_common_subtrees`'s sharing decision. +//! Wherever a [`TargetSubDAG`] already has two or more consumers (i.e. +//! `share_common_subtrees` already collapsed two or more workload +//! locations onto the same `Rc` — [`discover_targets`] below +//! does the identical workload-wide discovery for [`search_workload_with`]; +//! this module's own tests reuse the same dedup logic to build realistic +//! fixtures), it reports the two-way candidate CSE's own detection pass +//! deliberately declines to pick between on its own: build once and share +//! the already-interned subtree, or build it independently at each +//! consumer. [`crate::cost_model::CostModel::cse_share_decision`] is where +//! that choice actually gets made *today* (a fixed comparison, not a +//! search) — this strategy exposes the same two-way choice as an explicit, +//! inspectable pair of candidates instead of a cost model's already-decided +//! boolean. +//! +//! ## Non-goals (tracked separately, not attempted here) +//! +//! - **[`implementations_for_with`]'s own outward-facing behavior is +//! unchanged.** Same inputs still produce the same exhaustive, ranked +//! list — only its home moved (from a separate `implementation` module +//! into this one) and its own visibility dropped to module-private, since +//! [`SketchAlgorithmStrategy`] is now its only caller. +//! +//! ## Workload-wide search — merged in from the former `search.rs` (issue #252, part of #33) +//! +//! This section used to carry two more "non-goals" bullets here — "no +//! search/selection-across-a-whole-plan logic" and "no workload-wide +//! `TargetSubDAG` discovery pass" — describing work deliberately left for a +//! future Cascades/Volcano-style search engine (PR #263, +//! `feat/cascades-search-252`, over the [`ReplacementStrategy`] extension +//! point above). That engine is [`PlanSpace`]/[`MemoGroup`]/ +//! [`search_workload`]/[`search_workload_with`] below, merged into this +//! module rather than kept as a separate `search` module — the same "one +//! module, one step" reasoning the top of this file already uses for +//! decide-and-build: searching *across* a whole workload's worth of +//! [`TargetSubDAG`]s is a natural continuation of deciding and building +//! replacements *for* one, not a different concern that deserves its own +//! file. What follows (through "Cost-based final selection" below) is that +//! engine's own design documentation, preserved from `search.rs`. +//! +//! ### The pseudocode, and the two things it deliberately leaves open +//! +//! ```text +//! candidate_plans = { input_workload_plan } +//! loop: +//! new_plans = {} +//! for plan in candidate_plans: +//! for site in plan.bindable_sites(): +//! for strategy in registered_strategies: +//! if strategy.matches(site): +//! for replacement in strategy.replacements(site): +//! new_plans += substitute(plan, site, replacement) +//! new_plans -= candidate_plans +//! candidate_plans += new_plans +//! until new_plans is empty +//! return candidate_plans.sorted_by(cost_model) +//! ``` +//! +//! Read literally, this enumerates whole *plans* — full copies of the +//! workload's tree, one per combination of per-target choices. A workload +//! with `N` independently-choosable targets would produce up to `2^N` flat +//! plans, each one duplicating every untouched sibling subtree. This module +//! does not do that: +//! +//! 1. **MEMO groups, not flat plans.** [`MemoGroup`] is this engine's +//! Cascades-style "group": one distinct [`TargetSubDAG`] (identified by +//! its own `Rc` pointer identity — the same currency +//! [`asap_types::pre_asap::cse::share_common_subtrees`] already +//! established across the workload) holding every +//! [`ReplacementSubDAG`] alternative discovered for it. [`PlanSpace`] is +//! a collection of these groups, keyed by `TargetSubDAG` — a candidate +//! "plan" is never materialized as a distinct top-level `Rc` +//! at all; two logically-different overall choices at two different +//! targets are just two different entries in two different groups, +//! sharing every other node in the workload by construction (they *are* +//! the same `Rc`s — nothing was copied to make a second "plan"). +//! 2. **Dedup by structural hash + `PartialEq`, reusing `pre_asap::cse`'s own +//! discipline.** [`asap_types::pre_asap::cse::structural_hash`] (made +//! `pub` for exactly this reuse) is only ever a candidate-narrowing +//! filter; [`MemoGroup::add_candidate`]'s actual duplicate check is +//! `QueryExpr`'s derived `PartialEq` — the same "hash is a filter, +//! `PartialEq` is the decision, no exceptions" rule `cse.rs`'s own +//! "Correctness" section states and this module inherits rather than +//! reinvents. See [`is_duplicate_rewrite`] for the one deliberate +//! wrinkle this reuse needs (a `Rc`-identity case pure value equality +//! would get wrong). +//! +//! ### Where `TargetSubDAG` discovery comes from +//! +//! [`discover_targets`] is the workload-wide `TargetSubDAG` discovery pass +//! this section used to flag as explicitly *not* implemented ("no +//! workload-wide `TargetSubDAG` discovery pass is shipped either... wiring +//! it up automatically belongs to the same future search engine, not this +//! issue") — this is that future engine, so it's this module's job now, and +//! it's what the quoted pseudocode's `for site in plan.bindable_sites()` +//! line above stands for: every `TargetSubDAG` this pass discovers is one +//! iteration of that loop. It walks every workload root's whole DAG (the +//! same **relational-skeleton** operator-child scope +//! `asap_types::pre_asap::cse::share_common_subtrees` itself uses — see +//! that module's "Algorithm" section), discovering one `TargetSubDAG` per +//! distinct `Rc` and a *real* `consumer_count`: how many operator-child +//! positions anywhere in the workload reference that exact `Rc`, not just +//! how many of the workload's own top-level roots happen to be it — a +//! `SharedSubtreeStrategy` candidate three levels under an unshared +//! `Filter` is exactly as real a target as a shared whole root, so this +//! module's discovery can't stop at the top level. +//! +//! `discover_targets` duplicates (rather than reuses) this module's own +//! `#[cfg(test)]`-only `count_consumers` traversal (in the test module +//! below), which mirrors this exact shape for this module's own test +//! fixtures — that copy is intentionally test-only, so it isn't reachable +//! from this module's production code without either moving it into +//! shared, non-test-gated code or duplicating the (small, self-contained) +//! traversal here. Duplicating was judged simpler than restructuring a test +//! helper into shared production code for one caller. +//! +//! ### Termination +//! +//! Every discovered target is asked *once* per registered strategy, never +//! re-asked — [`search_workload_with`]'s loop processes each round's +//! frontier of not-yet-visited targets exactly one time each, so there is +//! no scenario where the same `(target, strategy)` pair is queried twice +//! (the `new_plans -= candidate_plans` dedup step the module-level +//! pseudocode describes is therefore never asked to recognize "the same +//! candidate, proposed again" as a special case — see +//! [`MemoGroup::add_candidate`]'s own doc on why that distinction matters +//! for [`Replacement::Summary`] specifically, where no real equality check +//! exists to make it safely). +//! +//! What *can* grow the frontier is a candidate's own reachable structure: +//! after a target is processed, every [`Replacement::Rewrite`] candidate's +//! **children** (never the candidate's own top-level node — that value is +//! an alternative *for* the target just processed, not a new target of its +//! own; see [`discover_new_descendant_targets`]) are scanned for pointers +//! not already known, and any found become next round's frontier. Both shipped +//! strategies are idempotent in exactly this sense: [`SketchAlgorithmStrategy`] +//! produces terminal [`Replacement::Summary`] candidates (no `QueryExpr` +//! children to scan at all), and [`SharedSubtreeStrategy`]'s two +//! [`Replacement::Rewrite`] candidates both reuse the target's own +//! already-known child `Rc`s verbatim (`Rc::clone`/a shallow top-level +//! `.clone()` — see that strategy's own doc). So for both, the frontier is +//! always empty after round one: real workloads converge in exactly one +//! round, regardless of size. +//! +//! That said, a future strategy whose `Replacement::Rewrite` candidates +//! invent brand-new descendant structure every time they're computed (e.g. +//! internal state that fabricates a fresh child node on every call) could +//! in principle keep the frontier non-empty forever. Since this crate has +//! no cardinality/statistics estimation to bound anything by, +//! [`search_workload_with`] enforces a generous, documented round cap +//! ([`MAX_SEARCH_ITERATIONS`]) instead: exceeding it panics with a clear +//! message naming the actual cause, rather than hanging silently — a test +//! ([`tests::a_pathologically_growing_strategy_trips_the_iteration_cap`]) +//! pins that this guard actually fires, by using exactly that shape of +//! pathological strategy. +//! +//! ### Cost-based final selection — reusing `CostModel`, not a second interface +//! +//! [`PlanSpace::cost_sorted`] is the `sorted_by(cost_model)` step, and it +//! reuses this crate's existing [`CostModel`] trait rather than inventing a +//! second cost interface (`docs/design_docs/cse-cost-model-decision.md`, +//! issue #237, explicitly reasoned about *why* a narrow, direct cost +//! comparison was enough for the CSE share/recompute decision alone, and +//! flagged that a real search engine — this module — is where that stops +//! being the whole story; it isn't a contradiction of #237, it's the scope +//! change #237 itself named). Concretely, per [`MemoGroup`]: +//! +//! - A group whose candidates are the [`SharedSubtreeStrategy`] +//! share-vs-recompute pair is ranked by calling +//! [`CostModel::cse_share_decision`] via this module's own +//! [`cse_preference`] — rather than re-deriving a competing comparison. +//! - A group whose candidates are [`SketchAlgorithmStrategy`]'s sketch-family +//! candidates is ranked via [`CostModel::rank_candidates`] (the same hook +//! `implementations_for_with` itself consults), applied to the +//! candidates' own [`SketchAlgorithm`]s. +//! - Any other shape (a single candidate, or a mix this module doesn't have +//! a defined comparison for) keeps discovery order — there is nothing to +//! rank, or no [`CostModel`] hook this module knows how to apply; it never +//! invents a comparison `CostModel` doesn't already define. + +use std::collections::HashMap; + +use asap_types::post_asap::{ + ExactKind, ExactParams, SamplingKind, SamplingParams, SketchAlgorithm, SketchKind, + SketchParams, SketchQuery as PostAsapSketchQuery, StatModelKind, StatModelParams, SummaryExpr, + SummaryFamilyType, SummaryField, SummaryNode, SummarySchema, WaveletKind, WaveletParams, +}; +use asap_types::pre_asap::agg_intent::{agg_is_mergeable, AggIntent}; +use asap_types::pre_asap::cse::{share_common_subtrees, structural_hash, HashCache}; +use asap_types::pre_asap::expr_ir::ColumnRef; +use asap_types::pre_asap::query_expr::{QueryExpr, QueryExprError, Reduction}; +use asap_types::pre_asap::schema::Schema; +use asap_types::types::AccuracyTarget; +use std::rc::Rc; +use thiserror::Error; + +use crate::cost_model::{CostModel, CseCandidate, DefaultCostModel, ShareDecision}; + +/// Errors from the pre-ASAP → post-ASAP replacement/construction path +/// ([`realize_child`] and [`keep_pre_asap`]). Moved here from the former +/// `bind.rs` (issue #251): this is what a [`ReplacementStrategy`] +/// implementor's own construction path can realistically fail with — +/// schema derivation over a pre-ASAP [`QueryExpr`] — not something specific +/// to workload-wide orchestration. +#[derive(Debug, Error)] +pub enum ImplementError { + /// Schema derivation failed while lifting an edge to `SummarySchema`. + #[error("schema derivation failed during pre-ASAP → post-ASAP binding: {0}")] + Schema(#[from] QueryExprError), +} + +/// A pre-ASAP sub-DAG a [`ReplacementStrategy`] knows how to replace. +/// +/// `root` is a reference into the workload's own [`QueryExpr`] tree (an +/// `Rc`, the same currency [`search_workload`] and +/// `asap_types::pre_asap::cse::share_common_subtrees` already thread through +/// this crate's public API — not a bare `&QueryExpr` — so a strategy that +/// needs the node's own `Rc` identity, not just its shape, has it available +/// without the caller re-deriving it). +/// +/// `consumer_count` is how many locations across the workload reference this +/// exact `Rc` — 1 for an ordinary single-use node and 2+ for a shared subtree. +/// [`search_workload_with`] computes the workload-wide value during target +/// discovery. [`TargetSubDAG::new`] defaults it to `1` for callers invoking a +/// strategy against one node in isolation. A strategy that only cares about +/// `root`'s shape (for example, [`SketchAlgorithmStrategy`]) can ignore the +/// count; [`SharedSubtreeStrategy`] consults it directly. +#[derive(Debug, Clone, Copy)] +pub struct TargetSubDAG<'a> { + pub root: &'a Rc, + pub consumer_count: usize, +} + +impl<'a> TargetSubDAG<'a> { + /// A target assumed to have exactly one consumer — the common case for a + /// caller that isn't already tracking cross-workload sharing. + pub fn new(root: &'a Rc) -> Self { + Self { + root, + consumer_count: 1, + } + } + + /// A target with an explicit `consumer_count`, used by workload discovery + /// and by callers that already know how many locations reference `root`. + pub fn with_consumer_count(root: &'a Rc, consumer_count: usize) -> Self { + Self { + root, + consumer_count, + } + } +} + +/// What a [`ReplacementSubDAG`] actually substitutes a [`TargetSubDAG`] with. +/// +/// Generalizes [`implementations_for_with`]'s two possible *kinds* of answer +/// — a post-ASAP binding decision, or a still-pre-ASAP structural alternative +/// — into "one candidate among several", each with its own +/// [`ReplacementSubDAG`]. +#[derive(Debug, Clone)] +pub enum Replacement { + /// A fully bound post-ASAP summary decision, for one particular + /// candidate realization of the target. + Summary(Rc), + /// A pre-ASAP rewrite: still a logical [`QueryExpr`], structurally + /// different from the target's own `root` (e.g. sharing vs. not sharing + /// a subtree) but semantically equivalent to it. + Rewrite(Rc), +} + +/// One candidate replacement for a [`TargetSubDAG`], plus a human-readable +/// `rationale` explaining why it's a valid candidate (meant for a +/// report/log/debugging a search engine's choices, not machine parsing — +/// [`crate::explanation::ReplacementExplanation::reason`] literally reuses +/// this same string rather than inventing new prose of its own. +#[derive(Debug, Clone)] +pub struct ReplacementSubDAG { + pub replacement: Replacement, + pub rationale: String, +} + +/// A replacement strategy: given a [`TargetSubDAG`], does this strategy have +/// an opinion on it at all (`matches`), and if so, every semantically valid +/// replacement (`replacements`)? +/// +/// The extension point this module exists for — the same shape +/// [`CostModel`] and [`Matcher`] already use elsewhere in this crate: a new +/// replacement source is a new `impl ReplacementStrategy`, no restructuring +/// of this trait or any existing strategy required. +/// +/// `replacements` is only meaningful when `matches` would return `true` for +/// the same target; both [`SketchAlgorithmStrategy`] and [`SharedSubtreeStrategy`] +/// return an empty `Vec` rather than panicking when called on a target they +/// don't match, so a caller that skips the `matches` check first still gets a +/// safe (merely uninformative) answer instead of a crash. +pub trait ReplacementStrategy { + /// Does this strategy have any replacement to offer for `target`? + fn matches(&self, target: &TargetSubDAG<'_>) -> bool; + + /// Every valid replacement for `target` — not ranked, not filtered. + /// Reporting "every valid candidate" is this method's whole job; picking + /// the best one is a [`CostModel`]'s job, out of scope here. + fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec; +} + +// ── Implementation: how one AggIntent may be realised ─────────────────────── + +/// How an [`AggIntent`] may be realised at post-ASAP binding time (issue +/// #98): by an approximate summary (sketch, sample, wavelet, statistical +/// model, …), by an exact mergeable accumulator, or by an ordinary exact +/// operator (pass-through). This is a post-ASAP concern — the pre-ASAP IR +/// carries only the intent + accuracy target, never the realization — and +/// it's a per-node decision, made once per `AggIntent`, not a plan-wide one. +/// +/// [`implementations_for_with`] is where every valid realization gets +/// enumerated, exhaustive and ranked (most-preferred first) — this crate has +/// no separate function that computes just "the one" `Implementation` +/// independently of that list. [`SketchAlgorithmStrategy`] is the sole +/// consumer: it wraps every entry of this list into its own bound +/// [`SummaryNode`] and returns all of them, ranked — a caller wanting a +/// single answer keeps the first one itself (see the module docs above). +#[derive(Debug, Clone, PartialEq)] +pub enum Implementation { + /// An exact **mergeable** accumulator (partial state ≡ the value + /// itself: `Sum` / `Count` / `MinMax` / `Rate` / `Increase`). The + /// built state *is* the answer already — no `SummaryEstimate` readout + /// step. + ExactAggregate { + kind: ExactKind, + params: ExactParams, + }, + /// An approximate sketch sized to the intent's [`AccuracyTarget`]. + /// Needs a `SummaryEstimate` readout to recover a value. Already + /// classified into its [`SketchKind`] category (`SketchKind::new` + /// having been called) — construction always goes through that + /// classifier, never this variant directly. + Sketch(SketchKind), + /// A sampling-based summary (a retained row subset). Needs a + /// `SummaryEstimate` readout. Not chosen by any core `AggIntent` + /// dispatch today — see the module docs. + Sample { + kind: SamplingKind, + params: SamplingParams, + }, + /// A wavelet-transform summary. Needs a `SummaryEstimate` readout. Not + /// chosen by any core `AggIntent` dispatch today — see the module docs. + Wavelet { + kind: WaveletKind, + params: WaveletParams, + }, + /// A fitted statistical/parametric-model summary. Needs a + /// `SummaryEstimate` readout. Not chosen by any core `AggIntent` + /// dispatch today — see the module docs. + StatModel { + kind: StatModelKind, + params: StatModelParams, + }, + /// No summary form — the node stays a logical pre-ASAP operator and is + /// executed exactly (per-series transforms, non-mergeable reducers, exact + /// quantile/top-k/cardinality, classic-bucket `HistogramQuantile`, …). + PassThrough, +} + +/// Does an already-**available** [`Implementation`] — e.g. a summary +/// instance a downstream deployment already materialized somewhere, found +/// via whatever inventory/index that deployment keeps — satisfy a +/// **required** [`Implementation`] (one of the candidates +/// [`implementations_for_with`] produced for some [`AggIntent`])? +/// +/// This is the query-optimization-literature "materialized view matching" +/// / "answering queries using views" question, narrowed to this crate's +/// summary vocabulary: not "can I build this from scratch" (that's what +/// [`implementations_for_with`] answers) but "does something that already +/// exists answer this". +/// +/// `asap-plan` deliberately ships no implementation of this trait and no +/// default method body — unlike [`implementations_for_with`], which decision +/// an available `Implementation` satisfies a required one is not a fact this +/// crate can settle on its own. Two real, reasonable answers already +/// diverge outside this crate: +/// +/// - A **pure sketch-algebra** answer would say a `Sketch{kind: Kll, ..}` +/// requirement is satisfied by an available `DDSketch` (both quantile +/// sketches), and that a heap-bearing top-k sketch also answers a bare +/// frequency point-query (the heap is additional info on the same +/// underlying matrix) — but not the reverse. +/// - A **deployment with its own storage-layout rules** may need more: +/// e.g. whether a multi-population accumulator can serve a +/// single-population query via re-aggregation is a fact about that +/// deployment's storage layout, not about any summary family's kind at +/// all — a family's own kind doesn't encode grouping (grouping lives on +/// the post-ASAP node's `by` instead), so there is nothing in this +/// crate's own vocabulary to subsume. +/// +/// Implementations are expected to consult `required`/`available`'s +/// `kind` (and whatever grouping/placement context the deployment tracks +/// alongside `Implementation`, which this trait's signature doesn't carry +/// because this crate has no inventory concept to carry it in). +pub trait Matcher { + fn is_satisfied_by(&self, required: &Implementation, available: &Implementation) -> bool; +} + +/// Confidence δ assumed when the target carries only an ε +/// (`AccuracyTarget::Epsilon`): the (ε, δ)-parameterised sketches (CMS) need +/// one. `ln(1/0.01) → depth 5`, matching the conventional CMS sizing. +pub const DEFAULT_DELTA: f64 = 0.01; + +/// The sketch kinds that can serve an intent, most-preferred first. +/// This is the `AggIntent → SketchAlgorithm` map of issue #98; +/// [`implementations_for_with`] sizes and ranks every entry via `cost_model`. +/// Listed here so the candidate set has one home. +pub fn summary_candidates(intent: &AggIntent) -> &'static [SketchAlgorithm] { + match intent { + AggIntent::Quantile { .. } => &[SketchAlgorithm::Kll, SketchAlgorithm::DDSketch], + AggIntent::Cardinality { .. } => &[ + SketchAlgorithm::Hll, + SketchAlgorithm::Theta, + SketchAlgorithm::Kmv, + ], + // Count-Sketch-with-heap is CMS-with-heap's balanced/zero-mean-error + // alternative for the same heavy-hitter shape. + AggIntent::TopK { .. } => &[ + SketchAlgorithm::CmsWithHeap, + SketchAlgorithm::CountSketchWithHeap, + ], + AggIntent::Count { .. } => &[SketchAlgorithm::Cms, SketchAlgorithm::CountSketch], + _ => &[], + } +} + +/// The [`AccuracyTarget`] threaded onto an approximate-capable intent +/// (`Quantile`/`Cardinality`/`Count`/`TopK`), or `None` for every other +/// intent (no sketch candidate applies — [`implementations_for_with`]'s own +/// match routes those elsewhere). Exposed so callers resolve the exact same +/// accuracy target [`implementations_for_with`] does, without re-deriving it +/// from scratch. +pub fn accuracy_target(intent: &AggIntent) -> Option<&AccuracyTarget> { + match intent { + AggIntent::Quantile { accuracy, .. } + | AggIntent::Cardinality { accuracy, .. } + | AggIntent::Count { accuracy } + | AggIntent::TopK { accuracy, .. } => Some(accuracy), + _ => None, + } +} + +/// Every valid [`Implementation`] for `intent`, exhaustive and ranked +/// (most-preferred first via `cost_model`) — the *only* place this crate +/// decides what an `AggIntent` may become. Nothing in this crate computes +/// "the one" `Implementation` independently of this list: +/// [`SketchAlgorithmStrategy`] keeps every entry as a candidate, and a caller +/// that wants a single executable answer takes the head of *that* strategy's +/// output itself. +/// +/// Exhaustive over the [`AggIntent`] vocabulary — adding a variant without an +/// explicit realization is a compile error, and the coverage-matrix test pins +/// each variant's category. +/// +/// Module-private: [`SketchAlgorithmStrategy::replacements`] is the only +/// caller — a caller outside this module has no use for the bare +/// `Implementation` list on its own, only for the bound +/// [`ReplacementSubDAG`]s that strategy produces from it. +fn implementations_for_with(intent: &AggIntent, cost_model: &dyn CostModel) -> Vec { + match intent { + // ── Approximate-capable intents — the AccuracyTarget decides ──────── + AggIntent::Quantile { accuracy, .. } + | AggIntent::Cardinality { accuracy, .. } + | AggIntent::Count { accuracy } + | AggIntent::TopK { accuracy, .. } => match accuracy { + AccuracyTarget::Exact => vec![exact_realization(intent)], + _ => sketch_implementations(intent, accuracy, cost_model), + }, + + // ── Exact mergeable accumulators ───────────────────────────────────── + AggIntent::Sum { .. } => vec![exact_accumulator(intent, ExactKind::Sum, ExactParams::Sum)], + AggIntent::Min { .. } | AggIntent::Max { .. } => { + vec![exact_accumulator( + intent, + ExactKind::MinMax, + ExactParams::MinMax, + )] + } + AggIntent::Rate => vec![exact_accumulator( + intent, + ExactKind::Rate, + ExactParams::Rate, + )], + AggIntent::Increase => { + vec![exact_accumulator( + intent, + ExactKind::Increase, + ExactParams::Increase, + )] + } + + // ── Exact, non-mergeable reducers — richer partial state than a + // single value (see `agg_is_mergeable`), so no accumulator form. + AggIntent::Avg { .. } | AggIntent::StdDev { .. } | AggIntent::Variance { .. } => { + vec![Implementation::PassThrough] + } + + // ── Classic-bucket histogram_quantile (#79): exact `le`-bucket + // interpolation over pre-aggregated counts — NOT re-sketchable. + // (The native/raw form lowers to the generic `Quantile` above.) + AggIntent::HistogramQuantile { .. } => vec![Implementation::PassThrough], + + // ── Per-series transforms and reductions with no sketch realization: + // counter-derivatives (#44), math (#45), time/calendar (#46), + // presence (#47), native-histogram accessors (#43), and the + // `*OverTime` reducers (#51). All exact by construction. + AggIntent::Changes + | AggIntent::Delta + | AggIntent::IDelta + | AggIntent::Deriv + | AggIntent::Resets + | AggIntent::PredictLinear { .. } + | AggIntent::DoubleExpSmoothing { .. } + | AggIntent::HistogramCount + | AggIntent::HistogramSum + | AggIntent::HistogramAvg + | AggIntent::HistogramStdDev + | AggIntent::HistogramStdVar + | AggIntent::HistogramFraction { .. } + | AggIntent::Math(_) + | AggIntent::Absent + | AggIntent::AbsentOverTime + | AggIntent::PresentOverTime + | AggIntent::TimeFn(_) + | AggIntent::LastOverTime + | AggIntent::FirstOverTime + | AggIntent::MadOverTime + | AggIntent::TsOfMinOverTime + | AggIntent::TsOfMaxOverTime + | AggIntent::TsOfFirstOverTime + | AggIntent::TsOfLastOverTime => vec![Implementation::PassThrough], + + // ── Group / count_values (#49): exact per `agg_is_exact`, but their + // output is structural (constant-1 / a synthesized label column), + // not a value a summary accumulator carries. + AggIntent::Group | AggIntent::CountValues { .. } => vec![Implementation::PassThrough], + + // ── Extension (deployment-model-specific, issue #131) — core has no + // realization opinion for a shape it doesn't know, so it defers + // entirely to the `CostModel` (issue #150): `realize_extension` + // defaults to `PassThrough`, preserving today's behavior for + // every deployment that doesn't override it. Core has no way to + // enumerate alternatives for an opaque deployment-defined shape, + // so this is always exactly one candidate. This is also the only + // path that can currently produce `Implementation::Sample`/ + // `Wavelet`/`StatModel` — see the module docs. + AggIntent::Extension { ext_kind, payload } => { + vec![cost_model.realize_extension(ext_kind, payload)] + } + } +} + +/// Exact realization of an approximate-capable intent whose target is +/// `AccuracyTarget::Exact`. `Count` has a mergeable exact accumulator; exact +/// quantile / top-k / cardinality have no single-value summary form (they +/// need the full multiset / heap / set) and pass through. +fn exact_realization(intent: &AggIntent) -> Implementation { + match intent { + AggIntent::Count { .. } => exact_accumulator(intent, ExactKind::Count, ExactParams::Count), + _ => Implementation::PassThrough, + } +} + +fn exact_accumulator(intent: &AggIntent, kind: ExactKind, params: ExactParams) -> Implementation { + // An exact accumulator is only sound when partial states merge + // (`agg(A ∪ B) = combine(agg(A), agg(B))`). + debug_assert!( + agg_is_mergeable(intent), + "accumulator for non-mergeable {intent:?}" + ); + Implementation::ExactAggregate { kind, params } +} + +/// Resolve an [`AccuracyTarget`] into the `(eps, delta)` budget +/// [`CostModel::size_params`] needs. Shared by [`sketch_implementations`] and +/// this crate's own sizing — one place this resolution happens, so nothing +/// can drift apart on it. +/// +/// `Exact` is unreachable via [`implementations_for_with`] (which routes +/// `Exact` to [`exact_realization`] instead); degrades to the tightest +/// parameters for a caller that resolves it directly anyway. +pub fn accuracy_budget(accuracy: &AccuracyTarget) -> (f64, f64) { + match accuracy { + AccuracyTarget::Exact => (f64::MIN_POSITIVE, DEFAULT_DELTA), + AccuracyTarget::Epsilon(e) => (*e, DEFAULT_DELTA), + AccuracyTarget::EpsilonDelta { epsilon, delta } => (*epsilon, *delta), + } +} + +/// Every candidate sketch [`Implementation`] for an approximate-capable +/// intent, sized to `accuracy` and ranked via `cost_model.rank_candidates` +/// (most-preferred first) — [`implementations_for_with`]'s Sketch branch. +fn sketch_implementations( + intent: &AggIntent, + accuracy: &AccuracyTarget, + cost_model: &dyn CostModel, +) -> Vec { + let (eps, delta) = accuracy_budget(accuracy); + let ranked = crate::cost_model::validated_candidate_ranking( + cost_model, + intent, + summary_candidates(intent), + ); + ranked + .into_iter() + .map(|algorithm| { + let params = cost_model.size_params(algorithm.clone(), intent, eps, delta); + Implementation::Sketch(SketchKind::new(algorithm, params)) + }) + .collect() +} + +/// `asap-plan`'s built-in `SketchParams` sizing, keyed off the resolved +/// `(eps, delta)` accuracy budget. [`CostModel::size_params`]'s default +/// body — factored out to a free function so a deployment's own +/// `CostModel` impl can still delegate to it for the candidates it +/// doesn't want to resize itself. +/// +/// Each formula inverts the sketch family's standard error bound to the +/// smallest parameter satisfying the target, clamped to the family's sane +/// range. A non-positive ε saturates to the clamp maximum (tightest +/// allowed). +pub fn default_size_params( + kind: SketchAlgorithm, + intent: &AggIntent, + eps: f64, + delta: f64, +) -> SketchParams { + match kind { + SketchAlgorithm::Kll => SketchParams::Kll { k: kll_k(eps) }, + SketchAlgorithm::Cms => SketchParams::Cms { + width: cms_width(eps), + depth: cms_depth(delta), + }, + SketchAlgorithm::Hll => SketchParams::Hll { + precision: hll_precision(eps), + }, + SketchAlgorithm::CmsWithHeap => { + let k = match intent { + AggIntent::TopK { k, .. } => *k, + _ => unreachable!("CmsWithHeap is only a TopK candidate"), + }; + SketchParams::CmsWithHeap { + width: cms_width(eps), + depth: cms_depth(delta), + heap_size: k as u32, + } + } + // Non-preferred candidates (DDSketch / Theta / Kmv / CountSketch / + // CountSketchWithHeap) are only reachable once a cost model picks + // them; sized here so that wiring is local. + SketchAlgorithm::DDSketch => SketchParams::DDSketch { alpha: eps }, + SketchAlgorithm::Theta => SketchParams::Theta { k: kmv_k(eps) }, + SketchAlgorithm::Kmv => SketchParams::Kmv { k: kmv_k(eps) }, + // Count-Sketch is CMS's balanced/zero-mean-error alternative — + // same (width, depth) shape, sized the same way for now (a + // Count-Sketch-specific bound uses an L2-norm error guarantee + // rather than CMS's L1-norm one; this is a placeholder pending + // that refinement, same status as the other non-preferred + // candidates above). + SketchAlgorithm::CountSketch => SketchParams::CountSketch { + width: cms_width(eps), + depth: cms_depth(delta), + }, + SketchAlgorithm::CountSketchWithHeap => { + let k = match intent { + AggIntent::TopK { k, .. } => *k, + _ => unreachable!("CountSketchWithHeap is only a TopK candidate"), + }; + SketchParams::CountSketchWithHeap { + width: cms_width(eps), + depth: cms_depth(delta), + heap_size: k as u32, + } + } + } +} + +/// A deployment's explicit bet about how "typical" (non-adversarial) its +/// workload's collision pattern is expected to be, consumed only by +/// [`posterior_aware_size_params`]. +/// +/// This is **not** derived from Chen et al.'s posterior-error-estimation +/// technique (issue #239, `asap_types::post_asap::query_time::error_estimation`) +/// — that technique computes a tighter bound *at query time* from a +/// sketch's real counter values, and this repo has no sketch runtime yet +/// for a real counter array to size against (see that module's docs, and +/// `asap_types::post_asap::query_time`'s module doc for why it's a +/// deliberately separate folder from this crate's own *plan-time* code). +/// This struct is this crate's own *plan-time* analogue of the same +/// underlying intuition — an expected-case (skewed / non-adversarial) +/// workload needs a smaller sketch than the adversarial worst case — +/// expressed as an explicit, caller-supplied assumption rather than +/// anything observed or proven. Issue #250 tracks actually connecting the +/// two: feeding query-time-observed posterior error back into a future +/// replan's `width_relaxation` instead of a bare caller guess. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ExpectedCaseSizing { + /// Fraction, in `(0, 1]`, of the traditional worst-case width + /// ([`cms_width`]) the caller is betting is enough. `1.0` (or any + /// value outside `(0, 1)`) reproduces the worst-case width exactly — + /// no risk taken. A smaller value shrinks the sketch proportionally, + /// at the cost documented on [`posterior_aware_size_params`]. + pub width_relaxation: f64, +} + +/// Opt-in alternative to [`default_size_params`] for the CMS-family kinds +/// (`Cms` / `CmsWithHeap` / `CountSketch` / `CountSketchWithHeap`): sizes +/// width to `assumption.width_relaxation` of the worst-case [`cms_width`], +/// trading the unconditional worst-case `(ε,δ)` guarantee for a smaller +/// sketch under an explicit, caller-stated non-adversarial-workload bet — +/// see [`ExpectedCaseSizing`]. +/// +/// **The tradeoff, spelled out:** [`default_size_params`]'s width guarantees +/// `Pr[error > ε·|F|₁] < δ` for *any* input, including an adversarial one +/// built to maximize collisions (§3.3 of the posterior-error-estimation +/// paper this issue is about — see +/// `asap_types::post_asap::query_time::error_estimation`'s module docs). +/// Shrinking +/// width below that only keeps the same `(ε,δ)` guarantee if the real +/// workload's collision load stays within `width_relaxation` of the +/// worst-case assumption — this function does not check that, cannot check +/// it (no data exists at plan time), and does not change the formal +/// guarantee's statement; it only changes how much hardware is spent +/// chasing it. Callers accept that gap explicitly by choosing +/// `width_relaxation < 1.0`. +/// +/// Depth ([`cms_depth`]) is left unchanged from [`default_size_params`]: +/// depth trades away confidence *exponentially* (`Pr[all r rows bad] = +/// p^r` — each extra row multiplies the failure probability down), a +/// differently-shaped and materially riskier tradeoff than width's linear +/// relaxation. Issue #239 asks for *a* tighter-sizing option under a +/// stated assumption, not a full redesign of the depth/width tradeoff +/// space, so depth relaxation is left as explicit future scope. +/// +/// For every `SketchAlgorithm` outside the CMS family, this is identical to +/// [`default_size_params`] — `width_relaxation` only ever touches the +/// [`cms_width`]-sized formulas this issue is about. +/// +/// [`default_size_params`]'s own behavior is completely unchanged by this +/// function's existence — this is a separate, additive entry point, never +/// called from [`default_size_params`] or [`implementations_for_with`]. +pub fn posterior_aware_size_params( + kind: SketchAlgorithm, + intent: &AggIntent, + eps: f64, + delta: f64, + assumption: ExpectedCaseSizing, +) -> SketchParams { + let relaxed_width = |eps: f64| -> u32 { + let base = cms_width(eps); + let f = assumption.width_relaxation; + if !(f.is_finite() && f > 0.0 && f < 1.0) { + return base; // out-of-range bet: no relaxation, fall back to worst case + } + saturating_ceil(base as f64 * f, 2, base) + }; + match kind { + SketchAlgorithm::Cms => SketchParams::Cms { + width: relaxed_width(eps), + depth: cms_depth(delta), + }, + SketchAlgorithm::CmsWithHeap => { + let k = match intent { + AggIntent::TopK { k, .. } => *k, + _ => unreachable!("CmsWithHeap is only a TopK candidate"), + }; + SketchParams::CmsWithHeap { + width: relaxed_width(eps), + depth: cms_depth(delta), + heap_size: k as u32, + } + } + SketchAlgorithm::CountSketch => SketchParams::CountSketch { + width: relaxed_width(eps), + depth: cms_depth(delta), + }, + SketchAlgorithm::CountSketchWithHeap => { + let k = match intent { + AggIntent::TopK { k, .. } => *k, + _ => unreachable!("CountSketchWithHeap is only a TopK candidate"), + }; + SketchParams::CountSketchWithHeap { + width: relaxed_width(eps), + depth: cms_depth(delta), + heap_size: k as u32, + } + } + // Every other kind is untouched by this issue's CMS-specific + // relaxation — defer to the existing formula verbatim. Spelled out + // exhaustively, matching `default_size_params`'s own match, rather + // than a wildcard arm: a future `SketchAlgorithm` variant then fails to + // compile *here* too, instead of silently inheriting worst-case + // sizing with no signal that this function never considered it. + SketchAlgorithm::Kll => default_size_params(kind, intent, eps, delta), + SketchAlgorithm::Hll => default_size_params(kind, intent, eps, delta), + SketchAlgorithm::DDSketch => default_size_params(kind, intent, eps, delta), + SketchAlgorithm::Theta => default_size_params(kind, intent, eps, delta), + SketchAlgorithm::Kmv => default_size_params(kind, intent, eps, delta), + } +} + +// ── Parameter sizing ────────────────────────────────────────────────────────── +// +// Each function inverts the sketch family's standard error bound to the +// smallest parameter satisfying the target, clamped to the family's sane +// range. A non-positive ε saturates to the clamp maximum (tightest allowed). + +/// KLL: rank error ε ≈ 2/k ⇒ `k = ⌈2/ε⌉`. ε = 0.01 → k = 200, matching the +/// design doc's worked example (`KLL{k=200}` satisfies ε=0.01). +fn kll_k(eps: f64) -> u32 { + saturating_ceil(2.0 / eps, 8, 65_535) +} + +/// HLL: standard error ≈ 1.04/√(2^p) ⇒ `p = ⌈log2((1.04/ε)²)⌉`. The default +/// `Cardinality` target (`asap-ir::default_cardinality`) inverts to p = 14. +fn hll_precision(eps: f64) -> u8 { + saturating_ceil((1.04 / eps).powi(2).log2(), 4, 18) as u8 +} + +/// CMS: over-count ≤ ε·N with width `w = ⌈e/ε⌉` columns. +fn cms_width(eps: f64) -> u32 { + saturating_ceil(std::f64::consts::E / eps, 2, 1 << 26) +} + +/// CMS: failure probability ≤ δ with depth `d = ⌈ln(1/δ)⌉` rows. +/// δ = 0.01 → depth 5. +fn cms_depth(delta: f64) -> u32 { + saturating_ceil((1.0 / delta).ln(), 1, 32) +} + +/// KMV / theta: relative error ≈ 1/√k ⇒ `k = ⌈1/ε²⌉`. +fn kmv_k(eps: f64) -> u32 { + saturating_ceil(1.0 / (eps * eps), 16, 1 << 26) +} + +/// `⌈x⌉` clamped to `[lo, hi]`; NaN / non-positive x saturate to `hi` +/// (a degenerate ε means "as accurate as this family goes"). +fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { + if !x.is_finite() || x <= 0.0 { + return hi; + } + (x.ceil() as u32).clamp(lo, hi) +} + +// ── SketchAlgorithmStrategy ───────────────────────────────────────────────── + +/// A single static instance so [`SketchAlgorithmStrategy::default_cost_model`] +/// can hand out a `&'static dyn CostModel` without heap-allocating one — +/// `DefaultCostModel` is a unit struct with no state, so one instance serves +/// every caller. +static DEFAULT_COST_MODEL: DefaultCostModel = DefaultCostModel; + +/// Wraps [`implementations_for_with`]'s exhaustive, ranked list directly: for +/// a bindable `Aggregate`, every valid candidate summary realization as its +/// own [`ReplacementSubDAG`]. +/// +/// Ranked (only to *order the enumeration*, never to drop a candidate) via a +/// [`CostModel`] — [`DefaultCostModel`] unless constructed with +/// [`SketchAlgorithmStrategy::new`] — so a deployment-specific cost model's +/// other hooks (`size_params`, `realize_extension`, `readout_extension`) are +/// still consulted while binding each candidate. +pub struct SketchAlgorithmStrategy<'a> { + cost_model: &'a dyn CostModel, +} + +impl SketchAlgorithmStrategy<'static> { + /// A strategy that ranks/binds via the built-in [`DefaultCostModel`] — + /// what a deployment gets with no custom cost model plugged in. + pub fn default_cost_model() -> Self { + Self { + cost_model: &DEFAULT_COST_MODEL, + } + } +} + +impl<'a> SketchAlgorithmStrategy<'a> { + /// A strategy that ranks/binds via `cost_model` instead of the built-in + /// static preference order — the same customization point + /// [`implementations_for_with`] already offers. + pub fn new(cost_model: &'a dyn CostModel) -> Self { + Self { cost_model } + } +} + +impl ReplacementStrategy for SketchAlgorithmStrategy<'_> { + fn matches(&self, target: &TargetSubDAG<'_>) -> bool { + bindable_intent(target.root).is_some() + } + + fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { + let Some(intent) = bindable_intent(target.root) else { + return Vec::new(); + }; + // `implementations_for_with` is already exhaustive and ranked — no + // separate dispatch needed here. Only `Sketch` has more than one + // candidate in practice (every other variant's own dispatch produces + // exactly one `Implementation`), but this loop doesn't need to know + // that; it just constructs whatever the list contains. + implementations_for_with(intent, self.cost_model) + .into_iter() + .filter_map(|implementation| { + let rationale = describe_implementation(intent, &implementation); + let node = construct_summary(target.root, implementation, self.cost_model).ok()?; + Some(ReplacementSubDAG { + replacement: Replacement::Summary(node), + rationale, + }) + }) + .collect() + } +} + +/// A human-readable rationale for one candidate `Implementation`, for +/// [`ReplacementSubDAG::rationale`] text. +fn describe_implementation(intent: &AggIntent, implementation: &Implementation) -> String { + match implementation { + Implementation::Sketch(kind) => format!( + "{} realizes as a {:?} sketch — one of summary_candidates' \ + alternatives for this intent (asap_aware_mapping::replacement::implementations_for_with)", + describe_intent(intent), + kind.algorithm() + ), + Implementation::ExactAggregate { kind, .. } => format!( + "{} realizes as an exact {kind:?} accumulator — the only realization \ + implementations_for_with produces for this intent (no approximate \ + candidate applies)", + describe_intent(intent) + ), + Implementation::PassThrough => format!( + "{} has no summary realization and stays a logical pass-through — the \ + only realization implementations_for_with produces for this intent", + describe_intent(intent) + ), + Implementation::Sample { kind, .. } => format!( + "{} realizes as a {kind:?} sample — the only realization the plugged-in \ + CostModel produced for this intent", + describe_intent(intent) + ), + Implementation::Wavelet { kind, .. } => format!( + "{} realizes as a {kind:?} wavelet transform — the only realization the \ + plugged-in CostModel produced for this intent", + describe_intent(intent) + ), + Implementation::StatModel { kind, .. } => format!( + "{} realizes as a {kind:?} statistical model — the only realization the \ + plugged-in CostModel produced for this intent", + describe_intent(intent) + ), + } +} + +/// A short human-readable label for an `AggIntent`, for +/// [`ReplacementSubDAG::rationale`] text. Not exhaustive by design (unlike +/// this crate's other `AggIntent` matches, e.g. [`implementations_for_with`]'s) +/// — this is prose for a rationale string, not a decision, so an unlisted +/// variant just falls back to its `Debug` tag rather than forcing every +/// future intent to be named here too. [`crate::explanation`] needs no +/// counterpart of its own: it reads a candidate's `rationale` — built from +/// this text — straight off [`ReplacementSubDAG`], rather than re-describing +/// the same intent a second time. +fn describe_intent(intent: &AggIntent) -> String { + match intent { + AggIntent::Quantile { q, .. } => format!("quantile(q={q})"), + AggIntent::Cardinality { .. } => "cardinality (distinct count)".to_string(), + AggIntent::TopK { k, .. } => format!("top-{k} heavy-hitters"), + AggIntent::Count { .. } => "count".to_string(), + other => format!("{other:?}"), + } +} + +// ── realize_child / keep_pre_asap: rank-and-take-first, and its fallback ── + +/// Rank-and-take-first selector for a single [`QueryExpr`] node: enumerate +/// every candidate via [`SketchAlgorithmStrategy::replacements`], keep the +/// `cost_model`-preferred (first) one, and fall back to [`keep_pre_asap`] +/// when there's no candidate at all — **not** a general single-answer API +/// for a whole workload (that "commit to one final answer" step is a +/// downstream deployment's job, out of this crate's scope — see the crate +/// doc's `## Status` section). `root` must already be the caller's own +/// `Rc`, never fabricated per call, so this never allocates beyond what the +/// caller already held. +/// +/// `pub(crate)`: reachable from this module's own construction helper +/// ([`construct_summary_agg`], so a nested aggregate gets its own +/// independent enumeration instead of inheriting the parent's forced +/// candidate), from this module's own [`realize_one`] (the representative +/// bound `SummaryNode` [`cse_preference`] needs for a +/// [`CostModel::cse_share_decision`] comparison), and from +/// [`crate::cost_model::DefaultCostModel::estimate_cost`] (the same +/// representative-node need, for a [`Replacement::Rewrite`] candidate's own +/// cost estimate). Every other caller goes through +/// [`SketchAlgorithmStrategy::replacements`] directly and decides for itself. +pub(crate) fn realize_child( + root: &Rc, + cost_model: &dyn CostModel, +) -> Result, ImplementError> { + let target = TargetSubDAG::new(root); + match SketchAlgorithmStrategy::new(cost_model) + .replacements(&target) + .into_iter() + .next() + { + Some(ReplacementSubDAG { + replacement: Replacement::Summary(node), + .. + }) => Ok(node), + Some(ReplacementSubDAG { + replacement: Replacement::Rewrite(_), + .. + }) => { + unreachable!("SketchAlgorithmStrategy never returns a Rewrite candidate") + } + // No candidate at all: `root` isn't `bindable_intent` shape (or its + // intent has no realization `implementations_for_with` can't + // produce — never happens, that match is exhaustive) — the same + // conservative fallback `SketchAlgorithmStrategy::matches` uses. + None => keep_pre_asap(root), + } +} + +/// Wrap an unrewritten pre-ASAP subtree, lifting its schema with every column +/// `SummaryFamilyType::Plain`. `pub` so a caller can fall back to this +/// explicitly — e.g. when `SketchAlgorithmStrategy::replacements()` returns no +/// candidate for a target, or a deployment wants to force a node its own +/// runtime can't actually implement — through the same fallback this +/// crate's own dispatch uses, without duplicating the schema-lift logic. +pub fn keep_pre_asap(expr: &QueryExpr) -> Result, ImplementError> { + let schema = expr.output_schema()?; + Ok(Rc::new(SummaryNode { + expr: SummaryExpr::KeepPreAsap(Box::new(expr.clone())), + schema: lift(&schema), + })) +} + +// ── Construction: turn one already-decided Implementation into a SummaryNode ─ + +/// The bindable shape [`SketchAlgorithmStrategy`] targets: a single intent, no +/// `HAVING`. A multi-intent node (SQL `SELECT SUM(a), AVG(b)`), or one with a +/// `HAVING` predicate (the filter would need the estimate first), stays +/// logical — conservative fallbacks: [`SummaryExpr::KeepPreAsap`] boxes a +/// whole pre-ASAP subtree with no post-ASAP children, so a *logical* +/// operator above a bindable aggregate (`Filter`/`BinaryOp`/… over a +/// quantile) subsumes the aggregate into the logical wrapper unbound too — +/// rewriting through logical parents is the post-ASAP rule engine's job +/// (#6/#33), not this pass's. +pub fn bindable_intent(node: &QueryExpr) -> Option<&AggIntent> { + if let QueryExpr::Aggregate { + measures, having, .. + } = node + { + if let ([intent], None) = (measures.as_slice(), having) { + return Some(intent); + } + } + None +} + +/// Construct `expr`'s [`ReplacementSubDAG`] payload for one already-decided +/// [`Implementation`] of its top intent — the mechanical half of +/// [`SketchAlgorithmStrategy::replacements`], called once per candidate that +/// method enumerates. +/// +/// `expr` must still be the [`bindable_intent`] shape for `implementation` to +/// have any effect; anything else falls back to [`keep_pre_asap`]. +/// Only `expr`'s own top-level decision is forced — recursion into `expr`'s +/// child goes back through [`realize_child`] (fresh candidate +/// enumeration, not a forced pick), so choosing one candidate for a target +/// never leaks into that target's own nested aggregates. +fn construct_summary( + expr: &QueryExpr, + implementation: Implementation, + cost_model: &dyn CostModel, +) -> Result, ImplementError> { + if let QueryExpr::Aggregate { + reduction, + measures, + having, + child, + .. + } = expr + { + // The bindable shape: exactly one intent, no HAVING. (Multi-intent + // nodes and HAVING stay logical — see `bindable_intent`.) + if let ([intent], None) = (measures.as_slice(), having) { + if let Some((family, estimate)) = summary_family(implementation) { + return construct_summary_agg( + expr, reduction, intent, child, family, estimate, cost_model, + ); + } + } + } + keep_pre_asap(expr) +} + +/// Translate an [`Implementation`] into the `(family, needs a +/// SummaryEstimate readout)` pair [`construct_summary_agg`] needs, or `None` +/// for `PassThrough` (the caller falls back to [`keep_pre_asap`]). +/// +/// Every family's partial state needs a readout to recover a value, except +/// `ExactAggregate` — its partial state *is* the value already, so no +/// estimate step follows it. +fn summary_family(implementation: Implementation) -> Option<(SummaryFamilyType, bool)> { + Some(match implementation { + Implementation::ExactAggregate { kind, params } => { + (SummaryFamilyType::ExactAggregate(kind, params), false) + } + Implementation::Sketch(kind) => (SummaryFamilyType::Sketch(kind), true), + Implementation::Sample { kind, params } => (SummaryFamilyType::Sample(kind, params), true), + Implementation::Wavelet { kind, params } => { + (SummaryFamilyType::Wavelet(kind, params), true) + } + Implementation::StatModel { kind, params } => { + (SummaryFamilyType::StatModel(kind, params), true) + } + Implementation::PassThrough => return None, + }) +} + +/// Emit `SummaryAgg` (recursively binding the child), plus the +/// `SummaryEstimate` readout when `estimate` is set. +#[allow(clippy::too_many_arguments)] +fn construct_summary_agg( + node: &QueryExpr, + reduction: &Reduction, + intent: &AggIntent, + child: &Rc, + family: SummaryFamilyType, + estimate: bool, + cost_model: &dyn CostModel, +) -> Result, ImplementError> { + let child_schema = child.output_schema()?; + // The single canonical pre-ASAP derivation (per-series vs cross-series, + // name overrides) already computes the row shape; binding only retypes + // the summary state column. + let per_series = matches!(reduction, Reduction::PerEntity); + let by: Vec = reduction + .group_keys() + .map(|g| g.to_vec()) + .unwrap_or_default(); + let out_schema = node.output_schema()?; + let state_idx = summary_col_index(&out_schema, &by, per_series); + + let col = summarised_column(intent, &child_schema); + let query = estimate.then(|| readout(intent, &col, cost_model)); + + let mut state_schema = lift(&out_schema); + if let Some(field) = state_schema.fields.get_mut(state_idx) { + field.dtype = family.clone(); + } + + // `reduction` is carried onto `SummaryAgg` verbatim — not flattened to a + // bare `Vec` — so `SummaryExecutor::find_candidates` can tell + // a genuine empty-`by` reduction apart from a per-entity shape with no + // grouping concept at all (issue #163). `construct_summary_agg` is the + // single place that decides this; nothing downstream re-derives it. + let agg = Rc::new(SummaryNode { + expr: SummaryExpr::SummaryAgg { + child: realize_child(child, cost_model)?, + family, + col, + reduction: reduction.clone(), + }, + schema: state_schema, + }); + match query { + // The readout: downstream of the estimate the schema is the plain + // pre-ASAP row shape again (the summary-state type does not + // propagate). + Some(query) => Ok(Rc::new(SummaryNode { + expr: SummaryExpr::SummaryEstimate { + summary_input: agg, + query, + }, + schema: lift(&out_schema), + })), + None => Ok(agg), + } +} + +/// Index of the summary-state column in the aggregate's output schema: +/// cross-series output is `by ++ [agg]` (the column after the keys); +/// a per-series reduction keeps every label and replaces the sample value +/// (named `value` — mirror `per_series_reduction_schema`'s fallback). +/// `per_series` is the caller's already-read `Reduction` (issue #165) — +/// this never re-derives it, so it can't disagree with the caller. +fn summary_col_index(out_schema: &Schema, by: &[usize], per_series: bool) -> usize { + if per_series { + out_schema + .column_id("value") + .or_else(|| (0..out_schema.columns.len()).find(|&i| Some(i) != out_schema.time_index)) + .unwrap_or(0) + } else { + by.len() + } +} + +/// The column fed into the summary: the intent's positional input column +/// resolved to a name against the child schema, or the PromQL sample value. +fn summarised_column(intent: &AggIntent, child_schema: &Schema) -> ColumnRef { + match intent + .input_col() + .and_then(|id| child_schema.columns.get(id)) + { + Some(c) => match &c.table { + Some(t) => ColumnRef::Qualified { + table: t.clone(), + name: c.name.clone(), + }, + None => ColumnRef::Named(c.name.clone()), + }, + None => ColumnRef::SampleValue, + } +} + +/// The `SummaryEstimate` readout for a summary-bound intent. +fn readout(intent: &AggIntent, col: &ColumnRef, cost_model: &dyn CostModel) -> PostAsapSketchQuery { + match intent { + AggIntent::Quantile { q, .. } => PostAsapSketchQuery::Quantile { q: *q }, + AggIntent::Cardinality { .. } => PostAsapSketchQuery::Cardinality, + AggIntent::TopK { k, .. } => PostAsapSketchQuery::TopK { k: *k }, + AggIntent::Count { .. } => PostAsapSketchQuery::PointCount { + key: col.clone(), + value: None, + }, + // Core doesn't know the shape of a deployment-specific `Extension` + // intent, so it can't build its readout either — delegate to the + // same `CostModel` that decided (via `realize_extension`) this + // intent gets a summary realization at all. See `readout_extension`'s + // doc for the invariant this depends on. + AggIntent::Extension { ext_kind, payload } => { + cost_model.readout_extension(ext_kind, payload, col) + } + other => { + unreachable!("no summary realization for {other:?} (implementations_for_with)") + } + } +} + +/// Lift a pre-ASAP [`Schema`] to a [`SummarySchema`] with every column +/// `SummaryFamilyType::Plain` — shared by [`construct_summary_agg`] and +/// [`keep_pre_asap`], both in this module. +fn lift(schema: &Schema) -> SummarySchema { + SummarySchema { + fields: schema + .columns + .iter() + .map(|c| SummaryField { + name: c.name.clone(), + dtype: SummaryFamilyType::Plain(c.dtype.clone()), + nullable: c.nullable, + }) + .collect(), + time_index: schema.time_index, + } +} + +// ── SharedSubtreeStrategy ──────────────────────────────────────────────── + +/// Wraps `asap_types::pre_asap::cse::share_common_subtrees`'s sharing +/// decision as an explicit candidate pair, wherever a [`TargetSubDAG`] +/// already has two or more consumers. +/// +/// This strategy does not decide sharing itself, nor does it discover which +/// nodes are shared — by the time a caller builds a `TargetSubDAG` with +/// `consumer_count >= 2`, `share_common_subtrees` has already made that +/// (legality-gated, `PartialEq`-checked) call; [`discover_targets`] below +/// discovers real consumer counts across a workload the same way for +/// [`search_workload_with`] (this module's own tests reuse the identical +/// dedup logic to build realistic fixtures — see the module docs' +/// "Non-goals" on why that traversal isn't itself part of this strategy). +/// This strategy only reframes "two or more consumers already share this +/// `Rc`" as the two-way choice a downstream cost model (today, +/// [`CostModel::cse_share_decision`]) picks between: build once and share, or +/// build independently at each consumer. +pub struct SharedSubtreeStrategy; + +impl ReplacementStrategy for SharedSubtreeStrategy { + fn matches(&self, target: &TargetSubDAG<'_>) -> bool { + target.consumer_count >= 2 + } + + fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { + if target.consumer_count < 2 { + return Vec::new(); + } + let count = target.consumer_count; + vec![ + ReplacementSubDAG { + // The already-interned `Rc` itself: reusing it verbatim *is* + // "build once and share" — no new node to construct. + replacement: Replacement::Rewrite(Rc::clone(target.root)), + rationale: format!( + "build once and share: share_common_subtrees already interned this \ + subtree once and reused it across {count} consumers — one build can \ + answer all of them instead of computing it {count} times" + ), + }, + ReplacementSubDAG { + // A structurally-identical but freshly-allocated `Rc`: same + // value (`PartialEq`), deliberately *not* the same pointer, + // representing "undo the sharing and recompute independently". + replacement: Replacement::Rewrite(Rc::new((**target.root).clone())), + rationale: format!( + "build independently: undo the sharing share_common_subtrees found and \ + recompute this subtree separately at each of its {count} consumers — \ + worth it only when independence outweighs the shared-maintenance cost, \ + a CostModel's call (e.g. CostModel::cse_share_decision) and not this \ + strategy's" + ), + }, + ] + } +} + +// ── Workload-wide search: MemoGroup / PlanSpace / search_workload ────────── +// +// Merged in from the former `search.rs` (issue #252, part of #33) — see this +// file's own top-level "Workload-wide search" doc section for the full +// design rationale. + +/// A generous, documented backstop against a hypothetically ill-behaved +/// future [`ReplacementStrategy`] (see the module docs' "Termination" +/// section) — not a bound either shipped strategy could ever approach. +/// [`SketchAlgorithmStrategy`] and [`SharedSubtreeStrategy`] both converge in +/// exactly 2 passes over a fixed target set, regardless of workload size. +pub const MAX_SEARCH_ITERATIONS: usize = 1_000; + +// ── MemoGroup ──────────────────────────────────────────────────────────── + +/// One Cascades-style MEMO group: a single distinct [`TargetSubDAG`] (its +/// own `target` `Rc`, keyed by pointer identity in +/// [`PlanSpace`]'s internal map — never re-derived by value) plus every +/// [`ReplacementSubDAG`] alternative any registered [`ReplacementStrategy`] +/// proposed for it. +/// +/// `candidates` is deliberately *not* required to be non-empty — a +/// `TargetSubDAG` no registered strategy has an opinion on still gets a +/// group (with an empty candidate list), so [`PlanSpace`] always has +/// exactly one group per discovered `TargetSubDAG`, not "one group per +/// `TargetSubDAG` something matched". +#[derive(Debug, Clone)] +pub struct MemoGroup { + /// The target sub-DAG this group is for. + pub target: Rc, + /// How many operator-child positions across the whole workload + /// reference this exact `Rc` — see [`discover_targets`]. + pub consumer_count: usize, + /// Every distinct alternative discovered for `target`, in discovery + /// order (not ranked — see [`PlanSpace::cost_sorted`] for the ranked + /// view). + pub candidates: Vec, +} + +impl MemoGroup { + fn new(target: Rc, consumer_count: usize) -> Self { + Self { + target, + consumer_count, + candidates: Vec::new(), + } + } + + /// Add `candidate` unless it's already present (see + /// [`is_duplicate_rewrite`]/[`is_duplicate_summary`] for what "already + /// present" means for each [`Replacement`] variant). Returns whether it + /// was actually added — [`search_workload_with`]'s fixpoint loop uses + /// this to detect when a pass made no progress. + fn add_candidate(&mut self, candidate: ReplacementSubDAG) -> bool { + let is_duplicate = self.candidates.iter().any(|existing| { + match (&existing.replacement, &candidate.replacement) { + (Replacement::Rewrite(existing_rc), Replacement::Rewrite(rc)) => { + is_duplicate_rewrite(existing_rc, rc, &self.target) + } + (Replacement::Summary(existing_node), Replacement::Summary(node)) => { + is_duplicate_summary(existing_node, node) + } + // A `Rewrite` and a `Summary` are never the same candidate — + // they're different `Replacement` variants entirely. + (Replacement::Rewrite(_), Replacement::Summary(_)) + | (Replacement::Summary(_), Replacement::Rewrite(_)) => false, + } + }); + if is_duplicate { + false + } else { + self.candidates.push(candidate); + true + } + } +} + +/// Are `existing` and `candidate` the same [`Replacement::Rewrite`] +/// candidate for a group targeting `target`? +/// +/// Structural (`QueryExpr`) value equality alone is *not* enough here: this +/// module's one shipped multi-candidate `Replacement::Rewrite` source, +/// [`SharedSubtreeStrategy`], deliberately returns **two** candidates that +/// are value-equal to each other (`build once and share` vs. `build +/// independently` — see that strategy's own doc) but represent genuinely +/// different physical choices, distinguished *only* by whether the +/// candidate's `Rc` is the group's own `target` `Rc` (share) or a freshly +/// allocated one (recompute independently) — this IR has no field that +/// records "materialized once and shared", so `Rc` identity against +/// `target` is the only signal that distinction exists in at all. Treating +/// those two as duplicates of each other via pure value equality would +/// silently collapse a real choice into one candidate — the "false-positive +/// dedup is a wrong answer, not a missed optimization" failure mode +/// `cse.rs`'s own "Correctness" section warns about, just one level up from +/// where that module states it. +/// +/// So: two candidates whose "is this the target's own `Rc`?" bit disagrees +/// are never duplicates of each other, full stop. Only when that bit +/// *agrees* does this fall through to the real dedup discipline — +/// [`structural_hash`] as a candidate-narrowing filter, `QueryExpr`'s +/// derived `PartialEq` as the actual decision — protecting against the +/// (currently hypothetical, since neither shipped strategy causes it) +/// case of the exact same alternative being proposed twice. A fresh +/// [`HashCache`] per call: this is a pairwise check between two candidates +/// for one group, not a bottom-up pass over a whole tree, so there is no +/// wider traversal to amortize the cache across the way `InternTable`'s own +/// use of `structural_hash` does. +fn is_duplicate_rewrite( + existing: &Rc, + candidate: &Rc, + target: &Rc, +) -> bool { + let existing_is_target = Rc::ptr_eq(existing, target); + let candidate_is_target = Rc::ptr_eq(candidate, target); + if existing_is_target != candidate_is_target { + return false; + } + let mut cache = HashCache::new(); + structural_hash(existing, &mut cache) == structural_hash(candidate, &mut cache) + && existing == candidate +} + +/// Are `existing` and `candidate` the same [`Replacement::Summary`] +/// candidate? +/// +/// [`SummaryNode`] derives neither `PartialEq` nor `Hash` (it embeds +/// `SketchParams`/`f64`-bearing accuracy targets deep inside `SummaryExpr`, +/// the same reason `QueryExpr` can't derive `Hash` either — see +/// [`structural_hash`]'s own doc). Per this module's inherited "hash is a +/// filter, `PartialEq` is the decision, no exceptions" rule, there is no +/// real equality check to back a dedup *decision* here — and skipping the +/// check is the only choice that rule permits: never merging two candidates +/// is harmless (at worst, a redundant entry in a group's candidate list), +/// while comparing by some proxy this module can't actually verify (e.g. +/// `Debug` text, or `ReplacementSubDAG::rationale` — documented elsewhere in +/// this crate as prose for a report, "not machine parsing") risks exactly +/// the false-positive merge the rule exists to prevent. Both strategies +/// shipped today already return a structurally distinct candidate for every +/// entry of one `replacements()` call, so this is future-proofing against a +/// hypothetical repeat call, not a gap either strategy's own tests exercise. +fn is_duplicate_summary(_existing: &Rc, _candidate: &Rc) -> bool { + false +} + +// ── PlanSpace ──────────────────────────────────────────────────────────── + +/// The deduped candidate space [`search_workload`]/[`search_workload_with`] +/// discover: one [`MemoGroup`] per distinct `TargetSubDAG` in the +/// (already-CSE'd) workload, plus the workload's own post-CSE roots so a +/// caller can still map a `Root`'s `Id` back to the `Rc` whose +/// group holds its alternatives. +pub struct PlanSpace { + /// The workload's roots, after the one `share_common_subtrees` pass + /// [`search_workload_with`] runs up front — the same post-CSE roots + /// every `TargetSubDAG` in `groups` was discovered from. + pub roots: Vec<(Id, Rc)>, + groups: HashMap<*const QueryExpr, MemoGroup>, + /// Discovery order — stable iteration for [`PlanSpace::groups`]/ + /// [`PlanSpace::cost_sorted`], since `HashMap` iteration order isn't. + order: Vec<*const QueryExpr>, +} + +impl PlanSpace { + /// Every discovered group, in discovery order. + pub fn groups(&self) -> impl Iterator { + self.order.iter().map(move |ptr| &self.groups[ptr]) + } + + /// How many distinct targets were discovered. + pub fn len(&self) -> usize { + self.groups.len() + } + + /// Whether no targets were discovered at all (an empty workload, or one + /// with no `QueryExpr` nodes reachable from any root — never true for a + /// non-empty `roots`, since every root is itself a target). + pub fn is_empty(&self) -> bool { + self.groups.is_empty() + } + + /// The group for `target`, if `target`'s own `Rc` is a discovered + /// `TargetSubDAG` (i.e. `Rc::ptr_eq` to some node reachable from + /// `roots`). + pub fn group_for(&self, target: &Rc) -> Option<&MemoGroup> { + self.groups.get(&Rc::as_ptr(target)) + } + + /// The `sorted_by(cost_model)` step: every group, each with its own + /// candidates ranked best-first under `cost_model` where this module + /// knows how (see the module docs' "Cost-based final selection" + /// section) — groups themselves stay in discovery order, since targets + /// are independent decision points, not alternatives competing with + /// each other. + /// + /// Ranking itself is decided entirely by [`rank_group`] before + /// [`RankedGroup::costs`] is ever computed — pairing each candidate with + /// [`CostModel::estimate_cost`]'s own number is an additive annotation + /// for a caller that wants to *display* a cost (e.g. a + /// DAG-visualization view), not a second ranking signal, so plugging in + /// a `CostModel` whose `estimate_cost` disagrees with its own + /// `rank_candidates`/`cse_share_decision` (a deployment bug, not + /// something this method tries to protect against) would show a + /// `RankedGroup` whose `costs` aren't monotonically non-decreasing — + /// `cost_sorted`'s own ordering guarantee is unaffected either way. + pub fn cost_sorted(&self, cost_model: &dyn CostModel) -> Vec> { + self.order + .iter() + .map(|ptr| { + let group = &self.groups[ptr]; + let candidates = rank_group(group, cost_model); + let target = TargetSubDAG::with_consumer_count(&group.target, group.consumer_count); + let costs = candidates + .iter() + .map(|c| cost_model.estimate_cost(c, &target)) + .collect(); + RankedGroup { + target: &group.target, + consumer_count: group.consumer_count, + candidates, + costs, + } + }) + .collect() + } +} + +/// One [`MemoGroup`]'s candidates, ranked best-first by +/// [`PlanSpace::cost_sorted`]. +#[derive(Debug)] +pub struct RankedGroup<'a> { + pub target: &'a Rc, + pub consumer_count: usize, + pub candidates: Vec<&'a ReplacementSubDAG>, + /// `costs[i]` is `candidates[i]`'s own [`CostModel::estimate_cost`] + /// estimate — aligned index-for-index with `candidates`, one number per + /// candidate, for a caller that wants an actual `f64` next to each + /// candidate (e.g. "candidate A costs ≈ X, candidate B costs ≈ Y") and + /// not just `candidates`' own relative order. `f64::NAN` throughout + /// unless `cost_model` overrides `estimate_cost` — see that method's own + /// doc. + pub costs: Vec, +} + +/// Rank `group`'s candidates best-first under `cost_model`, per the module +/// docs' "Cost-based final selection" section. Falls back to discovery +/// order whenever there's nothing to rank (0 or 1 candidates) or this +/// module doesn't have a defined `CostModel` comparison for the shape it +/// sees — it never invents one. +fn rank_group<'a>(group: &'a MemoGroup, cost_model: &dyn CostModel) -> Vec<&'a ReplacementSubDAG> { + let mut ranked: Vec<&ReplacementSubDAG> = group.candidates.iter().collect(); + if ranked.len() <= 1 { + return ranked; + } + + // Shape 1: a `SharedSubtreeStrategy` share-vs-recompute pair (every + // candidate is a `Rewrite`) — rank via `CostModel::cse_share_decision` + // (see `cse_preference` below). + if ranked + .iter() + .all(|c| matches!(c.replacement, Replacement::Rewrite(_))) + { + if let Some(prefer_target) = cse_preference(group, cost_model) { + ranked.sort_by_key(|c| { + let is_target = matches!( + &c.replacement, + Replacement::Rewrite(rc) if Rc::ptr_eq(rc, &group.target) + ); + u8::from(is_target != prefer_target) + }); + } + return ranked; + } + + // Shape 2: `SketchAlgorithmStrategy`'s sketch-family candidates (every + // candidate is a `Summary` that realizes a `SketchAlgorithm`) — rank via + // `CostModel::rank_candidates`, the same hook `implementations_for_with` + // itself consults. + if let Some(intent) = bindable_intent(&group.target) { + let kinds: Option> = ranked + .iter() + .map(|c| match &c.replacement { + Replacement::Summary(node) => sketch_kind_of(node), + Replacement::Rewrite(_) => None, + }) + .collect(); + if let Some(kinds) = kinds { + let order = crate::cost_model::validated_candidate_ranking(cost_model, intent, &kinds); + ranked.sort_by_key(|c| { + let kind = match &c.replacement { + Replacement::Summary(node) => sketch_kind_of(node), + Replacement::Rewrite(_) => None, + }; + kind.and_then(|k| order.iter().position(|o| *o == k)) + .unwrap_or(usize::MAX) + }); + return ranked; + } + } + + // A target may be handled by more than one strategy (for example, a + // shared aggregate has both bound-summary and share/recompute rewrite + // candidates). No shape-specific hook spans those different candidate + // types, so compare the numeric estimates the CostModel exposes for that + // purpose. `total_cmp` gives deterministic placement to a model's NaN + // placeholders without dropping any candidate. + let target = TargetSubDAG::with_consumer_count(&group.target, group.consumer_count); + ranked.sort_by(|a, b| { + cost_model + .estimate_cost(a, &target) + .total_cmp(&cost_model.estimate_cost(b, &target)) + }); + ranked +} + +/// For a group whose candidates are all [`Replacement::Rewrite`] (the +/// [`SharedSubtreeStrategy`] shape): does [`CostModel::cse_share_decision`] +/// prefer the candidate that shares `group.target`'s own `Rc` (`true`), or +/// the one that recomputes independently (`false`)? `None` when there's no +/// real comparison to make — fewer than 2 consumers (mirrors +/// [`SharedSubtreeStrategy::matches`]'s own gate), or `group.target` can't +/// actually be bound at all (no candidate and no logical fallback — never +/// expected in practice for a target that's already part of a legitimate +/// workload tree, but this degrades to "keep discovery order" rather than +/// panicking). +fn cse_preference(group: &MemoGroup, cost_model: &dyn CostModel) -> Option { + if group.consumer_count < 2 { + return None; + } + let bound = realize_one(&group.target, cost_model)?; + let candidate = CseCandidate { + subtree: &group.target, + bound_summary: &bound, + consumer_count: group.consumer_count, + }; + Some(match cost_model.cse_share_decision(&candidate) { + ShareDecision::Share => true, + ShareDecision::RecomputeIndependently => false, + }) +} + +/// [`cse_preference`] only needs one representative bound [`SummaryNode`] +/// for `target` (to build a [`CseCandidate`] for +/// [`CostModel::cse_share_decision`]), not the full ranked candidate list +/// [`SketchAlgorithmStrategy::replacements`] returns — so this just reuses +/// [`realize_child`], the same rank-and-take-first helper +/// `construct_summary_agg`'s own recursion and +/// [`crate::cost_model::DefaultCostModel::estimate_cost`] already use, +/// wrapped to swallow the (here, uninteresting) error into `None`. +fn realize_one(target: &Rc, cost_model: &dyn CostModel) -> Option> { + realize_child(target, cost_model).ok() +} + +/// The `SketchAlgorithm` a bound [`Replacement::Summary`] candidate ultimately +/// realizes, if any (`None` for an `ExactAggregate`/pass-through +/// `Summary` — nothing to rank against another `SketchAlgorithm`). +/// +/// Mirrors this module's own `#[cfg(test)]`-only `summary_family_algorithm` +/// helper (in the test module below), which does the identical +/// `SummaryEstimate`-unwrap-then-match for that module's own tests; that +/// copy is test-only, so this needs its own for real (non-test) ranking +/// code — the same "duplicate a small, self-contained traversal rather than +/// restructure a test helper" call this file's own top doc already makes +/// for [`discover_targets`]. +fn sketch_kind_of(node: &SummaryNode) -> Option { + match &node.expr { + SummaryExpr::SummaryEstimate { summary_input, .. } => sketch_kind_of(summary_input), + SummaryExpr::SummaryAgg { + family: SummaryFamilyType::Sketch(kind), + .. + } => Some(kind.algorithm().clone()), + _ => None, + } +} + +// ── default_strategies ────────────────────────────────────────────────── + +/// The strategies [`search_workload`] runs, in the built-in +/// [`DefaultCostModel`] configuration — mirrors this module's own two +/// shipped [`ReplacementStrategy`] impls. +/// [`crate::explanation::explain_replacements`] (issue #257) uses +/// this same set (via [`search_workload`]) rather than keeping a second, +/// explanation-specific list to stay in sync with. Use +/// [`default_strategies_with`] to plug in a deployment-specific +/// [`CostModel`] instead. +pub fn default_strategies() -> Vec> { + vec![ + Box::new(SketchAlgorithmStrategy::default_cost_model()), + Box::new(SharedSubtreeStrategy), + ] +} + +/// Like [`default_strategies`], but [`SketchAlgorithmStrategy`] ranks/binds via +/// `cost_model` instead of the built-in [`DefaultCostModel`] — the same +/// customization point [`SketchAlgorithmStrategy::new`] itself offers. +pub fn default_strategies_with<'a>( + cost_model: &'a dyn CostModel, +) -> Vec> { + vec![ + Box::new(SketchAlgorithmStrategy::new(cost_model)), + Box::new(SharedSubtreeStrategy), + ] +} + +// ── search_workload ────────────────────────────────────────────────────── + +/// Search a whole workload's pre-ASAP roots for every candidate replacement +/// [`default_strategies`] can find, deduped into a [`PlanSpace`]. Candidate +/// *generation* uses the built-in [`DefaultCostModel`] (via +/// [`default_strategies`], the same way [`SketchAlgorithmStrategy::default_cost_model`] +/// does); call [`PlanSpace::cost_sorted`] on the result for the final +/// `sorted_by(cost_model)` step. Use [`search_workload_with`] to plug in a +/// custom strategy set (e.g. built via [`default_strategies_with`] for a +/// deployment-specific [`CostModel`]). +pub fn search_workload(roots: Vec<(Id, Rc)>) -> PlanSpace { + search_workload_with(roots, &default_strategies()) +} + +/// Like [`search_workload`], but with an explicit `strategies` set (see +/// [`default_strategies_with`] to plug in a deployment-specific +/// [`CostModel`] for candidate generation). +/// +/// Runs [`share_common_subtrees`] once over `roots` first — so every +/// strategy (and, transitively, every +/// [`crate::explanation::ReplacementExplanation`] a caller reads off the +/// result) sees the same already-deduplicated tree — then discovers every +/// `TargetSubDAG` (see [`discover_targets`]) and runs the +/// fixpoint loop the module docs describe, capped at +/// [`MAX_SEARCH_ITERATIONS`] passes (see the module docs' "Termination" +/// section). Deduping candidate plans this way needs no +/// [`CostModel`] at all — that only enters at two well-defined points: each +/// [`ReplacementStrategy`] in `strategies` may already carry its own (e.g. +/// [`SketchAlgorithmStrategy::new`]'s), and [`PlanSpace::cost_sorted`]'s final +/// ranking step takes one explicitly. +pub fn search_workload_with<'s, Id>( + roots: Vec<(Id, Rc)>, + strategies: &[Box], +) -> PlanSpace { + // `share_common_subtrees` wants owned `QueryExpr`s, not already-`Rc` + // roots — the same `Rc::try_unwrap`-with-clone-fallback pattern + // `asap_types::pre_asap::cse::intern_child` itself uses to recover an + // owned node without cloning in the common (uniquely-owned) case. + let owned_roots: Vec<(Id, QueryExpr)> = roots + .into_iter() + .map(|(id, rc)| { + let expr = Rc::try_unwrap(rc).unwrap_or_else(|shared| (*shared).clone()); + (id, expr) + }) + .collect(); + let cse_roots = share_common_subtrees(owned_roots); + + let mut order = Vec::new(); + let mut nodes = HashMap::new(); + let mut counts: HashMap<*const QueryExpr, usize> = HashMap::new(); + discover_targets(&cse_roots, &mut order, &mut nodes, &mut counts); + + let mut groups: HashMap<*const QueryExpr, MemoGroup> = HashMap::new(); + for ptr in &order { + groups.insert(*ptr, MemoGroup::new(Rc::clone(&nodes[ptr]), counts[ptr])); + } + + // Round-based frontier: every target is asked exactly once per strategy + // (never re-asked — see the module docs' "Termination" section on why + // that matters for `Replacement::Summary` dedup specifically). A round + // can grow the *next* round's frontier only by a candidate's own + // reachable children exposing a genuinely new, not-yet-known `Rc` — see + // `discover_new_descendant_targets`. + let mut frontier = order.clone(); + let mut rounds = 0usize; + while !frontier.is_empty() { + rounds += 1; + assert!( + rounds <= MAX_SEARCH_ITERATIONS, + "search_workload: fixpoint search did not converge within {MAX_SEARCH_ITERATIONS} \ + rounds — a registered ReplacementStrategy's Replacement::Rewrite candidates keep \ + exposing new, never-before-seen descendant structure every round. \ + SketchAlgorithmStrategy/SharedSubtreeStrategy never do this (see replacement.rs's \ + module docs' \"Termination\" section); check any custom strategies passed to \ + search_workload_with.", + ); + + let targets_before = order.len(); + for ptr in &frontier { + let (root, consumer_count) = { + let group = &groups[ptr]; + (Rc::clone(&group.target), group.consumer_count) + }; + let target = TargetSubDAG::with_consumer_count(&root, consumer_count); + + let mut proposed = Vec::new(); + for strategy in strategies { + if strategy.matches(&target) { + proposed.extend(strategy.replacements(&target)); + } + } + + for candidate in &proposed { + if let Replacement::Rewrite(rc) = &candidate.replacement { + discover_new_descendant_targets(rc, &mut order, &mut nodes, &mut counts); + } + } + + let group = groups + .get_mut(ptr) + .expect("every discovered target has a group"); + for candidate in proposed { + group.add_candidate(candidate); + } + } + + // Any pointer `discover_new_descendant_targets` appended to `order` + // this round is a genuinely new target — give it a group and process + // it next round. Targets already in `groups` are never revisited. + let new_targets = &order[targets_before..]; + for ptr in new_targets { + groups + .entry(*ptr) + .or_insert_with(|| MemoGroup::new(Rc::clone(&nodes[ptr]), counts[ptr])); + } + frontier = new_targets.to_vec(); + } + + PlanSpace { + roots: cse_roots, + groups, + order, + } +} + +// ── target discovery ───────────────────────────────────────────────────── + +/// Walk every root's whole DAG, discovering one `TargetSubDAG` per distinct +/// `Rc` and its real `consumer_count` — see the module docs' "Where +/// `TargetSubDAG` discovery comes from" section for the full rationale. +fn discover_targets( + roots: &[(Id, Rc)], + order: &mut Vec<*const QueryExpr>, + nodes: &mut HashMap<*const QueryExpr, Rc>, + counts: &mut HashMap<*const QueryExpr, usize>, +) { + for (_, root) in roots { + walk(root, order, nodes, counts); + } +} + +/// Scan `candidate`'s **children** (deliberately never `candidate`'s own +/// top-level pointer — see the module docs' "Termination" section: a +/// [`Replacement::Rewrite`]'s value is an alternative *for* the target that +/// proposed it, never a new target of its own) for any `Rc` not already +/// known, appending each to `order`/`nodes`/`counts` so +/// [`search_workload_with`]'s next round processes it. A no-op when every +/// child is already known — the case both shipped strategies always produce +/// (see that section). +fn discover_new_descendant_targets( + candidate: &Rc, + order: &mut Vec<*const QueryExpr>, + nodes: &mut HashMap<*const QueryExpr, Rc>, + counts: &mut HashMap<*const QueryExpr, usize>, +) { + walk_children(candidate, order, nodes, counts); +} + +/// Visit `node`: count this occurrence, and — the first time this exact +/// `Rc` is seen — record it as a target and recurse into its children. +fn walk( + node: &Rc, + order: &mut Vec<*const QueryExpr>, + nodes: &mut HashMap<*const QueryExpr, Rc>, + counts: &mut HashMap<*const QueryExpr, usize>, +) { + let ptr = Rc::as_ptr(node); + let already_visited = counts.contains_key(&ptr); + *counts.entry(ptr).or_insert(0) += 1; + if !already_visited { + order.push(ptr); + nodes.insert(ptr, Rc::clone(node)); + walk_children(node, order, nodes, counts); + } +} + +/// `node`'s own **relational-skeleton** operator children — the same scope +/// `asap_types::pre_asap::cse::share_common_subtrees`/`rebuild_children` +/// itself uses (see that module's "Algorithm" section) and +/// `tests::count_consumers` mirrors for its own fixtures. Exhaustive over +/// every `QueryExpr` variant: a new variant fails to compile here until this +/// match is extended too. +fn walk_children( + node: &QueryExpr, + order: &mut Vec<*const QueryExpr>, + nodes: &mut HashMap<*const QueryExpr, Rc>, + counts: &mut HashMap<*const QueryExpr, usize>, +) { + use QueryExpr::*; + match node { + Scan { .. } | PromqlScalarBridge(_) | QueryTimestamp => {} + PromqlVectorFromScalar(c) | PromqlScalarFromVector(c) => walk(c, order, nodes, counts), + PromqlRelabel { child, .. } + | PromqlInfoEnrich { child, .. } + | PromqlSeriesSample { child, .. } + | Filter { child, .. } + | Project { child, .. } + | Aggregate { child, .. } + | Dedup { child, .. } + | PromqlSubquery { child, .. } + | TimeRange { child, .. } + | TimeShift { child, .. } + | SQLWindowFunc { child, .. } + | Sort { child, .. } + | Limit { child, .. } => walk(child, order, nodes, counts), + Concat { children } => { + for c in children { + walk_children(c, order, nodes, counts); + } + } + Join { left, right, .. } | SetOp { left, right, .. } => { + walk(left, order, nodes, counts); + walk(right, order, nodes, counts); + } + BinaryOp { lhs, rhs, .. } => { + walk(lhs, order, nodes, counts); + walk(rhs, order, nodes, counts); + } + Column(_) + | Literal(_) + | Compare { .. } + | BoolAnd(_) + | BoolOr(_) + | Not(_) + | IsNull(_) + | IsNotNull(_) + | Cast { .. } + | InList { .. } + | FunctionCall { .. } + | Arithmetic { .. } + | Case { .. } => {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + use asap_types::pre_asap::agg_intent::{ + agg_is_exact, default_cardinality, default_quantile, MathFunc, TimeFunc, + }; + use asap_types::pre_asap::query_expr::{Reduction as ReductionTy, Source}; + use asap_types::pre_asap::schema::{Column, DataType, Schema as SchemaTy}; + use asap_types::types::AccuracyTarget; + use std::collections::HashMap; + + fn eps(e: f64) -> AccuracyTarget { + AccuracyTarget::Epsilon(e) + } + + // ── implementations_for_with / sizing ─────────────────────────────── + + /// The most-preferred `Implementation` — `implementations_for_with(intent, + /// &DefaultCostModel)`'s head — for tests that only care about the + /// default pick, not the full candidate list. + fn preferred(intent: &AggIntent) -> Implementation { + implementations_for_with(intent, &DefaultCostModel) + .into_iter() + .next() + .expect("every intent has at least one Implementation") + } + + /// Shorthand for asserting the realization *category*. + #[derive(Debug, PartialEq)] + enum Cat { + Sketch(SketchAlgorithm), + Acc(ExactKind), + Pass, + } + + fn cat(intent: &AggIntent) -> Cat { + match preferred(intent) { + Implementation::ExactAggregate { kind, .. } => Cat::Acc(kind), + Implementation::Sketch(kind) => Cat::Sketch(kind.algorithm().clone()), + Implementation::PassThrough => Cat::Pass, + other => { + panic!("this coverage matrix expects only Exact/Sketch/PassThrough, got {other:?}") + } + } + } + + /// The `AggIntent → SummaryKind` coverage matrix (issue #98): every intent + /// variant maps to a sketch, an exact accumulator, or an explicit + /// pass-through. `implementations_for_with`'s match is exhaustive, so a + /// new variant cannot compile without a decision; this matrix pins what + /// each decision *is* (its preferred/first candidate). + #[test] + fn agg_intent_to_summary_kind_coverage_matrix() { + use AggIntent as A; + use Cat::*; + use ExactKind as E; + use SketchAlgorithm as K; + let matrix: Vec<(A, Cat)> = vec![ + // approximate-capable, at an ε target → sketch + (default_quantile(0.99), Sketch(K::Kll)), + (default_cardinality(), Sketch(K::Hll)), + ( + A::Count { + accuracy: eps(0.01), + }, + Sketch(K::Cms), + ), + ( + A::TopK { + k: 10, + accuracy: eps(0.01), + }, + Sketch(K::CmsWithHeap), + ), + // the same intents at Exact → exact realization + ( + A::Quantile { + col: None, + q: 0.5, + accuracy: AccuracyTarget::Exact, + }, + Pass, + ), + ( + A::Cardinality { + col: None, + accuracy: AccuracyTarget::Exact, + }, + Pass, + ), + ( + A::Count { + accuracy: AccuracyTarget::Exact, + }, + Acc(E::Count), + ), + ( + A::TopK { + k: 10, + accuracy: AccuracyTarget::Exact, + }, + Pass, + ), + // exact mergeable accumulators + (A::Sum { col: None }, Acc(E::Sum)), + (A::Min { col: None }, Acc(E::MinMax)), + (A::Max { col: None }, Acc(E::MinMax)), + (A::Rate, Acc(E::Rate)), + (A::Increase, Acc(E::Increase)), + // exact but non-mergeable → pass-through + (A::Avg { col: None }, Pass), + ( + A::StdDev { + col: None, + population: false, + }, + Pass, + ), + ( + A::Variance { + col: None, + population: true, + }, + Pass, + ), + // classic-bucket histogram_quantile is not re-sketchable (#79) + (A::HistogramQuantile { q: 0.99 }, Pass), + // counter-derivative / range-vector functions (#44) + (A::Changes, Pass), + (A::Delta, Pass), + (A::IDelta, Pass), + (A::Deriv, Pass), + (A::Resets, Pass), + (A::PredictLinear { seconds: 60.0 }, Pass), + ( + A::DoubleExpSmoothing { + smoothing: 0.5, + trend: 0.5, + }, + Pass, + ), + // native-histogram accessors (#43) + (A::HistogramCount, Pass), + (A::HistogramSum, Pass), + (A::HistogramAvg, Pass), + (A::HistogramStdDev, Pass), + (A::HistogramStdVar, Pass), + ( + A::HistogramFraction { + lower: 0.0, + upper: 1.0, + }, + Pass, + ), + // per-sample transforms (#45, #46) + presence (#47) + (A::Math(MathFunc::Abs), Pass), + (A::TimeFn(TimeFunc::Hour), Pass), + (A::Absent, Pass), + (A::AbsentOverTime, Pass), + (A::PresentOverTime, Pass), + // extended aggregations (#49) + (A::Group, Pass), + (A::CountValues { label: "v".into() }, Pass), + // additional range reducers (#51) + (A::LastOverTime, Pass), + (A::FirstOverTime, Pass), + (A::MadOverTime, Pass), + (A::TsOfMinOverTime, Pass), + (A::TsOfMaxOverTime, Pass), + (A::TsOfFirstOverTime, Pass), + (A::TsOfLastOverTime, Pass), + ]; + for (intent, expected) in &matrix { + assert_eq!(&cat(intent), expected, "realization for {intent:?}"); + } + // Every accumulator pick is mergeable; every sketch pick is on a + // genuinely approximate target (the `agg_is_*` helpers stay truthful). + for (intent, expected) in &matrix { + if let Cat::Acc(_) = expected { + assert!(agg_is_mergeable(intent), "{intent:?}"); + } + if let Cat::Sketch(_) = expected { + assert!( + !agg_is_exact(intent) || matches!(intent, AggIntent::Count { .. }), + "{intent:?} sketches only under an approximate target" + ); + } + } + } + + #[test] + fn accuracy_target_drives_the_boundary() { + // Same intent, three targets → three different decisions. + let exact = AggIntent::Quantile { + col: None, + q: 0.99, + accuracy: AccuracyTarget::Exact, + }; + assert_eq!(preferred(&exact), Implementation::PassThrough); + + let approx = default_quantile(0.99); // ε = 0.01 + assert_eq!( + preferred(&approx), + Implementation::Sketch(SketchKind::new( + SketchAlgorithm::Kll, + SketchParams::Kll { k: 200 }, // design.md worked example + )) + ); + + let looser = AggIntent::Quantile { + col: None, + q: 0.99, + accuracy: eps(0.05), + }; + assert_eq!( + preferred(&looser), + Implementation::Sketch(SketchKind::new( + SketchAlgorithm::Kll, + SketchParams::Kll { k: 40 }, // ⌈2/0.05⌉ + )) + ); + } + + #[test] + fn default_cardinality_inverts_to_hll_precision_14() { + // `default_cardinality` encodes HLL's standard error at p=14; the + // sizing must invert it back exactly. + assert_eq!( + preferred(&default_cardinality()), + Implementation::Sketch(SketchKind::new( + SketchAlgorithm::Hll, + SketchParams::Hll { precision: 14 }, + )) + ); + } + + #[test] + fn epsilon_delta_sizes_cms_depth() { + let intent = AggIntent::Count { + accuracy: AccuracyTarget::EpsilonDelta { + epsilon: 0.001, + delta: 0.001, + }, + }; + assert_eq!( + preferred(&intent), + Implementation::Sketch(SketchKind::new( + SketchAlgorithm::Cms, + SketchParams::Cms { + width: 2719, + depth: 7 + }, // ⌈e/0.001⌉, ⌈ln 1000⌉ + )) + ); + // Epsilon-only falls back to DEFAULT_DELTA → depth 5. + let intent = AggIntent::Count { + accuracy: eps(0.001), + }; + assert_eq!( + preferred(&intent), + Implementation::Sketch(SketchKind::new( + SketchAlgorithm::Cms, + SketchParams::Cms { + width: 2719, + depth: 5 + }, + )) + ); + } + + #[test] + fn topk_heap_size_tracks_k() { + let intent = AggIntent::TopK { + k: 25, + accuracy: eps(0.01), + }; + match preferred(&intent) { + Implementation::Sketch(kind) if kind.algorithm() == &SketchAlgorithm::CmsWithHeap => { + let SketchParams::CmsWithHeap { + width, + depth, + heap_size, + } = kind.params() + else { + unreachable!("SketchKind validates CmsWithHeap params") + }; + assert_eq!(*heap_size, 25); + assert_eq!(*width, 272); // ⌈e/0.01⌉ + assert_eq!(*depth, 5); + } + other => panic!("expected CmsWithHeap, got {other:?}"), + } + } + + #[test] + fn candidate_lists_match_the_issue_map() { + assert_eq!( + summary_candidates(&default_quantile(0.5)), + &[SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] + ); + assert_eq!( + summary_candidates(&default_cardinality()), + &[ + SketchAlgorithm::Hll, + SketchAlgorithm::Theta, + SketchAlgorithm::Kmv + ] + ); + assert_eq!( + summary_candidates(&AggIntent::TopK { + k: 5, + accuracy: eps(0.01) + }), + &[ + SketchAlgorithm::CmsWithHeap, + SketchAlgorithm::CountSketchWithHeap + ] + ); + assert_eq!( + summary_candidates(&AggIntent::Count { + accuracy: eps(0.01) + }), + &[SketchAlgorithm::Cms, SketchAlgorithm::CountSketch] + ); + assert!(summary_candidates(&AggIntent::Rate).is_empty()); + } + + #[test] + fn implementations_for_with_enumerates_every_candidate_ranked() { + // Quantile's candidate list is [Kll, DDSketch] — implementations_for_with + // must return both, ranked with the DefaultCostModel's preferred + // (Kll) first. + let kinds: Vec = + implementations_for_with(&default_quantile(0.99), &DefaultCostModel) + .into_iter() + .map(|implementation| match implementation { + Implementation::Sketch(kind) => kind.algorithm().clone(), + other => panic!("expected Sketch, got {other:?}"), + }) + .collect(); + assert_eq!(kinds, vec![SketchAlgorithm::Kll, SketchAlgorithm::DDSketch]); + } + + #[test] + fn degenerate_epsilon_saturates_to_tightest_params() { + let intent = AggIntent::Quantile { + col: None, + q: 0.99, + accuracy: eps(0.0), + }; + assert_eq!( + preferred(&intent), + Implementation::Sketch(SketchKind::new( + SketchAlgorithm::Kll, + SketchParams::Kll { k: 65_535 }, + )) + ); + } + + // ── posterior_aware_size_params (issue #239, integration point 2) ────── + + fn count_intent(e: f64) -> AggIntent { + AggIntent::Count { accuracy: eps(e) } + } + + #[test] + fn posterior_aware_sizing_shrinks_width_under_stated_assumption() { + let intent = count_intent(0.01); + let worst_case = default_size_params(SketchAlgorithm::Cms, &intent, 0.01, 0.01); + let relaxed = posterior_aware_size_params( + SketchAlgorithm::Cms, + &intent, + 0.01, + 0.01, + ExpectedCaseSizing { + width_relaxation: 0.5, + }, + ); + match (worst_case, relaxed) { + ( + SketchParams::Cms { + width: w0, + depth: d0, + }, + SketchParams::Cms { + width: w1, + depth: d1, + }, + ) => { + assert!( + w1 < w0, + "expected relaxed width {w1} to be strictly smaller than worst-case {w0}" + ); + assert_eq!(d0, d1, "depth must be unaffected by width_relaxation"); + } + other => panic!("expected Cms/Cms pair, got {other:?}"), + } + } + + #[test] + fn posterior_aware_sizing_at_full_relaxation_matches_worst_case() { + // width_relaxation = 1.0 must reproduce default_size_params exactly + // — the "no risk taken" boundary. + let intent = count_intent(0.01); + let worst_case = default_size_params(SketchAlgorithm::Cms, &intent, 0.01, 0.01); + let relaxed = posterior_aware_size_params( + SketchAlgorithm::Cms, + &intent, + 0.01, + 0.01, + ExpectedCaseSizing { + width_relaxation: 1.0, + }, + ); + assert_eq!(worst_case, relaxed); + } + + #[test] + fn posterior_aware_sizing_invalid_relaxation_falls_back_to_worst_case() { + let intent = count_intent(0.01); + let worst_case = default_size_params(SketchAlgorithm::Cms, &intent, 0.01, 0.01); + for bad in [0.0, -0.5, 1.5, f64::NAN, f64::INFINITY] { + let relaxed = posterior_aware_size_params( + SketchAlgorithm::Cms, + &intent, + 0.01, + 0.01, + ExpectedCaseSizing { + width_relaxation: bad, + }, + ); + assert_eq!( + worst_case, relaxed, + "width_relaxation={bad} should fall back to the worst-case width" + ); + } + } + + #[test] + fn posterior_aware_sizing_applies_to_every_cms_family_kind() { + let cms_heap_intent = AggIntent::TopK { + k: 7, + accuracy: eps(0.01), + }; + let assumption = ExpectedCaseSizing { + width_relaxation: 0.25, + }; + // CountSketch + assert_eq!( + posterior_aware_size_params( + SketchAlgorithm::CountSketch, + &count_intent(0.01), + 0.01, + 0.01, + assumption + ), + SketchParams::CountSketch { + width: 68, + depth: 5 + }, // ceil(272 * 0.25) + ); + // CmsWithHeap / CountSketchWithHeap carry k through untouched. + match posterior_aware_size_params( + SketchAlgorithm::CmsWithHeap, + &cms_heap_intent, + 0.01, + 0.01, + assumption, + ) { + SketchParams::CmsWithHeap { + width, + depth, + heap_size, + } => { + assert_eq!(width, 68); + assert_eq!(depth, 5); + assert_eq!(heap_size, 7); + } + other => panic!("expected CmsWithHeap, got {other:?}"), + } + } + + #[test] + fn posterior_aware_sizing_leaves_non_cms_kinds_unchanged() { + // Kll/Hll/etc. have no width_relaxation concept — must be byte-for- + // byte identical to default_size_params. + let intent = default_quantile(0.99); + let assumption = ExpectedCaseSizing { + width_relaxation: 0.1, + }; + assert_eq!( + posterior_aware_size_params(SketchAlgorithm::Kll, &intent, 0.01, 0.01, assumption), + default_size_params(SketchAlgorithm::Kll, &intent, 0.01, 0.01), + ); + } + + #[test] + fn default_size_params_unchanged_by_new_function_existing() { + // Regression pin: default_size_params's own worst-case behavior for + // existing callers must be untouched by adding + // posterior_aware_size_params alongside it. + assert_eq!( + default_size_params(SketchAlgorithm::Cms, &count_intent(0.001), 0.001, 0.001), + SketchParams::Cms { + width: 2719, + depth: 7 + }, + ); + } + + // ── SketchAlgorithmStrategy / SharedSubtreeStrategy fixtures ─────────── + + fn metric_scan(labels: &[&str]) -> QueryExpr { + let mut columns = vec![ + Column::new("ts", DataType::Timestamp, false), + Column::new("value", DataType::Float64, false), + ]; + columns.extend(labels.iter().map(|n| Column::new(*n, DataType::Utf8, true))); + QueryExpr::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: SchemaTy::with_time_index(columns, 0, vec![]), + } + } + + fn agg(by: Vec, intent: AggIntent, child: QueryExpr) -> QueryExpr { + QueryExpr::Aggregate { + reduction: ReductionTy::by(by), + measures: vec![intent], + output_names: vec![], + having: None, + child: Rc::new(child), + } + } + + // ── SketchAlgorithmStrategy ───────────────────────────────────────────── + + #[test] + fn matches_a_bindable_aggregate() { + let q = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); + let target = TargetSubDAG::new(&q); + assert!(SketchAlgorithmStrategy::default_cost_model().matches(&target)); + } + + #[test] + fn does_not_match_a_multi_intent_or_having_aggregate() { + let strategy = SketchAlgorithmStrategy::default_cost_model(); + + let multi = Rc::new(QueryExpr::Aggregate { + reduction: ReductionTy::by(vec![2]), + measures: vec![AggIntent::Sum { col: None }, AggIntent::Avg { col: None }], + output_names: vec![], + having: None, + child: Rc::new(metric_scan(&["job"])), + }); + let target = TargetSubDAG::new(&multi); + assert!(!strategy.matches(&target)); + assert!(strategy.replacements(&target).is_empty()); + + let mut having_q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + if let QueryExpr::Aggregate { having, .. } = &mut having_q { + *having = Some(asap_types::pre_asap::query_expr::Predicate(Rc::new( + QueryExpr::Literal(asap_types::pre_asap::expr_ir::ScalarValue::Boolean(true)), + ))); + } + let having_q = Rc::new(having_q); + let target = TargetSubDAG::new(&having_q); + assert!(!strategy.matches(&target)); + assert!(strategy.replacements(&target).is_empty()); + } + + #[test] + fn does_not_match_a_non_aggregate_node() { + let scan = Rc::new(metric_scan(&["job"])); + let target = TargetSubDAG::new(&scan); + assert!(!SketchAlgorithmStrategy::default_cost_model().matches(&target)); + assert!(SketchAlgorithmStrategy::default_cost_model() + .replacements(&target) + .is_empty()); + } + + #[test] + fn approximate_quantile_enumerates_every_summary_candidate() { + // Quantile's candidate list is [Kll, DDSketch] (summary_candidates) — + // every entry must come back as its own bound SummaryNode candidate, + // not just Kll (the CostModel-ranked head implementations_for_with commits to). + let q = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); + let target = TargetSubDAG::new(&q); + let replacements = SketchAlgorithmStrategy::default_cost_model().replacements(&target); + assert_eq!( + replacements.len(), + 2, + "expected 2 candidates, got {replacements:?}" + ); + + let kinds: Vec = replacements + .iter() + .map(|r| match &r.replacement { + Replacement::Summary(node) => summary_family_algorithm(node), + Replacement::Rewrite(_) => panic!("expected a Summary replacement"), + }) + .collect(); + assert!(kinds.contains(&SketchAlgorithm::Kll), "{kinds:?}"); + assert!(kinds.contains(&SketchAlgorithm::DDSketch), "{kinds:?}"); + assert!( + replacements.iter().all(|r| !r.rationale.is_empty()), + "every candidate must carry a rationale" + ); + } + + #[test] + fn cardinality_enumerates_all_three_summary_candidates() { + let q = Rc::new(agg(vec![2], default_cardinality(), metric_scan(&["job"]))); + let target = TargetSubDAG::new(&q); + let replacements = SketchAlgorithmStrategy::default_cost_model().replacements(&target); + let kinds: Vec = replacements + .iter() + .map(|r| match &r.replacement { + Replacement::Summary(node) => summary_family_algorithm(node), + Replacement::Rewrite(_) => panic!("expected a Summary replacement"), + }) + .collect(); + assert_eq!( + kinds, + vec![ + SketchAlgorithm::Hll, + SketchAlgorithm::Theta, + SketchAlgorithm::Kmv + ], + "expected every summary_candidates entry for Cardinality" + ); + } + + #[test] + fn exact_accuracy_target_yields_exactly_one_pass_through_candidate() { + // Exact quantile has no sketch candidate at all — implementations_for_with + // produces PassThrough, the only option, so exactly one candidate. + let intent = AggIntent::Quantile { + col: None, + q: 0.99, + accuracy: AccuracyTarget::Exact, + }; + let q = Rc::new(agg(vec![2], intent, metric_scan(&["job"]))); + let target = TargetSubDAG::new(&q); + let replacements = SketchAlgorithmStrategy::default_cost_model().replacements(&target); + assert_eq!(replacements.len(), 1, "{replacements:?}"); + assert!(matches!( + &replacements[0].replacement, + Replacement::Summary(node) if matches!( + node.expr, + asap_types::post_asap::SummaryExpr::KeepPreAsap(_) + ) + )); + assert!(replacements[0].rationale.contains("only realization")); + } + + #[test] + fn exact_mergeable_intent_yields_exactly_one_accumulator_candidate() { + let q = Rc::new(agg( + vec![2], + AggIntent::Sum { col: None }, + metric_scan(&["job"]), + )); + let target = TargetSubDAG::new(&q); + let replacements = SketchAlgorithmStrategy::default_cost_model().replacements(&target); + assert_eq!(replacements.len(), 1, "{replacements:?}"); + assert!(matches!( + &replacements[0].replacement, + Replacement::Summary(node) if matches!( + node.expr, + asap_types::post_asap::SummaryExpr::SummaryAgg { .. } + ) + )); + } + + /// A custom `CostModel` doesn't change *which* candidates are enumerated + /// (still every `summary_candidates` entry) — only which one + /// `implementations_for_with` itself would prefer first, and how each + /// candidate's own params are sized. + struct PreferDDSketch; + impl CostModel for PreferDDSketch { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + let mut v = candidates.to_vec(); + if let Some(pos) = v.iter().position(|k| *k == SketchAlgorithm::DDSketch) { + let dd = v.remove(pos); + v.insert(0, dd); + } + v + } + } + + #[test] + fn custom_cost_model_still_enumerates_every_candidate_not_just_its_own_pick() { + let q = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); + let target = TargetSubDAG::new(&q); + let custom = PreferDDSketch; + let replacements = SketchAlgorithmStrategy::new(&custom).replacements(&target); + let kinds: Vec = replacements + .iter() + .map(|r| match &r.replacement { + Replacement::Summary(node) => summary_family_algorithm(node), + Replacement::Rewrite(_) => panic!("expected a Summary replacement"), + }) + .collect(); + assert!(kinds.contains(&SketchAlgorithm::Kll)); + assert!(kinds.contains(&SketchAlgorithm::DDSketch)); + assert_eq!(kinds.len(), 2); + } + + /// Enumerating candidates for the *target* node must only steer that + /// node's own decision — a nested aggregate underneath it still gets its + /// own independent (`cost_model`-ranked) enumeration, not whatever the + /// caller happened to pick for the outer target. This is the behavior + /// [`construct_summary`]'s recursion (via [`realize_child`]) + /// gets for free: only the top node's `Implementation` is ever forced + /// from outside; the child is always re-enumerated fresh. + #[test] + fn enumerating_the_targets_candidates_does_not_leak_into_a_nested_aggregate() { + // outer: quantile(0.99, ...) over inner: quantile(0.5, m) — both + // Quantile, so both share the [Kll, DDSketch] candidate list. + let inner = agg(vec![2], default_quantile(0.5), metric_scan(&["job"])); + let outer = Rc::new(agg(vec![], default_quantile(0.99), inner)); + let target = TargetSubDAG::new(&outer); + let replacements = SketchAlgorithmStrategy::default_cost_model().replacements(&target); + + let ddsketch = replacements + .iter() + .find(|r| { + matches!(&r.replacement, Replacement::Summary(node) + if summary_family_algorithm(node) == SketchAlgorithm::DDSketch) + }) + .expect("the outer target's DDSketch candidate must be present"); + let Replacement::Summary(node) = &ddsketch.replacement else { + unreachable!("filtered on Replacement::Summary above"); + }; + assert_eq!( + summary_family_algorithm(node), + SketchAlgorithm::DDSketch, + "the outer (target) node must be the DDSketch candidate" + ); + + let asap_types::post_asap::SummaryExpr::SummaryEstimate { summary_input, .. } = &node.expr + else { + panic!("expected SummaryEstimate root, got {:?}", node.expr); + }; + let asap_types::post_asap::SummaryExpr::SummaryAgg { child, .. } = &summary_input.expr + else { + panic!("expected SummaryAgg, got {:?}", summary_input.expr); + }; + assert_eq!( + summary_family_algorithm(child), + SketchAlgorithm::Kll, + "the nested inner aggregate must still get the cost-model-ranked \ + default (Kll), not inherit the outer target's DDSketch candidate" + ); + } + + /// The `SummaryFamilyType`'s committed `SketchAlgorithm`, from the top + /// `SummaryAgg` reachable under a (possibly `SummaryEstimate`-wrapped) + /// bound root. + fn summary_family_algorithm(node: &SummaryNode) -> SketchAlgorithm { + match &node.expr { + asap_types::post_asap::SummaryExpr::SummaryEstimate { summary_input, .. } => { + summary_family_algorithm(summary_input) + } + asap_types::post_asap::SummaryExpr::SummaryAgg { family, .. } => match family { + asap_types::post_asap::SummaryFamilyType::Sketch(kind) => kind.algorithm().clone(), + other => panic!("expected a Sketch family, got {other:?}"), + }, + other => panic!("expected SummaryAgg/SummaryEstimate, got {other:?}"), + } + } + + // ── SharedSubtreeStrategy ──────────────────────────────────────────── + + #[test] + fn does_not_match_a_single_consumer_target() { + let q = Rc::new(agg( + vec![2], + AggIntent::Sum { col: None }, + metric_scan(&["job"]), + )); + let target = TargetSubDAG::new(&q); + assert_eq!(target.consumer_count, 1); + assert!(!SharedSubtreeStrategy.matches(&target)); + assert!(SharedSubtreeStrategy.replacements(&target).is_empty()); + } + + #[test] + fn two_or_more_consumers_yields_the_share_vs_independent_pair() { + let q = Rc::new(agg( + vec![2], + AggIntent::Sum { col: None }, + metric_scan(&["job"]), + )); + let target = TargetSubDAG::with_consumer_count(&q, 2); + assert!(SharedSubtreeStrategy.matches(&target)); + + let replacements = SharedSubtreeStrategy.replacements(&target); + assert_eq!(replacements.len(), 2, "{replacements:?}"); + + let shared = match &replacements[0].replacement { + Replacement::Rewrite(rc) => rc, + other => panic!("expected a Rewrite replacement, got {other:?}"), + }; + assert!( + Rc::ptr_eq(shared, &q), + "the 'build once and share' candidate must be the same Rc as the target" + ); + assert!(replacements[0].rationale.contains("build once and share")); + + let independent = match &replacements[1].replacement { + Replacement::Rewrite(rc) => rc, + other => panic!("expected a Rewrite replacement, got {other:?}"), + }; + assert!( + !Rc::ptr_eq(independent, &q), + "the 'build independently' candidate must be a distinct Rc from the target" + ); + assert_eq!( + **independent, *q, + "the 'build independently' candidate must still be structurally identical" + ); + assert!(replacements[1].rationale.contains("build independently")); + } + + #[test] + fn three_consumers_are_reported_verbatim_in_both_rationales() { + let q = Rc::new(agg( + vec![2], + AggIntent::Sum { col: None }, + metric_scan(&["job"]), + )); + let target = TargetSubDAG::with_consumer_count(&q, 3); + let replacements = SharedSubtreeStrategy.replacements(&target); + assert!(replacements[0].rationale.contains('3')); + assert!(replacements[1].rationale.contains('3')); + } + + /// Builds realistic multi-consumer `TargetSubDAG`s the same way this + /// module's own [`discover_targets`]/`walk` does: dedup by `Rc::as_ptr`, + /// walking only the relational-skeleton operator children + /// `asap_types::pre_asap::cse::share_common_subtrees` itself scopes to, + /// so a shared node nested below another shared node is only ever + /// counted at the highest (maximal) point sharing starts. Test-only: + /// this module deliberately does not ship a workload-wide discovery + /// pass of its own (see the module docs' "Non-goals"). + fn count_consumers(roots: &[Rc]) -> HashMap<*const QueryExpr, usize> { + fn walk(node: &Rc, counts: &mut HashMap<*const QueryExpr, usize>) { + let ptr = Rc::as_ptr(node); + let already_visited = counts.contains_key(&ptr); + *counts.entry(ptr).or_insert(0) += 1; + if !already_visited { + walk_children(node, counts); + } + } + fn walk_children(node: &QueryExpr, counts: &mut HashMap<*const QueryExpr, usize>) { + use QueryExpr::*; + match node { + Scan { .. } | PromqlScalarBridge(_) | QueryTimestamp => {} + PromqlVectorFromScalar(c) | PromqlScalarFromVector(c) => walk(c, counts), + PromqlRelabel { child, .. } + | PromqlInfoEnrich { child, .. } + | PromqlSeriesSample { child, .. } + | Filter { child, .. } + | Project { child, .. } + | Aggregate { child, .. } + | Dedup { child, .. } + | PromqlSubquery { child, .. } + | TimeRange { child, .. } + | TimeShift { child, .. } + | SQLWindowFunc { child, .. } + | Sort { child, .. } + | Limit { child, .. } => walk(child, counts), + Concat { children } => { + for c in children { + walk_children(c, counts); + } + } + Join { left, right, .. } | SetOp { left, right, .. } => { + walk(left, counts); + walk(right, counts); + } + BinaryOp { lhs, rhs, .. } => { + walk(lhs, counts); + walk(rhs, counts); + } + Column(_) + | Literal(_) + | Compare { .. } + | BoolAnd(_) + | BoolOr(_) + | Not(_) + | IsNull(_) + | IsNotNull(_) + | Cast { .. } + | InList { .. } + | FunctionCall { .. } + | Arithmetic { .. } + | Case { .. } => {} + } + } + + let mut counts = HashMap::new(); + for root in roots { + walk(root, &mut counts); + } + counts + } + + #[test] + fn realistic_cse_output_produces_a_two_consumer_target() { + // Two workload roots that `share_common_subtrees` collapses onto one + // Rc (mirrors `explanation`'s and `cse`'s own fixtures): a grouped + // Sum aggregate over the same scan, built independently at each root. + let a = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let b = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let shared = asap_types::pre_asap::cse::share_common_subtrees(vec![("a", a), ("b", b)]); + let [(_, ra), (_, rb)] = shared.as_slice() else { + panic!("expected 2 roots"); + }; + assert!(Rc::ptr_eq(ra, rb), "fixture sanity: the two roots merged"); + + let roots: Vec> = shared.into_iter().map(|(_, rc)| rc).collect(); + let counts = count_consumers(&roots); + let count = counts[&Rc::as_ptr(&roots[0])]; + assert_eq!(count, 2); + + let target = TargetSubDAG::with_consumer_count(&roots[0], count); + assert!(SharedSubtreeStrategy.matches(&target)); + assert_eq!(SharedSubtreeStrategy.replacements(&target).len(), 2); + } + + // ── search_workload / PlanSpace / MemoGroup (merged from search.rs) ── + // + // Reuses this test module's own `metric_scan`/`agg` fixture helpers + // above (identical to `search.rs`'s own copies, which are dropped here + // to avoid a duplicate-definition collision now that both test modules + // share one file) and `count_consumers` above (which mirrors + // `discover_targets`' own real, non-test traversal for these fixtures). + + // ── discovery + MEMO shape ─────────────────────────────────────────── + + #[test] + fn single_bindable_aggregate_gets_a_group_with_every_sketch_candidate() { + let root = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); + let space = search_workload(vec![("q", root)]); + + // One group for the Aggregate, one for its Scan child. + assert_eq!(space.len(), 2); + + let agg_group = space + .groups() + .find(|g| matches!(g.target.as_ref(), QueryExpr::Aggregate { .. })) + .expect("an Aggregate group must be discovered"); + assert_eq!(agg_group.consumer_count, 1); + assert_eq!( + agg_group.candidates.len(), + 2, + "quantile has 2 summary_candidates entries: {:?}", + agg_group.candidates + ); + assert!(agg_group + .candidates + .iter() + .all(|c| matches!(c.replacement, Replacement::Summary(_)))); + + let scan_group = space + .groups() + .find(|g| matches!(g.target.as_ref(), QueryExpr::Scan { .. })) + .expect("a Scan group must be discovered"); + assert_eq!(scan_group.consumer_count, 1); + assert!( + scan_group.candidates.is_empty(), + "no strategy matches a bare Scan" + ); + } + + #[test] + fn cardinality_group_gets_all_three_candidates() { + let root = Rc::new(agg(vec![2], default_cardinality(), metric_scan(&["job"]))); + let space = search_workload(vec![("q", root)]); + let agg_group = space + .groups() + .find(|g| matches!(g.target.as_ref(), QueryExpr::Aggregate { .. })) + .unwrap(); + assert_eq!(agg_group.candidates.len(), 3); + } + + #[test] + fn shared_aggregate_across_two_roots_gets_both_strategies_candidates() { + // Two independently-built, structurally identical Sum aggregates: + // share_common_subtrees (run inside search_workload) collapses them + // onto one Rc with consumer_count 2, so this single group should + // carry SketchAlgorithmStrategy's one ExactAggregate candidate *and* + // SharedSubtreeStrategy's share-vs-recompute pair. + let a = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let b = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let space = search_workload(vec![("a", Rc::new(a)), ("b", Rc::new(b))]); + + // roots[0] and roots[1] must have merged onto the same Rc. + assert!(Rc::ptr_eq(&space.roots[0].1, &space.roots[1].1)); + + let group = space.group_for(&space.roots[0].1).unwrap(); + assert_eq!(group.consumer_count, 2); + assert_eq!( + group.candidates.len(), + 3, + "1 ExactAggregate Summary + 2 Rewrite (share/recompute): {:?}", + group.candidates + ); + + let summary_count = group + .candidates + .iter() + .filter(|c| matches!(c.replacement, Replacement::Summary(_))) + .count(); + let rewrite_count = group + .candidates + .iter() + .filter(|c| matches!(c.replacement, Replacement::Rewrite(_))) + .count(); + assert_eq!(summary_count, 1); + assert_eq!(rewrite_count, 2); + + // The two Rewrite candidates must NOT have collapsed into one + // (the "false-positive dedup" failure mode `is_duplicate_rewrite` + // exists to prevent). + let one_is_the_target = group.candidates.iter().any( + |c| matches!(&c.replacement, Replacement::Rewrite(rc) if Rc::ptr_eq(rc, &group.target)), + ); + let one_is_not = group.candidates.iter().any(|c| { + matches!(&c.replacement, Replacement::Rewrite(rc) if !Rc::ptr_eq(rc, &group.target)) + }); + assert!(one_is_the_target && one_is_not); + } + + #[test] + fn nested_shared_subtree_below_an_unshared_parent_is_still_discovered() { + // A shared grouped Aggregate nested under two *different*, + // unshared Filter parents — real consumer_count must come from + // walking the whole DAG, not just root-level pointer identity + // (a naive whole-root-only consumer-count pass would miss this; + // this module's discover_targets must not). + use asap_types::pre_asap::expr_ir::ScalarValue; + use asap_types::pre_asap::query_expr::Predicate; + + let shared = Rc::new(agg( + vec![2], + AggIntent::Sum { col: None }, + metric_scan(&["job"]), + )); + // Different predicates so the two Filter *parents* stay distinct + // (don't themselves merge under CSE) — only their shared `child` + // should collapse onto one `Rc`. + let root_a = QueryExpr::Filter { + pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Int64(1)))), + child: Rc::clone(&shared), + }; + let root_b = QueryExpr::Filter { + pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Int64(2)))), + child: Rc::clone(&shared), + }; + + let space = search_workload(vec![("a", Rc::new(root_a)), ("b", Rc::new(root_b))]); + assert_eq!( + space.len(), + 4, + "2 distinct Filters + 1 shared Aggregate + 1 shared Scan" + ); + + // `share_common_subtrees` re-clones+re-interns anything that already + // had more than one owner going in (see `cse.rs`'s own doc on + // `intern_child`'s clone-fallback path) — so the post-CSE shared + // node is a *fresh* Rc, structurally equal to (but not the same + // pointer as) the pre-search `shared` variable. Recover it from the + // post-CSE root's own `child` field instead of the stale `shared` + // handle. + let QueryExpr::Filter { + child: post_cse_shared_a, + .. + } = space.roots[0].1.as_ref() + else { + panic!("expected a Filter root"); + }; + let QueryExpr::Filter { + child: post_cse_shared_b, + .. + } = space.roots[1].1.as_ref() + else { + panic!("expected a Filter root"); + }; + assert!( + Rc::ptr_eq(post_cse_shared_a, post_cse_shared_b), + "fixture sanity: the two Filters' children must still merge" + ); + let post_cse_shared = post_cse_shared_a; + let group = space + .group_for(post_cse_shared) + .expect("shared node must be a discovered target"); + assert_eq!(group.consumer_count, 2); + assert!( + SharedSubtreeStrategy.matches(&TargetSubDAG::with_consumer_count( + post_cse_shared, + group.consumer_count + )) + ); + } + + // ── dedup ──────────────────────────────────────────────────────────── + + #[test] + fn add_candidate_rejects_a_true_rewrite_duplicate() { + // SharedSubtreeStrategy's `Replacement::Rewrite` candidates are + // real `QueryExpr` values with `PartialEq`, so `add_candidate` can + // (and must) actually reject a genuine repeat — unlike the + // `Replacement::Summary` case (see the test below). + let root = Rc::new(agg( + vec![2], + AggIntent::Sum { col: None }, + metric_scan(&["job"]), + )); + let mut group = MemoGroup::new(Rc::clone(&root), 2); + let target = TargetSubDAG::with_consumer_count(&root, 2); + let mut inserted = 0; + for candidate in SharedSubtreeStrategy.replacements(&target) { + if group.add_candidate(candidate) { + inserted += 1; + } + } + assert_eq!(inserted, 2, "share + recompute-independently candidates"); + + // Re-adding the identical candidate list must add nothing new: the + // "share" candidate is literally the same Rc as before, and the + // "recompute independently" candidate is a fresh Rc but + // structurally identical value, both already covered by + // `is_duplicate_rewrite`. + let mut re_inserted = 0; + for candidate in SharedSubtreeStrategy.replacements(&target) { + if group.add_candidate(candidate) { + re_inserted += 1; + } + } + assert_eq!( + re_inserted, 0, + "re-proposing the same Rewrite candidates must not grow the group" + ); + assert_eq!(group.candidates.len(), 2); + } + + #[test] + fn add_candidate_never_dedups_summary_candidates() { + // Documented, deliberate consequence of `SummaryNode` deriving no + // `PartialEq` (see `is_duplicate_summary`'s own doc): re-proposing + // the same `Replacement::Summary` candidates DOES grow the group — + // this module refuses to guess at an equality check it can't back + // with a real `PartialEq`. `search_workload_with` never actually + // does this in practice (every target is asked exactly once — see the + // module docs' "Termination" section), so this test exists to pin + // the documented behavior, not to endorse calling `replacements` + // twice for the same target. + let root = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); + let mut group = MemoGroup::new(Rc::clone(&root), 1); + let strategy = SketchAlgorithmStrategy::default_cost_model(); + let target = TargetSubDAG::new(&root); + for candidate in strategy.replacements(&target) { + group.add_candidate(candidate); + } + assert_eq!(group.candidates.len(), 2); + + for candidate in strategy.replacements(&target) { + group.add_candidate(candidate); + } + assert_eq!( + group.candidates.len(), + 4, + "Summary candidates are never deduped by this module — see is_duplicate_summary" + ); + } + + #[test] + fn is_duplicate_rewrite_never_merges_share_with_recompute() { + let target = Rc::new(agg( + vec![2], + AggIntent::Sum { col: None }, + metric_scan(&["job"]), + )); + let share = Rc::clone(&target); + let recompute = Rc::new((*target).clone()); + assert!(!Rc::ptr_eq(&share, &recompute)); + assert_eq!( + *share, *recompute, + "fixture sanity: same value, different Rc" + ); + assert!(!is_duplicate_rewrite(&share, &recompute, &target)); + assert!(!is_duplicate_rewrite(&recompute, &share, &target)); + } + + #[test] + fn is_duplicate_rewrite_catches_a_real_repeat() { + let target = Rc::new(agg( + vec![2], + AggIntent::Sum { col: None }, + metric_scan(&["job"]), + )); + let first_recompute = Rc::new((*target).clone()); + let second_recompute = Rc::new((*target).clone()); + assert!(!Rc::ptr_eq(&first_recompute, &second_recompute)); + assert!(is_duplicate_rewrite( + &first_recompute, + &second_recompute, + &target + )); + } + + // ── cost-based ranking ─────────────────────────────────────────────── + + #[test] + fn cost_sorted_orders_shared_subtree_candidates_by_cse_share_decision() { + // Many consumers of a cheap-to-recompute, cheap-to-maintain exact + // accumulator: cse_share_decision should prefer Share (see + // cost_model.rs's own `cse_share_decision_shares_when_recompute_dominates_maintenance`). + let mut roots = Vec::new(); + let shared = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + for i in 0..20 { + roots.push((i, Rc::new(shared.clone()))); + } + let space = search_workload(roots); + let group = space.group_for(&space.roots[0].1).unwrap(); + assert_eq!(group.consumer_count, 20); + + let ranked = space.cost_sorted(&DefaultCostModel); + let ranked_group = ranked + .iter() + .find(|g| Rc::ptr_eq(g.target, &space.roots[0].1)) + .unwrap(); + assert!(matches!( + &ranked_group.candidates[0].replacement, + Replacement::Rewrite(rc) if Rc::ptr_eq(rc, &group.target) + )); + let rewrites: Vec<&ReplacementSubDAG> = ranked_group + .candidates + .iter() + .filter(|c| matches!(c.replacement, Replacement::Rewrite(_))) + .copied() + .collect(); + assert_eq!(rewrites.len(), 2); + let first_shares_target = match &rewrites[0].replacement { + Replacement::Rewrite(rc) => Rc::ptr_eq(rc, &group.target), + Replacement::Summary(_) => false, + }; + assert!( + first_shares_target, + "with 20 cheap consumers, Share should rank first: {rewrites:?}" + ); + } + + #[test] + fn cost_sorted_orders_sketch_candidates_by_rank_candidates() { + struct PreferDDSketch; + impl CostModel for PreferDDSketch { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + let mut v = candidates.to_vec(); + if let Some(pos) = v.iter().position(|k| *k == SketchAlgorithm::DDSketch) { + let dd = v.remove(pos); + v.insert(0, dd); + } + v + } + } + + let root = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); + let space = search_workload(vec![("q", root)]); + let ranked = space.cost_sorted(&PreferDDSketch); + let agg_group = ranked + .iter() + .find(|g| matches!(g.target.as_ref(), QueryExpr::Aggregate { .. })) + .unwrap(); + assert_eq!(agg_group.candidates.len(), 2); + let first_kind = match &agg_group.candidates[0].replacement { + Replacement::Summary(node) => sketch_kind_of(node), + Replacement::Rewrite(_) => None, + }; + assert_eq!(first_kind, Some(SketchAlgorithm::DDSketch)); + } + + /// [`RankedGroup::costs`] is a per-candidate annotation, aligned + /// index-for-index with `candidates` — each entry must equal what + /// calling [`CostModel::estimate_cost`] directly on that same candidate + /// and target produces, not some other (or stale) number. + #[test] + fn cost_sorted_pairs_each_candidate_with_its_own_estimate_cost() { + let root = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); + let space = search_workload(vec![("q", root)]); + let ranked = space.cost_sorted(&DefaultCostModel); + let agg_group = ranked + .iter() + .find(|g| matches!(g.target.as_ref(), QueryExpr::Aggregate { .. })) + .unwrap(); + assert_eq!( + agg_group.costs.len(), + agg_group.candidates.len(), + "costs must be aligned 1:1 with candidates" + ); + assert!(!agg_group.costs.is_empty()); + + let target = TargetSubDAG::with_consumer_count(agg_group.target, agg_group.consumer_count); + for (candidate, &cost) in agg_group.candidates.iter().zip(&agg_group.costs) { + assert_eq!( + cost, + DefaultCostModel.estimate_cost(candidate, &target), + "RankedGroup::costs must match calling CostModel::estimate_cost directly \ + for the same candidate/target" + ); + } + } + + // ── termination ────────────────────────────────────────────────────── + + #[test] + fn default_strategies_converge_without_hitting_the_iteration_cap() { + // A workload exercising both strategies at once; if this test + // completes at all, the fixpoint converged well under + // MAX_SEARCH_ITERATIONS (both strategies are idempotent — see the + // module docs — so this always converges in exactly 2 passes). + let a = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + let b = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + let space = search_workload(vec![("a", Rc::new(a)), ("b", Rc::new(b))]); + assert!(!space.is_empty()); + } + + /// A deliberately ill-behaved [`ReplacementStrategy`]: every call to + /// `replacements` wraps `target` in two `Filter` layers — the outer one + /// (ignored by target discovery — see [`discover_new_descendant_targets`]) + /// and an inner one carrying a monotonically-increasing counter, so the + /// inner layer is a **brand-new, never-before-seen `Rc` every call**. + /// Each round, `search_workload_with` discovers that inner layer as a + /// new target, processes it next round (this strategy matches + /// everything), and gets handed *another* fresh inner layer — the + /// frontier never empties, exactly the failure mode + /// [`MAX_SEARCH_ITERATIONS`] exists to catch. + struct AlwaysGrowingStrategy { + next: std::cell::Cell, + } + + impl ReplacementStrategy for AlwaysGrowingStrategy { + fn matches(&self, _target: &TargetSubDAG<'_>) -> bool { + true + } + + fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { + let n = self.next.get(); + self.next.set(n + 1); + use asap_types::pre_asap::expr_ir::ScalarValue; + use asap_types::pre_asap::query_expr::Predicate; + let fresh_inner_layer = QueryExpr::Filter { + pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Int64(n)))), + child: Rc::clone(target.root), + }; + let outer_wrapper = QueryExpr::Filter { + pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Boolean(true)))), + child: Rc::new(fresh_inner_layer), + }; + vec![ReplacementSubDAG { + replacement: Replacement::Rewrite(Rc::new(outer_wrapper)), + rationale: format!("pathological candidate #{n}"), + }] + } + } + + #[test] + #[should_panic(expected = "did not converge")] + fn a_pathologically_growing_strategy_trips_the_iteration_cap() { + let root = Rc::new(metric_scan(&["job"])); + let strategies: Vec> = vec![Box::new(AlwaysGrowingStrategy { + next: std::cell::Cell::new(0), + })]; + let _ = search_workload_with(vec![("q", root)], &strategies); + } + // ── realize_child / keep_pre_asap: end-to-end single-target realization ── + // + // Moved from the former `bind.rs` (issue #251): `bind.rs`'s own + // workload-wide orchestration (`implement_workload`/ + // `implement_workload_with`) was deleted as out of this crate's scope + // (see the crate doc's `## Status` section), but these tests exercise + // `construct_summary_agg`'s schema derivation end to end through + // `realize_child` — production logic that still lives in this module — + // so they move here rather than disappear. Unlike `bind.rs` (an + // external caller that had to reconstruct the rank-and-take-first + // pattern by hand since `realize_child` is `pub(crate)`), these tests + // call `realize_child` directly. + + fn agg_per_entity(intent: AggIntent, child: QueryExpr) -> QueryExpr { + QueryExpr::Aggregate { + reduction: ReductionTy::PerEntity, + measures: vec![intent], + output_names: vec![], + having: None, + child: Rc::new(child), + } + } + + fn field<'a>(schema: &'a SummarySchema, name: &str) -> &'a SummaryField { + schema + .fields + .iter() + .find(|f| f.name == name) + .unwrap_or_else(|| panic!("no field {name:?} in {schema:?}")) + } + + fn realize_first( + expr: &QueryExpr, + cost_model: &dyn CostModel, + ) -> Result, ImplementError> { + realize_child(&Rc::new(expr.clone()), cost_model) + } + + fn realize(expr: &QueryExpr) -> Result, ImplementError> { + realize_first(expr, &DefaultCostModel) + } + + #[test] + fn quantile_realizes_kll_wrapped_in_estimate() { + // quantile by (job) (m) at ε=0.01 → Estimate(Quantile) over + // SummaryAgg(Kll{k:200}) over KeepPreAsap(Scan). job = col 2. + let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + let root = realize(&q).unwrap(); + + let SummaryExpr::SummaryEstimate { + summary_input, + query, + } = &root.expr + else { + panic!("expected SummaryEstimate root, got {:?}", root.expr); + }; + assert!(matches!(query, PostAsapSketchQuery::Quantile { q } if *q == 0.99)); + // Estimate edge: plain row shape — group key + Float64 answer. + assert_eq!( + field(&root.schema, "quantile_0_99").dtype, + SummaryFamilyType::Plain(DataType::Float64) + ); + assert_eq!( + field(&root.schema, "job").dtype, + SummaryFamilyType::Plain(DataType::Utf8) + ); + + let SummaryExpr::SummaryAgg { + child, + family, + col, + reduction, + } = &summary_input.expr + else { + panic!("expected SummaryAgg, got {:?}", summary_input.expr); + }; + assert_eq!( + family, + &SummaryFamilyType::Sketch(SketchKind::new( + SketchAlgorithm::Kll, + SketchParams::Kll { k: 200 } + )) + ); + assert_eq!(col, &ColumnRef::SampleValue); + assert_eq!(reduction, &ReductionTy::by(vec![2])); + // SummaryAgg edge: the state column carries the committed family. + assert_eq!( + field(&summary_input.schema, "quantile_0_99").dtype, + SummaryFamilyType::Sketch(SketchKind::new( + SketchAlgorithm::Kll, + SketchParams::Kll { k: 200 } + )) + ); + assert!(matches!(child.expr, SummaryExpr::KeepPreAsap(ref e) + if matches!(**e, QueryExpr::Scan { .. }))); + } + + /// A deployment-supplied [`CostModel`] can override the default KLL + /// choice — `realize_first` (via `realize_child`) must actually consult + /// it, not just accept and ignore it (issue: cost model interface, see + /// `crate::cost_model`). + struct PreferDDSketchViaCostModel; + + impl CostModel for PreferDDSketchViaCostModel { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + let mut v = candidates.to_vec(); + if let Some(pos) = v.iter().position(|k| *k == SketchAlgorithm::DDSketch) { + let ddsketch = v.remove(pos); + v.insert(0, ddsketch); + } + v + } + } + + #[test] + fn realize_with_custom_cost_model_overrides_default_summary_choice() { + let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + + // Default: KLL (see `quantile_realizes_kll_wrapped_in_estimate` above). + let default_root = realize(&q).unwrap(); + let SummaryExpr::SummaryEstimate { summary_input, .. } = &default_root.expr else { + panic!("expected SummaryEstimate root, got {:?}", default_root.expr); + }; + let SummaryExpr::SummaryAgg { family, .. } = &summary_input.expr else { + panic!("expected SummaryAgg, got {:?}", summary_input.expr); + }; + assert!(matches!( + family, + SummaryFamilyType::Sketch(kind) if kind.algorithm() == &SketchAlgorithm::Kll + )); + + // With `PreferDDSketchViaCostModel`: DDSketch instead, same query. + let custom_root = realize_first(&q, &PreferDDSketchViaCostModel).unwrap(); + let SummaryExpr::SummaryEstimate { summary_input, .. } = &custom_root.expr else { + panic!("expected SummaryEstimate root, got {:?}", custom_root.expr); + }; + let SummaryExpr::SummaryAgg { family, .. } = &summary_input.expr else { + panic!("expected SummaryAgg, got {:?}", summary_input.expr); + }; + assert_eq!( + family, + &SummaryFamilyType::Sketch(SketchKind::new( + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { alpha: 0.01 } + )) + ); + } + + /// A deployment-supplied `CostModel` can realize an `AggIntent::Extension` + /// intent as a real sketch instead of the default `PassThrough` (issue + /// #150) — `implementations_for_with` must consult `realize_extension` + /// for the `Extension` arm, and `readout` must consult + /// `readout_extension` to build its `SketchQuery` without panicking. + struct FrequencyCostModel; + + impl CostModel for FrequencyCostModel { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + candidates.to_vec() + } + + fn realize_extension( + &self, + ext_kind: &str, + _payload: &serde_json::Value, + ) -> Implementation { + if ext_kind == "frequency" { + Implementation::Sketch(SketchKind::new( + SketchAlgorithm::CountSketch, + SketchParams::CountSketch { + width: 256, + depth: 4, + }, + )) + } else { + Implementation::PassThrough + } + } + + fn readout_extension( + &self, + ext_kind: &str, + payload: &serde_json::Value, + _col: &ColumnRef, + ) -> PostAsapSketchQuery { + assert_eq!(ext_kind, "frequency"); + let value = payload["item"].as_str().map(str::to_string); + PostAsapSketchQuery::PointCount { + key: ColumnRef::Named("item".into()), + value, + } + } + } + + #[test] + fn extension_intent_stays_logical_by_default() { + // Without a CostModel overriding `realize_extension`, an + // `Extension` intent must stay `PassThrough` -- today's behavior, + // unchanged. + let intent = AggIntent::Extension { + ext_kind: "frequency".to_string(), + payload: serde_json::json!({ "item": "checkout" }), + }; + let q = agg(vec![], intent, metric_scan(&[])); + let root = realize(&q).unwrap(); + assert!(matches!(root.expr, SummaryExpr::KeepPreAsap(_))); + } + + #[test] + fn extension_intent_realizes_via_custom_cost_model() { + let intent = AggIntent::Extension { + ext_kind: "frequency".to_string(), + payload: serde_json::json!({ "item": "checkout" }), + }; + let q = agg(vec![], intent, metric_scan(&[])); + let root = realize_first(&q, &FrequencyCostModel).unwrap(); + + let SummaryExpr::SummaryEstimate { + summary_input, + query, + } = &root.expr + else { + panic!("expected SummaryEstimate root, got {:?}", root.expr); + }; + assert!(matches!( + query, + PostAsapSketchQuery::PointCount { key: ColumnRef::Named(k), value: Some(v) } + if k == "item" && v == "checkout" + )); + + let SummaryExpr::SummaryAgg { family, .. } = &summary_input.expr else { + panic!("expected SummaryAgg, got {:?}", summary_input.expr); + }; + assert_eq!( + family, + &SummaryFamilyType::Sketch(SketchKind::new( + SketchAlgorithm::CountSketch, + SketchParams::CountSketch { + width: 256, + depth: 4 + } + )) + ); + } + + #[test] + fn exact_sum_realizes_accumulator_without_estimate() { + let q = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let root = realize(&q).unwrap(); + let SummaryExpr::SummaryAgg { family, .. } = &root.expr else { + panic!( + "expected bare SummaryAgg (no estimate), got {:?}", + root.expr + ); + }; + assert_eq!( + family, + &SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) + ); + assert_eq!( + field(&root.schema, "sum").dtype, + SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) + ); + } + + #[test] + fn per_series_rate_keeps_labels_and_retypes_value() { + // rate(m[5m]) — per-series: every label survives; the sample value + // column becomes the Rate accumulator state. + use std::time::Duration; + let q = agg_per_entity( + AggIntent::Rate, + QueryExpr::TimeRange { + range: Duration::from_secs(300), + child: Rc::new(metric_scan(&["job"])), + }, + ); + let root = realize(&q).unwrap(); + let SummaryExpr::SummaryAgg { family, .. } = &root.expr else { + panic!("expected SummaryAgg, got {:?}", root.expr); + }; + assert_eq!( + family, + &SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) + ); + assert_eq!( + root.schema + .fields + .iter() + .map(|f| f.name.as_str()) + .collect::>(), + vec!["ts", "value", "job"], + ); + assert_eq!( + field(&root.schema, "value").dtype, + SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) + ); + assert_eq!(root.schema.time_index, Some(0)); + } + + /// Issue #163, case 1: a bare per-series range function (e.g. + /// `quantile_over_time(...)`) realizes to `SummaryAgg { reduction: + /// PerEntity, .. }` — proving the pre-ASAP `Reduction` this crate + /// already computes (issue #165) is carried onto the post-ASAP node + /// verbatim, not flattened back into an ambiguous bare `Vec`. + #[test] + fn bare_per_series_aggregate_realizes_summary_agg_with_per_entity_reduction() { + use std::time::Duration; + let q = agg_per_entity( + default_quantile(0.99), + QueryExpr::TimeRange { + range: Duration::from_secs(10), + child: Rc::new(metric_scan(&["job"])), + }, + ); + let root = realize(&q).unwrap(); + let SummaryExpr::SummaryEstimate { summary_input, .. } = &root.expr else { + panic!("expected estimate root, got {:?}", root.expr); + }; + let SummaryExpr::SummaryAgg { reduction, .. } = &summary_input.expr else { + panic!("expected SummaryAgg, got {:?}", summary_input.expr); + }; + assert_eq!(reduction, &ReductionTy::PerEntity); + } + + /// Issue #163, case 2: an aggregation operator explicitly invoked with + /// no `by(...)` (e.g. `count(hll_metric)`) realizes to `SummaryAgg { + /// reduction: Reduce(vec![]), .. }` — byte-identical `by: []` to the + /// previous test at the old `Vec` shape; `reduction` is what + /// tells them apart now. + #[test] + fn explicit_empty_by_aggregate_realizes_summary_agg_with_reduce_reduction() { + let intent = AggIntent::Cardinality { + col: None, + accuracy: AccuracyTarget::Epsilon(0.01), + }; + let q = agg(vec![], intent, metric_scan(&["job"])); + let root = realize(&q).unwrap(); + let SummaryExpr::SummaryEstimate { summary_input, .. } = &root.expr else { + panic!("expected estimate root, got {:?}", root.expr); + }; + let SummaryExpr::SummaryAgg { reduction, .. } = &summary_input.expr else { + panic!("expected SummaryAgg, got {:?}", summary_input.expr); + }; + assert_eq!(reduction, &ReductionTy::by(vec![])); + } + + #[test] + fn nested_aggregates_realize_per_node() { + // quantile(0.9, sum by (job) (m)) — the implementation decision + // fires per node over the nested tree: KLL over an exact Sum + // accumulator. + let inner = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let outer = agg(vec![], default_quantile(0.9), inner); + let root = realize(&outer).unwrap(); + + let SummaryExpr::SummaryEstimate { summary_input, .. } = &root.expr else { + panic!("expected estimate root, got {:?}", root.expr); + }; + let SummaryExpr::SummaryAgg { child, family, .. } = &summary_input.expr else { + panic!("expected outer SummaryAgg, got {:?}", summary_input.expr); + }; + assert!(matches!( + family, + SummaryFamilyType::Sketch(kind) if kind.algorithm() == &SketchAlgorithm::Kll + )); + let SummaryExpr::SummaryAgg { + family: inner_family, + child: leaf, + .. + } = &child.expr + else { + panic!("expected inner SummaryAgg, got {:?}", child.expr); + }; + assert_eq!( + inner_family, + &SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) + ); + assert!(matches!(leaf.expr, SummaryExpr::KeepPreAsap(_))); + } + + /// Issue #115: the summary is built over the intent's own input column. + /// Before `Cardinality`/`Quantile` carried `col`, `summarised_column` always + /// fell through to `ColumnRef::SampleValue`, so an HLL was built over the + /// wrong column for every SQL `COUNT(DISTINCT c)`. + #[test] + fn sketch_realizes_over_the_intents_input_column() { + // `metric_scan(&["job"])` → columns [ts=0, value=1, job=2]. + let cases = [ + (Some(2), ColumnRef::Named("job".into())), + (Some(1), ColumnRef::Named("value".into())), + // PromQL convention: no column ⇒ the synthetic sample value. + (None, ColumnRef::SampleValue), + ]; + for (col, want) in cases { + let intent = AggIntent::Cardinality { + col, + accuracy: AccuracyTarget::Epsilon(0.01), + }; + let root = realize(&agg(vec![0], intent, metric_scan(&["job"]))).unwrap(); + let bound = find_summary_col(&root) + .unwrap_or_else(|| panic!("expected a SummaryAgg for col={col:?}")); + assert_eq!(bound, want, "wrong summarised column for col={col:?}"); + } + } + + /// The `col` of the first `SummaryAgg` in the tree. + fn find_summary_col(node: &SummaryNode) -> Option { + match &node.expr { + SummaryExpr::SummaryAgg { col, .. } => Some(col.clone()), + SummaryExpr::SummaryEstimate { summary_input, .. } => find_summary_col(summary_input), + _ => None, + } + } + + #[test] + fn pass_through_intents_stay_logical() { + // avg is exact but non-mergeable; histogram_quantile (classic + // buckets, #79) is never sketchable; exact quantile is exact by + // decree. All three stay whole logical subtrees. + for intent in [ + AggIntent::Avg { col: None }, + AggIntent::HistogramQuantile { q: 0.99 }, + AggIntent::Quantile { + col: None, + q: 0.99, + accuracy: AccuracyTarget::Exact, + }, + ] { + let q = agg(vec![2], intent.clone(), metric_scan(&["job"])); + let root = realize(&q).unwrap(); + assert!( + matches!(root.expr, SummaryExpr::KeepPreAsap(ref e) if **e == q), + "expected KeepPreAsap passthrough for {intent:?}" + ); + } + } + + #[test] + fn logical_parent_subsumes_bindable_child() { + // Filter over a bindable quantile: `KeepPreAsap` has no post-ASAP + // children, so the conservative fallback keeps the whole subtree + // logical. + use asap_types::pre_asap::expr_ir::{CompareOpKind, ScalarValue}; + use asap_types::pre_asap::query_expr::Predicate; + let q = QueryExpr::Filter { + pred: Predicate(Rc::new(QueryExpr::Compare { + left: Rc::new(QueryExpr::Column(0)), + op: CompareOpKind::Gt, + right: Rc::new(QueryExpr::Literal(ScalarValue::Float64(0.5))), + })), + child: Rc::new(agg(vec![], default_quantile(0.99), metric_scan(&[]))), + }; + let root = realize(&q).unwrap(); + assert!(matches!(root.expr, SummaryExpr::KeepPreAsap(ref e) if **e == q)); + } + + #[test] + fn having_and_multi_intent_stay_logical() { + use asap_types::pre_asap::expr_ir::ScalarValue; + use asap_types::pre_asap::query_expr::Predicate; + let mut q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + if let QueryExpr::Aggregate { having, .. } = &mut q { + *having = Some(Predicate(Rc::new(QueryExpr::Literal( + ScalarValue::Boolean(true), + )))); + } + assert!(matches!( + realize(&q).unwrap().expr, + SummaryExpr::KeepPreAsap(_) + )); + + let multi = QueryExpr::Aggregate { + reduction: ReductionTy::by(vec![2]), + measures: vec![AggIntent::Sum { col: None }, AggIntent::Avg { col: None }], + output_names: vec![], + having: None, + child: Rc::new(metric_scan(&["job"])), + }; + assert!(matches!( + realize(&multi).unwrap().expr, + SummaryExpr::KeepPreAsap(_) + )); + } + + #[test] + fn topk_realizes_cms_with_heap_and_topk_readout() { + let q = agg( + vec![2], + AggIntent::TopK { + k: 5, + accuracy: AccuracyTarget::Epsilon(0.01), + }, + metric_scan(&["job"]), + ); + let root = realize(&q).unwrap(); + let SummaryExpr::SummaryEstimate { + summary_input, + query, + } = &root.expr + else { + panic!("expected estimate root, got {:?}", root.expr); + }; + assert!(matches!(query, PostAsapSketchQuery::TopK { k: 5 })); + assert!(matches!( + &summary_input.expr, + SummaryExpr::SummaryAgg { + family: SummaryFamilyType::Sketch(kind), + .. + } if kind.algorithm() == &SketchAlgorithm::CmsWithHeap + )); + } + + #[test] + fn sql_reducer_resolves_named_input_column() { + // SUM(bytes) over a tabular scan: `col` resolves positionally to the + // named column, not the PromQL sample value. + let scan = QueryExpr::Scan { + source: Source::Table { + table_ref: "t".into(), + }, + predicates: vec![], + schema: SchemaTy { + columns: vec![ + Column::new("host", DataType::Utf8, false), + Column::new("bytes", DataType::Int64, false), + ], + time_index: None, + unique_keys: vec![], + closed: true, + }, + }; + let q = agg(vec![0], AggIntent::Sum { col: Some(1) }, scan); + let root = realize(&q).unwrap(); + let SummaryExpr::SummaryAgg { col, .. } = &root.expr else { + panic!("expected SummaryAgg, got {:?}", root.expr); + }; + assert_eq!(col, &ColumnRef::Named("bytes".into())); + } +} diff --git a/crates/devtools/src/bin/dag_export.rs b/crates/devtools/src/bin/dag_export.rs index 7c71eb68..591773c5 100644 --- a/crates/devtools/src/bin/dag_export.rs +++ b/crates/devtools/src/bin/dag_export.rs @@ -8,12 +8,22 @@ // cargo run -p asap-lower --bin dag_export -- --sql "..." --name q1 > /tmp/dag.json // // `--name` is optional; an unnamed query defaults to `q` (1-indexed). +// +// `--epsilon ` is optional and applies to every query in the run: it +// lowers with `AccuracyTarget::Epsilon()` instead of the default +// `AccuracyTarget::Exact`. Without it, every `AggIntent` lowers exact and +// `asap_aware_mapping::SketchAlgorithmStrategy` never has a genuine sketch +// alternative to report — so no node ever picks up a `SketchApproximation` +// note. Pass it to actually exercise that path, e.g.: +// cargo run -p asap-lower --bin dag_export -- \ +// --epsilon 0.01 --sql "SELECT quantile(0.99, latency) FROM metrics" --name p99 -use asap_devtools::{lower_promql, lower_sql, SqlCatalog}; -use asap_types::dag_export::{self, NamedGraph, WorkloadGraph}; +use asap_types::dag_export::{self, DagGraph, DagNote, NamedGraph, WorkloadGraph}; use asap_types::pre_asap::schema::{Column, DataType, Schema}; use asap_types::types::AccuracyTarget; +use asap_devtools::{lower_promql, lower_sql, SqlCatalog}; + enum Lang { Sql, PromQl, @@ -45,11 +55,13 @@ fn catalog() -> SqlCatalog { } /// Parses `--sql "" --name "