From 44036ef778e341a3a12762c9fca6fd723e1f0eeb Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 22 Aug 2026 19:08:22 -0600 Subject: [PATCH 01/49] feat(asap-aware-mapping): TargetSubDAG/ReplacementSubDAG/ReplacementStrategy (#251) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the `TargetSubDAG`/`ReplacementSubDAG`/`ReplacementStrategy` vocabulary `docs/asap_aware_mapping.md` stubs out under "Key concepts (not yet implemented)", scoped to the decision this crate already owns (summary-family/kind selection and CSE-sharing), not a new invention. `boundary::implementation_for_with` and `bind::implement_tree_with` each commit to exactly one answer per decision point, ranked via a CostModel — right for binding a query, but it throws away every alternative a cost model didn't pick. A future search/optimization engine needs the opposite shape: every semantically valid alternative, so a later cost-based search can explore and compare them. This adds that shape alongside the existing single-pick functions, wrapping rather than replacing them: - `TargetSubDAG` — a reference (by `Rc`, not a bare `&QueryExpr`) to a pre-ASAP node that's a replacement candidate, plus how many workload locations already reference it (`consumer_count`). - `ReplacementSubDAG` — one candidate replacement: a `Replacement` (`Summary(Rc)` for a binding decision, or `Rewrite(Rc)` for a pre-ASAP structural alternative) plus a human-readable rationale. - `ReplacementStrategy` — `matches`/`replacements`, exhaustive and not ranked or filtered; picking the best candidate stays a CostModel's job. Two real strategies, ported from existing decision logic with no new decision procedure: - `SketchFamilyStrategy` wraps `boundary::implementation_for_with`/ `summary_candidates`'s exhaustive match: for a bindable `Aggregate` (the same single-intent, no-`HAVING` shape `bind::implement_tree_with` requires), returns every candidate summary realization — every `SketchKind` `summary_candidates` lists when the boundary decision is approximate, built via a `ForceSketchKind` `CostModel` adapter that steers `implement_tree_with`'s own decision procedure towards each candidate in turn rather than reimplementing any part of it; the single exact-accumulator/pass-through outcome when that's the only option. - `SharedSubtreeStrategy` wraps `share_common_subtrees`'s sharing decision as an explicit build-independently-vs-build-once-and-share candidate pair, wherever a `TargetSubDAG` already has two or more consumers. `boundary.rs`/`bind.rs`/`applicability.rs`'s existing single-pick public behavior is unchanged; this only adds the exhaustive-candidate view alongside it. No search/selection logic is added — that's a separate Cascades/Volcano-style engine, tracked in #33's other sub-issues, built on top of this trait. `cargo build --workspace --all-targets`, `cargo test --workspace`, `cargo fmt --all -- --check`, and `cargo clippy --workspace --all-targets --all-features -- -D warnings` all pass clean. Co-Authored-By: Claude Sonnet 5 --- crates/asap-aware-mapping/src/lib.rs | 21 +- crates/asap-aware-mapping/src/replacement.rs | 914 +++++++++++++++++++ 2 files changed, 934 insertions(+), 1 deletion(-) create mode 100644 crates/asap-aware-mapping/src/replacement.rs diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index b428c2ed..b1a3fa2e 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -30,8 +30,22 @@ //! //! ## Status //! -//! Two real occupants and one stub: +//! Three real occupants and one stub: //! +//! - [`replacement`] — the `TargetSubDAG`/`ReplacementSubDAG`/ +//! `ReplacementStrategy` vocabulary `docs/asap_aware_mapping.md` stubs out +//! under "Key concepts (not yet implemented)", implemented for real (issue +//! #251, part of #33): a small extension-point trait reporting *every* +//! semantically valid replacement for a target sub-DAG, not just the one +//! [`boundary::implementation_for_with`]/[`bind::implement_tree_with`] +//! commit to. Two real strategies wrap those exact decision points instead +//! of re-deciding anything: +//! [`replacement::SketchFamilyStrategy`] (every candidate summary family +//! for a bindable `Aggregate`) and [`replacement::SharedSubtreeStrategy`] +//! (the build-independently-vs-build-once-and-share candidate pair for a +//! CSE-detected shared subtree). No search/ranking logic lives here — see +//! that module's docs for what's deliberately left to a future +//! Cascades/Volcano-style search engine. //! - [`boundary`] — the per-intent sketch-vs-exact (accuracy) decision: //! `AggIntent → Implementation` (a summary family's own `(Kind, Params)`, //! or an exact accumulator) sized to the `AccuracyTarget` (issue #98). @@ -90,6 +104,7 @@ pub mod bind; pub mod boundary; pub mod cost_model; +pub mod replacement; pub use bind::{ implement_tree, implement_tree_with, implement_workload, implement_workload_with, @@ -99,3 +114,7 @@ pub use boundary::{ implementation_for, implementation_for_with, summary_candidates, Implementation, Matcher, }; pub use cost_model::{CostModel, DefaultCostModel}; +pub use replacement::{ + Replacement, ReplacementStrategy, ReplacementSubDAG, SharedSubtreeStrategy, + SketchFamilyStrategy, TargetSubDAG, +}; diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs new file mode 100644 index 00000000..97cb8309 --- /dev/null +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -0,0 +1,914 @@ +//! `TargetSubDAG` / `ReplacementSubDAG` / `ReplacementStrategy` — the +//! candidate-replacement vocabulary `docs/asap_aware_mapping.md` stubs out +//! under "Key concepts (not yet implemented)", implemented for real (issue +//! #251, part of #33). +//! +//! ## Why this exists alongside [`boundary`] and [`bind`], not instead of them +//! +//! [`boundary::implementation_for_with`] and +//! [`bind::implement_tree_with`] each commit to exactly **one** answer per +//! decision point — a single [`Implementation`], a single bound +//! [`SummaryNode`] — ranked via a [`CostModel`]. That is exactly right for +//! *binding* a query (something has to actually run), but it means the +//! alternatives a cost model didn't pick are thrown away the moment they're +//! computed. A search/optimization engine (Cascades/Volcano-style, tracked +//! separately — see `docs/asap_aware_mapping.md`'s "Pseudocode for +//! Replacement Plan Searching") needs the *opposite* shape: every +//! semantically valid alternative for a sub-DAG, so a later cost-based search +//! can explore and compare them instead of being stuck with whatever +//! [`boundary`]/[`bind`] already locked in. +//! +//! This module is that alternative shape, built by **wrapping** the existing +//! decision points rather than re-deciding anything: +//! +//! - [`TargetSubDAG`] — a reference to a pre-ASAP [`QueryExpr`] node that is a +//! candidate for replacement, plus how many places in the workload already +//! reference it (its `consumer_count`) — the one piece of cross-node +//! context [`SharedSubtreeStrategy`] needs that a bare node reference alone +//! doesn't carry. +//! - [`ReplacementSubDAG`] — one candidate replacement for a `TargetSubDAG`: +//! either a fully bound [`SummaryNode`] (the same output +//! [`bind::implement_tree_with`] produces, for one particular candidate +//! instead of the one a `CostModel` ranked first) or a pre-ASAP +//! [`QueryExpr`] rewrite (still logical, structurally different from the +//! target but semantically equivalent) — see [`Replacement`] — plus a +//! human-readable `rationale`. +//! - [`ReplacementStrategy`] — `matches` + `replacements`, the same +//! extension-point shape [`CostModel`] and [`Matcher`](crate::boundary::Matcher) +//! already use in this crate (and the same shape issue #33's +//! applicability-rule framework, PR #247, uses for its own +//! `ApplicabilityRule`): a new replacement source is a new +//! `impl ReplacementStrategy`, not a restructuring of this trait or of any +//! existing strategy. `replacements` is **exhaustive, not ranked, not +//! filtered** — reporting "every valid candidate" is core's job; picking +//! the best one is a [`CostModel`]'s job, deliberately out of scope here +//! (see "Non-goals" below). +//! +//! ## The two strategies, and why these two +//! +//! Both wrap an existing, already-correct decision procedure into the +//! exhaustive-candidate shape — neither re-derives anything: +//! +//! - [`SketchFamilyStrategy`] wraps [`boundary::implementation_for_with`] / +//! [`boundary::summary_candidates`]'s exhaustive match over the +//! [`AggIntent`] vocabulary. For the same bindable-`Aggregate` shape +//! [`bind::implement_tree_with`] itself requires (single intent, no +//! `HAVING`), it returns every candidate [`boundary::implementation_for_with`] +//! could have committed to instead of the one it did: every `SketchKind` +//! [`boundary::summary_candidates`] lists for the intent when the boundary +//! decision is approximate, or the single exact-accumulator / +//! pass-through outcome when that's the *only* realization +//! [`boundary::implementation_for_with`]'s dispatch produces for this +//! intent (there is nothing else to enumerate in that case). +//! - [`SharedSubtreeStrategy`] wraps +//! `asap_types::pre_asap::cse::share_common_subtrees`'s sharing decision. +//! Wherever a [`TargetSubDAG`] already has two or more consumers (i.e. +//! `share_common_subtrees` already collapsed two or more workload +//! locations onto the same `Rc` — PR #247's own +//! `SharedSubexpressionRule` traversal/dedup logic, over in the +//! applicability-rule framework, does the identical discovery; this +//! module's own tests reuse the same dedup logic to build realistic +//! fixtures), it reports the two-way candidate CSE's own detection pass +//! deliberately declines to pick between on its own: build once and share +//! the already-interned subtree, or build it independently at each +//! consumer. [`crate::cost_model::CostModel::cse_share_decision`] is where +//! that choice actually gets made *today* (a fixed comparison, not a +//! search) — this strategy exposes the same two-way choice as an explicit, +//! inspectable pair of candidates instead of a cost model's already-decided +//! boolean. +//! +//! ## Non-goals (tracked separately, not attempted here) +//! +//! - **No search/selection logic.** `docs/asap_aware_mapping.md`'s +//! replacement-plan-searching pseudocode — trying every `ReplacementStrategy` +//! against every candidate plan, deduplicating, iterating to a fixpoint, +//! then ranking by a `CostModel` — is a Cascades/Volcano-style search +//! engine, tracked as a separate follow-up. This module only needs +//! `replacements()` to be exhaustive and correct for one `TargetSubDAG` at +//! a time, mirroring [`boundary::implementation_for_with`]'s own +//! "exhaustive match, no silent fallthrough" discipline. +//! - **No workload-wide `TargetSubDAG` discovery pass.** Finding every +//! candidate node in a whole workload (walking every root, deduplicating by +//! `Rc` identity, computing real consumer counts) is exactly what PR #247's +//! `SharedSubexpressionRule`/`SketchApplicabilityRule` traversals already +//! do — reusable, but wiring it up to feed this module's strategies +//! automatically is part of the same future search engine, not this issue. +//! This module's own tests build `TargetSubDAG`s directly, the same +//! hand-rolled-fixture style `bind.rs`/`boundary.rs`/`cost_model.rs`'s own +//! tests already use. +//! - **[`boundary`]/[`bind`]'s existing single-pick public behavior is +//! unchanged.** Nothing here modifies [`boundary::implementation_for`], +//! [`bind::implement_tree_with`], or how either is called; this module adds +//! the exhaustive-candidate view alongside them, it does not replace or +//! rewire either yet (a later issue's job). + +use std::rc::Rc; + +use asap_types::post_asap::{SketchKind, SketchParams, SketchQuery, SummaryNode}; +use asap_types::pre_asap::agg_intent::AggIntent; +use asap_types::pre_asap::expr_ir::ColumnRef; +use asap_types::pre_asap::query_expr::QueryExpr; + +use crate::bind::implement_tree_with; +use crate::boundary::{implementation_for_with, summary_candidates, Implementation}; +use crate::cost_model::{CostModel, CseCandidate, DefaultCostModel, ShareDecision}; + +/// A pre-ASAP sub-DAG a [`ReplacementStrategy`] knows how to replace. +/// +/// `root` is a reference into the workload's own [`QueryExpr`] tree (an +/// `Rc`, the same currency [`bind::implement_workload`] and +/// `asap_types::pre_asap::cse::share_common_subtrees` already thread through +/// this crate's public API — not a bare `&QueryExpr` — so a strategy that +/// needs the node's own `Rc` identity, not just its shape, has it available +/// without the caller re-deriving it). +/// +/// `consumer_count` is how many locations across the workload already +/// reference this exact `Rc` — 1 for an ordinary single-use node, 2+ when the +/// caller already ran `share_common_subtrees` and found this subtree shared. +/// A strategy that only cares about `root`'s own shape (e.g. +/// [`SketchFamilyStrategy`]) can ignore it entirely; [`SharedSubtreeStrategy`] +/// is the one strategy that consults it. Computing a *real* consumer count +/// across a whole workload is a traversal this module deliberately does not +/// own (see the module docs' "Non-goals") — [`TargetSubDAG::new`] defaults it +/// to `1`, the safe assumption for a caller inspecting one node in isolation. +#[derive(Debug, Clone, Copy)] +pub struct TargetSubDAG<'a> { + pub root: &'a Rc, + pub consumer_count: usize, +} + +impl<'a> TargetSubDAG<'a> { + /// A target assumed to have exactly one consumer — the common case for a + /// caller that isn't already tracking cross-workload sharing. + pub fn new(root: &'a Rc) -> Self { + Self { + root, + consumer_count: 1, + } + } + + /// A target with an explicit `consumer_count` — for a caller that already + /// knows (e.g. from `share_common_subtrees`, or from this module's own + /// test helpers) how many workload locations reference `root`. + pub fn with_consumer_count(root: &'a Rc, consumer_count: usize) -> Self { + Self { + root, + consumer_count, + } + } +} + +/// What a [`ReplacementSubDAG`] actually substitutes a [`TargetSubDAG`] with. +/// +/// Generalizes [`bind::implement_tree_with`]'s and +/// [`boundary::implementation_for_with`]'s two possible *kinds* of answer — +/// a post-ASAP binding decision, or a still-pre-ASAP structural alternative — +/// from "the one they commit to" into "one candidate among several". +#[derive(Debug, Clone)] +pub enum Replacement { + /// A fully bound post-ASAP summary decision — the same + /// [`SummaryNode`] shape [`bind::implement_tree_with`] itself produces, + /// for one particular candidate realization of the target. + Summary(Rc), + /// A pre-ASAP rewrite: still a logical [`QueryExpr`], structurally + /// different from the target's own `root` (e.g. sharing vs. not sharing + /// a subtree) but semantically equivalent to it. + Rewrite(Rc), +} + +/// One candidate replacement for a [`TargetSubDAG`], plus a human-readable +/// `rationale` explaining why it's a valid candidate (meant for a +/// report/log/debugging a search engine's choices, not machine parsing — +/// the same role an `ApplicabilityFinding`'s `reason` field plays for +/// applicability findings, over in PR #247's applicability-rule framework). +#[derive(Debug, Clone)] +pub struct ReplacementSubDAG { + pub replacement: Replacement, + pub rationale: String, +} + +/// A replacement strategy: given a [`TargetSubDAG`], does this strategy have +/// an opinion on it at all (`matches`), and if so, every semantically valid +/// replacement (`replacements`)? +/// +/// The extension point this module exists for — the same shape +/// [`CostModel`] and [`crate::boundary::Matcher`] already use elsewhere in +/// this crate: a new replacement source is a new `impl ReplacementStrategy`, +/// no restructuring of this trait or any existing strategy required. +/// +/// `replacements` is only meaningful when `matches` would return `true` for +/// the same target; both [`SketchFamilyStrategy`] and [`SharedSubtreeStrategy`] +/// return an empty `Vec` rather than panicking when called on a target they +/// don't match, so a caller that skips the `matches` check first still gets a +/// safe (merely uninformative) answer instead of a crash. +pub trait ReplacementStrategy { + /// Does this strategy have any replacement to offer for `target`? + fn matches(&self, target: &TargetSubDAG<'_>) -> bool; + + /// Every valid replacement for `target` — not ranked, not filtered. + /// Reporting "every valid candidate" is this method's whole job; picking + /// the best one is a [`CostModel`]'s job, out of scope here. + fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec; +} + +// ── SketchFamilyStrategy ───────────────────────────────────────────────── + +/// The bindable shape [`bind::implement_tree_with`] itself requires: a single +/// intent, no `HAVING` (a multi-intent node, or one with a `HAVING` +/// predicate, stays logical at actual binding time too — see `bind.rs`'s own +/// module docs on why — so neither is a target this strategy has an opinion +/// on). +fn bindable_intent(node: &QueryExpr) -> Option<&AggIntent> { + if let QueryExpr::Aggregate { + measures, having, .. + } = node + { + if let ([intent], None) = (measures.as_slice(), having) { + return Some(intent); + } + } + None +} + +/// A single static instance so [`SketchFamilyStrategy::default_cost_model`] +/// can hand out a `&'static dyn CostModel` without heap-allocating one — +/// `DefaultCostModel` is a unit struct with no state, so one instance serves +/// every caller (same pattern `applicability::SketchApplicabilityRule` uses). +static DEFAULT_COST_MODEL: DefaultCostModel = DefaultCostModel; + +/// Wraps [`boundary::implementation_for_with`]/[`boundary::summary_candidates`]'s +/// exhaustive match over the [`AggIntent`] vocabulary: for a bindable +/// `Aggregate`, every candidate summary realization instead of just the one +/// [`boundary::implementation_for_with`] commits to. +/// +/// Ranks (only to *order the enumeration*, never to drop a candidate) via a +/// [`CostModel`] — [`DefaultCostModel`] unless constructed with +/// [`SketchFamilyStrategy::new`] — so a deployment-specific cost model's +/// other hooks (`size_params`, `realize_extension`, `readout_extension`) are +/// still consulted while binding each candidate, exactly as +/// [`bind::implement_tree_with`] would. +pub struct SketchFamilyStrategy<'a> { + cost_model: &'a dyn CostModel, +} + +impl SketchFamilyStrategy<'static> { + /// A strategy that ranks/binds via the built-in [`DefaultCostModel`], + /// matching what a deployment gets from [`bind::implement_tree`] with no + /// custom cost model plugged in. + pub fn default_cost_model() -> Self { + Self { + cost_model: &DEFAULT_COST_MODEL, + } + } +} + +impl<'a> SketchFamilyStrategy<'a> { + /// A strategy that ranks/binds via `cost_model` instead of the built-in + /// static preference order — the same customization point + /// [`bind::implement_tree_with`] and [`boundary::implementation_for_with`] + /// already offer. + pub fn new(cost_model: &'a dyn CostModel) -> Self { + Self { cost_model } + } +} + +impl ReplacementStrategy for SketchFamilyStrategy<'_> { + fn matches(&self, target: &TargetSubDAG<'_>) -> bool { + bindable_intent(target.root).is_some() + } + + fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { + let Some(intent) = bindable_intent(target.root) else { + return Vec::new(); + }; + // Exhaustive over `Implementation`'s variants — the same "no silent + // fallthrough" discipline `boundary::implementation_for_with`'s own + // match uses. Only `Sketch` has more than one candidate to enumerate + // (`summary_candidates`); every other variant is the *only* + // realization `boundary::implementation_for_with`'s dispatch + // produces for this intent, so there's nothing else to offer. + match implementation_for_with(intent, self.cost_model) { + Implementation::Sketch { .. } => summary_candidates(intent) + .iter() + .filter_map(|kind| { + sketch_candidate(target.root, intent, kind.clone(), self.cost_model) + }) + .collect(), + Implementation::ExactAggregate { kind, .. } => single_candidate( + target.root, + self.cost_model, + format!( + "{} realizes as an exact {kind:?} accumulator — the only realization \ + boundary::implementation_for produces for this intent (no approximate \ + candidate applies)", + describe_intent(intent) + ), + ), + Implementation::PassThrough => single_candidate( + target.root, + self.cost_model, + format!( + "{} has no summary realization and stays a logical pass-through — the \ + only realization boundary::implementation_for produces for this intent", + describe_intent(intent) + ), + ), + Implementation::Sample { kind, .. } => single_candidate( + target.root, + self.cost_model, + format!( + "{} realizes as a {kind:?} sample — the only realization the plugged-in \ + CostModel produced for this intent", + describe_intent(intent) + ), + ), + Implementation::Wavelet { kind, .. } => single_candidate( + target.root, + self.cost_model, + format!( + "{} realizes as a {kind:?} wavelet transform — the only realization the \ + plugged-in CostModel produced for this intent", + describe_intent(intent) + ), + ), + Implementation::StatModel { kind, .. } => single_candidate( + target.root, + self.cost_model, + format!( + "{} realizes as a {kind:?} statistical model — the only realization the \ + plugged-in CostModel produced for this intent", + describe_intent(intent) + ), + ), + } + } +} + +/// Bind `root` via `cost_model` unchanged (no candidate to steer towards) — +/// the "only option" path for any [`Implementation`] category besides +/// `Sketch`. Schema derivation cannot fail for a target that was already a +/// legitimate part of the workload's tree, but [`implement_tree_with`]'s +/// signature is fallible (`ImplementError`), so a failure here — never +/// expected in practice — degrades to "no candidate" rather than a panic, +/// same conservatism as the rest of this strategy. +fn single_candidate( + root: &QueryExpr, + cost_model: &dyn CostModel, + rationale: String, +) -> Vec { + implement_tree_with(root, cost_model) + .ok() + .map(|node| ReplacementSubDAG { + replacement: Replacement::Summary(node), + rationale, + }) + .into_iter() + .collect() +} + +/// Bind `root` forcing the sketch-vs-exact boundary towards `kind` — one +/// entry of [`summary_candidates(intent)`](summary_candidates) — instead of +/// whichever candidate `cost_model` would otherwise rank first, by steering +/// [`ForceSketchKind`] through the exact same [`implement_tree_with`] call +/// [`bind::implement_tree_with`] itself would make. This reuses +/// `implement_tree_with`'s whole decision procedure (schema derivation, +/// column resolution, readout construction) unchanged — it does not +/// reimplement any part of it. +fn sketch_candidate( + root: &QueryExpr, + intent: &AggIntent, + kind: SketchKind, + cost_model: &dyn CostModel, +) -> Option { + let forced = ForceSketchKind { + kind: kind.clone(), + inner: cost_model, + }; + let node = implement_tree_with(root, &forced).ok()?; + Some(ReplacementSubDAG { + replacement: Replacement::Summary(node), + rationale: format!( + "{} realizes as a {kind:?} sketch — one of boundary::summary_candidates' \ + alternatives for this intent (asap_aware_mapping::boundary::implementation_for)", + describe_intent(intent) + ), + }) +} + +/// A [`CostModel`] adapter that forces +/// [`rank_candidates`](CostModel::rank_candidates) to put `kind` first, +/// forwarding every other hook to `inner` unchanged — so binding a specific +/// candidate still respects whatever deployment-specific sizing/extension +/// behavior `inner` provides, exactly like binding via `inner` directly +/// would. Only ever constructed with a `kind` already present in the +/// `candidates` slice `implementation_for_with` passes to `rank_candidates` +/// (drawn straight from [`summary_candidates(intent)`](summary_candidates)), +/// so this never invents a candidate the underlying model didn't already +/// have available — see [`CostModel::rank_candidates`]'s own contract. +struct ForceSketchKind<'a> { + kind: SketchKind, + inner: &'a dyn CostModel, +} + +impl CostModel for ForceSketchKind<'_> { + fn rank_candidates(&self, intent: &AggIntent, candidates: &[SketchKind]) -> Vec { + let mut ranked = self.inner.rank_candidates(intent, candidates); + ranked.retain(|k| *k != self.kind); + ranked.insert(0, self.kind.clone()); + ranked + } + + fn size_params( + &self, + kind: SketchKind, + intent: &AggIntent, + eps: f64, + delta: f64, + ) -> SketchParams { + self.inner.size_params(kind, intent, eps, delta) + } + + fn realize_extension(&self, ext_kind: &str, payload: &serde_json::Value) -> Implementation { + self.inner.realize_extension(ext_kind, payload) + } + + fn readout_extension( + &self, + ext_kind: &str, + payload: &serde_json::Value, + col: &ColumnRef, + ) -> SketchQuery { + self.inner.readout_extension(ext_kind, payload, col) + } + + fn cse_recompute_cost(&self, candidate: &CseCandidate) -> f64 { + self.inner.cse_recompute_cost(candidate) + } + + fn cse_shared_maintenance_cost(&self, candidate: &CseCandidate) -> f64 { + self.inner.cse_shared_maintenance_cost(candidate) + } + + fn cse_share_decision(&self, candidate: &CseCandidate) -> ShareDecision { + self.inner.cse_share_decision(candidate) + } +} + +/// A short human-readable label for an `AggIntent`, for +/// [`ReplacementSubDAG::rationale`] text. Not exhaustive by design (unlike +/// this crate's other `AggIntent` matches, e.g. +/// [`boundary::implementation_for_with`]'s) — this is prose for a +/// rationale string, not a decision, so an unlisted variant just falls back +/// to its `Debug` tag rather than forcing every future intent to be named +/// here too (same rationale, and same shape, as +/// `applicability`'s own private `describe_intent`). +fn describe_intent(intent: &AggIntent) -> String { + match intent { + AggIntent::Quantile { q, .. } => format!("quantile(q={q})"), + AggIntent::Cardinality { .. } => "cardinality (distinct count)".to_string(), + AggIntent::TopK { k, .. } => format!("top-{k} heavy-hitters"), + AggIntent::Count { .. } => "count".to_string(), + other => format!("{other:?}"), + } +} + +// ── SharedSubtreeStrategy ──────────────────────────────────────────────── + +/// Wraps `asap_types::pre_asap::cse::share_common_subtrees`'s sharing +/// decision as an explicit candidate pair, wherever a [`TargetSubDAG`] +/// already has two or more consumers. +/// +/// This strategy does not decide sharing itself, nor does it discover which +/// nodes are shared — by the time a caller builds a `TargetSubDAG` with +/// `consumer_count >= 2`, `share_common_subtrees` has already made that +/// (legality-gated, `PartialEq`-checked) call; PR #247's +/// `SharedSubexpressionRule` traversal discovers real consumer counts across +/// a workload the same way (this module's own tests reuse the identical +/// dedup logic to build realistic fixtures — see the module docs' +/// "Non-goals" on why that traversal isn't itself part of this strategy). +/// This strategy only reframes "two or more consumers already share this +/// `Rc`" as the two-way choice a downstream cost model (today, +/// [`CostModel::cse_share_decision`]) picks between: build once and share, or +/// build independently at each consumer. +pub struct SharedSubtreeStrategy; + +impl ReplacementStrategy for SharedSubtreeStrategy { + fn matches(&self, target: &TargetSubDAG<'_>) -> bool { + target.consumer_count >= 2 + } + + fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { + if target.consumer_count < 2 { + return Vec::new(); + } + let count = target.consumer_count; + vec![ + ReplacementSubDAG { + // The already-interned `Rc` itself: reusing it verbatim *is* + // "build once and share" — no new node to construct. + replacement: Replacement::Rewrite(Rc::clone(target.root)), + rationale: format!( + "build once and share: share_common_subtrees already interned this \ + subtree once and reused it across {count} consumers — one build can \ + answer all of them instead of computing it {count} times" + ), + }, + ReplacementSubDAG { + // A structurally-identical but freshly-allocated `Rc`: same + // value (`PartialEq`), deliberately *not* the same pointer, + // representing "undo the sharing and recompute independently". + replacement: Replacement::Rewrite(Rc::new((**target.root).clone())), + rationale: format!( + "build independently: undo the sharing share_common_subtrees found and \ + recompute this subtree separately at each of its {count} consumers — \ + worth it only when independence outweighs the shared-maintenance cost, \ + a CostModel's call (e.g. CostModel::cse_share_decision) and not this \ + strategy's" + ), + }, + ] + } +} + +#[cfg(test)] +mod tests { + use super::*; + use asap_types::pre_asap::agg_intent::{default_cardinality, default_quantile}; + use asap_types::pre_asap::query_expr::{Reduction, Source}; + use asap_types::pre_asap::schema::{Column, DataType, Schema}; + use asap_types::types::AccuracyTarget; + use std::collections::HashMap; + + fn metric_scan(labels: &[&str]) -> QueryExpr { + let mut columns = vec![ + Column::new("ts", DataType::Timestamp, false), + Column::new("value", DataType::Float64, false), + ]; + columns.extend(labels.iter().map(|n| Column::new(*n, DataType::Utf8, true))); + QueryExpr::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index(columns, 0, vec![]), + } + } + + fn agg(by: Vec, intent: AggIntent, child: QueryExpr) -> QueryExpr { + QueryExpr::Aggregate { + reduction: Reduction::by(by), + measures: vec![intent], + output_names: vec![], + having: None, + child: Rc::new(child), + } + } + + // ── SketchFamilyStrategy ───────────────────────────────────────────── + + #[test] + fn matches_a_bindable_aggregate() { + let q = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); + let target = TargetSubDAG::new(&q); + assert!(SketchFamilyStrategy::default_cost_model().matches(&target)); + } + + #[test] + fn does_not_match_a_multi_intent_or_having_aggregate() { + let strategy = SketchFamilyStrategy::default_cost_model(); + + let multi = Rc::new(QueryExpr::Aggregate { + reduction: Reduction::by(vec![2]), + measures: vec![AggIntent::Sum { col: None }, AggIntent::Avg { col: None }], + output_names: vec![], + having: None, + child: Rc::new(metric_scan(&["job"])), + }); + let target = TargetSubDAG::new(&multi); + assert!(!strategy.matches(&target)); + assert!(strategy.replacements(&target).is_empty()); + + let mut having_q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + if let QueryExpr::Aggregate { having, .. } = &mut having_q { + *having = Some(asap_types::pre_asap::query_expr::Predicate(Rc::new( + QueryExpr::Literal(asap_types::pre_asap::expr_ir::ScalarValue::Boolean(true)), + ))); + } + let having_q = Rc::new(having_q); + let target = TargetSubDAG::new(&having_q); + assert!(!strategy.matches(&target)); + assert!(strategy.replacements(&target).is_empty()); + } + + #[test] + fn does_not_match_a_non_aggregate_node() { + let scan = Rc::new(metric_scan(&["job"])); + let target = TargetSubDAG::new(&scan); + assert!(!SketchFamilyStrategy::default_cost_model().matches(&target)); + assert!(SketchFamilyStrategy::default_cost_model() + .replacements(&target) + .is_empty()); + } + + #[test] + fn approximate_quantile_enumerates_every_summary_candidate() { + // Quantile's candidate list is [Kll, DDSketch] (boundary::summary_candidates) — + // every entry must come back as its own bound SummaryNode candidate, + // not just Kll (the CostModel-ranked head implementation_for_with commits to). + let q = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); + let target = TargetSubDAG::new(&q); + let replacements = SketchFamilyStrategy::default_cost_model().replacements(&target); + assert_eq!( + replacements.len(), + 2, + "expected 2 candidates, got {replacements:?}" + ); + + let kinds: Vec = replacements + .iter() + .map(|r| match &r.replacement { + Replacement::Summary(node) => summary_family_kind(node), + Replacement::Rewrite(_) => panic!("expected a Summary replacement"), + }) + .collect(); + assert!(kinds.contains(&SketchKind::Kll), "{kinds:?}"); + assert!(kinds.contains(&SketchKind::DDSketch), "{kinds:?}"); + assert!( + replacements.iter().all(|r| !r.rationale.is_empty()), + "every candidate must carry a rationale" + ); + } + + #[test] + fn cardinality_enumerates_all_three_summary_candidates() { + let q = Rc::new(agg(vec![2], default_cardinality(), metric_scan(&["job"]))); + let target = TargetSubDAG::new(&q); + let replacements = SketchFamilyStrategy::default_cost_model().replacements(&target); + let kinds: Vec = replacements + .iter() + .map(|r| match &r.replacement { + Replacement::Summary(node) => summary_family_kind(node), + Replacement::Rewrite(_) => panic!("expected a Summary replacement"), + }) + .collect(); + assert_eq!( + kinds, + vec![SketchKind::Hll, SketchKind::Theta, SketchKind::Kmv], + "expected every boundary::summary_candidates entry for Cardinality" + ); + } + + #[test] + fn exact_accuracy_target_yields_exactly_one_pass_through_candidate() { + // Exact quantile has no sketch candidate at all — implementation_for + // commits to PassThrough, the only option, so exactly one candidate. + let intent = AggIntent::Quantile { + col: None, + q: 0.99, + accuracy: AccuracyTarget::Exact, + }; + let q = Rc::new(agg(vec![2], intent, metric_scan(&["job"]))); + let target = TargetSubDAG::new(&q); + let replacements = SketchFamilyStrategy::default_cost_model().replacements(&target); + assert_eq!(replacements.len(), 1, "{replacements:?}"); + assert!(matches!( + &replacements[0].replacement, + Replacement::Summary(node) if matches!( + node.expr, + asap_types::post_asap::SummaryExpr::Logical(_) + ) + )); + assert!(replacements[0].rationale.contains("only realization")); + } + + #[test] + fn exact_mergeable_intent_yields_exactly_one_accumulator_candidate() { + let q = Rc::new(agg( + vec![2], + AggIntent::Sum { col: None }, + metric_scan(&["job"]), + )); + let target = TargetSubDAG::new(&q); + let replacements = SketchFamilyStrategy::default_cost_model().replacements(&target); + assert_eq!(replacements.len(), 1, "{replacements:?}"); + assert!(matches!( + &replacements[0].replacement, + Replacement::Summary(node) if matches!( + node.expr, + asap_types::post_asap::SummaryExpr::SummaryAgg { .. } + ) + )); + } + + /// A custom `CostModel` doesn't change *which* candidates are enumerated + /// (still every `summary_candidates` entry) — only which one + /// `implementation_for_with` itself would have picked, and how each + /// candidate's own params are sized. + struct PreferDDSketch; + impl CostModel for PreferDDSketch { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchKind], + ) -> Vec { + let mut v = candidates.to_vec(); + if let Some(pos) = v.iter().position(|k| *k == SketchKind::DDSketch) { + let dd = v.remove(pos); + v.insert(0, dd); + } + v + } + } + + #[test] + fn custom_cost_model_still_enumerates_every_candidate_not_just_its_own_pick() { + let q = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); + let target = TargetSubDAG::new(&q); + let custom = PreferDDSketch; + let replacements = SketchFamilyStrategy::new(&custom).replacements(&target); + let kinds: Vec = replacements + .iter() + .map(|r| match &r.replacement { + Replacement::Summary(node) => summary_family_kind(node), + Replacement::Rewrite(_) => panic!("expected a Summary replacement"), + }) + .collect(); + assert!(kinds.contains(&SketchKind::Kll)); + assert!(kinds.contains(&SketchKind::DDSketch)); + assert_eq!(kinds.len(), 2); + } + + /// The `SummaryFamilyType`'s `SketchKind`, from the top `SummaryAgg` + /// reachable under a (possibly `SummaryEstimate`-wrapped) bound root. + fn summary_family_kind(node: &SummaryNode) -> SketchKind { + match &node.expr { + asap_types::post_asap::SummaryExpr::SummaryEstimate { summary_input, .. } => { + summary_family_kind(summary_input) + } + asap_types::post_asap::SummaryExpr::SummaryAgg { family, .. } => match family { + asap_types::post_asap::SummaryFamilyType::Sketch(kind, _) => kind.clone(), + other => panic!("expected a Sketch family, got {other:?}"), + }, + other => panic!("expected SummaryAgg/SummaryEstimate, got {other:?}"), + } + } + + // ── SharedSubtreeStrategy ──────────────────────────────────────────── + + #[test] + fn does_not_match_a_single_consumer_target() { + let q = Rc::new(agg( + vec![2], + AggIntent::Sum { col: None }, + metric_scan(&["job"]), + )); + let target = TargetSubDAG::new(&q); + assert_eq!(target.consumer_count, 1); + assert!(!SharedSubtreeStrategy.matches(&target)); + assert!(SharedSubtreeStrategy.replacements(&target).is_empty()); + } + + #[test] + fn two_or_more_consumers_yields_the_share_vs_independent_pair() { + let q = Rc::new(agg( + vec![2], + AggIntent::Sum { col: None }, + metric_scan(&["job"]), + )); + let target = TargetSubDAG::with_consumer_count(&q, 2); + assert!(SharedSubtreeStrategy.matches(&target)); + + let replacements = SharedSubtreeStrategy.replacements(&target); + assert_eq!(replacements.len(), 2, "{replacements:?}"); + + let shared = match &replacements[0].replacement { + Replacement::Rewrite(rc) => rc, + other => panic!("expected a Rewrite replacement, got {other:?}"), + }; + assert!( + Rc::ptr_eq(shared, &q), + "the 'build once and share' candidate must be the same Rc as the target" + ); + assert!(replacements[0].rationale.contains("build once and share")); + + let independent = match &replacements[1].replacement { + Replacement::Rewrite(rc) => rc, + other => panic!("expected a Rewrite replacement, got {other:?}"), + }; + assert!( + !Rc::ptr_eq(independent, &q), + "the 'build independently' candidate must be a distinct Rc from the target" + ); + assert_eq!( + **independent, *q, + "the 'build independently' candidate must still be structurally identical" + ); + assert!(replacements[1].rationale.contains("build independently")); + } + + #[test] + fn three_consumers_are_reported_verbatim_in_both_rationales() { + let q = Rc::new(agg( + vec![2], + AggIntent::Sum { col: None }, + metric_scan(&["job"]), + )); + let target = TargetSubDAG::with_consumer_count(&q, 3); + let replacements = SharedSubtreeStrategy.replacements(&target); + assert!(replacements[0].rationale.contains('3')); + assert!(replacements[1].rationale.contains('3')); + } + + /// Builds realistic multi-consumer `TargetSubDAG`s the same way + /// `applicability::SharedSubexpressionRule`'s `register_site`/ + /// `walk_rc_children` traversal does: dedup by `Rc::as_ptr`, walking + /// only the relational-skeleton operator children + /// `asap_types::pre_asap::cse::share_common_subtrees` itself scopes to, + /// so a shared node nested below another shared node is only ever + /// counted at the highest (maximal) point sharing starts. Test-only: + /// this module deliberately does not ship a workload-wide discovery + /// pass of its own (see the module docs' "Non-goals"). + fn count_consumers(roots: &[Rc]) -> HashMap<*const QueryExpr, usize> { + fn walk(node: &Rc, counts: &mut HashMap<*const QueryExpr, usize>) { + let ptr = Rc::as_ptr(node); + let already_visited = counts.contains_key(&ptr); + *counts.entry(ptr).or_insert(0) += 1; + if !already_visited { + walk_children(node, counts); + } + } + fn walk_children(node: &QueryExpr, counts: &mut HashMap<*const QueryExpr, usize>) { + use QueryExpr::*; + match node { + Scan { .. } | PromqlScalarBridge(_) | QueryTimestamp => {} + PromqlVectorFromScalar(c) | PromqlScalarFromVector(c) => walk(c, counts), + PromqlRelabel { child, .. } + | PromqlInfoEnrich { child, .. } + | PromqlSeriesSample { child, .. } + | Filter { child, .. } + | Project { child, .. } + | Aggregate { child, .. } + | Dedup { child, .. } + | PromqlSubquery { child, .. } + | TimeRange { child, .. } + | TimeShift { child, .. } + | SQLWindowFunc { child, .. } + | Sort { child, .. } + | Limit { child, .. } => walk(child, counts), + Concat { children } => { + for c in children { + walk_children(c, counts); + } + } + Join { left, right, .. } | SetOp { left, right, .. } => { + walk(left, counts); + walk(right, counts); + } + BinaryOp { lhs, rhs, .. } => { + walk(lhs, counts); + walk(rhs, counts); + } + Column(_) + | Literal(_) + | Compare { .. } + | BoolAnd(_) + | BoolOr(_) + | Not(_) + | IsNull(_) + | IsNotNull(_) + | Cast { .. } + | InList { .. } + | FunctionCall { .. } + | Arithmetic { .. } + | Case { .. } => {} + } + } + + let mut counts = HashMap::new(); + for root in roots { + walk(root, &mut counts); + } + counts + } + + #[test] + fn realistic_cse_output_produces_a_two_consumer_target() { + // Two workload roots that `share_common_subtrees` collapses onto one + // Rc (mirrors `applicability`'s and `cse`'s own fixtures): a grouped + // Sum aggregate over the same scan, built independently at each root. + let a = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let b = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let shared = asap_types::pre_asap::cse::share_common_subtrees(vec![("a", a), ("b", b)]); + let [(_, ra), (_, rb)] = shared.as_slice() else { + panic!("expected 2 roots"); + }; + assert!(Rc::ptr_eq(ra, rb), "fixture sanity: the two roots merged"); + + let roots: Vec> = shared.into_iter().map(|(_, rc)| rc).collect(); + let counts = count_consumers(&roots); + let count = counts[&Rc::as_ptr(&roots[0])]; + assert_eq!(count, 2); + + let target = TargetSubDAG::with_consumer_count(&roots[0], count); + assert!(SharedSubtreeStrategy.matches(&target)); + assert_eq!(SharedSubtreeStrategy.replacements(&target).len(), 2); + } +} From ab876e7f179682ceea4f2ad8447948efe1bed9c6 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 23 Aug 2026 12:16:58 -0600 Subject: [PATCH 02/49] docs(asap-aware-mapping): add ReplacementStrategy dev guide (#259) Milind's review comment on #259: this is going to be critical code for anyone contributing to ASAPPlanner, and needs docs beyond the module's own rustdoc (which already covers what each type is and why). docs/replacement-strategy-dev-guide.md is the narrower companion: how a contributor actually writes a new ReplacementStrategy -- the contract's non-obvious rules (exhaustive not ranked, don't panic on non-match, wrap existing decision logic instead of inventing new), a full worked walkthrough of SharedSubtreeStrategy, a testing checklist, and explicit non-goals. Aimed at whoever picks up #253/#254/#256/#257 next. --- docs/replacement-strategy-dev-guide.md | 180 +++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 docs/replacement-strategy-dev-guide.md diff --git a/docs/replacement-strategy-dev-guide.md b/docs/replacement-strategy-dev-guide.md new file mode 100644 index 00000000..e8766139 --- /dev/null +++ b/docs/replacement-strategy-dev-guide.md @@ -0,0 +1,180 @@ +# Writing a `ReplacementStrategy` (dev guide) + +This is the practical companion to [`docs/asap_aware_mapping.md`](asap_aware_mapping.md) +(design rationale for *why* this vocabulary exists) and the module docs on +[`crates/asap-aware-mapping/src/replacement.rs`](../crates/asap-aware-mapping/src/replacement.rs) +(the authoritative reference for the exact contract of each type). Read those +first if you haven't. This doc exists for one narrower question: **you're +implementing #253/#254/#256/#257 or a similar optimization — what do you +actually write?** + +## The three types, as a contributor uses them + +```rust +pub struct TargetSubDAG<'a> { + pub root: &'a Rc, // the node you might replace + pub consumer_count: usize, // how many workload locations reference it +} + +pub enum Replacement { + Summary(Rc), // "bind it to this post-ASAP summary" + Rewrite(Rc), // "replace it with this equivalent pre-ASAP tree" +} + +pub struct ReplacementSubDAG { + pub replacement: Replacement, + pub rationale: String, // human-readable, for logs/debugging — not parsed +} + +pub trait ReplacementStrategy { + fn matches(&self, target: &TargetSubDAG<'_>) -> bool; + fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec; +} +``` + +A strategy is stateless business logic over one `TargetSubDAG` at a time. It +does not walk a workload, does not rank, does not decide. Those are, in +order, the future search engine's job (#252), a `CostModel`'s job, and the +search engine's job again. + +## The contract, made explicit + +- **`matches` must be cheap and side-effect-free.** The future search engine + (#252) will call it on every `TargetSubDAG` in a workload, against every + registered strategy, on every iteration of its fixpoint loop. Don't do + binding work here — that's `replacements`' job, and only after `matches` + said yes. +- **`replacements` must be exhaustive.** List *every* semantically valid + candidate, not just the ones you'd guess are good. Pruning is explicitly + out of scope for a strategy — a `CostModel` (used by the search engine, + #252) is what narrows candidates down, and it can only compare candidates + it was actually given. An under-enumerating strategy silently shrinks the + search space and can hide the globally best plan. +- **`replacements` must not panic when `matches` would have returned + `false`.** Return an empty `Vec` instead. A caller that skips the + `matches` check first (or a future search engine that calls + `replacements` speculatively) should get a safe, merely-uninformative + answer, not a crash. `SketchFamilyStrategy` and `SharedSubtreeStrategy` + both follow this; keep doing it. +- **`rationale` is prose for a human, not a machine.** It's meant for a + debug log or a search engine's explanation of why it considered a + candidate — write it as you would a code comment explaining a decision, + not as a structured value another piece of code will parse. +- **Wrap an existing decision procedure; don't invent a new one.** Every + strategy so far (`SketchFamilyStrategy` wrapping + `boundary::implementation_for_with`/`summary_candidates`, + `SharedSubtreeStrategy` wrapping `cse::share_common_subtrees`) reuses + logic that's already correct and already tested elsewhere in this crate. + If your strategy needs a decision nothing in the codebase makes yet, + that decision procedure is its own piece of work — build and test it on + its own terms first, then wrap it in a thin `ReplacementStrategy` + `impl`, the same way these two do. + +## Worked example: `SharedSubtreeStrategy` + +This is the smallest real strategy in the module — a good template to copy +from. + +```rust +pub struct SharedSubtreeStrategy; + +impl ReplacementStrategy for SharedSubtreeStrategy { + fn matches(&self, target: &TargetSubDAG<'_>) -> bool { + target.consumer_count >= 2 + } + + fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { + if target.consumer_count < 2 { + return Vec::new(); + } + let count = target.consumer_count; + vec![ + ReplacementSubDAG { + // The already-interned `Rc` itself: reusing it verbatim *is* + // "build once and share" — no new node to construct. + replacement: Replacement::Rewrite(Rc::clone(target.root)), + rationale: format!( + "build once and share: share_common_subtrees already interned this \ + subtree once and reused it across {count} consumers — one build can \ + answer all of them instead of computing it {count} times" + ), + }, + ReplacementSubDAG { + // A structurally-identical but freshly-allocated `Rc`: same + // value (`PartialEq`), deliberately *not* the same pointer, + // representing "undo the sharing and recompute independently". + replacement: Replacement::Rewrite(Rc::new((**target.root).clone())), + rationale: format!( + "build independently: undo the sharing share_common_subtrees found and \ + recompute this subtree separately at each of its {count} consumers — \ + worth it only when independence outweighs the shared-maintenance cost, \ + a CostModel's call and not this strategy's" + ), + }, + ] + } +} +``` + +What to notice, as a pattern to reuse: + +1. **`matches` is a one-line predicate** over data `TargetSubDAG` already + carries (`consumer_count`). It doesn't re-derive anything the caller + could have told it directly. +2. **The strategy doesn't decide** which of the two candidates is better — + it returns both, unconditionally, whenever it matches. "Which is + cheaper" is explicitly deferred to `CostModel::cse_share_decision` + (today) / the search engine's cost-based ranking (once #252 lands). +3. **Both branches build a `Replacement::Rewrite`**, because sharing-vs-not + is a structural (pre-ASAP) choice, not a binding decision. Use + `Replacement::Summary` instead when your strategy's candidates are + post-ASAP binding choices — see `SketchFamilyStrategy` for that shape + (it wraps `bind::implement_tree_with` via a `CostModel` adapter, + `ForceSketchKind`, to steer the existing binding procedure toward each + candidate `SketchKind` in turn, rather than reimplementing binding). +4. **Guard `replacements` even though `matches` already checked the same + condition.** The `if target.consumer_count < 2 { return Vec::new() }` + inside `replacements` looks redundant next to `matches`, but it's what + keeps `replacements` safe to call on its own — see the contract above. + +## Testing a new strategy + +Follow the hand-rolled-fixture style `bind.rs`/`boundary.rs`/`cost_model.rs` +already use — build a small `QueryExpr` tree by hand, wrap it in a +`TargetSubDAG`, and assert on `matches`/`replacements` directly. At minimum: + +- **A positive case**: a target your strategy matches, asserting the full + set of `replacements()` it returns (not just its length — check which + `Replacement` variants and values came back). +- **A negative case**: a target your strategy does *not* match, asserting + `matches` returns `false` and `replacements` returns an empty `Vec`. +- If your strategy depends on cross-node context (like + `SharedSubtreeStrategy`'s `consumer_count`), add one test that builds it + from the real upstream computation instead of a hand-set field — see + `replacement.rs`'s own test that runs `SharedSubtreeStrategy` over a + `TargetSubDAG` built from real `share_common_subtrees` output, via a + small test-only traversal mirroring PR #247's `SharedSubexpressionRule` + dedup logic. A hand-set `consumer_count` alone can't catch a bug in how + that value gets computed for real. + +## What you are not responsible for + +- **Discovering `TargetSubDAG`s across a workload.** Your strategy receives + one `TargetSubDAG` at a time; walking a whole workload's tree to find + candidates is the search engine's job (#252) — or, today, ad hoc test + fixtures / `applicability.rs`'s existing traversal. +- **Ranking or filtering your own candidates.** Return everything valid; + a `CostModel` picks. +- **Wiring your strategy into anything automatically.** Until #252 lands, + a new `impl ReplacementStrategy` is just a type other code can construct + and call directly (e.g. from a test, or from `applicability.rs`) — there + is no registry or dispatcher yet. + +## Where this fits + +`docs/asap_aware_mapping.md` is the design doc this vocabulary implements +(read it for *why* `ReplacementStrategy` is shaped as "exhaustive, not +decisive," why the eventual search is MEMO-style and iterative, and how a +`CostModel` fits in). This guide is the narrower "how do I add one" doc for +whoever's picking up #253 (semantic rewrite), #254 (roll-up), #256 (Hydra +grouping), or #257 (applicability-as-a-view) next. From bb24367aedb0cb5c08b0066b82109d1c7b28364f Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 23 Aug 2026 12:23:13 -0600 Subject: [PATCH 03/49] docs(asap-aware-mapping): broaden dev guide to cover CostModel and Matcher Rename replacement-strategy-dev-guide.md -> extension-points-dev-guide.md. CostModel shares ReplacementStrategy's extension-point shape (small trait, sensible defaults, override just what you need) and deserved the same worked-example/contract treatment: the rank_candidates/size_params/ realize_extension+readout_extension pairing, the cse_share_decision default composing the two cost hooks (delegate to those, don't reimplement the comparison), and the ForceSketchKind delegation pattern for adapting an existing CostModel. Matcher gets a short pointer -- unlike the other two it ships with no default body and no implementation in this crate at all. --- ...guide.md => extension-points-dev-guide.md} | 153 ++++++++++++++++-- 1 file changed, 137 insertions(+), 16 deletions(-) rename docs/{replacement-strategy-dev-guide.md => extension-points-dev-guide.md} (52%) diff --git a/docs/replacement-strategy-dev-guide.md b/docs/extension-points-dev-guide.md similarity index 52% rename from docs/replacement-strategy-dev-guide.md rename to docs/extension-points-dev-guide.md index e8766139..81f7a3d1 100644 --- a/docs/replacement-strategy-dev-guide.md +++ b/docs/extension-points-dev-guide.md @@ -1,14 +1,26 @@ -# Writing a `ReplacementStrategy` (dev guide) +# Extension points (dev guide) -This is the practical companion to [`docs/asap_aware_mapping.md`](asap_aware_mapping.md) -(design rationale for *why* this vocabulary exists) and the module docs on -[`crates/asap-aware-mapping/src/replacement.rs`](../crates/asap-aware-mapping/src/replacement.rs) -(the authoritative reference for the exact contract of each type). Read those -first if you haven't. This doc exists for one narrower question: **you're -implementing #253/#254/#256/#257 or a similar optimization — what do you -actually write?** +`crates/asap-aware-mapping` has three extension points a contributor or a +downstream deployment plugs into: [`ReplacementStrategy`](#replacementstrategy), +[`CostModel`](#costmodel), and [`Matcher`](#matcher-boundaryrs). All three +share the same shape — a small trait, a default or no-default body, and one +job each: `ReplacementStrategy` reports *what's possible*, `CostModel` +decides *what's preferable*, `Matcher` decides *what already satisfies a +requirement*. This doc is the practical "what do I actually write" companion +to [`docs/asap_aware_mapping.md`](asap_aware_mapping.md) (the design +rationale for *why* this vocabulary exists) and each type's own rustdoc (the +authoritative reference for its exact contract). Read those first if you +haven't. -## The three types, as a contributor uses them +## `ReplacementStrategy` + +Defined in [`crates/asap-aware-mapping/src/replacement.rs`](../crates/asap-aware-mapping/src/replacement.rs). +Relevant if you're implementing #253 (semantic rewrite), #254 (roll-up), +#256 (Hydra grouping), #257 (applicability-as-a-view), or any future +optimization that should show up as a candidate in the eventual search +(#252). + +### The three types, as a contributor uses them ```rust pub struct TargetSubDAG<'a> { @@ -37,7 +49,7 @@ does not walk a workload, does not rank, does not decide. Those are, in order, the future search engine's job (#252), a `CostModel`'s job, and the search engine's job again. -## The contract, made explicit +### The contract, made explicit - **`matches` must be cheap and side-effect-free.** The future search engine (#252) will call it on every `TargetSubDAG` in a workload, against every @@ -70,7 +82,7 @@ search engine's job again. its own terms first, then wrap it in a thin `ReplacementStrategy` `impl`, the same way these two do. -## Worked example: `SharedSubtreeStrategy` +### Worked example: `SharedSubtreeStrategy` This is the smallest real strategy in the module — a good template to copy from. @@ -137,7 +149,7 @@ What to notice, as a pattern to reuse: inside `replacements` looks redundant next to `matches`, but it's what keeps `replacements` safe to call on its own — see the contract above. -## Testing a new strategy +### Testing a new strategy Follow the hand-rolled-fixture style `bind.rs`/`boundary.rs`/`cost_model.rs` already use — build a small `QueryExpr` tree by hand, wrap it in a @@ -157,7 +169,7 @@ already use — build a small `QueryExpr` tree by hand, wrap it in a dedup logic. A hand-set `consumer_count` alone can't catch a bug in how that value gets computed for real. -## What you are not responsible for +### What you are not responsible for - **Discovering `TargetSubDAG`s across a workload.** Your strategy receives one `TargetSubDAG` at a time; walking a whole workload's tree to find @@ -170,11 +182,120 @@ already use — build a small `QueryExpr` tree by hand, wrap it in a and call directly (e.g. from a test, or from `applicability.rs`) — there is no registry or dispatcher yet. +## `CostModel` + +Defined in [`crates/asap-aware-mapping/src/cost_model.rs`](../crates/asap-aware-mapping/src/cost_model.rs). +Relevant if you're wiring a deployment (ASAPCollector, ASAPFusion, …) with +real cost knowledge (bandwidth budget, memory footprint, site count, +observed drift, …) into candidate selection, instead of accepting this +crate's built-in static preferences. + +### The extension-point discipline + +`asap-aware-mapping` deliberately has no cost model *implementation* of its +own beyond [`DefaultCostModel`] — ranking real candidates by real cost needs +deployment knowledge this crate doesn't have and, per the crate's layering +invariant, shouldn't acquire. `CostModel` is the one interface every +deployment plugs its own knowledge into instead of forking `boundary`/`bind`. +Every method has a default body that reproduces today's static behavior +exactly — implement only the hooks you actually need to change: + +```rust +pub trait CostModel { + // No default: candidate selection needs a real answer. + fn rank_candidates(&self, intent: &AggIntent, candidates: &[SketchKind]) -> Vec; + + // Defaults to asap-plan's built-in per-family sizing formulas. + fn size_params(&self, kind: SketchKind, intent: &AggIntent, eps: f64, delta: f64) -> SketchParams { ... } + + // Defaults to Implementation::PassThrough for AggIntent::Extension. + fn realize_extension(&self, ext_kind: &str, payload: &serde_json::Value) -> Implementation { ... } + + // Panics if realize_extension ever returned Sketch without this being overridden too. + fn readout_extension(&self, ext_kind: &str, payload: &serde_json::Value, col: &ColumnRef) -> SketchQuery { ... } + + // CSE sharing cost hooks (issue #237) — defaults are structural-size / + // per-family-weight proxies; override with real cost knowledge. + fn cse_recompute_cost(&self, candidate: &CseCandidate) -> f64 { ... } + fn cse_shared_maintenance_cost(&self, candidate: &CseCandidate) -> f64 { ... } + fn cse_share_decision(&self, candidate: &CseCandidate) -> ShareDecision { ... } // composes the two above +} +``` + +### The contract, made explicit + +- **`rank_candidates` may reorder or drop entries, never invent one.** + Returning a `SketchKind` that wasn't in `candidates` will panic downstream + (`boundary::implementation_for_with` has no sizing logic for an unknown + kind). Returning an empty `Vec` is valid — it means "no candidate is + acceptable here." +- **Prefer overriding the narrowest hook.** `cse_share_decision`'s default + body composes `cse_recompute_cost` and `cse_shared_maintenance_cost` into + a Volcano/Cascades-style comparison (share iff maintaining one shared + summary costs no more than recomputing it everywhere it's used) — a + deployment with real cost numbers should override the two cost hooks and + keep the comparison, not reimplement the comparison itself. Override + `cse_share_decision` directly only for a genuinely different policy (e.g. + something other than a cost threshold). See + [`docs/cse-cost-model-decision.md`](cse-cost-model-decision.md) for the + full design discussion behind this split. +- **`realize_extension` and `readout_extension` are a matched pair.** If you + override `realize_extension` to return `Implementation::Sketch` for some + `ext_kind`, you MUST also override `readout_extension` for that same + `ext_kind` — the default panics loudly (not silently) the first time that + intent is actually read out, specifically so a mismatch is caught instead + of misinterpreting `payload`. +- **Delegate, don't reimplement, when adapting an existing `CostModel`.** + `SketchFamilyStrategy`'s `ForceSketchKind` (in `replacement.rs`) is the + reference example: it wraps another `&dyn CostModel`, overrides only + `rank_candidates` to force one specific kind first, and forwards every + other method to the wrapped model unchanged — so a deployment's own + sizing/extension behavior still applies while one method's behavior is + steered. Reach for this pattern before writing a `CostModel` from scratch + when you only need to adjust one hook's behavior. + +### Testing a new `CostModel` + +`cost_model.rs`'s own tests are the reference style: construct a minimal +`CostModel` `impl` inline in the test (only overriding the method(s) under +test), build fixture `QueryExpr`/`SummaryNode` values by hand, and assert +directly on the method's return value — e.g. +`custom_cost_model_can_reorder_candidates`, +`custom_cost_model_can_override_sizing_independently_of_ranking`, and +`cse_share_decision_default_body_composes_the_two_cost_hooks`. For the +default-preserving behavior of any method you don't override, add a test +asserting it matches the crate's built-in default exactly (see +`default_cost_model_preserves_static_order`) — that's the guarantee a +deployment relies on when it only overrides one hook. + +## `Matcher` (`boundary.rs`) + +Defined in [`crates/asap-aware-mapping/src/boundary.rs`](../crates/asap-aware-mapping/src/boundary.rs). +Much smaller surface than the other two, but shares the same "extension +point instead of a fork" role, so it's worth knowing it exists: + +```rust +pub trait Matcher { + fn is_satisfied_by(&self, required: &Implementation, available: &Implementation) -> bool; +} +``` + +Unlike `CostModel`, `asap-plan` ships **no implementation and no default +body** for this trait at all — whether an available summary satisfies a +required one (e.g. does an available `DDSketch` satisfy a required `Kll` +request? does a multi-population accumulator satisfy a single-population +query?) depends on facts this crate deliberately doesn't settle: a pure +sketch-algebra answer and a deployment's own storage-layout rules can +legitimately disagree. If you need this, you're implementing the whole +trait for your deployment from scratch — there's no default to lean on and +no existing implementation in this crate to use as a worked example. + ## Where this fits `docs/asap_aware_mapping.md` is the design doc this vocabulary implements (read it for *why* `ReplacementStrategy` is shaped as "exhaustive, not decisive," why the eventual search is MEMO-style and iterative, and how a -`CostModel` fits in). This guide is the narrower "how do I add one" doc for -whoever's picking up #253 (semantic rewrite), #254 (roll-up), #256 (Hydra -grouping), or #257 (applicability-as-a-view) next. +`CostModel` fits into that search). This guide is the narrower "how do I +add one" doc — for `ReplacementStrategy`, aimed at whoever's picking up +#253/#254/#256/#257 next; for `CostModel`, aimed at whoever's wiring a real +deployment's cost knowledge into candidate selection. From ed231e4d5981cfb0b38d9331c4f0825418ebb8df Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 23 Aug 2026 12:39:33 -0600 Subject: [PATCH 04/49] docs(asap-aware-mapping): replace extension-points guide with user's rewrite Rewritten as a fuller how-to: mental model, glossary, worked examples for both SketchFamilyStrategy and SharedSubtreeStrategy, a strategy contract with numbered rules, a per-hook CostModel reference, common mistakes, an extension checklist, and an extension-point lookup table. Verified against current code (ForceSketchKind body, SketchFamilyStrategy dispatch, CostModel hook signatures, CseCandidate/ShareDecision, test names) -- no discrepancies found. On top of the original draft: normalized heading levels (was mixing H1/H2 for sections 4-18), defined 'readout' where it first appears, marked illustrative (My*/Prefer*) vs. real code, linked docs/asap_aware_mapping.md, and added a Matcher section (confirmed via grep: no impl Matcher for anywhere in the crate, so it ships with neither a default body nor an implementation, unlike CostModel). --- docs/extension-points-dev-guide.md | 1470 +++++++++++++++++++++++----- 1 file changed, 1207 insertions(+), 263 deletions(-) diff --git a/docs/extension-points-dev-guide.md b/docs/extension-points-dev-guide.md index 81f7a3d1..8a002a07 100644 --- a/docs/extension-points-dev-guide.md +++ b/docs/extension-points-dev-guide.md @@ -1,301 +1,1245 @@ -# Extension points (dev guide) +# ASAP-Aware Mapping: Developer Guide -`crates/asap-aware-mapping` has three extension points a contributor or a -downstream deployment plugs into: [`ReplacementStrategy`](#replacementstrategy), -[`CostModel`](#costmodel), and [`Matcher`](#matcher-boundaryrs). All three -share the same shape — a small trait, a default or no-default body, and one -job each: `ReplacementStrategy` reports *what's possible*, `CostModel` -decides *what's preferable*, `Matcher` decides *what already satisfies a -requirement*. This doc is the practical "what do I actually write" companion -to [`docs/asap_aware_mapping.md`](asap_aware_mapping.md) (the design -rationale for *why* this vocabulary exists) and each type's own rustdoc (the -authoritative reference for its exact contract). Read those first if you -haven't. +This document explains how to extend ASAP-aware mapping in the current codebase. -## `ReplacementStrategy` +It is written for developers who want to: -Defined in [`crates/asap-aware-mapping/src/replacement.rs`](../crates/asap-aware-mapping/src/replacement.rs). -Relevant if you're implementing #253 (semantic rewrite), #254 (roll-up), -#256 (Hydra grouping), #257 (applicability-as-a-view), or any future -optimization that should show up as a candidate in the eventual search -(#252). +- add a new `ReplacementStrategy`, +- add or customize a `CostModel`, +- understand how strategies, binding, and costing interact, +- add a new kind of replacement without duplicating existing planner logic, +- and write the tests expected for a new extension. -### The three types, as a contributor uses them +The focus here is the **current code interfaces and their contracts**. For the higher-level motivation and future search design, see the separate design document, [`docs/asap_aware_mapping.md`](asap_aware_mapping.md). + +Code samples named `My*` or `Prefer*` (`MyStrategy`, `MyCostModel`, `PreferDDSketch`, …) below are illustrative sketches of a pattern, not code that ships in this crate. Samples that name a real type (`SketchFamilyStrategy`, `ForceSketchKind`, `SharedSubtreeStrategy`, …) are copied verbatim from `replacement.rs`/`cost_model.rs`. + +--- + +## 1. Mental model + +ASAP-aware mapping has two different jobs that should remain separate: + +1. **Generate valid alternatives.** +2. **Choose among alternatives.** + +`ReplacementStrategy` is responsible for the first job. + +`CostModel` is responsible for the second. + +A strategy should answer: + +> Does this transformation apply here, and if so, what are all semantically valid replacements? + +A cost model should answer: + +> Given valid choices, which choices are preferable, and how should they be parameterized? + +Do not put cost-based pruning into a `ReplacementStrategy`. A strategy should enumerate valid alternatives even when one of them is obviously more expensive under the default cost model. (This is the single most important rule in this guide — later sections restate it in context rather than re-arguing it; see §5 Rule 2 for the canonical statement.) + +--- + +## 2. Glossary + +### `TargetSubDAG` + +A pre-ASAP `QueryExpr` node that a strategy may replace. ```rust pub struct TargetSubDAG<'a> { - pub root: &'a Rc, // the node you might replace - pub consumer_count: usize, // how many workload locations reference it + pub root: &'a Rc, + pub consumer_count: usize, } +``` + +`root` is the actual `Rc` from the workload. + +`consumer_count` records how many workload locations refer to that exact shared node. + +Use: + +```rust +let target = TargetSubDAG::new(&root); +``` + +when you are inspecting a node in isolation. + +Use: + +```rust +let target = TargetSubDAG::with_consumer_count(&root, count); +``` + +when the caller already knows the real number of consumers. + +`TargetSubDAG::new` assumes one consumer. + +--- + +### `Replacement` +The actual object that substitutes the target. + +There are currently two forms: + +```rust pub enum Replacement { - Summary(Rc), // "bind it to this post-ASAP summary" - Rewrite(Rc), // "replace it with this equivalent pre-ASAP tree" + Summary(Rc), + Rewrite(Rc), } +``` + +Use `Replacement::Summary` when the alternative is already bound into a post-ASAP summary plan. + +Use `Replacement::Rewrite` when the alternative is still a logical pre-ASAP `QueryExpr`. + +Examples: + +```text +Quantile(...) + -> KLL SummaryNode +``` + +is a `Summary`. + +```text +compute independently + vs. +reuse an already shared logical subtree +``` + +is represented as a `Rewrite`. + +--- + +### `ReplacementSubDAG` + +One candidate replacement plus an explanation. +```rust pub struct ReplacementSubDAG { pub replacement: Replacement, - pub rationale: String, // human-readable, for logs/debugging — not parsed + pub rationale: String, } +``` + +The `rationale` is for debugging, reporting, and explaining planner choices. It is human-readable text, not a machine-readable protocol. + +Every candidate should carry a useful rationale. +--- + +### `ReplacementStrategy` + +The main extension point for adding a new optimization or replacement source. + +```rust pub trait ReplacementStrategy { fn matches(&self, target: &TargetSubDAG<'_>) -> bool; - fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec; + + fn replacements( + &self, + target: &TargetSubDAG<'_>, + ) -> Vec; } ``` -A strategy is stateless business logic over one `TargetSubDAG` at a time. It -does not walk a workload, does not rank, does not decide. Those are, in -order, the future search engine's job (#252), a `CostModel`'s job, and the -search engine's job again. - -### The contract, made explicit - -- **`matches` must be cheap and side-effect-free.** The future search engine - (#252) will call it on every `TargetSubDAG` in a workload, against every - registered strategy, on every iteration of its fixpoint loop. Don't do - binding work here — that's `replacements`' job, and only after `matches` - said yes. -- **`replacements` must be exhaustive.** List *every* semantically valid - candidate, not just the ones you'd guess are good. Pruning is explicitly - out of scope for a strategy — a `CostModel` (used by the search engine, - #252) is what narrows candidates down, and it can only compare candidates - it was actually given. An under-enumerating strategy silently shrinks the - search space and can hide the globally best plan. -- **`replacements` must not panic when `matches` would have returned - `false`.** Return an empty `Vec` instead. A caller that skips the - `matches` check first (or a future search engine that calls - `replacements` speculatively) should get a safe, merely-uninformative - answer, not a crash. `SketchFamilyStrategy` and `SharedSubtreeStrategy` - both follow this; keep doing it. -- **`rationale` is prose for a human, not a machine.** It's meant for a - debug log or a search engine's explanation of why it considered a - candidate — write it as you would a code comment explaining a decision, - not as a structured value another piece of code will parse. -- **Wrap an existing decision procedure; don't invent a new one.** Every - strategy so far (`SketchFamilyStrategy` wrapping - `boundary::implementation_for_with`/`summary_candidates`, - `SharedSubtreeStrategy` wrapping `cse::share_common_subtrees`) reuses - logic that's already correct and already tested elsewhere in this crate. - If your strategy needs a decision nothing in the codebase makes yet, - that decision procedure is its own piece of work — build and test it on - its own terms first, then wrap it in a thin `ReplacementStrategy` - `impl`, the same way these two do. - -### Worked example: `SharedSubtreeStrategy` - -This is the smallest real strategy in the module — a good template to copy -from. - -```rust -pub struct SharedSubtreeStrategy; - -impl ReplacementStrategy for SharedSubtreeStrategy { +The two methods have intentionally different responsibilities. + +`matches` answers: + +> Does this strategy have anything to offer for this target? + +`replacements` answers: + +> What are all semantically valid alternatives for this target? + +`replacements` must be **exhaustive, not ranked, and not cost-filtered**. + +--- + +### `CostModel` + +`CostModel` controls ranking, sizing, and cost-sensitive decisions. + +The current strategy code exercises the following hooks: + +```rust +fn rank_candidates( + &self, + intent: &AggIntent, + candidates: &[SketchKind], +) -> Vec; + +fn size_params( + &self, + kind: SketchKind, + intent: &AggIntent, + eps: f64, + delta: f64, +) -> SketchParams; + +fn realize_extension( + &self, + ext_kind: &str, + payload: &serde_json::Value, +) -> Implementation; + +fn readout_extension( + &self, + ext_kind: &str, + payload: &serde_json::Value, + col: &ColumnRef, +) -> SketchQuery; + +fn cse_recompute_cost( + &self, + candidate: &CseCandidate, +) -> f64; + +fn cse_shared_maintenance_cost( + &self, + candidate: &CseCandidate, +) -> f64; + +fn cse_share_decision( + &self, + candidate: &CseCandidate, +) -> ShareDecision; +``` + +*"Readout" here means the query-time **read** path — how an already-bound summary answers a query — as distinct from `realize_extension`, which decides how the summary is **built and maintained**. `realize_extension` and `readout_extension` are a matched pair: `realize_extension` picks what gets maintained, `readout_extension` says how to query it. + +A custom cost model does not necessarily need to override every hook. The current tests include a model that overrides only `rank_candidates`, relying on defaults for the rest. + +--- + +## 3. How the current pieces fit together + +There are two related paths in the current code. + +### Binding path + +The binding path needs one executable answer. + +Conceptually: + +```text +AggIntent + | + v +boundary::implementation_for_with(...) + | + v +one Implementation + | + v +bind::implement_tree_with(...) + | + v +one bound SummaryNode +``` + +A `CostModel` is used here because binding must eventually commit to one implementation. + +--- + +### Replacement-strategy path + +The strategy path exists to expose alternatives rather than immediately commit to one. + +Conceptually: + +```text +TargetSubDAG + | + v +ReplacementStrategy::matches(...) + | + v +ReplacementStrategy::replacements(...) + | + +-------------------------------+ + | | | +candidate A candidate B candidate C +``` + +The important rule is: + +> Strategies should reuse existing decision and binding logic where possible instead of reimplementing it. + +For example, `SketchFamilyStrategy` uses the existing boundary and binding machinery to enumerate each valid sketch realization. + +--- + +## 4. Adding a new `ReplacementStrategy` + +A new optimization should normally be introduced as a new implementation of `ReplacementStrategy`. + +Do not change the trait just because a new optimization is added. + +Start with this skeleton (illustrative — `MyStrategy` is not a real type in this crate): + +```rust +pub struct MyStrategy; + +impl ReplacementStrategy for MyStrategy { fn matches(&self, target: &TargetSubDAG<'_>) -> bool { - target.consumer_count >= 2 + // Return true only when this transformation is applicable. + todo!() } - fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { - if target.consumer_count < 2 { + fn replacements( + &self, + target: &TargetSubDAG<'_>, + ) -> Vec { + if !self.matches(target) { return Vec::new(); } - let count = target.consumer_count; - vec![ - ReplacementSubDAG { - // The already-interned `Rc` itself: reusing it verbatim *is* - // "build once and share" — no new node to construct. - replacement: Replacement::Rewrite(Rc::clone(target.root)), - rationale: format!( - "build once and share: share_common_subtrees already interned this \ - subtree once and reused it across {count} consumers — one build can \ - answer all of them instead of computing it {count} times" - ), - }, - ReplacementSubDAG { - // A structurally-identical but freshly-allocated `Rc`: same - // value (`PartialEq`), deliberately *not* the same pointer, - // representing "undo the sharing and recompute independently". - replacement: Replacement::Rewrite(Rc::new((**target.root).clone())), - rationale: format!( - "build independently: undo the sharing share_common_subtrees found and \ - recompute this subtree separately at each of its {count} consumers — \ - worth it only when independence outweighs the shared-maintenance cost, \ - a CostModel's call and not this strategy's" - ), - }, - ] + + // Enumerate every semantically valid replacement. + todo!() } } ``` -What to notice, as a pattern to reuse: - -1. **`matches` is a one-line predicate** over data `TargetSubDAG` already - carries (`consumer_count`). It doesn't re-derive anything the caller - could have told it directly. -2. **The strategy doesn't decide** which of the two candidates is better — - it returns both, unconditionally, whenever it matches. "Which is - cheaper" is explicitly deferred to `CostModel::cse_share_decision` - (today) / the search engine's cost-based ranking (once #252 lands). -3. **Both branches build a `Replacement::Rewrite`**, because sharing-vs-not - is a structural (pre-ASAP) choice, not a binding decision. Use - `Replacement::Summary` instead when your strategy's candidates are - post-ASAP binding choices — see `SketchFamilyStrategy` for that shape - (it wraps `bind::implement_tree_with` via a `CostModel` adapter, - `ForceSketchKind`, to steer the existing binding procedure toward each - candidate `SketchKind` in turn, rather than reimplementing binding). -4. **Guard `replacements` even though `matches` already checked the same - condition.** The `if target.consumer_count < 2 { return Vec::new() }` - inside `replacements` looks redundant next to `matches`, but it's what - keeps `replacements` safe to call on its own — see the contract above. - -### Testing a new strategy - -Follow the hand-rolled-fixture style `bind.rs`/`boundary.rs`/`cost_model.rs` -already use — build a small `QueryExpr` tree by hand, wrap it in a -`TargetSubDAG`, and assert on `matches`/`replacements` directly. At minimum: - -- **A positive case**: a target your strategy matches, asserting the full - set of `replacements()` it returns (not just its length — check which - `Replacement` variants and values came back). -- **A negative case**: a target your strategy does *not* match, asserting - `matches` returns `false` and `replacements` returns an empty `Vec`. -- If your strategy depends on cross-node context (like - `SharedSubtreeStrategy`'s `consumer_count`), add one test that builds it - from the real upstream computation instead of a hand-set field — see - `replacement.rs`'s own test that runs `SharedSubtreeStrategy` over a - `TargetSubDAG` built from real `share_common_subtrees` output, via a - small test-only traversal mirroring PR #247's `SharedSubexpressionRule` - dedup logic. A hand-set `consumer_count` alone can't catch a bug in how - that value gets computed for real. - -### What you are not responsible for - -- **Discovering `TargetSubDAG`s across a workload.** Your strategy receives - one `TargetSubDAG` at a time; walking a whole workload's tree to find - candidates is the search engine's job (#252) — or, today, ad hoc test - fixtures / `applicability.rs`'s existing traversal. -- **Ranking or filtering your own candidates.** Return everything valid; - a `CostModel` picks. -- **Wiring your strategy into anything automatically.** Until #252 lands, - a new `impl ReplacementStrategy` is just a type other code can construct - and call directly (e.g. from a test, or from `applicability.rs`) — there - is no registry or dispatcher yet. - -## `CostModel` - -Defined in [`crates/asap-aware-mapping/src/cost_model.rs`](../crates/asap-aware-mapping/src/cost_model.rs). -Relevant if you're wiring a deployment (ASAPCollector, ASAPFusion, …) with -real cost knowledge (bandwidth budget, memory footprint, site count, -observed drift, …) into candidate selection, instead of accepting this -crate's built-in static preferences. - -### The extension-point discipline - -`asap-aware-mapping` deliberately has no cost model *implementation* of its -own beyond [`DefaultCostModel`] — ranking real candidates by real cost needs -deployment knowledge this crate doesn't have and, per the crate's layering -invariant, shouldn't acquire. `CostModel` is the one interface every -deployment plugs its own knowledge into instead of forking `boundary`/`bind`. -Every method has a default body that reproduces today's static behavior -exactly — implement only the hooks you actually need to change: - -```rust -pub trait CostModel { - // No default: candidate selection needs a real answer. - fn rank_candidates(&self, intent: &AggIntent, candidates: &[SketchKind]) -> Vec; - - // Defaults to asap-plan's built-in per-family sizing formulas. - fn size_params(&self, kind: SketchKind, intent: &AggIntent, eps: f64, delta: f64) -> SketchParams { ... } - - // Defaults to Implementation::PassThrough for AggIntent::Extension. - fn realize_extension(&self, ext_kind: &str, payload: &serde_json::Value) -> Implementation { ... } - - // Panics if realize_extension ever returned Sketch without this being overridden too. - fn readout_extension(&self, ext_kind: &str, payload: &serde_json::Value, col: &ColumnRef) -> SketchQuery { ... } - - // CSE sharing cost hooks (issue #237) — defaults are structural-size / - // per-family-weight proxies; override with real cost knowledge. - fn cse_recompute_cost(&self, candidate: &CseCandidate) -> f64 { ... } - fn cse_shared_maintenance_cost(&self, candidate: &CseCandidate) -> f64 { ... } - fn cse_share_decision(&self, candidate: &CseCandidate) -> ShareDecision { ... } // composes the two above +There are four decisions to make. + +--- + +### 4.1 Define the target shape + +`matches` should contain the minimum structural and semantic checks needed to determine whether the strategy applies. + +For example, `SketchFamilyStrategy` only matches the aggregate shape that the existing binder can actually bind: + +- the node is an `Aggregate`, +- it has one aggregation intent, +- it does not have `HAVING`. + +A strategy that depends on cross-query context may additionally inspect `consumer_count`. + +For example: + +```rust +fn matches(&self, target: &TargetSubDAG<'_>) -> bool { + target.consumer_count >= 2 } ``` -### The contract, made explicit - -- **`rank_candidates` may reorder or drop entries, never invent one.** - Returning a `SketchKind` that wasn't in `candidates` will panic downstream - (`boundary::implementation_for_with` has no sizing logic for an unknown - kind). Returning an empty `Vec` is valid — it means "no candidate is - acceptable here." -- **Prefer overriding the narrowest hook.** `cse_share_decision`'s default - body composes `cse_recompute_cost` and `cse_shared_maintenance_cost` into - a Volcano/Cascades-style comparison (share iff maintaining one shared - summary costs no more than recomputing it everywhere it's used) — a - deployment with real cost numbers should override the two cost hooks and - keep the comparison, not reimplement the comparison itself. Override - `cse_share_decision` directly only for a genuinely different policy (e.g. - something other than a cost threshold). See - [`docs/cse-cost-model-decision.md`](cse-cost-model-decision.md) for the - full design discussion behind this split. -- **`realize_extension` and `readout_extension` are a matched pair.** If you - override `realize_extension` to return `Implementation::Sketch` for some - `ext_kind`, you MUST also override `readout_extension` for that same - `ext_kind` — the default panics loudly (not silently) the first time that - intent is actually read out, specifically so a mismatch is caught instead - of misinterpreting `payload`. -- **Delegate, don't reimplement, when adapting an existing `CostModel`.** - `SketchFamilyStrategy`'s `ForceSketchKind` (in `replacement.rs`) is the - reference example: it wraps another `&dyn CostModel`, overrides only - `rank_candidates` to force one specific kind first, and forwards every - other method to the wrapped model unchanged — so a deployment's own - sizing/extension behavior still applies while one method's behavior is - steered. Reach for this pattern before writing a `CostModel` from scratch - when you only need to adjust one hook's behavior. - -### Testing a new `CostModel` - -`cost_model.rs`'s own tests are the reference style: construct a minimal -`CostModel` `impl` inline in the test (only overriding the method(s) under -test), build fixture `QueryExpr`/`SummaryNode` values by hand, and assert -directly on the method's return value — e.g. -`custom_cost_model_can_reorder_candidates`, -`custom_cost_model_can_override_sizing_independently_of_ranking`, and -`cse_share_decision_default_body_composes_the_two_cost_hooks`. For the -default-preserving behavior of any method you don't override, add a test -asserting it matches the crate's built-in default exactly (see -`default_cost_model_preserves_static_order`) — that's the guarantee a -deployment relies on when it only overrides one hook. - -## `Matcher` (`boundary.rs`) - -Defined in [`crates/asap-aware-mapping/src/boundary.rs`](../crates/asap-aware-mapping/src/boundary.rs). -Much smaller surface than the other two, but shares the same "extension -point instead of a fork" role, so it's worth knowing it exists: +is enough for the current shared-subtree strategy. + +#### Guideline + +Keep `matches` cheap and unsurprising. + +It should answer applicability, not perform ranking or choose a winner. + +--- + +### 4.2 Enumerate every valid alternative + +`replacements` should return every semantically valid candidate for a matched target. + +For example, if a quantile can be represented by: + +```text +KLL +DDSketch +``` + +then both should be returned. + +Do not write: ```rust -pub trait Matcher { - fn is_satisfied_by(&self, required: &Implementation, available: &Implementation) -> bool; +if kll_is_cheaper { + vec![kll] +} else { + vec![ddsketch] +} +``` + +inside a strategy. + +That would throw away part of the search space before the cost model can compare complete plans. + +Instead: + +```rust +vec![kll, ddsketch] +``` + +and let costing decide later. + +--- + +### 4.3 Choose `Summary` vs. `Rewrite` + +Return: + +```rust +Replacement::Summary(...) +``` + +when the candidate is a fully bound post-ASAP summary. + +Return: + +```rust +Replacement::Rewrite(...) +``` + +when the candidate is a logical pre-ASAP rewrite. + +This distinction matters because a rewrite may enable more transformations later, while a bound summary represents a concrete summary realization. + +--- + +### 4.4 Add a rationale + +Every `ReplacementSubDAG` should explain why the candidate exists. + +Good: + +```rust +ReplacementSubDAG { + replacement: ..., + rationale: "derive service-level aggregate by rolling up the \ + mergeable service+region aggregate".into(), +} +``` + +Less useful: + +```rust +rationale: "candidate 2".into() +``` + +The rationale should help a developer understand a planner trace without reading the strategy implementation. + +Do not encode machine-readable state into the string. + +--- + +## 5. Strategy contract + +Every new strategy should follow these rules. + +### Rule 1: `matches == false` should be safe + +The existing strategies return an empty vector when `replacements` is called on a target they do not match. + +Follow the same convention: + +```rust +if !self.matches(target) { + return Vec::new(); +} +``` + +Do not panic simply because the caller skipped a prior `matches` call. + +--- + +### Rule 2: enumerate; do not rank + +A strategy owns **legality and enumeration**. + +A cost model owns **preference and costing**. + +This separation is the most important extension rule in this module. + +--- + +### Rule 3: do not duplicate an existing decision procedure + +If another module already knows how to determine whether something is legal or how to bind it, wrap that logic. + +Do not create a second implementation of the same semantics inside the strategy. + +The existing `SketchFamilyStrategy` is the model to follow: it reuses the boundary candidate list and the existing binder. + +--- + +### Rule 4: preserve semantics + +Every returned replacement must be semantically valid for the target. + +Cost differences do not justify semantic differences. + +If a transformation is only valid under additional summary properties, grouping assumptions, or accuracy constraints, check those conditions before returning the candidate. + +--- + +### Rule 5: a strategy does not need to discover the whole workload + +`TargetSubDAG` is passed into the strategy. + +The strategy is responsible for deciding what to do with that target, not for walking every workload root to discover targets. + +If your transformation requires context not currently represented in `TargetSubDAG`, that is a design question about target metadata or search/discovery — not a reason to hide a second workload traversal inside `replacements`. + +--- + +## 6. Example: current `SketchFamilyStrategy` + +`SketchFamilyStrategy` is the reference implementation for a strategy that produces bound summaries. + +Construction: + +```rust +let strategy = + SketchFamilyStrategy::default_cost_model(); +``` + +or with a custom cost model: + +```rust +let model = MyCostModel; // illustrative +let strategy = SketchFamilyStrategy::new(&model); +``` + +The strategy matches bindable aggregate nodes. + +At a high level: + +```text +Target Aggregate + | + v +extract AggIntent + | + v +boundary::implementation_for_with(...) + | + +--------------------------+ + | + | approximate sketch case + v +boundary::summary_candidates(...) + | + v +bind each sketch candidate separately + | + v +Vec +``` + +For an approximate quantile, the current candidate list includes both KLL and DDSketch. + +The strategy returns both, even if the cost model ranks one ahead of the other. + +For cases where boundary selection has only one realization, such as an exact accumulator or pass-through, the strategy returns that single realization. + +--- + +### Why `ForceSketchKind` exists + +The ordinary binding path asks the cost model to rank sketch candidates and then binds the first choice. + +That is correct for normal binding, but a replacement strategy needs to bind **each** valid sketch candidate. + +`ForceSketchKind` is a small `CostModel` adapter used for that purpose. It's real code — copied verbatim from `replacement.rs`. + +It overrides only ranking: + +```rust +fn rank_candidates( + &self, + intent: &AggIntent, + candidates: &[SketchKind], +) -> Vec { + let mut ranked = + self.inner.rank_candidates(intent, candidates); + + ranked.retain(|k| *k != self.kind); + ranked.insert(0, self.kind.clone()); + ranked } ``` -Unlike `CostModel`, `asap-plan` ships **no implementation and no default -body** for this trait at all — whether an available summary satisfies a -required one (e.g. does an available `DDSketch` satisfy a required `Kll` -request? does a multi-population accumulator satisfy a single-population -query?) depends on facts this crate deliberately doesn't settle: a pure -sketch-algebra answer and a deployment's own storage-layout rules can -legitimately disagree. If you need this, you're implementing the whole -trait for your deployment from scratch — there's no default to lean on and -no existing implementation in this crate to use as a worked example. - -## Where this fits - -`docs/asap_aware_mapping.md` is the design doc this vocabulary implements -(read it for *why* `ReplacementStrategy` is shaped as "exhaustive, not -decisive," why the eventual search is MEMO-style and iterative, and how a -`CostModel` fits into that search). This guide is the narrower "how do I -add one" doc — for `ReplacementStrategy`, aimed at whoever's picking up -#253/#254/#256/#257 next; for `CostModel`, aimed at whoever's wiring a real -deployment's cost knowledge into candidate selection. +Every other cost-model hook is forwarded to the underlying model. + +The result is: + +> bind this specific sketch family, but keep using the caller's real cost model for sizing and all other behavior. + +This is an important pattern to preserve when adding another strategy that needs to enumerate alternatives through an API that normally chooses one. + +--- + +## 7. Example: current `SharedSubtreeStrategy` + +`SharedSubtreeStrategy` is the reference implementation for a logical rewrite strategy. + +It applies when: + +```rust +target.consumer_count >= 2 +``` + +and returns two alternatives: + +```text +1. Build once and share. +2. Build independently for each consumer. +``` + +The shared candidate reuses the same `Rc`: + +```rust +Replacement::Rewrite(Rc::clone(target.root)) +``` + +The independent candidate creates a structurally equal but separately allocated node: + +```rust +Replacement::Rewrite( + Rc::new((**target.root).clone()) +) +``` + +This strategy does **not** decide whether sharing is cheaper. + +That decision belongs to the cost model. + +This example is useful when implementing transformations such as: + +- roll-up vs. recompute, +- shared grouping vs. per-group instances, +- semantic rewrite vs. original expression. + +Each strategy can expose the alternatives without choosing between them. + +--- + +## 8. Adding a custom `CostModel` + +Use a custom `CostModel` when you want to change preferences or cost assumptions without changing transformation legality. + +A minimal model can override only the hook it cares about. + +For example (illustrative — `PreferDDSketch` is not a real type in this crate): + +```rust +struct PreferDDSketch; + +impl CostModel for PreferDDSketch { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchKind], + ) -> Vec { + let mut ranked = candidates.to_vec(); + + if let Some(pos) = + ranked.iter().position( + |k| *k == SketchKind::DDSketch + ) + { + let dd = ranked.remove(pos); + ranked.insert(0, dd); + } + + ranked + } +} +``` + +Then inject it into code that accepts a `&dyn CostModel`: + +```rust +let model = PreferDDSketch; + +let strategy = + SketchFamilyStrategy::new(&model); + +let replacements = + strategy.replacements(&target); +``` + +Important: changing `rank_candidates` changes the preferred ordering, but `SketchFamilyStrategy` still enumerates every valid sketch candidate. + +A custom cost model should not change which alternatives are semantically legal. + +--- + +## 9. Which `CostModel` hook should I implement? + +Use this as a practical guide. + +### `rank_candidates` + +Use when you want to change the preference among valid sketch families. + +Example: + +```text +KLL vs. DDSketch +HLL vs. Theta vs. KMV +``` + +Signature: + +```rust +fn rank_candidates( + &self, + intent: &AggIntent, + candidates: &[SketchKind], +) -> Vec; +``` + +The returned vector should rank candidates from most to least preferred. + +It should rank candidates that were supplied to it rather than invent unrelated sketch kinds. + +--- + +### `size_params` + +Use when the sketch family is already known and you want to choose its parameters from an accuracy target. + +Signature: + +```rust +fn size_params( + &self, + kind: SketchKind, + intent: &AggIntent, + eps: f64, + delta: f64, +) -> SketchParams; +``` + +Typical uses include: + +- choosing KLL capacity, +- choosing HLL precision, +- selecting sketch-specific error parameters. + +Conceptually: + +```text +SketchKind + AggIntent + accuracy target + | + v + SketchParams +``` + +--- + +### `realize_extension` + +Use for extension-defined implementation kinds. + +```rust +fn realize_extension( + &self, + ext_kind: &str, + payload: &serde_json::Value, +) -> Implementation; +``` + +This is the hook for turning an extension description into a concrete `Implementation`. + +Use it for implementation families that are intentionally outside the built-in enum dispatch. + +--- + +### `readout_extension` + +Use when an extension-defined summary also needs custom query/readout behavior. + +```rust +fn readout_extension( + &self, + ext_kind: &str, + payload: &serde_json::Value, + col: &ColumnRef, +) -> SketchQuery; +``` + +This complements `realize_extension`: realization defines what gets maintained; readout defines how it is queried (see the definition in §2's `CostModel` glossary entry if "readout" is unfamiliar). + +--- + +### `cse_recompute_cost` + +Use to estimate the cost of computing a common subtree independently at each consumer. + +```rust +fn cse_recompute_cost( + &self, + candidate: &CseCandidate, +) -> f64; +``` + +--- + +### `cse_shared_maintenance_cost` + +Use to estimate the cost of computing and maintaining a shared subtree. + +```rust +fn cse_shared_maintenance_cost( + &self, + candidate: &CseCandidate, +) -> f64; +``` + +--- + +### `cse_share_decision` + +Use when the current binding/planning path needs the final share-vs.-recompute decision. + +```rust +fn cse_share_decision( + &self, + candidate: &CseCandidate, +) -> ShareDecision; +``` + +The replacement-strategy layer should still expose both valid alternatives where appropriate. This hook is the cost-sensitive decision point for code paths that need to commit to one answer. + +--- + +## 10. `Matcher` (`boundary.rs`) + +There is a third extension point in this crate, much smaller in surface area than `ReplacementStrategy` or `CostModel`, but worth knowing about: + +```rust +pub trait Matcher { + fn is_satisfied_by(&self, required: &Implementation, available: &Implementation) -> bool; +} +``` + +`Matcher` decides whether an already-available `Implementation` satisfies a required one (e.g. does an available `DDSketch` satisfy a request for `Kll`? does a multi-population accumulator satisfy a single-population query?). + +Unlike `CostModel`, this crate ships **no default body and no implementation** for `Matcher` at all — the answer depends on facts this crate deliberately doesn't settle (a pure sketch-algebra answer and a deployment's own storage-layout rules can legitimately disagree). If you need this, you're implementing the whole trait for your deployment from scratch; there's no default to lean on and no existing implementation in this crate to copy from. + +--- + +## 11. Adding both a strategy and a cost model + +Some features require both. + +For example, suppose we add: + +```text +roll up a finer group-by +vs. +compute the coarser group-by independently +``` + +The responsibilities should be divided as follows. + +### Strategy + +The new strategy determines: + +- whether the two group-bys have the required relationship, +- whether the aggregation is mergeable, +- whether roll-up preserves semantics, +- and then returns both valid alternatives. + +Conceptually: + +```rust +vec![ + rollup_candidate, + recompute_candidate, +] +``` + +### Cost model + +The cost model determines: + +- maintenance cost of the finer-grained summary, +- cost of roll-up, +- cost of independent computation, +- expected query frequency, +- and which complete plan is cheaper. + +Do not encode: + +```text +"only return roll-up when roll-up is cheaper" +``` + +inside the strategy. + +That turns a cost decision into a legality decision and prevents later global plan search from seeing both options. + +--- + +## 12. Adding a new sketch family + +A new sketch family generally touches more than `ReplacementStrategy`. + +The strategy should not maintain its own private list of sketch kinds. + +`SketchFamilyStrategy` obtains sketch alternatives through the existing boundary interface: + +```rust +summary_candidates(intent) +``` + +and binds them through the normal binder. + +Therefore, when adding a new built-in sketch family, the intended flow is: + +```text +1. Teach the boundary layer that the sketch is a valid candidate + for the relevant AggIntent. + +2. Teach the cost model how to rank and size it. + +3. Ensure the binder can realize the sketch family. + +4. SketchFamilyStrategy will then enumerate it through the + existing candidate/binding path. +``` + +This keeps one source of truth for sketch applicability. + +Do not special-case the new sketch inside `SketchFamilyStrategy` unless the strategy itself needs fundamentally new behavior. + +--- + +## 13. Using a strategy + +The basic calling pattern is: + +```rust +let target = TargetSubDAG::new(&root); +let strategy = + SketchFamilyStrategy::default_cost_model(); + +if strategy.matches(&target) { + let candidates = + strategy.replacements(&target); + + for candidate in candidates { + println!("{}", candidate.rationale); + } +} +``` + +A caller may also safely call `replacements` directly and treat an empty vector as "not applicable": + +```rust +let candidates = + strategy.replacements(&target); + +if candidates.is_empty() { + // No candidate from this strategy. +} +``` + +For strategies that require workload context: + +```rust +let target = + TargetSubDAG::with_consumer_count( + &root, + consumer_count, + ); +``` + +The caller is responsible for providing correct cross-workload metadata. + +--- + +## 14. Testing a new strategy + +Every new strategy should have focused tests for its contract. + +At minimum, test the following. + +### Applicability + +A matching target should satisfy: + +```rust +assert!(strategy.matches(&target)); +``` + +A non-matching target should satisfy: + +```rust +assert!(!strategy.matches(&target)); +``` + +--- + +### Safe non-match behavior + +Also call `replacements` on a non-matching target: + +```rust +assert!( + strategy.replacements(&target).is_empty() +); +``` + +This verifies that the strategy does not depend on callers always invoking `matches` first. + +--- + +### Exhaustive candidate enumeration + +If the target has N valid alternatives: + +```rust +let replacements = + strategy.replacements(&target); + +assert_eq!(replacements.len(), N); +``` + +Check the identities or kinds of all candidates, not just the preferred one. + +The current sketch-family tests explicitly verify that: + +- quantile returns both KLL and DDSketch, +- cardinality returns HLL, Theta, and KMV. + +This is the most important regression test for a strategy. + +--- + +### Rationale + +Verify that every candidate has a non-empty rationale: + +```rust +assert!( + replacements + .iter() + .all(|r| !r.rationale.is_empty()) +); +``` + +For a strategy whose explanation includes important context, also test that context. + +For example, the shared-subtree tests verify that the consumer count appears in the rationale. + +--- + +### Structural semantics + +For logical rewrites, test the structural property that distinguishes the alternatives. + +For example, the current shared-subtree tests verify: + +```rust +Rc::ptr_eq(shared, &q) +``` + +for the shared candidate, and: + +```rust +!Rc::ptr_eq(independent, &q) +``` + +plus structural equality for the independent candidate. + +Do not test only the rationale string; test the actual replacement semantics. + +--- + +### Custom cost model behavior + +If a strategy accepts a cost model, verify that a custom model changes the intended costing behavior without changing the exhaustive candidate set. + +The current sketch strategy does exactly this: + +```text +custom model prefers DDSketch + | + v +strategy still returns +KLL + DDSketch +``` + +That is the expected separation between enumeration and ranking. + +--- + +## 15. Testing a new cost model + +A cost-model test should focus on the hook being customized. + +For ranking: + +```rust +let ranked = + model.rank_candidates( + &intent, + &[SketchKind::Kll, + SketchKind::DDSketch], + ); + +assert_eq!( + ranked[0], + SketchKind::DDSketch +); +``` + +Then test integration through a consumer of the cost model. + +For example: + +```rust +let strategy = + SketchFamilyStrategy::new(&model); + +let replacements = + strategy.replacements(&target); +``` + +The important assertion is usually not that other valid candidates disappeared. They should not. + +Instead verify that: + +- the model changes ordering or parameters as intended, +- all legal candidates remain available to the replacement layer. + +For sizing, test representative accuracy targets and assert the resulting `SketchParams`. + +For CSE costing, create a representative `CseCandidate` and test recompute cost, shared-maintenance cost, and the resulting `ShareDecision`. + +--- + +## 16. Common mistakes + +### Mistake: choosing the cheapest candidate inside a strategy + +Wrong: + +```rust +fn replacements(...) -> Vec { + vec![choose_cheapest_candidate()] +} +``` + +Right: + +```rust +fn replacements(...) -> Vec { + all_valid_candidates() +} +``` + +--- + +### Mistake: maintaining a second sketch-applicability table + +If the boundary layer already defines which sketch families satisfy an `AggIntent`, reuse that source. + +Otherwise the binder and replacement strategy can silently disagree. + +--- + +### Mistake: reimplementing binding inside a strategy + +If the candidate should produce a normal `SummaryNode`, use the existing binding path. + +A strategy should steer or wrap that path when necessary, not recreate schema derivation, column resolution, readout construction, or parameter sizing. + +--- + +### Mistake: treating `rationale` as planner state + +`rationale` is explanatory text. + +If downstream logic needs a fact, represent it in the plan or another typed structure instead of parsing the rationale. + +--- + +### Mistake: hiding workload traversal in a strategy + +A `ReplacementStrategy` operates on the `TargetSubDAG` it is given. + +Workload-wide target discovery, deduplication, and consumer counting are separate concerns. + +--- + +### Mistake: assuming `Rc` structural equality and identity mean the same thing + +For CSE-style decisions, pointer identity can encode actual sharing. + +Two `Rc` values can be structurally equal but deliberately represent independent computation. + +Use the distinction intentionally. + +--- + +## 17. Extension checklist + +When adding a new strategy: + +- [ ] Define the exact target shape. +- [ ] Implement `ReplacementStrategy::matches`. +- [ ] Implement `ReplacementStrategy::replacements`. +- [ ] Return every semantically valid replacement. +- [ ] Return an empty vector for non-matching targets. +- [ ] Use `Replacement::Summary` for bound post-ASAP output. +- [ ] Use `Replacement::Rewrite` for logical pre-ASAP alternatives. +- [ ] Add a useful rationale to every candidate. +- [ ] Reuse existing legality/binding logic instead of duplicating it. +- [ ] Keep ranking and cost-based pruning out of the strategy. +- [ ] Test positive and negative applicability. +- [ ] Test exhaustive enumeration. +- [ ] Test the actual structural semantics of each replacement. +- [ ] Test behavior with a custom cost model if the strategy uses one. + +When adding a new cost model: + +- [ ] Override only the hooks whose behavior should change. +- [ ] Keep semantic applicability outside the cost model. +- [ ] Use `rank_candidates` for family preference. +- [ ] Use `size_params` for accuracy-to-parameter mapping. +- [ ] Use extension hooks for extension-defined implementations/readouts. +- [ ] Use CSE hooks for recompute-vs.-sharing costs. +- [ ] Test the hook directly. +- [ ] Test integration through a consumer such as `SketchFamilyStrategy`. +- [ ] Verify that changing cost preferences does not silently remove valid replacement candidates. + +--- + +## 18. Current extension map + +Use this table to find the right place for a change. + +| I want to... | Primary extension point | +|---|---| +| Add a new logical optimization | new `impl ReplacementStrategy` | +| Add a new replacement for an existing target shape | `ReplacementStrategy::replacements` | +| Change when a strategy applies | `ReplacementStrategy::matches` | +| Add a new built-in sketch candidate | boundary summary-candidate mapping | +| Prefer one sketch family over another | `CostModel::rank_candidates` | +| Change sketch sizing for an accuracy target | `CostModel::size_params` | +| Add extension-defined implementation behavior | `CostModel::realize_extension` | +| Add extension-defined readout behavior | `CostModel::readout_extension` | +| Change CSE recomputation cost | `CostModel::cse_recompute_cost` | +| Change shared-maintenance cost | `CostModel::cse_shared_maintenance_cost` | +| Change current share/recompute choice | `CostModel::cse_share_decision` | +| Decide whether an available implementation satisfies a required one | `impl Matcher` | +| Produce a normal bound summary candidate | reuse `bind::implement_tree_with` | +| Enumerate valid sketch kinds | reuse `boundary::summary_candidates` | +| Build a target with no workload context | `TargetSubDAG::new` | +| Build a target with known sharing context | `TargetSubDAG::with_consumer_count` | + +--- + +## 19. Design rule to remember + +If there is one rule to keep in mind when extending this layer, it is: + +> **Strategies define the valid search space; cost models express preferences within that search space.** + +Keeping those responsibilities separate makes new optimizations composable, keeps one source of truth for semantics, and allows a future whole-plan search engine to compare interactions between choices rather than inheriting irreversible local decisions. From efdab66f578d035fbfa51a063170a1d8a6ae9240 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 23 Aug 2026 12:41:12 -0600 Subject: [PATCH 05/49] docs(asap-aware-mapping): rename extension-points-dev-guide.md -> ASAP-aware-mapping-developer-guide.md --- ...-points-dev-guide.md => ASAP-aware-mapping-developer-guide.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{extension-points-dev-guide.md => ASAP-aware-mapping-developer-guide.md} (100%) diff --git a/docs/extension-points-dev-guide.md b/docs/ASAP-aware-mapping-developer-guide.md similarity index 100% rename from docs/extension-points-dev-guide.md rename to docs/ASAP-aware-mapping-developer-guide.md From 706e8b7f265f801fce98c8d70068c0d92379f374 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 23 Aug 2026 12:43:15 -0600 Subject: [PATCH 06/49] docs(asap-aware-mapping): define each CostModel hook individually in the glossary Was a bare signature dump with only readout_extension explained. Each of the 7 hooks now gets a one-line definition plus its default behavior (or 'no default' for rank_candidates, the one hook every CostModel must implement) right next to its signature, so the glossary entry is useful on its own without jumping to SS9's fuller per-hook reference. --- docs/ASAP-aware-mapping-developer-guide.md | 74 ++++++++++------------ 1 file changed, 34 insertions(+), 40 deletions(-) diff --git a/docs/ASAP-aware-mapping-developer-guide.md b/docs/ASAP-aware-mapping-developer-guide.md index 8a002a07..a65f7063 100644 --- a/docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/ASAP-aware-mapping-developer-guide.md @@ -160,55 +160,49 @@ The two methods have intentionally different responsibilities. ### `CostModel` -`CostModel` controls ranking, sizing, and cost-sensitive decisions. +`CostModel` controls ranking, sizing, and cost-sensitive decisions. Every hook but one has a default body that reproduces this crate's built-in static behavior — override only what you need to change: -The current strategy code exercises the following hooks: +- **`rank_candidates`** — order a set of sketch candidates for one `AggIntent`, best choice first. **No default** — this is the one hook every `CostModel` must implement; candidate selection needs a real answer from somewhere. -```rust -fn rank_candidates( - &self, - intent: &AggIntent, - candidates: &[SketchKind], -) -> Vec; + ```rust + fn rank_candidates(&self, intent: &AggIntent, candidates: &[SketchKind]) -> Vec; + ``` -fn size_params( - &self, - kind: SketchKind, - intent: &AggIntent, - eps: f64, - delta: f64, -) -> SketchParams; +- **`size_params`** — pick concrete parameters (e.g. sketch capacity) for one already-chosen `SketchKind`, given an accuracy target `(eps, delta)`. Default: `boundary::default_size_params`, this crate's built-in per-family sizing formulas. -fn realize_extension( - &self, - ext_kind: &str, - payload: &serde_json::Value, -) -> Implementation; + ```rust + fn size_params(&self, kind: SketchKind, intent: &AggIntent, eps: f64, delta: f64) -> SketchParams; + ``` -fn readout_extension( - &self, - ext_kind: &str, - payload: &serde_json::Value, - col: &ColumnRef, -) -> SketchQuery; +- **`realize_extension`** — decide what post-ASAP `Implementation` a deployment-defined `AggIntent::Extension` maps to (a shape core has no built-in opinion on). Default: `Implementation::PassThrough`. -fn cse_recompute_cost( - &self, - candidate: &CseCandidate, -) -> f64; + ```rust + fn realize_extension(&self, ext_kind: &str, payload: &serde_json::Value) -> Implementation; + ``` -fn cse_shared_maintenance_cost( - &self, - candidate: &CseCandidate, -) -> f64; +- **`readout_extension`** — build the query-time read for an `Extension` intent this same `CostModel` already realized as `Sketch` via `realize_extension`. "Readout" means the query-time **read** path — how an already-bound summary answers a query — as distinct from `realize_extension`, which decides how the summary is **built and maintained**; the two are a matched pair, so overriding `realize_extension` to return `Sketch` for some `ext_kind` requires overriding this for that same `ext_kind` too. Default: panics (deliberately — a silent wrong answer here is worse than a loud crash) if that pairing was left incomplete. -fn cse_share_decision( - &self, - candidate: &CseCandidate, -) -> ShareDecision; -``` + ```rust + fn readout_extension(&self, ext_kind: &str, payload: &serde_json::Value, col: &ColumnRef) -> SketchQuery; + ``` + +- **`cse_recompute_cost`** — estimate the one-time cost of recomputing a CSE candidate's subtree independently at a single consumer. Default: `default_cse_recompute_cost`, a structural-size proxy. + + ```rust + fn cse_recompute_cost(&self, candidate: &CseCandidate) -> f64; + ``` + +- **`cse_shared_maintenance_cost`** — estimate the cost of maintaining one shared summary continuously for the life of the workload. Default: `default_cse_shared_maintenance_cost`, a per-family weight table. + + ```rust + fn cse_shared_maintenance_cost(&self, candidate: &CseCandidate) -> f64; + ``` + +- **`cse_share_decision`** — decide whether to share one summary across all of a CSE candidate's consumers, or recompute independently at each. Default: composes the two cost hooks above (share iff shared-maintenance cost ≤ total recompute cost across every consumer) — override the two cost hooks and keep this comparison unless you need a genuinely different policy, not a different cost input. -*"Readout" here means the query-time **read** path — how an already-bound summary answers a query — as distinct from `realize_extension`, which decides how the summary is **built and maintained**. `realize_extension` and `readout_extension` are a matched pair: `realize_extension` picks what gets maintained, `readout_extension` says how to query it. + ```rust + fn cse_share_decision(&self, candidate: &CseCandidate) -> ShareDecision; + ``` A custom cost model does not necessarily need to override every hook. The current tests include a model that overrides only `rank_candidates`, relying on defaults for the rest. From 4b40b5737e0da5cdfcec6b9a163610345a721125 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 23 Aug 2026 12:56:45 -0600 Subject: [PATCH 07/49] docs: move dev guide into docs/developer_docs/ Companion to docs/reorganize-into-folders (main) and docs/replacement-strategy-search-design (#258), which move the design docs into docs/design_docs/ and docs/user-guide.md into docs/user-guide/. All three branches converge on the same final docs/ layout. Updated the link to the design doc to point at its eventual docs/design_docs/asap_aware_mapping.md location -- note this link is only valid once this branch merges after (or alongside) the reorg; on this branch alone, asap_aware_mapping.md is still at its old top-level docs/ path, since moving it is #258's change, not this one's. --- docs/{ => developer_docs}/ASAP-aware-mapping-developer-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename docs/{ => developer_docs}/ASAP-aware-mapping-developer-guide.md (99%) diff --git a/docs/ASAP-aware-mapping-developer-guide.md b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md similarity index 99% rename from docs/ASAP-aware-mapping-developer-guide.md rename to docs/developer_docs/ASAP-aware-mapping-developer-guide.md index a65f7063..a52c4cfa 100644 --- a/docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md @@ -10,7 +10,7 @@ It is written for developers who want to: - add a new kind of replacement without duplicating existing planner logic, - and write the tests expected for a new extension. -The focus here is the **current code interfaces and their contracts**. For the higher-level motivation and future search design, see the separate design document, [`docs/asap_aware_mapping.md`](asap_aware_mapping.md). +The focus here is the **current code interfaces and their contracts**. For the higher-level motivation and future search design, see the separate design document, [`docs/design_docs/asap_aware_mapping.md`](../design_docs/asap_aware_mapping.md). Code samples named `My*` or `Prefer*` (`MyStrategy`, `MyCostModel`, `PreferDDSketch`, …) below are illustrative sketches of a pattern, not code that ships in this crate. Samples that name a real type (`SketchFamilyStrategy`, `ForceSketchKind`, `SharedSubtreeStrategy`, …) are copied verbatim from `replacement.rs`/`cost_model.rs`. From e73fff1ca01b57b42d9f6be7432dccc145b96255 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 23 Aug 2026 13:46:25 -0600 Subject: [PATCH 08/49] docs(asap-aware-mapping): address zzylol's review comments on the dev guide Ten review comments, addressed with clarifying edits (two -- the cost-struct suggestion and the "boundary is a weird name" rename -- are real design/code decisions out of scope for a docs pass, noted rather than silently acted on): - CostModel::size_params: explain why sizing counts as a cost decision, not just configuration -- same "real-world cost knowledge this crate can't hardcode" reasoning as ranking. - realize_extension: define AggIntent::Extension (issue #131's deployment escape hatch) with a concrete ext_kind/payload example. - cse_recompute_cost: note the f64-vs-struct question honestly, point at it as a separate future issue rather than deciding here. - Sec 3 "How the current pieces fit together": state explicitly which path is production (binding) vs. additive (replacement-strategy), and connect ReplacementSubDAG/TargetSubDAG to the design doc's alternatives/candidates language. - Sec 4.1 Guideline: disambiguate "applicability" (plain English) from applicability.rs's formal ApplicabilityRule/ApplicabilityFinding concept. - ForceSketchKind: add the missing "why" paragraph before the code block -- what problem forcing rank_candidates actually solves. - SharedSubtreeStrategy's two alternatives: explain why the strategy must enumerate both even though CSE detection already makes "share" free to construct and CostModel::cse_share_decision already has an answer for the production binding path -- neither fact licenses skipping enumeration. - Matcher: add a concrete "existing DDSketch answering a Kll request" example -- what question it answers and why this crate can't answer it itself. --- .../ASAP-aware-mapping-developer-guide.md | 42 +++++++++++++++---- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md index a52c4cfa..0a6455a5 100644 --- a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md @@ -160,7 +160,7 @@ The two methods have intentionally different responsibilities. ### `CostModel` -`CostModel` controls ranking, sizing, and cost-sensitive decisions. Every hook but one has a default body that reproduces this crate's built-in static behavior — override only what you need to change: +`CostModel`'s job is broader than its name suggests — it's the one extension point every deployment-specific numeric/configuration decision plugs into, not just "which candidate is cheapest." Sizing counts as a cost decision for the same reason ranking does: a bigger sketch (more memory, more update cost) buys more accuracy, and how you want to spend that budget is exactly the kind of real-world cost knowledge this crate deliberately doesn't hardcode — see the crate doc's layering note at the top of `cost_model.rs` (`asap-plan` depends only on `asap_ir`, never on a runtime or deployment model, so it can't know real costs itself). Every hook but one has a default body that reproduces this crate's built-in static behavior — override only what you need to change: - **`rank_candidates`** — order a set of sketch candidates for one `AggIntent`, best choice first. **No default** — this is the one hook every `CostModel` must implement; candidate selection needs a real answer from somewhere. @@ -168,7 +168,7 @@ The two methods have intentionally different responsibilities. fn rank_candidates(&self, intent: &AggIntent, candidates: &[SketchKind]) -> Vec; ``` -- **`size_params`** — pick concrete parameters (e.g. sketch capacity) for one already-chosen `SketchKind`, given an accuracy target `(eps, delta)`. Default: `boundary::default_size_params`, this crate's built-in per-family sizing formulas. +- **`size_params`** — pick concrete parameters (e.g. sketch capacity) for one already-chosen `SketchKind`, given an accuracy target `(eps, delta)`. This is a separate hook from `rank_candidates` specifically so a deployment can override *just* sizing (e.g. an empirically-tuned table, or discrete capacity rungs a downstream catalog requires) without also forking candidate selection — same "one extension point per decision" shape as everything else in this trait. Default: `boundary::default_size_params`, this crate's built-in per-family sizing formulas. ```rust fn size_params(&self, kind: SketchKind, intent: &AggIntent, eps: f64, delta: f64) -> SketchParams; @@ -176,6 +176,20 @@ The two methods have intentionally different responsibilities. - **`realize_extension`** — decide what post-ASAP `Implementation` a deployment-defined `AggIntent::Extension` maps to (a shape core has no built-in opinion on). Default: `Implementation::PassThrough`. + `AggIntent::Extension { ext_kind: String, payload: serde_json::Value }` is the escape hatch for an intent shape only *your* deployment needs — core's `AggIntent` enum deliberately doesn't grow a variant for every capability a single deployment wants (issue #131), so instead a deployment tags its own shape with a `ext_kind` string and puts whatever it needs in `payload`; core treats both opaquely. Example: a deployment wants an approximate-frequency intent core has no variant for. It tags queries with `Extension { ext_kind: "frequency".into(), payload: ... }`, then overrides `realize_extension` to recognize that tag: + + ```rust + fn realize_extension(&self, ext_kind: &str, _payload: &serde_json::Value) -> Implementation { + if ext_kind == "frequency" { + Implementation::Sketch { kind: SketchKind::CountSketch, params: /* ... */ } + } else { + Implementation::PassThrough // fall back to the default for anything else + } + } + ``` + + Every `ext_kind` your deployment doesn't recognize should still fall through to `Implementation::PassThrough` (the default), not panic — only the `ext_kind`s you've deliberately implemented get a real realization. + ```rust fn realize_extension(&self, ext_kind: &str, payload: &serde_json::Value) -> Implementation; ``` @@ -186,7 +200,7 @@ The two methods have intentionally different responsibilities. fn readout_extension(&self, ext_kind: &str, payload: &serde_json::Value, col: &ColumnRef) -> SketchQuery; ``` -- **`cse_recompute_cost`** — estimate the one-time cost of recomputing a CSE candidate's subtree independently at a single consumer. Default: `default_cse_recompute_cost`, a structural-size proxy. +- **`cse_recompute_cost`** — estimate the one-time cost of recomputing a CSE candidate's subtree independently at a single consumer. Default: `default_cse_recompute_cost`, a structural-size proxy. Returns a bare `f64` today (a unitless scalar compared directly against `cse_shared_maintenance_cost`'s own `f64`) — a richer cost type (e.g. a struct separating CPU/memory/network) would let deployments compare along more than one axis, but is a real signature change across every hook in this trait, not a doc fix; worth its own issue rather than deciding here. ```rust fn cse_recompute_cost(&self, candidate: &CseCandidate) -> f64; @@ -210,7 +224,12 @@ A custom cost model does not necessarily need to override every hook. The curren ## 3. How the current pieces fit together -There are two related paths in the current code. +There are two related paths in the current code, and it's worth being explicit about how they relate before looking at either one, since neither name says it outright: **the binding path is older and still the one thing that actually runs in production; the replacement-strategy path is new and additive, not a replacement for it** (`replacement.rs`'s own module docs put this as "why this exists alongside `boundary`/`bind`, not instead of them"). Both start from the same input (an `AggIntent`) and both eventually reuse the exact same underlying decision logic (`boundary::implementation_for_with`, `bind::implement_tree_with`) — they differ only in *how much of the answer* they keep: + +- The **binding path** is what runs today when a query actually gets bound: it commits to one `Implementation` and discards every alternative a `CostModel` didn't pick, because something has to actually execute. +- The **replacement-strategy path** is what `ReplacementStrategy::replacements()` (§2) produces: instead of committing to one, it *keeps every alternative* the binding path would have thrown away, packaged as `ReplacementSubDAG`s. + +This is exactly the "alternatives"/"candidates" language in [the design doc](../design_docs/asap_aware_mapping.md): a `ReplacementSubDAG` **is** one candidate; a `TargetSubDAG` with its full `replacements()` list **is** the set of alternatives for one spot in the plan. The design doc describes a *future* search (not built yet, tracked separately) that would compare whole candidate plans built from these alternatives — the two paths below are what that future search would draw on, not the search itself. ### Binding path @@ -327,7 +346,7 @@ is enough for the current shared-subtree strategy. Keep `matches` cheap and unsurprising. -It should answer applicability, not perform ranking or choose a winner. +It should answer *whether the strategy applies here* in the plain English sense — not `applicability.rs`'s formal `ApplicabilityRule`/`ApplicabilityFinding` concept (issue #247, a separate reporting layer covered in the design doc's "Applicability reporting" section). `matches` isn't that machinery and doesn't need to produce anything it consumes; it should also not perform ranking or choose a winner. --- @@ -536,7 +555,7 @@ That is correct for normal binding, but a replacement strategy needs to bind **e `ForceSketchKind` is a small `CostModel` adapter used for that purpose. It's real code — copied verbatim from `replacement.rs`. -It overrides only ranking: +The trick: `implement_tree_with` always binds whichever candidate `rank_candidates` puts *first* — there's no way to tell it directly "bind KLL this time." So to bind KLL specifically, wrap the real cost model in a `ForceSketchKind { kind: Kll, inner: real_cost_model }` and pass *that* into `implement_tree_with` instead. `ForceSketchKind::rank_candidates` forces `kind` (here, `Kll`) to the front of whatever `inner` would have ranked, so `implement_tree_with`'s "take the first candidate" logic ends up binding `Kll` — while every other decision (sizing, extension realization, …) still comes from `inner` unchanged, since only `rank_candidates` is overridden. Do this once per candidate (`ForceSketchKind { kind: Kll, .. }`, then separately `ForceSketchKind { kind: DDSketch, .. }`) and each gets bound through the exact same real machinery, one at a time, without touching `implement_tree_with`'s internals at all. It overrides only ranking: ```rust fn rank_candidates( @@ -596,7 +615,10 @@ Replacement::Rewrite( This strategy does **not** decide whether sharing is cheaper. -That decision belongs to the cost model. +That decision belongs to the cost model — and today, `CostModel::cse_share_decision`'s default body already makes it, via a real cost comparison (§9), every time the *production binding path* (§3) sees a shared subtree. So it's fair to ask: isn't "build independently" (#2) already the exception, not the default — and doesn't that comparison already decide things? It's worth being precise about what "default" means here: + +- `target.root` arriving with `consumer_count >= 2` means CSE detection (`share_common_subtrees`) already merged it onto one shared `Rc` *before* this strategy ever runs — so "build once and share" (#1) costs nothing to produce (`Rc::clone` of what's already there); "build independently" (#2) is the one requiring real work (an actual deep clone). +- But `cse_share_decision`'s cost comparison is a *separate, already-existing* mechanism that only the production binding path consults — it isn't something `SharedSubtreeStrategy` calls or defers to. This strategy has to enumerate **both** regardless of which one is cheap to construct or which one that other comparison would currently pick, because per §1's core rule, a strategy is never allowed to pre-decide based on cost — collapsing to just #1 here would be exactly the "cost-based pruning inside a strategy" anti-pattern §5 Rule 2 forbids, even though today's separate binding path already has an answer. The whole reason this module exists (§0) is to keep both alternatives around for a future search to compare *in the context of a full plan* — which might disagree with `cse_share_decision`'s isolated, pairwise-only comparison. This example is useful when implementing transformations such as: @@ -809,9 +831,11 @@ pub trait Matcher { } ``` -`Matcher` decides whether an already-available `Implementation` satisfies a required one (e.g. does an available `DDSketch` satisfy a request for `Kll`? does a multi-population accumulator satisfy a single-population query?). +`Matcher` answers a different question than everything above it: not "how do I *build* a summary for this intent" (that's `boundary`/`bind`/`ReplacementStrategy`), but "is there already a summary sitting around somewhere that answers this without building anything new?" This is the query-optimization-literature "answering queries using views" question, applied to summaries — the same idea as a database reusing an existing materialized view or index instead of recomputing from scratch. + +Concrete example: a deployment already has a `DDSketch` built earlier for some other query on `latency`, and a new query needs `Kll` quantiles on `latency`. Can the existing `DDSketch` answer it without building a new `Kll` sketch? A pure sketch-algebra answer says yes — both are quantile sketches, mutually substitutable at the query level. But a deployment with its own storage-layout rules might say no, e.g. if its inventory ties a stored summary to a specific algorithm identity it won't re-interpret. `Matcher::is_satisfied_by(required, available)` is where a deployment plugs in whichever answer is actually true for its own storage layer — `required` is what a query needs (an `Implementation` `boundary`/`bind` computed), `available` is what already exists somewhere in that deployment's inventory. -Unlike `CostModel`, this crate ships **no default body and no implementation** for `Matcher` at all — the answer depends on facts this crate deliberately doesn't settle (a pure sketch-algebra answer and a deployment's own storage-layout rules can legitimately disagree). If you need this, you're implementing the whole trait for your deployment from scratch; there's no default to lean on and no existing implementation in this crate to copy from. +Unlike `CostModel`, this crate ships **no default body and no implementation** for `Matcher` at all — the answer genuinely depends on facts this crate deliberately doesn't settle (see the example above: a pure sketch-algebra answer and a deployment's own storage-layout rules can legitimately disagree, and this crate has no inventory concept of its own to judge between them). If you need this, you're implementing the whole trait for your deployment from scratch; there's no default to lean on and no existing implementation in this crate to copy from. --- From c89b3c55aad4cc293c3be8f9d9ce7219a02786a2 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 23 Aug 2026 13:47:00 -0600 Subject: [PATCH 09/49] docs(asap-aware-mapping): clarify consumer_count is a structural reference count, not a runtime one Addresses the comment on TargetSubDAG.consumer_count -- 'refer or consume? can you give an example' -- with a worked example: two queries sharing a CSE'd rate() subtree, not execution counting. --- docs/developer_docs/ASAP-aware-mapping-developer-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md index 0a6455a5..5a134e00 100644 --- a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md @@ -54,7 +54,7 @@ pub struct TargetSubDAG<'a> { `root` is the actual `Rc` from the workload. -`consumer_count` records how many workload locations refer to that exact shared node. +`consumer_count` counts **structural references**, not runtime reads/executions: how many places in the workload's `Rc` DAG point at this exact `root` node. Example — two separate top-level queries, `sum by (service) (rate(m[5m]))` and `avg by (service) (rate(m[5m]))`, share an identical `rate(m[5m])` subtree; after CSE (`share_common_subtrees`) collapses that into one `Rc`, both queries' trees point at the same node, so its `TargetSubDAG.consumer_count` is `2` — regardless of how many times either query actually executes. Use: From 8ef85870f59b45b4072571a896b51e033dcd0c04 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 23 Aug 2026 13:55:36 -0600 Subject: [PATCH 10/49] refactor(cost_model): encode CSE cost hooks as Cost, not bare f64 Per milind's review comment on the dev guide's cse_recompute_cost entry: "Might be worth encoding the cost as a struct, instead of f64. Will help with extensibility later on." New Cost(pub f64) newtype -- PartialOrd/Add/Mul/Display -- gives a future richer cost type (e.g. separate CPU/memory/network fields) a place to grow into without a second signature-breaking change across every hook. Still a single unitless scalar today; behavior is unchanged. Updated: - CostModel::cse_recompute_cost / cse_shared_maintenance_cost -> Cost - default_cse_recompute_cost / default_cse_shared_maintenance_cost -> Cost - cse_share_decision's default body: consumer_count multiplication now goes through Cost's Mul instead of a raw `as f64` cast - ForceSketchKind's forwarding impl (replacement.rs) - every test construction/comparison site in cost_model.rs - the dev guide's CostModel hook signatures (both the glossary and the per-hook reference section) cargo build/test/fmt/clippy --workspace all pass clean. --- crates/asap-aware-mapping/src/cost_model.rs | 60 +++++++++++++++---- crates/asap-aware-mapping/src/replacement.rs | 6 +- .../ASAP-aware-mapping-developer-guide.md | 12 ++-- 3 files changed, 58 insertions(+), 20 deletions(-) diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index 89cddb98..10cd268b 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -73,6 +73,42 @@ pub struct CseCandidate<'a> { pub consumer_count: usize, } +/// A cost estimate produced by a [`CostModel`] hook. A newtype around `f64` +/// rather than a bare `f64` return type, so a future cost dimension (e.g. +/// separate CPU/memory/network estimates, once a deployment actually needs +/// to compare along more than one axis) can be added as a field here +/// without changing every hook's signature a second time. Today it's still +/// a single unitless scalar — the same magnitude convention +/// [`default_cse_recompute_cost`]/[`default_cse_shared_maintenance_cost`] +/// already used as bare `f64`s, just wrapped. +#[derive(Debug, Clone, Copy, PartialEq, PartialOrd)] +pub struct Cost(pub f64); + +impl Cost { + /// The cost of an operation that costs nothing at all. + pub const ZERO: Cost = Cost(0.0); +} + +impl std::fmt::Display for Cost { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl std::ops::Add for Cost { + type Output = Cost; + fn add(self, rhs: Cost) -> Cost { + Cost(self.0 + rhs.0) + } +} + +impl std::ops::Mul for Cost { + type Output = Cost; + fn mul(self, rhs: usize) -> Cost { + Cost(self.0 * rhs as f64) + } +} + /// The decision [`CostModel::cse_share_decision`] returns for one /// [`CseCandidate`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -100,8 +136,8 @@ pub enum ShareDecision { /// leaf costs little to recompute, a deep multi-join subtree costs a lot. /// A deployment with real per-row/per-update cost knowledge should /// override [`CostModel::cse_recompute_cost`] instead of relying on this. -pub fn default_cse_recompute_cost(subtree: &QueryExpr) -> f64 { - asap_types::pre_asap::cse::dag_node_count(subtree) as f64 +pub fn default_cse_recompute_cost(subtree: &QueryExpr) -> Cost { + Cost(asap_types::pre_asap::cse::dag_node_count(subtree) as f64) } /// Default [`CostModel::cse_shared_maintenance_cost`]: a small @@ -116,7 +152,7 @@ pub fn default_cse_recompute_cost(subtree: &QueryExpr) -> f64 { /// deployment with real memory/update-cost numbers should override /// [`CostModel::cse_shared_maintenance_cost`] instead of relying on this /// table. -pub fn default_cse_shared_maintenance_cost(family: &SummaryFamilyType) -> f64 { +pub fn default_cse_shared_maintenance_cost(family: &SummaryFamilyType) -> Cost { const UNIT: f64 = 1.0; let weight = match family { SummaryFamilyType::Plain(_) => 1.0, @@ -126,7 +162,7 @@ pub fn default_cse_shared_maintenance_cost(family: &SummaryFamilyType) -> f64 { SummaryFamilyType::Wavelet(..) => 5.0, SummaryFamilyType::StatModel(..) => 6.0, }; - weight * UNIT + Cost(weight * UNIT) } /// Ranks the candidate summary families for one [`AggIntent`], best choice @@ -212,7 +248,7 @@ pub trait CostModel { /// independently at a single use site. Default: /// [`default_cse_recompute_cost`] (a structural-size proxy). See /// `docs/design_docs/cse-cost-model-decision.md`. - fn cse_recompute_cost(&self, candidate: &CseCandidate) -> f64 { + fn cse_recompute_cost(&self, candidate: &CseCandidate) -> Cost { default_cse_recompute_cost(candidate.subtree) } @@ -224,7 +260,7 @@ pub trait CostModel { /// state (falls back to the cheapest, `Plain`, weight if none does — /// e.g. `bound_summary` is a passthrough `Logical` node with nothing /// summary-shaped to maintain). See `docs/design_docs/cse-cost-model-decision.md`. - fn cse_shared_maintenance_cost(&self, candidate: &CseCandidate) -> f64 { + fn cse_shared_maintenance_cost(&self, candidate: &CseCandidate) -> Cost { let family = candidate .bound_summary .schema @@ -252,7 +288,7 @@ pub trait CostModel { /// (keeping this comparison), or override this method directly for a /// wholly different policy. fn cse_share_decision(&self, candidate: &CseCandidate) -> ShareDecision { - let recompute_total = self.cse_recompute_cost(candidate) * candidate.consumer_count as f64; + let recompute_total = self.cse_recompute_cost(candidate) * candidate.consumer_count; let shared = self.cse_shared_maintenance_cost(candidate); if shared <= recompute_total { ShareDecision::Share @@ -425,7 +461,7 @@ mod tests { cols: vec![0], child: std::rc::Rc::new(leaf.clone()), }; - assert!(default_cse_recompute_cost(&leaf) > 0.0); + assert!(default_cse_recompute_cost(&leaf) > Cost::ZERO); assert!(default_cse_recompute_cost(&nested) > default_cse_recompute_cost(&leaf)); } @@ -460,12 +496,12 @@ mod tests { }; assert_eq!( default_cse_recompute_cost(&no_sharing), - 3.0, + Cost(3.0), "no sharing: Join + 2 independent Scans = 3 unique nodes" ); assert_eq!( default_cse_recompute_cost(&with_sharing), - 2.0, + Cost(2.0), "internal sharing: Join + 1 shared Scan (referenced twice) = \ 2 unique nodes, not 3 — a tree-shaped size measure would \ wrongly charge for the shared Scan twice" @@ -540,8 +576,8 @@ mod tests { ) -> Vec { candidates.to_vec() } - fn cse_recompute_cost(&self, _candidate: &CseCandidate) -> f64 { - 1e9 + fn cse_recompute_cost(&self, _candidate: &CseCandidate) -> Cost { + Cost(1e9) } } diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index 97cb8309..b6a93034 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -111,7 +111,7 @@ use asap_types::pre_asap::query_expr::QueryExpr; use crate::bind::implement_tree_with; use crate::boundary::{implementation_for_with, summary_candidates, Implementation}; -use crate::cost_model::{CostModel, CseCandidate, DefaultCostModel, ShareDecision}; +use crate::cost_model::{Cost, CostModel, CseCandidate, DefaultCostModel, ShareDecision}; /// A pre-ASAP sub-DAG a [`ReplacementStrategy`] knows how to replace. /// @@ -441,11 +441,11 @@ impl CostModel for ForceSketchKind<'_> { self.inner.readout_extension(ext_kind, payload, col) } - fn cse_recompute_cost(&self, candidate: &CseCandidate) -> f64 { + fn cse_recompute_cost(&self, candidate: &CseCandidate) -> Cost { self.inner.cse_recompute_cost(candidate) } - fn cse_shared_maintenance_cost(&self, candidate: &CseCandidate) -> f64 { + fn cse_shared_maintenance_cost(&self, candidate: &CseCandidate) -> Cost { self.inner.cse_shared_maintenance_cost(candidate) } diff --git a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md index 5a134e00..37c786bc 100644 --- a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md @@ -200,18 +200,20 @@ The two methods have intentionally different responsibilities. fn readout_extension(&self, ext_kind: &str, payload: &serde_json::Value, col: &ColumnRef) -> SketchQuery; ``` -- **`cse_recompute_cost`** — estimate the one-time cost of recomputing a CSE candidate's subtree independently at a single consumer. Default: `default_cse_recompute_cost`, a structural-size proxy. Returns a bare `f64` today (a unitless scalar compared directly against `cse_shared_maintenance_cost`'s own `f64`) — a richer cost type (e.g. a struct separating CPU/memory/network) would let deployments compare along more than one axis, but is a real signature change across every hook in this trait, not a doc fix; worth its own issue rather than deciding here. +- **`cse_recompute_cost`** — estimate the one-time cost of recomputing a CSE candidate's subtree independently at a single consumer. Default: `default_cse_recompute_cost`, a structural-size proxy. ```rust - fn cse_recompute_cost(&self, candidate: &CseCandidate) -> f64; + fn cse_recompute_cost(&self, candidate: &CseCandidate) -> Cost; ``` - **`cse_shared_maintenance_cost`** — estimate the cost of maintaining one shared summary continuously for the life of the workload. Default: `default_cse_shared_maintenance_cost`, a per-family weight table. ```rust - fn cse_shared_maintenance_cost(&self, candidate: &CseCandidate) -> f64; + fn cse_shared_maintenance_cost(&self, candidate: &CseCandidate) -> Cost; ``` + Both hooks return `Cost`, a newtype around `f64` rather than a bare `f64` — a deliberately minimal wrapper today (still one unitless scalar, `Cost(f64)`, comparable via `PartialOrd`/`Add`/`Mul`), but one that gives a future richer cost type (e.g. separate CPU/memory/network fields) a place to grow into without changing every hook's signature a second time. + - **`cse_share_decision`** — decide whether to share one summary across all of a CSE candidate's consumers, or recompute independently at each. Default: composes the two cost hooks above (share iff shared-maintenance cost ≤ total recompute cost across every consumer) — override the two cost hooks and keep this comparison unless you need a genuinely different policy, not a different cost input. ```rust @@ -788,7 +790,7 @@ Use to estimate the cost of computing a common subtree independently at each con fn cse_recompute_cost( &self, candidate: &CseCandidate, -) -> f64; +) -> Cost; ``` --- @@ -801,7 +803,7 @@ Use to estimate the cost of computing and maintaining a shared subtree. fn cse_shared_maintenance_cost( &self, candidate: &CseCandidate, -) -> f64; +) -> Cost; ``` --- From 66c772d6078060c7cecaa872e5ad654f7a0a47ec Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 23 Aug 2026 13:59:31 -0600 Subject: [PATCH 11/49] docs: fix asap_aware_mapping.md path refs after rebase onto main's docs reorg replacement.rs/lib.rs are new files this PR creates, so the rebase onto main (which already moved docs/asap_aware_mapping.md -> docs/design_docs/) had nothing on main's side to reconcile these against -- fix by hand. --- crates/asap-aware-mapping/src/lib.rs | 2 +- crates/asap-aware-mapping/src/replacement.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index b1a3fa2e..1fc9ea5e 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -33,7 +33,7 @@ //! Three real occupants and one stub: //! //! - [`replacement`] — the `TargetSubDAG`/`ReplacementSubDAG`/ -//! `ReplacementStrategy` vocabulary `docs/asap_aware_mapping.md` stubs out +//! `ReplacementStrategy` vocabulary `docs/design_docs/asap_aware_mapping.md` stubs out //! under "Key concepts (not yet implemented)", implemented for real (issue //! #251, part of #33): a small extension-point trait reporting *every* //! semantically valid replacement for a target sub-DAG, not just the one diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index b6a93034..e2ef5de4 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -1,5 +1,5 @@ //! `TargetSubDAG` / `ReplacementSubDAG` / `ReplacementStrategy` — the -//! candidate-replacement vocabulary `docs/asap_aware_mapping.md` stubs out +//! candidate-replacement vocabulary `docs/design_docs/asap_aware_mapping.md` stubs out //! under "Key concepts (not yet implemented)", implemented for real (issue //! #251, part of #33). //! @@ -12,7 +12,7 @@ //! *binding* a query (something has to actually run), but it means the //! alternatives a cost model didn't pick are thrown away the moment they're //! computed. A search/optimization engine (Cascades/Volcano-style, tracked -//! separately — see `docs/asap_aware_mapping.md`'s "Pseudocode for +//! separately — see `docs/design_docs/asap_aware_mapping.md`'s "Pseudocode for //! Replacement Plan Searching") needs the *opposite* shape: every //! semantically valid alternative for a sub-DAG, so a later cost-based search //! can explore and compare them instead of being stuck with whatever @@ -79,7 +79,7 @@ //! //! ## Non-goals (tracked separately, not attempted here) //! -//! - **No search/selection logic.** `docs/asap_aware_mapping.md`'s +//! - **No search/selection logic.** `docs/design_docs/asap_aware_mapping.md`'s //! replacement-plan-searching pseudocode — trying every `ReplacementStrategy` //! against every candidate plan, deduplicating, iterating to a fixpoint, //! then ranking by a `CostModel` — is a Cascades/Volcano-style search From 8fe5710ac76f5a3254487913f9549eb3f88c39aa Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 23 Aug 2026 14:21:50 -0600 Subject: [PATCH 12/49] refactor(asap-aware-mapping): rename boundary.rs -> implementation.rs Per milind's review comment on the dev guide ("boundary is a weird name, should rename (after this PR)") and zzylol's follow-up decision to do it now: this module's dominant exported type/functions (Implementation, implementation_for, implementation_for_with, Matcher::is_satisfied_by( required: &Implementation, ...)) already center on "Implementation", not "boundary" -- lib.rs's own "Terminology" section documents that this crate deliberately settled on "implementation" (Cascades/Volcano's logical -> physical "implementation rule") as its vocabulary for this exact decision (6fb0113), everywhere except the module filename itself. This finishes that rename and matches this crate's own file-naming convention (file named after its primary type: cost_model.rs <-> CostModel, replacement.rs <-> ReplacementStrategy). - crates/asap-aware-mapping/src/boundary.rs -> implementation.rs, module doc rewritten to lead with `Implementation` rather than the "boundary" framing. - lib.rs: mod/pub use updated; the Terminology section's own table (which argues for "implementation" as the deliberate term) now consistently names this module `implementation` throughout. - bind.rs, cost_model.rs, replacement.rs: import paths and every doc-link reference updated. Prose uses of "boundary" as the accuracy-decision *concept* (not the module) are left alone -- e.g. "the sketch-vs-exact boundary decision" is still accurate English, independent of what the file implementing it is called. - Cross-crate stale references fixed for consistency: crates/types' cse.rs, query_time/mod.rs, and query_time/error_estimation.rs (including three test names literally encoding "matches boundary.rs's behavior" -- renamed to "implementation_rs" to match). - docs/developer_docs/ASAP-aware-mapping-developer-guide.md: every `boundary`/`boundary.rs` reference updated to `implementation`/ `implementation.rs`. cargo build/test/fmt/clippy --workspace all pass clean. cargo doc's pre-existing intra-doc-link warning count is unchanged (47 before and after) -- this rename doesn't introduce any new broken links, just renames already-broken ones consistently. --- crates/asap-aware-mapping/src/bind.rs | 23 ++++---- crates/asap-aware-mapping/src/cost_model.rs | 36 ++++++------ .../src/{boundary.rs => implementation.rs} | 17 +++--- crates/asap-aware-mapping/src/lib.rs | 42 +++++++------- crates/asap-aware-mapping/src/replacement.rs | 56 +++++++++---------- .../integration-tests/tests/l4_binding_sql.rs | 5 +- .../post_asap/query_time/error_estimation.rs | 32 +++++------ crates/types/src/post_asap/query_time/mod.rs | 2 +- crates/types/src/pre_asap/cse.rs | 2 +- .../ASAP-aware-mapping-developer-guide.md | 32 +++++------ 10 files changed, 127 insertions(+), 120 deletions(-) rename crates/asap-aware-mapping/src/{boundary.rs => implementation.rs} (98%) diff --git a/crates/asap-aware-mapping/src/bind.rs b/crates/asap-aware-mapping/src/bind.rs index 60a2508c..ef728f9c 100644 --- a/crates/asap-aware-mapping/src/bind.rs +++ b/crates/asap-aware-mapping/src/bind.rs @@ -2,7 +2,7 @@ //! //! Walks a canonical pre-ASAP [`QueryExpr`] and emits the summary-bound //! post-ASAP IR ([`SummaryExpr`] / [`SummaryNode`] in `asap-sketch`). Per -//! node, the [`boundary`](crate::boundary) decision picks the realization: +//! node, the [`implementation`](crate::implementation) decision picks the realization: //! //! - **Summary family** (sketch / sample / wavelet / statistical model) — //! the `Aggregate` becomes a [`SummaryExpr::SummaryAgg`] carrying the @@ -44,8 +44,8 @@ use asap_types::pre_asap::query_expr::{QueryExpr, QueryExprError, Reduction}; use asap_types::pre_asap::schema::Schema; use thiserror::Error; -use crate::boundary::{implementation_for_with, Implementation}; use crate::cost_model::{CostModel, CseCandidate, DefaultCostModel, ShareDecision}; +use crate::implementation::{implementation_for_with, Implementation}; /// Errors from the pre-ASAP → post-ASAP binding pass. #[derive(Debug, Error)] @@ -109,7 +109,7 @@ pub fn implement_tree_with( /// needs to recognize when it already bound the exact `Rc` a later root /// hands back, and is deliberately *not* a general "does an available /// `Implementation` satisfy this one" lookup — that subsumption question is -/// `asap_aware_mapping::boundary::Matcher`'s documented, deliberately-unfilled +/// `asap_aware_mapping::implementation::Matcher`'s documented, deliberately-unfilled /// job, not this one's. /// /// A first pass over `roots` counts each distinct `Rc` pointer's @@ -321,7 +321,9 @@ fn readout(intent: &AggIntent, col: &ColumnRef, cost_model: &dyn CostModel) -> S cost_model.readout_extension(ext_kind, payload, col) } other => { - unreachable!("no summary realization for {other:?} (boundary::implementation_for)") + unreachable!( + "no summary realization for {other:?} (implementation::implementation_for)" + ) } } } @@ -329,7 +331,7 @@ fn readout(intent: &AggIntent, col: &ColumnRef, cost_model: &dyn CostModel) -> S /// Wrap an unrewritten pre-ASAP subtree, lifting its schema with every column /// `SummaryFamilyType::Plain`. Public so a deployment can force a node it /// knows `implement_tree_in_with` would otherwise actively (mis)bind — -/// e.g. an intent this crate's `boundary::implementation_for` maps to an +/// e.g. an intent this crate's `implementation::implementation_for` maps to an /// accumulator kind the deployment's runtime doesn't actually implement — /// through the same fallback this crate's own dispatch uses, without /// duplicating the schema-lift logic. @@ -534,9 +536,9 @@ mod tests { &self, ext_kind: &str, _payload: &serde_json::Value, - ) -> crate::boundary::Implementation { + ) -> crate::implementation::Implementation { if ext_kind == "frequency" { - crate::boundary::Implementation::Sketch { + crate::implementation::Implementation::Sketch { kind: SketchKind::CountSketch, params: SketchParams::CountSketch { width: 256, @@ -544,7 +546,7 @@ mod tests { }, } } else { - crate::boundary::Implementation::PassThrough + crate::implementation::Implementation::PassThrough } } @@ -716,8 +718,9 @@ mod tests { #[test] fn nested_aggregates_bind_per_node() { - // quantile(0.9, sum by (job) (m)) — the boundary fires per node over - // the nested tree: KLL over an exact Sum accumulator. + // quantile(0.9, sum by (job) (m)) — the implementation decision + // fires per node over the nested tree: KLL over an exact Sum + // accumulator. let inner = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); let outer = agg(vec![], default_quantile(0.9), inner); let root = implement_tree(&outer).unwrap(); diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index 10cd268b..c644c44e 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -6,10 +6,10 @@ //! needs knowledge this crate doesn't have and shouldn't acquire: the crate //! doc's layering invariant is that `asap-plan` depends only on [`asap_ir`], //! never on a runtime or a deployment model. What it *can* own is the -//! interface every deployment's cost model plugs into, so [`boundary`]'s +//! interface every deployment's cost model plugs into, so [`implementation`]'s //! summary selection has exactly one extension point instead of forcing //! each downstream (ASAPCollector + ASAPQuery-backend, ASAPFusion, …) to -//! fork [`boundary::implementation_for`]. +//! fork [`implementation::implementation_for`]. //! //! This trait is scoped to the approximate-**sketch** family specifically //! ([`CostModel::rank_candidates`]/[`size_params`](CostModel::size_params) @@ -26,7 +26,7 @@ //! than overloading these ones across incompatible `Kind`/`Params` types. //! //! Every entry point that doesn't take an explicit `&dyn CostModel` -//! ([`implementation_for`](crate::boundary::implementation_for), +//! ([`implementation_for`](crate::implementation::implementation_for), //! [`implement_tree`](crate::bind::implement_tree)) runs against //! [`DefaultCostModel`], so a deployment that never plugs in its own cost //! model keeps today's static-preference-order behavior exactly, byte for @@ -51,7 +51,7 @@ use asap_types::pre_asap::agg_intent::AggIntent; use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::query_expr::QueryExpr; -use crate::boundary::Implementation; +use crate::implementation::Implementation; /// A CSE-detected, legality-gated shared subtree with two or more consumers /// — the unit [`CostModel::cse_share_decision`] decides over. Built by @@ -168,22 +168,22 @@ pub fn default_cse_shared_maintenance_cost(family: &SummaryFamilyType) -> Cost { /// Ranks the candidate summary families for one [`AggIntent`], best choice /// first. /// -/// [`boundary::summary_candidates`] returns every family that *can* answer an +/// [`implementation::summary_candidates`] returns every family that *can* answer an /// intent, in an arbitrary static preference order (issue #98's "one home" /// for the candidate set). A `CostModel` re-orders that list under real, /// deployment-specific cost knowledge this crate has no way to know about — -/// [`boundary::implementation_for_with`] implements whichever candidate ends +/// [`implementation::implementation_for_with`] implements whichever candidate ends /// up first after ranking. pub trait CostModel { /// Rank `candidates` (as returned by - /// [`summary_candidates`](crate::boundary::summary_candidates)) for + /// [`summary_candidates`](crate::implementation::summary_candidates)) for /// `intent`, best choice first. /// /// Implementations MAY reorder freely and MAY drop entries that aren't /// available in their deployment, but MUST NOT invent a candidate that /// wasn't in the input — an unknown [`SketchKind`] has no /// [`SketchParams`](asap_types::post_asap::SketchParams) sizing logic in - /// [`boundary::implementation_for_with`] and binding it will panic. + /// [`implementation::implementation_for_with`] and binding it will panic. /// Returning an empty `Vec` means "no candidate is acceptable"; /// `implementation_for_with` treats that the same as `candidates` /// having been empty to begin with. @@ -196,9 +196,9 @@ pub trait CostModel { /// Splitting sizing out from candidate selection lets a deployment own /// its own parameter-sizing math (e.g. an empirically-tuned table, or /// discrete rungs required by a downstream catalog) without forking - /// [`boundary::implementation_for_with`] — the same "one extension + /// [`implementation::implementation_for_with`] — the same "one extension /// point" rationale as `rank_candidates`, one level deeper. Default: - /// [`boundary::default_size_params`], `asap-plan`'s built-in formulas + /// [`implementation::default_size_params`], `asap-plan`'s built-in formulas /// (unchanged) — a deployment that only needs to reorder candidates, /// not resize them, can leave this method unimplemented. fn size_params( @@ -208,12 +208,12 @@ pub trait CostModel { eps: f64, delta: f64, ) -> SketchParams { - crate::boundary::default_size_params(kind, intent, eps, delta) + crate::implementation::default_size_params(kind, intent, eps, delta) } /// Realize an `AggIntent::Extension { ext_kind, payload }` — a /// deployment-specific intent shape core has no realization opinion - /// for (issue #131). `boundary::implementation_for_with` consults this + /// for (issue #131). `implementation::implementation_for_with` consults this /// for every `Extension` node instead of hardcoding `PassThrough` /// (issue #150). Default: `PassThrough` — preserves today's behavior /// for every deployment that doesn't override this, exactly like @@ -299,9 +299,9 @@ pub trait CostModel { } /// The default cost model: preserves [`summary_candidates`]'s built-in static -/// order and [`boundary::default_size_params`]'s built-in sizing unchanged. +/// order and [`implementation::default_size_params`]'s built-in sizing unchanged. /// -/// [`summary_candidates`]: crate::boundary::summary_candidates +/// [`summary_candidates`]: crate::implementation::summary_candidates pub struct DefaultCostModel; impl CostModel for DefaultCostModel { @@ -313,7 +313,7 @@ impl CostModel for DefaultCostModel { #[cfg(test)] mod tests { use super::*; - use crate::boundary::summary_candidates; + use crate::implementation::summary_candidates; use asap_types::pre_asap::agg_intent::default_cardinality; #[test] @@ -356,7 +356,7 @@ mod tests { let intent = default_cardinality(); assert_eq!( AlwaysPreferLast.size_params(SketchKind::Hll, &intent, 0.01, 0.01), - crate::boundary::default_size_params(SketchKind::Hll, &intent, 0.01, 0.01), + crate::implementation::default_size_params(SketchKind::Hll, &intent, 0.01, 0.01), ); } @@ -387,7 +387,7 @@ mod tests { let k = if eps >= 0.01 { 200 } else { 2048 }; SketchParams::Kll { k } } - other => crate::boundary::default_size_params(other, intent, eps, delta), + other => crate::implementation::default_size_params(other, intent, eps, delta), } } } @@ -404,7 +404,7 @@ mod tests { // Untouched kinds still fall through to the default formula. assert_eq!( DiscreteKllRungs.size_params(SketchKind::Hll, &intent, 0.01, 0.01), - crate::boundary::default_size_params(SketchKind::Hll, &intent, 0.01, 0.01), + crate::implementation::default_size_params(SketchKind::Hll, &intent, 0.01, 0.01), ); } diff --git a/crates/asap-aware-mapping/src/boundary.rs b/crates/asap-aware-mapping/src/implementation.rs similarity index 98% rename from crates/asap-aware-mapping/src/boundary.rs rename to crates/asap-aware-mapping/src/implementation.rs index ff3f2b39..5155c59c 100644 --- a/crates/asap-aware-mapping/src/boundary.rs +++ b/crates/asap-aware-mapping/src/implementation.rs @@ -1,12 +1,13 @@ -//! Sketch-vs-exact boundary — the per-intent accuracy decision (issue #98). +//! [`Implementation`] — how one [`AggIntent`] gets realized at post-ASAP +//! binding time (issue #98): by an approximate summary (sketch, sample, +//! wavelet, statistical model, …), by an exact mergeable accumulator, or by +//! an ordinary exact operator (pass-through). This is a post-ASAP concern — +//! the pre-ASAP IR carries only the intent + accuracy target, never the +//! realization — and it's a per-node decision, made once per `AggIntent`, +//! not a plan-wide one. //! -//! The per-node choice of how an [`AggIntent`] is *realised*: by an -//! approximate summary (sketch, sample, wavelet, statistical model, …), by -//! an exact mergeable accumulator, or by an ordinary exact operator -//! (pass-through). This is a post-ASAP concern: the pre-ASAP IR carries only -//! the intent + accuracy target, never the realization. -//! -//! The decision consumes three inputs: +//! [`implementation_for`]/[`implementation_for_with`] are where that +//! decision actually gets made. Three inputs feed it: //! //! - the [`AccuracyTarget`] threaded onto the approximate-capable intents //! (`Quantile` / `Cardinality` / `Count` / `TopK`) — `Exact` forbids a diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 1fc9ea5e..898df458 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -37,7 +37,7 @@ //! under "Key concepts (not yet implemented)", implemented for real (issue //! #251, part of #33): a small extension-point trait reporting *every* //! semantically valid replacement for a target sub-DAG, not just the one -//! [`boundary::implementation_for_with`]/[`bind::implement_tree_with`] +//! [`implementation::implementation_for_with`]/[`bind::implement_tree_with`] //! commit to. Two real strategies wrap those exact decision points instead //! of re-deciding anything: //! [`replacement::SketchFamilyStrategy`] (every candidate summary family @@ -46,21 +46,22 @@ //! CSE-detected shared subtree). No search/ranking logic lives here — see //! that module's docs for what's deliberately left to a future //! Cascades/Volcano-style search engine. -//! - [`boundary`] — the per-intent sketch-vs-exact (accuracy) decision: +//! - [`implementation`] — the per-intent sketch-vs-exact (accuracy) decision: //! `AggIntent → Implementation` (a summary family's own `(Kind, Params)`, //! or an exact accumulator) sized to the `AccuracyTarget` (issue #98). -//! [`boundary::implementation_for`] is the per-node decision; +//! [`implementation::implementation_for`] is the per-node decision; //! [`bind::implement_tree`] drives it over a whole tree, and //! [`bind::implement_workload`] drives it over a whole workload's roots — //! memoized on `Rc` identity so two roots that //! `asap_types::pre_asap::cse::share_common_subtrees` already collapsed //! onto one shared subtree bind to one shared `SummaryNode` too (issue -//! #212, #222, #223) — see the terminology section below for why these are -//! named around "implementation" rather than "bind". +//! #212, #222, #223) — see the terminology section below for why this +//! module (and `implement_tree`) are named around "implementation" rather +//! than "bind". //! - [`cost_model`] — the [`CostModel`](cost_model::CostModel) trait every //! deployment's cost-based sketch selection plugs into (issues #6, #33). //! `asap-plan` itself only ships [`DefaultCostModel`](cost_model::DefaultCostModel), -//! which preserves [`boundary`]'s built-in static preference order. +//! which preserves [`implementation`]'s built-in static preference order. //! //! ## Terminology — "bind" already means three different things nearby; //! this crate's own logical→physical step is named "implementation" instead @@ -68,20 +69,21 @@ //! `asap-plan` and its downstream consumers (e.g. `ASAPQuery-backend`'s //! `control_plane`) independently reused the word "bind" for three //! *different*, layer-specific meanings — none of which is what this -//! crate's [`boundary`]/[`bind`] modules do. To avoid becoming a fourth, -//! colliding sense of the same word, this crate names its own logical -//! intent → physical realization step after the term the query-optimization -//! literature already uses for exactly that step: **implementation** -//! (Cascades/Volcano's "implementation rule", logical → physical, as -//! distinct from a *transformation rule*, logical → logical — see Graefe, -//! *The Cascades Framework for Query Optimization*): +//! crate's [`implementation`]/[`bind`] modules do. To avoid becoming a +//! fourth, colliding sense of the same word, this crate names its own +//! logical intent → physical realization step after the term the +//! query-optimization literature already uses for exactly that step: +//! **implementation** (Cascades/Volcano's "implementation rule", logical → +//! physical, as distinct from a *transformation rule*, logical → logical — +//! see Graefe, *The Cascades Framework for Query Optimization*) — and this +//! module is named after that same term for the same reason. //! //! | Term | Stage | Meaning | Lives in | //! |---|---|---|---| //! | **Parse** | parse | text (PromQL/SQL) → AST | `asap-frontend-promql` / `asap-frontend-sql` | //! | **Bind #1** | name resolution | `ColumnRef` (a name) → `ColumnId` (a concrete schema column) — the classic RDBMS "Parse → **Bind** → Optimize" pipeline sense (e.g. SQL Server's query-processor terminology) | [`asap_types::pre_asap::binder::Binder`](https://docs.rs/asap-types) | -//! | **Implementation** — [`boundary::implementation_for`] | pre-ASAP → post-ASAP, *one node* | choosing a concrete physical realization (a sketch family, an exact accumulator, or pass-through) for one [`AggIntent`](asap_types::pre_asap::agg_intent::AggIntent) | [`boundary`] | -//! | **`implement_tree`** — [`bind::implement_tree`] | pre-ASAP → post-ASAP, *whole tree* | walk a whole `QueryExpr` tree, calling [`boundary::implementation_for`] per node, and emit the complete post-ASAP [`SummaryExpr`](asap_types::post_asap::SummaryExpr)/`SummaryNode` DAG — named after "implementation" too rather than reusing "bind" a second time | [`bind`] | +//! | **Implementation** — [`implementation::implementation_for`] | pre-ASAP → post-ASAP, *one node* | choosing a concrete physical realization (a sketch family, an exact accumulator, or pass-through) for one [`AggIntent`](asap_types::pre_asap::agg_intent::AggIntent) | [`implementation`] | +//! | **`implement_tree`** — [`bind::implement_tree`] | pre-ASAP → post-ASAP, *whole tree* | walk a whole `QueryExpr` tree, calling [`implementation::implementation_for`] per node, and emit the complete post-ASAP [`SummaryExpr`](asap_types::post_asap::SummaryExpr)/`SummaryNode` DAG — named after "implementation" too rather than reusing "bind" a second time | [`bind`] | //! | **Bind #2** (downstream, not in this crate) | post-ASAP → deployment placement | a *deployment's* own physical binder, additionally deciding **placement** (edge vs. backend, wire format, …) — a genuinely different, deployment-specific decision this crate doesn't model at all | e.g. `control_plane::sketch_algebra::rules::bind_*` (as of this writing; expected to fold into that deployment's cost-model layer rather than stay a separate "bind" concept) | //! //! A related question (tracked alongside issues #6/#33): whether this @@ -89,8 +91,8 @@ //! *available* `Implementation` satisfy a *required* one" — the way a //! database's materialized-view matching / "answering queries using //! views" layer does. It owns the *question*, not an *answer*: -//! [`boundary::Matcher`] is a trait with no default implementation and no -//! shipped instance, the same shape as [`cost_model::CostModel`] and for +//! [`implementation::Matcher`] is a trait with no default implementation and +//! no shipped instance, the same shape as [`cost_model::CostModel`] and for //! the same reason — which `Implementation`s are actually *available* //! anywhere is entirely a downstream deployment's concern (an inventory //! this crate has no way to see), and even the pure sketch-algebra @@ -102,18 +104,18 @@ //! reference downstream implementation. pub mod bind; -pub mod boundary; pub mod cost_model; +pub mod implementation; pub mod replacement; pub use bind::{ implement_tree, implement_tree_with, implement_workload, implement_workload_with, ImplementError, }; -pub use boundary::{ +pub use cost_model::{CostModel, DefaultCostModel}; +pub use implementation::{ implementation_for, implementation_for_with, summary_candidates, Implementation, Matcher, }; -pub use cost_model::{CostModel, DefaultCostModel}; pub use replacement::{ Replacement, ReplacementStrategy, ReplacementSubDAG, SharedSubtreeStrategy, SketchFamilyStrategy, TargetSubDAG, diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index e2ef5de4..af4a0213 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -3,9 +3,9 @@ //! under "Key concepts (not yet implemented)", implemented for real (issue //! #251, part of #33). //! -//! ## Why this exists alongside [`boundary`] and [`bind`], not instead of them +//! ## Why this exists alongside [`implementation`] and [`bind`], not instead of them //! -//! [`boundary::implementation_for_with`] and +//! [`implementation::implementation_for_with`] and //! [`bind::implement_tree_with`] each commit to exactly **one** answer per //! decision point — a single [`Implementation`], a single bound //! [`SummaryNode`] — ranked via a [`CostModel`]. That is exactly right for @@ -16,7 +16,7 @@ //! Replacement Plan Searching") needs the *opposite* shape: every //! semantically valid alternative for a sub-DAG, so a later cost-based search //! can explore and compare them instead of being stuck with whatever -//! [`boundary`]/[`bind`] already locked in. +//! [`implementation`]/[`bind`] already locked in. //! //! This module is that alternative shape, built by **wrapping** the existing //! decision points rather than re-deciding anything: @@ -34,7 +34,7 @@ //! target but semantically equivalent) — see [`Replacement`] — plus a //! human-readable `rationale`. //! - [`ReplacementStrategy`] — `matches` + `replacements`, the same -//! extension-point shape [`CostModel`] and [`Matcher`](crate::boundary::Matcher) +//! extension-point shape [`CostModel`] and [`Matcher`](crate::implementation::Matcher) //! already use in this crate (and the same shape issue #33's //! applicability-rule framework, PR #247, uses for its own //! `ApplicabilityRule`): a new replacement source is a new @@ -49,16 +49,16 @@ //! Both wrap an existing, already-correct decision procedure into the //! exhaustive-candidate shape — neither re-derives anything: //! -//! - [`SketchFamilyStrategy`] wraps [`boundary::implementation_for_with`] / -//! [`boundary::summary_candidates`]'s exhaustive match over the +//! - [`SketchFamilyStrategy`] wraps [`implementation::implementation_for_with`] / +//! [`implementation::summary_candidates`]'s exhaustive match over the //! [`AggIntent`] vocabulary. For the same bindable-`Aggregate` shape //! [`bind::implement_tree_with`] itself requires (single intent, no -//! `HAVING`), it returns every candidate [`boundary::implementation_for_with`] +//! `HAVING`), it returns every candidate [`implementation::implementation_for_with`] //! could have committed to instead of the one it did: every `SketchKind` -//! [`boundary::summary_candidates`] lists for the intent when the boundary +//! [`implementation::summary_candidates`] lists for the intent when the boundary //! decision is approximate, or the single exact-accumulator / //! pass-through outcome when that's the *only* realization -//! [`boundary::implementation_for_with`]'s dispatch produces for this +//! [`implementation::implementation_for_with`]'s dispatch produces for this //! intent (there is nothing else to enumerate in that case). //! - [`SharedSubtreeStrategy`] wraps //! `asap_types::pre_asap::cse::share_common_subtrees`'s sharing decision. @@ -85,7 +85,7 @@ //! then ranking by a `CostModel` — is a Cascades/Volcano-style search //! engine, tracked as a separate follow-up. This module only needs //! `replacements()` to be exhaustive and correct for one `TargetSubDAG` at -//! a time, mirroring [`boundary::implementation_for_with`]'s own +//! a time, mirroring [`implementation::implementation_for_with`]'s own //! "exhaustive match, no silent fallthrough" discipline. //! - **No workload-wide `TargetSubDAG` discovery pass.** Finding every //! candidate node in a whole workload (walking every root, deduplicating by @@ -94,10 +94,10 @@ //! do — reusable, but wiring it up to feed this module's strategies //! automatically is part of the same future search engine, not this issue. //! This module's own tests build `TargetSubDAG`s directly, the same -//! hand-rolled-fixture style `bind.rs`/`boundary.rs`/`cost_model.rs`'s own +//! hand-rolled-fixture style `bind.rs`/`implementation.rs`/`cost_model.rs`'s own //! tests already use. -//! - **[`boundary`]/[`bind`]'s existing single-pick public behavior is -//! unchanged.** Nothing here modifies [`boundary::implementation_for`], +//! - **[`implementation`]/[`bind`]'s existing single-pick public behavior is +//! unchanged.** Nothing here modifies [`implementation::implementation_for`], //! [`bind::implement_tree_with`], or how either is called; this module adds //! the exhaustive-candidate view alongside them, it does not replace or //! rewire either yet (a later issue's job). @@ -110,8 +110,8 @@ use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::query_expr::QueryExpr; use crate::bind::implement_tree_with; -use crate::boundary::{implementation_for_with, summary_candidates, Implementation}; use crate::cost_model::{Cost, CostModel, CseCandidate, DefaultCostModel, ShareDecision}; +use crate::implementation::{implementation_for_with, summary_candidates, Implementation}; /// A pre-ASAP sub-DAG a [`ReplacementStrategy`] knows how to replace. /// @@ -161,7 +161,7 @@ impl<'a> TargetSubDAG<'a> { /// What a [`ReplacementSubDAG`] actually substitutes a [`TargetSubDAG`] with. /// /// Generalizes [`bind::implement_tree_with`]'s and -/// [`boundary::implementation_for_with`]'s two possible *kinds* of answer — +/// [`implementation::implementation_for_with`]'s two possible *kinds* of answer — /// a post-ASAP binding decision, or a still-pre-ASAP structural alternative — /// from "the one they commit to" into "one candidate among several". #[derive(Debug, Clone)] @@ -192,7 +192,7 @@ pub struct ReplacementSubDAG { /// replacement (`replacements`)? /// /// The extension point this module exists for — the same shape -/// [`CostModel`] and [`crate::boundary::Matcher`] already use elsewhere in +/// [`CostModel`] and [`crate::implementation::Matcher`] already use elsewhere in /// this crate: a new replacement source is a new `impl ReplacementStrategy`, /// no restructuring of this trait or any existing strategy required. /// @@ -236,10 +236,10 @@ fn bindable_intent(node: &QueryExpr) -> Option<&AggIntent> { /// every caller (same pattern `applicability::SketchApplicabilityRule` uses). static DEFAULT_COST_MODEL: DefaultCostModel = DefaultCostModel; -/// Wraps [`boundary::implementation_for_with`]/[`boundary::summary_candidates`]'s +/// Wraps [`implementation::implementation_for_with`]/[`implementation::summary_candidates`]'s /// exhaustive match over the [`AggIntent`] vocabulary: for a bindable /// `Aggregate`, every candidate summary realization instead of just the one -/// [`boundary::implementation_for_with`] commits to. +/// [`implementation::implementation_for_with`] commits to. /// /// Ranks (only to *order the enumeration*, never to drop a candidate) via a /// [`CostModel`] — [`DefaultCostModel`] unless constructed with @@ -265,7 +265,7 @@ impl SketchFamilyStrategy<'static> { impl<'a> SketchFamilyStrategy<'a> { /// A strategy that ranks/binds via `cost_model` instead of the built-in /// static preference order — the same customization point - /// [`bind::implement_tree_with`] and [`boundary::implementation_for_with`] + /// [`bind::implement_tree_with`] and [`implementation::implementation_for_with`] /// already offer. pub fn new(cost_model: &'a dyn CostModel) -> Self { Self { cost_model } @@ -282,10 +282,10 @@ impl ReplacementStrategy for SketchFamilyStrategy<'_> { return Vec::new(); }; // Exhaustive over `Implementation`'s variants — the same "no silent - // fallthrough" discipline `boundary::implementation_for_with`'s own + // fallthrough" discipline `implementation::implementation_for_with`'s own // match uses. Only `Sketch` has more than one candidate to enumerate // (`summary_candidates`); every other variant is the *only* - // realization `boundary::implementation_for_with`'s dispatch + // realization `implementation::implementation_for_with`'s dispatch // produces for this intent, so there's nothing else to offer. match implementation_for_with(intent, self.cost_model) { Implementation::Sketch { .. } => summary_candidates(intent) @@ -299,7 +299,7 @@ impl ReplacementStrategy for SketchFamilyStrategy<'_> { self.cost_model, format!( "{} realizes as an exact {kind:?} accumulator — the only realization \ - boundary::implementation_for produces for this intent (no approximate \ + implementation::implementation_for produces for this intent (no approximate \ candidate applies)", describe_intent(intent) ), @@ -309,7 +309,7 @@ impl ReplacementStrategy for SketchFamilyStrategy<'_> { self.cost_model, format!( "{} has no summary realization and stays a logical pass-through — the \ - only realization boundary::implementation_for produces for this intent", + only realization implementation::implementation_for produces for this intent", describe_intent(intent) ), ), @@ -388,8 +388,8 @@ fn sketch_candidate( Some(ReplacementSubDAG { replacement: Replacement::Summary(node), rationale: format!( - "{} realizes as a {kind:?} sketch — one of boundary::summary_candidates' \ - alternatives for this intent (asap_aware_mapping::boundary::implementation_for)", + "{} realizes as a {kind:?} sketch — one of implementation::summary_candidates' \ + alternatives for this intent (asap_aware_mapping::implementation::implementation_for)", describe_intent(intent) ), }) @@ -457,7 +457,7 @@ impl CostModel for ForceSketchKind<'_> { /// A short human-readable label for an `AggIntent`, for /// [`ReplacementSubDAG::rationale`] text. Not exhaustive by design (unlike /// this crate's other `AggIntent` matches, e.g. -/// [`boundary::implementation_for_with`]'s) — this is prose for a +/// [`implementation::implementation_for_with`]'s) — this is prose for a /// rationale string, not a decision, so an unlisted variant just falls back /// to its `Debug` tag rather than forcing every future intent to be named /// here too (same rationale, and same shape, as @@ -610,7 +610,7 @@ mod tests { #[test] fn approximate_quantile_enumerates_every_summary_candidate() { - // Quantile's candidate list is [Kll, DDSketch] (boundary::summary_candidates) — + // Quantile's candidate list is [Kll, DDSketch] (implementation::summary_candidates) — // every entry must come back as its own bound SummaryNode candidate, // not just Kll (the CostModel-ranked head implementation_for_with commits to). let q = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); @@ -652,7 +652,7 @@ mod tests { assert_eq!( kinds, vec![SketchKind::Hll, SketchKind::Theta, SketchKind::Kmv], - "expected every boundary::summary_candidates entry for Cardinality" + "expected every implementation::summary_candidates entry for Cardinality" ); } diff --git a/crates/integration-tests/tests/l4_binding_sql.rs b/crates/integration-tests/tests/l4_binding_sql.rs index d6b3d077..2d32f3b5 100644 --- a/crates/integration-tests/tests/l4_binding_sql.rs +++ b/crates/integration-tests/tests/l4_binding_sql.rs @@ -201,8 +201,9 @@ async fn sql_quantile_binds_kll_sketch_over_named_column() { /// `SELECT COUNT(DISTINCT service) FROM metrics` at ε = 0.01 lowers to /// `AggIntent::Cardinality` (`sql_lowering.rs::count_distinct_is_cardinality`) /// — unlike `Quantile`/`Count`/`TopK`, its preferred candidate is HLL, not -/// KLL/CMS (`boundary::summary_candidates`), so this exercises a distinct -/// branch of the sketch-vs-exact boundary than the quantile test above. +/// KLL/CMS (`implementation::summary_candidates`), so this exercises a +/// distinct branch of the sketch-vs-exact decision than the quantile test +/// above. #[tokio::test] async fn sql_count_distinct_binds_hll_sketch_over_named_column() { let l3 = lower( diff --git a/crates/types/src/post_asap/query_time/error_estimation.rs b/crates/types/src/post_asap/query_time/error_estimation.rs index 28afae40..c343214a 100644 --- a/crates/types/src/post_asap/query_time/error_estimation.rs +++ b/crates/types/src/post_asap/query_time/error_estimation.rs @@ -8,7 +8,7 @@ //! The traditional CMS/Count-Sketch/CU-Sketch guarantee is an *a priori*, //! worst-case bound derived before any data is seen: e.g. classic CMS sizing //! (`w = ⌈e/ε⌉` counters/row, `r = ⌈ln(1/δ)⌉` rows — exactly -//! `crates/asap-aware-mapping/src/boundary.rs`'s `cms_width`/`cms_depth`) +//! `crates/asap-aware-mapping/src/implementation.rs`'s `cms_width`/`cms_depth`) //! guarantees `Pr[error > ε·|F|₁] < δ` obliviously to the real data //! distribution, by construction assuming the adversarial worst case (§3, //! §3.3 of the paper). @@ -47,7 +47,7 @@ //! vendored CMS/CountSketch/CU-Sketch runtime and no counter-array data //! structure anywhere** — confirmed by inspecting //! `crates/types/src/post_asap/sketch.rs` (`SketchKind`, `SketchParams`, -//! this module's neighbors) and `crates/asap-aware-mapping/src/boundary.rs` +//! this module's neighbors) and `crates/asap-aware-mapping/src/implementation.rs` //! (`default_size_params`, `cms_width`, `cms_depth`): both are purely //! planning-time sizing metadata. There is no `A[row][col]` counter matrix //! anywhere in the workspace for these functions to be handed at query @@ -65,7 +65,7 @@ //! Integration point (2) — tighter *plan-time* sizing under an expected-case //! (non-adversarial) assumption — is wired for real, since it touches code //! that already exists: see -//! `crates/asap-aware-mapping/src/boundary.rs::posterior_aware_size_params`. +//! `crates/asap-aware-mapping/src/implementation.rs::posterior_aware_size_params`. // ── Posterior (query-time) estimators ─────────────────────────────────────── @@ -192,10 +192,10 @@ pub fn count_sketch_posterior_error_bound(row: &[i64], rows: u32, delta: f64) -> /// `⌈x⌉` clamped to `[lo, hi]`; NaN / non-positive `x` saturate to `hi` (a /// degenerate accuracy target means "as accurate as this family goes"). -/// Byte-for-byte the same policy as `boundary.rs`'s private +/// Byte-for-byte the same policy as `implementation.rs`'s private /// `saturating_ceil` — duplicated here, not imported, for the same /// layering reason [`classic_cms_sizing`] itself is duplicated rather than -/// calling `boundary.rs` directly. +/// calling `implementation.rs` directly. fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { if !x.is_finite() || x <= 0.0 { return hi; @@ -205,19 +205,19 @@ fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { /// The classic CMS `(ε, δ)` sizing formula (§3.3, restated just before /// Eq. 6; identical — formula *and* clamping — to -/// `crates/asap-aware-mapping/src/boundary.rs`'s private +/// `crates/asap-aware-mapping/src/implementation.rs`'s private /// `cms_width`/`cms_depth`, reimplemented here — deliberately, not by /// accident — because `asap-types` sits below `asap-aware-mapping` in the /// workspace's dependency layering and cannot import from it): `w = ⌈e/ε⌉` /// counters/row, clamped to `[2, 2²⁶]`; `r = ⌈ln(1/δ)⌉` rows, clamped to /// `[1, 32]`. /// -/// Returns `(width, depth)`. Matching `boundary.rs`'s own clamp semantics +/// Returns `(width, depth)`. Matching `implementation.rs`'s own clamp semantics /// exactly (not just its in-range formula) matters here specifically: /// this function's whole purpose is giving comparison/test code (and, /// per issue #250, a future replan) the traditional bound to compare /// this module's posterior bound against — an un-clamped reimplementation -/// would silently diverge from `boundary.rs`'s real sizing outside a +/// would silently diverge from `implementation.rs`'s real sizing outside a /// narrow "nothing saturates" range of `(eps, delta)`, exactly the range /// most tests default to, making the divergence easy to miss. pub fn classic_cms_sizing(eps: f64, delta: f64) -> (u32, u32) { @@ -475,9 +475,9 @@ mod tests { // ── Traditional bound / classic sizing ────────────────────────────────── #[test] - fn classic_cms_sizing_matches_boundary_rs_formula() { + fn classic_cms_sizing_matches_implementation_rs_formula() { // Same worked example as - // `asap-aware-mapping::boundary::tests::epsilon_delta_sizes_cms_depth` + // `asap-aware-mapping::implementation::tests::epsilon_delta_sizes_cms_depth` // (eps=0.001, delta=0.001 ⇒ width=2719, depth=7), pinned here too so // the two independent (layering-forced) reimplementations can't // silently drift apart undetected. @@ -486,12 +486,12 @@ mod tests { } #[test] - fn classic_cms_sizing_degenerate_inputs_saturate_like_boundary_rs() { + fn classic_cms_sizing_degenerate_inputs_saturate_like_implementation_rs() { // Degenerate width/depth saturate to their hi clamp (2^26 / 32), - // matching `boundary.rs`'s `saturating_ceil` exactly — not `0` (see + // matching `implementation.rs`'s `saturating_ceil` exactly — not `0` (see // the correctness fix on `classic_cms_sizing`'s doc comment: an // earlier version returned `(0, 0)` here, silently diverging from - // `boundary.rs`'s real degenerate-input behavior). + // `implementation.rs`'s real degenerate-input behavior). assert_eq!(classic_cms_sizing(0.0, 0.01), (1 << 26, 5)); assert_eq!(classic_cms_sizing(0.01, 0.0), (272, 32)); assert_eq!(classic_cms_sizing(0.01, 1.0), (272, 32)); @@ -500,13 +500,13 @@ mod tests { /// Pinned extreme-range values — the clamp actually engaging, not just /// the in-range formula — computed by hand against the same - /// `[2, 2²⁶]` / `[1, 32]` bounds `boundary.rs`'s `cms_width`/`cms_depth` + /// `[2, 2²⁶]` / `[1, 32]` bounds `implementation.rs`'s `cms_width`/`cms_depth` /// use, so a future edit that reintroduces the un-clamped bug (an /// earlier version of this function silently diverged from - /// `boundary.rs` outside the narrow range most other tests exercise) + /// `implementation.rs` outside the narrow range most other tests exercise) /// gets caught here. #[test] - fn classic_cms_sizing_clamps_extreme_ranges_like_boundary_rs() { + fn classic_cms_sizing_clamps_extreme_ranges_like_implementation_rs() { // eps=1e-10: raw width e/eps ≈ 2.7e10, hi-clamped to 2^26. assert_eq!(classic_cms_sizing(1e-10, 0.01), (1 << 26, 5)); // eps=3.0: raw width e/3 < 1, lo-clamped to 2. diff --git a/crates/types/src/post_asap/query_time/mod.rs b/crates/types/src/post_asap/query_time/mod.rs index 1a6360d0..b9bd8f67 100644 --- a/crates/types/src/post_asap/query_time/mod.rs +++ b/crates/types/src/post_asap/query_time/mod.rs @@ -21,7 +21,7 @@ //! doesn't exist yet in this workspace (see [`error_estimation`]'s own //! docs for exactly what's blocked and why). //! -//! Contrast with `asap-aware-mapping::boundary::posterior_aware_size_params` +//! Contrast with `asap-aware-mapping::implementation::posterior_aware_size_params` //! (issue #239, PR #248): that function is real, wired *planning*-time //! code — it lives in the planning crate, not here, and does not call //! into this module. It borrows the same underlying intuition (a diff --git a/crates/types/src/pre_asap/cse.rs b/crates/types/src/pre_asap/cse.rs index f27cb6ee..dc139790 100644 --- a/crates/types/src/pre_asap/cse.rs +++ b/crates/types/src/pre_asap/cse.rs @@ -52,7 +52,7 @@ //! intentionally conservative: it only recognizes *exact* structural //! matches, not "a stricter-accuracy summary could also answer a looser //! request." That subsumption question already has a documented, -//! deliberately-unfilled home (`asap_aware_mapping::boundary::Matcher`) — +//! deliberately-unfilled home (`asap_aware_mapping::implementation::Matcher`) — //! CSE here does not attempt it. //! //! ## Legality: gated by `Schema::unique_keys` diff --git a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md index 37c786bc..ea91cb9c 100644 --- a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md @@ -168,7 +168,7 @@ The two methods have intentionally different responsibilities. fn rank_candidates(&self, intent: &AggIntent, candidates: &[SketchKind]) -> Vec; ``` -- **`size_params`** — pick concrete parameters (e.g. sketch capacity) for one already-chosen `SketchKind`, given an accuracy target `(eps, delta)`. This is a separate hook from `rank_candidates` specifically so a deployment can override *just* sizing (e.g. an empirically-tuned table, or discrete capacity rungs a downstream catalog requires) without also forking candidate selection — same "one extension point per decision" shape as everything else in this trait. Default: `boundary::default_size_params`, this crate's built-in per-family sizing formulas. +- **`size_params`** — pick concrete parameters (e.g. sketch capacity) for one already-chosen `SketchKind`, given an accuracy target `(eps, delta)`. This is a separate hook from `rank_candidates` specifically so a deployment can override *just* sizing (e.g. an empirically-tuned table, or discrete capacity rungs a downstream catalog requires) without also forking candidate selection — same "one extension point per decision" shape as everything else in this trait. Default: `implementation::default_size_params`, this crate's built-in per-family sizing formulas. ```rust fn size_params(&self, kind: SketchKind, intent: &AggIntent, eps: f64, delta: f64) -> SketchParams; @@ -226,7 +226,7 @@ A custom cost model does not necessarily need to override every hook. The curren ## 3. How the current pieces fit together -There are two related paths in the current code, and it's worth being explicit about how they relate before looking at either one, since neither name says it outright: **the binding path is older and still the one thing that actually runs in production; the replacement-strategy path is new and additive, not a replacement for it** (`replacement.rs`'s own module docs put this as "why this exists alongside `boundary`/`bind`, not instead of them"). Both start from the same input (an `AggIntent`) and both eventually reuse the exact same underlying decision logic (`boundary::implementation_for_with`, `bind::implement_tree_with`) — they differ only in *how much of the answer* they keep: +There are two related paths in the current code, and it's worth being explicit about how they relate before looking at either one, since neither name says it outright: **the binding path is older and still the one thing that actually runs in production; the replacement-strategy path is new and additive, not a replacement for it** (`replacement.rs`'s own module docs put this as "why this exists alongside `implementation`/`bind`, not instead of them"). Both start from the same input (an `AggIntent`) and both eventually reuse the exact same underlying decision logic (`implementation::implementation_for_with`, `bind::implement_tree_with`) — they differ only in *how much of the answer* they keep: - The **binding path** is what runs today when a query actually gets bound: it commits to one `Implementation` and discards every alternative a `CostModel` didn't pick, because something has to actually execute. - The **replacement-strategy path** is what `ReplacementStrategy::replacements()` (§2) produces: instead of committing to one, it *keeps every alternative* the binding path would have thrown away, packaged as `ReplacementSubDAG`s. @@ -243,7 +243,7 @@ Conceptually: AggIntent | v -boundary::implementation_for_with(...) +implementation::implementation_for_with(...) | v one Implementation @@ -283,7 +283,7 @@ The important rule is: > Strategies should reuse existing decision and binding logic where possible instead of reimplementing it. -For example, `SketchFamilyStrategy` uses the existing boundary and binding machinery to enumerate each valid sketch realization. +For example, `SketchFamilyStrategy` uses the existing implementation-selection and binding machinery to enumerate each valid sketch realization. --- @@ -473,7 +473,7 @@ If another module already knows how to determine whether something is legal or h Do not create a second implementation of the same semantics inside the strategy. -The existing `SketchFamilyStrategy` is the model to follow: it reuses the boundary candidate list and the existing binder. +The existing `SketchFamilyStrategy` is the model to follow: it reuses `implementation.rs`'s existing candidate list and the existing binder. --- @@ -526,13 +526,13 @@ Target Aggregate extract AggIntent | v -boundary::implementation_for_with(...) +implementation::implementation_for_with(...) | +--------------------------+ | | approximate sketch case v -boundary::summary_candidates(...) +implementation::summary_candidates(...) | v bind each sketch candidate separately @@ -545,7 +545,7 @@ For an approximate quantile, the current candidate list includes both KLL and DD The strategy returns both, even if the cost model ranks one ahead of the other. -For cases where boundary selection has only one realization, such as an exact accumulator or pass-through, the strategy returns that single realization. +For cases where the implementation decision has only one realization, such as an exact accumulator or pass-through, the strategy returns that single realization. --- @@ -823,7 +823,7 @@ The replacement-strategy layer should still expose both valid alternatives where --- -## 10. `Matcher` (`boundary.rs`) +## 10. `Matcher` (`implementation.rs`) There is a third extension point in this crate, much smaller in surface area than `ReplacementStrategy` or `CostModel`, but worth knowing about: @@ -833,9 +833,9 @@ pub trait Matcher { } ``` -`Matcher` answers a different question than everything above it: not "how do I *build* a summary for this intent" (that's `boundary`/`bind`/`ReplacementStrategy`), but "is there already a summary sitting around somewhere that answers this without building anything new?" This is the query-optimization-literature "answering queries using views" question, applied to summaries — the same idea as a database reusing an existing materialized view or index instead of recomputing from scratch. +`Matcher` answers a different question than everything above it: not "how do I *build* a summary for this intent" (that's `implementation`/`bind`/`ReplacementStrategy`), but "is there already a summary sitting around somewhere that answers this without building anything new?" This is the query-optimization-literature "answering queries using views" question, applied to summaries — the same idea as a database reusing an existing materialized view or index instead of recomputing from scratch. -Concrete example: a deployment already has a `DDSketch` built earlier for some other query on `latency`, and a new query needs `Kll` quantiles on `latency`. Can the existing `DDSketch` answer it without building a new `Kll` sketch? A pure sketch-algebra answer says yes — both are quantile sketches, mutually substitutable at the query level. But a deployment with its own storage-layout rules might say no, e.g. if its inventory ties a stored summary to a specific algorithm identity it won't re-interpret. `Matcher::is_satisfied_by(required, available)` is where a deployment plugs in whichever answer is actually true for its own storage layer — `required` is what a query needs (an `Implementation` `boundary`/`bind` computed), `available` is what already exists somewhere in that deployment's inventory. +Concrete example: a deployment already has a `DDSketch` built earlier for some other query on `latency`, and a new query needs `Kll` quantiles on `latency`. Can the existing `DDSketch` answer it without building a new `Kll` sketch? A pure sketch-algebra answer says yes — both are quantile sketches, mutually substitutable at the query level. But a deployment with its own storage-layout rules might say no, e.g. if its inventory ties a stored summary to a specific algorithm identity it won't re-interpret. `Matcher::is_satisfied_by(required, available)` is where a deployment plugs in whichever answer is actually true for its own storage layer — `required` is what a query needs (an `Implementation` value that `implementation`/`bind` already computed), `available` is what already exists somewhere in that deployment's inventory. Unlike `CostModel`, this crate ships **no default body and no implementation** for `Matcher` at all — the answer genuinely depends on facts this crate deliberately doesn't settle (see the example above: a pure sketch-algebra answer and a deployment's own storage-layout rules can legitimately disagree, and this crate has no inventory concept of its own to judge between them). If you need this, you're implementing the whole trait for your deployment from scratch; there's no default to lean on and no existing implementation in this crate to copy from. @@ -901,7 +901,7 @@ A new sketch family generally touches more than `ReplacementStrategy`. The strategy should not maintain its own private list of sketch kinds. -`SketchFamilyStrategy` obtains sketch alternatives through the existing boundary interface: +`SketchFamilyStrategy` obtains sketch alternatives through `implementation.rs`'s existing interface: ```rust summary_candidates(intent) @@ -912,7 +912,7 @@ and binds them through the normal binder. Therefore, when adding a new built-in sketch family, the intended flow is: ```text -1. Teach the boundary layer that the sketch is a valid candidate +1. Teach `implementation.rs` that the sketch is a valid candidate for the relevant AggIntent. 2. Teach the cost model how to rank and size it. @@ -1158,7 +1158,7 @@ fn replacements(...) -> Vec { ### Mistake: maintaining a second sketch-applicability table -If the boundary layer already defines which sketch families satisfy an `AggIntent`, reuse that source. +If `implementation.rs` already defines which sketch families satisfy an `AggIntent`, reuse that source. Otherwise the binder and replacement strategy can silently disagree. @@ -1240,7 +1240,7 @@ Use this table to find the right place for a change. | Add a new logical optimization | new `impl ReplacementStrategy` | | Add a new replacement for an existing target shape | `ReplacementStrategy::replacements` | | Change when a strategy applies | `ReplacementStrategy::matches` | -| Add a new built-in sketch candidate | boundary summary-candidate mapping | +| Add a new built-in sketch candidate | `implementation.rs`'s summary-candidate mapping | | Prefer one sketch family over another | `CostModel::rank_candidates` | | Change sketch sizing for an accuracy target | `CostModel::size_params` | | Add extension-defined implementation behavior | `CostModel::realize_extension` | @@ -1250,7 +1250,7 @@ Use this table to find the right place for a change. | Change current share/recompute choice | `CostModel::cse_share_decision` | | Decide whether an available implementation satisfies a required one | `impl Matcher` | | Produce a normal bound summary candidate | reuse `bind::implement_tree_with` | -| Enumerate valid sketch kinds | reuse `boundary::summary_candidates` | +| Enumerate valid sketch kinds | reuse `implementation::summary_candidates` | | Build a target with no workload context | `TargetSubDAG::new` | | Build a target with known sharing context | `TargetSubDAG::with_consumer_count` | From 145e9e9a611ee793d8dac45d712e0cb87feb26f3 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 23 Aug 2026 19:02:36 -0600 Subject: [PATCH 13/49] docs(asap-aware-mapping): address milind's high-level review on the dev guide - Remove Sec 19 'Design rule to remember' -- a closing 'one rule to keep in mind' framing reads as advice for a human's attention/memory, not useful as documentation content; the rule itself (enumerate vs. rank) is already stated where it matters (Sec 1, Sec 5 Rule 2). - Add a Terminology section up front defining 'implementation' and 'bind'/'binding' -- the latter is genuinely overloaded three ways across this crate and its downstream consumers (lib.rs's own Terminology section documents all three; this is the short version a reader needs before the rest of the guide uses 'the binding path' and 'implementation::implementation_for_with' without explanation). Sec 16-18 (Common mistakes / Extension checklist / Extension map) -- no change requested, called out as a pattern worth reusing for other dev docs. --- .../ASAP-aware-mapping-developer-guide.md | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md index ea91cb9c..b700b3af 100644 --- a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md @@ -16,6 +16,20 @@ Code samples named `My*` or `Prefer*` (`MyStrategy`, `MyCostModel`, `PreferDDSke --- +## Terminology + +Two words come up constantly below and are worth pinning down before anything else, since neither is self-explanatory from context alone: + +- **`implementation`** (the module `implementation.rs`) — the *per-node* decision of how one `AggIntent` gets realized: as an approximate sketch, an exact mergeable accumulator, or a pass-through (no summary at all). `implementation::implementation_for`/`implementation_for_with` make this decision for **one** node at a time; they don't walk anything. (This module used to be named `boundary.rs` — "the sketch-vs-exact boundary decision" — renamed to match its actual exported type, `Implementation`.) +- **`bind` / "binding"** — this word is genuinely overloaded across this crate and its downstream consumers; `lib.rs`'s own "Terminology" section documents all three senses in full, but the short version: + 1. *Name resolution* (classic RDBMS "Parse → Bind → Optimize" sense) — turning a column reference into a resolved schema column. Lives in `asap_types::pre_asap::binder::Binder`, **not** this crate. + 2. *This crate's `bind.rs`* — `implement_tree`/`implement_tree_with` walk a **whole** `QueryExpr` tree, calling `implementation::implementation_for` per node, and emit the complete post-ASAP `SummaryExpr`/`SummaryNode` DAG. This is what "the binding path" means everywhere in this guide (§3 onward): the *existing, already-shipping* code path that commits to one `Implementation` per node because something has to actually execute. + 3. *A downstream deployment's own physical binder* (post-ASAP → deployment placement, e.g. edge vs. backend) — a different, later decision this crate doesn't model at all. + +So: `implementation` decides **what** one node becomes; `bind`/`implement_tree` (sense 2) is **what walks a whole tree** applying that decision everywhere; `ReplacementStrategy` (this guide's main subject) is the newer, additive path that keeps every alternative `implementation` could have picked instead of collapsing to the one `bind` commits to. + +--- + ## 1. Mental model ASAP-aware mapping has two different jobs that should remain separate: @@ -1253,13 +1267,3 @@ Use this table to find the right place for a change. | Enumerate valid sketch kinds | reuse `implementation::summary_candidates` | | Build a target with no workload context | `TargetSubDAG::new` | | Build a target with known sharing context | `TargetSubDAG::with_consumer_count` | - ---- - -## 19. Design rule to remember - -If there is one rule to keep in mind when extending this layer, it is: - -> **Strategies define the valid search space; cost models express preferences within that search space.** - -Keeping those responsibilities separate makes new optimizations composable, keeps one source of truth for semantics, and allows a future whole-plan search engine to compare interactions between choices rather than inheriting irreversible local decisions. From 265497b5acdc10c66f44199c0ece5d9529d8c3d6 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 23 Aug 2026 20:07:01 -0600 Subject: [PATCH 14/49] refactor(asap-aware-mapping): unify binding path and ReplacementStrategy on one primitive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bind::implement_tree_with (the single-answer production binding path) and replacement::SketchFamilyStrategy (the exhaustive-candidate path) used to be two independent implementations of "turn a chosen Implementation into a SummaryNode": SketchFamilyStrategy got there by wrapping the real CostModel in a ForceSketchKind adapter that lied to rank_candidates, then routing through implement_tree_with anyway. This adds one shared low-level primitive, bind::bind_with_implementation(expr, implementation, cost_model), that both now bottom out in: - implement_tree_with resolves one ranked Implementation via implementation_for_with and hands it to bind_with_implementation. - SketchFamilyStrategy sizes each candidate itself (via two new small helpers, implementation::accuracy_target/accuracy_budget, extracted out of bind_summary_with) and hands each one to bind_with_implementation in turn. ForceSketchKind is deleted entirely — no more CostModel-forcing hack. This also fixes a latent bug: ForceSketchKind forced rank_candidates for the *whole* recursive bind, so enumerating a candidate for one target could leak into that target's own nested aggregates (e.g. a nested Quantile silently forced to the outer target's DDSketch pick). bind_with_implementation only forces the top node; recursion into children re-ranks normally. New regression test: forcing_the_targets_candidate_does_not_leak_into_a_nested_aggregate. Outward-facing behavior is unchanged: implementation_for/implement_tree_with still return exactly what they always did for the same inputs. Deliberately NOT done: making implement_tree_with call SketchFamilyStrategy::replacements() and take the first result literally. That would size and fully bind every candidate at every sketch-capable node, recursively, just to discard all but one — multiplying binding cost by the branching factor on a path that still runs in production. The design doc's future Cascades/Volcano-style search engine (not built yet, tracked separately) is the actual "ReplacementStrategy + CostModel-based selection replaces binding" story, at plan granularity — not this. Dev guide (§3, §6) rewritten to document the shared primitive and this tradeoff explicitly; Terminology section trimmed to drop the two "bind" senses that live outside this crate (name resolution, a downstream deployment's own binder) since they're not this guide's subject. Verified: cargo build/test/fmt/clippy --workspace all clean. Co-Authored-By: Claude Sonnet 5 --- crates/asap-aware-mapping/src/bind.rs | 58 ++++- .../asap-aware-mapping/src/implementation.rs | 44 +++- crates/asap-aware-mapping/src/replacement.rs | 212 +++++++++--------- .../ASAP-aware-mapping-developer-guide.md | 118 ++++++---- 4 files changed, 273 insertions(+), 159 deletions(-) diff --git a/crates/asap-aware-mapping/src/bind.rs b/crates/asap-aware-mapping/src/bind.rs index ef728f9c..7feab9b4 100644 --- a/crates/asap-aware-mapping/src/bind.rs +++ b/crates/asap-aware-mapping/src/bind.rs @@ -55,6 +55,24 @@ pub enum ImplementError { Schema(#[from] QueryExprError), } +/// The bindable shape this pass (and +/// [`crate::replacement::SketchFamilyStrategy`], which targets the exact +/// same shape) requires: a single intent, no `HAVING`. A multi-intent node +/// (SQL `SELECT SUM(a), AVG(b)`), or one with a `HAVING` predicate (the +/// filter would need the estimate first), stays logical — see the module +/// docs' "Conservative fallbacks". +pub fn bindable_intent(node: &QueryExpr) -> Option<&AggIntent> { + if let QueryExpr::Aggregate { + measures, having, .. + } = node + { + if let ([intent], None) = (measures.as_slice(), having) { + return Some(intent); + } + } + None +} + /// Bind a single query to the post-ASAP IR. Ranks candidate summaries via /// [`DefaultCostModel`] (`asap-plan`'s built-in static preference order, /// unchanged); use [`implement_tree_with`] to plug in a deployment-specific @@ -65,9 +83,48 @@ pub fn implement_tree(expr: &QueryExpr) -> Result, ImplementErro /// Like [`implement_tree`], but ranks candidate summaries via `cost_model` (see /// [`crate::cost_model`]) instead of the built-in static preference order. +/// +/// Thin: resolve *which* [`Implementation`] `cost_model` ranks first for +/// `expr`'s intent, then hand it to [`bind_with_implementation`] — the same +/// primitive [`crate::replacement::SketchFamilyStrategy`] calls once per +/// *candidate* `Implementation` instead of just the ranked-first one. This is +/// the one place the two paths' outputs are actually assembled into a +/// `SummaryNode`; ranking is the only thing that differs between them. pub fn implement_tree_with( expr: &QueryExpr, cost_model: &dyn CostModel, +) -> Result, ImplementError> { + match bindable_intent(expr) { + Some(intent) => bind_with_implementation( + expr, + implementation_for_with(intent, cost_model), + cost_model, + ), + None => logical(expr), + } +} + +/// Bind `expr` to an already-decided [`Implementation`] for its top intent, +/// instead of deriving one via [`implementation_for_with`]. The shared +/// low-level primitive: [`implement_tree_with`] computes the +/// `cost_model`-ranked `Implementation` and calls this; +/// [`crate::replacement::SketchFamilyStrategy`] sizes each candidate +/// `Implementation` itself (see +/// [`implementation::accuracy_budget`](crate::implementation::accuracy_budget)) +/// and calls this once per candidate — one function turns a chosen +/// `Implementation` into a `SummaryNode`, used by both the single-answer +/// production path and the exhaustive-candidate path. +/// +/// `expr` must still be the [`bindable_intent`] shape for `implementation` to +/// have any effect; anything else falls back to [`logical`], same as +/// [`implement_tree_with`]. Only `expr`'s own top-level decision is forced — +/// recursion into `expr`'s child re-ranks normally via `cost_model`, the same +/// as an ordinary [`implement_tree_with`] call, so forcing one target's +/// candidate never leaks into that target's own nested aggregates. +pub fn bind_with_implementation( + expr: &QueryExpr, + implementation: Implementation, + cost_model: &dyn CostModel, ) -> Result, ImplementError> { if let QueryExpr::Aggregate { reduction, @@ -80,7 +137,6 @@ pub fn implement_tree_with( // The bindable shape: exactly one intent, no HAVING. (Multi-intent // nodes and HAVING stay logical — see the module docs.) if let ([intent], None) = (measures.as_slice(), having) { - let implementation = implementation_for_with(intent, cost_model); if let Some((family, estimate)) = summary_family(implementation) { return bind_summary_agg( expr, reduction, intent, child, family, estimate, cost_model, diff --git a/crates/asap-aware-mapping/src/implementation.rs b/crates/asap-aware-mapping/src/implementation.rs index 5155c59c..24350ff4 100644 --- a/crates/asap-aware-mapping/src/implementation.rs +++ b/crates/asap-aware-mapping/src/implementation.rs @@ -160,6 +160,24 @@ pub fn implementation_for(intent: &AggIntent) -> Implementation { implementation_for_with(intent, &DefaultCostModel) } +/// The [`AccuracyTarget`] threaded onto an approximate-capable intent +/// (`Quantile`/`Cardinality`/`Count`/`TopK`), or `None` for every other +/// intent (no sketch candidate applies — [`implementation_for_with`]'s own +/// match routes those elsewhere). Exposed so +/// [`crate::replacement::SketchFamilyStrategy`] can resolve the same +/// accuracy target [`implementation_for_with`] would, without re-deriving +/// [`implementation_for_with`]'s whole dispatch just to read one field back +/// out. +pub fn accuracy_target(intent: &AggIntent) -> Option<&AccuracyTarget> { + match intent { + AggIntent::Quantile { accuracy, .. } + | AggIntent::Cardinality { accuracy, .. } + | AggIntent::Count { accuracy } + | AggIntent::TopK { accuracy, .. } => Some(accuracy), + _ => None, + } +} + /// Like [`implementation_for`], but ranks candidate summaries via `cost_model` (see /// [`crate::cost_model`]) instead of the built-in static preference order. pub fn implementation_for_with(intent: &AggIntent, cost_model: &dyn CostModel) -> Implementation { @@ -263,6 +281,24 @@ fn exact_accumulator(intent: &AggIntent, kind: ExactKind, params: ExactParams) - Implementation::ExactAggregate { kind, params } } +/// Resolve an [`AccuracyTarget`] into the `(eps, delta)` budget +/// [`CostModel::size_params`] needs. Shared by [`bind_summary_with`] (the +/// binding path's own choice) and +/// [`crate::replacement::SketchFamilyStrategy`] (which sizes every +/// candidate, not just the one the binding path would have picked) — one +/// place this resolution happens, so the two paths can't drift apart on it. +/// +/// `Exact` is unreachable via [`implementation_for`] (which routes `Exact` +/// to [`exact_realization`] instead); degrades to the tightest parameters +/// for a caller that resolves it directly anyway. +pub fn accuracy_budget(accuracy: &AccuracyTarget) -> (f64, f64) { + match accuracy { + AccuracyTarget::Exact => (f64::MIN_POSITIVE, DEFAULT_DELTA), + AccuracyTarget::Epsilon(e) => (*e, DEFAULT_DELTA), + AccuracyTarget::EpsilonDelta { epsilon, delta } => (*epsilon, *delta), + } +} + /// Bind the preferred candidate sketch, with parameters sized to the /// target, ranking [`summary_candidates`] via `cost_model` (see /// [`crate::cost_model`]) instead of taking the static-order head @@ -272,13 +308,7 @@ fn bind_summary_with( accuracy: &AccuracyTarget, cost_model: &dyn CostModel, ) -> Implementation { - let (eps, delta) = match accuracy { - // Unreachable via `implementation_for` (Exact routes to `exact_realization`); - // degrade to the tightest parameters if called directly. - AccuracyTarget::Exact => (f64::MIN_POSITIVE, DEFAULT_DELTA), - AccuracyTarget::Epsilon(e) => (*e, DEFAULT_DELTA), - AccuracyTarget::EpsilonDelta { epsilon, delta } => (*epsilon, *delta), - }; + let (eps, delta) = accuracy_budget(accuracy); let ranked = cost_model.rank_candidates(intent, summary_candidates(intent)); let kind = ranked .into_iter() diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index af4a0213..5ef97503 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -96,22 +96,29 @@ //! This module's own tests build `TargetSubDAG`s directly, the same //! hand-rolled-fixture style `bind.rs`/`implementation.rs`/`cost_model.rs`'s own //! tests already use. -//! - **[`implementation`]/[`bind`]'s existing single-pick public behavior is -//! unchanged.** Nothing here modifies [`implementation::implementation_for`], -//! [`bind::implement_tree_with`], or how either is called; this module adds -//! the exhaustive-candidate view alongside them, it does not replace or -//! rewire either yet (a later issue's job). +//! - **[`implementation`]/[`bind`]'s existing single-pick *outward-facing* +//! behavior is unchanged.** [`implementation::implementation_for`] and +//! [`bind::implement_tree_with`] still return exactly the same +//! `Implementation`/`SummaryNode` they always did, for the same inputs. +//! What changed is internal: [`bind::implement_tree_with`] and this +//! module's [`SketchFamilyStrategy`] now both bottom out in the same +//! [`bind::bind_with_implementation`] primitive — the binding path +//! resolves one ranked `Implementation` and hands it there; this module +//! sizes every candidate `Implementation` itself and hands each one there +//! in turn — rather than this module working around the binding path via +//! a `CostModel`-forcing adapter, as it used to. use std::rc::Rc; -use asap_types::post_asap::{SketchKind, SketchParams, SketchQuery, SummaryNode}; +use asap_types::post_asap::{SketchKind, SummaryNode}; use asap_types::pre_asap::agg_intent::AggIntent; -use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::query_expr::QueryExpr; -use crate::bind::implement_tree_with; -use crate::cost_model::{Cost, CostModel, CseCandidate, DefaultCostModel, ShareDecision}; -use crate::implementation::{implementation_for_with, summary_candidates, Implementation}; +use crate::bind::{bind_with_implementation, bindable_intent}; +use crate::cost_model::{CostModel, DefaultCostModel}; +use crate::implementation::{ + accuracy_budget, accuracy_target, implementation_for_with, summary_candidates, Implementation, +}; /// A pre-ASAP sub-DAG a [`ReplacementStrategy`] knows how to replace. /// @@ -213,23 +220,6 @@ pub trait ReplacementStrategy { // ── SketchFamilyStrategy ───────────────────────────────────────────────── -/// The bindable shape [`bind::implement_tree_with`] itself requires: a single -/// intent, no `HAVING` (a multi-intent node, or one with a `HAVING` -/// predicate, stays logical at actual binding time too — see `bind.rs`'s own -/// module docs on why — so neither is a target this strategy has an opinion -/// on). -fn bindable_intent(node: &QueryExpr) -> Option<&AggIntent> { - if let QueryExpr::Aggregate { - measures, having, .. - } = node - { - if let ([intent], None) = (measures.as_slice(), having) { - return Some(intent); - } - } - None -} - /// A single static instance so [`SketchFamilyStrategy::default_cost_model`] /// can hand out a `&'static dyn CostModel` without heap-allocating one — /// `DefaultCostModel` is a unit struct with no state, so one instance serves @@ -286,8 +276,11 @@ impl ReplacementStrategy for SketchFamilyStrategy<'_> { // match uses. Only `Sketch` has more than one candidate to enumerate // (`summary_candidates`); every other variant is the *only* // realization `implementation::implementation_for_with`'s dispatch - // produces for this intent, so there's nothing else to offer. - match implementation_for_with(intent, self.cost_model) { + // produces for this intent, so there's nothing else to offer — bind + // the one `implementation` this branch already computed rather than + // re-deriving it a second time. + let implementation = implementation_for_with(intent, self.cost_model); + match &implementation { Implementation::Sketch { .. } => summary_candidates(intent) .iter() .filter_map(|kind| { @@ -296,6 +289,7 @@ impl ReplacementStrategy for SketchFamilyStrategy<'_> { .collect(), Implementation::ExactAggregate { kind, .. } => single_candidate( target.root, + implementation.clone(), self.cost_model, format!( "{} realizes as an exact {kind:?} accumulator — the only realization \ @@ -306,6 +300,7 @@ impl ReplacementStrategy for SketchFamilyStrategy<'_> { ), Implementation::PassThrough => single_candidate( target.root, + implementation.clone(), self.cost_model, format!( "{} has no summary realization and stays a logical pass-through — the \ @@ -315,6 +310,7 @@ impl ReplacementStrategy for SketchFamilyStrategy<'_> { ), Implementation::Sample { kind, .. } => single_candidate( target.root, + implementation.clone(), self.cost_model, format!( "{} realizes as a {kind:?} sample — the only realization the plugged-in \ @@ -324,6 +320,7 @@ impl ReplacementStrategy for SketchFamilyStrategy<'_> { ), Implementation::Wavelet { kind, .. } => single_candidate( target.root, + implementation.clone(), self.cost_model, format!( "{} realizes as a {kind:?} wavelet transform — the only realization the \ @@ -333,6 +330,7 @@ impl ReplacementStrategy for SketchFamilyStrategy<'_> { ), Implementation::StatModel { kind, .. } => single_candidate( target.root, + implementation.clone(), self.cost_model, format!( "{} realizes as a {kind:?} statistical model — the only realization the \ @@ -344,19 +342,21 @@ impl ReplacementStrategy for SketchFamilyStrategy<'_> { } } -/// Bind `root` via `cost_model` unchanged (no candidate to steer towards) — -/// the "only option" path for any [`Implementation`] category besides -/// `Sketch`. Schema derivation cannot fail for a target that was already a -/// legitimate part of the workload's tree, but [`implement_tree_with`]'s -/// signature is fallible (`ImplementError`), so a failure here — never -/// expected in practice — degrades to "no candidate" rather than a panic, -/// same conservatism as the rest of this strategy. +/// Bind `root` to the already-computed `implementation` (no candidate to +/// steer towards) — the "only option" path for any [`Implementation`] +/// category besides `Sketch`. Schema derivation cannot fail for a target +/// that was already a legitimate part of the workload's tree, but +/// [`bind_with_implementation`]'s signature is fallible (`ImplementError`), +/// so a failure here — never expected in practice — degrades to "no +/// candidate" rather than a panic, same conservatism as the rest of this +/// strategy. fn single_candidate( root: &QueryExpr, + implementation: Implementation, cost_model: &dyn CostModel, rationale: String, ) -> Vec { - implement_tree_with(root, cost_model) + bind_with_implementation(root, implementation, cost_model) .ok() .map(|node| ReplacementSubDAG { replacement: Replacement::Summary(node), @@ -366,25 +366,30 @@ fn single_candidate( .collect() } -/// Bind `root` forcing the sketch-vs-exact boundary towards `kind` — one -/// entry of [`summary_candidates(intent)`](summary_candidates) — instead of -/// whichever candidate `cost_model` would otherwise rank first, by steering -/// [`ForceSketchKind`] through the exact same [`implement_tree_with`] call -/// [`bind::implement_tree_with`] itself would make. This reuses -/// `implement_tree_with`'s whole decision procedure (schema derivation, -/// column resolution, readout construction) unchanged — it does not -/// reimplement any part of it. +/// Bind `root` to `kind` — one entry of +/// [`summary_candidates(intent)`](summary_candidates) — sized the same way +/// [`implementation::bind_summary_with`] would (via +/// [`implementation::accuracy_budget`] and `cost_model.size_params`), then +/// handed to [`bind_with_implementation`] — the same primitive +/// [`bind::implement_tree_with`] itself bottoms out in for whichever +/// candidate `cost_model` ranks first. This reuses that whole decision +/// procedure (schema derivation, column resolution, readout construction) +/// unchanged; it only supplies a different top-level `Implementation` than +/// the ranked-first one. fn sketch_candidate( root: &QueryExpr, intent: &AggIntent, kind: SketchKind, cost_model: &dyn CostModel, ) -> Option { - let forced = ForceSketchKind { + let accuracy = accuracy_target(intent)?; + let (eps, delta) = accuracy_budget(accuracy); + let params = cost_model.size_params(kind.clone(), intent, eps, delta); + let implementation = Implementation::Sketch { kind: kind.clone(), - inner: cost_model, + params, }; - let node = implement_tree_with(root, &forced).ok()?; + let node = bind_with_implementation(root, implementation, cost_model).ok()?; Some(ReplacementSubDAG { replacement: Replacement::Summary(node), rationale: format!( @@ -395,65 +400,6 @@ fn sketch_candidate( }) } -/// A [`CostModel`] adapter that forces -/// [`rank_candidates`](CostModel::rank_candidates) to put `kind` first, -/// forwarding every other hook to `inner` unchanged — so binding a specific -/// candidate still respects whatever deployment-specific sizing/extension -/// behavior `inner` provides, exactly like binding via `inner` directly -/// would. Only ever constructed with a `kind` already present in the -/// `candidates` slice `implementation_for_with` passes to `rank_candidates` -/// (drawn straight from [`summary_candidates(intent)`](summary_candidates)), -/// so this never invents a candidate the underlying model didn't already -/// have available — see [`CostModel::rank_candidates`]'s own contract. -struct ForceSketchKind<'a> { - kind: SketchKind, - inner: &'a dyn CostModel, -} - -impl CostModel for ForceSketchKind<'_> { - fn rank_candidates(&self, intent: &AggIntent, candidates: &[SketchKind]) -> Vec { - let mut ranked = self.inner.rank_candidates(intent, candidates); - ranked.retain(|k| *k != self.kind); - ranked.insert(0, self.kind.clone()); - ranked - } - - fn size_params( - &self, - kind: SketchKind, - intent: &AggIntent, - eps: f64, - delta: f64, - ) -> SketchParams { - self.inner.size_params(kind, intent, eps, delta) - } - - fn realize_extension(&self, ext_kind: &str, payload: &serde_json::Value) -> Implementation { - self.inner.realize_extension(ext_kind, payload) - } - - fn readout_extension( - &self, - ext_kind: &str, - payload: &serde_json::Value, - col: &ColumnRef, - ) -> SketchQuery { - self.inner.readout_extension(ext_kind, payload, col) - } - - fn cse_recompute_cost(&self, candidate: &CseCandidate) -> Cost { - self.inner.cse_recompute_cost(candidate) - } - - fn cse_shared_maintenance_cost(&self, candidate: &CseCandidate) -> Cost { - self.inner.cse_shared_maintenance_cost(candidate) - } - - fn cse_share_decision(&self, candidate: &CseCandidate) -> ShareDecision { - self.inner.cse_share_decision(candidate) - } -} - /// A short human-readable label for an `AggIntent`, for /// [`ReplacementSubDAG::rationale`] text. Not exhaustive by design (unlike /// this crate's other `AggIntent` matches, e.g. @@ -736,6 +682,58 @@ mod tests { assert_eq!(kinds.len(), 2); } + /// Enumerating a candidate for the *target* node must only steer that + /// node's own decision — a nested aggregate underneath it still gets the + /// `cost_model`-ranked default, not the target's forced candidate. This + /// is the behavior [`sketch_candidate`]'s [`bind_with_implementation`] + /// call gets for free (only the top node's `Implementation` is forced; + /// `bind_summary_agg`'s recursion into the child re-ranks normally via + /// `cost_model`) — the old `ForceSketchKind`-`CostModel`-adapter + /// implementation forced *every* `rank_candidates` call for the whole + /// recursive bind, which would have silently forced a nested Quantile to + /// DDSketch too whenever the outer target's DDSketch candidate was + /// enumerated. + #[test] + fn forcing_the_targets_candidate_does_not_leak_into_a_nested_aggregate() { + // outer: quantile(0.99, ...) over inner: quantile(0.5, m) — both + // Quantile, so both share the [Kll, DDSketch] candidate list. + let inner = agg(vec![2], default_quantile(0.5), metric_scan(&["job"])); + let outer = Rc::new(agg(vec![], default_quantile(0.99), inner)); + let target = TargetSubDAG::new(&outer); + let replacements = SketchFamilyStrategy::default_cost_model().replacements(&target); + + let ddsketch = replacements + .iter() + .find(|r| { + matches!(&r.replacement, Replacement::Summary(node) + if summary_family_kind(node) == SketchKind::DDSketch) + }) + .expect("the outer target's DDSketch candidate must be present"); + let Replacement::Summary(node) = &ddsketch.replacement else { + unreachable!("filtered on Replacement::Summary above"); + }; + assert_eq!( + summary_family_kind(node), + SketchKind::DDSketch, + "the outer (target) node must be the forced candidate" + ); + + let asap_types::post_asap::SummaryExpr::SummaryEstimate { summary_input, .. } = &node.expr + else { + panic!("expected SummaryEstimate root, got {:?}", node.expr); + }; + let asap_types::post_asap::SummaryExpr::SummaryAgg { child, .. } = &summary_input.expr + else { + panic!("expected SummaryAgg, got {:?}", summary_input.expr); + }; + assert_eq!( + summary_family_kind(child), + SketchKind::Kll, + "the nested inner aggregate must still get the cost-model-ranked \ + default (Kll), not inherit the outer target's forced DDSketch" + ); + } + /// The `SummaryFamilyType`'s `SketchKind`, from the top `SummaryAgg` /// reachable under a (possibly `SummaryEstimate`-wrapped) bound root. fn summary_family_kind(node: &SummaryNode) -> SketchKind { diff --git a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md index b700b3af..6da01d46 100644 --- a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md @@ -12,7 +12,7 @@ It is written for developers who want to: The focus here is the **current code interfaces and their contracts**. For the higher-level motivation and future search design, see the separate design document, [`docs/design_docs/asap_aware_mapping.md`](../design_docs/asap_aware_mapping.md). -Code samples named `My*` or `Prefer*` (`MyStrategy`, `MyCostModel`, `PreferDDSketch`, …) below are illustrative sketches of a pattern, not code that ships in this crate. Samples that name a real type (`SketchFamilyStrategy`, `ForceSketchKind`, `SharedSubtreeStrategy`, …) are copied verbatim from `replacement.rs`/`cost_model.rs`. +Code samples named `My*` or `Prefer*` (`MyStrategy`, `MyCostModel`, `PreferDDSketch`, …) below are illustrative sketches of a pattern, not code that ships in this crate. Samples that name a real type (`SketchFamilyStrategy`, `SharedSubtreeStrategy`, `bind_with_implementation`, …) are copied verbatim from `replacement.rs`/`bind.rs`/`cost_model.rs`. --- @@ -20,13 +20,10 @@ Code samples named `My*` or `Prefer*` (`MyStrategy`, `MyCostModel`, `PreferDDSke Two words come up constantly below and are worth pinning down before anything else, since neither is self-explanatory from context alone: -- **`implementation`** (the module `implementation.rs`) — the *per-node* decision of how one `AggIntent` gets realized: as an approximate sketch, an exact mergeable accumulator, or a pass-through (no summary at all). `implementation::implementation_for`/`implementation_for_with` make this decision for **one** node at a time; they don't walk anything. (This module used to be named `boundary.rs` — "the sketch-vs-exact boundary decision" — renamed to match its actual exported type, `Implementation`.) -- **`bind` / "binding"** — this word is genuinely overloaded across this crate and its downstream consumers; `lib.rs`'s own "Terminology" section documents all three senses in full, but the short version: - 1. *Name resolution* (classic RDBMS "Parse → Bind → Optimize" sense) — turning a column reference into a resolved schema column. Lives in `asap_types::pre_asap::binder::Binder`, **not** this crate. - 2. *This crate's `bind.rs`* — `implement_tree`/`implement_tree_with` walk a **whole** `QueryExpr` tree, calling `implementation::implementation_for` per node, and emit the complete post-ASAP `SummaryExpr`/`SummaryNode` DAG. This is what "the binding path" means everywhere in this guide (§3 onward): the *existing, already-shipping* code path that commits to one `Implementation` per node because something has to actually execute. - 3. *A downstream deployment's own physical binder* (post-ASAP → deployment placement, e.g. edge vs. backend) — a different, later decision this crate doesn't model at all. +- **`implementation`** (the module `implementation.rs`) — the *per-node* decision of how one `AggIntent` gets realized: as an approximate sketch, an exact mergeable accumulator, or a pass-through (no summary at all). `implementation::implementation_for`/`implementation_for_with` make this decision for **one** node at a time; they don't walk anything. +- **`bind` / "binding"** — this crate's own `bind.rs`: `implement_tree`/`implement_tree_with` walk a **whole** `QueryExpr` tree, calling `implementation::implementation_for` per node, and emit the complete post-ASAP `SummaryExpr`/`SummaryNode` DAG. This is what "the binding path" means everywhere in this guide (§3 onward): the code path that commits to one `Implementation` per node because something has to actually execute. (The word "bind" means other things elsewhere in this crate's downstream consumers — see `lib.rs`'s own "Terminology" section if you need the full picture — but within this guide, "bind"/"binding" always means this.) -So: `implementation` decides **what** one node becomes; `bind`/`implement_tree` (sense 2) is **what walks a whole tree** applying that decision everywhere; `ReplacementStrategy` (this guide's main subject) is the newer, additive path that keeps every alternative `implementation` could have picked instead of collapsing to the one `bind` commits to. +So: `implementation` decides **what** one node becomes; `bind`/`implement_tree` is **what walks a whole tree** applying that decision everywhere; `ReplacementStrategy` (this guide's main subject) is the newer, additive path that keeps every alternative `implementation` could have picked instead of collapsing to the one `bind` commits to. --- @@ -240,18 +237,46 @@ A custom cost model does not necessarily need to override every hook. The curren ## 3. How the current pieces fit together -There are two related paths in the current code, and it's worth being explicit about how they relate before looking at either one, since neither name says it outright: **the binding path is older and still the one thing that actually runs in production; the replacement-strategy path is new and additive, not a replacement for it** (`replacement.rs`'s own module docs put this as "why this exists alongside `implementation`/`bind`, not instead of them"). Both start from the same input (an `AggIntent`) and both eventually reuse the exact same underlying decision logic (`implementation::implementation_for_with`, `bind::implement_tree_with`) — they differ only in *how much of the answer* they keep: +There are two related paths in the current code, and it's worth being explicit about how they relate before looking at either one, since neither name says it outright: **the binding path is older and still the one thing that actually runs in production; the replacement-strategy path is new and additive, not a replacement for it.** They differ only in *how much of the answer* they keep — but under the hood, they now share one primitive rather than two independent implementations of "turn a chosen `Implementation` into a `SummaryNode`": -- The **binding path** is what runs today when a query actually gets bound: it commits to one `Implementation` and discards every alternative a `CostModel` didn't pick, because something has to actually execute. -- The **replacement-strategy path** is what `ReplacementStrategy::replacements()` (§2) produces: instead of committing to one, it *keeps every alternative* the binding path would have thrown away, packaged as `ReplacementSubDAG`s. +- The **binding path** (`bind::implement_tree_with`) is what runs today when a query actually gets bound: it ranks candidates via a `CostModel`, keeps only the one ranked first, and discards every alternative. +- The **replacement-strategy path** (`ReplacementStrategy::replacements()`, §2) is what enumerates candidates instead of picking one: for each valid `Implementation` a target could realize as, it *keeps* the bound result instead of throwing it away, packaged as a `ReplacementSubDAG`. -This is exactly the "alternatives"/"candidates" language in [the design doc](../design_docs/asap_aware_mapping.md): a `ReplacementSubDAG` **is** one candidate; a `TargetSubDAG` with its full `replacements()` list **is** the set of alternatives for one spot in the plan. The design doc describes a *future* search (not built yet, tracked separately) that would compare whole candidate plans built from these alternatives — the two paths below are what that future search would draw on, not the search itself. +This is exactly the "alternatives"/"candidates" language in [the design doc](../design_docs/asap_aware_mapping.md): a `ReplacementSubDAG` **is** one candidate; a `TargetSubDAG` with its full `replacements()` list **is** the set of alternatives for one spot in the plan. -### Binding path +**Is binding "replaced by" `ReplacementStrategy` + `CostModel`-based selection?** Not today, and not literally, on purpose. `implement_tree_with` does not call `SketchFamilyStrategy::replacements()` and take the first result — it resolves its one `Implementation` directly (`implementation_for_with`, which ranks and takes the head) and binds only that. Routing production binding through `replacements()` would mean *sizing and fully binding every candidate* (both KLL and DDSketch, say) at *every* sketch-capable node, recursively, just to discard all but one — multiplying the cost of ordinary binding by the branching factor at each such node. That's why the two paths share a low-level primitive instead of one being layered on top of the other. What *is* true, and tracked separately, not built yet: the design doc's future Cascades/Volcano-style search engine — generate candidates via `ReplacementStrategy` across a whole plan, evaluate/select via `CostModel` — would be exactly "binding replaced by enumerate-then-select," just at plan granularity rather than reimplementing `implement_tree_with` itself. The two paths described below are what that future search would draw on, not the search itself. -The binding path needs one executable answer. +### The shared primitive: `bind_with_implementation` -Conceptually: +Both paths bottom out in the same function, `bind::bind_with_implementation(expr, implementation, cost_model)` — *given* an already-decided `Implementation` for `expr`'s top intent, bind it into a `SummaryNode` (or fall back to a logical passthrough). Neither path re-decides how a chosen `Implementation` becomes a `SummaryNode`; they only differ in **which** `Implementation`(s) they hand it: + +```text + chooses which Implementation(s) + | + binding path | replacement-strategy path + (implement_tree_with) | (SketchFamilyStrategy) + | | | + v | v + implementation_for_with(...) | summary_candidates(intent) + | | .iter().map(size each) + v | | + one ranked Implementation | every candidate Implementation + | | | + +----------------+----------------+----------+ + | + v + bind_with_implementation(expr, implementation, cost_model) + | + v + one SummaryNode + (one per Implementation handed in) +``` + +Only `expr`'s own top-level decision is ever forced — recursion into `expr`'s child re-ranks normally via `cost_model`, so enumerating a candidate for one target never leaks into that target's own nested aggregates. + +### Binding path + +The binding path needs one executable answer: rank via `cost_model`, take the head, bind it. ```text AggIntent @@ -263,21 +288,17 @@ implementation::implementation_for_with(...) one Implementation | v -bind::implement_tree_with(...) +bind::bind_with_implementation(...) | v one bound SummaryNode ``` -A `CostModel` is used here because binding must eventually commit to one implementation. - --- ### Replacement-strategy path -The strategy path exists to expose alternatives rather than immediately commit to one. - -Conceptually: +The strategy path exists to expose alternatives rather than immediately commit to one: for every candidate `Implementation`, size it and hand it to the same `bind_with_implementation` the binding path uses. ```text TargetSubDAG @@ -288,16 +309,19 @@ ReplacementStrategy::matches(...) v ReplacementStrategy::replacements(...) | - +-------------------------------+ - | | | -candidate A candidate B candidate C + +---------------------------------------------+ + | | | +bind_with_implementation(..., candidate A, ...) ... + | | | + v v v +candidate A candidate B candidate C ``` The important rule is: > Strategies should reuse existing decision and binding logic where possible instead of reimplementing it. -For example, `SketchFamilyStrategy` uses the existing implementation-selection and binding machinery to enumerate each valid sketch realization. +`SketchFamilyStrategy` follows this literally: it doesn't reimplement any part of binding, and it doesn't work around it via a `CostModel`-forcing adapter either — it sizes each candidate the same way `implementation_for_with` would (`implementation::accuracy_budget` + `cost_model.size_params`) and calls `bind_with_implementation` directly, once per candidate. --- @@ -563,38 +587,43 @@ For cases where the implementation decision has only one realization, such as an --- -### Why `ForceSketchKind` exists - -The ordinary binding path asks the cost model to rank sketch candidates and then binds the first choice. +### How each candidate actually gets bound: `bind_with_implementation` -That is correct for normal binding, but a replacement strategy needs to bind **each** valid sketch candidate. +The ordinary binding path (`implement_tree_with`) asks the cost model to rank sketch candidates and then binds only the first choice. -`ForceSketchKind` is a small `CostModel` adapter used for that purpose. It's real code — copied verbatim from `replacement.rs`. +That's correct for normal binding, but a replacement strategy needs to bind **each** valid sketch candidate — and it does so through the *same* underlying primitive `implement_tree_with` itself uses, `bind::bind_with_implementation(expr, implementation, cost_model)`, rather than working around `implement_tree_with`'s "always binds the ranked-first candidate" behavior. -The trick: `implement_tree_with` always binds whichever candidate `rank_candidates` puts *first* — there's no way to tell it directly "bind KLL this time." So to bind KLL specifically, wrap the real cost model in a `ForceSketchKind { kind: Kll, inner: real_cost_model }` and pass *that* into `implement_tree_with` instead. `ForceSketchKind::rank_candidates` forces `kind` (here, `Kll`) to the front of whatever `inner` would have ranked, so `implement_tree_with`'s "take the first candidate" logic ends up binding `Kll` — while every other decision (sizing, extension realization, …) still comes from `inner` unchanged, since only `rank_candidates` is overridden. Do this once per candidate (`ForceSketchKind { kind: Kll, .. }`, then separately `ForceSketchKind { kind: DDSketch, .. }`) and each gets bound through the exact same real machinery, one at a time, without touching `implement_tree_with`'s internals at all. It overrides only ranking: +`bind_with_implementation` takes an already-decided `Implementation` directly — no ranking involved — and binds `expr` to it. `implement_tree_with` is a thin wrapper: rank via `cost_model`, take the head, hand it to `bind_with_implementation`. `SketchFamilyStrategy` skips the ranking step entirely and constructs the `Implementation` for each candidate itself: ```rust -fn rank_candidates( - &self, +fn sketch_candidate( + root: &QueryExpr, intent: &AggIntent, - candidates: &[SketchKind], -) -> Vec { - let mut ranked = - self.inner.rank_candidates(intent, candidates); - - ranked.retain(|k| *k != self.kind); - ranked.insert(0, self.kind.clone()); - ranked + kind: SketchKind, + cost_model: &dyn CostModel, +) -> Option { + let accuracy = accuracy_target(intent)?; + let (eps, delta) = accuracy_budget(accuracy); + let params = cost_model.size_params(kind.clone(), intent, eps, delta); + let implementation = Implementation::Sketch { + kind: kind.clone(), + params, + }; + let node = + bind_with_implementation(root, implementation, cost_model).ok()?; + // ... wrap `node` in a ReplacementSubDAG with a rationale } ``` -Every other cost-model hook is forwarded to the underlying model. +`accuracy_target`/`accuracy_budget` (both in `implementation.rs`) are the same accuracy-resolution logic `implementation_for_with`'s own ranked pick goes through — sizing one candidate this way and letting `implementation_for_with` size the ranked-first candidate can never drift apart, because both call the same two functions. The result is: -> bind this specific sketch family, but keep using the caller's real cost model for sizing and all other behavior. +> bind this specific sketch family, sized exactly the way the ordinary binding path would size it, without touching any of `bind_with_implementation`'s internals. + +Because only `expr`'s own top-level `Implementation` is forced — `bind_with_implementation` recurses into `expr`'s child via the ordinary `cost_model`-ranked path, not a forced one — enumerating a candidate for one target never leaks into that target's own nested aggregates. (An earlier version of this strategy forced ranking via a `CostModel`-wrapping adapter for the whole recursive bind, which had exactly that leak as a latent bug; `bind_with_implementation` doesn't have the problem because it never re-ranks anything below the top node.) -This is an important pattern to preserve when adding another strategy that needs to enumerate alternatives through an API that normally chooses one. +This is the pattern to preserve when adding another strategy that needs to enumerate alternatives through an API that normally chooses one: construct the `Implementation` (or equivalent) explicitly, and hand it to the shared low-level primitive — don't wrap the `CostModel` to trick the ranked-first path into producing what you want. --- @@ -1263,7 +1292,8 @@ Use this table to find the right place for a change. | Change shared-maintenance cost | `CostModel::cse_shared_maintenance_cost` | | Change current share/recompute choice | `CostModel::cse_share_decision` | | Decide whether an available implementation satisfies a required one | `impl Matcher` | -| Produce a normal bound summary candidate | reuse `bind::implement_tree_with` | +| Produce a normal (ranked-first) bound summary | reuse `bind::implement_tree_with` | +| Bind a specific, already-chosen `Implementation` | reuse `bind::bind_with_implementation` | | Enumerate valid sketch kinds | reuse `implementation::summary_candidates` | | Build a target with no workload context | `TargetSubDAG::new` | | Build a target with known sharing context | `TargetSubDAG::with_consumer_count` | From 9cc65f02e9a38322fc42eb2d023bd9e6232103e7 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 23 Aug 2026 20:40:24 -0600 Subject: [PATCH 15/49] refactor(asap-aware-mapping): make binding literally a selector over ReplacementStrategy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per explicit follow-up request: only the replacement-strategy path should decide what an AggIntent may become; the binding path should be a thin selector over it, not an independent decision procedure that happens to agree with it. implementation.rs: - Replace implementation_for/implementation_for_with (single-answer) with implementations_for_with(intent, cost_model) -> Vec: exhaustive over the AggIntent vocabulary (same "no silent fallthrough" match as before) and ranked, most-preferred first. This is now the *only* function in the crate that decides what an AggIntent may become. bind.rs: - implement_tree_with no longer computes an Implementation on its own. It builds a TargetSubDAG, asks SketchFamilyStrategy::replacements() for every candidate, and keeps the first (cost_model-preferred) one. A new internal select_and_bind(&Rc, cost_model) is the actual selector, reused by implement_workload_with and by bind_summary_agg's own recursion into a child (so nested aggregates also go through selection, not a forced pick). - bind_summary_agg's child parameter is now &Rc (was &QueryExpr) so recursion passes the caller's existing Rc straight to select_and_bind instead of fabricating a new one per node; only the outermost implement_tree_with call clones once to get an Rc for TargetSubDAG. replacement.rs: - SketchFamilyStrategy::replacements() collapses from a 6-arm per-category match (single_candidate/sketch_candidate, each rebuilding rationale text) into one loop over implementations_for_with(intent, cost_model), binding each entry via bind_with_implementation and describing it via a new describe_implementation helper. - Module docs rewritten: this crate now has one place (implementation.rs) that enumerates every valid Implementation; bind.rs is a selector over it, not a second decision path. Documented plainly: an ordinary bind now sizes and fully constructs every sketch candidate at every sketch-capable node, not just the one kept, in exchange for one place owning what an AggIntent may become. What's still NOT built: comparing whole candidate *plans* (the design doc's future Cascades/Volcano-style search) - what exists is a single-node stand-in (cost_model.rank_candidates) applied one node at a time. Outward-facing behavior is unchanged for the same inputs — all 53 asap-aware-mapping unit tests and every integration test pass unmodified in assertions (only import/rename churn), confirming implement_tree/ implement_tree_with/implement_workload/implement_workload_with still produce byte-identical output to before this refactor. Dev guide (Terminology, §3, §6) rewritten to match: binding is now literally "call SketchFamilyStrategy::replacements(), keep the head", not a parallel implementation sharing a low-level primitive. Verified: cargo build/test/fmt/clippy --workspace all clean. Co-Authored-By: Claude Sonnet 5 --- crates/asap-aware-mapping/src/bind.rs | 129 ++++--- crates/asap-aware-mapping/src/cost_model.rs | 15 +- .../asap-aware-mapping/src/implementation.rs | 214 +++++++----- crates/asap-aware-mapping/src/lib.rs | 56 +-- crates/asap-aware-mapping/src/replacement.rs | 328 +++++++----------- .../ASAP-aware-mapping-developer-guide.md | 142 ++++---- 6 files changed, 441 insertions(+), 443 deletions(-) diff --git a/crates/asap-aware-mapping/src/bind.rs b/crates/asap-aware-mapping/src/bind.rs index 7feab9b4..5b63733f 100644 --- a/crates/asap-aware-mapping/src/bind.rs +++ b/crates/asap-aware-mapping/src/bind.rs @@ -2,7 +2,9 @@ //! //! Walks a canonical pre-ASAP [`QueryExpr`] and emits the summary-bound //! post-ASAP IR ([`SummaryExpr`] / [`SummaryNode`] in `asap-sketch`). Per -//! node, the [`implementation`](crate::implementation) decision picks the realization: +//! node, [`replacement::SketchFamilyStrategy`](crate::replacement::SketchFamilyStrategy) +//! enumerates every valid realization and this pass keeps the first +//! (`cost_model`-preferred) one: //! //! - **Summary family** (sketch / sample / wavelet / statistical model) — //! the `Aggregate` becomes a [`SummaryExpr::SummaryAgg`] carrying the @@ -20,7 +22,25 @@ //! //! Binding recurses through the `Aggregate` spine, so nested aggregates each //! get their own decision (`quantile(0.9, sum by (svc) (rate(m[5m]))))` binds -//! KLL over an exact `Sum` accumulator over an exact `Rate` accumulator). +//! KLL over an exact `Sum` accumulator over an exact `Rate` accumulator) — +//! and each nested node's own candidate enumeration and selection is +//! independent of its parent's. +//! +//! ## This pass is a selector over `ReplacementStrategy`, not a separate decision path +//! +//! [`implement_tree_with`] does not compute an `Implementation` on its own. +//! It builds a [`TargetSubDAG`](crate::replacement::TargetSubDAG) for the +//! node being bound, asks +//! [`replacement::SketchFamilyStrategy`](crate::replacement::SketchFamilyStrategy) +//! for every candidate [`Replacement`](crate::replacement::Replacement), and +//! keeps the first one — the same list a caller who wants every alternative +//! (rather than just the first) would get from calling +//! `SketchFamilyStrategy::replacements` directly. [`bind_with_implementation`] +//! is the shared low-level primitive both this pass and `SketchFamilyStrategy` +//! bottom out in: given an *already-decided* [`Implementation`], turn it into +//! a `SummaryNode`. See [`crate::replacement`]'s module docs for the reverse +//! half of this relationship — `SketchFamilyStrategy` calls back into this +//! module for that primitive and for [`bindable_intent`]. //! //! ## Conservative fallbacks //! @@ -45,7 +65,10 @@ use asap_types::pre_asap::schema::Schema; use thiserror::Error; use crate::cost_model::{CostModel, CseCandidate, DefaultCostModel, ShareDecision}; -use crate::implementation::{implementation_for_with, Implementation}; +use crate::implementation::Implementation; +use crate::replacement::{ + Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG, +}; /// Errors from the pre-ASAP → post-ASAP binding pass. #[derive(Debug, Error)] @@ -84,43 +107,67 @@ pub fn implement_tree(expr: &QueryExpr) -> Result, ImplementErro /// Like [`implement_tree`], but ranks candidate summaries via `cost_model` (see /// [`crate::cost_model`]) instead of the built-in static preference order. /// -/// Thin: resolve *which* [`Implementation`] `cost_model` ranks first for -/// `expr`'s intent, then hand it to [`bind_with_implementation`] — the same -/// primitive [`crate::replacement::SketchFamilyStrategy`] calls once per -/// *candidate* `Implementation` instead of just the ranked-first one. This is -/// the one place the two paths' outputs are actually assembled into a -/// `SummaryNode`; ranking is the only thing that differs between them. +/// A thin selector, not a second decision path — see the module docs. Binds +/// a fresh `Rc` for `expr` (cheap: `QueryExpr::clone` only clones +/// the top node, every descendant stays `Rc`-shared) so +/// [`TargetSubDAG`](crate::replacement::TargetSubDAG) has the identity it +/// needs; a caller that already holds `expr` as an `Rc` (e.g. +/// [`implement_workload_with`]) should prefer passing that `Rc` straight +/// into `SketchFamilyStrategy` itself to skip the extra clone. pub fn implement_tree_with( expr: &QueryExpr, cost_model: &dyn CostModel, ) -> Result, ImplementError> { - match bindable_intent(expr) { - Some(intent) => bind_with_implementation( - expr, - implementation_for_with(intent, cost_model), - cost_model, - ), - None => logical(expr), + select_and_bind(&Rc::new(expr.clone()), cost_model) +} + +/// The actual selector: build a [`TargetSubDAG`] for `root`, ask +/// [`SketchFamilyStrategy`] for every candidate, and keep the first +/// (`cost_model`-preferred) one. `root` must already be the caller's own +/// `Rc` — never fabricated per recursion step — so this stays a single +/// allocation at the outermost [`implement_tree_with`] call, not one per +/// node. +fn select_and_bind( + root: &Rc, + cost_model: &dyn CostModel, +) -> Result, ImplementError> { + let target = TargetSubDAG::new(root); + match SketchFamilyStrategy::new(cost_model) + .replacements(&target) + .into_iter() + .next() + { + Some(ReplacementSubDAG { + replacement: Replacement::Summary(node), + .. + }) => Ok(node), + Some(ReplacementSubDAG { + replacement: Replacement::Rewrite(_), + .. + }) => { + unreachable!("SketchFamilyStrategy never returns a Rewrite candidate") + } + // No candidate at all: `root` isn't `bindable_intent` shape (or its + // intent has no realization `implementations_for_with` can't + // produce — never happens, that match is exhaustive) — the same + // conservative fallback `SketchFamilyStrategy::matches` uses. + None => logical(root), } } -/// Bind `expr` to an already-decided [`Implementation`] for its top intent, -/// instead of deriving one via [`implementation_for_with`]. The shared -/// low-level primitive: [`implement_tree_with`] computes the -/// `cost_model`-ranked `Implementation` and calls this; -/// [`crate::replacement::SketchFamilyStrategy`] sizes each candidate -/// `Implementation` itself (see -/// [`implementation::accuracy_budget`](crate::implementation::accuracy_budget)) -/// and calls this once per candidate — one function turns a chosen -/// `Implementation` into a `SummaryNode`, used by both the single-answer -/// production path and the exhaustive-candidate path. +/// Bind `expr` to an already-decided [`Implementation`] for its top intent. +/// The shared low-level primitive: [`select_and_bind`] (and so +/// [`implement_tree_with`]) calls this with whichever candidate it kept; +/// [`crate::replacement::SketchFamilyStrategy`] calls this once per +/// candidate it enumerates. One function turns a chosen `Implementation` +/// into a `SummaryNode`, used identically by both. /// /// `expr` must still be the [`bindable_intent`] shape for `implementation` to -/// have any effect; anything else falls back to [`logical`], same as -/// [`implement_tree_with`]. Only `expr`'s own top-level decision is forced — -/// recursion into `expr`'s child re-ranks normally via `cost_model`, the same -/// as an ordinary [`implement_tree_with`] call, so forcing one target's -/// candidate never leaks into that target's own nested aggregates. +/// have any effect; anything else falls back to [`logical`]. Only `expr`'s +/// own top-level decision is forced — recursion into `expr`'s child goes +/// back through [`select_and_bind`] (fresh candidate enumeration, not a +/// forced pick), so choosing one candidate for a target never leaks into +/// that target's own nested aggregates. pub fn bind_with_implementation( expr: &QueryExpr, implementation: Implementation, @@ -214,9 +261,9 @@ pub fn implement_workload_with( let result = match memo.get(&ptr) { Some((cached, ShareDecision::Share)) => Ok(Rc::clone(cached)), Some((_, ShareDecision::RecomputeIndependently)) => { - implement_tree_with(&expr, cost_model) + select_and_bind(&expr, cost_model) } - None => implement_tree_with(&expr, cost_model).inspect(|node| { + None => select_and_bind(&expr, cost_model).inspect(|node| { let count = consumer_count[&ptr]; let decision = if count > 1 { let candidate = CseCandidate { @@ -269,7 +316,7 @@ fn bind_summary_agg( node: &QueryExpr, reduction: &Reduction, intent: &AggIntent, - child: &QueryExpr, + child: &Rc, family: SummaryFamilyType, estimate: bool, cost_model: &dyn CostModel, @@ -301,7 +348,7 @@ fn bind_summary_agg( // place that decides this; nothing downstream re-derives it. let agg = Rc::new(SummaryNode { expr: SummaryExpr::SummaryAgg { - child: implement_tree_with(child, cost_model)?, + child: select_and_bind(child, cost_model)?, family, col, reduction: reduction.clone(), @@ -378,7 +425,7 @@ fn readout(intent: &AggIntent, col: &ColumnRef, cost_model: &dyn CostModel) -> S } other => { unreachable!( - "no summary realization for {other:?} (implementation::implementation_for)" + "no summary realization for {other:?} (implementation::implementations_for_with)" ) } } @@ -386,11 +433,11 @@ fn readout(intent: &AggIntent, col: &ColumnRef, cost_model: &dyn CostModel) -> S /// Wrap an unrewritten pre-ASAP subtree, lifting its schema with every column /// `SummaryFamilyType::Plain`. Public so a deployment can force a node it -/// knows `implement_tree_in_with` would otherwise actively (mis)bind — -/// e.g. an intent this crate's `implementation::implementation_for` maps to an -/// accumulator kind the deployment's runtime doesn't actually implement — -/// through the same fallback this crate's own dispatch uses, without -/// duplicating the schema-lift logic. +/// knows [`implement_tree_with`] would otherwise actively (mis)bind — +/// e.g. an intent this crate's `implementation::implementations_for_with` +/// maps to an accumulator kind the deployment's runtime doesn't actually +/// implement — through the same fallback this crate's own dispatch uses, +/// without duplicating the schema-lift logic. pub fn logical(expr: &QueryExpr) -> Result, ImplementError> { let schema = expr.output_schema()?; Ok(Rc::new(SummaryNode { diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index c644c44e..e4ff888d 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -9,7 +9,7 @@ //! interface every deployment's cost model plugs into, so [`implementation`]'s //! summary selection has exactly one extension point instead of forcing //! each downstream (ASAPCollector + ASAPQuery-backend, ASAPFusion, …) to -//! fork [`implementation::implementation_for`]. +//! fork [`implementation::implementations_for_with`]. //! //! This trait is scoped to the approximate-**sketch** family specifically //! ([`CostModel::rank_candidates`]/[`size_params`](CostModel::size_params) @@ -26,8 +26,7 @@ //! than overloading these ones across incompatible `Kind`/`Params` types. //! //! Every entry point that doesn't take an explicit `&dyn CostModel` -//! ([`implementation_for`](crate::implementation::implementation_for), -//! [`implement_tree`](crate::bind::implement_tree)) runs against +//! ([`implement_tree`](crate::bind::implement_tree)) runs against //! [`DefaultCostModel`], so a deployment that never plugs in its own cost //! model keeps today's static-preference-order behavior exactly, byte for //! byte. @@ -172,7 +171,7 @@ pub fn default_cse_shared_maintenance_cost(family: &SummaryFamilyType) -> Cost { /// intent, in an arbitrary static preference order (issue #98's "one home" /// for the candidate set). A `CostModel` re-orders that list under real, /// deployment-specific cost knowledge this crate has no way to know about — -/// [`implementation::implementation_for_with`] implements whichever candidate ends +/// [`implementation::implementations_for_with`] implements whichever candidate ends /// up first after ranking. pub trait CostModel { /// Rank `candidates` (as returned by @@ -183,9 +182,9 @@ pub trait CostModel { /// available in their deployment, but MUST NOT invent a candidate that /// wasn't in the input — an unknown [`SketchKind`] has no /// [`SketchParams`](asap_types::post_asap::SketchParams) sizing logic in - /// [`implementation::implementation_for_with`] and binding it will panic. + /// [`implementation::implementations_for_with`] and binding it will panic. /// Returning an empty `Vec` means "no candidate is acceptable"; - /// `implementation_for_with` treats that the same as `candidates` + /// `implementations_for_with` treats that the same as `candidates` /// having been empty to begin with. fn rank_candidates(&self, intent: &AggIntent, candidates: &[SketchKind]) -> Vec; @@ -196,7 +195,7 @@ pub trait CostModel { /// Splitting sizing out from candidate selection lets a deployment own /// its own parameter-sizing math (e.g. an empirically-tuned table, or /// discrete rungs required by a downstream catalog) without forking - /// [`implementation::implementation_for_with`] — the same "one extension + /// [`implementation::implementations_for_with`] — the same "one extension /// point" rationale as `rank_candidates`, one level deeper. Default: /// [`implementation::default_size_params`], `asap-plan`'s built-in formulas /// (unchanged) — a deployment that only needs to reorder candidates, @@ -213,7 +212,7 @@ pub trait CostModel { /// Realize an `AggIntent::Extension { ext_kind, payload }` — a /// deployment-specific intent shape core has no realization opinion - /// for (issue #131). `implementation::implementation_for_with` consults this + /// for (issue #131). `implementation::implementations_for_with` consults this /// for every `Extension` node instead of hardcoding `PassThrough` /// (issue #150). Default: `PassThrough` — preserves today's behavior /// for every deployment that doesn't override this, exactly like diff --git a/crates/asap-aware-mapping/src/implementation.rs b/crates/asap-aware-mapping/src/implementation.rs index 24350ff4..b09fd524 100644 --- a/crates/asap-aware-mapping/src/implementation.rs +++ b/crates/asap-aware-mapping/src/implementation.rs @@ -1,4 +1,4 @@ -//! [`Implementation`] — how one [`AggIntent`] gets realized at post-ASAP +//! [`Implementation`] — how one [`AggIntent`] can be realized at post-ASAP //! binding time (issue #98): by an approximate summary (sketch, sample, //! wavelet, statistical model, …), by an exact mergeable accumulator, or by //! an ordinary exact operator (pass-through). This is a post-ASAP concern — @@ -6,8 +6,13 @@ //! realization — and it's a per-node decision, made once per `AggIntent`, //! not a plan-wide one. //! -//! [`implementation_for`]/[`implementation_for_with`] are where that -//! decision actually gets made. Three inputs feed it: +//! [`implementations_for_with`] is where every valid realization gets +//! enumerated, exhaustive and ranked (most-preferred first) — this crate has +//! no separate function that computes just "the one" `Implementation` +//! independently of that list. [`crate::bind::implement_tree_with`] and +//! [`crate::replacement::SketchFamilyStrategy`] both consume this same +//! list; they differ only in how much of it they keep (see `bind.rs`'s and +//! `replacement.rs`'s module docs). Three inputs feed it: //! //! - the [`AccuracyTarget`] threaded onto the approximate-capable intents //! (`Quantile` / `Cardinality` / `Count` / `TopK`) — `Exact` forbids a @@ -21,12 +26,11 @@ //! pre-aggregated bucket counts can't feed a quantile sketch — while the //! generic `Quantile` path (native histograms / raw samples) is. //! -//! [`implementation_for`] is a single exhaustive match over the intent vocabulary, so a -//! new `AggIntent` variant fails to compile until it is given an explicit -//! realization — there is no silent fall-through. The [`bind`](crate::bind) -//! pass fires it per node over nested trees. +//! [`implementations_for_with`] is a single exhaustive match over the intent +//! vocabulary, so a new `AggIntent` variant fails to compile until it is +//! given an explicit realization — there is no silent fall-through. //! -//! Today's core dispatch only ever picks [`Implementation::ExactAggregate`], +//! Today's core dispatch only ever produces [`Implementation::ExactAggregate`], //! [`Implementation::Sketch`], or [`Implementation::PassThrough`] — no //! `AggIntent` variant maps to sampling/wavelet/statistical-model yet. //! [`Implementation::Sample`]/[`Wavelet`](Implementation::Wavelet)/ @@ -42,9 +46,9 @@ use asap_types::post_asap::{ use asap_types::pre_asap::agg_intent::{agg_is_mergeable, AggIntent}; use asap_types::types::AccuracyTarget; -use crate::cost_model::{CostModel, DefaultCostModel}; +use crate::cost_model::CostModel; -/// How an [`AggIntent`] is realised at post-ASAP binding time. +/// How an [`AggIntent`] may be realised at post-ASAP binding time. #[derive(Debug, Clone, PartialEq)] pub enum Implementation { /// An exact **mergeable** accumulator (partial state ≡ the value @@ -90,18 +94,18 @@ pub enum Implementation { /// Does an already-**available** [`Implementation`] — e.g. a summary /// instance a downstream deployment already materialized somewhere, found /// via whatever inventory/index that deployment keeps — satisfy a -/// **required** [`Implementation`] (what [`implementation_for`]/ -/// [`implementation_for_with`] computed for some [`AggIntent`])? +/// **required** [`Implementation`] (one of the candidates +/// [`implementations_for_with`] produced for some [`AggIntent`])? /// /// This is the query-optimization-literature "materialized view matching" /// / "answering queries using views" question, narrowed to this crate's /// summary vocabulary: not "can I build this from scratch" (that's what -/// `implementation_for` answers) but "does something that already exists -/// answer this". +/// [`implementations_for_with`] answers) but "does something that already +/// exists answer this". /// /// `asap-plan` deliberately ships no implementation of this trait and no -/// default method body — unlike [`implementation_for`], which decision an -/// available `Implementation` satisfies a required one is not a fact this +/// default method body — unlike [`implementations_for_with`], which decision +/// an available `Implementation` satisfies a required one is not a fact this /// crate can settle on its own. Two real, reasonable answers already /// diverge outside this crate: /// @@ -131,11 +135,10 @@ pub trait Matcher { /// one. `ln(1/0.01) → depth 5`, matching the conventional CMS sizing. pub const DEFAULT_DELTA: f64 = 0.01; -/// The sketch families that can serve an intent, most-preferred first. -/// This is the `AggIntent → SketchKind` map of issue #98; [`implementation_for`] binds -/// the head of the list. The tail entries are the alternatives a future cost -/// model (#6/#33) may pick instead — listed here so the candidate set has one -/// home. +/// The sketch kinds that can serve an intent, most-preferred first. +/// This is the `AggIntent → SketchKind` map of issue #98; +/// [`implementations_for_with`] sizes and ranks every entry via `cost_model`. +/// Listed here so the candidate set has one home. pub fn summary_candidates(intent: &AggIntent) -> &'static [SketchKind] { match intent { AggIntent::Quantile { .. } => &[SketchKind::Kll, SketchKind::DDSketch], @@ -148,26 +151,13 @@ pub fn summary_candidates(intent: &AggIntent) -> &'static [SketchKind] { } } -/// The sketch-vs-exact boundary decision for one intent. -/// -/// Exhaustive over the [`AggIntent`] vocabulary — adding a variant without an -/// explicit realization is a compile error, and the coverage-matrix test pins -/// each variant's category. Ranks candidate summaries via -/// [`DefaultCostModel`] (`asap-plan`'s built-in static preference order, -/// unchanged); use [`implementation_for_with`] to plug in a deployment-specific -/// [`CostModel`] instead. -pub fn implementation_for(intent: &AggIntent) -> Implementation { - implementation_for_with(intent, &DefaultCostModel) -} - /// The [`AccuracyTarget`] threaded onto an approximate-capable intent /// (`Quantile`/`Cardinality`/`Count`/`TopK`), or `None` for every other -/// intent (no sketch candidate applies — [`implementation_for_with`]'s own +/// intent (no sketch candidate applies — [`implementations_for_with`]'s own /// match routes those elsewhere). Exposed so -/// [`crate::replacement::SketchFamilyStrategy`] can resolve the same -/// accuracy target [`implementation_for_with`] would, without re-deriving -/// [`implementation_for_with`]'s whole dispatch just to read one field back -/// out. +/// [`crate::replacement::SketchFamilyStrategy`] and +/// [`implementations_for_with`] resolve the exact same accuracy target, +/// without either re-deriving it from scratch. pub fn accuracy_target(intent: &AggIntent) -> Option<&AccuracyTarget> { match intent { AggIntent::Quantile { accuracy, .. } @@ -178,39 +168,65 @@ pub fn accuracy_target(intent: &AggIntent) -> Option<&AccuracyTarget> { } } -/// Like [`implementation_for`], but ranks candidate summaries via `cost_model` (see -/// [`crate::cost_model`]) instead of the built-in static preference order. -pub fn implementation_for_with(intent: &AggIntent, cost_model: &dyn CostModel) -> Implementation { +/// Every valid [`Implementation`] for `intent`, exhaustive and ranked +/// (most-preferred first via `cost_model`) — the *only* place this crate +/// decides what an `AggIntent` may become. Nothing in this crate computes +/// "the one" `Implementation` independently of this list: +/// [`crate::bind::implement_tree_with`] takes the head for the single +/// executable answer production binding needs; +/// [`crate::replacement::SketchFamilyStrategy`] keeps every entry as a +/// candidate. Both differ only in *how much of this list* they keep — see +/// their own module docs. +/// +/// Exhaustive over the [`AggIntent`] vocabulary — adding a variant without an +/// explicit realization is a compile error, and the coverage-matrix test pins +/// each variant's category. +pub fn implementations_for_with( + intent: &AggIntent, + cost_model: &dyn CostModel, +) -> Vec { match intent { // ── Approximate-capable intents — the AccuracyTarget decides ──────── AggIntent::Quantile { accuracy, .. } | AggIntent::Cardinality { accuracy, .. } | AggIntent::Count { accuracy } | AggIntent::TopK { accuracy, .. } => match accuracy { - AccuracyTarget::Exact => exact_realization(intent), - _ => bind_summary_with(intent, accuracy, cost_model), + AccuracyTarget::Exact => vec![exact_realization(intent)], + _ => sketch_implementations(intent, accuracy, cost_model), }, // ── Exact mergeable accumulators ───────────────────────────────────── - AggIntent::Sum { .. } => exact_accumulator(intent, ExactKind::Sum, ExactParams::Sum), + AggIntent::Sum { .. } => vec![exact_accumulator(intent, ExactKind::Sum, ExactParams::Sum)], AggIntent::Min { .. } | AggIntent::Max { .. } => { - exact_accumulator(intent, ExactKind::MinMax, ExactParams::MinMax) + vec![exact_accumulator( + intent, + ExactKind::MinMax, + ExactParams::MinMax, + )] } - AggIntent::Rate => exact_accumulator(intent, ExactKind::Rate, ExactParams::Rate), + AggIntent::Rate => vec![exact_accumulator( + intent, + ExactKind::Rate, + ExactParams::Rate, + )], AggIntent::Increase => { - exact_accumulator(intent, ExactKind::Increase, ExactParams::Increase) + vec![exact_accumulator( + intent, + ExactKind::Increase, + ExactParams::Increase, + )] } // ── Exact, non-mergeable reducers — richer partial state than a // single value (see `agg_is_mergeable`), so no accumulator form. AggIntent::Avg { .. } | AggIntent::StdDev { .. } | AggIntent::Variance { .. } => { - Implementation::PassThrough + vec![Implementation::PassThrough] } // ── Classic-bucket histogram_quantile (#79): exact `le`-bucket // interpolation over pre-aggregated counts — NOT re-sketchable. // (The native/raw form lowers to the generic `Quantile` above.) - AggIntent::HistogramQuantile { .. } => Implementation::PassThrough, + AggIntent::HistogramQuantile { .. } => vec![Implementation::PassThrough], // ── Per-series transforms and reductions with no sketch realization: // counter-derivatives (#44), math (#45), time/calendar (#46), @@ -240,22 +256,24 @@ pub fn implementation_for_with(intent: &AggIntent, cost_model: &dyn CostModel) - | AggIntent::TsOfMinOverTime | AggIntent::TsOfMaxOverTime | AggIntent::TsOfFirstOverTime - | AggIntent::TsOfLastOverTime => Implementation::PassThrough, + | AggIntent::TsOfLastOverTime => vec![Implementation::PassThrough], // ── Group / count_values (#49): exact per `agg_is_exact`, but their // output is structural (constant-1 / a synthesized label column), // not a value a summary accumulator carries. - AggIntent::Group | AggIntent::CountValues { .. } => Implementation::PassThrough, + AggIntent::Group | AggIntent::CountValues { .. } => vec![Implementation::PassThrough], // ── Extension (deployment-model-specific, issue #131) — core has no // realization opinion for a shape it doesn't know, so it defers // entirely to the `CostModel` (issue #150): `realize_extension` // defaults to `PassThrough`, preserving today's behavior for - // every deployment that doesn't override it. This is also the - // only path that can currently produce `Implementation::Sample`/ + // every deployment that doesn't override it. Core has no way to + // enumerate alternatives for an opaque deployment-defined shape, + // so this is always exactly one candidate. This is also the only + // path that can currently produce `Implementation::Sample`/ // `Wavelet`/`StatModel` — see the module docs. AggIntent::Extension { ext_kind, payload } => { - cost_model.realize_extension(ext_kind, payload) + vec![cost_model.realize_extension(ext_kind, payload)] } } } @@ -282,15 +300,13 @@ fn exact_accumulator(intent: &AggIntent, kind: ExactKind, params: ExactParams) - } /// Resolve an [`AccuracyTarget`] into the `(eps, delta)` budget -/// [`CostModel::size_params`] needs. Shared by [`bind_summary_with`] (the -/// binding path's own choice) and -/// [`crate::replacement::SketchFamilyStrategy`] (which sizes every -/// candidate, not just the one the binding path would have picked) — one -/// place this resolution happens, so the two paths can't drift apart on it. +/// [`CostModel::size_params`] needs. Shared by [`sketch_implementations`] and +/// [`crate::replacement`]'s own sizing — one place this resolution happens, +/// so nothing can drift apart on it. /// -/// `Exact` is unreachable via [`implementation_for`] (which routes `Exact` -/// to [`exact_realization`] instead); degrades to the tightest parameters -/// for a caller that resolves it directly anyway. +/// `Exact` is unreachable via [`implementations_for_with`] (which routes +/// `Exact` to [`exact_realization`] instead); degrades to the tightest +/// parameters for a caller that resolves it directly anyway. pub fn accuracy_budget(accuracy: &AccuracyTarget) -> (f64, f64) { match accuracy { AccuracyTarget::Exact => (f64::MIN_POSITIVE, DEFAULT_DELTA), @@ -299,23 +315,23 @@ pub fn accuracy_budget(accuracy: &AccuracyTarget) -> (f64, f64) { } } -/// Bind the preferred candidate sketch, with parameters sized to the -/// target, ranking [`summary_candidates`] via `cost_model` (see -/// [`crate::cost_model`]) instead of taking the static-order head -/// unconditionally. -fn bind_summary_with( +/// Every candidate sketch [`Implementation`] for an approximate-capable +/// intent, sized to `accuracy` and ranked via `cost_model.rank_candidates` +/// (most-preferred first) — [`implementations_for_with`]'s Sketch branch. +fn sketch_implementations( intent: &AggIntent, accuracy: &AccuracyTarget, cost_model: &dyn CostModel, -) -> Implementation { +) -> Vec { let (eps, delta) = accuracy_budget(accuracy); let ranked = cost_model.rank_candidates(intent, summary_candidates(intent)); - let kind = ranked + ranked .into_iter() - .next() - .expect("approximate intent has at least one candidate summary"); - let params = cost_model.size_params(kind.clone(), intent, eps, delta); - Implementation::Sketch { kind, params } + .map(|kind| { + let params = cost_model.size_params(kind.clone(), intent, eps, delta); + Implementation::Sketch { kind, params } + }) + .collect() } /// `asap-plan`'s built-in `SketchParams` sizing, keyed off the resolved @@ -447,7 +463,7 @@ pub struct ExpectedCaseSizing { /// /// [`default_size_params`]'s own behavior is completely unchanged by this /// function's existence — this is a separate, additive entry point, never -/// called from [`default_size_params`] or [`implementation_for`]. +/// called from [`default_size_params`] or [`implementations_for_with`]. pub fn posterior_aware_size_params( kind: SketchKind, intent: &AggIntent, @@ -554,6 +570,7 @@ fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { #[cfg(test)] mod tests { use super::*; + use crate::cost_model::DefaultCostModel; use asap_types::pre_asap::agg_intent::{ agg_is_exact, default_cardinality, default_quantile, MathFunc, TimeFunc, }; @@ -562,6 +579,16 @@ mod tests { AccuracyTarget::Epsilon(e) } + /// The most-preferred `Implementation` — `implementations_for_with(intent, + /// &DefaultCostModel)`'s head — for tests that only care about the + /// default pick, not the full candidate list. + fn preferred(intent: &AggIntent) -> Implementation { + implementations_for_with(intent, &DefaultCostModel) + .into_iter() + .next() + .expect("every intent has at least one Implementation") + } + /// Shorthand for asserting the realization *category*. #[derive(Debug, PartialEq)] enum Cat { @@ -571,7 +598,7 @@ mod tests { } fn cat(intent: &AggIntent) -> Cat { - match implementation_for(intent) { + match preferred(intent) { Implementation::ExactAggregate { kind, .. } => Cat::Acc(kind), Implementation::Sketch { kind, .. } => Cat::Sketch(kind), Implementation::PassThrough => Cat::Pass, @@ -583,8 +610,9 @@ mod tests { /// The `AggIntent → SummaryKind` coverage matrix (issue #98): every intent /// variant maps to a sketch, an exact accumulator, or an explicit - /// pass-through. `implementation_for`'s match is exhaustive, so a new variant cannot - /// compile without a decision; this matrix pins what each decision *is*. + /// pass-through. `implementations_for_with`'s match is exhaustive, so a + /// new variant cannot compile without a decision; this matrix pins what + /// each decision *is* (its preferred/first candidate). #[test] fn agg_intent_to_summary_kind_coverage_matrix() { use AggIntent as A; @@ -732,11 +760,11 @@ mod tests { q: 0.99, accuracy: AccuracyTarget::Exact, }; - assert_eq!(implementation_for(&exact), Implementation::PassThrough); + assert_eq!(preferred(&exact), Implementation::PassThrough); let approx = default_quantile(0.99); // ε = 0.01 assert_eq!( - implementation_for(&approx), + preferred(&approx), Implementation::Sketch { kind: SketchKind::Kll, params: SketchParams::Kll { k: 200 }, // design.md worked example @@ -749,7 +777,7 @@ mod tests { accuracy: eps(0.05), }; assert_eq!( - implementation_for(&looser), + preferred(&looser), Implementation::Sketch { kind: SketchKind::Kll, params: SketchParams::Kll { k: 40 }, // ⌈2/0.05⌉ @@ -762,7 +790,7 @@ mod tests { // `default_cardinality` encodes HLL's standard error at p=14; the // sizing must invert it back exactly. assert_eq!( - implementation_for(&default_cardinality()), + preferred(&default_cardinality()), Implementation::Sketch { kind: SketchKind::Hll, params: SketchParams::Hll { precision: 14 }, @@ -779,7 +807,7 @@ mod tests { }, }; assert_eq!( - implementation_for(&intent), + preferred(&intent), Implementation::Sketch { kind: SketchKind::Cms, params: SketchParams::Cms { @@ -793,7 +821,7 @@ mod tests { accuracy: eps(0.001), }; assert_eq!( - implementation_for(&intent), + preferred(&intent), Implementation::Sketch { kind: SketchKind::Cms, params: SketchParams::Cms { @@ -810,7 +838,7 @@ mod tests { k: 25, accuracy: eps(0.01), }; - match implementation_for(&intent) { + match preferred(&intent) { Implementation::Sketch { kind: SketchKind::CmsWithHeap, params: @@ -854,6 +882,22 @@ mod tests { assert!(summary_candidates(&AggIntent::Rate).is_empty()); } + #[test] + fn implementations_for_with_enumerates_every_candidate_ranked() { + // Quantile's candidate list is [Kll, DDSketch] — implementations_for_with + // must return both, ranked with the DefaultCostModel's preferred + // (Kll) first. + let kinds: Vec = + implementations_for_with(&default_quantile(0.99), &DefaultCostModel) + .into_iter() + .map(|implementation| match implementation { + Implementation::Sketch { kind, .. } => kind, + other => panic!("expected Sketch, got {other:?}"), + }) + .collect(); + assert_eq!(kinds, vec![SketchKind::Kll, SketchKind::DDSketch]); + } + #[test] fn degenerate_epsilon_saturates_to_tightest_params() { let intent = AggIntent::Quantile { @@ -862,7 +906,7 @@ mod tests { accuracy: eps(0.0), }; assert_eq!( - implementation_for(&intent), + preferred(&intent), Implementation::Sketch { kind: SketchKind::Kll, params: SketchParams::Kll { k: 65_535 }, diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 898df458..db4e093e 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -32,32 +32,34 @@ //! //! Three real occupants and one stub: //! +//! - [`implementation`] — [`implementation::implementations_for_with`] is +//! where every valid realization of an `AggIntent` gets decided: exhaustive +//! and ranked (most-preferred first via a `CostModel`), sized to the +//! `AccuracyTarget` (issue #98). Nothing else in this crate computes an +//! `Implementation` independently of this list. //! - [`replacement`] — the `TargetSubDAG`/`ReplacementSubDAG`/ //! `ReplacementStrategy` vocabulary `docs/design_docs/asap_aware_mapping.md` stubs out //! under "Key concepts (not yet implemented)", implemented for real (issue -//! #251, part of #33): a small extension-point trait reporting *every* -//! semantically valid replacement for a target sub-DAG, not just the one -//! [`implementation::implementation_for_with`]/[`bind::implement_tree_with`] -//! commit to. Two real strategies wrap those exact decision points instead -//! of re-deciding anything: -//! [`replacement::SketchFamilyStrategy`] (every candidate summary family -//! for a bindable `Aggregate`) and [`replacement::SharedSubtreeStrategy`] -//! (the build-independently-vs-build-once-and-share candidate pair for a -//! CSE-detected shared subtree). No search/ranking logic lives here — see -//! that module's docs for what's deliberately left to a future -//! Cascades/Volcano-style search engine. -//! - [`implementation`] — the per-intent sketch-vs-exact (accuracy) decision: -//! `AggIntent → Implementation` (a summary family's own `(Kind, Params)`, -//! or an exact accumulator) sized to the `AccuracyTarget` (issue #98). -//! [`implementation::implementation_for`] is the per-node decision; -//! [`bind::implement_tree`] drives it over a whole tree, and -//! [`bind::implement_workload`] drives it over a whole workload's roots — -//! memoized on `Rc` identity so two roots that -//! `asap_types::pre_asap::cse::share_common_subtrees` already collapsed -//! onto one shared subtree bind to one shared `SummaryNode` too (issue -//! #212, #222, #223) — see the terminology section below for why this -//! module (and `implement_tree`) are named around "implementation" rather -//! than "bind". +//! #251, part of #33): [`replacement::SketchFamilyStrategy`] wraps +//! [`implementation::implementations_for_with`]'s list directly, keeping +//! *every* candidate as its own bound [`SummaryNode`](asap_types::post_asap::SummaryNode) +//! instead of just one — [`replacement::SharedSubtreeStrategy`] does the +//! same for the build-independently-vs-build-once-and-share choice at a +//! CSE-detected shared subtree. No search/ranking-across-a-whole-plan logic +//! lives here — see that module's docs for what's deliberately left to a +//! future Cascades/Volcano-style search engine. +//! - [`bind`] — [`bind::implement_tree`]/[`bind::implement_tree_with`] walk a +//! whole tree, keeping only the first (`cost_model`-preferred) candidate +//! [`replacement::SketchFamilyStrategy`] enumerates at each node — a thin +//! selector over `replacement`, not a second decision path (see +//! `bind.rs`'s and `replacement.rs`'s own module docs for the full +//! relationship). [`bind::implement_workload`] drives the same selection +//! over a whole workload's roots, memoized on `Rc` identity so two roots +//! that `asap_types::pre_asap::cse::share_common_subtrees` already +//! collapsed onto one shared subtree bind to one shared `SummaryNode` too +//! (issue #212, #222, #223) — see the terminology section below for why +//! this module (and `implement_tree`) are named around "implementation" +//! rather than "bind". //! - [`cost_model`] — the [`CostModel`](cost_model::CostModel) trait every //! deployment's cost-based sketch selection plugs into (issues #6, #33). //! `asap-plan` itself only ships [`DefaultCostModel`](cost_model::DefaultCostModel), @@ -82,8 +84,8 @@ //! |---|---|---|---| //! | **Parse** | parse | text (PromQL/SQL) → AST | `asap-frontend-promql` / `asap-frontend-sql` | //! | **Bind #1** | name resolution | `ColumnRef` (a name) → `ColumnId` (a concrete schema column) — the classic RDBMS "Parse → **Bind** → Optimize" pipeline sense (e.g. SQL Server's query-processor terminology) | [`asap_types::pre_asap::binder::Binder`](https://docs.rs/asap-types) | -//! | **Implementation** — [`implementation::implementation_for`] | pre-ASAP → post-ASAP, *one node* | choosing a concrete physical realization (a sketch family, an exact accumulator, or pass-through) for one [`AggIntent`](asap_types::pre_asap::agg_intent::AggIntent) | [`implementation`] | -//! | **`implement_tree`** — [`bind::implement_tree`] | pre-ASAP → post-ASAP, *whole tree* | walk a whole `QueryExpr` tree, calling [`implementation::implementation_for`] per node, and emit the complete post-ASAP [`SummaryExpr`](asap_types::post_asap::SummaryExpr)/`SummaryNode` DAG — named after "implementation" too rather than reusing "bind" a second time | [`bind`] | +//! | **Implementation** — [`implementation::implementations_for_with`] | pre-ASAP → post-ASAP, *one node* | enumerating every concrete physical realization (a sketch family, an exact accumulator, or pass-through) for one [`AggIntent`](asap_types::pre_asap::agg_intent::AggIntent) | [`implementation`] | +//! | **`implement_tree`** — [`bind::implement_tree`] | pre-ASAP → post-ASAP, *whole tree* | walk a whole `QueryExpr` tree, keeping the first candidate [`implementation::implementations_for_with`] produces per node, and emit the complete post-ASAP [`SummaryExpr`](asap_types::post_asap::SummaryExpr)/`SummaryNode` DAG — named after "implementation" too rather than reusing "bind" a second time | [`bind`] | //! | **Bind #2** (downstream, not in this crate) | post-ASAP → deployment placement | a *deployment's* own physical binder, additionally deciding **placement** (edge vs. backend, wire format, …) — a genuinely different, deployment-specific decision this crate doesn't model at all | e.g. `control_plane::sketch_algebra::rules::bind_*` (as of this writing; expected to fold into that deployment's cost-model layer rather than stay a separate "bind" concept) | //! //! A related question (tracked alongside issues #6/#33): whether this @@ -113,9 +115,7 @@ pub use bind::{ ImplementError, }; pub use cost_model::{CostModel, DefaultCostModel}; -pub use implementation::{ - implementation_for, implementation_for_with, summary_candidates, Implementation, Matcher, -}; +pub use implementation::{implementations_for_with, summary_candidates, Implementation, Matcher}; pub use replacement::{ Replacement, ReplacementStrategy, ReplacementSubDAG, SharedSubtreeStrategy, SketchFamilyStrategy, TargetSubDAG, diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index 5ef97503..dbc8fdb2 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -3,23 +3,12 @@ //! under "Key concepts (not yet implemented)", implemented for real (issue //! #251, part of #33). //! -//! ## Why this exists alongside [`implementation`] and [`bind`], not instead of them +//! ## This is where every `Implementation` gets decided — `bind` is a selector over it //! -//! [`implementation::implementation_for_with`] and -//! [`bind::implement_tree_with`] each commit to exactly **one** answer per -//! decision point — a single [`Implementation`], a single bound -//! [`SummaryNode`] — ranked via a [`CostModel`]. That is exactly right for -//! *binding* a query (something has to actually run), but it means the -//! alternatives a cost model didn't pick are thrown away the moment they're -//! computed. A search/optimization engine (Cascades/Volcano-style, tracked -//! separately — see `docs/design_docs/asap_aware_mapping.md`'s "Pseudocode for -//! Replacement Plan Searching") needs the *opposite* shape: every -//! semantically valid alternative for a sub-DAG, so a later cost-based search -//! can explore and compare them instead of being stuck with whatever -//! [`implementation`]/[`bind`] already locked in. -//! -//! This module is that alternative shape, built by **wrapping** the existing -//! decision points rather than re-deciding anything: +//! [`implementation::implementations_for_with`] is the one place this crate +//! enumerates every valid [`Implementation`] for an [`AggIntent`], exhaustive +//! and ranked (most-preferred first via a [`CostModel`]). This module wraps +//! that list into the exhaustive-candidate shape a future search needs: //! //! - [`TargetSubDAG`] — a reference to a pre-ASAP [`QueryExpr`] node that is a //! candidate for replacement, plus how many places in the workload already @@ -28,11 +17,10 @@ //! doesn't carry. //! - [`ReplacementSubDAG`] — one candidate replacement for a `TargetSubDAG`: //! either a fully bound [`SummaryNode`] (the same output -//! [`bind::implement_tree_with`] produces, for one particular candidate -//! instead of the one a `CostModel` ranked first) or a pre-ASAP -//! [`QueryExpr`] rewrite (still logical, structurally different from the -//! target but semantically equivalent) — see [`Replacement`] — plus a -//! human-readable `rationale`. +//! [`bind::implement_tree_with`] itself keeps, for one particular candidate +//! instead of the one it kept) or a pre-ASAP [`QueryExpr`] rewrite (still +//! logical, structurally different from the target but semantically +//! equivalent) — see [`Replacement`] — plus a human-readable `rationale`. //! - [`ReplacementStrategy`] — `matches` + `replacements`, the same //! extension-point shape [`CostModel`] and [`Matcher`](crate::implementation::Matcher) //! already use in this crate (and the same shape issue #33's @@ -41,25 +29,35 @@ //! `impl ReplacementStrategy`, not a restructuring of this trait or of any //! existing strategy. `replacements` is **exhaustive, not ranked, not //! filtered** — reporting "every valid candidate" is core's job; picking -//! the best one is a [`CostModel`]'s job, deliberately out of scope here -//! (see "Non-goals" below). +//! the best one is a [`CostModel`]'s job (see "Selection" below). //! -//! ## The two strategies, and why these two +//! ## Selection: `bind::implement_tree_with` keeps the head, nothing else //! -//! Both wrap an existing, already-correct decision procedure into the -//! exhaustive-candidate shape — neither re-derives anything: +//! [`bind::implement_tree_with`] does not compute an `Implementation` +//! independently — it builds a `TargetSubDAG` for the node being bound, +//! calls [`SketchFamilyStrategy::replacements`], and keeps the first +//! (`cost_model`-preferred) candidate; everything else it discards. That's +//! the *only* difference between "the binding path" and "the +//! replacement-strategy path": how much of this module's own output each one +//! keeps. Both bottom out in the exact same low-level primitive, +//! [`bind::bind_with_implementation`] — given an *already-decided* +//! `Implementation`, turn it into a `SummaryNode` — which this module calls +//! once per candidate it enumerates, and `bind::implement_tree_with` calls +//! once for the one candidate it kept. See `bind.rs`'s own module docs for +//! the reverse half of this relationship. //! -//! - [`SketchFamilyStrategy`] wraps [`implementation::implementation_for_with`] / -//! [`implementation::summary_candidates`]'s exhaustive match over the -//! [`AggIntent`] vocabulary. For the same bindable-`Aggregate` shape -//! [`bind::implement_tree_with`] itself requires (single intent, no -//! `HAVING`), it returns every candidate [`implementation::implementation_for_with`] -//! could have committed to instead of the one it did: every `SketchKind` -//! [`implementation::summary_candidates`] lists for the intent when the boundary -//! decision is approximate, or the single exact-accumulator / -//! pass-through outcome when that's the *only* realization -//! [`implementation::implementation_for_with`]'s dispatch produces for this -//! intent (there is nothing else to enumerate in that case). +//! This means an ordinary bind now sizes and fully constructs *every* +//! sketch candidate at every sketch-capable node (not just the one it keeps) +//! — a deliberate tradeoff, made so there is exactly one place in this crate +//! that decides what an `AggIntent` may become, at the cost of extra work +//! per bind proportional to each node's own candidate count. +//! +//! ## The two strategies, and why these two +//! +//! - [`SketchFamilyStrategy`] wraps [`implementation::implementations_for_with`]'s +//! exhaustive, ranked list directly: for the same bindable-`Aggregate` +//! shape [`bind::implement_tree_with`] itself requires (single intent, no +//! `HAVING`), every entry becomes its own bound candidate. //! - [`SharedSubtreeStrategy`] wraps //! `asap_types::pre_asap::cse::share_common_subtrees`'s sharing decision. //! Wherever a [`TargetSubDAG`] already has two or more consumers (i.e. @@ -79,14 +77,14 @@ //! //! ## Non-goals (tracked separately, not attempted here) //! -//! - **No search/selection logic.** `docs/design_docs/asap_aware_mapping.md`'s +//! - **No search/selection-across-a-whole-plan logic.** `docs/design_docs/asap_aware_mapping.md`'s //! replacement-plan-searching pseudocode — trying every `ReplacementStrategy` //! against every candidate plan, deduplicating, iterating to a fixpoint, //! then ranking by a `CostModel` — is a Cascades/Volcano-style search -//! engine, tracked as a separate follow-up. This module only needs -//! `replacements()` to be exhaustive and correct for one `TargetSubDAG` at -//! a time, mirroring [`implementation::implementation_for_with`]'s own -//! "exhaustive match, no silent fallthrough" discipline. +//! engine, tracked as a separate follow-up. `bind::implement_tree_with`'s +//! "keep the head" selection is a single-node stand-in for that, not the +//! real thing: it never compares whole candidate *plans*, only one node's +//! own candidates against each other via `cost_model.rank_candidates`. //! - **No workload-wide `TargetSubDAG` discovery pass.** Finding every //! candidate node in a whole workload (walking every root, deduplicating by //! `Rc` identity, computing real consumer counts) is exactly what PR #247's @@ -96,29 +94,20 @@ //! This module's own tests build `TargetSubDAG`s directly, the same //! hand-rolled-fixture style `bind.rs`/`implementation.rs`/`cost_model.rs`'s own //! tests already use. -//! - **[`implementation`]/[`bind`]'s existing single-pick *outward-facing* -//! behavior is unchanged.** [`implementation::implementation_for`] and -//! [`bind::implement_tree_with`] still return exactly the same -//! `Implementation`/`SummaryNode` they always did, for the same inputs. -//! What changed is internal: [`bind::implement_tree_with`] and this -//! module's [`SketchFamilyStrategy`] now both bottom out in the same -//! [`bind::bind_with_implementation`] primitive — the binding path -//! resolves one ranked `Implementation` and hands it there; this module -//! sizes every candidate `Implementation` itself and hands each one there -//! in turn — rather than this module working around the binding path via -//! a `CostModel`-forcing adapter, as it used to. +//! - **[`implementation::implementations_for_with`]'s own outward-facing +//! behavior is unchanged.** Same inputs still produce the same exhaustive, +//! ranked list. What changed is who calls it and how much of the result +//! they keep — see "Selection" above. use std::rc::Rc; -use asap_types::post_asap::{SketchKind, SummaryNode}; +use asap_types::post_asap::SummaryNode; use asap_types::pre_asap::agg_intent::AggIntent; use asap_types::pre_asap::query_expr::QueryExpr; use crate::bind::{bind_with_implementation, bindable_intent}; use crate::cost_model::{CostModel, DefaultCostModel}; -use crate::implementation::{ - accuracy_budget, accuracy_target, implementation_for_with, summary_candidates, Implementation, -}; +use crate::implementation::{implementations_for_with, Implementation}; /// A pre-ASAP sub-DAG a [`ReplacementStrategy`] knows how to replace. /// @@ -168,9 +157,9 @@ impl<'a> TargetSubDAG<'a> { /// What a [`ReplacementSubDAG`] actually substitutes a [`TargetSubDAG`] with. /// /// Generalizes [`bind::implement_tree_with`]'s and -/// [`implementation::implementation_for_with`]'s two possible *kinds* of answer — -/// a post-ASAP binding decision, or a still-pre-ASAP structural alternative — -/// from "the one they commit to" into "one candidate among several". +/// [`implementation::implementations_for_with`]'s two possible *kinds* of +/// answer — a post-ASAP binding decision, or a still-pre-ASAP structural +/// alternative — from "the one kept" into "one candidate among several". #[derive(Debug, Clone)] pub enum Replacement { /// A fully bound post-ASAP summary decision — the same @@ -226,12 +215,11 @@ pub trait ReplacementStrategy { /// every caller (same pattern `applicability::SketchApplicabilityRule` uses). static DEFAULT_COST_MODEL: DefaultCostModel = DefaultCostModel; -/// Wraps [`implementation::implementation_for_with`]/[`implementation::summary_candidates`]'s -/// exhaustive match over the [`AggIntent`] vocabulary: for a bindable -/// `Aggregate`, every candidate summary realization instead of just the one -/// [`implementation::implementation_for_with`] commits to. +/// Wraps [`implementation::implementations_for_with`]'s exhaustive, ranked +/// list directly: for a bindable `Aggregate`, every valid candidate summary +/// realization as its own [`ReplacementSubDAG`]. /// -/// Ranks (only to *order the enumeration*, never to drop a candidate) via a +/// Ranked (only to *order the enumeration*, never to drop a candidate) via a /// [`CostModel`] — [`DefaultCostModel`] unless constructed with /// [`SketchFamilyStrategy::new`] — so a deployment-specific cost model's /// other hooks (`size_params`, `realize_extension`, `readout_extension`) are @@ -255,8 +243,8 @@ impl SketchFamilyStrategy<'static> { impl<'a> SketchFamilyStrategy<'a> { /// A strategy that ranks/binds via `cost_model` instead of the built-in /// static preference order — the same customization point - /// [`bind::implement_tree_with`] and [`implementation::implementation_for_with`] - /// already offer. + /// [`bind::implement_tree_with`] and + /// [`implementation::implementations_for_with`] already offer. pub fn new(cost_model: &'a dyn CostModel) -> Self { Self { cost_model } } @@ -271,139 +259,69 @@ impl ReplacementStrategy for SketchFamilyStrategy<'_> { let Some(intent) = bindable_intent(target.root) else { return Vec::new(); }; - // Exhaustive over `Implementation`'s variants — the same "no silent - // fallthrough" discipline `implementation::implementation_for_with`'s own - // match uses. Only `Sketch` has more than one candidate to enumerate - // (`summary_candidates`); every other variant is the *only* - // realization `implementation::implementation_for_with`'s dispatch - // produces for this intent, so there's nothing else to offer — bind - // the one `implementation` this branch already computed rather than - // re-deriving it a second time. - let implementation = implementation_for_with(intent, self.cost_model); - match &implementation { - Implementation::Sketch { .. } => summary_candidates(intent) - .iter() - .filter_map(|kind| { - sketch_candidate(target.root, intent, kind.clone(), self.cost_model) + // `implementations_for_with` is already exhaustive and ranked — no + // separate dispatch needed here. Only `Sketch` has more than one + // candidate in practice (every other variant's own dispatch produces + // exactly one `Implementation`), but this loop doesn't need to know + // that; it just binds whatever the list contains. + implementations_for_with(intent, self.cost_model) + .into_iter() + .filter_map(|implementation| { + let rationale = describe_implementation(intent, &implementation); + let node = + bind_with_implementation(target.root, implementation, self.cost_model).ok()?; + Some(ReplacementSubDAG { + replacement: Replacement::Summary(node), + rationale, }) - .collect(), - Implementation::ExactAggregate { kind, .. } => single_candidate( - target.root, - implementation.clone(), - self.cost_model, - format!( - "{} realizes as an exact {kind:?} accumulator — the only realization \ - implementation::implementation_for produces for this intent (no approximate \ - candidate applies)", - describe_intent(intent) - ), - ), - Implementation::PassThrough => single_candidate( - target.root, - implementation.clone(), - self.cost_model, - format!( - "{} has no summary realization and stays a logical pass-through — the \ - only realization implementation::implementation_for produces for this intent", - describe_intent(intent) - ), - ), - Implementation::Sample { kind, .. } => single_candidate( - target.root, - implementation.clone(), - self.cost_model, - format!( - "{} realizes as a {kind:?} sample — the only realization the plugged-in \ - CostModel produced for this intent", - describe_intent(intent) - ), - ), - Implementation::Wavelet { kind, .. } => single_candidate( - target.root, - implementation.clone(), - self.cost_model, - format!( - "{} realizes as a {kind:?} wavelet transform — the only realization the \ - plugged-in CostModel produced for this intent", - describe_intent(intent) - ), - ), - Implementation::StatModel { kind, .. } => single_candidate( - target.root, - implementation.clone(), - self.cost_model, - format!( - "{} realizes as a {kind:?} statistical model — the only realization the \ - plugged-in CostModel produced for this intent", - describe_intent(intent) - ), - ), - } + }) + .collect() } } -/// Bind `root` to the already-computed `implementation` (no candidate to -/// steer towards) — the "only option" path for any [`Implementation`] -/// category besides `Sketch`. Schema derivation cannot fail for a target -/// that was already a legitimate part of the workload's tree, but -/// [`bind_with_implementation`]'s signature is fallible (`ImplementError`), -/// so a failure here — never expected in practice — degrades to "no -/// candidate" rather than a panic, same conservatism as the rest of this -/// strategy. -fn single_candidate( - root: &QueryExpr, - implementation: Implementation, - cost_model: &dyn CostModel, - rationale: String, -) -> Vec { - bind_with_implementation(root, implementation, cost_model) - .ok() - .map(|node| ReplacementSubDAG { - replacement: Replacement::Summary(node), - rationale, - }) - .into_iter() - .collect() -} - -/// Bind `root` to `kind` — one entry of -/// [`summary_candidates(intent)`](summary_candidates) — sized the same way -/// [`implementation::bind_summary_with`] would (via -/// [`implementation::accuracy_budget`] and `cost_model.size_params`), then -/// handed to [`bind_with_implementation`] — the same primitive -/// [`bind::implement_tree_with`] itself bottoms out in for whichever -/// candidate `cost_model` ranks first. This reuses that whole decision -/// procedure (schema derivation, column resolution, readout construction) -/// unchanged; it only supplies a different top-level `Implementation` than -/// the ranked-first one. -fn sketch_candidate( - root: &QueryExpr, - intent: &AggIntent, - kind: SketchKind, - cost_model: &dyn CostModel, -) -> Option { - let accuracy = accuracy_target(intent)?; - let (eps, delta) = accuracy_budget(accuracy); - let params = cost_model.size_params(kind.clone(), intent, eps, delta); - let implementation = Implementation::Sketch { - kind: kind.clone(), - params, - }; - let node = bind_with_implementation(root, implementation, cost_model).ok()?; - Some(ReplacementSubDAG { - replacement: Replacement::Summary(node), - rationale: format!( +/// A human-readable rationale for one candidate `Implementation`, for +/// [`ReplacementSubDAG::rationale`] text. +fn describe_implementation(intent: &AggIntent, implementation: &Implementation) -> String { + match implementation { + Implementation::Sketch { kind, .. } => format!( "{} realizes as a {kind:?} sketch — one of implementation::summary_candidates' \ - alternatives for this intent (asap_aware_mapping::implementation::implementation_for)", + alternatives for this intent \ + (asap_aware_mapping::implementation::implementations_for_with)", + describe_intent(intent) + ), + Implementation::ExactAggregate { kind, .. } => format!( + "{} realizes as an exact {kind:?} accumulator — the only realization \ + implementation::implementations_for_with produces for this intent (no approximate \ + candidate applies)", + describe_intent(intent) + ), + Implementation::PassThrough => format!( + "{} has no summary realization and stays a logical pass-through — the \ + only realization implementation::implementations_for_with produces for this intent", + describe_intent(intent) + ), + Implementation::Sample { kind, .. } => format!( + "{} realizes as a {kind:?} sample — the only realization the plugged-in \ + CostModel produced for this intent", + describe_intent(intent) + ), + Implementation::Wavelet { kind, .. } => format!( + "{} realizes as a {kind:?} wavelet transform — the only realization the \ + plugged-in CostModel produced for this intent", describe_intent(intent) ), - }) + Implementation::StatModel { kind, .. } => format!( + "{} realizes as a {kind:?} statistical model — the only realization the \ + plugged-in CostModel produced for this intent", + describe_intent(intent) + ), + } } /// A short human-readable label for an `AggIntent`, for /// [`ReplacementSubDAG::rationale`] text. Not exhaustive by design (unlike /// this crate's other `AggIntent` matches, e.g. -/// [`implementation::implementation_for_with`]'s) — this is prose for a +/// [`implementation::implementations_for_with`]'s) — this is prose for a /// rationale string, not a decision, so an unlisted variant just falls back /// to its `Debug` tag rather than forcing every future intent to be named /// here too (same rationale, and same shape, as @@ -479,6 +397,7 @@ impl ReplacementStrategy for SharedSubtreeStrategy { #[cfg(test)] mod tests { use super::*; + use asap_types::post_asap::SketchKind; use asap_types::pre_asap::agg_intent::{default_cardinality, default_quantile}; use asap_types::pre_asap::query_expr::{Reduction, Source}; use asap_types::pre_asap::schema::{Column, DataType, Schema}; @@ -558,7 +477,7 @@ mod tests { fn approximate_quantile_enumerates_every_summary_candidate() { // Quantile's candidate list is [Kll, DDSketch] (implementation::summary_candidates) — // every entry must come back as its own bound SummaryNode candidate, - // not just Kll (the CostModel-ranked head implementation_for_with commits to). + // not just Kll (the CostModel-ranked head implementations_for_with commits to). let q = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); let target = TargetSubDAG::new(&q); let replacements = SketchFamilyStrategy::default_cost_model().replacements(&target); @@ -604,8 +523,8 @@ mod tests { #[test] fn exact_accuracy_target_yields_exactly_one_pass_through_candidate() { - // Exact quantile has no sketch candidate at all — implementation_for - // commits to PassThrough, the only option, so exactly one candidate. + // Exact quantile has no sketch candidate at all — implementations_for_with + // produces PassThrough, the only option, so exactly one candidate. let intent = AggIntent::Quantile { col: None, q: 0.99, @@ -646,7 +565,7 @@ mod tests { /// A custom `CostModel` doesn't change *which* candidates are enumerated /// (still every `summary_candidates` entry) — only which one - /// `implementation_for_with` itself would have picked, and how each + /// `implementations_for_with` itself would prefer first, and how each /// candidate's own params are sized. struct PreferDDSketch; impl CostModel for PreferDDSketch { @@ -682,19 +601,16 @@ mod tests { assert_eq!(kinds.len(), 2); } - /// Enumerating a candidate for the *target* node must only steer that - /// node's own decision — a nested aggregate underneath it still gets the - /// `cost_model`-ranked default, not the target's forced candidate. This - /// is the behavior [`sketch_candidate`]'s [`bind_with_implementation`] - /// call gets for free (only the top node's `Implementation` is forced; - /// `bind_summary_agg`'s recursion into the child re-ranks normally via - /// `cost_model`) — the old `ForceSketchKind`-`CostModel`-adapter - /// implementation forced *every* `rank_candidates` call for the whole - /// recursive bind, which would have silently forced a nested Quantile to - /// DDSketch too whenever the outer target's DDSketch candidate was - /// enumerated. + /// Enumerating candidates for the *target* node must only steer that + /// node's own decision — a nested aggregate underneath it still gets its + /// own independent (`cost_model`-ranked) enumeration, not whatever the + /// caller happened to pick for the outer target. This is the behavior + /// [`bind::bind_with_implementation`]'s recursion (via + /// `bind::select_and_bind`) gets for free: only the top node's + /// `Implementation` is ever forced from outside; the child is always + /// re-enumerated fresh. #[test] - fn forcing_the_targets_candidate_does_not_leak_into_a_nested_aggregate() { + fn enumerating_the_targets_candidates_does_not_leak_into_a_nested_aggregate() { // outer: quantile(0.99, ...) over inner: quantile(0.5, m) — both // Quantile, so both share the [Kll, DDSketch] candidate list. let inner = agg(vec![2], default_quantile(0.5), metric_scan(&["job"])); @@ -715,7 +631,7 @@ mod tests { assert_eq!( summary_family_kind(node), SketchKind::DDSketch, - "the outer (target) node must be the forced candidate" + "the outer (target) node must be the DDSketch candidate" ); let asap_types::post_asap::SummaryExpr::SummaryEstimate { summary_input, .. } = &node.expr @@ -730,7 +646,7 @@ mod tests { summary_family_kind(child), SketchKind::Kll, "the nested inner aggregate must still get the cost-model-ranked \ - default (Kll), not inherit the outer target's forced DDSketch" + default (Kll), not inherit the outer target's DDSketch candidate" ); } diff --git a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md index 6da01d46..91e7903d 100644 --- a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md @@ -20,10 +20,10 @@ Code samples named `My*` or `Prefer*` (`MyStrategy`, `MyCostModel`, `PreferDDSke Two words come up constantly below and are worth pinning down before anything else, since neither is self-explanatory from context alone: -- **`implementation`** (the module `implementation.rs`) — the *per-node* decision of how one `AggIntent` gets realized: as an approximate sketch, an exact mergeable accumulator, or a pass-through (no summary at all). `implementation::implementation_for`/`implementation_for_with` make this decision for **one** node at a time; they don't walk anything. -- **`bind` / "binding"** — this crate's own `bind.rs`: `implement_tree`/`implement_tree_with` walk a **whole** `QueryExpr` tree, calling `implementation::implementation_for` per node, and emit the complete post-ASAP `SummaryExpr`/`SummaryNode` DAG. This is what "the binding path" means everywhere in this guide (§3 onward): the code path that commits to one `Implementation` per node because something has to actually execute. (The word "bind" means other things elsewhere in this crate's downstream consumers — see `lib.rs`'s own "Terminology" section if you need the full picture — but within this guide, "bind"/"binding" always means this.) +- **`implementation`** (the module `implementation.rs`) — every valid way one `AggIntent` could be realized: as an approximate sketch, an exact mergeable accumulator, or a pass-through (no summary at all). `implementation::implementations_for_with` computes this **one node at a time**, exhaustive and ranked; it doesn't walk anything, and it doesn't pick a favorite — picking is left to its callers. +- **`bind` / "binding"** — this crate's own `bind.rs`: `implement_tree`/`implement_tree_with` walk a **whole** `QueryExpr` tree, and at each node, keep only the first (`cost_model`-preferred) entry from `implementation::implementations_for_with`'s list — the rest is a thin selector, not a second decision procedure (see §3). This is what "the binding path" means everywhere in this guide: the code path that commits to one `Implementation` per node because something has to actually execute. (The word "bind" means other things elsewhere in this crate's downstream consumers — see `lib.rs`'s own "Terminology" section if you need the full picture — but within this guide, "bind"/"binding" always means this.) -So: `implementation` decides **what** one node becomes; `bind`/`implement_tree` is **what walks a whole tree** applying that decision everywhere; `ReplacementStrategy` (this guide's main subject) is the newer, additive path that keeps every alternative `implementation` could have picked instead of collapsing to the one `bind` commits to. +So: `implementation` enumerates **every** way one node could become something; `bind`/`implement_tree` walks a whole tree, keeping only the first of each node's list; `ReplacementStrategy` (this guide's main subject) is the other consumer of that same list, keeping every entry instead of just the first — see §3 for exactly how these three connect. --- @@ -237,55 +237,63 @@ A custom cost model does not necessarily need to override every hook. The curren ## 3. How the current pieces fit together -There are two related paths in the current code, and it's worth being explicit about how they relate before looking at either one, since neither name says it outright: **the binding path is older and still the one thing that actually runs in production; the replacement-strategy path is new and additive, not a replacement for it.** They differ only in *how much of the answer* they keep — but under the hood, they now share one primitive rather than two independent implementations of "turn a chosen `Implementation` into a `SummaryNode`": - -- The **binding path** (`bind::implement_tree_with`) is what runs today when a query actually gets bound: it ranks candidates via a `CostModel`, keeps only the one ranked first, and discards every alternative. -- The **replacement-strategy path** (`ReplacementStrategy::replacements()`, §2) is what enumerates candidates instead of picking one: for each valid `Implementation` a target could realize as, it *keeps* the bound result instead of throwing it away, packaged as a `ReplacementSubDAG`. +There is one place this crate decides what an `AggIntent` may become — +[`implementation::implementations_for_with`] — and two consumers of its +output that differ only in *how much of the list* they keep. Binding is +**literally implemented in terms of the replacement-strategy path**, not a +separate decision procedure that happens to agree with it: + +- **`implementation::implementations_for_with(intent, cost_model)`** enumerates + every valid `Implementation` for `intent`, exhaustive and ranked + (most-preferred first via `cost_model`). +- **`replacement::SketchFamilyStrategy::replacements()`** (§2) wraps that list + directly: every entry becomes its own bound `ReplacementSubDAG`, none + discarded. +- **`bind::implement_tree_with`** builds a `TargetSubDAG` for the node being + bound, calls `SketchFamilyStrategy::replacements()`, and keeps only the + *first* (`cost_model`-preferred) candidate — everything else it throws + away. It does not compute an `Implementation` on its own. This is exactly the "alternatives"/"candidates" language in [the design doc](../design_docs/asap_aware_mapping.md): a `ReplacementSubDAG` **is** one candidate; a `TargetSubDAG` with its full `replacements()` list **is** the set of alternatives for one spot in the plan. -**Is binding "replaced by" `ReplacementStrategy` + `CostModel`-based selection?** Not today, and not literally, on purpose. `implement_tree_with` does not call `SketchFamilyStrategy::replacements()` and take the first result — it resolves its one `Implementation` directly (`implementation_for_with`, which ranks and takes the head) and binds only that. Routing production binding through `replacements()` would mean *sizing and fully binding every candidate* (both KLL and DDSketch, say) at *every* sketch-capable node, recursively, just to discard all but one — multiplying the cost of ordinary binding by the branching factor at each such node. That's why the two paths share a low-level primitive instead of one being layered on top of the other. What *is* true, and tracked separately, not built yet: the design doc's future Cascades/Volcano-style search engine — generate candidates via `ReplacementStrategy` across a whole plan, evaluate/select via `CostModel` — would be exactly "binding replaced by enumerate-then-select," just at plan granularity rather than reimplementing `implement_tree_with` itself. The two paths described below are what that future search would draw on, not the search itself. +**The tradeoff, stated plainly:** because binding now goes through the same enumeration `SketchFamilyStrategy` uses, an ordinary bind sizes and fully constructs *every* sketch candidate at every sketch-capable node — not just the one it keeps — before selecting the head. That's strictly more work per bind than a version that only ever computed the preferred candidate, in exchange for there being exactly one place in this crate that decides what an `AggIntent` may become. Recursion into a node's child goes back through the same enumerate-then-select step fresh, so choosing (or forcing) a candidate for one target never leaks into that target's own nested aggregates. + +What this is *not*: the design doc's future Cascades/Volcano-style search engine — generate candidates via `ReplacementStrategy` across a *whole plan*, evaluate/select via `CostModel` across whole candidate plans — is a separate, not-yet-built piece of work. What exists today is a **single-node** stand-in for that selection (`cost_model.rank_candidates`, applied one node at a time), not the real thing. -### The shared primitive: `bind_with_implementation` +### The shared low-level primitive: `bind_with_implementation` -Both paths bottom out in the same function, `bind::bind_with_implementation(expr, implementation, cost_model)` — *given* an already-decided `Implementation` for `expr`'s top intent, bind it into a `SummaryNode` (or fall back to a logical passthrough). Neither path re-decides how a chosen `Implementation` becomes a `SummaryNode`; they only differ in **which** `Implementation`(s) they hand it: +Underneath both `implement_tree_with` and `SketchFamilyStrategy::replacements()` sits one more function, `bind::bind_with_implementation(expr, implementation, cost_model)` — *given* an already-decided `Implementation` for `expr`'s top intent, bind it into a `SummaryNode` (or fall back to a logical passthrough). Neither caller re-decides how a chosen `Implementation` becomes a `SummaryNode`; they only differ in **which** `Implementation`(s) they hand it, and how many times: ```text - chooses which Implementation(s) + implementations_for_with(intent, cost_model) + every valid Implementation, ranked | - binding path | replacement-strategy path - (implement_tree_with) | (SketchFamilyStrategy) - | | | - v | v - implementation_for_with(...) | summary_candidates(intent) - | | .iter().map(size each) - v | | - one ranked Implementation | every candidate Implementation - | | | - +----------------+----------------+----------+ - | - v - bind_with_implementation(expr, implementation, cost_model) - | - v - one SummaryNode - (one per Implementation handed in) + +----------------+----------------+ + | | + SketchFamilyStrategy::replacements() bind::implement_tree_with + keeps every candidate keeps candidate[0] only + | | + v v + bind_with_implementation(...) bind_with_implementation(...) + once per candidate once, for the kept candidate + | | + v v + N ReplacementSubDAGs one bound SummaryNode ``` -Only `expr`'s own top-level decision is ever forced — recursion into `expr`'s child re-ranks normally via `cost_model`, so enumerating a candidate for one target never leaks into that target's own nested aggregates. - ### Binding path -The binding path needs one executable answer: rank via `cost_model`, take the head, bind it. - ```text AggIntent | v -implementation::implementation_for_with(...) +implementation::implementations_for_with(...) | v -one Implementation +every ranked Implementation + | + v +keep candidate[0] | v bind::bind_with_implementation(...) @@ -298,8 +306,6 @@ one bound SummaryNode ### Replacement-strategy path -The strategy path exists to expose alternatives rather than immediately commit to one: for every candidate `Implementation`, size it and hand it to the same `bind_with_implementation` the binding path uses. - ```text TargetSubDAG | @@ -321,7 +327,7 @@ The important rule is: > Strategies should reuse existing decision and binding logic where possible instead of reimplementing it. -`SketchFamilyStrategy` follows this literally: it doesn't reimplement any part of binding, and it doesn't work around it via a `CostModel`-forcing adapter either — it sizes each candidate the same way `implementation_for_with` would (`implementation::accuracy_budget` + `cost_model.size_params`) and calls `bind_with_implementation` directly, once per candidate. +`SketchFamilyStrategy` follows this literally: it doesn't reimplement any part of binding — it hands `implementations_for_with`'s own list straight to `bind_with_implementation`, once per candidate. --- @@ -564,66 +570,52 @@ Target Aggregate extract AggIntent | v -implementation::implementation_for_with(...) - | - +--------------------------+ +implementation::implementations_for_with(...) | - | approximate sketch case v -implementation::summary_candidates(...) +every ranked Implementation | v -bind each sketch candidate separately +bind each candidate separately | v Vec ``` -For an approximate quantile, the current candidate list includes both KLL and DDSketch. - -The strategy returns both, even if the cost model ranks one ahead of the other. - -For cases where the implementation decision has only one realization, such as an exact accumulator or pass-through, the strategy returns that single realization. +For an approximate quantile, the current candidate list includes both KLL and DDSketch — the strategy returns both, even though the cost model ranks one ahead of the other. For cases where `implementations_for_with` has only one realization, such as an exact accumulator or pass-through, the strategy returns that single realization. There is no separate per-category dispatch inside the strategy: whatever `implementations_for_with` produces, the strategy binds, one entry at a time. --- ### How each candidate actually gets bound: `bind_with_implementation` -The ordinary binding path (`implement_tree_with`) asks the cost model to rank sketch candidates and then binds only the first choice. - -That's correct for normal binding, but a replacement strategy needs to bind **each** valid sketch candidate — and it does so through the *same* underlying primitive `implement_tree_with` itself uses, `bind::bind_with_implementation(expr, implementation, cost_model)`, rather than working around `implement_tree_with`'s "always binds the ranked-first candidate" behavior. - -`bind_with_implementation` takes an already-decided `Implementation` directly — no ranking involved — and binds `expr` to it. `implement_tree_with` is a thin wrapper: rank via `cost_model`, take the head, hand it to `bind_with_implementation`. `SketchFamilyStrategy` skips the ranking step entirely and constructs the `Implementation` for each candidate itself: +`SketchFamilyStrategy`'s whole `replacements()` body is one loop: ```rust -fn sketch_candidate( - root: &QueryExpr, - intent: &AggIntent, - kind: SketchKind, - cost_model: &dyn CostModel, -) -> Option { - let accuracy = accuracy_target(intent)?; - let (eps, delta) = accuracy_budget(accuracy); - let params = cost_model.size_params(kind.clone(), intent, eps, delta); - let implementation = Implementation::Sketch { - kind: kind.clone(), - params, +fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { + let Some(intent) = bindable_intent(target.root) else { + return Vec::new(); }; - let node = - bind_with_implementation(root, implementation, cost_model).ok()?; - // ... wrap `node` in a ReplacementSubDAG with a rationale + implementations_for_with(intent, self.cost_model) + .into_iter() + .filter_map(|implementation| { + let rationale = describe_implementation(intent, &implementation); + let node = + bind_with_implementation(target.root, implementation, self.cost_model) + .ok()?; + Some(ReplacementSubDAG { + replacement: Replacement::Summary(node), + rationale, + }) + }) + .collect() } ``` -`accuracy_target`/`accuracy_budget` (both in `implementation.rs`) are the same accuracy-resolution logic `implementation_for_with`'s own ranked pick goes through — sizing one candidate this way and letting `implementation_for_with` size the ranked-first candidate can never drift apart, because both call the same two functions. - -The result is: - -> bind this specific sketch family, sized exactly the way the ordinary binding path would size it, without touching any of `bind_with_implementation`'s internals. +`implementations_for_with` already did the hard part — enumerating and sizing every candidate, ranked. This loop's only job is to hand each one to `bind::bind_with_implementation(expr, implementation, cost_model)` — the same low-level primitive `bind::implement_tree_with` bottoms out in for whichever candidate it keeps (see §3) — and package the result. No per-candidate sizing logic lives here; there's nothing to keep in sync with the ordinary binding path, because both call the exact same enumeration function. -Because only `expr`'s own top-level `Implementation` is forced — `bind_with_implementation` recurses into `expr`'s child via the ordinary `cost_model`-ranked path, not a forced one — enumerating a candidate for one target never leaks into that target's own nested aggregates. (An earlier version of this strategy forced ranking via a `CostModel`-wrapping adapter for the whole recursive bind, which had exactly that leak as a latent bug; `bind_with_implementation` doesn't have the problem because it never re-ranks anything below the top node.) +Because only `expr`'s own top-level `Implementation` is ever supplied from outside — `bind_with_implementation` recurses into `expr`'s child via a fresh call back into the ordinary `implement_tree_with` selection, not a forced one — enumerating a candidate for one target never leaks into that target's own nested aggregates. (An earlier version of this strategy forced ranking via a `CostModel`-wrapping adapter for the whole recursive bind, which had exactly that leak as a latent bug; today's design doesn't have the problem, because nothing below the top node is ever forced.) -This is the pattern to preserve when adding another strategy that needs to enumerate alternatives through an API that normally chooses one: construct the `Implementation` (or equivalent) explicitly, and hand it to the shared low-level primitive — don't wrap the `CostModel` to trick the ranked-first path into producing what you want. +This is the pattern to preserve when adding another strategy that needs to enumerate alternatives through an API that normally chooses one: get the exhaustive list from the same enumeration function the single-answer path uses, and hand each entry to the shared low-level binding primitive — never wrap the `CostModel` to trick a ranked-first path into producing what you want. --- From 8b61a8492c0e135b412a6600e7b5fba21c4571f2 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 24 Aug 2026 06:38:18 -0600 Subject: [PATCH 16/49] refactor(asap-aware-mapping): delete bind::implement_tree/_with, SketchFamilyStrategy is the sole bound-output API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `bind::implement_tree`/`implement_tree_with` were a "keep candidate[0] only" public entry point sitting alongside `SketchFamilyStrategy:: replacements()`, which already returns every candidate. Delete them: `SketchFamilyStrategy::replacements()` is now the only public way to get bound output for a target, always returning every candidate, ranked. A caller that wants a single executable answer takes the first entry itself (`.into_iter().next()`) and falls back to `bind::logical` if the list is empty — the same conservative pass-through this crate's own dispatch already used internally. `implement_workload`/`implement_workload_with` keep the take-the-head selection internally, since workload-wide CSE sharing memoizes on `Rc` pointer identity and needs one canonical decision per shared root to key sharing on — there's no meaningful "N candidates" answer to memoize against. That's the one place inside this crate a single-answer selection still lives. Migrated the four call sites outside asap-aware-mapping (l4_binding.rs, l4_binding_sql.rs, promql_corpus.rs, show_post_asap_ir.rs) to a local take-the-head test/debug helper following the same pattern. Rewrote doc references across bind.rs, replacement.rs, implementation.rs, cost_model.rs, lib.rs, the ASAP-aware-mapping developer guide, and the user guide. Verified: cargo build --workspace --all-targets, cargo test --workspace (0 failures), cargo fmt --all -- --check, cargo clippy --workspace --all-targets --all-features -- -D warnings — all clean. --- crates/asap-aware-mapping/src/bind.rs | 183 ++++++++---------- crates/asap-aware-mapping/src/cost_model.rs | 3 +- .../asap-aware-mapping/src/implementation.rs | 16 +- crates/asap-aware-mapping/src/lib.rs | 44 +++-- crates/asap-aware-mapping/src/replacement.rs | 91 ++++----- crates/devtools/src/bin/show_post_asap_ir.rs | 31 ++- .../tests/observability/promql_corpus.rs | 38 +++- crates/integration-tests/tests/l4_binding.rs | 46 ++++- .../integration-tests/tests/l4_binding_sql.rs | 80 +++++--- .../ASAP-aware-mapping-developer-guide.md | 81 +++----- docs/user-guide/user-guide.md | 46 +++-- 11 files changed, 380 insertions(+), 279 deletions(-) diff --git a/crates/asap-aware-mapping/src/bind.rs b/crates/asap-aware-mapping/src/bind.rs index 5b63733f..d64619e2 100644 --- a/crates/asap-aware-mapping/src/bind.rs +++ b/crates/asap-aware-mapping/src/bind.rs @@ -1,10 +1,15 @@ -//! The pre-ASAP → post-ASAP binding pass (issue #98). +//! The pre-ASAP → post-ASAP binding primitives (issue #98). //! -//! Walks a canonical pre-ASAP [`QueryExpr`] and emits the summary-bound -//! post-ASAP IR ([`SummaryExpr`] / [`SummaryNode`] in `asap-sketch`). Per -//! node, [`replacement::SketchFamilyStrategy`](crate::replacement::SketchFamilyStrategy) -//! enumerates every valid realization and this pass keeps the first -//! (`cost_model`-preferred) one: +//! This module does **not** expose a "bind me one tree" entry point. +//! [`replacement::SketchFamilyStrategy`](crate::replacement::SketchFamilyStrategy) +//! is the only public way to get bound output for a target — it always +//! returns every candidate [`ReplacementSubDAG`](crate::replacement::ReplacementSubDAG), +//! ranked; a caller that wants a single executable answer takes the first +//! entry itself (`.into_iter().next()`) and decides what to do if the list +//! is empty (this module's [`logical`] is the same pass-through fallback +//! this crate's own dispatch would otherwise use). What this module +//! provides is the shared low-level primitive that turns one *already-decided* +//! candidate into a real [`SummaryNode`]: //! //! - **Summary family** (sketch / sample / wavelet / statistical model) — //! the `Aggregate` becomes a [`SummaryExpr::SummaryAgg`] carrying the @@ -20,27 +25,23 @@ //! [`SummaryExpr::Logical`], schema lifted with every column //! `SummaryFamilyType::Plain`. //! -//! Binding recurses through the `Aggregate` spine, so nested aggregates each -//! get their own decision (`quantile(0.9, sum by (svc) (rate(m[5m]))))` binds -//! KLL over an exact `Sum` accumulator over an exact `Rate` accumulator) — -//! and each nested node's own candidate enumeration and selection is -//! independent of its parent's. +//! Construction recurses through the `Aggregate` spine, so nested aggregates +//! each get their own independent candidate enumeration and selection +//! (`quantile(0.9, sum by (svc) (rate(m[5m]))))` binds KLL over an exact +//! `Sum` accumulator over an exact `Rate` accumulator) — see +//! [`bind_with_implementation`]. //! -//! ## This pass is a selector over `ReplacementStrategy`, not a separate decision path +//! ## Why [`implement_workload`]/[`implement_workload_with`] are still here //! -//! [`implement_tree_with`] does not compute an `Implementation` on its own. -//! It builds a [`TargetSubDAG`](crate::replacement::TargetSubDAG) for the -//! node being bound, asks -//! [`replacement::SketchFamilyStrategy`](crate::replacement::SketchFamilyStrategy) -//! for every candidate [`Replacement`](crate::replacement::Replacement), and -//! keeps the first one — the same list a caller who wants every alternative -//! (rather than just the first) would get from calling -//! `SketchFamilyStrategy::replacements` directly. [`bind_with_implementation`] -//! is the shared low-level primitive both this pass and `SketchFamilyStrategy` -//! bottom out in: given an *already-decided* [`Implementation`], turn it into -//! a `SummaryNode`. See [`crate::replacement`]'s module docs for the reverse -//! half of this relationship — `SketchFamilyStrategy` calls back into this -//! module for that primitive and for [`bindable_intent`]. +//! Workload-level CSE sharing ([`asap_types::pre_asap::cse::share_common_subtrees`]) +//! memoizes on `Rc` pointer identity: two workload roots that collapsed onto +//! the same `Rc` must resolve to the *same* canonical decision to +//! be shareable at all — there is no meaningful "N candidates" answer to +//! memoize against. So this one entry point keeps the old +//! rank-and-take-first behavior internally (via a private selector), scoped +//! to workload-wide CSE memoization specifically. It is not a general +//! "bind me one tree" API — for a single target, go through +//! `SketchFamilyStrategy` and decide what to keep yourself. //! //! ## Conservative fallbacks //! @@ -78,9 +79,8 @@ pub enum ImplementError { Schema(#[from] QueryExprError), } -/// The bindable shape this pass (and -/// [`crate::replacement::SketchFamilyStrategy`], which targets the exact -/// same shape) requires: a single intent, no `HAVING`. A multi-intent node +/// The bindable shape [`crate::replacement::SketchFamilyStrategy`] targets: +/// a single intent, no `HAVING`. A multi-intent node /// (SQL `SELECT SUM(a), AVG(b)`), or one with a `HAVING` predicate (the /// filter would need the estimate first), stays logical — see the module /// docs' "Conservative fallbacks". @@ -96,37 +96,12 @@ pub fn bindable_intent(node: &QueryExpr) -> Option<&AggIntent> { None } -/// Bind a single query to the post-ASAP IR. Ranks candidate summaries via -/// [`DefaultCostModel`] (`asap-plan`'s built-in static preference order, -/// unchanged); use [`implement_tree_with`] to plug in a deployment-specific -/// [`CostModel`] instead. -pub fn implement_tree(expr: &QueryExpr) -> Result, ImplementError> { - implement_tree_with(expr, &DefaultCostModel) -} - -/// Like [`implement_tree`], but ranks candidate summaries via `cost_model` (see -/// [`crate::cost_model`]) instead of the built-in static preference order. -/// -/// A thin selector, not a second decision path — see the module docs. Binds -/// a fresh `Rc` for `expr` (cheap: `QueryExpr::clone` only clones -/// the top node, every descendant stays `Rc`-shared) so -/// [`TargetSubDAG`](crate::replacement::TargetSubDAG) has the identity it -/// needs; a caller that already holds `expr` as an `Rc` (e.g. -/// [`implement_workload_with`]) should prefer passing that `Rc` straight -/// into `SketchFamilyStrategy` itself to skip the extra clone. -pub fn implement_tree_with( - expr: &QueryExpr, - cost_model: &dyn CostModel, -) -> Result, ImplementError> { - select_and_bind(&Rc::new(expr.clone()), cost_model) -} - -/// The actual selector: build a [`TargetSubDAG`] for `root`, ask -/// [`SketchFamilyStrategy`] for every candidate, and keep the first -/// (`cost_model`-preferred) one. `root` must already be the caller's own -/// `Rc` — never fabricated per recursion step — so this stays a single -/// allocation at the outermost [`implement_tree_with`] call, not one per -/// node. +/// Rank-and-take-first selector, scoped to [`implement_workload_with`]'s +/// CSE memoization and to this module's own recursion into a node's child +/// (see the module docs' "Why `implement_workload`/`implement_workload_with` +/// are still here") — **not** a general single-answer API. `root` must +/// already be the caller's own `Rc`, never fabricated per call, so this +/// never allocates beyond what the caller already held. fn select_and_bind( root: &Rc, cost_model: &dyn CostModel, @@ -156,11 +131,11 @@ fn select_and_bind( } /// Bind `expr` to an already-decided [`Implementation`] for its top intent. -/// The shared low-level primitive: [`select_and_bind`] (and so -/// [`implement_tree_with`]) calls this with whichever candidate it kept; -/// [`crate::replacement::SketchFamilyStrategy`] calls this once per -/// candidate it enumerates. One function turns a chosen `Implementation` -/// into a `SummaryNode`, used identically by both. +/// The shared low-level primitive: [`select_and_bind`] (workload-CSE +/// memoization and this module's own recursion) calls this with whichever +/// candidate it kept; [`crate::replacement::SketchFamilyStrategy`] calls +/// this once per candidate it enumerates. One function turns a chosen +/// `Implementation` into a `SummaryNode`, used identically by both. /// /// `expr` must still be the [`bindable_intent`] shape for `implementation` to /// have any effect; anything else falls back to [`logical`]. Only `expr`'s @@ -227,8 +202,8 @@ pub fn bind_with_implementation( /// independently. /// /// Only whole-root sharing is memoized (matching two workload roots that are -/// themselves the same `Rc` after CSE) — [`implement_tree`] is -/// called at most once per distinct root pointer when the decision is +/// themselves the same `Rc` after CSE) — a root is bound at most +/// once per distinct root pointer when the decision is /// `Share`, but it still walks each such tree's own internal structure /// fresh; a subtree shared only *below* two different roots' top level does /// not additionally memoize inside that walk. Widening this to sub-root @@ -432,12 +407,11 @@ fn readout(intent: &AggIntent, col: &ColumnRef, cost_model: &dyn CostModel) -> S } /// Wrap an unrewritten pre-ASAP subtree, lifting its schema with every column -/// `SummaryFamilyType::Plain`. Public so a deployment can force a node it -/// knows [`implement_tree_with`] would otherwise actively (mis)bind — -/// e.g. an intent this crate's `implementation::implementations_for_with` -/// maps to an accumulator kind the deployment's runtime doesn't actually -/// implement — through the same fallback this crate's own dispatch uses, -/// without duplicating the schema-lift logic. +/// `SummaryFamilyType::Plain`. Public so a caller can fall back to this +/// explicitly — e.g. when `SketchFamilyStrategy::replacements()` returns no +/// candidate for a target, or a deployment wants to force a node its own +/// runtime can't actually implement — through the same fallback this +/// crate's own dispatch uses, without duplicating the schema-lift logic. pub fn logical(expr: &QueryExpr) -> Result, ImplementError> { let schema = expr.output_schema()?; Ok(Rc::new(SummaryNode { @@ -516,12 +490,29 @@ mod tests { .unwrap_or_else(|| panic!("no field {name:?} in {schema:?}")) } + /// This module no longer exposes a single-answer public API — the tests + /// below use the same "rank via `cost_model`, keep the first candidate" + /// pattern `select_and_bind` already implements, since `bind.rs`'s own + /// tests are the one internal caller allowed to reach it directly. + /// An external caller doesn't have this shortcut — it goes through + /// `SketchFamilyStrategy::replacements()` itself (see the module docs). + fn bind_first( + expr: &QueryExpr, + cost_model: &dyn CostModel, + ) -> Result, ImplementError> { + select_and_bind(&Rc::new(expr.clone()), cost_model) + } + + fn bind(expr: &QueryExpr) -> Result, ImplementError> { + bind_first(expr, &DefaultCostModel) + } + #[test] fn quantile_binds_kll_wrapped_in_estimate() { // quantile by (job) (m) at ε=0.01 → Estimate(Quantile) over // SummaryAgg(Kll{k:200}) over Logical(Scan). job = col 2. let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); - let root = implement_tree(&q).unwrap(); + let root = bind(&q).unwrap(); let SummaryExpr::SummaryEstimate { summary_input, @@ -566,8 +557,9 @@ mod tests { } /// A deployment-supplied [`CostModel`] can override the default KLL - /// choice — `implement_tree_with` must actually consult it, not just accept and - /// ignore it (issue: cost model interface, see `crate::cost_model`). + /// choice — `bind_first` (via `select_and_bind`) must actually consult it, + /// not just accept and ignore it (issue: cost model interface, see + /// `crate::cost_model`). struct PreferDDSketch; impl CostModel for PreferDDSketch { @@ -590,7 +582,7 @@ mod tests { let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); // Default: KLL (see `quantile_binds_kll_wrapped_in_estimate` above). - let default_root = implement_tree(&q).unwrap(); + let default_root = bind(&q).unwrap(); let SummaryExpr::SummaryEstimate { summary_input, .. } = &default_root.expr else { panic!("expected SummaryEstimate root, got {:?}", default_root.expr); }; @@ -603,7 +595,7 @@ mod tests { )); // With `PreferDDSketch`: DDSketch instead, same query. - let custom_root = implement_tree_with(&q, &PreferDDSketch).unwrap(); + let custom_root = bind_first(&q, &PreferDDSketch).unwrap(); let SummaryExpr::SummaryEstimate { summary_input, .. } = &custom_root.expr else { panic!("expected SummaryEstimate root, got {:?}", custom_root.expr); }; @@ -621,9 +613,9 @@ mod tests { /// A deployment-supplied `CostModel` can realize an `AggIntent::Extension` /// intent as a real sketch instead of the default `PassThrough` (issue - /// #150) — `implement_tree_with` must consult `realize_extension` for - /// the `Extension` arm, and `readout` must consult `readout_extension` - /// to build its `SketchQuery` without panicking. + /// #150) — `implementations_for_with` must consult `realize_extension` + /// for the `Extension` arm, and `readout` must consult + /// `readout_extension` to build its `SketchQuery` without panicking. struct FrequencyCostModel; impl CostModel for FrequencyCostModel { @@ -678,7 +670,7 @@ mod tests { payload: serde_json::json!({ "item": "checkout" }), }; let q = agg(vec![], intent, metric_scan(&[])); - let root = implement_tree(&q).unwrap(); + let root = bind(&q).unwrap(); assert!(matches!(root.expr, SummaryExpr::Logical(_))); } @@ -689,7 +681,7 @@ mod tests { payload: serde_json::json!({ "item": "checkout" }), }; let q = agg(vec![], intent, metric_scan(&[])); - let root = implement_tree_with(&q, &FrequencyCostModel).unwrap(); + let root = bind_first(&q, &FrequencyCostModel).unwrap(); let SummaryExpr::SummaryEstimate { summary_input, @@ -722,7 +714,7 @@ mod tests { #[test] fn exact_sum_binds_accumulator_without_estimate() { let q = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let root = implement_tree(&q).unwrap(); + let root = bind(&q).unwrap(); let SummaryExpr::SummaryAgg { family, .. } = &root.expr else { panic!( "expected bare SummaryAgg (no estimate), got {:?}", @@ -750,7 +742,7 @@ mod tests { child: Rc::new(metric_scan(&["job"])), }, ); - let root = implement_tree(&q).unwrap(); + let root = bind(&q).unwrap(); let SummaryExpr::SummaryAgg { family, .. } = &root.expr else { panic!("expected SummaryAgg, got {:?}", root.expr); }; @@ -787,7 +779,7 @@ mod tests { child: Rc::new(metric_scan(&["job"])), }, ); - let root = implement_tree(&q).unwrap(); + let root = bind(&q).unwrap(); let SummaryExpr::SummaryEstimate { summary_input, .. } = &root.expr else { panic!("expected estimate root, got {:?}", root.expr); }; @@ -809,7 +801,7 @@ mod tests { accuracy: AccuracyTarget::Epsilon(0.01), }; let q = agg(vec![], intent, metric_scan(&["job"])); - let root = implement_tree(&q).unwrap(); + let root = bind(&q).unwrap(); let SummaryExpr::SummaryEstimate { summary_input, .. } = &root.expr else { panic!("expected estimate root, got {:?}", root.expr); }; @@ -826,7 +818,7 @@ mod tests { // accumulator. let inner = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); let outer = agg(vec![], default_quantile(0.9), inner); - let root = implement_tree(&outer).unwrap(); + let root = bind(&outer).unwrap(); let SummaryExpr::SummaryEstimate { summary_input, .. } = &root.expr else { panic!("expected estimate root, got {:?}", root.expr); @@ -871,7 +863,7 @@ mod tests { col, accuracy: AccuracyTarget::Epsilon(0.01), }; - let root = implement_tree(&agg(vec![0], intent, metric_scan(&["job"]))).unwrap(); + let root = bind(&agg(vec![0], intent, metric_scan(&["job"]))).unwrap(); let bound = find_summary_col(&root) .unwrap_or_else(|| panic!("expected a SummaryAgg for col={col:?}")); assert_eq!(bound, want, "wrong summarised column for col={col:?}"); @@ -902,7 +894,7 @@ mod tests { }, ] { let q = agg(vec![2], intent.clone(), metric_scan(&["job"])); - let root = implement_tree(&q).unwrap(); + let root = bind(&q).unwrap(); assert!( matches!(root.expr, SummaryExpr::Logical(ref e) if **e == q), "expected Logical passthrough for {intent:?}" @@ -923,7 +915,7 @@ mod tests { })), child: Rc::new(agg(vec![], default_quantile(0.99), metric_scan(&[]))), }; - let root = implement_tree(&q).unwrap(); + let root = bind(&q).unwrap(); assert!(matches!(root.expr, SummaryExpr::Logical(ref e) if **e == q)); } @@ -935,10 +927,7 @@ mod tests { ScalarValue::Boolean(true), )))); } - assert!(matches!( - implement_tree(&q).unwrap().expr, - SummaryExpr::Logical(_) - )); + assert!(matches!(bind(&q).unwrap().expr, SummaryExpr::Logical(_))); let multi = QueryExpr::Aggregate { reduction: Reduction::by(vec![2]), @@ -948,7 +937,7 @@ mod tests { child: Rc::new(metric_scan(&["job"])), }; assert!(matches!( - implement_tree(&multi).unwrap().expr, + bind(&multi).unwrap().expr, SummaryExpr::Logical(_) )); } @@ -963,7 +952,7 @@ mod tests { }, metric_scan(&["job"]), ); - let root = implement_tree(&q).unwrap(); + let root = bind(&q).unwrap(); let SummaryExpr::SummaryEstimate { summary_input, query, @@ -1001,7 +990,7 @@ mod tests { }, }; let q = agg(vec![0], AggIntent::Sum { col: Some(1) }, scan); - let root = implement_tree(&q).unwrap(); + let root = bind(&q).unwrap(); let SummaryExpr::SummaryAgg { col, .. } = &root.expr else { panic!("expected SummaryAgg, got {:?}", root.expr); }; diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index e4ff888d..027d626a 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -26,7 +26,8 @@ //! than overloading these ones across incompatible `Kind`/`Params` types. //! //! Every entry point that doesn't take an explicit `&dyn CostModel` -//! ([`implement_tree`](crate::bind::implement_tree)) runs against +//! ([`SketchFamilyStrategy::default_cost_model`](crate::replacement::SketchFamilyStrategy::default_cost_model), +//! [`implement_workload`](crate::bind::implement_workload)) runs against //! [`DefaultCostModel`], so a deployment that never plugs in its own cost //! model keeps today's static-preference-order behavior exactly, byte for //! byte. diff --git a/crates/asap-aware-mapping/src/implementation.rs b/crates/asap-aware-mapping/src/implementation.rs index b09fd524..dd14d9d9 100644 --- a/crates/asap-aware-mapping/src/implementation.rs +++ b/crates/asap-aware-mapping/src/implementation.rs @@ -9,10 +9,12 @@ //! [`implementations_for_with`] is where every valid realization gets //! enumerated, exhaustive and ranked (most-preferred first) — this crate has //! no separate function that computes just "the one" `Implementation` -//! independently of that list. [`crate::bind::implement_tree_with`] and -//! [`crate::replacement::SketchFamilyStrategy`] both consume this same -//! list; they differ only in how much of it they keep (see `bind.rs`'s and -//! `replacement.rs`'s module docs). Three inputs feed it: +//! independently of that list. +//! [`crate::replacement::SketchFamilyStrategy`] is the sole public consumer: +//! it wraps every entry of this list into its own bound +//! [`SummaryNode`](asap_types::post_asap::SummaryNode) and returns all of +//! them, ranked — a caller wanting a single answer keeps the first one +//! itself (see `replacement.rs`'s module docs). Three inputs feed it: //! //! - the [`AccuracyTarget`] threaded onto the approximate-capable intents //! (`Quantile` / `Cardinality` / `Count` / `TopK`) — `Exact` forbids a @@ -172,11 +174,9 @@ pub fn accuracy_target(intent: &AggIntent) -> Option<&AccuracyTarget> { /// (most-preferred first via `cost_model`) — the *only* place this crate /// decides what an `AggIntent` may become. Nothing in this crate computes /// "the one" `Implementation` independently of this list: -/// [`crate::bind::implement_tree_with`] takes the head for the single -/// executable answer production binding needs; /// [`crate::replacement::SketchFamilyStrategy`] keeps every entry as a -/// candidate. Both differ only in *how much of this list* they keep — see -/// their own module docs. +/// candidate, and a caller that wants a single executable answer takes the +/// head of *that* strategy's output itself — see its own module docs. /// /// Exhaustive over the [`AggIntent`] vocabulary — adding a variant without an /// explicit realization is a compile error, and the coverage-matrix test pins diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index db4e093e..1cf35bc2 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -8,12 +8,11 @@ //! **Common sub-expression elimination (CSE) is not this crate's job.** //! Detection is a primary pass over the pre-ASAP `QueryExpr` IR itself //! (`asap_types::pre_asap`, design tracked in issue #223), run before a -//! tree ever reaches [`bind::implement_tree`] — see issue #222 for why -//! (batch query optimization needs to see shared work across a +//! tree ever reaches [`replacement::SketchFamilyStrategy`] — see issue #222 +//! for why (batch query optimization needs to see shared work across a //! `QueryWorkload` before summary binding, not after). This crate may //! eventually run a second, narrower CSE pass of its own over an -//! already-[`implement_tree`](bind::implement_tree)'d -//! `SummaryExpr`/`SummaryNode` DAG, recognizing sharing that's invisible +//! already-bound `SummaryExpr`/`SummaryNode` DAG, recognizing sharing that's invisible //! at the pre-ASAP level by construction — e.g. `Quantile(x, 0.99)` and //! `Quantile(x, 0.95)` are structurally distinct `AggIntent`s but can //! still share one built sketch, read out twice. That post-ASAP pass is @@ -48,18 +47,23 @@ //! CSE-detected shared subtree. No search/ranking-across-a-whole-plan logic //! lives here — see that module's docs for what's deliberately left to a //! future Cascades/Volcano-style search engine. -//! - [`bind`] — [`bind::implement_tree`]/[`bind::implement_tree_with`] walk a -//! whole tree, keeping only the first (`cost_model`-preferred) candidate -//! [`replacement::SketchFamilyStrategy`] enumerates at each node — a thin -//! selector over `replacement`, not a second decision path (see -//! `bind.rs`'s and `replacement.rs`'s own module docs for the full -//! relationship). [`bind::implement_workload`] drives the same selection -//! over a whole workload's roots, memoized on `Rc` identity so two roots -//! that `asap_types::pre_asap::cse::share_common_subtrees` already -//! collapsed onto one shared subtree bind to one shared `SummaryNode` too -//! (issue #212, #222, #223) — see the terminology section below for why -//! this module (and `implement_tree`) are named around "implementation" -//! rather than "bind". +//! - [`bind`] — has no "bind me one tree" entry point of its own. +//! [`replacement::SketchFamilyStrategy`] is the only public way to get +//! bound output for a target, and it always returns *every* candidate; a +//! caller that wants one answer takes the first entry itself. What +//! `bind` provides is the shared low-level primitive +//! ([`bind::bind_with_implementation`]) that turns one already-decided +//! candidate into a real [`SummaryNode`](asap_types::post_asap::SummaryNode), +//! plus [`bind::implement_workload`]/[`bind::implement_workload_with`], +//! which drive rank-and-take-first selection over a whole workload's +//! roots, memoized on `Rc` identity so two roots that +//! `asap_types::pre_asap::cse::share_common_subtrees` already collapsed +//! onto one shared subtree bind to one shared `SummaryNode` too (issue +//! #212, #222, #223) — memoization needs one canonical decision per +//! shared root to key sharing on, so that entry point is the one place a +//! single-answer selection still lives. See the terminology section below +//! for why this crate's logical→physical step (and this module) are named +//! around "implementation" rather than "bind". //! - [`cost_model`] — the [`CostModel`](cost_model::CostModel) trait every //! deployment's cost-based sketch selection plugs into (issues #6, #33). //! `asap-plan` itself only ships [`DefaultCostModel`](cost_model::DefaultCostModel), @@ -85,7 +89,8 @@ //! | **Parse** | parse | text (PromQL/SQL) → AST | `asap-frontend-promql` / `asap-frontend-sql` | //! | **Bind #1** | name resolution | `ColumnRef` (a name) → `ColumnId` (a concrete schema column) — the classic RDBMS "Parse → **Bind** → Optimize" pipeline sense (e.g. SQL Server's query-processor terminology) | [`asap_types::pre_asap::binder::Binder`](https://docs.rs/asap-types) | //! | **Implementation** — [`implementation::implementations_for_with`] | pre-ASAP → post-ASAP, *one node* | enumerating every concrete physical realization (a sketch family, an exact accumulator, or pass-through) for one [`AggIntent`](asap_types::pre_asap::agg_intent::AggIntent) | [`implementation`] | -//! | **`implement_tree`** — [`bind::implement_tree`] | pre-ASAP → post-ASAP, *whole tree* | walk a whole `QueryExpr` tree, keeping the first candidate [`implementation::implementations_for_with`] produces per node, and emit the complete post-ASAP [`SummaryExpr`](asap_types::post_asap::SummaryExpr)/`SummaryNode` DAG — named after "implementation" too rather than reusing "bind" a second time | [`bind`] | +//! | **Replacement** — [`replacement::SketchFamilyStrategy::replacements`] | pre-ASAP → post-ASAP, *one target, every candidate* | wrap each [`implementation::implementations_for_with`] candidate into its own bound [`SummaryNode`](asap_types::post_asap::SummaryNode), ranked — a caller wanting one answer takes the first entry itself | [`replacement`] | +//! | **`implement_workload`** — [`bind::implement_workload`] | pre-ASAP → post-ASAP, *whole workload* | walk every root of a `QueryWorkload`, keeping the first (`cost_model`-preferred) candidate per node, sharing one bound `SummaryNode` across roots CSE already collapsed onto one `Rc` — emits the complete post-ASAP `SummaryExpr`/`SummaryNode` DAG | [`bind`] | //! | **Bind #2** (downstream, not in this crate) | post-ASAP → deployment placement | a *deployment's* own physical binder, additionally deciding **placement** (edge vs. backend, wire format, …) — a genuinely different, deployment-specific decision this crate doesn't model at all | e.g. `control_plane::sketch_algebra::rules::bind_*` (as of this writing; expected to fold into that deployment's cost-model layer rather than stay a separate "bind" concept) | //! //! A related question (tracked alongside issues #6/#33): whether this @@ -110,10 +115,7 @@ pub mod cost_model; pub mod implementation; pub mod replacement; -pub use bind::{ - implement_tree, implement_tree_with, implement_workload, implement_workload_with, - ImplementError, -}; +pub use bind::{implement_workload, implement_workload_with, ImplementError}; pub use cost_model::{CostModel, DefaultCostModel}; pub use implementation::{implementations_for_with, summary_candidates, Implementation, Matcher}; pub use replacement::{ diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index dbc8fdb2..66f97939 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -3,12 +3,14 @@ //! under "Key concepts (not yet implemented)", implemented for real (issue //! #251, part of #33). //! -//! ## This is where every `Implementation` gets decided — `bind` is a selector over it +//! ## This is where every `Implementation` gets decided — and the only place bound output comes from //! //! [`implementation::implementations_for_with`] is the one place this crate //! enumerates every valid [`Implementation`] for an [`AggIntent`], exhaustive //! and ranked (most-preferred first via a [`CostModel`]). This module wraps -//! that list into the exhaustive-candidate shape a future search needs: +//! that list into the exhaustive-candidate shape a future search needs, and +//! is the *only* public way to get bound output for a target — see +//! "Selection" below. //! //! - [`TargetSubDAG`] — a reference to a pre-ASAP [`QueryExpr`] node that is a //! candidate for replacement, plus how many places in the workload already @@ -16,10 +18,8 @@ //! context [`SharedSubtreeStrategy`] needs that a bare node reference alone //! doesn't carry. //! - [`ReplacementSubDAG`] — one candidate replacement for a `TargetSubDAG`: -//! either a fully bound [`SummaryNode`] (the same output -//! [`bind::implement_tree_with`] itself keeps, for one particular candidate -//! instead of the one it kept) or a pre-ASAP [`QueryExpr`] rewrite (still -//! logical, structurally different from the target but semantically +//! either a fully bound [`SummaryNode`] or a pre-ASAP [`QueryExpr`] rewrite +//! (still logical, structurally different from the target but semantically //! equivalent) — see [`Replacement`] — plus a human-readable `rationale`. //! - [`ReplacementStrategy`] — `matches` + `replacements`, the same //! extension-point shape [`CostModel`] and [`Matcher`](crate::implementation::Matcher) @@ -29,35 +29,38 @@ //! `impl ReplacementStrategy`, not a restructuring of this trait or of any //! existing strategy. `replacements` is **exhaustive, not ranked, not //! filtered** — reporting "every valid candidate" is core's job; picking -//! the best one is a [`CostModel`]'s job (see "Selection" below). +//! the best one is left to the caller (see "Selection" below). //! -//! ## Selection: `bind::implement_tree_with` keeps the head, nothing else +//! ## Selection: this crate reports every candidate; a caller keeps what it wants //! -//! [`bind::implement_tree_with`] does not compute an `Implementation` -//! independently — it builds a `TargetSubDAG` for the node being bound, -//! calls [`SketchFamilyStrategy::replacements`], and keeps the first -//! (`cost_model`-preferred) candidate; everything else it discards. That's -//! the *only* difference between "the binding path" and "the -//! replacement-strategy path": how much of this module's own output each one -//! keeps. Both bottom out in the exact same low-level primitive, -//! [`bind::bind_with_implementation`] — given an *already-decided* -//! `Implementation`, turn it into a `SummaryNode` — which this module calls -//! once per candidate it enumerates, and `bind::implement_tree_with` calls -//! once for the one candidate it kept. See `bind.rs`'s own module docs for -//! the reverse half of this relationship. +//! [`SketchFamilyStrategy::replacements`] builds a `TargetSubDAG` for the +//! node being replaced, then enumerates [`implementation::implementations_for_with`]'s +//! ranked list, calling [`bind::bind_with_implementation`] once per +//! candidate — given an *already-decided* `Implementation`, turn it into a +//! `SummaryNode`. Every candidate comes back; nothing is discarded here. A +//! caller that wants one executable answer takes the first +//! (`cost_model`-preferred) entry itself (`.into_iter().next()`) — that +//! "keep the head" step now lives entirely on the calling side, not behind +//! a second module-level entry point. `bind.rs`'s own +//! [`bind::implement_workload`]/[`bind::implement_workload_with`] are the +//! one place inside this crate that still performs that take-first step +//! internally, because workload-wide CSE memoization needs one canonical +//! decision per shared root to key sharing on (see `bind.rs`'s own module +//! docs). Every other caller goes through `SketchFamilyStrategy::replacements` +//! directly and decides for itself. //! -//! This means an ordinary bind now sizes and fully constructs *every* -//! sketch candidate at every sketch-capable node (not just the one it keeps) -//! — a deliberate tradeoff, made so there is exactly one place in this crate -//! that decides what an `AggIntent` may become, at the cost of extra work -//! per bind proportional to each node's own candidate count. +//! This means an ordinary single-target bind now sizes and fully constructs +//! *every* sketch candidate at every sketch-capable node (not just the one a +//! caller keeps) — a deliberate tradeoff, made so there is exactly one place +//! in this crate that decides what an `AggIntent` may become, at the cost of +//! extra work per bind proportional to each node's own candidate count. //! //! ## The two strategies, and why these two //! //! - [`SketchFamilyStrategy`] wraps [`implementation::implementations_for_with`]'s //! exhaustive, ranked list directly: for the same bindable-`Aggregate` -//! shape [`bind::implement_tree_with`] itself requires (single intent, no -//! `HAVING`), every entry becomes its own bound candidate. +//! shape this crate binds (single intent, no `HAVING`), every entry +//! becomes its own bound candidate. //! - [`SharedSubtreeStrategy`] wraps //! `asap_types::pre_asap::cse::share_common_subtrees`'s sharing decision. //! Wherever a [`TargetSubDAG`] already has two or more consumers (i.e. @@ -81,10 +84,11 @@ //! replacement-plan-searching pseudocode — trying every `ReplacementStrategy` //! against every candidate plan, deduplicating, iterating to a fixpoint, //! then ranking by a `CostModel` — is a Cascades/Volcano-style search -//! engine, tracked as a separate follow-up. `bind::implement_tree_with`'s -//! "keep the head" selection is a single-node stand-in for that, not the -//! real thing: it never compares whole candidate *plans*, only one node's -//! own candidates against each other via `cost_model.rank_candidates`. +//! engine, tracked as a separate follow-up. Taking the first candidate off +//! [`SketchFamilyStrategy::replacements`] is a single-node stand-in for +//! that, not the real thing: it never compares whole candidate *plans*, +//! only one node's own candidates against each other via +//! `cost_model.rank_candidates`. //! - **No workload-wide `TargetSubDAG` discovery pass.** Finding every //! candidate node in a whole workload (walking every root, deduplicating by //! `Rc` identity, computing real consumer counts) is exactly what PR #247's @@ -156,15 +160,15 @@ impl<'a> TargetSubDAG<'a> { /// What a [`ReplacementSubDAG`] actually substitutes a [`TargetSubDAG`] with. /// -/// Generalizes [`bind::implement_tree_with`]'s and -/// [`implementation::implementations_for_with`]'s two possible *kinds* of -/// answer — a post-ASAP binding decision, or a still-pre-ASAP structural -/// alternative — from "the one kept" into "one candidate among several". +/// Generalizes [`implementation::implementations_for_with`]'s two possible +/// *kinds* of answer — a post-ASAP binding decision, or a still-pre-ASAP +/// structural alternative — into "one candidate among several", each with +/// its own [`ReplacementSubDAG`]. #[derive(Debug, Clone)] pub enum Replacement { - /// A fully bound post-ASAP summary decision — the same - /// [`SummaryNode`] shape [`bind::implement_tree_with`] itself produces, - /// for one particular candidate realization of the target. + /// A fully bound post-ASAP summary decision — the same [`SummaryNode`] + /// shape [`bind::bind_with_implementation`] produces, for one particular + /// candidate realization of the target. Summary(Rc), /// A pre-ASAP rewrite: still a logical [`QueryExpr`], structurally /// different from the target's own `root` (e.g. sharing vs. not sharing @@ -223,16 +227,14 @@ static DEFAULT_COST_MODEL: DefaultCostModel = DefaultCostModel; /// [`CostModel`] — [`DefaultCostModel`] unless constructed with /// [`SketchFamilyStrategy::new`] — so a deployment-specific cost model's /// other hooks (`size_params`, `realize_extension`, `readout_extension`) are -/// still consulted while binding each candidate, exactly as -/// [`bind::implement_tree_with`] would. +/// still consulted while binding each candidate. pub struct SketchFamilyStrategy<'a> { cost_model: &'a dyn CostModel, } impl SketchFamilyStrategy<'static> { - /// A strategy that ranks/binds via the built-in [`DefaultCostModel`], - /// matching what a deployment gets from [`bind::implement_tree`] with no - /// custom cost model plugged in. + /// A strategy that ranks/binds via the built-in [`DefaultCostModel`] — + /// what a deployment gets with no custom cost model plugged in. pub fn default_cost_model() -> Self { Self { cost_model: &DEFAULT_COST_MODEL, @@ -243,8 +245,7 @@ impl SketchFamilyStrategy<'static> { impl<'a> SketchFamilyStrategy<'a> { /// A strategy that ranks/binds via `cost_model` instead of the built-in /// static preference order — the same customization point - /// [`bind::implement_tree_with`] and - /// [`implementation::implementations_for_with`] already offer. + /// [`implementation::implementations_for_with`] already offers. pub fn new(cost_model: &'a dyn CostModel) -> Self { Self { cost_model } } diff --git a/crates/devtools/src/bin/show_post_asap_ir.rs b/crates/devtools/src/bin/show_post_asap_ir.rs index 1092a662..ec50b8ca 100644 --- a/crates/devtools/src/bin/show_post_asap_ir.rs +++ b/crates/devtools/src/bin/show_post_asap_ir.rs @@ -20,14 +20,41 @@ // `metrics(ts, service, region, latency, bytes)` catalog — the same table // used in cross_language.rs and topk_ir.rs. -use asap_aware_mapping::implement_tree; +use asap_aware_mapping::bind::logical; +use asap_aware_mapping::{ + Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG, +}; use asap_devtools::{lower_promql, lower_sql, SqlCatalog}; +use asap_types::pre_asap::query_expr::QueryExpr; use asap_types::pre_asap::schema::{Column, DataType, Schema}; use asap_types::types::AccuracyTarget; use std::io::Read; +use std::rc::Rc; const ACCURACY: AccuracyTarget = AccuracyTarget::Epsilon(0.01); +/// `asap-aware-mapping` has no "bind me one tree" public API any more — +/// `SketchFamilyStrategy::replacements` always returns every candidate, and +/// a caller decides what to keep. This debug tool just wants one +/// representative binding per query, so it takes the first +/// (`cost_model`-preferred) candidate the same way a production caller +/// would. +fn bind(expr: &QueryExpr) -> Result, String> { + let root = Rc::new(expr.clone()); + let target = TargetSubDAG::new(&root); + match SketchFamilyStrategy::default_cost_model() + .replacements(&target) + .into_iter() + .next() + { + Some(ReplacementSubDAG { + replacement: Replacement::Summary(node), + .. + }) => Ok(node), + _ => logical(&root).map_err(|e| e.to_string()), + } +} + fn col(name: &str, dtype: DataType) -> Column { Column::new(name, dtype, false) } @@ -82,7 +109,7 @@ async fn main() { println!(); continue; }; - match l3.and_then(|expr| implement_tree(&expr).map_err(|e| e.to_string())) { + match l3.and_then(|expr| bind(&expr)) { Ok(l4) => println!("{:#?}", l4.expr), Err(e) => println!("ERR: {e}"), } diff --git a/crates/frontend-promql/tests/observability/promql_corpus.rs b/crates/frontend-promql/tests/observability/promql_corpus.rs index 526c02bf..babde60a 100644 --- a/crates/frontend-promql/tests/observability/promql_corpus.rs +++ b/crates/frontend-promql/tests/observability/promql_corpus.rs @@ -13,11 +13,39 @@ //! that is the guarantee. A coverage floor guards against a change silently //! tanking how much of the corpus we can lower. -use asap_aware_mapping::implement_tree; +use std::rc::Rc; + +use asap_aware_mapping::bind::{logical, ImplementError}; +use asap_aware_mapping::{ + Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG, +}; use asap_frontend_promql::{lower_promql, PromqlError as LoweringError}; -use asap_types::post_asap::SummaryExpr; +use asap_types::post_asap::{SummaryExpr, SummaryNode}; +use asap_types::pre_asap::query_expr::QueryExpr; use asap_types::types::AccuracyTarget; +/// This crate has no "bind me one tree" public API any more — +/// `SketchFamilyStrategy::replacements` always returns every candidate, and +/// a caller decides what to keep. This test-only helper reproduces the +/// take-the-first-(`cost_model`-preferred)-candidate pattern so [`bind_tally`] +/// gets one representative `Result` per query, matching what a totality +/// check over the whole corpus wants. +fn bind(expr: &QueryExpr) -> Result, ImplementError> { + let root = Rc::new(expr.clone()); + let target = TargetSubDAG::new(&root); + match SketchFamilyStrategy::default_cost_model() + .replacements(&target) + .into_iter() + .next() + { + Some(ReplacementSubDAG { + replacement: Replacement::Summary(node), + .. + }) => Ok(node), + _ => logical(&root), + } +} + const DOCS: &str = include_str!("data/promql_corpus_docs.txt"); const TESTDATA: &str = include_str!("data/promql_corpus_testdata.txt"); @@ -66,7 +94,7 @@ struct BindTally { transformed: usize, /// Root stayed `Logical` — the pass left the query untouched. unchanged: usize, - /// `implement_tree` returned `Err` (schema derivation failed). + /// [`bind`] returned `Err` (schema derivation failed). errored: usize, } @@ -76,7 +104,7 @@ fn bind_tally(corpus: &str, accuracy: AccuracyTarget) -> BindTally { let Ok(tree) = lower_promql(q, accuracy.clone()) else { continue; }; - match implement_tree(&tree) { + match bind(&tree) { Ok(bound) if matches!(bound.expr, SummaryExpr::Logical(_)) => t.unchanged += 1, Ok(_) => t.transformed += 1, Err(_) => t.errored += 1, @@ -93,7 +121,7 @@ fn binding_is_total_over_the_entire_corpus() { eprintln!("docs corpus post-ASAP binding: {docs:?}"); eprintln!("testdata corpus post-ASAP binding: {td:?}"); - // Same totality guarantee as lowering: reaching here means `implement_tree` + // Same totality guarantee as lowering: reaching here means `bind` // never panicked over any lowerable query in the corpus. assert_eq!(docs.errored, 0, "post-ASAP binding errored: {docs:?}"); assert_eq!(td.errored, 0, "post-ASAP binding errored: {td:?}"); diff --git a/crates/integration-tests/tests/l4_binding.rs b/crates/integration-tests/tests/l4_binding.rs index 0eddcf26..470feb8b 100644 --- a/crates/integration-tests/tests/l4_binding.rs +++ b/crates/integration-tests/tests/l4_binding.rs @@ -1,23 +1,49 @@ //! End-to-end query-string → post-ASAP IR pin (issue #98). //! //! Drives the full pipeline — PromQL text → pre-ASAP `QueryExpr` -//! (`lower_promql`) → post-ASAP `SummaryExpr` DAG -//! (`asap_aware_mapping::implement_tree`) — and pins the summary-bound shape -//! node by node, including the family `(Kind, Params)` committed on each -//! edge's schema. This is the design doc's §"L4 — sketch algebra" worked -//! example, running for real. +//! (`lower_promql`) → post-ASAP `SummaryExpr` DAG (via +//! `SketchFamilyStrategy::replacements`, see [`bind`] below) — and pins the +//! summary-bound shape node by node, including the family `(Kind, Params)` +//! committed on each edge's schema. This is the design doc's §"L4 — sketch +//! algebra" worked example, running for real. -use asap_aware_mapping::implement_tree; +use std::rc::Rc; + +use asap_aware_mapping::bind::{logical, ImplementError}; +use asap_aware_mapping::{ + Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG, +}; use asap_frontend_promql::lower_promql; use asap_types::post_asap::{ ExactKind, ExactParams, SketchKind, SketchParams, SketchQuery, SummaryExpr, SummaryFamilyType, - SummarySchema, + SummaryNode, SummarySchema, }; use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::query_expr::{QueryExpr, Reduction}; use asap_types::pre_asap::schema::DataType; use asap_types::types::AccuracyTarget; +/// This crate has no "bind me one tree" public API any more — +/// `SketchFamilyStrategy::replacements` always returns every candidate, and +/// a caller decides what to keep. This test-only helper reproduces the +/// take-the-first-(`cost_model`-preferred)-candidate pattern so the +/// single-answer pins below don't all repeat it by hand. +fn bind(expr: &QueryExpr) -> Result, ImplementError> { + let root = Rc::new(expr.clone()); + let target = TargetSubDAG::new(&root); + match SketchFamilyStrategy::default_cost_model() + .replacements(&target) + .into_iter() + .next() + { + Some(ReplacementSubDAG { + replacement: Replacement::Summary(node), + .. + }) => Ok(node), + _ => logical(&root), + } +} + fn dtype<'a>(schema: &'a SummarySchema, name: &str) -> &'a SummaryFamilyType { &schema .fields @@ -46,7 +72,7 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { AccuracyTarget::Epsilon(0.01), ) .expect("lowering failed"); - let root = implement_tree(&l3).expect("binding failed"); + let root = bind(&l3).expect("binding failed"); // Root: the sketch readout, back to a plain row shape. let SummaryExpr::SummaryEstimate { @@ -144,7 +170,7 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { fn promql_exact_workload_binds_accumulators_not_sketches() { let l3 = lower_promql("sum by (job) (http_requests_total)", AccuracyTarget::Exact) .expect("lowering failed"); - let root = implement_tree(&l3).expect("binding failed"); + let root = bind(&l3).expect("binding failed"); let SummaryExpr::SummaryAgg { family, reduction, .. } = &root.expr @@ -168,7 +194,7 @@ fn promql_exact_workload_binds_accumulators_not_sketches() { let l3 = lower_promql("avg(http_requests_total)", AccuracyTarget::Exact).expect("lowering failed"); - let root = implement_tree(&l3).expect("binding failed"); + let root = bind(&l3).expect("binding failed"); assert!( matches!(root.expr, SummaryExpr::Logical(_)), "avg has no mergeable accumulator — stays logical" diff --git a/crates/integration-tests/tests/l4_binding_sql.rs b/crates/integration-tests/tests/l4_binding_sql.rs index 2d32f3b5..6cfa97ca 100644 --- a/crates/integration-tests/tests/l4_binding_sql.rs +++ b/crates/integration-tests/tests/l4_binding_sql.rs @@ -1,44 +1,70 @@ //! End-to-end SQL query-string → post-ASAP IR pin (issue #191). //! //! The SQL counterpart of `l4_binding.rs`: drives SQL text — `lower_sql` -//! (text → pre-ASAP `QueryExpr`) → `asap_aware_mapping::implement_tree` -//! (pre-ASAP → post-ASAP `SummaryExpr`) — and pins the resulting -//! sketch-vs-exact-accumulator binding shape node by node, the way +//! (text → pre-ASAP `QueryExpr`) → `SketchFamilyStrategy::replacements` +//! (pre-ASAP → post-ASAP `SummaryExpr`, see [`bind`] below) — and pins the +//! resulting sketch-vs-exact-accumulator binding shape node by node, the way //! `l4_binding.rs` does for PromQL. //! //! ## A structural wrinkle PromQL doesn't have //! //! `lower_promql` returns a *bare* `QueryExpr::Aggregate` for a top-level -//! aggregation (`sum by (job) (m)`, `quantile(0.99, …)`), so `implement_tree` -//! can bind it directly at the tree root. `lower_sql` never does: DataFusion's +//! aggregation (`sum by (job) (m)`, `quantile(0.99, …)`), so [`bind`] can +//! bind it directly at the tree root. `lower_sql` never does: DataFusion's //! planner always wraps even a single, unaliased aggregate in an identity //! `Project` (confirmed below), so a SQL tree's *root* is always `Project { -//! child: Aggregate { .. } }`. `implement_tree` only fires -//! `bind_summary_agg` when the node it's looking at is itself a bindable -//! `QueryExpr::Aggregate` (see `bind.rs`'s module docs on the "logical parent -//! subsumes bindable child" conservative fallback); a `Project` at the root -//! is exactly such a logical parent, so feeding a raw `lower_sql` result -//! straight into `implement_tree` always yields a whole-tree -//! `SummaryExpr::Logical` — never a genuine sketch or accumulator binding. +//! child: Aggregate { .. } }`. Binding only fires `bind_summary_agg` when +//! the node it's looking at is itself a bindable `QueryExpr::Aggregate` (see +//! `bind.rs`'s module docs on the "logical parent subsumes bindable child" +//! conservative fallback); a `Project` at the root is exactly such a logical +//! parent, so feeding a raw `lower_sql` result straight into [`bind`] always +//! yields a whole-tree `SummaryExpr::Logical` — never a genuine sketch or +//! accumulator binding. //! //! The tests below extract the inner `Aggregate` node the same way this //! crate's own `frontend-sql/tests/sql_lowering.rs` does (its //! `find_aggregate`/`find_aggregate_node` helpers) and hand that to -//! `implement_tree` directly, which is the shape a future Project-elision -//! rewrite (tracked with the rest of the post-ASAP rule engine, issues -//! #6/#33) would present to this pass in production. +//! [`bind`] directly, which is the shape a future Project-elision rewrite +//! (tracked with the rest of the post-ASAP rule engine, issues #6/#33) would +//! present to this pass in production. -use asap_aware_mapping::implement_tree; +use std::rc::Rc; + +use asap_aware_mapping::bind::{logical, ImplementError}; +use asap_aware_mapping::{ + Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG, +}; use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::post_asap::{ ExactKind, ExactParams, SketchKind, SketchParams, SketchQuery, SummaryExpr, SummaryFamilyType, - SummarySchema, + SummaryNode, SummarySchema, }; use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::query_expr::{QueryExpr, Reduction}; use asap_types::pre_asap::schema::{Column, DataType, Schema}; use asap_types::types::AccuracyTarget; +/// This crate has no "bind me one tree" public API any more — +/// `SketchFamilyStrategy::replacements` always returns every candidate, and +/// a caller decides what to keep. This test-only helper reproduces the +/// take-the-first-(`cost_model`-preferred)-candidate pattern so the +/// single-answer pins below don't all repeat it by hand. +fn bind(expr: &QueryExpr) -> Result, ImplementError> { + let root = Rc::new(expr.clone()); + let target = TargetSubDAG::new(&root); + match SketchFamilyStrategy::default_cost_model() + .replacements(&target) + .into_iter() + .next() + { + Some(ReplacementSubDAG { + replacement: Replacement::Summary(node), + .. + }) => Ok(node), + _ => logical(&root), + } +} + fn dtype<'a>(schema: &'a SummarySchema, name: &str) -> &'a SummaryFamilyType { &schema .fields @@ -87,9 +113,9 @@ fn inner_aggregate(qe: &QueryExpr) -> &QueryExpr { } /// Sanity + documentation: feeding a raw `lower_sql` root straight into -/// `implement_tree` never binds anything — the wrapping `Project` always -/// subsumes the `Aggregate` beneath it into one logical passthrough. This is -/// the "conservative fallback" `bind.rs`'s module docs describe, hitting +/// [`bind`] never binds anything — the wrapping `Project` always subsumes +/// the `Aggregate` beneath it into one logical passthrough. This is the +/// "conservative fallback" `bind.rs`'s module docs describe, hitting /// unconditionally for SQL because of the Project DataFusion always inserts. #[tokio::test] async fn sql_full_query_root_stays_logical_under_the_identity_projection() { @@ -102,11 +128,11 @@ async fn sql_full_query_root_stays_logical_under_the_identity_projection() { matches!(l3, QueryExpr::Project { .. }), "sanity: a SQL root is a Project, unlike lower_promql's bare Aggregate" ); - let root = implement_tree(&l3).expect("binding failed"); + let root = bind(&l3).expect("binding failed"); assert!( matches!(root.expr, SummaryExpr::Logical(ref e) if **e == l3), - "implement_tree does not look inside a Project to find a bindable \ - Aggregate child, so the whole Project{{Aggregate}} tree stays logical" + "bind does not look inside a Project to find a bindable Aggregate \ + child, so the whole Project{{Aggregate}} tree stays logical" ); } @@ -132,7 +158,7 @@ async fn sql_quantile_binds_kll_sketch_over_named_column() { ) .await; let agg = inner_aggregate(&l3); - let root = implement_tree(agg).expect("binding failed"); + let root = bind(agg).expect("binding failed"); let SummaryExpr::SummaryEstimate { summary_input, @@ -212,7 +238,7 @@ async fn sql_count_distinct_binds_hll_sketch_over_named_column() { ) .await; let agg = inner_aggregate(&l3); - let root = implement_tree(agg).expect("binding failed"); + let root = bind(agg).expect("binding failed"); let SummaryExpr::SummaryEstimate { summary_input, @@ -264,7 +290,7 @@ async fn sql_exact_workload_binds_accumulators_not_sketches() { ) .await; let agg = inner_aggregate(&l3); - let root = implement_tree(agg).expect("binding failed"); + let root = bind(agg).expect("binding failed"); let SummaryExpr::SummaryAgg { family, reduction, .. } = &root.expr @@ -288,7 +314,7 @@ async fn sql_exact_workload_binds_accumulators_not_sketches() { let l3 = lower("SELECT AVG(bytes) FROM metrics", AccuracyTarget::Exact).await; let agg = inner_aggregate(&l3); - let root = implement_tree(agg).expect("binding failed"); + let root = bind(agg).expect("binding failed"); assert!( matches!(root.expr, SummaryExpr::Logical(_)), "avg has no mergeable accumulator — stays logical" diff --git a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md index 91e7903d..a8e870d1 100644 --- a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md @@ -21,9 +21,9 @@ Code samples named `My*` or `Prefer*` (`MyStrategy`, `MyCostModel`, `PreferDDSke Two words come up constantly below and are worth pinning down before anything else, since neither is self-explanatory from context alone: - **`implementation`** (the module `implementation.rs`) — every valid way one `AggIntent` could be realized: as an approximate sketch, an exact mergeable accumulator, or a pass-through (no summary at all). `implementation::implementations_for_with` computes this **one node at a time**, exhaustive and ranked; it doesn't walk anything, and it doesn't pick a favorite — picking is left to its callers. -- **`bind` / "binding"** — this crate's own `bind.rs`: `implement_tree`/`implement_tree_with` walk a **whole** `QueryExpr` tree, and at each node, keep only the first (`cost_model`-preferred) entry from `implementation::implementations_for_with`'s list — the rest is a thin selector, not a second decision procedure (see §3). This is what "the binding path" means everywhere in this guide: the code path that commits to one `Implementation` per node because something has to actually execute. (The word "bind" means other things elsewhere in this crate's downstream consumers — see `lib.rs`'s own "Terminology" section if you need the full picture — but within this guide, "bind"/"binding" always means this.) +- **`bind` / "binding"** — this crate's own `bind.rs`. It has no "bind me one tree" entry point of its own: `replacement::SketchFamilyStrategy::replacements()` is the only public way to get bound output for a target, and it always returns *every* candidate; a caller that wants a single executable answer takes the first entry itself (see §3). What `bind.rs` provides is the shared low-level primitive, `bind_with_implementation`, that turns one already-decided candidate into a real `SummaryNode`, plus `implement_workload`/`implement_workload_with`, which drive that same take-the-head selection over a whole workload's roots (see §3's closing note on why those two still keep it internally). This is what "the binding path" means everywhere in this guide: the code that commits to one `Implementation` per node because something has to actually execute. (The word "bind" means other things elsewhere in this crate's downstream consumers — see `lib.rs`'s own "Terminology" section if you need the full picture — but within this guide, "bind"/"binding" always means this.) -So: `implementation` enumerates **every** way one node could become something; `bind`/`implement_tree` walks a whole tree, keeping only the first of each node's list; `ReplacementStrategy` (this guide's main subject) is the other consumer of that same list, keeping every entry instead of just the first — see §3 for exactly how these three connect. +So: `implementation` enumerates **every** way one node could become something; `ReplacementStrategy` (this guide's main subject) wraps that list into one `ReplacementSubDAG` per candidate, keeping all of them; a caller that wants one answer takes the first entry itself — see §3 for exactly how these connect. --- @@ -238,72 +238,54 @@ A custom cost model does not necessarily need to override every hook. The curren ## 3. How the current pieces fit together There is one place this crate decides what an `AggIntent` may become — -[`implementation::implementations_for_with`] — and two consumers of its -output that differ only in *how much of the list* they keep. Binding is -**literally implemented in terms of the replacement-strategy path**, not a -separate decision procedure that happens to agree with it: +[`implementation::implementations_for_with`] — and one place bound output +comes from. There is no separate "binding path" that computes its own +answer and happens to agree with the replacement-strategy path; there is +only the replacement-strategy path: - **`implementation::implementations_for_with(intent, cost_model)`** enumerates every valid `Implementation` for `intent`, exhaustive and ranked (most-preferred first via `cost_model`). - **`replacement::SketchFamilyStrategy::replacements()`** (§2) wraps that list directly: every entry becomes its own bound `ReplacementSubDAG`, none - discarded. -- **`bind::implement_tree_with`** builds a `TargetSubDAG` for the node being - bound, calls `SketchFamilyStrategy::replacements()`, and keeps only the - *first* (`cost_model`-preferred) candidate — everything else it throws - away. It does not compute an `Implementation` on its own. + discarded. This is the *only* public way to get bound output for a + target — there is no second, single-answer entry point sitting behind it. +- **A caller that wants one executable answer** takes the first + (`cost_model`-preferred) entry off `replacements()` itself + (`.into_iter().next()`) and decides what to do if the list comes back + empty (`bind::logical` is the same conservative pass-through fallback this + crate's own dispatch would otherwise use). This is exactly the "alternatives"/"candidates" language in [the design doc](../design_docs/asap_aware_mapping.md): a `ReplacementSubDAG` **is** one candidate; a `TargetSubDAG` with its full `replacements()` list **is** the set of alternatives for one spot in the plan. -**The tradeoff, stated plainly:** because binding now goes through the same enumeration `SketchFamilyStrategy` uses, an ordinary bind sizes and fully constructs *every* sketch candidate at every sketch-capable node — not just the one it keeps — before selecting the head. That's strictly more work per bind than a version that only ever computed the preferred candidate, in exchange for there being exactly one place in this crate that decides what an `AggIntent` may become. Recursion into a node's child goes back through the same enumerate-then-select step fresh, so choosing (or forcing) a candidate for one target never leaks into that target's own nested aggregates. +**The tradeoff, stated plainly:** because a single-target bind now goes through the same enumeration `SketchFamilyStrategy` uses, it sizes and fully constructs *every* sketch candidate at every sketch-capable node — not just the one a caller keeps — before that caller selects the head. That's strictly more work per bind than a version that only ever computed the preferred candidate, in exchange for there being exactly one place in this crate that decides what an `AggIntent` may become and exactly one place bound output comes from. Recursion into a node's child goes back through the same enumerate-then-select step fresh, so choosing (or forcing) a candidate for one target never leaks into that target's own nested aggregates. What this is *not*: the design doc's future Cascades/Volcano-style search engine — generate candidates via `ReplacementStrategy` across a *whole plan*, evaluate/select via `CostModel` across whole candidate plans — is a separate, not-yet-built piece of work. What exists today is a **single-node** stand-in for that selection (`cost_model.rank_candidates`, applied one node at a time), not the real thing. +**One exception:** `bind::implement_workload`/`implement_workload_with` still keep the take-the-head step internally, because workload-wide CSE sharing memoizes on `Rc` pointer identity — two workload roots that collapsed onto the same `Rc` must resolve to the *same* canonical decision to be shareable at all, so there's no meaningful "N candidates" answer to memoize against. That's the one place inside this crate a single-answer selection still lives; every other caller goes through `SketchFamilyStrategy::replacements()` directly. + ### The shared low-level primitive: `bind_with_implementation` -Underneath both `implement_tree_with` and `SketchFamilyStrategy::replacements()` sits one more function, `bind::bind_with_implementation(expr, implementation, cost_model)` — *given* an already-decided `Implementation` for `expr`'s top intent, bind it into a `SummaryNode` (or fall back to a logical passthrough). Neither caller re-decides how a chosen `Implementation` becomes a `SummaryNode`; they only differ in **which** `Implementation`(s) they hand it, and how many times: +Underneath `SketchFamilyStrategy::replacements()` sits one more function, `bind::bind_with_implementation(expr, implementation, cost_model)` — *given* an already-decided `Implementation` for `expr`'s top intent, bind it into a `SummaryNode` (or fall back to a logical passthrough). It doesn't re-decide how a chosen `Implementation` becomes a `SummaryNode`; `replacements()` just calls it once per candidate: ```text implementations_for_with(intent, cost_model) every valid Implementation, ranked | - +----------------+----------------+ - | | - SketchFamilyStrategy::replacements() bind::implement_tree_with - keeps every candidate keeps candidate[0] only - | | - v v - bind_with_implementation(...) bind_with_implementation(...) - once per candidate once, for the kept candidate - | | - v v - N ReplacementSubDAGs one bound SummaryNode -``` - -### Binding path + v + SketchFamilyStrategy::replacements() + keeps every candidate + | + v + bind_with_implementation(...) + once per candidate + | + v + N ReplacementSubDAGs -```text -AggIntent - | - v -implementation::implementations_for_with(...) - | - v -every ranked Implementation - | - v -keep candidate[0] - | - v -bind::bind_with_implementation(...) - | - v -one bound SummaryNode + a caller wanting one answer: replacements(...).into_iter().next() ``` ---- - ### Replacement-strategy path ```text @@ -611,9 +593,9 @@ fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { } ``` -`implementations_for_with` already did the hard part — enumerating and sizing every candidate, ranked. This loop's only job is to hand each one to `bind::bind_with_implementation(expr, implementation, cost_model)` — the same low-level primitive `bind::implement_tree_with` bottoms out in for whichever candidate it keeps (see §3) — and package the result. No per-candidate sizing logic lives here; there's nothing to keep in sync with the ordinary binding path, because both call the exact same enumeration function. +`implementations_for_with` already did the hard part — enumerating and sizing every candidate, ranked. This loop's only job is to hand each one to `bind::bind_with_implementation(expr, implementation, cost_model)` — the shared low-level primitive every caller of this crate bottoms out in, whether it keeps one candidate or all of them (see §3) — and package the result. No per-candidate sizing logic lives here. -Because only `expr`'s own top-level `Implementation` is ever supplied from outside — `bind_with_implementation` recurses into `expr`'s child via a fresh call back into the ordinary `implement_tree_with` selection, not a forced one — enumerating a candidate for one target never leaks into that target's own nested aggregates. (An earlier version of this strategy forced ranking via a `CostModel`-wrapping adapter for the whole recursive bind, which had exactly that leak as a latent bug; today's design doesn't have the problem, because nothing below the top node is ever forced.) +Because only `expr`'s own top-level `Implementation` is ever supplied from outside — `bind_with_implementation` recurses into `expr`'s child via a fresh internal selection over that child's own candidates, not a forced one — enumerating a candidate for one target never leaks into that target's own nested aggregates. (An earlier version of this strategy forced ranking via a `CostModel`-wrapping adapter for the whole recursive bind, which had exactly that leak as a latent bug; today's design doesn't have the problem, because nothing below the top node is ever forced.) This is the pattern to preserve when adding another strategy that needs to enumerate alternatives through an API that normally chooses one: get the exhaustive list from the same enumeration function the single-answer path uses, and hand each entry to the shared low-level binding primitive — never wrap the `CostModel` to trick a ranked-first path into producing what you want. @@ -1284,7 +1266,8 @@ Use this table to find the right place for a change. | Change shared-maintenance cost | `CostModel::cse_shared_maintenance_cost` | | Change current share/recompute choice | `CostModel::cse_share_decision` | | Decide whether an available implementation satisfies a required one | `impl Matcher` | -| Produce a normal (ranked-first) bound summary | reuse `bind::implement_tree_with` | +| Produce a normal (ranked-first) bound summary for one target | `SketchFamilyStrategy::replacements(...).into_iter().next()` | +| Bind a whole workload's roots, sharing across CSE-collapsed roots | reuse `bind::implement_workload`/`implement_workload_with` | | Bind a specific, already-chosen `Implementation` | reuse `bind::bind_with_implementation` | | Enumerate valid sketch kinds | reuse `implementation::summary_candidates` | | Build a target with no workload context | `TargetSubDAG::new` | diff --git a/docs/user-guide/user-guide.md b/docs/user-guide/user-guide.md index 301b9fa5..747e62c2 100644 --- a/docs/user-guide/user-guide.md +++ b/docs/user-guide/user-guide.md @@ -85,26 +85,44 @@ allowed, `Epsilon(e)` / `EpsilonDelta{epsilon, delta}` otherwise. ### Step 2 — get the post-ASAP IR Feed the `QueryExpr` to `asap-aware-mapping`. This crate depends only on `asap-types`, never on a -front end, so it's agnostic to which language produced the tree. +front end, so it's agnostic to which language produced the tree. There is no "bind me one tree" +entry point: `SketchFamilyStrategy::replacements()` always returns every valid candidate for a +target, ranked, and you take the one you want. ```rust -use asap_aware_mapping::implement_tree; - -let post_asap = implement_tree(&pre_asap)?; // Rc — the SummaryExpr DAG +use asap_aware_mapping::{Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG}; +use std::rc::Rc; + +let root = Rc::new(pre_asap); +let target = TargetSubDAG::new(&root); +let candidates = SketchFamilyStrategy::default_cost_model().replacements(&target); + +// Take the cost-model-preferred candidate — the common case. +let Some(ReplacementSubDAG { replacement: Replacement::Summary(post_asap), .. }) = + candidates.into_iter().next() +else { + // No candidate (e.g. the node isn't a bindable Aggregate) — fall back to + // `asap_aware_mapping::bind::logical(&root)`, the same conservative + // pass-through this crate's own dispatch uses. + panic!("no candidate for this target"); +}; +// post_asap: Rc — the SummaryExpr DAG ``` -Two entry points, both re-exported from the crate root: +`implementations_for_with(&AggIntent, &dyn CostModel) -> Vec` is the lower-level +enumeration `SketchFamilyStrategy` wraps, if you only need the per-node decision (sketch, exact +accumulator, or pass-through) without binding it into a `SummaryNode`. -- `implementation_for(&AggIntent) -> Implementation` — the single-node decision (sketch, exact - accumulator, or pass-through) for one aggregation. -- `implement_tree(&QueryExpr) -> Result, ImplementError>` — walks a whole tree, calling - the per-node decision at every `Aggregate` and emitting the full post-ASAP DAG. +`SketchFamilyStrategy::new(&dyn CostModel)` (vs. `default_cost_model()`) is the extension point for +a deployment that wants its own candidate ranking or parameter sizing instead of this crate's +built-in static preference order (`DefaultCostModel` — what `default_cost_model()` uses). +See the `CostModel` trait doc in `crates/asap-aware-mapping/src/cost_model.rs` for its overridable +hooks (`rank_candidates`, `size_params`, `realize_extension`, …). -Each has a `_with(..., &dyn CostModel)` variant. A `CostModel` is the extension point for a -deployment that wants its own candidate ranking or parameter sizing instead of this crate's -built-in static preference order (`DefaultCostModel` — what the plain, non-`_with` functions use). -See the `CostModel` trait doc in `crates/asap-aware-mapping/src/cost_model.rs` for its three -overridable hooks (`rank_candidates`, `size_params`, `realize_extension`). +To bind every root of a whole workload at once — sharing one bound `SummaryNode` across roots CSE +already collapsed onto the same `Rc` — use `asap_aware_mapping::implement_workload`/ +`implement_workload_with` instead; those two still keep only the cost-model-preferred candidate per +root, since sharing needs one canonical decision to key on. ### Reading the result From 94bb9678be7c72a5c46677afde91d45e2fe37801 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 24 Aug 2026 07:14:55 -0600 Subject: [PATCH 17/49] docs(asap-aware-mapping): explain the into_iter().next() take-first idiom MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A reader asked what the diagram's "a caller wanting one answer: replacements(...).into_iter().next()" line actually does — spell it out inline instead of assuming it's self-evident. --- docs/developer_docs/ASAP-aware-mapping-developer-guide.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md index a8e870d1..84f18481 100644 --- a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md @@ -286,6 +286,8 @@ Underneath `SketchFamilyStrategy::replacements()` sits one more function, `bind: a caller wanting one answer: replacements(...).into_iter().next() ``` +(`replacements(...)` returns a `Vec` ranked most-preferred-first; `.into_iter().next()` takes just that first entry and drops the rest — the same "keep candidate[0]" step `implement_tree_with` used to do internally, now written out explicitly on the calling side instead of hidden behind a second entry point.) + ### Replacement-strategy path ```text From 72bcb991de5188abb61b845a82dac033ae79ff34 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 24 Aug 2026 08:04:46 -0600 Subject: [PATCH 18/49] =?UTF-8?q?docs(asap-aware-mapping):=20readability?= =?UTF-8?q?=20rewrite=20of=20developer=20guide=20intro/=C2=A70-10?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Condenses the terminology, mental-model, CostModel-hooks, §3 data-flow, SharedSubtreeStrategy, and Matcher sections for readability while preserving all technical claims. Sections 11-18 are unchanged. Co-Authored-By: Claude Sonnet 5 --- .../ASAP-aware-mapping-developer-guide.md | 107 ++++++++++-------- 1 file changed, 57 insertions(+), 50 deletions(-) diff --git a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md index 84f18481..03637a1b 100644 --- a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md @@ -1,29 +1,29 @@ # ASAP-Aware Mapping: Developer Guide -This document explains how to extend ASAP-aware mapping in the current codebase. - -It is written for developers who want to: +This guide explains how to extend ASAP-aware mapping in the current codebase. Use it when you want to: - add a new `ReplacementStrategy`, - add or customize a `CostModel`, - understand how strategies, binding, and costing interact, - add a new kind of replacement without duplicating existing planner logic, -- and write the tests expected for a new extension. +- write the tests expected for a new extension. The focus here is the **current code interfaces and their contracts**. For the higher-level motivation and future search design, see the separate design document, [`docs/design_docs/asap_aware_mapping.md`](../design_docs/asap_aware_mapping.md). -Code samples named `My*` or `Prefer*` (`MyStrategy`, `MyCostModel`, `PreferDDSketch`, …) below are illustrative sketches of a pattern, not code that ships in this crate. Samples that name a real type (`SketchFamilyStrategy`, `SharedSubtreeStrategy`, `bind_with_implementation`, …) are copied verbatim from `replacement.rs`/`bind.rs`/`cost_model.rs`. +Names such as `MyStrategy`, `MyCostModel`, and `PreferDDSketch` are illustrative; they do not ship with this crate. Samples that use real types—such as `SketchFamilyStrategy`, `SharedSubtreeStrategy`, and `bind_with_implementation`—are copied from `replacement.rs`, `bind.rs`, or `cost_model.rs`. + +If you only need to find the right extension point, start with the [extension map](#18-current-extension-map). If you are implementing a strategy, read sections 1–5 first. --- ## Terminology -Two words come up constantly below and are worth pinning down before anything else, since neither is self-explanatory from context alone: +Two terms are central to this guide: -- **`implementation`** (the module `implementation.rs`) — every valid way one `AggIntent` could be realized: as an approximate sketch, an exact mergeable accumulator, or a pass-through (no summary at all). `implementation::implementations_for_with` computes this **one node at a time**, exhaustive and ranked; it doesn't walk anything, and it doesn't pick a favorite — picking is left to its callers. -- **`bind` / "binding"** — this crate's own `bind.rs`. It has no "bind me one tree" entry point of its own: `replacement::SketchFamilyStrategy::replacements()` is the only public way to get bound output for a target, and it always returns *every* candidate; a caller that wants a single executable answer takes the first entry itself (see §3). What `bind.rs` provides is the shared low-level primitive, `bind_with_implementation`, that turns one already-decided candidate into a real `SummaryNode`, plus `implement_workload`/`implement_workload_with`, which drive that same take-the-head selection over a whole workload's roots (see §3's closing note on why those two still keep it internally). This is what "the binding path" means everywhere in this guide: the code that commits to one `Implementation` per node because something has to actually execute. (The word "bind" means other things elsewhere in this crate's downstream consumers — see `lib.rs`'s own "Terminology" section if you need the full picture — but within this guide, "bind"/"binding" always means this.) +- **Implementation**: one valid realization of an `AggIntent`. It may be an approximate sketch, an exact mergeable accumulator, or a pass-through with no summary. `implementation::implementations_for_with` enumerates the valid implementations for one node and ranks them. It does not walk the plan or select a winner. +- **Binding**: converting a chosen `Implementation` into an executable `SummaryNode`. The shared primitive is `bind_with_implementation`. For one target, `SketchFamilyStrategy::replacements()` binds every candidate. A caller that needs one result takes the first candidate. Workload-wide helpers—`implement_workload` and `implement_workload_with`—perform that selection internally so shared `Rc` roots receive one consistent decision. -So: `implementation` enumerates **every** way one node could become something; `ReplacementStrategy` (this guide's main subject) wraps that list into one `ReplacementSubDAG` per candidate, keeping all of them; a caller that wants one answer takes the first entry itself — see §3 for exactly how these connect. +In short: `implementation` enumerates, `ReplacementStrategy` packages and binds every candidate, and the caller selects when it needs a single executable answer. Section 3 shows the complete flow. --- @@ -46,7 +46,7 @@ A cost model should answer: > Given valid choices, which choices are preferable, and how should they be parameterized? -Do not put cost-based pruning into a `ReplacementStrategy`. A strategy should enumerate valid alternatives even when one of them is obviously more expensive under the default cost model. (This is the single most important rule in this guide — later sections restate it in context rather than re-arguing it; see §5 Rule 2 for the canonical statement.) +Do not put cost-based pruning into a `ReplacementStrategy`. A strategy must enumerate every valid alternative, even when the default cost model clearly prefers one. See [Rule 2](#rule-2-enumerate-do-not-rank). --- @@ -65,7 +65,14 @@ pub struct TargetSubDAG<'a> { `root` is the actual `Rc` from the workload. -`consumer_count` counts **structural references**, not runtime reads/executions: how many places in the workload's `Rc` DAG point at this exact `root` node. Example — two separate top-level queries, `sum by (service) (rate(m[5m]))` and `avg by (service) (rate(m[5m]))`, share an identical `rate(m[5m])` subtree; after CSE (`share_common_subtrees`) collapses that into one `Rc`, both queries' trees point at the same node, so its `TargetSubDAG.consumer_count` is `2` — regardless of how many times either query actually executes. +`consumer_count` counts structural references, not runtime executions. It is the number of places in the workload DAG that point to this exact `Rc` node. + +For example, consider two top-level queries: + +- `sum by (service) (rate(m[5m]))` +- `avg by (service) (rate(m[5m]))` + +After `share_common_subtrees` merges their identical `rate(m[5m])` subtrees, both query trees point to the same `Rc`. That node's `consumer_count` is `2`, regardless of how often either query executes. Use: @@ -171,23 +178,35 @@ The two methods have intentionally different responsibilities. ### `CostModel` -`CostModel`'s job is broader than its name suggests — it's the one extension point every deployment-specific numeric/configuration decision plugs into, not just "which candidate is cheapest." Sizing counts as a cost decision for the same reason ranking does: a bigger sketch (more memory, more update cost) buys more accuracy, and how you want to spend that budget is exactly the kind of real-world cost knowledge this crate deliberately doesn't hardcode — see the crate doc's layering note at the top of `cost_model.rs` (`asap-plan` depends only on `asap_ir`, never on a runtime or deployment model, so it can't know real costs itself). Every hook but one has a default body that reproduces this crate's built-in static behavior — override only what you need to change: +`CostModel` covers every deployment-specific numeric or configuration decision—not only which candidate is cheapest. For example, sketch sizing trades memory and update cost for accuracy, so it belongs here too. + +The crate cannot hardcode real deployment costs: `asap-plan` depends on `asap_ir`, not on a runtime or deployment model. Most hooks therefore provide the crate's built-in static behavior as a default. Override only the decisions your deployment needs to change. + +| Hook | Use it to | Default? | +|---|---|---| +| `rank_candidates` | Order valid sketch families | No | +| `size_params` | Convert an accuracy target into sketch parameters | Yes | +| `realize_extension` | Map a custom intent to an implementation | Yes | +| `readout_extension` | Query a custom extension summary | Panics until paired with a custom realization | +| `cse_recompute_cost` | Estimate independent recomputation | Yes | +| `cse_shared_maintenance_cost` | Estimate shared maintenance | Yes | +| `cse_share_decision` | Choose sharing or recomputation | Yes | -- **`rank_candidates`** — order a set of sketch candidates for one `AggIntent`, best choice first. **No default** — this is the one hook every `CostModel` must implement; candidate selection needs a real answer from somewhere. +- **`rank_candidates`** — order the sketch candidates for one `AggIntent`, best first. This is the only required hook. ```rust fn rank_candidates(&self, intent: &AggIntent, candidates: &[SketchKind]) -> Vec; ``` -- **`size_params`** — pick concrete parameters (e.g. sketch capacity) for one already-chosen `SketchKind`, given an accuracy target `(eps, delta)`. This is a separate hook from `rank_candidates` specifically so a deployment can override *just* sizing (e.g. an empirically-tuned table, or discrete capacity rungs a downstream catalog requires) without also forking candidate selection — same "one extension point per decision" shape as everything else in this trait. Default: `implementation::default_size_params`, this crate's built-in per-family sizing formulas. +- **`size_params`** — choose parameters, such as sketch capacity, for an already-selected `SketchKind` and accuracy target `(eps, delta)`. It is separate from ranking so a deployment can customize sizing without changing family selection. The default is `implementation::default_size_params`. ```rust fn size_params(&self, kind: SketchKind, intent: &AggIntent, eps: f64, delta: f64) -> SketchParams; ``` -- **`realize_extension`** — decide what post-ASAP `Implementation` a deployment-defined `AggIntent::Extension` maps to (a shape core has no built-in opinion on). Default: `Implementation::PassThrough`. +- **`realize_extension`** — map a deployment-defined `AggIntent::Extension` to a post-ASAP `Implementation`. The default is `Implementation::PassThrough`. - `AggIntent::Extension { ext_kind: String, payload: serde_json::Value }` is the escape hatch for an intent shape only *your* deployment needs — core's `AggIntent` enum deliberately doesn't grow a variant for every capability a single deployment wants (issue #131), so instead a deployment tags its own shape with a `ext_kind` string and puts whatever it needs in `payload`; core treats both opaquely. Example: a deployment wants an approximate-frequency intent core has no variant for. It tags queries with `Extension { ext_kind: "frequency".into(), payload: ... }`, then overrides `realize_extension` to recognize that tag: + Use `AggIntent::Extension { ext_kind, payload }` for intent shapes that only your deployment needs. Core treats both fields as opaque. For example, a deployment can tag an approximate-frequency intent with `ext_kind: "frequency"` and recognize it in `realize_extension`: ```rust fn realize_extension(&self, ext_kind: &str, _payload: &serde_json::Value) -> Implementation { @@ -199,13 +218,13 @@ The two methods have intentionally different responsibilities. } ``` - Every `ext_kind` your deployment doesn't recognize should still fall through to `Implementation::PassThrough` (the default), not panic — only the `ext_kind`s you've deliberately implemented get a real realization. + Return `Implementation::PassThrough` for unrecognized extension kinds. Do not panic. ```rust fn realize_extension(&self, ext_kind: &str, payload: &serde_json::Value) -> Implementation; ``` -- **`readout_extension`** — build the query-time read for an `Extension` intent this same `CostModel` already realized as `Sketch` via `realize_extension`. "Readout" means the query-time **read** path — how an already-bound summary answers a query — as distinct from `realize_extension`, which decides how the summary is **built and maintained**; the two are a matched pair, so overriding `realize_extension` to return `Sketch` for some `ext_kind` requires overriding this for that same `ext_kind` too. Default: panics (deliberately — a silent wrong answer here is worse than a loud crash) if that pairing was left incomplete. +- **`readout_extension`** — define how queries read an extension summary that `realize_extension` mapped to a `Sketch`. The two hooks are a pair: realization defines what is maintained; readout defines how it is queried. Override both for the same `ext_kind`. The default readout panics to prevent a silent wrong answer. ```rust fn readout_extension(&self, ext_kind: &str, payload: &serde_json::Value, col: &ColumnRef) -> SketchQuery; @@ -223,9 +242,9 @@ The two methods have intentionally different responsibilities. fn cse_shared_maintenance_cost(&self, candidate: &CseCandidate) -> Cost; ``` - Both hooks return `Cost`, a newtype around `f64` rather than a bare `f64` — a deliberately minimal wrapper today (still one unitless scalar, `Cost(f64)`, comparable via `PartialOrd`/`Add`/`Mul`), but one that gives a future richer cost type (e.g. separate CPU/memory/network fields) a place to grow into without changing every hook's signature a second time. + Both hooks return `Cost`, currently a unitless `f64` newtype. The wrapper allows the type to grow later—for example, to separate CPU, memory, and network cost—without changing every hook signature. -- **`cse_share_decision`** — decide whether to share one summary across all of a CSE candidate's consumers, or recompute independently at each. Default: composes the two cost hooks above (share iff shared-maintenance cost ≤ total recompute cost across every consumer) — override the two cost hooks and keep this comparison unless you need a genuinely different policy, not a different cost input. +- **`cse_share_decision`** — choose between one shared summary and independent recomputation at each consumer. By default, it shares when maintenance cost is no greater than total recomputation cost. Override the two cost inputs first; override this decision hook only when you need a different policy. ```rust fn cse_share_decision(&self, candidate: &CseCandidate) -> ShareDecision; @@ -237,36 +256,23 @@ A custom cost model does not necessarily need to override every hook. The curren ## 3. How the current pieces fit together -There is one place this crate decides what an `AggIntent` may become — -[`implementation::implementations_for_with`] — and one place bound output -comes from. There is no separate "binding path" that computes its own -answer and happens to agree with the replacement-strategy path; there is -only the replacement-strategy path: +The crate has one source of truth for valid implementations and one public path for producing bound candidates: -- **`implementation::implementations_for_with(intent, cost_model)`** enumerates - every valid `Implementation` for `intent`, exhaustive and ranked - (most-preferred first via `cost_model`). -- **`replacement::SketchFamilyStrategy::replacements()`** (§2) wraps that list - directly: every entry becomes its own bound `ReplacementSubDAG`, none - discarded. This is the *only* public way to get bound output for a - target — there is no second, single-answer entry point sitting behind it. -- **A caller that wants one executable answer** takes the first - (`cost_model`-preferred) entry off `replacements()` itself - (`.into_iter().next()`) and decides what to do if the list comes back - empty (`bind::logical` is the same conservative pass-through fallback this - crate's own dispatch would otherwise use). +- `implementation::implementations_for_with(intent, cost_model)` returns every valid `Implementation`, ranked by preference. +- `replacement::SketchFamilyStrategy::replacements()` binds each entry and returns one `ReplacementSubDAG` per candidate. It does not discard any candidate. +- A caller that needs one executable answer uses `.into_iter().next()` and handles the empty case. The crate's conservative fallback is `bind::logical`. -This is exactly the "alternatives"/"candidates" language in [the design doc](../design_docs/asap_aware_mapping.md): a `ReplacementSubDAG` **is** one candidate; a `TargetSubDAG` with its full `replacements()` list **is** the set of alternatives for one spot in the plan. +In the [design document](../design_docs/asap_aware_mapping.md), each `ReplacementSubDAG` is a candidate. The complete `replacements()` result is the set of alternatives for one location in the plan. -**The tradeoff, stated plainly:** because a single-target bind now goes through the same enumeration `SketchFamilyStrategy` uses, it sizes and fully constructs *every* sketch candidate at every sketch-capable node — not just the one a caller keeps — before that caller selects the head. That's strictly more work per bind than a version that only ever computed the preferred candidate, in exchange for there being exactly one place in this crate that decides what an `AggIntent` may become and exactly one place bound output comes from. Recursion into a node's child goes back through the same enumerate-then-select step fresh, so choosing (or forcing) a candidate for one target never leaks into that target's own nested aggregates. +**Tradeoff:** a single-target bind sizes and constructs every valid sketch candidate before the caller keeps the first one. This costs more than constructing only the preferred candidate, but it preserves one implementation-enumeration path and one binding path. Child nodes repeat selection independently, so a choice at one target does not force choices in nested aggregates. -What this is *not*: the design doc's future Cascades/Volcano-style search engine — generate candidates via `ReplacementStrategy` across a *whole plan*, evaluate/select via `CostModel` across whole candidate plans — is a separate, not-yet-built piece of work. What exists today is a **single-node** stand-in for that selection (`cost_model.rank_candidates`, applied one node at a time), not the real thing. +This is not yet the whole-plan Cascades/Volcano-style search described in the design document. Today, `cost_model.rank_candidates` ranks one node at a time. Whole-plan candidate generation and selection remain future work. -**One exception:** `bind::implement_workload`/`implement_workload_with` still keep the take-the-head step internally, because workload-wide CSE sharing memoizes on `Rc` pointer identity — two workload roots that collapsed onto the same `Rc` must resolve to the *same* canonical decision to be shareable at all, so there's no meaningful "N candidates" answer to memoize against. That's the one place inside this crate a single-answer selection still lives; every other caller goes through `SketchFamilyStrategy::replacements()` directly. +**One exception:** `bind::implement_workload` and `implement_workload_with` select the first candidate internally. Workload-wide CSE memoizes by `Rc` pointer identity, so roots that share an `Rc` must use the same canonical decision. Other callers use `SketchFamilyStrategy::replacements()` directly. ### The shared low-level primitive: `bind_with_implementation` -Underneath `SketchFamilyStrategy::replacements()` sits one more function, `bind::bind_with_implementation(expr, implementation, cost_model)` — *given* an already-decided `Implementation` for `expr`'s top intent, bind it into a `SummaryNode` (or fall back to a logical passthrough). It doesn't re-decide how a chosen `Implementation` becomes a `SummaryNode`; `replacements()` just calls it once per candidate: +`bind::bind_with_implementation(expr, implementation, cost_model)` converts an already-selected top-level `Implementation` into a `SummaryNode`, or falls back to a logical pass-through. `replacements()` calls it once per candidate: ```text implementations_for_with(intent, cost_model) @@ -286,7 +292,7 @@ Underneath `SketchFamilyStrategy::replacements()` sits one more function, `bind: a caller wanting one answer: replacements(...).into_iter().next() ``` -(`replacements(...)` returns a `Vec` ranked most-preferred-first; `.into_iter().next()` takes just that first entry and drops the rest — the same "keep candidate[0]" step `implement_tree_with` used to do internally, now written out explicitly on the calling side instead of hidden behind a second entry point.) +`replacements(...)` returns candidates in preferred-first order. `.into_iter().next()` keeps the first and drops the rest. This makes the selection explicit at the call site. ### Replacement-strategy path @@ -636,10 +642,11 @@ Replacement::Rewrite( This strategy does **not** decide whether sharing is cheaper. -That decision belongs to the cost model — and today, `CostModel::cse_share_decision`'s default body already makes it, via a real cost comparison (§9), every time the *production binding path* (§3) sees a shared subtree. So it's fair to ask: isn't "build independently" (#2) already the exception, not the default — and doesn't that comparison already decide things? It's worth being precise about what "default" means here: +That choice belongs to the cost model. The production binding path calls `CostModel::cse_share_decision` when it encounters a shared subtree. The strategy still returns both alternatives because enumeration and selection are separate steps: -- `target.root` arriving with `consumer_count >= 2` means CSE detection (`share_common_subtrees`) already merged it onto one shared `Rc` *before* this strategy ever runs — so "build once and share" (#1) costs nothing to produce (`Rc::clone` of what's already there); "build independently" (#2) is the one requiring real work (an actual deep clone). -- But `cse_share_decision`'s cost comparison is a *separate, already-existing* mechanism that only the production binding path consults — it isn't something `SharedSubtreeStrategy` calls or defers to. This strategy has to enumerate **both** regardless of which one is cheap to construct or which one that other comparison would currently pick, because per §1's core rule, a strategy is never allowed to pre-decide based on cost — collapsing to just #1 here would be exactly the "cost-based pruning inside a strategy" anti-pattern §5 Rule 2 forbids, even though today's separate binding path already has an answer. The whole reason this module exists (§0) is to keep both alternatives around for a future search to compare *in the context of a full plan* — which might disagree with `cse_share_decision`'s isolated, pairwise-only comparison. +- `consumer_count >= 2` means `share_common_subtrees` has already merged the expression into one shared `Rc`. The shared alternative is therefore an `Rc::clone`; the independent alternative requires a deep clone. +- `cse_share_decision` is used by the production binding path, not by `SharedSubtreeStrategy`. +- The strategy must return both valid alternatives even if the current cost model strongly prefers one. A future whole-plan search may choose differently from today's local comparison. This example is useful when implementing transformations such as: @@ -844,7 +851,7 @@ The replacement-strategy layer should still expose both valid alternatives where ## 10. `Matcher` (`implementation.rs`) -There is a third extension point in this crate, much smaller in surface area than `ReplacementStrategy` or `CostModel`, but worth knowing about: +`Matcher` is a smaller, separate extension point: ```rust pub trait Matcher { @@ -852,11 +859,11 @@ pub trait Matcher { } ``` -`Matcher` answers a different question than everything above it: not "how do I *build* a summary for this intent" (that's `implementation`/`bind`/`ReplacementStrategy`), but "is there already a summary sitting around somewhere that answers this without building anything new?" This is the query-optimization-literature "answering queries using views" question, applied to summaries — the same idea as a database reusing an existing materialized view or index instead of recomputing from scratch. +`Matcher` does not decide how to build a summary. It asks whether an existing summary can satisfy a required implementation without building anything new. This is similar to a database reusing a materialized view or index. -Concrete example: a deployment already has a `DDSketch` built earlier for some other query on `latency`, and a new query needs `Kll` quantiles on `latency`. Can the existing `DDSketch` answer it without building a new `Kll` sketch? A pure sketch-algebra answer says yes — both are quantile sketches, mutually substitutable at the query level. But a deployment with its own storage-layout rules might say no, e.g. if its inventory ties a stored summary to a specific algorithm identity it won't re-interpret. `Matcher::is_satisfied_by(required, available)` is where a deployment plugs in whichever answer is actually true for its own storage layer — `required` is what a query needs (an `Implementation` value that `implementation`/`bind` already computed), `available` is what already exists somewhere in that deployment's inventory. +For example, suppose a deployment already has a `DDSketch` for `latency`, while a new query requests KLL quantiles. Sketch algebra may allow substitution because both answer quantile queries. A deployment's storage rules may forbid it because stored summaries retain a specific algorithm identity. `Matcher::is_satisfied_by(required, available)` lets the deployment make that decision: `required` is what the query needs, and `available` is what the inventory already contains. -Unlike `CostModel`, this crate ships **no default body and no implementation** for `Matcher` at all — the answer genuinely depends on facts this crate deliberately doesn't settle (see the example above: a pure sketch-algebra answer and a deployment's own storage-layout rules can legitimately disagree, and this crate has no inventory concept of its own to judge between them). If you need this, you're implementing the whole trait for your deployment from scratch; there's no default to lean on and no existing implementation in this crate to copy from. +The crate provides no default `Matcher` implementation because the answer depends on deployment-specific inventory and storage rules. If you need one, implement the complete trait for your deployment. --- From e4451494f0a3c9fbba11b6e86e0cfe59636a49cf Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 24 Aug 2026 08:09:21 -0600 Subject: [PATCH 19/49] docs(user-guide): readability rewrite of the no-code asap-devtools walkthrough Restructures the no-code section into numbered steps plus 'Other useful commands'/'Additional examples', and fixes the dag-viewer RUNNING.md relative link (was missing a '../' level: docs/user-guide/ -> tools/ needs two levels up, not one). The 'As a library' section is unchanged. Co-Authored-By: Claude Sonnet 5 --- docs/user-guide/user-guide.md | 84 +++++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 23 deletions(-) diff --git a/docs/user-guide/user-guide.md b/docs/user-guide/user-guide.md index 747e62c2..f5d755f0 100644 --- a/docs/user-guide/user-guide.md +++ b/docs/user-guide/user-guide.md @@ -1,51 +1,89 @@ # User guide: getting the pre-ASAP and post-ASAP IR for a query -Two ways to do this: no-code (the `asap-devtools` binaries) or as a library embedded in your own -codebase. +The `asap-devtools` package provides command-line tools for inspecting the pre-ASAP and post-ASAP IR generated for SQL and PromQL queries. -## No-code: `asap-devtools` binaries +## 1. Create a query file -Write a file of `sql>`/`promql>`-prefixed queries (one per line; blank lines and `#` comments -ignored): +Create a text file containing one query per line. Prefix each query with `sql>` or `promql>`. + +For example, `queries.txt`: ```text promql> quantile(0.99, rate(http_requests_total[5m])) sql> SELECT service, COUNT(*) FROM metrics GROUP BY service ``` -**Pre-ASAP IR** (ASAP-agnostic `QueryExpr`/`AggIntent`): +Blank lines and lines beginning with `#` are ignored. + +## 2. Show the pre-ASAP IR + +Run: ```sh cargo run -p asap-devtools --bin show_pre_asap_ir -- queries.txt ``` -**Post-ASAP IR** (ASAP-aware IR with `SummaryExpr`/`L4Node`, one layer downstream, with -concrete `SummaryKind`/`SummaryParams` committed per aggregate): +You can also provide the queries through stdin: + +```sh +cargo run -p asap-devtools --bin show_pre_asap_ir < queries.txt +``` + +## 3. Show the post-ASAP IR + +Run: ```sh cargo run -p asap-devtools --bin show_post_asap_ir -- queries.txt ``` -Both accept stdin instead of a file (`... < queries.txt`). `show_post_asap_ir` lowers at -ε = 0.01 rather than `Exact` — an exact target only ever exercises the mergeable-accumulator -arm of the binding decision, never a real sketch, so it wouldn't show the interesting case. +Or through stdin: + +```sh +cargo run -p asap-devtools --bin show_post_asap_ir < queries.txt +``` + +`show_post_asap_ir` uses an approximation target of ε = 0.01 so that the output can exercise sketch-based implementations rather than only exact aggregation. + +## Other useful commands + +### Export a query DAG + +Export pre-ASAP IR for SQL or PromQL queries for use with the interactive DAG viewer: + +```sh +cargo run -p asap-devtools --bin dag_export -- --sql "" +``` + +or: + +```sh +cargo run -p asap-devtools --bin dag_export -- --promql "" +``` + +See [`tools/dag-viewer/RUNNING.md`](../../tools/dag-viewer/RUNNING.md) for instructions on running the DAG viewer. + +### Check IR variant coverage -### Other dev tools +Parse the query corpora in the repository and report which pre-ASAP IR variants are exercised: -`crates/devtools` ships more debugging binaries and examples for poking at the lowering pipeline. Binaries (`cargo run -p asap-devtools --bin `): +```sh +cargo run -p asap-devtools --bin variant_coverage +``` + +## Additional examples + +Print pre-ASAP IR for several top-k queries: -- **`show_pre_asap_ir`** — see above -- **`show_post_asap_ir`** — see above -- **`dag_export`** — dumps pre-ASAP IRs for given `--sql`/`--promql` queries for - [`tools/dag-viewer`](../tools/dag-viewer/index.html), an interactive DAG viewer - (see [`tools/dag-viewer/RUNNING.md`](../tools/dag-viewer/RUNNING.md) for - end-to-end setup, including running it over a remote tunnel). -- **`variant_coverage`** — parses and canonicalizes every query corpus in the repo to pre-ASAP IR and reports which `QueryExpr` variants get exercised. +```sh +cargo run -p asap-devtools --example topk_ir +``` -Examples (`cargo run -p asap-devtools --example `): +Print representative queries covering the pre-ASAP IR variants: -- **`topk_ir`** — prints pre-ASAP IR for a hardcoded set of topk-shaped SQL/PromQL queries -- **`canonical_examples`** — prints pre-ASAP IR for one canonical query per `QueryExpr` variant, and custom join/set-op/distinct/CTE probes, to eyeball their shape. +```sh +cargo run -p asap-devtools --example canonical_examples +``` ## As a library From 728d5d6f089b49e55bf5f5aca09206072ca079b5 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 24 Aug 2026 09:55:06 -0600 Subject: [PATCH 20/49] refactor(asap-aware-mapping): merge implementation.rs into replacement.rs, delete bind_with_implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why Review of #259 flagged that the dev guide's mental model and glossary (Sec 1/2) only describe TargetSubDAG -> ReplacementStrategy -> ReplacementSubDAG, CostModel ranks/sizes -- but the actual code had a third, undocumented layer in between: implementation.rs's Implementation enum + implementations_for_with, bridged into a real SummaryNode by bind::bind_with_implementation. That bridge function wasn't part of Sec 1/2's vocabulary at all, so it read as unexplained extra machinery, and it wasn't clear from the design docs why binding needed a second decision procedure alongside the replacement-strategy one. ## What implementations_for_with already produces a fully-decided value -- Implementation::Sketch{kind, params} already has params sized to the target's accuracy via CostModel::size_params. Everything bind_with_implementation/bind_summary_agg did after that (derive child schema, resolve the summarized column, build the readout, recurse into the child, construct the SummaryNode) is mechanical construction, not a second decision -- it has to run regardless of how (kind, params) were chosen, so there's no real justification for a separate, differently-named, differently-modularized bridge function sitting between "decide" and "here's your candidate." - implementation.rs's content (Implementation enum, Matcher trait, implementations_for_with, summary_candidates, accuracy_budget, default_size_params, posterior_aware_size_params, the sizing formulas) moves directly into replacement.rs -- not a submodule, flattened into the same namespace as TargetSubDAG/Replacement/ ReplacementSubDAG/ReplacementStrategy. implementation.rs is deleted. implementations_for_with itself drops off the crate's public surface (module-private now) since SketchFamilyStrategy is its only caller. - bind::bind_with_implementation is deleted. Its body (renamed construct_summary) and helpers (summary_family, construct_summary_agg, summary_col_index, summarised_column, readout, lift) move into replacement.rs, called directly from SketchFamilyStrategy::replacements()'s loop. - bind.rs shrinks to: bindable_intent (moved to replacement.rs), select_and_bind (pub(crate), the "take first candidate" helper used for recursion into a child and by workload-level binding), implement_workload/implement_workload_with (public entry points, unchanged signatures), and logical. ## How This is a relocation plus one deletion, not a behavior change: the exact same functions, doing the exact same work, just moved between files -- verified by running the full existing test suite unchanged (53/53 asap-aware-mapping tests pass byte-for-byte identical, same candidates, same SummaryNode output, same rationale strings). The "core algorithm" isn't new; what changed is that SketchFamilyStrategy::replacements() now both decides (enumerate every valid Implementation, ranked and sized) and constructs (turn each one into a real SummaryNode) in one method, instead of the decide step living in one module and the construct step living in another, bridged by a public function whose existence Sec 1/2 never explained. lib.rs, cost_model.rs, and the developer guide were updated to match (Implementation/Matcher/summary_candidates/default_size_params now live under replacement::, not implementation::; the dev guide's §3 diagram and §6 walkthrough no longer show a separate bind_with_implementation box). ## Output $ cargo test -p asap-aware-mapping 2>&1 | tail -5 test result: ok. 53 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out $ cargo build --workspace --all-targets && cargo fmt --all -- --check \ && cargo clippy --workspace --all-targets --all-features -- -D warnings Finished `dev` profile [unoptimized + debuginfo] target(s) (fmt: no output) Finished `dev` profile [unoptimized + debuginfo] target(s) $ grep -rln 'bind_with_implementation\|crate::implementation::' --include='*.rs' --include='*.md' . (no output -- zero stale references left in the tree) Co-Authored-By: Claude Sonnet 5 --- crates/asap-aware-mapping/src/bind.rs | 318 +--- crates/asap-aware-mapping/src/cost_model.rs | 34 +- .../asap-aware-mapping/src/implementation.rs | 1067 ------------ crates/asap-aware-mapping/src/lib.rs | 63 +- crates/asap-aware-mapping/src/replacement.rs | 1433 ++++++++++++++++- .../ASAP-aware-mapping-developer-guide.md | 69 +- 6 files changed, 1452 insertions(+), 1532 deletions(-) delete mode 100644 crates/asap-aware-mapping/src/implementation.rs diff --git a/crates/asap-aware-mapping/src/bind.rs b/crates/asap-aware-mapping/src/bind.rs index d64619e2..a0ea5b0c 100644 --- a/crates/asap-aware-mapping/src/bind.rs +++ b/crates/asap-aware-mapping/src/bind.rs @@ -1,45 +1,27 @@ -//! The pre-ASAP → post-ASAP binding primitives (issue #98). +//! Workload-level pre-ASAP → post-ASAP binding orchestration (issue #98). //! -//! This module does **not** expose a "bind me one tree" entry point. -//! [`replacement::SketchFamilyStrategy`](crate::replacement::SketchFamilyStrategy) -//! is the only public way to get bound output for a target — it always -//! returns every candidate [`ReplacementSubDAG`](crate::replacement::ReplacementSubDAG), +//! This module does **not** expose a "bind me one tree" entry point for a +//! single target. [`crate::replacement::SketchFamilyStrategy`] is the only +//! public way to get bound output for one target — it always returns every +//! candidate [`ReplacementSubDAG`](crate::replacement::ReplacementSubDAG), //! ranked; a caller that wants a single executable answer takes the first //! entry itself (`.into_iter().next()`) and decides what to do if the list -//! is empty (this module's [`logical`] is the same pass-through fallback -//! this crate's own dispatch would otherwise use). What this module -//! provides is the shared low-level primitive that turns one *already-decided* -//! candidate into a real [`SummaryNode`]: +//! is empty ([`logical`] is the same pass-through fallback this crate's own +//! dispatch would otherwise use — see `crate::replacement`'s own module docs +//! for how deciding *and* constructing each candidate both happen inside +//! that one strategy). //! -//! - **Summary family** (sketch / sample / wavelet / statistical model) — -//! the `Aggregate` becomes a [`SummaryExpr::SummaryAgg`] carrying the -//! committed `family: SummaryFamilyType` (that family's own -//! `(kind, params)`), wrapped in a [`SummaryExpr::SummaryEstimate`] that -//! reads the answer back out (the summary-state column type does not -//! propagate past the estimate); -//! - **Exact accumulator** — a `SummaryAgg` with `family: -//! SummaryFamilyType::ExactAggregate` (`Sum`/`Count`/`MinMax`/`Rate`/ -//! `Increase`) and no estimate: the partial state *is* the value, so a -//! deployment's later finalization step is the identity; -//! - **Pass-through** — the whole pre-ASAP subtree is wrapped as -//! [`SummaryExpr::Logical`], schema lifted with every column -//! `SummaryFamilyType::Plain`. +//! What this module *does* provide is workload-wide orchestration: //! -//! Construction recurses through the `Aggregate` spine, so nested aggregates -//! each get their own independent candidate enumeration and selection -//! (`quantile(0.9, sum by (svc) (rate(m[5m]))))` binds KLL over an exact -//! `Sum` accumulator over an exact `Rate` accumulator) — see -//! [`bind_with_implementation`]. -//! -//! ## Why [`implement_workload`]/[`implement_workload_with`] are still here +//! ## Why [`implement_workload`]/[`implement_workload_with`] are here //! //! Workload-level CSE sharing ([`asap_types::pre_asap::cse::share_common_subtrees`]) //! memoizes on `Rc` pointer identity: two workload roots that collapsed onto //! the same `Rc` must resolve to the *same* canonical decision to //! be shareable at all — there is no meaningful "N candidates" answer to -//! memoize against. So this one entry point keeps the old -//! rank-and-take-first behavior internally (via a private selector), scoped -//! to workload-wide CSE memoization specifically. It is not a general +//! memoize against. So this one entry point keeps a rank-and-take-first +//! behavior internally (via the private [`select_and_bind`]), scoped to +//! workload-wide CSE memoization specifically. It is not a general //! "bind me one tree" API — for a single target, go through //! `SketchFamilyStrategy` and decide what to keep yourself. //! @@ -52,21 +34,15 @@ //! post-ASAP rule engine's job (#6/#33), not this pass's. Similarly //! conservative: multi-intent `Aggregate` nodes (SQL `SELECT SUM(a), AVG(b)`) //! and aggregates with a `HAVING` predicate (the filter would need the -//! estimate first) stay logical. +//! estimate first) stay logical — see [`crate::replacement::bindable_intent`]. use std::rc::Rc; -use asap_types::post_asap::{ - SketchQuery, SummaryExpr, SummaryFamilyType, SummaryField, SummaryNode, SummarySchema, -}; -use asap_types::pre_asap::agg_intent::AggIntent; -use asap_types::pre_asap::expr_ir::ColumnRef; -use asap_types::pre_asap::query_expr::{QueryExpr, QueryExprError, Reduction}; -use asap_types::pre_asap::schema::Schema; +use asap_types::post_asap::{SummaryExpr, SummaryNode}; +use asap_types::pre_asap::query_expr::{QueryExpr, QueryExprError}; use thiserror::Error; use crate::cost_model::{CostModel, CseCandidate, DefaultCostModel, ShareDecision}; -use crate::implementation::Implementation; use crate::replacement::{ Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG, }; @@ -79,30 +55,17 @@ pub enum ImplementError { Schema(#[from] QueryExprError), } -/// The bindable shape [`crate::replacement::SketchFamilyStrategy`] targets: -/// a single intent, no `HAVING`. A multi-intent node -/// (SQL `SELECT SUM(a), AVG(b)`), or one with a `HAVING` predicate (the -/// filter would need the estimate first), stays logical — see the module -/// docs' "Conservative fallbacks". -pub fn bindable_intent(node: &QueryExpr) -> Option<&AggIntent> { - if let QueryExpr::Aggregate { - measures, having, .. - } = node - { - if let ([intent], None) = (measures.as_slice(), having) { - return Some(intent); - } - } - None -} - /// Rank-and-take-first selector, scoped to [`implement_workload_with`]'s -/// CSE memoization and to this module's own recursion into a node's child -/// (see the module docs' "Why `implement_workload`/`implement_workload_with` -/// are still here") — **not** a general single-answer API. `root` must -/// already be the caller's own `Rc`, never fabricated per call, so this -/// never allocates beyond what the caller already held. -fn select_and_bind( +/// CSE memoization and to [`crate::replacement`]'s own recursion into a +/// node's child (see the module docs' "Why `implement_workload`/ +/// `implement_workload_with` are here") — **not** a general single-answer +/// API. `root` must already be the caller's own `Rc`, never fabricated per +/// call, so this never allocates beyond what the caller already held. +/// +/// `pub(crate)`: [`crate::replacement`]'s own construction helper calls this +/// to bind a target's child, so a nested aggregate gets its own independent +/// enumeration instead of inheriting the parent's forced candidate. +pub(crate) fn select_and_bind( root: &Rc, cost_model: &dyn CostModel, ) -> Result, ImplementError> { @@ -130,45 +93,6 @@ fn select_and_bind( } } -/// Bind `expr` to an already-decided [`Implementation`] for its top intent. -/// The shared low-level primitive: [`select_and_bind`] (workload-CSE -/// memoization and this module's own recursion) calls this with whichever -/// candidate it kept; [`crate::replacement::SketchFamilyStrategy`] calls -/// this once per candidate it enumerates. One function turns a chosen -/// `Implementation` into a `SummaryNode`, used identically by both. -/// -/// `expr` must still be the [`bindable_intent`] shape for `implementation` to -/// have any effect; anything else falls back to [`logical`]. Only `expr`'s -/// own top-level decision is forced — recursion into `expr`'s child goes -/// back through [`select_and_bind`] (fresh candidate enumeration, not a -/// forced pick), so choosing one candidate for a target never leaks into -/// that target's own nested aggregates. -pub fn bind_with_implementation( - expr: &QueryExpr, - implementation: Implementation, - cost_model: &dyn CostModel, -) -> Result, ImplementError> { - if let QueryExpr::Aggregate { - reduction, - measures, - having, - child, - .. - } = expr - { - // The bindable shape: exactly one intent, no HAVING. (Multi-intent - // nodes and HAVING stay logical — see the module docs.) - if let ([intent], None) = (measures.as_slice(), having) { - if let Some((family, estimate)) = summary_family(implementation) { - return bind_summary_agg( - expr, reduction, intent, child, family, estimate, cost_model, - ); - } - } - } - logical(expr) -} - /// Bind a whole workload's worth of already-CSE'd roots /// ([`asap_types::pre_asap::cse::share_common_subtrees`]'s output), reusing /// one bound [`SummaryNode`] wherever two roots share the same `Rc` *and* @@ -187,7 +111,7 @@ pub fn bind_with_implementation( /// needs to recognize when it already bound the exact `Rc` a later root /// hands back, and is deliberately *not* a general "does an available /// `Implementation` satisfy this one" lookup — that subsumption question is -/// `asap_aware_mapping::implementation::Matcher`'s documented, deliberately-unfilled +/// `asap_aware_mapping::replacement::Matcher`'s documented, deliberately-unfilled /// job, not this one's. /// /// A first pass over `roots` counts each distinct `Rc` pointer's @@ -260,152 +184,6 @@ pub fn implement_workload_with( .collect() } -/// Translate an [`Implementation`] into the `(family, needs a -/// SummaryEstimate readout)` pair [`bind_summary_agg`] needs, or `None` for -/// `PassThrough` (the caller falls back to [`logical`]). -/// -/// Every family's partial state needs a readout to recover a value, except -/// `ExactAggregate` — its partial state *is* the value already, so no -/// estimate step follows it. -fn summary_family(implementation: Implementation) -> Option<(SummaryFamilyType, bool)> { - Some(match implementation { - Implementation::ExactAggregate { kind, params } => { - (SummaryFamilyType::ExactAggregate(kind, params), false) - } - Implementation::Sketch { kind, params } => (SummaryFamilyType::Sketch(kind, params), true), - Implementation::Sample { kind, params } => (SummaryFamilyType::Sample(kind, params), true), - Implementation::Wavelet { kind, params } => { - (SummaryFamilyType::Wavelet(kind, params), true) - } - Implementation::StatModel { kind, params } => { - (SummaryFamilyType::StatModel(kind, params), true) - } - Implementation::PassThrough => return None, - }) -} - -/// Emit `SummaryAgg` (recursively binding the child), plus the -/// `SummaryEstimate` readout when `estimate` is set. -#[allow(clippy::too_many_arguments)] -fn bind_summary_agg( - node: &QueryExpr, - reduction: &Reduction, - intent: &AggIntent, - child: &Rc, - family: SummaryFamilyType, - estimate: bool, - cost_model: &dyn CostModel, -) -> Result, ImplementError> { - let child_schema = child.output_schema()?; - // The single canonical pre-ASAP derivation (per-series vs cross-series, - // name overrides) already computes the row shape; binding only retypes - // the summary state column. - let per_series = matches!(reduction, Reduction::PerEntity); - let by: Vec = reduction - .group_keys() - .map(|g| g.to_vec()) - .unwrap_or_default(); - let out_schema = node.output_schema()?; - let state_idx = summary_col_index(&out_schema, &by, per_series); - - let col = summarised_column(intent, &child_schema); - let query = estimate.then(|| readout(intent, &col, cost_model)); - - let mut state_schema = lift(&out_schema); - if let Some(field) = state_schema.fields.get_mut(state_idx) { - field.dtype = family.clone(); - } - - // `reduction` is carried onto `SummaryAgg` verbatim — not flattened to a - // bare `Vec` — so `SummaryExecutor::find_candidates` can tell - // a genuine empty-`by` reduction apart from a per-entity shape with no - // grouping concept at all (issue #163). `bind_summary_agg` is the single - // place that decides this; nothing downstream re-derives it. - let agg = Rc::new(SummaryNode { - expr: SummaryExpr::SummaryAgg { - child: select_and_bind(child, cost_model)?, - family, - col, - reduction: reduction.clone(), - }, - schema: state_schema, - }); - match query { - // The readout: downstream of the estimate the schema is the plain - // pre-ASAP row shape again (the summary-state type does not - // propagate). - Some(query) => Ok(Rc::new(SummaryNode { - expr: SummaryExpr::SummaryEstimate { - summary_input: agg, - query, - }, - schema: lift(&out_schema), - })), - None => Ok(agg), - } -} - -/// Index of the summary-state column in the aggregate's output schema: -/// cross-series output is `by ++ [agg]` (the column after the keys); -/// a per-series reduction keeps every label and replaces the sample value -/// (named `value` — mirror `per_series_reduction_schema`'s fallback). -/// `per_series` is the caller's already-read `Reduction` (issue #165) — -/// this never re-derives it, so it can't disagree with the caller. -fn summary_col_index(out_schema: &Schema, by: &[usize], per_series: bool) -> usize { - if per_series { - out_schema - .column_id("value") - .or_else(|| (0..out_schema.columns.len()).find(|&i| Some(i) != out_schema.time_index)) - .unwrap_or(0) - } else { - by.len() - } -} - -/// The column fed into the summary: the intent's positional input column -/// resolved to a name against the child schema, or the PromQL sample value. -fn summarised_column(intent: &AggIntent, child_schema: &Schema) -> ColumnRef { - match intent - .input_col() - .and_then(|id| child_schema.columns.get(id)) - { - Some(c) => match &c.table { - Some(t) => ColumnRef::Qualified { - table: t.clone(), - name: c.name.clone(), - }, - None => ColumnRef::Named(c.name.clone()), - }, - None => ColumnRef::SampleValue, - } -} - -/// The `SummaryEstimate` readout for a summary-bound intent. -fn readout(intent: &AggIntent, col: &ColumnRef, cost_model: &dyn CostModel) -> SketchQuery { - match intent { - AggIntent::Quantile { q, .. } => SketchQuery::Quantile { q: *q }, - AggIntent::Cardinality { .. } => SketchQuery::Cardinality, - AggIntent::TopK { k, .. } => SketchQuery::TopK { k: *k }, - AggIntent::Count { .. } => SketchQuery::PointCount { - key: col.clone(), - value: None, - }, - // Core doesn't know the shape of a deployment-specific `Extension` - // intent, so it can't build its readout either — delegate to the - // same `CostModel` that decided (via `realize_extension`) this - // intent gets a summary realization at all. See `readout_extension`'s - // doc for the invariant this depends on. - AggIntent::Extension { ext_kind, payload } => { - cost_model.readout_extension(ext_kind, payload, col) - } - other => { - unreachable!( - "no summary realization for {other:?} (implementation::implementations_for_with)" - ) - } - } -} - /// Wrap an unrewritten pre-ASAP subtree, lifting its schema with every column /// `SummaryFamilyType::Plain`. Public so a caller can fall back to this /// explicitly — e.g. when `SketchFamilyStrategy::replacements()` returns no @@ -416,33 +194,20 @@ pub fn logical(expr: &QueryExpr) -> Result, ImplementError> { let schema = expr.output_schema()?; Ok(Rc::new(SummaryNode { expr: SummaryExpr::Logical(Box::new(expr.clone())), - schema: lift(&schema), + schema: crate::replacement::lift(&schema), })) } -fn lift(schema: &Schema) -> SummarySchema { - SummarySchema { - fields: schema - .columns - .iter() - .map(|c| SummaryField { - name: c.name.clone(), - dtype: SummaryFamilyType::Plain(c.dtype.clone()), - nullable: c.nullable, - }) - .collect(), - time_index: schema.time_index, - } -} - #[cfg(test)] mod tests { use super::*; - use asap_types::post_asap::{ExactKind, ExactParams, SketchKind, SketchParams}; - use asap_types::pre_asap::agg_intent::default_quantile; - use asap_types::pre_asap::expr_ir::{CompareOpKind, ScalarValue}; - use asap_types::pre_asap::query_expr::{Predicate, Source}; - use asap_types::pre_asap::schema::{Column, DataType}; + use asap_types::post_asap::{ + ExactKind, ExactParams, SketchKind, SketchParams, SketchQuery, SummaryFamilyType, + }; + use asap_types::pre_asap::agg_intent::{default_quantile, AggIntent}; + use asap_types::pre_asap::expr_ir::{ColumnRef, CompareOpKind, ScalarValue}; + use asap_types::pre_asap::query_expr::{Predicate, Reduction, Source}; + use asap_types::pre_asap::schema::{Column, DataType, Schema}; use asap_types::types::AccuracyTarget; use std::time::Duration; @@ -482,7 +247,10 @@ mod tests { } } - fn field<'a>(schema: &'a SummarySchema, name: &str) -> &'a SummaryField { + fn field<'a>( + schema: &'a asap_types::post_asap::SummarySchema, + name: &str, + ) -> &'a asap_types::post_asap::SummaryField { schema .fields .iter() @@ -631,9 +399,9 @@ mod tests { &self, ext_kind: &str, _payload: &serde_json::Value, - ) -> crate::implementation::Implementation { + ) -> crate::replacement::Implementation { if ext_kind == "frequency" { - crate::implementation::Implementation::Sketch { + crate::replacement::Implementation::Sketch { kind: SketchKind::CountSketch, params: SketchParams::CountSketch { width: 256, @@ -641,7 +409,7 @@ mod tests { }, } } else { - crate::implementation::Implementation::PassThrough + crate::replacement::Implementation::PassThrough } } diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index 027d626a..10194c80 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -6,10 +6,10 @@ //! needs knowledge this crate doesn't have and shouldn't acquire: the crate //! doc's layering invariant is that `asap-plan` depends only on [`asap_ir`], //! never on a runtime or a deployment model. What it *can* own is the -//! interface every deployment's cost model plugs into, so [`implementation`]'s +//! interface every deployment's cost model plugs into, so [`replacement`]'s //! summary selection has exactly one extension point instead of forcing //! each downstream (ASAPCollector + ASAPQuery-backend, ASAPFusion, …) to -//! fork [`implementation::implementations_for_with`]. +//! fork `replacement::implementations_for_with`. //! //! This trait is scoped to the approximate-**sketch** family specifically //! ([`CostModel::rank_candidates`]/[`size_params`](CostModel::size_params) @@ -51,7 +51,7 @@ use asap_types::pre_asap::agg_intent::AggIntent; use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::query_expr::QueryExpr; -use crate::implementation::Implementation; +use crate::replacement::Implementation; /// A CSE-detected, legality-gated shared subtree with two or more consumers /// — the unit [`CostModel::cse_share_decision`] decides over. Built by @@ -168,22 +168,22 @@ pub fn default_cse_shared_maintenance_cost(family: &SummaryFamilyType) -> Cost { /// Ranks the candidate summary families for one [`AggIntent`], best choice /// first. /// -/// [`implementation::summary_candidates`] returns every family that *can* answer an +/// [`replacement::summary_candidates`] returns every family that *can* answer an /// intent, in an arbitrary static preference order (issue #98's "one home" /// for the candidate set). A `CostModel` re-orders that list under real, /// deployment-specific cost knowledge this crate has no way to know about — -/// [`implementation::implementations_for_with`] implements whichever candidate ends +/// `replacement::implementations_for_with` implements whichever candidate ends /// up first after ranking. pub trait CostModel { /// Rank `candidates` (as returned by - /// [`summary_candidates`](crate::implementation::summary_candidates)) for + /// [`summary_candidates`](crate::replacement::summary_candidates)) for /// `intent`, best choice first. /// /// Implementations MAY reorder freely and MAY drop entries that aren't /// available in their deployment, but MUST NOT invent a candidate that /// wasn't in the input — an unknown [`SketchKind`] has no /// [`SketchParams`](asap_types::post_asap::SketchParams) sizing logic in - /// [`implementation::implementations_for_with`] and binding it will panic. + /// `replacement::implementations_for_with` and binding it will panic. /// Returning an empty `Vec` means "no candidate is acceptable"; /// `implementations_for_with` treats that the same as `candidates` /// having been empty to begin with. @@ -196,9 +196,9 @@ pub trait CostModel { /// Splitting sizing out from candidate selection lets a deployment own /// its own parameter-sizing math (e.g. an empirically-tuned table, or /// discrete rungs required by a downstream catalog) without forking - /// [`implementation::implementations_for_with`] — the same "one extension + /// `replacement::implementations_for_with` — the same "one extension /// point" rationale as `rank_candidates`, one level deeper. Default: - /// [`implementation::default_size_params`], `asap-plan`'s built-in formulas + /// [`replacement::default_size_params`], `asap-plan`'s built-in formulas /// (unchanged) — a deployment that only needs to reorder candidates, /// not resize them, can leave this method unimplemented. fn size_params( @@ -208,12 +208,12 @@ pub trait CostModel { eps: f64, delta: f64, ) -> SketchParams { - crate::implementation::default_size_params(kind, intent, eps, delta) + crate::replacement::default_size_params(kind, intent, eps, delta) } /// Realize an `AggIntent::Extension { ext_kind, payload }` — a /// deployment-specific intent shape core has no realization opinion - /// for (issue #131). `implementation::implementations_for_with` consults this + /// for (issue #131). `replacement::implementations_for_with` consults this /// for every `Extension` node instead of hardcoding `PassThrough` /// (issue #150). Default: `PassThrough` — preserves today's behavior /// for every deployment that doesn't override this, exactly like @@ -299,9 +299,9 @@ pub trait CostModel { } /// The default cost model: preserves [`summary_candidates`]'s built-in static -/// order and [`implementation::default_size_params`]'s built-in sizing unchanged. +/// order and [`replacement::default_size_params`]'s built-in sizing unchanged. /// -/// [`summary_candidates`]: crate::implementation::summary_candidates +/// [`summary_candidates`]: crate::replacement::summary_candidates pub struct DefaultCostModel; impl CostModel for DefaultCostModel { @@ -313,7 +313,7 @@ impl CostModel for DefaultCostModel { #[cfg(test)] mod tests { use super::*; - use crate::implementation::summary_candidates; + use crate::replacement::summary_candidates; use asap_types::pre_asap::agg_intent::default_cardinality; #[test] @@ -356,7 +356,7 @@ mod tests { let intent = default_cardinality(); assert_eq!( AlwaysPreferLast.size_params(SketchKind::Hll, &intent, 0.01, 0.01), - crate::implementation::default_size_params(SketchKind::Hll, &intent, 0.01, 0.01), + crate::replacement::default_size_params(SketchKind::Hll, &intent, 0.01, 0.01), ); } @@ -387,7 +387,7 @@ mod tests { let k = if eps >= 0.01 { 200 } else { 2048 }; SketchParams::Kll { k } } - other => crate::implementation::default_size_params(other, intent, eps, delta), + other => crate::replacement::default_size_params(other, intent, eps, delta), } } } @@ -404,7 +404,7 @@ mod tests { // Untouched kinds still fall through to the default formula. assert_eq!( DiscreteKllRungs.size_params(SketchKind::Hll, &intent, 0.01, 0.01), - crate::implementation::default_size_params(SketchKind::Hll, &intent, 0.01, 0.01), + crate::replacement::default_size_params(SketchKind::Hll, &intent, 0.01, 0.01), ); } diff --git a/crates/asap-aware-mapping/src/implementation.rs b/crates/asap-aware-mapping/src/implementation.rs deleted file mode 100644 index dd14d9d9..00000000 --- a/crates/asap-aware-mapping/src/implementation.rs +++ /dev/null @@ -1,1067 +0,0 @@ -//! [`Implementation`] — how one [`AggIntent`] can be realized at post-ASAP -//! binding time (issue #98): by an approximate summary (sketch, sample, -//! wavelet, statistical model, …), by an exact mergeable accumulator, or by -//! an ordinary exact operator (pass-through). This is a post-ASAP concern — -//! the pre-ASAP IR carries only the intent + accuracy target, never the -//! realization — and it's a per-node decision, made once per `AggIntent`, -//! not a plan-wide one. -//! -//! [`implementations_for_with`] is where every valid realization gets -//! enumerated, exhaustive and ranked (most-preferred first) — this crate has -//! no separate function that computes just "the one" `Implementation` -//! independently of that list. -//! [`crate::replacement::SketchFamilyStrategy`] is the sole public consumer: -//! it wraps every entry of this list into its own bound -//! [`SummaryNode`](asap_types::post_asap::SummaryNode) and returns all of -//! them, ranked — a caller wanting a single answer keeps the first one -//! itself (see `replacement.rs`'s module docs). Three inputs feed it: -//! -//! - the [`AccuracyTarget`] threaded onto the approximate-capable intents -//! (`Quantile` / `Cardinality` / `Count` / `TopK`) — `Exact` forbids a -//! sketch; `Epsilon` / `EpsilonDelta` size the sketch parameters; -//! - the [`agg_is_exact`] / [`agg_is_mergeable`] helpers in `asap-ir` — -//! an exact accumulator exists only for mergeable intents -//! (`Avg`/`StdDev`/`Variance` need richer partial state, so they -//! pass through to an ordinary exact operator); -//! - histogram sketchability (#79): [`AggIntent::HistogramQuantile`] -//! (classic cumulative-`le`-bucket interpolation) is **not** re-sketchable — -//! pre-aggregated bucket counts can't feed a quantile sketch — while the -//! generic `Quantile` path (native histograms / raw samples) is. -//! -//! [`implementations_for_with`] is a single exhaustive match over the intent -//! vocabulary, so a new `AggIntent` variant fails to compile until it is -//! given an explicit realization — there is no silent fall-through. -//! -//! Today's core dispatch only ever produces [`Implementation::ExactAggregate`], -//! [`Implementation::Sketch`], or [`Implementation::PassThrough`] — no -//! `AggIntent` variant maps to sampling/wavelet/statistical-model yet. -//! [`Implementation::Sample`]/[`Wavelet`](Implementation::Wavelet)/ -//! [`StatModel`](Implementation::StatModel) exist so a deployment's own -//! [`CostModel::realize_extension`](crate::cost_model::CostModel::realize_extension) -//! can choose one of them for an `AggIntent::Extension` node — core has no -//! opinion on when that's the right choice. - -use asap_types::post_asap::{ - ExactKind, ExactParams, SamplingKind, SamplingParams, SketchKind, SketchParams, StatModelKind, - StatModelParams, WaveletKind, WaveletParams, -}; -use asap_types::pre_asap::agg_intent::{agg_is_mergeable, AggIntent}; -use asap_types::types::AccuracyTarget; - -use crate::cost_model::CostModel; - -/// How an [`AggIntent`] may be realised at post-ASAP binding time. -#[derive(Debug, Clone, PartialEq)] -pub enum Implementation { - /// An exact **mergeable** accumulator (partial state ≡ the value - /// itself: `Sum` / `Count` / `MinMax` / `Rate` / `Increase`). The - /// built state *is* the answer already — no `SummaryEstimate` readout - /// step. - ExactAggregate { - kind: ExactKind, - params: ExactParams, - }, - /// An approximate sketch sized to the intent's [`AccuracyTarget`]. - /// Needs a `SummaryEstimate` readout to recover a value. - Sketch { - kind: SketchKind, - params: SketchParams, - }, - /// A sampling-based summary (a retained row subset). Needs a - /// `SummaryEstimate` readout. Not chosen by any core `AggIntent` - /// dispatch today — see the module docs. - Sample { - kind: SamplingKind, - params: SamplingParams, - }, - /// A wavelet-transform summary. Needs a `SummaryEstimate` readout. Not - /// chosen by any core `AggIntent` dispatch today — see the module docs. - Wavelet { - kind: WaveletKind, - params: WaveletParams, - }, - /// A fitted statistical/parametric-model summary. Needs a - /// `SummaryEstimate` readout. Not chosen by any core `AggIntent` - /// dispatch today — see the module docs. - StatModel { - kind: StatModelKind, - params: StatModelParams, - }, - /// No summary form — the node stays a logical pre-ASAP operator and is - /// executed exactly (per-series transforms, non-mergeable reducers, exact - /// quantile/top-k/cardinality, classic-bucket `HistogramQuantile`, …). - PassThrough, -} - -/// Does an already-**available** [`Implementation`] — e.g. a summary -/// instance a downstream deployment already materialized somewhere, found -/// via whatever inventory/index that deployment keeps — satisfy a -/// **required** [`Implementation`] (one of the candidates -/// [`implementations_for_with`] produced for some [`AggIntent`])? -/// -/// This is the query-optimization-literature "materialized view matching" -/// / "answering queries using views" question, narrowed to this crate's -/// summary vocabulary: not "can I build this from scratch" (that's what -/// [`implementations_for_with`] answers) but "does something that already -/// exists answer this". -/// -/// `asap-plan` deliberately ships no implementation of this trait and no -/// default method body — unlike [`implementations_for_with`], which decision -/// an available `Implementation` satisfies a required one is not a fact this -/// crate can settle on its own. Two real, reasonable answers already -/// diverge outside this crate: -/// -/// - A **pure sketch-algebra** answer would say a `Sketch{kind: Kll, ..}` -/// requirement is satisfied by an available `DDSketch` (both quantile -/// sketches), and that a heap-bearing top-k sketch also answers a bare -/// frequency point-query (the heap is additional info on the same -/// underlying matrix) — but not the reverse. -/// - A **deployment with its own storage-layout rules** may need more: -/// e.g. whether a multi-population accumulator can serve a -/// single-population query via re-aggregation is a fact about that -/// deployment's storage layout, not about any summary family's kind at -/// all — a family's own kind doesn't encode grouping (grouping lives on -/// the post-ASAP node's `by` instead), so there is nothing in this -/// crate's own vocabulary to subsume. -/// -/// Implementations are expected to consult `required`/`available`'s -/// `kind` (and whatever grouping/placement context the deployment tracks -/// alongside `Implementation`, which this trait's signature doesn't carry -/// because this crate has no inventory concept to carry it in). -pub trait Matcher { - fn is_satisfied_by(&self, required: &Implementation, available: &Implementation) -> bool; -} - -/// Confidence δ assumed when the target carries only an ε -/// (`AccuracyTarget::Epsilon`): the (ε, δ)-parameterised sketches (CMS) need -/// one. `ln(1/0.01) → depth 5`, matching the conventional CMS sizing. -pub const DEFAULT_DELTA: f64 = 0.01; - -/// The sketch kinds that can serve an intent, most-preferred first. -/// This is the `AggIntent → SketchKind` map of issue #98; -/// [`implementations_for_with`] sizes and ranks every entry via `cost_model`. -/// Listed here so the candidate set has one home. -pub fn summary_candidates(intent: &AggIntent) -> &'static [SketchKind] { - match intent { - AggIntent::Quantile { .. } => &[SketchKind::Kll, SketchKind::DDSketch], - AggIntent::Cardinality { .. } => &[SketchKind::Hll, SketchKind::Theta, SketchKind::Kmv], - // Count-Sketch-with-heap is CMS-with-heap's balanced/zero-mean-error - // alternative for the same heavy-hitter shape. - AggIntent::TopK { .. } => &[SketchKind::CmsWithHeap, SketchKind::CountSketchWithHeap], - AggIntent::Count { .. } => &[SketchKind::Cms, SketchKind::CountSketch], - _ => &[], - } -} - -/// The [`AccuracyTarget`] threaded onto an approximate-capable intent -/// (`Quantile`/`Cardinality`/`Count`/`TopK`), or `None` for every other -/// intent (no sketch candidate applies — [`implementations_for_with`]'s own -/// match routes those elsewhere). Exposed so -/// [`crate::replacement::SketchFamilyStrategy`] and -/// [`implementations_for_with`] resolve the exact same accuracy target, -/// without either re-deriving it from scratch. -pub fn accuracy_target(intent: &AggIntent) -> Option<&AccuracyTarget> { - match intent { - AggIntent::Quantile { accuracy, .. } - | AggIntent::Cardinality { accuracy, .. } - | AggIntent::Count { accuracy } - | AggIntent::TopK { accuracy, .. } => Some(accuracy), - _ => None, - } -} - -/// Every valid [`Implementation`] for `intent`, exhaustive and ranked -/// (most-preferred first via `cost_model`) — the *only* place this crate -/// decides what an `AggIntent` may become. Nothing in this crate computes -/// "the one" `Implementation` independently of this list: -/// [`crate::replacement::SketchFamilyStrategy`] keeps every entry as a -/// candidate, and a caller that wants a single executable answer takes the -/// head of *that* strategy's output itself — see its own module docs. -/// -/// Exhaustive over the [`AggIntent`] vocabulary — adding a variant without an -/// explicit realization is a compile error, and the coverage-matrix test pins -/// each variant's category. -pub fn implementations_for_with( - intent: &AggIntent, - cost_model: &dyn CostModel, -) -> Vec { - match intent { - // ── Approximate-capable intents — the AccuracyTarget decides ──────── - AggIntent::Quantile { accuracy, .. } - | AggIntent::Cardinality { accuracy, .. } - | AggIntent::Count { accuracy } - | AggIntent::TopK { accuracy, .. } => match accuracy { - AccuracyTarget::Exact => vec![exact_realization(intent)], - _ => sketch_implementations(intent, accuracy, cost_model), - }, - - // ── Exact mergeable accumulators ───────────────────────────────────── - AggIntent::Sum { .. } => vec![exact_accumulator(intent, ExactKind::Sum, ExactParams::Sum)], - AggIntent::Min { .. } | AggIntent::Max { .. } => { - vec![exact_accumulator( - intent, - ExactKind::MinMax, - ExactParams::MinMax, - )] - } - AggIntent::Rate => vec![exact_accumulator( - intent, - ExactKind::Rate, - ExactParams::Rate, - )], - AggIntent::Increase => { - vec![exact_accumulator( - intent, - ExactKind::Increase, - ExactParams::Increase, - )] - } - - // ── Exact, non-mergeable reducers — richer partial state than a - // single value (see `agg_is_mergeable`), so no accumulator form. - AggIntent::Avg { .. } | AggIntent::StdDev { .. } | AggIntent::Variance { .. } => { - vec![Implementation::PassThrough] - } - - // ── Classic-bucket histogram_quantile (#79): exact `le`-bucket - // interpolation over pre-aggregated counts — NOT re-sketchable. - // (The native/raw form lowers to the generic `Quantile` above.) - AggIntent::HistogramQuantile { .. } => vec![Implementation::PassThrough], - - // ── Per-series transforms and reductions with no sketch realization: - // counter-derivatives (#44), math (#45), time/calendar (#46), - // presence (#47), native-histogram accessors (#43), and the - // `*OverTime` reducers (#51). All exact by construction. - AggIntent::Changes - | AggIntent::Delta - | AggIntent::IDelta - | AggIntent::Deriv - | AggIntent::Resets - | AggIntent::PredictLinear { .. } - | AggIntent::DoubleExpSmoothing { .. } - | AggIntent::HistogramCount - | AggIntent::HistogramSum - | AggIntent::HistogramAvg - | AggIntent::HistogramStdDev - | AggIntent::HistogramStdVar - | AggIntent::HistogramFraction { .. } - | AggIntent::Math(_) - | AggIntent::Absent - | AggIntent::AbsentOverTime - | AggIntent::PresentOverTime - | AggIntent::TimeFn(_) - | AggIntent::LastOverTime - | AggIntent::FirstOverTime - | AggIntent::MadOverTime - | AggIntent::TsOfMinOverTime - | AggIntent::TsOfMaxOverTime - | AggIntent::TsOfFirstOverTime - | AggIntent::TsOfLastOverTime => vec![Implementation::PassThrough], - - // ── Group / count_values (#49): exact per `agg_is_exact`, but their - // output is structural (constant-1 / a synthesized label column), - // not a value a summary accumulator carries. - AggIntent::Group | AggIntent::CountValues { .. } => vec![Implementation::PassThrough], - - // ── Extension (deployment-model-specific, issue #131) — core has no - // realization opinion for a shape it doesn't know, so it defers - // entirely to the `CostModel` (issue #150): `realize_extension` - // defaults to `PassThrough`, preserving today's behavior for - // every deployment that doesn't override it. Core has no way to - // enumerate alternatives for an opaque deployment-defined shape, - // so this is always exactly one candidate. This is also the only - // path that can currently produce `Implementation::Sample`/ - // `Wavelet`/`StatModel` — see the module docs. - AggIntent::Extension { ext_kind, payload } => { - vec![cost_model.realize_extension(ext_kind, payload)] - } - } -} - -/// Exact realization of an approximate-capable intent whose target is -/// `AccuracyTarget::Exact`. `Count` has a mergeable exact accumulator; exact -/// quantile / top-k / cardinality have no single-value summary form (they -/// need the full multiset / heap / set) and pass through. -fn exact_realization(intent: &AggIntent) -> Implementation { - match intent { - AggIntent::Count { .. } => exact_accumulator(intent, ExactKind::Count, ExactParams::Count), - _ => Implementation::PassThrough, - } -} - -fn exact_accumulator(intent: &AggIntent, kind: ExactKind, params: ExactParams) -> Implementation { - // An exact accumulator is only sound when partial states merge - // (`agg(A ∪ B) = combine(agg(A), agg(B))`). - debug_assert!( - agg_is_mergeable(intent), - "accumulator for non-mergeable {intent:?}" - ); - Implementation::ExactAggregate { kind, params } -} - -/// Resolve an [`AccuracyTarget`] into the `(eps, delta)` budget -/// [`CostModel::size_params`] needs. Shared by [`sketch_implementations`] and -/// [`crate::replacement`]'s own sizing — one place this resolution happens, -/// so nothing can drift apart on it. -/// -/// `Exact` is unreachable via [`implementations_for_with`] (which routes -/// `Exact` to [`exact_realization`] instead); degrades to the tightest -/// parameters for a caller that resolves it directly anyway. -pub fn accuracy_budget(accuracy: &AccuracyTarget) -> (f64, f64) { - match accuracy { - AccuracyTarget::Exact => (f64::MIN_POSITIVE, DEFAULT_DELTA), - AccuracyTarget::Epsilon(e) => (*e, DEFAULT_DELTA), - AccuracyTarget::EpsilonDelta { epsilon, delta } => (*epsilon, *delta), - } -} - -/// Every candidate sketch [`Implementation`] for an approximate-capable -/// intent, sized to `accuracy` and ranked via `cost_model.rank_candidates` -/// (most-preferred first) — [`implementations_for_with`]'s Sketch branch. -fn sketch_implementations( - intent: &AggIntent, - accuracy: &AccuracyTarget, - cost_model: &dyn CostModel, -) -> Vec { - let (eps, delta) = accuracy_budget(accuracy); - let ranked = cost_model.rank_candidates(intent, summary_candidates(intent)); - ranked - .into_iter() - .map(|kind| { - let params = cost_model.size_params(kind.clone(), intent, eps, delta); - Implementation::Sketch { kind, params } - }) - .collect() -} - -/// `asap-plan`'s built-in `SketchParams` sizing, keyed off the resolved -/// `(eps, delta)` accuracy budget. [`CostModel::size_params`]'s default -/// body — factored out to a free function so a deployment's own -/// `CostModel` impl can still delegate to it for the candidates it -/// doesn't want to resize itself. -/// -/// Each formula inverts the sketch family's standard error bound to the -/// smallest parameter satisfying the target, clamped to the family's sane -/// range. A non-positive ε saturates to the clamp maximum (tightest -/// allowed). -pub fn default_size_params( - kind: SketchKind, - intent: &AggIntent, - eps: f64, - delta: f64, -) -> SketchParams { - match kind { - SketchKind::Kll => SketchParams::Kll { k: kll_k(eps) }, - SketchKind::Cms => SketchParams::Cms { - width: cms_width(eps), - depth: cms_depth(delta), - }, - SketchKind::Hll => SketchParams::Hll { - precision: hll_precision(eps), - }, - SketchKind::CmsWithHeap => { - let k = match intent { - AggIntent::TopK { k, .. } => *k, - _ => unreachable!("CmsWithHeap is only a TopK candidate"), - }; - SketchParams::CmsWithHeap { - width: cms_width(eps), - depth: cms_depth(delta), - heap_size: k as u32, - } - } - // Non-preferred candidates (DDSketch / Theta / Kmv / CountSketch / - // CountSketchWithHeap) are only reachable once a cost model picks - // them; sized here so that wiring is local. - SketchKind::DDSketch => SketchParams::DDSketch { alpha: eps }, - SketchKind::Theta => SketchParams::Theta { k: kmv_k(eps) }, - SketchKind::Kmv => SketchParams::Kmv { k: kmv_k(eps) }, - // Count-Sketch is CMS's balanced/zero-mean-error alternative — - // same (width, depth) shape, sized the same way for now (a - // Count-Sketch-specific bound uses an L2-norm error guarantee - // rather than CMS's L1-norm one; this is a placeholder pending - // that refinement, same status as the other non-preferred - // candidates above). - SketchKind::CountSketch => SketchParams::CountSketch { - width: cms_width(eps), - depth: cms_depth(delta), - }, - SketchKind::CountSketchWithHeap => { - let k = match intent { - AggIntent::TopK { k, .. } => *k, - _ => unreachable!("CountSketchWithHeap is only a TopK candidate"), - }; - SketchParams::CountSketchWithHeap { - width: cms_width(eps), - depth: cms_depth(delta), - heap_size: k as u32, - } - } - } -} - -/// A deployment's explicit bet about how "typical" (non-adversarial) its -/// workload's collision pattern is expected to be, consumed only by -/// [`posterior_aware_size_params`]. -/// -/// This is **not** derived from Chen et al.'s posterior-error-estimation -/// technique (issue #239, `asap_types::post_asap::query_time::error_estimation`) -/// — that technique computes a tighter bound *at query time* from a -/// sketch's real counter values, and this repo has no sketch runtime yet -/// for a real counter array to size against (see that module's docs, and -/// `asap_types::post_asap::query_time`'s module doc for why it's a -/// deliberately separate folder from this crate's own *plan-time* code). -/// This struct is this crate's own *plan-time* analogue of the same -/// underlying intuition — an expected-case (skewed / non-adversarial) -/// workload needs a smaller sketch than the adversarial worst case — -/// expressed as an explicit, caller-supplied assumption rather than -/// anything observed or proven. Issue #250 tracks actually connecting the -/// two: feeding query-time-observed posterior error back into a future -/// replan's `width_relaxation` instead of a bare caller guess. -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct ExpectedCaseSizing { - /// Fraction, in `(0, 1]`, of the traditional worst-case width - /// ([`cms_width`]) the caller is betting is enough. `1.0` (or any - /// value outside `(0, 1)`) reproduces the worst-case width exactly — - /// no risk taken. A smaller value shrinks the sketch proportionally, - /// at the cost documented on [`posterior_aware_size_params`]. - pub width_relaxation: f64, -} - -/// Opt-in alternative to [`default_size_params`] for the CMS-family kinds -/// (`Cms` / `CmsWithHeap` / `CountSketch` / `CountSketchWithHeap`): sizes -/// width to `assumption.width_relaxation` of the worst-case [`cms_width`], -/// trading the unconditional worst-case `(ε,δ)` guarantee for a smaller -/// sketch under an explicit, caller-stated non-adversarial-workload bet — -/// see [`ExpectedCaseSizing`]. -/// -/// **The tradeoff, spelled out:** [`default_size_params`]'s width guarantees -/// `Pr[error > ε·|F|₁] < δ` for *any* input, including an adversarial one -/// built to maximize collisions (§3.3 of the posterior-error-estimation -/// paper this issue is about — see -/// `asap_types::post_asap::query_time::error_estimation`'s module docs). -/// Shrinking -/// width below that only keeps the same `(ε,δ)` guarantee if the real -/// workload's collision load stays within `width_relaxation` of the -/// worst-case assumption — this function does not check that, cannot check -/// it (no data exists at plan time), and does not change the formal -/// guarantee's statement; it only changes how much hardware is spent -/// chasing it. Callers accept that gap explicitly by choosing -/// `width_relaxation < 1.0`. -/// -/// Depth ([`cms_depth`]) is left unchanged from [`default_size_params`]: -/// depth trades away confidence *exponentially* (`Pr[all r rows bad] = -/// p^r` — each extra row multiplies the failure probability down), a -/// differently-shaped and materially riskier tradeoff than width's linear -/// relaxation. Issue #239 asks for *a* tighter-sizing option under a -/// stated assumption, not a full redesign of the depth/width tradeoff -/// space, so depth relaxation is left as explicit future scope. -/// -/// For every `SketchKind` outside the CMS family, this is identical to -/// [`default_size_params`] — `width_relaxation` only ever touches the -/// [`cms_width`]-sized formulas this issue is about. -/// -/// [`default_size_params`]'s own behavior is completely unchanged by this -/// function's existence — this is a separate, additive entry point, never -/// called from [`default_size_params`] or [`implementations_for_with`]. -pub fn posterior_aware_size_params( - kind: SketchKind, - intent: &AggIntent, - eps: f64, - delta: f64, - assumption: ExpectedCaseSizing, -) -> SketchParams { - let relaxed_width = |eps: f64| -> u32 { - let base = cms_width(eps); - let f = assumption.width_relaxation; - if !(f.is_finite() && f > 0.0 && f < 1.0) { - return base; // out-of-range bet: no relaxation, fall back to worst case - } - saturating_ceil(base as f64 * f, 2, base) - }; - match kind { - SketchKind::Cms => SketchParams::Cms { - width: relaxed_width(eps), - depth: cms_depth(delta), - }, - SketchKind::CmsWithHeap => { - let k = match intent { - AggIntent::TopK { k, .. } => *k, - _ => unreachable!("CmsWithHeap is only a TopK candidate"), - }; - SketchParams::CmsWithHeap { - width: relaxed_width(eps), - depth: cms_depth(delta), - heap_size: k as u32, - } - } - SketchKind::CountSketch => SketchParams::CountSketch { - width: relaxed_width(eps), - depth: cms_depth(delta), - }, - SketchKind::CountSketchWithHeap => { - let k = match intent { - AggIntent::TopK { k, .. } => *k, - _ => unreachable!("CountSketchWithHeap is only a TopK candidate"), - }; - SketchParams::CountSketchWithHeap { - width: relaxed_width(eps), - depth: cms_depth(delta), - heap_size: k as u32, - } - } - // Every other kind is untouched by this issue's CMS-specific - // relaxation — defer to the existing formula verbatim. Spelled out - // exhaustively, matching `default_size_params`'s own match, rather - // than a wildcard arm: a future `SketchKind` variant then fails to - // compile *here* too, instead of silently inheriting worst-case - // sizing with no signal that this function never considered it. - SketchKind::Kll => default_size_params(kind, intent, eps, delta), - SketchKind::Hll => default_size_params(kind, intent, eps, delta), - SketchKind::DDSketch => default_size_params(kind, intent, eps, delta), - SketchKind::Theta => default_size_params(kind, intent, eps, delta), - SketchKind::Kmv => default_size_params(kind, intent, eps, delta), - } -} - -// ── Parameter sizing ────────────────────────────────────────────────────────── -// -// Each function inverts the sketch family's standard error bound to the -// smallest parameter satisfying the target, clamped to the family's sane -// range. A non-positive ε saturates to the clamp maximum (tightest allowed). - -/// KLL: rank error ε ≈ 2/k ⇒ `k = ⌈2/ε⌉`. ε = 0.01 → k = 200, matching the -/// design doc's worked example (`KLL{k=200}` satisfies ε=0.01). -fn kll_k(eps: f64) -> u32 { - saturating_ceil(2.0 / eps, 8, 65_535) -} - -/// HLL: standard error ≈ 1.04/√(2^p) ⇒ `p = ⌈log2((1.04/ε)²)⌉`. The default -/// `Cardinality` target (`asap-ir::default_cardinality`) inverts to p = 14. -fn hll_precision(eps: f64) -> u8 { - saturating_ceil((1.04 / eps).powi(2).log2(), 4, 18) as u8 -} - -/// CMS: over-count ≤ ε·N with width `w = ⌈e/ε⌉` columns. -fn cms_width(eps: f64) -> u32 { - saturating_ceil(std::f64::consts::E / eps, 2, 1 << 26) -} - -/// CMS: failure probability ≤ δ with depth `d = ⌈ln(1/δ)⌉` rows. -/// δ = 0.01 → depth 5. -fn cms_depth(delta: f64) -> u32 { - saturating_ceil((1.0 / delta).ln(), 1, 32) -} - -/// KMV / theta: relative error ≈ 1/√k ⇒ `k = ⌈1/ε²⌉`. -fn kmv_k(eps: f64) -> u32 { - saturating_ceil(1.0 / (eps * eps), 16, 1 << 26) -} - -/// `⌈x⌉` clamped to `[lo, hi]`; NaN / non-positive x saturate to `hi` -/// (a degenerate ε means "as accurate as this family goes"). -fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { - if !x.is_finite() || x <= 0.0 { - return hi; - } - (x.ceil() as u32).clamp(lo, hi) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::cost_model::DefaultCostModel; - use asap_types::pre_asap::agg_intent::{ - agg_is_exact, default_cardinality, default_quantile, MathFunc, TimeFunc, - }; - - fn eps(e: f64) -> AccuracyTarget { - AccuracyTarget::Epsilon(e) - } - - /// The most-preferred `Implementation` — `implementations_for_with(intent, - /// &DefaultCostModel)`'s head — for tests that only care about the - /// default pick, not the full candidate list. - fn preferred(intent: &AggIntent) -> Implementation { - implementations_for_with(intent, &DefaultCostModel) - .into_iter() - .next() - .expect("every intent has at least one Implementation") - } - - /// Shorthand for asserting the realization *category*. - #[derive(Debug, PartialEq)] - enum Cat { - Sketch(SketchKind), - Acc(ExactKind), - Pass, - } - - fn cat(intent: &AggIntent) -> Cat { - match preferred(intent) { - Implementation::ExactAggregate { kind, .. } => Cat::Acc(kind), - Implementation::Sketch { kind, .. } => Cat::Sketch(kind), - Implementation::PassThrough => Cat::Pass, - other => { - panic!("this coverage matrix expects only Exact/Sketch/PassThrough, got {other:?}") - } - } - } - - /// The `AggIntent → SummaryKind` coverage matrix (issue #98): every intent - /// variant maps to a sketch, an exact accumulator, or an explicit - /// pass-through. `implementations_for_with`'s match is exhaustive, so a - /// new variant cannot compile without a decision; this matrix pins what - /// each decision *is* (its preferred/first candidate). - #[test] - fn agg_intent_to_summary_kind_coverage_matrix() { - use AggIntent as A; - use Cat::*; - use ExactKind as E; - use SketchKind as K; - let matrix: Vec<(A, Cat)> = vec![ - // approximate-capable, at an ε target → sketch - (default_quantile(0.99), Sketch(K::Kll)), - (default_cardinality(), Sketch(K::Hll)), - ( - A::Count { - accuracy: eps(0.01), - }, - Sketch(K::Cms), - ), - ( - A::TopK { - k: 10, - accuracy: eps(0.01), - }, - Sketch(K::CmsWithHeap), - ), - // the same intents at Exact → exact realization - ( - A::Quantile { - col: None, - q: 0.5, - accuracy: AccuracyTarget::Exact, - }, - Pass, - ), - ( - A::Cardinality { - col: None, - accuracy: AccuracyTarget::Exact, - }, - Pass, - ), - ( - A::Count { - accuracy: AccuracyTarget::Exact, - }, - Acc(E::Count), - ), - ( - A::TopK { - k: 10, - accuracy: AccuracyTarget::Exact, - }, - Pass, - ), - // exact mergeable accumulators - (A::Sum { col: None }, Acc(E::Sum)), - (A::Min { col: None }, Acc(E::MinMax)), - (A::Max { col: None }, Acc(E::MinMax)), - (A::Rate, Acc(E::Rate)), - (A::Increase, Acc(E::Increase)), - // exact but non-mergeable → pass-through - (A::Avg { col: None }, Pass), - ( - A::StdDev { - col: None, - population: false, - }, - Pass, - ), - ( - A::Variance { - col: None, - population: true, - }, - Pass, - ), - // classic-bucket histogram_quantile is not re-sketchable (#79) - (A::HistogramQuantile { q: 0.99 }, Pass), - // counter-derivative / range-vector functions (#44) - (A::Changes, Pass), - (A::Delta, Pass), - (A::IDelta, Pass), - (A::Deriv, Pass), - (A::Resets, Pass), - (A::PredictLinear { seconds: 60.0 }, Pass), - ( - A::DoubleExpSmoothing { - smoothing: 0.5, - trend: 0.5, - }, - Pass, - ), - // native-histogram accessors (#43) - (A::HistogramCount, Pass), - (A::HistogramSum, Pass), - (A::HistogramAvg, Pass), - (A::HistogramStdDev, Pass), - (A::HistogramStdVar, Pass), - ( - A::HistogramFraction { - lower: 0.0, - upper: 1.0, - }, - Pass, - ), - // per-sample transforms (#45, #46) + presence (#47) - (A::Math(MathFunc::Abs), Pass), - (A::TimeFn(TimeFunc::Hour), Pass), - (A::Absent, Pass), - (A::AbsentOverTime, Pass), - (A::PresentOverTime, Pass), - // extended aggregations (#49) - (A::Group, Pass), - (A::CountValues { label: "v".into() }, Pass), - // additional range reducers (#51) - (A::LastOverTime, Pass), - (A::FirstOverTime, Pass), - (A::MadOverTime, Pass), - (A::TsOfMinOverTime, Pass), - (A::TsOfMaxOverTime, Pass), - (A::TsOfFirstOverTime, Pass), - (A::TsOfLastOverTime, Pass), - ]; - for (intent, expected) in &matrix { - assert_eq!(&cat(intent), expected, "realization for {intent:?}"); - } - // Every accumulator pick is mergeable; every sketch pick is on a - // genuinely approximate target (the `agg_is_*` helpers stay truthful). - for (intent, expected) in &matrix { - if let Cat::Acc(_) = expected { - assert!(agg_is_mergeable(intent), "{intent:?}"); - } - if let Cat::Sketch(_) = expected { - assert!( - !agg_is_exact(intent) || matches!(intent, AggIntent::Count { .. }), - "{intent:?} sketches only under an approximate target" - ); - } - } - } - - #[test] - fn accuracy_target_drives_the_boundary() { - // Same intent, three targets → three different decisions. - let exact = AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::Exact, - }; - assert_eq!(preferred(&exact), Implementation::PassThrough); - - let approx = default_quantile(0.99); // ε = 0.01 - assert_eq!( - preferred(&approx), - Implementation::Sketch { - kind: SketchKind::Kll, - params: SketchParams::Kll { k: 200 }, // design.md worked example - } - ); - - let looser = AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: eps(0.05), - }; - assert_eq!( - preferred(&looser), - Implementation::Sketch { - kind: SketchKind::Kll, - params: SketchParams::Kll { k: 40 }, // ⌈2/0.05⌉ - } - ); - } - - #[test] - fn default_cardinality_inverts_to_hll_precision_14() { - // `default_cardinality` encodes HLL's standard error at p=14; the - // sizing must invert it back exactly. - assert_eq!( - preferred(&default_cardinality()), - Implementation::Sketch { - kind: SketchKind::Hll, - params: SketchParams::Hll { precision: 14 }, - } - ); - } - - #[test] - fn epsilon_delta_sizes_cms_depth() { - let intent = AggIntent::Count { - accuracy: AccuracyTarget::EpsilonDelta { - epsilon: 0.001, - delta: 0.001, - }, - }; - assert_eq!( - preferred(&intent), - Implementation::Sketch { - kind: SketchKind::Cms, - params: SketchParams::Cms { - width: 2719, - depth: 7 - }, // ⌈e/0.001⌉, ⌈ln 1000⌉ - } - ); - // Epsilon-only falls back to DEFAULT_DELTA → depth 5. - let intent = AggIntent::Count { - accuracy: eps(0.001), - }; - assert_eq!( - preferred(&intent), - Implementation::Sketch { - kind: SketchKind::Cms, - params: SketchParams::Cms { - width: 2719, - depth: 5 - }, - } - ); - } - - #[test] - fn topk_heap_size_tracks_k() { - let intent = AggIntent::TopK { - k: 25, - accuracy: eps(0.01), - }; - match preferred(&intent) { - Implementation::Sketch { - kind: SketchKind::CmsWithHeap, - params: - SketchParams::CmsWithHeap { - width, - depth, - heap_size, - }, - } => { - assert_eq!(heap_size, 25); - assert_eq!(width, 272); // ⌈e/0.01⌉ - assert_eq!(depth, 5); - } - other => panic!("expected CmsWithHeap, got {other:?}"), - } - } - - #[test] - fn candidate_lists_match_the_issue_map() { - assert_eq!( - summary_candidates(&default_quantile(0.5)), - &[SketchKind::Kll, SketchKind::DDSketch] - ); - assert_eq!( - summary_candidates(&default_cardinality()), - &[SketchKind::Hll, SketchKind::Theta, SketchKind::Kmv] - ); - assert_eq!( - summary_candidates(&AggIntent::TopK { - k: 5, - accuracy: eps(0.01) - }), - &[SketchKind::CmsWithHeap, SketchKind::CountSketchWithHeap] - ); - assert_eq!( - summary_candidates(&AggIntent::Count { - accuracy: eps(0.01) - }), - &[SketchKind::Cms, SketchKind::CountSketch] - ); - assert!(summary_candidates(&AggIntent::Rate).is_empty()); - } - - #[test] - fn implementations_for_with_enumerates_every_candidate_ranked() { - // Quantile's candidate list is [Kll, DDSketch] — implementations_for_with - // must return both, ranked with the DefaultCostModel's preferred - // (Kll) first. - let kinds: Vec = - implementations_for_with(&default_quantile(0.99), &DefaultCostModel) - .into_iter() - .map(|implementation| match implementation { - Implementation::Sketch { kind, .. } => kind, - other => panic!("expected Sketch, got {other:?}"), - }) - .collect(); - assert_eq!(kinds, vec![SketchKind::Kll, SketchKind::DDSketch]); - } - - #[test] - fn degenerate_epsilon_saturates_to_tightest_params() { - let intent = AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: eps(0.0), - }; - assert_eq!( - preferred(&intent), - Implementation::Sketch { - kind: SketchKind::Kll, - params: SketchParams::Kll { k: 65_535 }, - } - ); - } - - // ── posterior_aware_size_params (issue #239, integration point 2) ────── - - fn count_intent(e: f64) -> AggIntent { - AggIntent::Count { accuracy: eps(e) } - } - - #[test] - fn posterior_aware_sizing_shrinks_width_under_stated_assumption() { - let intent = count_intent(0.01); - let worst_case = default_size_params(SketchKind::Cms, &intent, 0.01, 0.01); - let relaxed = posterior_aware_size_params( - SketchKind::Cms, - &intent, - 0.01, - 0.01, - ExpectedCaseSizing { - width_relaxation: 0.5, - }, - ); - match (worst_case, relaxed) { - ( - SketchParams::Cms { - width: w0, - depth: d0, - }, - SketchParams::Cms { - width: w1, - depth: d1, - }, - ) => { - assert!( - w1 < w0, - "expected relaxed width {w1} to be strictly smaller than worst-case {w0}" - ); - assert_eq!(d0, d1, "depth must be unaffected by width_relaxation"); - } - other => panic!("expected Cms/Cms pair, got {other:?}"), - } - } - - #[test] - fn posterior_aware_sizing_at_full_relaxation_matches_worst_case() { - // width_relaxation = 1.0 must reproduce default_size_params exactly - // — the "no risk taken" boundary. - let intent = count_intent(0.01); - let worst_case = default_size_params(SketchKind::Cms, &intent, 0.01, 0.01); - let relaxed = posterior_aware_size_params( - SketchKind::Cms, - &intent, - 0.01, - 0.01, - ExpectedCaseSizing { - width_relaxation: 1.0, - }, - ); - assert_eq!(worst_case, relaxed); - } - - #[test] - fn posterior_aware_sizing_invalid_relaxation_falls_back_to_worst_case() { - let intent = count_intent(0.01); - let worst_case = default_size_params(SketchKind::Cms, &intent, 0.01, 0.01); - for bad in [0.0, -0.5, 1.5, f64::NAN, f64::INFINITY] { - let relaxed = posterior_aware_size_params( - SketchKind::Cms, - &intent, - 0.01, - 0.01, - ExpectedCaseSizing { - width_relaxation: bad, - }, - ); - assert_eq!( - worst_case, relaxed, - "width_relaxation={bad} should fall back to the worst-case width" - ); - } - } - - #[test] - fn posterior_aware_sizing_applies_to_every_cms_family_kind() { - let cms_heap_intent = AggIntent::TopK { - k: 7, - accuracy: eps(0.01), - }; - let assumption = ExpectedCaseSizing { - width_relaxation: 0.25, - }; - // CountSketch - assert_eq!( - posterior_aware_size_params( - SketchKind::CountSketch, - &count_intent(0.01), - 0.01, - 0.01, - assumption - ), - SketchParams::CountSketch { - width: 68, - depth: 5 - }, // ceil(272 * 0.25) - ); - // CmsWithHeap / CountSketchWithHeap carry k through untouched. - match posterior_aware_size_params( - SketchKind::CmsWithHeap, - &cms_heap_intent, - 0.01, - 0.01, - assumption, - ) { - SketchParams::CmsWithHeap { - width, - depth, - heap_size, - } => { - assert_eq!(width, 68); - assert_eq!(depth, 5); - assert_eq!(heap_size, 7); - } - other => panic!("expected CmsWithHeap, got {other:?}"), - } - } - - #[test] - fn posterior_aware_sizing_leaves_non_cms_kinds_unchanged() { - // Kll/Hll/etc. have no width_relaxation concept — must be byte-for- - // byte identical to default_size_params. - let intent = default_quantile(0.99); - let assumption = ExpectedCaseSizing { - width_relaxation: 0.1, - }; - assert_eq!( - posterior_aware_size_params(SketchKind::Kll, &intent, 0.01, 0.01, assumption), - default_size_params(SketchKind::Kll, &intent, 0.01, 0.01), - ); - } - - #[test] - fn default_size_params_unchanged_by_new_function_existing() { - // Regression pin: default_size_params's own worst-case behavior for - // existing callers must be untouched by adding - // posterior_aware_size_params alongside it. - assert_eq!( - default_size_params(SketchKind::Cms, &count_intent(0.001), 0.001, 0.001), - SketchParams::Cms { - width: 2719, - depth: 7 - }, - ); - } -} diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 1cf35bc2..48322a7d 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -29,45 +29,39 @@ //! //! ## Status //! -//! Three real occupants and one stub: -//! -//! - [`implementation`] — [`implementation::implementations_for_with`] is -//! where every valid realization of an `AggIntent` gets decided: exhaustive -//! and ranked (most-preferred first via a `CostModel`), sized to the -//! `AccuracyTarget` (issue #98). Nothing else in this crate computes an -//! `Implementation` independently of this list. //! - [`replacement`] — the `TargetSubDAG`/`ReplacementSubDAG`/ //! `ReplacementStrategy` vocabulary `docs/design_docs/asap_aware_mapping.md` stubs out //! under "Key concepts (not yet implemented)", implemented for real (issue -//! #251, part of #33): [`replacement::SketchFamilyStrategy`] wraps -//! [`implementation::implementations_for_with`]'s list directly, keeping -//! *every* candidate as its own bound [`SummaryNode`](asap_types::post_asap::SummaryNode) -//! instead of just one — [`replacement::SharedSubtreeStrategy`] does the -//! same for the build-independently-vs-build-once-and-share choice at a -//! CSE-detected shared subtree. No search/ranking-across-a-whole-plan logic -//! lives here — see that module's docs for what's deliberately left to a -//! future Cascades/Volcano-style search engine. -//! - [`bind`] — has no "bind me one tree" entry point of its own. +//! #251, part of #33). One module, one step: +//! [`replacement::SketchFamilyStrategy::replacements`] both *decides* what +//! an `AggIntent` may become ([`replacement::implementations_for_with`], +//! exhaustive and ranked via a `CostModel`, sized to the `AccuracyTarget`) +//! and *constructs* each candidate's bound +//! [`SummaryNode`](asap_types::post_asap::SummaryNode) — every candidate +//! comes back, not just one. [`replacement::SharedSubtreeStrategy`] does +//! the analogous job for the build-independently-vs-build-once-and-share +//! choice at a CSE-detected shared subtree. No search/ranking-across-a- +//! whole-plan logic lives here — see that module's docs for what's +//! deliberately left to a future Cascades/Volcano-style search engine. +//! - [`bind`] — has no "bind me one tree" entry point for a single target. //! [`replacement::SketchFamilyStrategy`] is the only public way to get //! bound output for a target, and it always returns *every* candidate; a -//! caller that wants one answer takes the first entry itself. What -//! `bind` provides is the shared low-level primitive -//! ([`bind::bind_with_implementation`]) that turns one already-decided -//! candidate into a real [`SummaryNode`](asap_types::post_asap::SummaryNode), -//! plus [`bind::implement_workload`]/[`bind::implement_workload_with`], -//! which drive rank-and-take-first selection over a whole workload's -//! roots, memoized on `Rc` identity so two roots that +//! caller that wants one answer takes the first entry itself. What `bind` +//! provides is workload-wide orchestration: +//! [`bind::implement_workload`]/[`bind::implement_workload_with`], which +//! drive rank-and-take-first selection over a whole workload's roots, +//! memoized on `Rc` identity so two roots that //! `asap_types::pre_asap::cse::share_common_subtrees` already collapsed //! onto one shared subtree bind to one shared `SummaryNode` too (issue //! #212, #222, #223) — memoization needs one canonical decision per //! shared root to key sharing on, so that entry point is the one place a //! single-answer selection still lives. See the terminology section below -//! for why this crate's logical→physical step (and this module) are named -//! around "implementation" rather than "bind". +//! for why this crate's logical→physical step is named "implementation" +//! rather than "bind". //! - [`cost_model`] — the [`CostModel`](cost_model::CostModel) trait every //! deployment's cost-based sketch selection plugs into (issues #6, #33). //! `asap-plan` itself only ships [`DefaultCostModel`](cost_model::DefaultCostModel), -//! which preserves [`implementation`]'s built-in static preference order. +//! which preserves [`replacement`]'s built-in static preference order. //! //! ## Terminology — "bind" already means three different things nearby; //! this crate's own logical→physical step is named "implementation" instead @@ -75,21 +69,20 @@ //! `asap-plan` and its downstream consumers (e.g. `ASAPQuery-backend`'s //! `control_plane`) independently reused the word "bind" for three //! *different*, layer-specific meanings — none of which is what this -//! crate's [`implementation`]/[`bind`] modules do. To avoid becoming a +//! crate's [`replacement`]/[`bind`] modules do. To avoid becoming a //! fourth, colliding sense of the same word, this crate names its own //! logical intent → physical realization step after the term the //! query-optimization literature already uses for exactly that step: //! **implementation** (Cascades/Volcano's "implementation rule", logical → //! physical, as distinct from a *transformation rule*, logical → logical — -//! see Graefe, *The Cascades Framework for Query Optimization*) — and this -//! module is named after that same term for the same reason. +//! see Graefe, *The Cascades Framework for Query Optimization*). //! //! | Term | Stage | Meaning | Lives in | //! |---|---|---|---| //! | **Parse** | parse | text (PromQL/SQL) → AST | `asap-frontend-promql` / `asap-frontend-sql` | //! | **Bind #1** | name resolution | `ColumnRef` (a name) → `ColumnId` (a concrete schema column) — the classic RDBMS "Parse → **Bind** → Optimize" pipeline sense (e.g. SQL Server's query-processor terminology) | [`asap_types::pre_asap::binder::Binder`](https://docs.rs/asap-types) | -//! | **Implementation** — [`implementation::implementations_for_with`] | pre-ASAP → post-ASAP, *one node* | enumerating every concrete physical realization (a sketch family, an exact accumulator, or pass-through) for one [`AggIntent`](asap_types::pre_asap::agg_intent::AggIntent) | [`implementation`] | -//! | **Replacement** — [`replacement::SketchFamilyStrategy::replacements`] | pre-ASAP → post-ASAP, *one target, every candidate* | wrap each [`implementation::implementations_for_with`] candidate into its own bound [`SummaryNode`](asap_types::post_asap::SummaryNode), ranked — a caller wanting one answer takes the first entry itself | [`replacement`] | +//! | **Implementation** — `replacement::implementations_for_with` | pre-ASAP → post-ASAP, *one node* | enumerating every concrete physical realization (a sketch family, an exact accumulator, or pass-through) for one [`AggIntent`](asap_types::pre_asap::agg_intent::AggIntent) | [`replacement`] | +//! | **Replacement** — [`replacement::SketchFamilyStrategy::replacements`] | pre-ASAP → post-ASAP, *one target, every candidate* | wrap each `implementations_for_with` candidate into its own bound [`SummaryNode`](asap_types::post_asap::SummaryNode), ranked — a caller wanting one answer takes the first entry itself | [`replacement`] | //! | **`implement_workload`** — [`bind::implement_workload`] | pre-ASAP → post-ASAP, *whole workload* | walk every root of a `QueryWorkload`, keeping the first (`cost_model`-preferred) candidate per node, sharing one bound `SummaryNode` across roots CSE already collapsed onto one `Rc` — emits the complete post-ASAP `SummaryExpr`/`SummaryNode` DAG | [`bind`] | //! | **Bind #2** (downstream, not in this crate) | post-ASAP → deployment placement | a *deployment's* own physical binder, additionally deciding **placement** (edge vs. backend, wire format, …) — a genuinely different, deployment-specific decision this crate doesn't model at all | e.g. `control_plane::sketch_algebra::rules::bind_*` (as of this writing; expected to fold into that deployment's cost-model layer rather than stay a separate "bind" concept) | //! @@ -98,7 +91,7 @@ //! *available* `Implementation` satisfy a *required* one" — the way a //! database's materialized-view matching / "answering queries using //! views" layer does. It owns the *question*, not an *answer*: -//! [`implementation::Matcher`] is a trait with no default implementation and +//! [`replacement::Matcher`] is a trait with no default implementation and //! no shipped instance, the same shape as [`cost_model::CostModel`] and for //! the same reason — which `Implementation`s are actually *available* //! anywhere is entirely a downstream deployment's concern (an inventory @@ -112,13 +105,11 @@ pub mod bind; pub mod cost_model; -pub mod implementation; pub mod replacement; pub use bind::{implement_workload, implement_workload_with, ImplementError}; pub use cost_model::{CostModel, DefaultCostModel}; -pub use implementation::{implementations_for_with, summary_candidates, Implementation, Matcher}; pub use replacement::{ - Replacement, ReplacementStrategy, ReplacementSubDAG, SharedSubtreeStrategy, - SketchFamilyStrategy, TargetSubDAG, + summary_candidates, Implementation, Matcher, Replacement, ReplacementStrategy, + ReplacementSubDAG, SharedSubtreeStrategy, SketchFamilyStrategy, TargetSubDAG, }; diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index 66f97939..ae91de70 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -3,14 +3,33 @@ //! under "Key concepts (not yet implemented)", implemented for real (issue //! #251, part of #33). //! -//! ## This is where every `Implementation` gets decided — and the only place bound output comes from +//! ## One step, not two: `SketchFamilyStrategy::replacements()` decides *and* builds //! -//! [`implementation::implementations_for_with`] is the one place this crate -//! enumerates every valid [`Implementation`] for an [`AggIntent`], exhaustive -//! and ranked (most-preferred first via a [`CostModel`]). This module wraps -//! that list into the exhaustive-candidate shape a future search needs, and -//! is the *only* public way to get bound output for a target — see -//! "Selection" below. +//! For a bindable `Aggregate`, `SketchFamilyStrategy::replacements()` is the +//! single place this crate both decides what an `AggIntent` may become and +//! turns each of those candidates into a real, executable +//! [`ReplacementSubDAG`]: +//! +//! 1. **Decide**: [`implementations_for_with`] enumerates every valid +//! [`Implementation`] for the target's intent — exhaustive, and ranked +//! most-preferred-first via a [`CostModel`] (candidate sketch family/kind, +//! already sized to the target's own accuracy target: `Implementation::Sketch`'s +//! `params` are the output of inverting that accuracy target through +//! `CostModel::size_params`, not a placeholder filled in later). +//! 2. **Build**: for each candidate in that list, [`construct_summary`] +//! mechanically turns the already-decided `(kind, params)` into a real +//! [`SummaryNode`] — derives the child schema, resolves the summarized +//! column, builds the readout query, recurses into the child (via +//! [`crate::bind::select_and_bind`], so a nested aggregate gets its own +//! independent enumeration, never the outer target's forced choice), and +//! assembles the `SummaryAgg`/`SummaryEstimate` node. +//! +//! There is no separate decision step and construction step living in +//! different modules bridged by a named "given an `Implementation`, bind it" +//! function — step 2 is *not* a second decision (nothing about which +//! candidate to prefer happens there), it is mechanical construction that +//! has to run regardless of how `(kind, params)` were chosen, so it lives +//! directly inside the one method that needs it. //! //! - [`TargetSubDAG`] — a reference to a pre-ASAP [`QueryExpr`] node that is a //! candidate for replacement, plus how many places in the workload already @@ -22,34 +41,26 @@ //! (still logical, structurally different from the target but semantically //! equivalent) — see [`Replacement`] — plus a human-readable `rationale`. //! - [`ReplacementStrategy`] — `matches` + `replacements`, the same -//! extension-point shape [`CostModel`] and [`Matcher`](crate::implementation::Matcher) -//! already use in this crate (and the same shape issue #33's -//! applicability-rule framework, PR #247, uses for its own -//! `ApplicabilityRule`): a new replacement source is a new -//! `impl ReplacementStrategy`, not a restructuring of this trait or of any -//! existing strategy. `replacements` is **exhaustive, not ranked, not +//! extension-point shape [`CostModel`] and [`Matcher`] already use in this +//! crate (and the same shape issue #33's applicability-rule framework, PR +//! #247, uses for its own `ApplicabilityRule`): a new replacement source is +//! a new `impl ReplacementStrategy`, not a restructuring of this trait or of +//! any existing strategy. `replacements` is **exhaustive, not ranked, not //! filtered** — reporting "every valid candidate" is core's job; picking -//! the best one is left to the caller (see "Selection" below). -//! -//! ## Selection: this crate reports every candidate; a caller keeps what it wants +//! the best one is left to the caller. //! -//! [`SketchFamilyStrategy::replacements`] builds a `TargetSubDAG` for the -//! node being replaced, then enumerates [`implementation::implementations_for_with`]'s -//! ranked list, calling [`bind::bind_with_implementation`] once per -//! candidate — given an *already-decided* `Implementation`, turn it into a -//! `SummaryNode`. Every candidate comes back; nothing is discarded here. A -//! caller that wants one executable answer takes the first -//! (`cost_model`-preferred) entry itself (`.into_iter().next()`) — that -//! "keep the head" step now lives entirely on the calling side, not behind -//! a second module-level entry point. `bind.rs`'s own -//! [`bind::implement_workload`]/[`bind::implement_workload_with`] are the -//! one place inside this crate that still performs that take-first step -//! internally, because workload-wide CSE memoization needs one canonical -//! decision per shared root to key sharing on (see `bind.rs`'s own module -//! docs). Every other caller goes through `SketchFamilyStrategy::replacements` -//! directly and decides for itself. +//! A caller that wants one executable answer takes the first +//! (`cost_model`-preferred) entry off `replacements()` itself +//! (`.into_iter().next()`) — that "keep the head" step lives entirely on the +//! calling side, not behind a second module-level entry point. `bind.rs`'s +//! own [`crate::bind::implement_workload`]/[`crate::bind::implement_workload_with`] +//! are the one place inside this crate that still perform that take-first +//! step internally, because workload-wide CSE memoization needs one +//! canonical decision per shared root to key sharing on (see `bind.rs`'s own +//! module docs). Every other caller goes through +//! `SketchFamilyStrategy::replacements` directly and decides for itself. //! -//! This means an ordinary single-target bind now sizes and fully constructs +//! This means an ordinary single-target bind sizes and fully constructs //! *every* sketch candidate at every sketch-capable node (not just the one a //! caller keeps) — a deliberate tradeoff, made so there is exactly one place //! in this crate that decides what an `AggIntent` may become, at the cost of @@ -57,10 +68,10 @@ //! //! ## The two strategies, and why these two //! -//! - [`SketchFamilyStrategy`] wraps [`implementation::implementations_for_with`]'s -//! exhaustive, ranked list directly: for the same bindable-`Aggregate` -//! shape this crate binds (single intent, no `HAVING`), every entry -//! becomes its own bound candidate. +//! - [`SketchFamilyStrategy`] wraps [`implementations_for_with`]'s exhaustive, +//! ranked list directly: for the same bindable-`Aggregate` shape this crate +//! binds (single intent, no `HAVING`), every entry becomes its own bound +//! candidate. //! - [`SharedSubtreeStrategy`] wraps //! `asap_types::pre_asap::cse::share_common_subtrees`'s sharing decision. //! Wherever a [`TargetSubDAG`] already has two or more consumers (i.e. @@ -96,27 +107,33 @@ //! do — reusable, but wiring it up to feed this module's strategies //! automatically is part of the same future search engine, not this issue. //! This module's own tests build `TargetSubDAG`s directly, the same -//! hand-rolled-fixture style `bind.rs`/`implementation.rs`/`cost_model.rs`'s own -//! tests already use. -//! - **[`implementation::implementations_for_with`]'s own outward-facing -//! behavior is unchanged.** Same inputs still produce the same exhaustive, -//! ranked list. What changed is who calls it and how much of the result -//! they keep — see "Selection" above. +//! hand-rolled-fixture style `bind.rs`/`cost_model.rs`'s own tests already +//! use. +//! - **[`implementations_for_with`]'s own outward-facing behavior is +//! unchanged.** Same inputs still produce the same exhaustive, ranked +//! list — only its home moved (from a separate `implementation` module +//! into this one) and its own visibility dropped to module-private, since +//! [`SketchFamilyStrategy`] is now its only caller. +use asap_types::post_asap::{ + ExactKind, ExactParams, SamplingKind, SamplingParams, SketchKind, SketchParams, + SketchQuery as PostAsapSketchQuery, StatModelKind, StatModelParams, SummaryExpr, + SummaryFamilyType, SummaryField, SummaryNode, SummarySchema, WaveletKind, WaveletParams, +}; +use asap_types::pre_asap::agg_intent::{agg_is_mergeable, AggIntent}; +use asap_types::pre_asap::expr_ir::ColumnRef; +use asap_types::pre_asap::query_expr::{QueryExpr, Reduction}; +use asap_types::pre_asap::schema::Schema; +use asap_types::types::AccuracyTarget; use std::rc::Rc; -use asap_types::post_asap::SummaryNode; -use asap_types::pre_asap::agg_intent::AggIntent; -use asap_types::pre_asap::query_expr::QueryExpr; - -use crate::bind::{bind_with_implementation, bindable_intent}; +use crate::bind::{select_and_bind, ImplementError}; use crate::cost_model::{CostModel, DefaultCostModel}; -use crate::implementation::{implementations_for_with, Implementation}; /// A pre-ASAP sub-DAG a [`ReplacementStrategy`] knows how to replace. /// /// `root` is a reference into the workload's own [`QueryExpr`] tree (an -/// `Rc`, the same currency [`bind::implement_workload`] and +/// `Rc`, the same currency [`crate::bind::implement_workload`] and /// `asap_types::pre_asap::cse::share_common_subtrees` already thread through /// this crate's public API — not a bare `&QueryExpr` — so a strategy that /// needs the node's own `Rc` identity, not just its shape, has it available @@ -160,14 +177,13 @@ impl<'a> TargetSubDAG<'a> { /// What a [`ReplacementSubDAG`] actually substitutes a [`TargetSubDAG`] with. /// -/// Generalizes [`implementation::implementations_for_with`]'s two possible -/// *kinds* of answer — a post-ASAP binding decision, or a still-pre-ASAP -/// structural alternative — into "one candidate among several", each with -/// its own [`ReplacementSubDAG`]. +/// Generalizes [`implementations_for_with`]'s two possible *kinds* of answer +/// — a post-ASAP binding decision, or a still-pre-ASAP structural alternative +/// — into "one candidate among several", each with its own +/// [`ReplacementSubDAG`]. #[derive(Debug, Clone)] pub enum Replacement { - /// A fully bound post-ASAP summary decision — the same [`SummaryNode`] - /// shape [`bind::bind_with_implementation`] produces, for one particular + /// A fully bound post-ASAP summary decision, for one particular /// candidate realization of the target. Summary(Rc), /// A pre-ASAP rewrite: still a logical [`QueryExpr`], structurally @@ -192,9 +208,9 @@ pub struct ReplacementSubDAG { /// replacement (`replacements`)? /// /// The extension point this module exists for — the same shape -/// [`CostModel`] and [`crate::implementation::Matcher`] already use elsewhere in -/// this crate: a new replacement source is a new `impl ReplacementStrategy`, -/// no restructuring of this trait or any existing strategy required. +/// [`CostModel`] and [`Matcher`] already use elsewhere in this crate: a new +/// replacement source is a new `impl ReplacementStrategy`, no restructuring +/// of this trait or any existing strategy required. /// /// `replacements` is only meaningful when `matches` would return `true` for /// the same target; both [`SketchFamilyStrategy`] and [`SharedSubtreeStrategy`] @@ -211,6 +227,539 @@ pub trait ReplacementStrategy { fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec; } +// ── Implementation: how one AggIntent may be realised ─────────────────────── + +/// How an [`AggIntent`] may be realised at post-ASAP binding time (issue +/// #98): by an approximate summary (sketch, sample, wavelet, statistical +/// model, …), by an exact mergeable accumulator, or by an ordinary exact +/// operator (pass-through). This is a post-ASAP concern — the pre-ASAP IR +/// carries only the intent + accuracy target, never the realization — and +/// it's a per-node decision, made once per `AggIntent`, not a plan-wide one. +/// +/// [`implementations_for_with`] is where every valid realization gets +/// enumerated, exhaustive and ranked (most-preferred first) — this crate has +/// no separate function that computes just "the one" `Implementation` +/// independently of that list. [`SketchFamilyStrategy`] is the sole +/// consumer: it wraps every entry of this list into its own bound +/// [`SummaryNode`] and returns all of them, ranked — a caller wanting a +/// single answer keeps the first one itself (see the module docs above). +#[derive(Debug, Clone, PartialEq)] +pub enum Implementation { + /// An exact **mergeable** accumulator (partial state ≡ the value + /// itself: `Sum` / `Count` / `MinMax` / `Rate` / `Increase`). The + /// built state *is* the answer already — no `SummaryEstimate` readout + /// step. + ExactAggregate { + kind: ExactKind, + params: ExactParams, + }, + /// An approximate sketch sized to the intent's [`AccuracyTarget`]. + /// Needs a `SummaryEstimate` readout to recover a value. + Sketch { + kind: SketchKind, + params: SketchParams, + }, + /// A sampling-based summary (a retained row subset). Needs a + /// `SummaryEstimate` readout. Not chosen by any core `AggIntent` + /// dispatch today — see the module docs. + Sample { + kind: SamplingKind, + params: SamplingParams, + }, + /// A wavelet-transform summary. Needs a `SummaryEstimate` readout. Not + /// chosen by any core `AggIntent` dispatch today — see the module docs. + Wavelet { + kind: WaveletKind, + params: WaveletParams, + }, + /// A fitted statistical/parametric-model summary. Needs a + /// `SummaryEstimate` readout. Not chosen by any core `AggIntent` + /// dispatch today — see the module docs. + StatModel { + kind: StatModelKind, + params: StatModelParams, + }, + /// No summary form — the node stays a logical pre-ASAP operator and is + /// executed exactly (per-series transforms, non-mergeable reducers, exact + /// quantile/top-k/cardinality, classic-bucket `HistogramQuantile`, …). + PassThrough, +} + +/// Does an already-**available** [`Implementation`] — e.g. a summary +/// instance a downstream deployment already materialized somewhere, found +/// via whatever inventory/index that deployment keeps — satisfy a +/// **required** [`Implementation`] (one of the candidates +/// [`implementations_for_with`] produced for some [`AggIntent`])? +/// +/// This is the query-optimization-literature "materialized view matching" +/// / "answering queries using views" question, narrowed to this crate's +/// summary vocabulary: not "can I build this from scratch" (that's what +/// [`implementations_for_with`] answers) but "does something that already +/// exists answer this". +/// +/// `asap-plan` deliberately ships no implementation of this trait and no +/// default method body — unlike [`implementations_for_with`], which decision +/// an available `Implementation` satisfies a required one is not a fact this +/// crate can settle on its own. Two real, reasonable answers already +/// diverge outside this crate: +/// +/// - A **pure sketch-algebra** answer would say a `Sketch{kind: Kll, ..}` +/// requirement is satisfied by an available `DDSketch` (both quantile +/// sketches), and that a heap-bearing top-k sketch also answers a bare +/// frequency point-query (the heap is additional info on the same +/// underlying matrix) — but not the reverse. +/// - A **deployment with its own storage-layout rules** may need more: +/// e.g. whether a multi-population accumulator can serve a +/// single-population query via re-aggregation is a fact about that +/// deployment's storage layout, not about any summary family's kind at +/// all — a family's own kind doesn't encode grouping (grouping lives on +/// the post-ASAP node's `by` instead), so there is nothing in this +/// crate's own vocabulary to subsume. +/// +/// Implementations are expected to consult `required`/`available`'s +/// `kind` (and whatever grouping/placement context the deployment tracks +/// alongside `Implementation`, which this trait's signature doesn't carry +/// because this crate has no inventory concept to carry it in). +pub trait Matcher { + fn is_satisfied_by(&self, required: &Implementation, available: &Implementation) -> bool; +} + +/// Confidence δ assumed when the target carries only an ε +/// (`AccuracyTarget::Epsilon`): the (ε, δ)-parameterised sketches (CMS) need +/// one. `ln(1/0.01) → depth 5`, matching the conventional CMS sizing. +pub const DEFAULT_DELTA: f64 = 0.01; + +/// The sketch kinds that can serve an intent, most-preferred first. +/// This is the `AggIntent → SketchKind` map of issue #98; +/// [`implementations_for_with`] sizes and ranks every entry via `cost_model`. +/// Listed here so the candidate set has one home. +pub fn summary_candidates(intent: &AggIntent) -> &'static [SketchKind] { + match intent { + AggIntent::Quantile { .. } => &[SketchKind::Kll, SketchKind::DDSketch], + AggIntent::Cardinality { .. } => &[SketchKind::Hll, SketchKind::Theta, SketchKind::Kmv], + // Count-Sketch-with-heap is CMS-with-heap's balanced/zero-mean-error + // alternative for the same heavy-hitter shape. + AggIntent::TopK { .. } => &[SketchKind::CmsWithHeap, SketchKind::CountSketchWithHeap], + AggIntent::Count { .. } => &[SketchKind::Cms, SketchKind::CountSketch], + _ => &[], + } +} + +/// The [`AccuracyTarget`] threaded onto an approximate-capable intent +/// (`Quantile`/`Cardinality`/`Count`/`TopK`), or `None` for every other +/// intent (no sketch candidate applies — [`implementations_for_with`]'s own +/// match routes those elsewhere). Exposed so callers resolve the exact same +/// accuracy target [`implementations_for_with`] does, without re-deriving it +/// from scratch. +pub fn accuracy_target(intent: &AggIntent) -> Option<&AccuracyTarget> { + match intent { + AggIntent::Quantile { accuracy, .. } + | AggIntent::Cardinality { accuracy, .. } + | AggIntent::Count { accuracy } + | AggIntent::TopK { accuracy, .. } => Some(accuracy), + _ => None, + } +} + +/// Every valid [`Implementation`] for `intent`, exhaustive and ranked +/// (most-preferred first via `cost_model`) — the *only* place this crate +/// decides what an `AggIntent` may become. Nothing in this crate computes +/// "the one" `Implementation` independently of this list: +/// [`SketchFamilyStrategy`] keeps every entry as a candidate, and a caller +/// that wants a single executable answer takes the head of *that* strategy's +/// output itself. +/// +/// Exhaustive over the [`AggIntent`] vocabulary — adding a variant without an +/// explicit realization is a compile error, and the coverage-matrix test pins +/// each variant's category. +/// +/// Module-private: [`SketchFamilyStrategy::replacements`] is the only +/// caller — a caller outside this module has no use for the bare +/// `Implementation` list on its own, only for the bound +/// [`ReplacementSubDAG`]s that strategy produces from it. +fn implementations_for_with(intent: &AggIntent, cost_model: &dyn CostModel) -> Vec { + match intent { + // ── Approximate-capable intents — the AccuracyTarget decides ──────── + AggIntent::Quantile { accuracy, .. } + | AggIntent::Cardinality { accuracy, .. } + | AggIntent::Count { accuracy } + | AggIntent::TopK { accuracy, .. } => match accuracy { + AccuracyTarget::Exact => vec![exact_realization(intent)], + _ => sketch_implementations(intent, accuracy, cost_model), + }, + + // ── Exact mergeable accumulators ───────────────────────────────────── + AggIntent::Sum { .. } => vec![exact_accumulator(intent, ExactKind::Sum, ExactParams::Sum)], + AggIntent::Min { .. } | AggIntent::Max { .. } => { + vec![exact_accumulator( + intent, + ExactKind::MinMax, + ExactParams::MinMax, + )] + } + AggIntent::Rate => vec![exact_accumulator( + intent, + ExactKind::Rate, + ExactParams::Rate, + )], + AggIntent::Increase => { + vec![exact_accumulator( + intent, + ExactKind::Increase, + ExactParams::Increase, + )] + } + + // ── Exact, non-mergeable reducers — richer partial state than a + // single value (see `agg_is_mergeable`), so no accumulator form. + AggIntent::Avg { .. } | AggIntent::StdDev { .. } | AggIntent::Variance { .. } => { + vec![Implementation::PassThrough] + } + + // ── Classic-bucket histogram_quantile (#79): exact `le`-bucket + // interpolation over pre-aggregated counts — NOT re-sketchable. + // (The native/raw form lowers to the generic `Quantile` above.) + AggIntent::HistogramQuantile { .. } => vec![Implementation::PassThrough], + + // ── Per-series transforms and reductions with no sketch realization: + // counter-derivatives (#44), math (#45), time/calendar (#46), + // presence (#47), native-histogram accessors (#43), and the + // `*OverTime` reducers (#51). All exact by construction. + AggIntent::Changes + | AggIntent::Delta + | AggIntent::IDelta + | AggIntent::Deriv + | AggIntent::Resets + | AggIntent::PredictLinear { .. } + | AggIntent::DoubleExpSmoothing { .. } + | AggIntent::HistogramCount + | AggIntent::HistogramSum + | AggIntent::HistogramAvg + | AggIntent::HistogramStdDev + | AggIntent::HistogramStdVar + | AggIntent::HistogramFraction { .. } + | AggIntent::Math(_) + | AggIntent::Absent + | AggIntent::AbsentOverTime + | AggIntent::PresentOverTime + | AggIntent::TimeFn(_) + | AggIntent::LastOverTime + | AggIntent::FirstOverTime + | AggIntent::MadOverTime + | AggIntent::TsOfMinOverTime + | AggIntent::TsOfMaxOverTime + | AggIntent::TsOfFirstOverTime + | AggIntent::TsOfLastOverTime => vec![Implementation::PassThrough], + + // ── Group / count_values (#49): exact per `agg_is_exact`, but their + // output is structural (constant-1 / a synthesized label column), + // not a value a summary accumulator carries. + AggIntent::Group | AggIntent::CountValues { .. } => vec![Implementation::PassThrough], + + // ── Extension (deployment-model-specific, issue #131) — core has no + // realization opinion for a shape it doesn't know, so it defers + // entirely to the `CostModel` (issue #150): `realize_extension` + // defaults to `PassThrough`, preserving today's behavior for + // every deployment that doesn't override it. Core has no way to + // enumerate alternatives for an opaque deployment-defined shape, + // so this is always exactly one candidate. This is also the only + // path that can currently produce `Implementation::Sample`/ + // `Wavelet`/`StatModel` — see the module docs. + AggIntent::Extension { ext_kind, payload } => { + vec![cost_model.realize_extension(ext_kind, payload)] + } + } +} + +/// Exact realization of an approximate-capable intent whose target is +/// `AccuracyTarget::Exact`. `Count` has a mergeable exact accumulator; exact +/// quantile / top-k / cardinality have no single-value summary form (they +/// need the full multiset / heap / set) and pass through. +fn exact_realization(intent: &AggIntent) -> Implementation { + match intent { + AggIntent::Count { .. } => exact_accumulator(intent, ExactKind::Count, ExactParams::Count), + _ => Implementation::PassThrough, + } +} + +fn exact_accumulator(intent: &AggIntent, kind: ExactKind, params: ExactParams) -> Implementation { + // An exact accumulator is only sound when partial states merge + // (`agg(A ∪ B) = combine(agg(A), agg(B))`). + debug_assert!( + agg_is_mergeable(intent), + "accumulator for non-mergeable {intent:?}" + ); + Implementation::ExactAggregate { kind, params } +} + +/// Resolve an [`AccuracyTarget`] into the `(eps, delta)` budget +/// [`CostModel::size_params`] needs. Shared by [`sketch_implementations`] and +/// this crate's own sizing — one place this resolution happens, so nothing +/// can drift apart on it. +/// +/// `Exact` is unreachable via [`implementations_for_with`] (which routes +/// `Exact` to [`exact_realization`] instead); degrades to the tightest +/// parameters for a caller that resolves it directly anyway. +pub fn accuracy_budget(accuracy: &AccuracyTarget) -> (f64, f64) { + match accuracy { + AccuracyTarget::Exact => (f64::MIN_POSITIVE, DEFAULT_DELTA), + AccuracyTarget::Epsilon(e) => (*e, DEFAULT_DELTA), + AccuracyTarget::EpsilonDelta { epsilon, delta } => (*epsilon, *delta), + } +} + +/// Every candidate sketch [`Implementation`] for an approximate-capable +/// intent, sized to `accuracy` and ranked via `cost_model.rank_candidates` +/// (most-preferred first) — [`implementations_for_with`]'s Sketch branch. +fn sketch_implementations( + intent: &AggIntent, + accuracy: &AccuracyTarget, + cost_model: &dyn CostModel, +) -> Vec { + let (eps, delta) = accuracy_budget(accuracy); + let ranked = cost_model.rank_candidates(intent, summary_candidates(intent)); + ranked + .into_iter() + .map(|kind| { + let params = cost_model.size_params(kind.clone(), intent, eps, delta); + Implementation::Sketch { kind, params } + }) + .collect() +} + +/// `asap-plan`'s built-in `SketchParams` sizing, keyed off the resolved +/// `(eps, delta)` accuracy budget. [`CostModel::size_params`]'s default +/// body — factored out to a free function so a deployment's own +/// `CostModel` impl can still delegate to it for the candidates it +/// doesn't want to resize itself. +/// +/// Each formula inverts the sketch family's standard error bound to the +/// smallest parameter satisfying the target, clamped to the family's sane +/// range. A non-positive ε saturates to the clamp maximum (tightest +/// allowed). +pub fn default_size_params( + kind: SketchKind, + intent: &AggIntent, + eps: f64, + delta: f64, +) -> SketchParams { + match kind { + SketchKind::Kll => SketchParams::Kll { k: kll_k(eps) }, + SketchKind::Cms => SketchParams::Cms { + width: cms_width(eps), + depth: cms_depth(delta), + }, + SketchKind::Hll => SketchParams::Hll { + precision: hll_precision(eps), + }, + SketchKind::CmsWithHeap => { + let k = match intent { + AggIntent::TopK { k, .. } => *k, + _ => unreachable!("CmsWithHeap is only a TopK candidate"), + }; + SketchParams::CmsWithHeap { + width: cms_width(eps), + depth: cms_depth(delta), + heap_size: k as u32, + } + } + // Non-preferred candidates (DDSketch / Theta / Kmv / CountSketch / + // CountSketchWithHeap) are only reachable once a cost model picks + // them; sized here so that wiring is local. + SketchKind::DDSketch => SketchParams::DDSketch { alpha: eps }, + SketchKind::Theta => SketchParams::Theta { k: kmv_k(eps) }, + SketchKind::Kmv => SketchParams::Kmv { k: kmv_k(eps) }, + // Count-Sketch is CMS's balanced/zero-mean-error alternative — + // same (width, depth) shape, sized the same way for now (a + // Count-Sketch-specific bound uses an L2-norm error guarantee + // rather than CMS's L1-norm one; this is a placeholder pending + // that refinement, same status as the other non-preferred + // candidates above). + SketchKind::CountSketch => SketchParams::CountSketch { + width: cms_width(eps), + depth: cms_depth(delta), + }, + SketchKind::CountSketchWithHeap => { + let k = match intent { + AggIntent::TopK { k, .. } => *k, + _ => unreachable!("CountSketchWithHeap is only a TopK candidate"), + }; + SketchParams::CountSketchWithHeap { + width: cms_width(eps), + depth: cms_depth(delta), + heap_size: k as u32, + } + } + } +} + +/// A deployment's explicit bet about how "typical" (non-adversarial) its +/// workload's collision pattern is expected to be, consumed only by +/// [`posterior_aware_size_params`]. +/// +/// This is **not** derived from Chen et al.'s posterior-error-estimation +/// technique (issue #239, `asap_types::post_asap::query_time::error_estimation`) +/// — that technique computes a tighter bound *at query time* from a +/// sketch's real counter values, and this repo has no sketch runtime yet +/// for a real counter array to size against (see that module's docs, and +/// `asap_types::post_asap::query_time`'s module doc for why it's a +/// deliberately separate folder from this crate's own *plan-time* code). +/// This struct is this crate's own *plan-time* analogue of the same +/// underlying intuition — an expected-case (skewed / non-adversarial) +/// workload needs a smaller sketch than the adversarial worst case — +/// expressed as an explicit, caller-supplied assumption rather than +/// anything observed or proven. Issue #250 tracks actually connecting the +/// two: feeding query-time-observed posterior error back into a future +/// replan's `width_relaxation` instead of a bare caller guess. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ExpectedCaseSizing { + /// Fraction, in `(0, 1]`, of the traditional worst-case width + /// ([`cms_width`]) the caller is betting is enough. `1.0` (or any + /// value outside `(0, 1)`) reproduces the worst-case width exactly — + /// no risk taken. A smaller value shrinks the sketch proportionally, + /// at the cost documented on [`posterior_aware_size_params`]. + pub width_relaxation: f64, +} + +/// Opt-in alternative to [`default_size_params`] for the CMS-family kinds +/// (`Cms` / `CmsWithHeap` / `CountSketch` / `CountSketchWithHeap`): sizes +/// width to `assumption.width_relaxation` of the worst-case [`cms_width`], +/// trading the unconditional worst-case `(ε,δ)` guarantee for a smaller +/// sketch under an explicit, caller-stated non-adversarial-workload bet — +/// see [`ExpectedCaseSizing`]. +/// +/// **The tradeoff, spelled out:** [`default_size_params`]'s width guarantees +/// `Pr[error > ε·|F|₁] < δ` for *any* input, including an adversarial one +/// built to maximize collisions (§3.3 of the posterior-error-estimation +/// paper this issue is about — see +/// `asap_types::post_asap::query_time::error_estimation`'s module docs). +/// Shrinking +/// width below that only keeps the same `(ε,δ)` guarantee if the real +/// workload's collision load stays within `width_relaxation` of the +/// worst-case assumption — this function does not check that, cannot check +/// it (no data exists at plan time), and does not change the formal +/// guarantee's statement; it only changes how much hardware is spent +/// chasing it. Callers accept that gap explicitly by choosing +/// `width_relaxation < 1.0`. +/// +/// Depth ([`cms_depth`]) is left unchanged from [`default_size_params`]: +/// depth trades away confidence *exponentially* (`Pr[all r rows bad] = +/// p^r` — each extra row multiplies the failure probability down), a +/// differently-shaped and materially riskier tradeoff than width's linear +/// relaxation. Issue #239 asks for *a* tighter-sizing option under a +/// stated assumption, not a full redesign of the depth/width tradeoff +/// space, so depth relaxation is left as explicit future scope. +/// +/// For every `SketchKind` outside the CMS family, this is identical to +/// [`default_size_params`] — `width_relaxation` only ever touches the +/// [`cms_width`]-sized formulas this issue is about. +/// +/// [`default_size_params`]'s own behavior is completely unchanged by this +/// function's existence — this is a separate, additive entry point, never +/// called from [`default_size_params`] or [`implementations_for_with`]. +pub fn posterior_aware_size_params( + kind: SketchKind, + intent: &AggIntent, + eps: f64, + delta: f64, + assumption: ExpectedCaseSizing, +) -> SketchParams { + let relaxed_width = |eps: f64| -> u32 { + let base = cms_width(eps); + let f = assumption.width_relaxation; + if !(f.is_finite() && f > 0.0 && f < 1.0) { + return base; // out-of-range bet: no relaxation, fall back to worst case + } + saturating_ceil(base as f64 * f, 2, base) + }; + match kind { + SketchKind::Cms => SketchParams::Cms { + width: relaxed_width(eps), + depth: cms_depth(delta), + }, + SketchKind::CmsWithHeap => { + let k = match intent { + AggIntent::TopK { k, .. } => *k, + _ => unreachable!("CmsWithHeap is only a TopK candidate"), + }; + SketchParams::CmsWithHeap { + width: relaxed_width(eps), + depth: cms_depth(delta), + heap_size: k as u32, + } + } + SketchKind::CountSketch => SketchParams::CountSketch { + width: relaxed_width(eps), + depth: cms_depth(delta), + }, + SketchKind::CountSketchWithHeap => { + let k = match intent { + AggIntent::TopK { k, .. } => *k, + _ => unreachable!("CountSketchWithHeap is only a TopK candidate"), + }; + SketchParams::CountSketchWithHeap { + width: relaxed_width(eps), + depth: cms_depth(delta), + heap_size: k as u32, + } + } + // Every other kind is untouched by this issue's CMS-specific + // relaxation — defer to the existing formula verbatim. Spelled out + // exhaustively, matching `default_size_params`'s own match, rather + // than a wildcard arm: a future `SketchKind` variant then fails to + // compile *here* too, instead of silently inheriting worst-case + // sizing with no signal that this function never considered it. + SketchKind::Kll => default_size_params(kind, intent, eps, delta), + SketchKind::Hll => default_size_params(kind, intent, eps, delta), + SketchKind::DDSketch => default_size_params(kind, intent, eps, delta), + SketchKind::Theta => default_size_params(kind, intent, eps, delta), + SketchKind::Kmv => default_size_params(kind, intent, eps, delta), + } +} + +// ── Parameter sizing ────────────────────────────────────────────────────────── +// +// Each function inverts the sketch family's standard error bound to the +// smallest parameter satisfying the target, clamped to the family's sane +// range. A non-positive ε saturates to the clamp maximum (tightest allowed). + +/// KLL: rank error ε ≈ 2/k ⇒ `k = ⌈2/ε⌉`. ε = 0.01 → k = 200, matching the +/// design doc's worked example (`KLL{k=200}` satisfies ε=0.01). +fn kll_k(eps: f64) -> u32 { + saturating_ceil(2.0 / eps, 8, 65_535) +} + +/// HLL: standard error ≈ 1.04/√(2^p) ⇒ `p = ⌈log2((1.04/ε)²)⌉`. The default +/// `Cardinality` target (`asap-ir::default_cardinality`) inverts to p = 14. +fn hll_precision(eps: f64) -> u8 { + saturating_ceil((1.04 / eps).powi(2).log2(), 4, 18) as u8 +} + +/// CMS: over-count ≤ ε·N with width `w = ⌈e/ε⌉` columns. +fn cms_width(eps: f64) -> u32 { + saturating_ceil(std::f64::consts::E / eps, 2, 1 << 26) +} + +/// CMS: failure probability ≤ δ with depth `d = ⌈ln(1/δ)⌉` rows. +/// δ = 0.01 → depth 5. +fn cms_depth(delta: f64) -> u32 { + saturating_ceil((1.0 / delta).ln(), 1, 32) +} + +/// KMV / theta: relative error ≈ 1/√k ⇒ `k = ⌈1/ε²⌉`. +fn kmv_k(eps: f64) -> u32 { + saturating_ceil(1.0 / (eps * eps), 16, 1 << 26) +} + +/// `⌈x⌉` clamped to `[lo, hi]`; NaN / non-positive x saturate to `hi` +/// (a degenerate ε means "as accurate as this family goes"). +fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { + if !x.is_finite() || x <= 0.0 { + return hi; + } + (x.ceil() as u32).clamp(lo, hi) +} + // ── SketchFamilyStrategy ───────────────────────────────────────────────── /// A single static instance so [`SketchFamilyStrategy::default_cost_model`] @@ -219,9 +768,9 @@ pub trait ReplacementStrategy { /// every caller (same pattern `applicability::SketchApplicabilityRule` uses). static DEFAULT_COST_MODEL: DefaultCostModel = DefaultCostModel; -/// Wraps [`implementation::implementations_for_with`]'s exhaustive, ranked -/// list directly: for a bindable `Aggregate`, every valid candidate summary -/// realization as its own [`ReplacementSubDAG`]. +/// Wraps [`implementations_for_with`]'s exhaustive, ranked list directly: for +/// a bindable `Aggregate`, every valid candidate summary realization as its +/// own [`ReplacementSubDAG`]. /// /// Ranked (only to *order the enumeration*, never to drop a candidate) via a /// [`CostModel`] — [`DefaultCostModel`] unless constructed with @@ -245,7 +794,7 @@ impl SketchFamilyStrategy<'static> { impl<'a> SketchFamilyStrategy<'a> { /// A strategy that ranks/binds via `cost_model` instead of the built-in /// static preference order — the same customization point - /// [`implementation::implementations_for_with`] already offers. + /// [`implementations_for_with`] already offers. pub fn new(cost_model: &'a dyn CostModel) -> Self { Self { cost_model } } @@ -264,13 +813,12 @@ impl ReplacementStrategy for SketchFamilyStrategy<'_> { // separate dispatch needed here. Only `Sketch` has more than one // candidate in practice (every other variant's own dispatch produces // exactly one `Implementation`), but this loop doesn't need to know - // that; it just binds whatever the list contains. + // that; it just constructs whatever the list contains. implementations_for_with(intent, self.cost_model) .into_iter() .filter_map(|implementation| { let rationale = describe_implementation(intent, &implementation); - let node = - bind_with_implementation(target.root, implementation, self.cost_model).ok()?; + let node = construct_summary(target.root, implementation, self.cost_model).ok()?; Some(ReplacementSubDAG { replacement: Replacement::Summary(node), rationale, @@ -285,20 +833,19 @@ impl ReplacementStrategy for SketchFamilyStrategy<'_> { fn describe_implementation(intent: &AggIntent, implementation: &Implementation) -> String { match implementation { Implementation::Sketch { kind, .. } => format!( - "{} realizes as a {kind:?} sketch — one of implementation::summary_candidates' \ - alternatives for this intent \ - (asap_aware_mapping::implementation::implementations_for_with)", + "{} realizes as a {kind:?} sketch — one of summary_candidates' \ + alternatives for this intent (asap_aware_mapping::replacement::implementations_for_with)", describe_intent(intent) ), Implementation::ExactAggregate { kind, .. } => format!( "{} realizes as an exact {kind:?} accumulator — the only realization \ - implementation::implementations_for_with produces for this intent (no approximate \ + implementations_for_with produces for this intent (no approximate \ candidate applies)", describe_intent(intent) ), Implementation::PassThrough => format!( "{} has no summary realization and stays a logical pass-through — the \ - only realization implementation::implementations_for_with produces for this intent", + only realization implementations_for_with produces for this intent", describe_intent(intent) ), Implementation::Sample { kind, .. } => format!( @@ -321,11 +868,10 @@ fn describe_implementation(intent: &AggIntent, implementation: &Implementation) /// A short human-readable label for an `AggIntent`, for /// [`ReplacementSubDAG::rationale`] text. Not exhaustive by design (unlike -/// this crate's other `AggIntent` matches, e.g. -/// [`implementation::implementations_for_with`]'s) — this is prose for a -/// rationale string, not a decision, so an unlisted variant just falls back -/// to its `Debug` tag rather than forcing every future intent to be named -/// here too (same rationale, and same shape, as +/// this crate's other `AggIntent` matches, e.g. [`implementations_for_with`]'s) +/// — this is prose for a rationale string, not a decision, so an unlisted +/// variant just falls back to its `Debug` tag rather than forcing every +/// future intent to be named here too (same rationale, and same shape, as /// `applicability`'s own private `describe_intent`). fn describe_intent(intent: &AggIntent) -> String { match intent { @@ -337,6 +883,223 @@ fn describe_intent(intent: &AggIntent) -> String { } } +// ── Construction: turn one already-decided Implementation into a SummaryNode ─ + +/// The bindable shape [`SketchFamilyStrategy`] targets: a single intent, no +/// `HAVING`. A multi-intent node (SQL `SELECT SUM(a), AVG(b)`), or one with a +/// `HAVING` predicate (the filter would need the estimate first), stays +/// logical — see the module docs' "Conservative fallbacks" in `bind.rs`. +pub fn bindable_intent(node: &QueryExpr) -> Option<&AggIntent> { + if let QueryExpr::Aggregate { + measures, having, .. + } = node + { + if let ([intent], None) = (measures.as_slice(), having) { + return Some(intent); + } + } + None +} + +/// Construct `expr`'s [`ReplacementSubDAG`] payload for one already-decided +/// [`Implementation`] of its top intent — the mechanical half of +/// [`SketchFamilyStrategy::replacements`], called once per candidate that +/// method enumerates. +/// +/// `expr` must still be the [`bindable_intent`] shape for `implementation` to +/// have any effect; anything else falls back to [`crate::bind::logical`]. +/// Only `expr`'s own top-level decision is forced — recursion into `expr`'s +/// child goes back through [`crate::bind::select_and_bind`] (fresh candidate +/// enumeration, not a forced pick), so choosing one candidate for a target +/// never leaks into that target's own nested aggregates. +fn construct_summary( + expr: &QueryExpr, + implementation: Implementation, + cost_model: &dyn CostModel, +) -> Result, ImplementError> { + if let QueryExpr::Aggregate { + reduction, + measures, + having, + child, + .. + } = expr + { + // The bindable shape: exactly one intent, no HAVING. (Multi-intent + // nodes and HAVING stay logical — see `bindable_intent`.) + if let ([intent], None) = (measures.as_slice(), having) { + if let Some((family, estimate)) = summary_family(implementation) { + return construct_summary_agg( + expr, reduction, intent, child, family, estimate, cost_model, + ); + } + } + } + crate::bind::logical(expr) +} + +/// Translate an [`Implementation`] into the `(family, needs a +/// SummaryEstimate readout)` pair [`construct_summary_agg`] needs, or `None` +/// for `PassThrough` (the caller falls back to [`crate::bind::logical`]). +/// +/// Every family's partial state needs a readout to recover a value, except +/// `ExactAggregate` — its partial state *is* the value already, so no +/// estimate step follows it. +fn summary_family(implementation: Implementation) -> Option<(SummaryFamilyType, bool)> { + Some(match implementation { + Implementation::ExactAggregate { kind, params } => { + (SummaryFamilyType::ExactAggregate(kind, params), false) + } + Implementation::Sketch { kind, params } => (SummaryFamilyType::Sketch(kind, params), true), + Implementation::Sample { kind, params } => (SummaryFamilyType::Sample(kind, params), true), + Implementation::Wavelet { kind, params } => { + (SummaryFamilyType::Wavelet(kind, params), true) + } + Implementation::StatModel { kind, params } => { + (SummaryFamilyType::StatModel(kind, params), true) + } + Implementation::PassThrough => return None, + }) +} + +/// Emit `SummaryAgg` (recursively binding the child), plus the +/// `SummaryEstimate` readout when `estimate` is set. +#[allow(clippy::too_many_arguments)] +fn construct_summary_agg( + node: &QueryExpr, + reduction: &Reduction, + intent: &AggIntent, + child: &Rc, + family: SummaryFamilyType, + estimate: bool, + cost_model: &dyn CostModel, +) -> Result, ImplementError> { + let child_schema = child.output_schema()?; + // The single canonical pre-ASAP derivation (per-series vs cross-series, + // name overrides) already computes the row shape; binding only retypes + // the summary state column. + let per_series = matches!(reduction, Reduction::PerEntity); + let by: Vec = reduction + .group_keys() + .map(|g| g.to_vec()) + .unwrap_or_default(); + let out_schema = node.output_schema()?; + let state_idx = summary_col_index(&out_schema, &by, per_series); + + let col = summarised_column(intent, &child_schema); + let query = estimate.then(|| readout(intent, &col, cost_model)); + + let mut state_schema = lift(&out_schema); + if let Some(field) = state_schema.fields.get_mut(state_idx) { + field.dtype = family.clone(); + } + + // `reduction` is carried onto `SummaryAgg` verbatim — not flattened to a + // bare `Vec` — so `SummaryExecutor::find_candidates` can tell + // a genuine empty-`by` reduction apart from a per-entity shape with no + // grouping concept at all (issue #163). `construct_summary_agg` is the + // single place that decides this; nothing downstream re-derives it. + let agg = Rc::new(SummaryNode { + expr: SummaryExpr::SummaryAgg { + child: select_and_bind(child, cost_model)?, + family, + col, + reduction: reduction.clone(), + }, + schema: state_schema, + }); + match query { + // The readout: downstream of the estimate the schema is the plain + // pre-ASAP row shape again (the summary-state type does not + // propagate). + Some(query) => Ok(Rc::new(SummaryNode { + expr: SummaryExpr::SummaryEstimate { + summary_input: agg, + query, + }, + schema: lift(&out_schema), + })), + None => Ok(agg), + } +} + +/// Index of the summary-state column in the aggregate's output schema: +/// cross-series output is `by ++ [agg]` (the column after the keys); +/// a per-series reduction keeps every label and replaces the sample value +/// (named `value` — mirror `per_series_reduction_schema`'s fallback). +/// `per_series` is the caller's already-read `Reduction` (issue #165) — +/// this never re-derives it, so it can't disagree with the caller. +fn summary_col_index(out_schema: &Schema, by: &[usize], per_series: bool) -> usize { + if per_series { + out_schema + .column_id("value") + .or_else(|| (0..out_schema.columns.len()).find(|&i| Some(i) != out_schema.time_index)) + .unwrap_or(0) + } else { + by.len() + } +} + +/// The column fed into the summary: the intent's positional input column +/// resolved to a name against the child schema, or the PromQL sample value. +fn summarised_column(intent: &AggIntent, child_schema: &Schema) -> ColumnRef { + match intent + .input_col() + .and_then(|id| child_schema.columns.get(id)) + { + Some(c) => match &c.table { + Some(t) => ColumnRef::Qualified { + table: t.clone(), + name: c.name.clone(), + }, + None => ColumnRef::Named(c.name.clone()), + }, + None => ColumnRef::SampleValue, + } +} + +/// The `SummaryEstimate` readout for a summary-bound intent. +fn readout(intent: &AggIntent, col: &ColumnRef, cost_model: &dyn CostModel) -> PostAsapSketchQuery { + match intent { + AggIntent::Quantile { q, .. } => PostAsapSketchQuery::Quantile { q: *q }, + AggIntent::Cardinality { .. } => PostAsapSketchQuery::Cardinality, + AggIntent::TopK { k, .. } => PostAsapSketchQuery::TopK { k: *k }, + AggIntent::Count { .. } => PostAsapSketchQuery::PointCount { + key: col.clone(), + value: None, + }, + // Core doesn't know the shape of a deployment-specific `Extension` + // intent, so it can't build its readout either — delegate to the + // same `CostModel` that decided (via `realize_extension`) this + // intent gets a summary realization at all. See `readout_extension`'s + // doc for the invariant this depends on. + AggIntent::Extension { ext_kind, payload } => { + cost_model.readout_extension(ext_kind, payload, col) + } + other => { + unreachable!("no summary realization for {other:?} (implementations_for_with)") + } + } +} + +/// Lift a pre-ASAP [`Schema`] to a [`SummarySchema`] with every column +/// `SummaryFamilyType::Plain` — shared by [`construct_summary_agg`] and +/// [`crate::bind::logical`] (`pub(crate)` for that cross-module reuse). +pub(crate) fn lift(schema: &Schema) -> SummarySchema { + SummarySchema { + fields: schema + .columns + .iter() + .map(|c| SummaryField { + name: c.name.clone(), + dtype: SummaryFamilyType::Plain(c.dtype.clone()), + nullable: c.nullable, + }) + .collect(), + time_index: schema.time_index, + } +} + // ── SharedSubtreeStrategy ──────────────────────────────────────────────── /// Wraps `asap_types::pre_asap::cse::share_common_subtrees`'s sharing @@ -398,13 +1161,508 @@ impl ReplacementStrategy for SharedSubtreeStrategy { #[cfg(test)] mod tests { use super::*; - use asap_types::post_asap::SketchKind; - use asap_types::pre_asap::agg_intent::{default_cardinality, default_quantile}; - use asap_types::pre_asap::query_expr::{Reduction, Source}; - use asap_types::pre_asap::schema::{Column, DataType, Schema}; + use asap_types::pre_asap::agg_intent::{ + agg_is_exact, default_cardinality, default_quantile, MathFunc, TimeFunc, + }; + use asap_types::pre_asap::query_expr::{Reduction as ReductionTy, Source}; + use asap_types::pre_asap::schema::{Column, DataType, Schema as SchemaTy}; use asap_types::types::AccuracyTarget; use std::collections::HashMap; + fn eps(e: f64) -> AccuracyTarget { + AccuracyTarget::Epsilon(e) + } + + // ── implementations_for_with / sizing ─────────────────────────────── + + /// The most-preferred `Implementation` — `implementations_for_with(intent, + /// &DefaultCostModel)`'s head — for tests that only care about the + /// default pick, not the full candidate list. + fn preferred(intent: &AggIntent) -> Implementation { + implementations_for_with(intent, &DefaultCostModel) + .into_iter() + .next() + .expect("every intent has at least one Implementation") + } + + /// Shorthand for asserting the realization *category*. + #[derive(Debug, PartialEq)] + enum Cat { + Sketch(SketchKind), + Acc(ExactKind), + Pass, + } + + fn cat(intent: &AggIntent) -> Cat { + match preferred(intent) { + Implementation::ExactAggregate { kind, .. } => Cat::Acc(kind), + Implementation::Sketch { kind, .. } => Cat::Sketch(kind), + Implementation::PassThrough => Cat::Pass, + other => { + panic!("this coverage matrix expects only Exact/Sketch/PassThrough, got {other:?}") + } + } + } + + /// The `AggIntent → SummaryKind` coverage matrix (issue #98): every intent + /// variant maps to a sketch, an exact accumulator, or an explicit + /// pass-through. `implementations_for_with`'s match is exhaustive, so a + /// new variant cannot compile without a decision; this matrix pins what + /// each decision *is* (its preferred/first candidate). + #[test] + fn agg_intent_to_summary_kind_coverage_matrix() { + use AggIntent as A; + use Cat::*; + use ExactKind as E; + use SketchKind as K; + let matrix: Vec<(A, Cat)> = vec![ + // approximate-capable, at an ε target → sketch + (default_quantile(0.99), Sketch(K::Kll)), + (default_cardinality(), Sketch(K::Hll)), + ( + A::Count { + accuracy: eps(0.01), + }, + Sketch(K::Cms), + ), + ( + A::TopK { + k: 10, + accuracy: eps(0.01), + }, + Sketch(K::CmsWithHeap), + ), + // the same intents at Exact → exact realization + ( + A::Quantile { + col: None, + q: 0.5, + accuracy: AccuracyTarget::Exact, + }, + Pass, + ), + ( + A::Cardinality { + col: None, + accuracy: AccuracyTarget::Exact, + }, + Pass, + ), + ( + A::Count { + accuracy: AccuracyTarget::Exact, + }, + Acc(E::Count), + ), + ( + A::TopK { + k: 10, + accuracy: AccuracyTarget::Exact, + }, + Pass, + ), + // exact mergeable accumulators + (A::Sum { col: None }, Acc(E::Sum)), + (A::Min { col: None }, Acc(E::MinMax)), + (A::Max { col: None }, Acc(E::MinMax)), + (A::Rate, Acc(E::Rate)), + (A::Increase, Acc(E::Increase)), + // exact but non-mergeable → pass-through + (A::Avg { col: None }, Pass), + ( + A::StdDev { + col: None, + population: false, + }, + Pass, + ), + ( + A::Variance { + col: None, + population: true, + }, + Pass, + ), + // classic-bucket histogram_quantile is not re-sketchable (#79) + (A::HistogramQuantile { q: 0.99 }, Pass), + // counter-derivative / range-vector functions (#44) + (A::Changes, Pass), + (A::Delta, Pass), + (A::IDelta, Pass), + (A::Deriv, Pass), + (A::Resets, Pass), + (A::PredictLinear { seconds: 60.0 }, Pass), + ( + A::DoubleExpSmoothing { + smoothing: 0.5, + trend: 0.5, + }, + Pass, + ), + // native-histogram accessors (#43) + (A::HistogramCount, Pass), + (A::HistogramSum, Pass), + (A::HistogramAvg, Pass), + (A::HistogramStdDev, Pass), + (A::HistogramStdVar, Pass), + ( + A::HistogramFraction { + lower: 0.0, + upper: 1.0, + }, + Pass, + ), + // per-sample transforms (#45, #46) + presence (#47) + (A::Math(MathFunc::Abs), Pass), + (A::TimeFn(TimeFunc::Hour), Pass), + (A::Absent, Pass), + (A::AbsentOverTime, Pass), + (A::PresentOverTime, Pass), + // extended aggregations (#49) + (A::Group, Pass), + (A::CountValues { label: "v".into() }, Pass), + // additional range reducers (#51) + (A::LastOverTime, Pass), + (A::FirstOverTime, Pass), + (A::MadOverTime, Pass), + (A::TsOfMinOverTime, Pass), + (A::TsOfMaxOverTime, Pass), + (A::TsOfFirstOverTime, Pass), + (A::TsOfLastOverTime, Pass), + ]; + for (intent, expected) in &matrix { + assert_eq!(&cat(intent), expected, "realization for {intent:?}"); + } + // Every accumulator pick is mergeable; every sketch pick is on a + // genuinely approximate target (the `agg_is_*` helpers stay truthful). + for (intent, expected) in &matrix { + if let Cat::Acc(_) = expected { + assert!(agg_is_mergeable(intent), "{intent:?}"); + } + if let Cat::Sketch(_) = expected { + assert!( + !agg_is_exact(intent) || matches!(intent, AggIntent::Count { .. }), + "{intent:?} sketches only under an approximate target" + ); + } + } + } + + #[test] + fn accuracy_target_drives_the_boundary() { + // Same intent, three targets → three different decisions. + let exact = AggIntent::Quantile { + col: None, + q: 0.99, + accuracy: AccuracyTarget::Exact, + }; + assert_eq!(preferred(&exact), Implementation::PassThrough); + + let approx = default_quantile(0.99); // ε = 0.01 + assert_eq!( + preferred(&approx), + Implementation::Sketch { + kind: SketchKind::Kll, + params: SketchParams::Kll { k: 200 }, // design.md worked example + } + ); + + let looser = AggIntent::Quantile { + col: None, + q: 0.99, + accuracy: eps(0.05), + }; + assert_eq!( + preferred(&looser), + Implementation::Sketch { + kind: SketchKind::Kll, + params: SketchParams::Kll { k: 40 }, // ⌈2/0.05⌉ + } + ); + } + + #[test] + fn default_cardinality_inverts_to_hll_precision_14() { + // `default_cardinality` encodes HLL's standard error at p=14; the + // sizing must invert it back exactly. + assert_eq!( + preferred(&default_cardinality()), + Implementation::Sketch { + kind: SketchKind::Hll, + params: SketchParams::Hll { precision: 14 }, + } + ); + } + + #[test] + fn epsilon_delta_sizes_cms_depth() { + let intent = AggIntent::Count { + accuracy: AccuracyTarget::EpsilonDelta { + epsilon: 0.001, + delta: 0.001, + }, + }; + assert_eq!( + preferred(&intent), + Implementation::Sketch { + kind: SketchKind::Cms, + params: SketchParams::Cms { + width: 2719, + depth: 7 + }, // ⌈e/0.001⌉, ⌈ln 1000⌉ + } + ); + // Epsilon-only falls back to DEFAULT_DELTA → depth 5. + let intent = AggIntent::Count { + accuracy: eps(0.001), + }; + assert_eq!( + preferred(&intent), + Implementation::Sketch { + kind: SketchKind::Cms, + params: SketchParams::Cms { + width: 2719, + depth: 5 + }, + } + ); + } + + #[test] + fn topk_heap_size_tracks_k() { + let intent = AggIntent::TopK { + k: 25, + accuracy: eps(0.01), + }; + match preferred(&intent) { + Implementation::Sketch { + kind: SketchKind::CmsWithHeap, + params: + SketchParams::CmsWithHeap { + width, + depth, + heap_size, + }, + } => { + assert_eq!(heap_size, 25); + assert_eq!(width, 272); // ⌈e/0.01⌉ + assert_eq!(depth, 5); + } + other => panic!("expected CmsWithHeap, got {other:?}"), + } + } + + #[test] + fn candidate_lists_match_the_issue_map() { + assert_eq!( + summary_candidates(&default_quantile(0.5)), + &[SketchKind::Kll, SketchKind::DDSketch] + ); + assert_eq!( + summary_candidates(&default_cardinality()), + &[SketchKind::Hll, SketchKind::Theta, SketchKind::Kmv] + ); + assert_eq!( + summary_candidates(&AggIntent::TopK { + k: 5, + accuracy: eps(0.01) + }), + &[SketchKind::CmsWithHeap, SketchKind::CountSketchWithHeap] + ); + assert_eq!( + summary_candidates(&AggIntent::Count { + accuracy: eps(0.01) + }), + &[SketchKind::Cms, SketchKind::CountSketch] + ); + assert!(summary_candidates(&AggIntent::Rate).is_empty()); + } + + #[test] + fn implementations_for_with_enumerates_every_candidate_ranked() { + // Quantile's candidate list is [Kll, DDSketch] — implementations_for_with + // must return both, ranked with the DefaultCostModel's preferred + // (Kll) first. + let kinds: Vec = + implementations_for_with(&default_quantile(0.99), &DefaultCostModel) + .into_iter() + .map(|implementation| match implementation { + Implementation::Sketch { kind, .. } => kind, + other => panic!("expected Sketch, got {other:?}"), + }) + .collect(); + assert_eq!(kinds, vec![SketchKind::Kll, SketchKind::DDSketch]); + } + + #[test] + fn degenerate_epsilon_saturates_to_tightest_params() { + let intent = AggIntent::Quantile { + col: None, + q: 0.99, + accuracy: eps(0.0), + }; + assert_eq!( + preferred(&intent), + Implementation::Sketch { + kind: SketchKind::Kll, + params: SketchParams::Kll { k: 65_535 }, + } + ); + } + + // ── posterior_aware_size_params (issue #239, integration point 2) ────── + + fn count_intent(e: f64) -> AggIntent { + AggIntent::Count { accuracy: eps(e) } + } + + #[test] + fn posterior_aware_sizing_shrinks_width_under_stated_assumption() { + let intent = count_intent(0.01); + let worst_case = default_size_params(SketchKind::Cms, &intent, 0.01, 0.01); + let relaxed = posterior_aware_size_params( + SketchKind::Cms, + &intent, + 0.01, + 0.01, + ExpectedCaseSizing { + width_relaxation: 0.5, + }, + ); + match (worst_case, relaxed) { + ( + SketchParams::Cms { + width: w0, + depth: d0, + }, + SketchParams::Cms { + width: w1, + depth: d1, + }, + ) => { + assert!( + w1 < w0, + "expected relaxed width {w1} to be strictly smaller than worst-case {w0}" + ); + assert_eq!(d0, d1, "depth must be unaffected by width_relaxation"); + } + other => panic!("expected Cms/Cms pair, got {other:?}"), + } + } + + #[test] + fn posterior_aware_sizing_at_full_relaxation_matches_worst_case() { + // width_relaxation = 1.0 must reproduce default_size_params exactly + // — the "no risk taken" boundary. + let intent = count_intent(0.01); + let worst_case = default_size_params(SketchKind::Cms, &intent, 0.01, 0.01); + let relaxed = posterior_aware_size_params( + SketchKind::Cms, + &intent, + 0.01, + 0.01, + ExpectedCaseSizing { + width_relaxation: 1.0, + }, + ); + assert_eq!(worst_case, relaxed); + } + + #[test] + fn posterior_aware_sizing_invalid_relaxation_falls_back_to_worst_case() { + let intent = count_intent(0.01); + let worst_case = default_size_params(SketchKind::Cms, &intent, 0.01, 0.01); + for bad in [0.0, -0.5, 1.5, f64::NAN, f64::INFINITY] { + let relaxed = posterior_aware_size_params( + SketchKind::Cms, + &intent, + 0.01, + 0.01, + ExpectedCaseSizing { + width_relaxation: bad, + }, + ); + assert_eq!( + worst_case, relaxed, + "width_relaxation={bad} should fall back to the worst-case width" + ); + } + } + + #[test] + fn posterior_aware_sizing_applies_to_every_cms_family_kind() { + let cms_heap_intent = AggIntent::TopK { + k: 7, + accuracy: eps(0.01), + }; + let assumption = ExpectedCaseSizing { + width_relaxation: 0.25, + }; + // CountSketch + assert_eq!( + posterior_aware_size_params( + SketchKind::CountSketch, + &count_intent(0.01), + 0.01, + 0.01, + assumption + ), + SketchParams::CountSketch { + width: 68, + depth: 5 + }, // ceil(272 * 0.25) + ); + // CmsWithHeap / CountSketchWithHeap carry k through untouched. + match posterior_aware_size_params( + SketchKind::CmsWithHeap, + &cms_heap_intent, + 0.01, + 0.01, + assumption, + ) { + SketchParams::CmsWithHeap { + width, + depth, + heap_size, + } => { + assert_eq!(width, 68); + assert_eq!(depth, 5); + assert_eq!(heap_size, 7); + } + other => panic!("expected CmsWithHeap, got {other:?}"), + } + } + + #[test] + fn posterior_aware_sizing_leaves_non_cms_kinds_unchanged() { + // Kll/Hll/etc. have no width_relaxation concept — must be byte-for- + // byte identical to default_size_params. + let intent = default_quantile(0.99); + let assumption = ExpectedCaseSizing { + width_relaxation: 0.1, + }; + assert_eq!( + posterior_aware_size_params(SketchKind::Kll, &intent, 0.01, 0.01, assumption), + default_size_params(SketchKind::Kll, &intent, 0.01, 0.01), + ); + } + + #[test] + fn default_size_params_unchanged_by_new_function_existing() { + // Regression pin: default_size_params's own worst-case behavior for + // existing callers must be untouched by adding + // posterior_aware_size_params alongside it. + assert_eq!( + default_size_params(SketchKind::Cms, &count_intent(0.001), 0.001, 0.001), + SketchParams::Cms { + width: 2719, + depth: 7 + }, + ); + } + + // ── SketchFamilyStrategy / SharedSubtreeStrategy fixtures ─────────── + fn metric_scan(labels: &[&str]) -> QueryExpr { let mut columns = vec![ Column::new("ts", DataType::Timestamp, false), @@ -414,13 +1672,13 @@ mod tests { QueryExpr::Scan { source: Source::TimeSeries { metric: "m".into() }, predicates: vec![], - schema: Schema::with_time_index(columns, 0, vec![]), + schema: SchemaTy::with_time_index(columns, 0, vec![]), } } fn agg(by: Vec, intent: AggIntent, child: QueryExpr) -> QueryExpr { QueryExpr::Aggregate { - reduction: Reduction::by(by), + reduction: ReductionTy::by(by), measures: vec![intent], output_names: vec![], having: None, @@ -442,7 +1700,7 @@ mod tests { let strategy = SketchFamilyStrategy::default_cost_model(); let multi = Rc::new(QueryExpr::Aggregate { - reduction: Reduction::by(vec![2]), + reduction: ReductionTy::by(vec![2]), measures: vec![AggIntent::Sum { col: None }, AggIntent::Avg { col: None }], output_names: vec![], having: None, @@ -476,7 +1734,7 @@ mod tests { #[test] fn approximate_quantile_enumerates_every_summary_candidate() { - // Quantile's candidate list is [Kll, DDSketch] (implementation::summary_candidates) — + // Quantile's candidate list is [Kll, DDSketch] (summary_candidates) — // every entry must come back as its own bound SummaryNode candidate, // not just Kll (the CostModel-ranked head implementations_for_with commits to). let q = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); @@ -518,7 +1776,7 @@ mod tests { assert_eq!( kinds, vec![SketchKind::Hll, SketchKind::Theta, SketchKind::Kmv], - "expected every implementation::summary_candidates entry for Cardinality" + "expected every summary_candidates entry for Cardinality" ); } @@ -606,10 +1864,9 @@ mod tests { /// node's own decision — a nested aggregate underneath it still gets its /// own independent (`cost_model`-ranked) enumeration, not whatever the /// caller happened to pick for the outer target. This is the behavior - /// [`bind::bind_with_implementation`]'s recursion (via - /// `bind::select_and_bind`) gets for free: only the top node's - /// `Implementation` is ever forced from outside; the child is always - /// re-enumerated fresh. + /// [`construct_summary`]'s recursion (via `crate::bind::select_and_bind`) + /// gets for free: only the top node's `Implementation` is ever forced + /// from outside; the child is always re-enumerated fresh. #[test] fn enumerating_the_targets_candidates_does_not_leak_into_a_nested_aggregate() { // outer: quantile(0.99, ...) over inner: quantile(0.5, m) — both diff --git a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md index 03637a1b..5adde4ff 100644 --- a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md @@ -10,7 +10,7 @@ This guide explains how to extend ASAP-aware mapping in the current codebase. Us The focus here is the **current code interfaces and their contracts**. For the higher-level motivation and future search design, see the separate design document, [`docs/design_docs/asap_aware_mapping.md`](../design_docs/asap_aware_mapping.md). -Names such as `MyStrategy`, `MyCostModel`, and `PreferDDSketch` are illustrative; they do not ship with this crate. Samples that use real types—such as `SketchFamilyStrategy`, `SharedSubtreeStrategy`, and `bind_with_implementation`—are copied from `replacement.rs`, `bind.rs`, or `cost_model.rs`. +Names such as `MyStrategy`, `MyCostModel`, and `PreferDDSketch` are illustrative; they do not ship with this crate. Samples that use real types—such as `SketchFamilyStrategy`, `SharedSubtreeStrategy`, and `implementations_for_with`—are copied from `replacement.rs`, `bind.rs`, or `cost_model.rs`. If you only need to find the right extension point, start with the [extension map](#18-current-extension-map). If you are implementing a strategy, read sections 1–5 first. @@ -18,12 +18,11 @@ If you only need to find the right extension point, start with the [extension ma ## Terminology -Two terms are central to this guide: +One term is central to this guide: -- **Implementation**: one valid realization of an `AggIntent`. It may be an approximate sketch, an exact mergeable accumulator, or a pass-through with no summary. `implementation::implementations_for_with` enumerates the valid implementations for one node and ranks them. It does not walk the plan or select a winner. -- **Binding**: converting a chosen `Implementation` into an executable `SummaryNode`. The shared primitive is `bind_with_implementation`. For one target, `SketchFamilyStrategy::replacements()` binds every candidate. A caller that needs one result takes the first candidate. Workload-wide helpers—`implement_workload` and `implement_workload_with`—perform that selection internally so shared `Rc` roots receive one consistent decision. +- **Implementation**: one valid realization of an `AggIntent`. It may be an approximate sketch, an exact mergeable accumulator, or a pass-through with no summary. `implementations_for_with` (a `replacement.rs`-private function) enumerates the valid implementations for one node and ranks them. It does not walk the plan or select a winner. `SketchFamilyStrategy::replacements()` is its only caller: for one target, it constructs every candidate's `SummaryNode` directly and returns all of them — deciding and constructing happen in the same step, not two steps bridged by a separately-named function in another module. A caller that needs one result takes the first candidate. Workload-wide helpers—`implement_workload` and `implement_workload_with`—perform that selection internally so shared `Rc` roots receive one consistent decision. -In short: `implementation` enumerates, `ReplacementStrategy` packages and binds every candidate, and the caller selects when it needs a single executable answer. Section 3 shows the complete flow. +In short: `ReplacementStrategy` enumerates, packages, and binds every candidate, and the caller selects when it needs a single executable answer. Section 3 shows the complete flow. --- @@ -198,7 +197,7 @@ The crate cannot hardcode real deployment costs: `asap-plan` depends on `asap_ir fn rank_candidates(&self, intent: &AggIntent, candidates: &[SketchKind]) -> Vec; ``` -- **`size_params`** — choose parameters, such as sketch capacity, for an already-selected `SketchKind` and accuracy target `(eps, delta)`. It is separate from ranking so a deployment can customize sizing without changing family selection. The default is `implementation::default_size_params`. +- **`size_params`** — choose parameters, such as sketch capacity, for an already-selected `SketchKind` and accuracy target `(eps, delta)`. It is separate from ranking so a deployment can customize sizing without changing family selection. The default is `replacement::default_size_params`. ```rust fn size_params(&self, kind: SketchKind, intent: &AggIntent, eps: f64, delta: f64) -> SketchParams; @@ -256,44 +255,19 @@ A custom cost model does not necessarily need to override every hook. The curren ## 3. How the current pieces fit together -The crate has one source of truth for valid implementations and one public path for producing bound candidates: +The crate has one source of truth for valid implementations, and deciding and constructing a candidate happen in the same step — not two steps bridged by a separately-named function: -- `implementation::implementations_for_with(intent, cost_model)` returns every valid `Implementation`, ranked by preference. -- `replacement::SketchFamilyStrategy::replacements()` binds each entry and returns one `ReplacementSubDAG` per candidate. It does not discard any candidate. +- `SketchFamilyStrategy::replacements(target)`, for a bindable `Aggregate`, calls `implementations_for_with(intent, cost_model)` (a `replacement.rs`-private function) to get every valid `Implementation`, ranked by preference and already sized to the target's own accuracy target — then, for each candidate, constructs its bound `SummaryNode` directly (child schema, summarized column, readout, recursion into the child) and returns it as a `ReplacementSubDAG`. It does not discard any candidate. - A caller that needs one executable answer uses `.into_iter().next()` and handles the empty case. The crate's conservative fallback is `bind::logical`. In the [design document](../design_docs/asap_aware_mapping.md), each `ReplacementSubDAG` is a candidate. The complete `replacements()` result is the set of alternatives for one location in the plan. -**Tradeoff:** a single-target bind sizes and constructs every valid sketch candidate before the caller keeps the first one. This costs more than constructing only the preferred candidate, but it preserves one implementation-enumeration path and one binding path. Child nodes repeat selection independently, so a choice at one target does not force choices in nested aggregates. +**Tradeoff:** a single-target bind sizes and constructs every valid sketch candidate before the caller keeps the first one. This costs more than constructing only the preferred candidate, but it means there is exactly one place in the crate that decides what an `AggIntent` may become and exactly one place bound output comes from. Child nodes repeat selection independently (via `bind::select_and_bind`), so a choice at one target does not force choices in nested aggregates. This is not yet the whole-plan Cascades/Volcano-style search described in the design document. Today, `cost_model.rank_candidates` ranks one node at a time. Whole-plan candidate generation and selection remain future work. **One exception:** `bind::implement_workload` and `implement_workload_with` select the first candidate internally. Workload-wide CSE memoizes by `Rc` pointer identity, so roots that share an `Rc` must use the same canonical decision. Other callers use `SketchFamilyStrategy::replacements()` directly. -### The shared low-level primitive: `bind_with_implementation` - -`bind::bind_with_implementation(expr, implementation, cost_model)` converts an already-selected top-level `Implementation` into a `SummaryNode`, or falls back to a logical pass-through. `replacements()` calls it once per candidate: - -```text - implementations_for_with(intent, cost_model) - every valid Implementation, ranked - | - v - SketchFamilyStrategy::replacements() - keeps every candidate - | - v - bind_with_implementation(...) - once per candidate - | - v - N ReplacementSubDAGs - - a caller wanting one answer: replacements(...).into_iter().next() -``` - -`replacements(...)` returns candidates in preferred-first order. `.into_iter().next()` keeps the first and drops the rest. This makes the selection explicit at the call site. - ### Replacement-strategy path ```text @@ -304,12 +278,10 @@ ReplacementStrategy::matches(...) | v ReplacementStrategy::replacements(...) - | + | (SketchFamilyStrategy: decide via implementations_for_with, + | then construct each candidate's SummaryNode, in one method) +---------------------------------------------+ | | | -bind_with_implementation(..., candidate A, ...) ... - | | | - v v v candidate A candidate B candidate C ``` @@ -317,7 +289,7 @@ The important rule is: > Strategies should reuse existing decision and binding logic where possible instead of reimplementing it. -`SketchFamilyStrategy` follows this literally: it doesn't reimplement any part of binding — it hands `implementations_for_with`'s own list straight to `bind_with_implementation`, once per candidate. +`SketchFamilyStrategy` follows this literally: `implementations_for_with` is the single source of truth for what an `AggIntent` may become, and every candidate it produces gets constructed the same way. --- @@ -560,23 +532,23 @@ Target Aggregate extract AggIntent | v -implementation::implementations_for_with(...) +implementations_for_with(...) | v every ranked Implementation | v -bind each candidate separately +construct each candidate's SummaryNode separately | v Vec ``` -For an approximate quantile, the current candidate list includes both KLL and DDSketch — the strategy returns both, even though the cost model ranks one ahead of the other. For cases where `implementations_for_with` has only one realization, such as an exact accumulator or pass-through, the strategy returns that single realization. There is no separate per-category dispatch inside the strategy: whatever `implementations_for_with` produces, the strategy binds, one entry at a time. +For an approximate quantile, the current candidate list includes both KLL and DDSketch — the strategy returns both, even though the cost model ranks one ahead of the other. For cases where `implementations_for_with` has only one realization, such as an exact accumulator or pass-through, the strategy returns that single realization. There is no separate per-category dispatch inside the strategy: whatever `implementations_for_with` produces, the strategy constructs, one entry at a time. --- -### How each candidate actually gets bound: `bind_with_implementation` +### How each candidate actually gets constructed `SketchFamilyStrategy`'s whole `replacements()` body is one loop: @@ -590,7 +562,7 @@ fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { .filter_map(|implementation| { let rationale = describe_implementation(intent, &implementation); let node = - bind_with_implementation(target.root, implementation, self.cost_model) + construct_summary(target.root, implementation, self.cost_model) .ok()?; Some(ReplacementSubDAG { replacement: Replacement::Summary(node), @@ -601,11 +573,11 @@ fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { } ``` -`implementations_for_with` already did the hard part — enumerating and sizing every candidate, ranked. This loop's only job is to hand each one to `bind::bind_with_implementation(expr, implementation, cost_model)` — the shared low-level primitive every caller of this crate bottoms out in, whether it keeps one candidate or all of them (see §3) — and package the result. No per-candidate sizing logic lives here. +`implementations_for_with` already did the hard part — enumerating and sizing every candidate, ranked. This loop's only job is to hand each one to `construct_summary(expr, implementation, cost_model)` — a private helper in the same file, not a separately-named public bridge in a different module — which derives the child schema, resolves the summarized column, builds the readout, recurses into the child, and assembles the `SummaryNode`. No per-candidate sizing logic lives here: sizing already happened inside `implementations_for_with`. -Because only `expr`'s own top-level `Implementation` is ever supplied from outside — `bind_with_implementation` recurses into `expr`'s child via a fresh internal selection over that child's own candidates, not a forced one — enumerating a candidate for one target never leaks into that target's own nested aggregates. (An earlier version of this strategy forced ranking via a `CostModel`-wrapping adapter for the whole recursive bind, which had exactly that leak as a latent bug; today's design doesn't have the problem, because nothing below the top node is ever forced.) +Because only `expr`'s own top-level `Implementation` is ever supplied from outside — `construct_summary` recurses into `expr`'s child via `bind::select_and_bind`, a fresh internal selection over that child's own candidates, not a forced one — enumerating a candidate for one target never leaks into that target's own nested aggregates. (An earlier version of this strategy forced ranking via a `CostModel`-wrapping adapter for the whole recursive bind, which had exactly that leak as a latent bug; today's design doesn't have the problem, because nothing below the top node is ever forced.) -This is the pattern to preserve when adding another strategy that needs to enumerate alternatives through an API that normally chooses one: get the exhaustive list from the same enumeration function the single-answer path uses, and hand each entry to the shared low-level binding primitive — never wrap the `CostModel` to trick a ranked-first path into producing what you want. +This is the pattern to preserve when adding another strategy that needs to enumerate alternatives through an API that normally chooses one: get the exhaustive list from the same enumeration function the single-answer path uses, and construct each entry directly — never wrap the `CostModel` to trick a ranked-first path into producing what you want. --- @@ -1277,7 +1249,6 @@ Use this table to find the right place for a change. | Decide whether an available implementation satisfies a required one | `impl Matcher` | | Produce a normal (ranked-first) bound summary for one target | `SketchFamilyStrategy::replacements(...).into_iter().next()` | | Bind a whole workload's roots, sharing across CSE-collapsed roots | reuse `bind::implement_workload`/`implement_workload_with` | -| Bind a specific, already-chosen `Implementation` | reuse `bind::bind_with_implementation` | -| Enumerate valid sketch kinds | reuse `implementation::summary_candidates` | +| Enumerate valid sketch kinds | reuse `replacement::summary_candidates` | | Build a target with no workload context | `TargetSubDAG::new` | | Build a target with known sharing context | `TargetSubDAG::with_consumer_count` | From 9a696786cde2d19d4a05ac9a3dd31250c79e49a7 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 24 Aug 2026 10:35:35 -0600 Subject: [PATCH 21/49] refactor(types): rename SummaryExpr::Logical to KeepPreAsap "Logical" imports classic query-optimization-literature terminology ("logical vs physical plan") that's redundant with, and less precise than, the pre-ASAP/post-ASAP vocabulary this codebase already uses for the exact same distinction (asap_types::pre_asap, asap_types::post_asap). KeepPreAsap says directly what the variant does: it's the conservative fallback that keeps a pre-ASAP subtree as-is when no binding rule rewrote it into post-ASAP form. Pure rename, no behavior change: - SummaryExpr::Logical(Box) -> SummaryExpr::KeepPreAsap(...) in crates/types/src/post_asap/expr.rs, doc comment updated to lead with the "keep pre-ASAP form" framing. - bind::logical -> bind::keep_pre_asap in crates/asap-aware-mapping/src/bind.rs, plus every call site (replacement.rs's construct_summary fallback, devtools' show_post_asap_ir, and test code across asap-aware-mapping, frontend-promql, and integration-tests). - All match/construction sites of the enum variant updated (bind.rs, replacement.rs, cost_model.rs, and the l4_binding / l4_binding_sql / promql_corpus test suites), including ASCII-art IR diagrams and panic messages that named the old variant. - Docs updated: ASAP-aware-mapping-developer-guide.md and user-guide.md's references to bind::logical / Logical(...). Unrelated uses of the word "logical" (logical AND/OR, logical join, lib.rs's Cascades/Volcano "logical vs physical" terminology discussion) are untouched -- they're English words, not references to this variant. Co-Authored-By: Claude Sonnet 5 --- crates/asap-aware-mapping/src/bind.rs | 33 ++++++++++--------- crates/asap-aware-mapping/src/cost_model.rs | 4 +-- crates/asap-aware-mapping/src/replacement.rs | 10 +++--- crates/devtools/src/bin/show_post_asap_ir.rs | 6 ++-- .../tests/observability/promql_corpus.rs | 8 ++--- crates/integration-tests/tests/l4_binding.rs | 16 ++++----- .../integration-tests/tests/l4_binding_sql.rs | 18 +++++----- crates/types/src/post_asap/expr.rs | 11 ++++--- .../ASAP-aware-mapping-developer-guide.md | 2 +- docs/user-guide/user-guide.md | 4 +-- 10 files changed, 58 insertions(+), 54 deletions(-) diff --git a/crates/asap-aware-mapping/src/bind.rs b/crates/asap-aware-mapping/src/bind.rs index a0ea5b0c..b4530f8f 100644 --- a/crates/asap-aware-mapping/src/bind.rs +++ b/crates/asap-aware-mapping/src/bind.rs @@ -6,7 +6,7 @@ //! candidate [`ReplacementSubDAG`](crate::replacement::ReplacementSubDAG), //! ranked; a caller that wants a single executable answer takes the first //! entry itself (`.into_iter().next()`) and decides what to do if the list -//! is empty ([`logical`] is the same pass-through fallback this crate's own +//! is empty ([`keep_pre_asap`] is the same pass-through fallback this crate's own //! dispatch would otherwise use — see `crate::replacement`'s own module docs //! for how deciding *and* constructing each candidate both happen inside //! that one strategy). @@ -27,7 +27,7 @@ //! //! ## Conservative fallbacks //! -//! [`SummaryExpr::Logical`] boxes a whole pre-ASAP subtree — it has no +//! [`SummaryExpr::KeepPreAsap`] boxes a whole pre-ASAP subtree — it has no //! post-ASAP children — so a *logical* operator above a bindable aggregate //! (`Filter`/`BinaryOp`/… over a quantile) subsumes the aggregate into the //! logical wrapper unbound. Rewriting through logical parents is the @@ -89,7 +89,7 @@ pub(crate) fn select_and_bind( // intent has no realization `implementations_for_with` can't // produce — never happens, that match is exhaustive) — the same // conservative fallback `SketchFamilyStrategy::matches` uses. - None => logical(root), + None => keep_pre_asap(root), } } @@ -190,10 +190,10 @@ pub fn implement_workload_with( /// candidate for a target, or a deployment wants to force a node its own /// runtime can't actually implement — through the same fallback this /// crate's own dispatch uses, without duplicating the schema-lift logic. -pub fn logical(expr: &QueryExpr) -> Result, ImplementError> { +pub fn keep_pre_asap(expr: &QueryExpr) -> Result, ImplementError> { let schema = expr.output_schema()?; Ok(Rc::new(SummaryNode { - expr: SummaryExpr::Logical(Box::new(expr.clone())), + expr: SummaryExpr::KeepPreAsap(Box::new(expr.clone())), schema: crate::replacement::lift(&schema), })) } @@ -278,7 +278,7 @@ mod tests { #[test] fn quantile_binds_kll_wrapped_in_estimate() { // quantile by (job) (m) at ε=0.01 → Estimate(Quantile) over - // SummaryAgg(Kll{k:200}) over Logical(Scan). job = col 2. + // SummaryAgg(Kll{k:200}) over KeepPreAsap(Scan). job = col 2. let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); let root = bind(&q).unwrap(); @@ -320,7 +320,7 @@ mod tests { field(&summary_input.schema, "quantile_0_99").dtype, SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) ); - assert!(matches!(child.expr, SummaryExpr::Logical(ref e) + assert!(matches!(child.expr, SummaryExpr::KeepPreAsap(ref e) if matches!(**e, QueryExpr::Scan { .. }))); } @@ -439,7 +439,7 @@ mod tests { }; let q = agg(vec![], intent, metric_scan(&[])); let root = bind(&q).unwrap(); - assert!(matches!(root.expr, SummaryExpr::Logical(_))); + assert!(matches!(root.expr, SummaryExpr::KeepPreAsap(_))); } #[test] @@ -610,7 +610,7 @@ mod tests { inner_family, &SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) ); - assert!(matches!(leaf.expr, SummaryExpr::Logical(_))); + assert!(matches!(leaf.expr, SummaryExpr::KeepPreAsap(_))); } /// Issue #115: the summary is built over the intent's own input column. @@ -664,15 +664,15 @@ mod tests { let q = agg(vec![2], intent.clone(), metric_scan(&["job"])); let root = bind(&q).unwrap(); assert!( - matches!(root.expr, SummaryExpr::Logical(ref e) if **e == q), - "expected Logical passthrough for {intent:?}" + matches!(root.expr, SummaryExpr::KeepPreAsap(ref e) if **e == q), + "expected KeepPreAsap passthrough for {intent:?}" ); } } #[test] fn logical_parent_subsumes_bindable_child() { - // Filter over a bindable quantile: `Logical` has no post-ASAP + // Filter over a bindable quantile: `KeepPreAsap` has no post-ASAP // children, so the conservative fallback keeps the whole subtree // logical. let q = QueryExpr::Filter { @@ -684,7 +684,7 @@ mod tests { child: Rc::new(agg(vec![], default_quantile(0.99), metric_scan(&[]))), }; let root = bind(&q).unwrap(); - assert!(matches!(root.expr, SummaryExpr::Logical(ref e) if **e == q)); + assert!(matches!(root.expr, SummaryExpr::KeepPreAsap(ref e) if **e == q)); } #[test] @@ -695,7 +695,10 @@ mod tests { ScalarValue::Boolean(true), )))); } - assert!(matches!(bind(&q).unwrap().expr, SummaryExpr::Logical(_))); + assert!(matches!( + bind(&q).unwrap().expr, + SummaryExpr::KeepPreAsap(_) + )); let multi = QueryExpr::Aggregate { reduction: Reduction::by(vec![2]), @@ -706,7 +709,7 @@ mod tests { }; assert!(matches!( bind(&multi).unwrap().expr, - SummaryExpr::Logical(_) + SummaryExpr::KeepPreAsap(_) )); } diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index 10194c80..1564e5ec 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -258,7 +258,7 @@ pub trait CostModel { /// weight table), applied to whichever field of /// `candidate.bound_summary`'s output schema actually carries summary /// state (falls back to the cheapest, `Plain`, weight if none does — - /// e.g. `bound_summary` is a passthrough `Logical` node with nothing + /// e.g. `bound_summary` is a passthrough `KeepPreAsap` node with nothing /// summary-shaped to maintain). See `docs/design_docs/cse-cost-model-decision.md`. fn cse_shared_maintenance_cost(&self, candidate: &CseCandidate) -> Cost { let family = candidate @@ -433,7 +433,7 @@ mod tests { SummaryNode { expr: SummaryExpr::SummaryAgg { child: std::rc::Rc::new(SummaryNode { - expr: SummaryExpr::Logical(Box::new(scan())), + expr: SummaryExpr::KeepPreAsap(Box::new(scan())), schema: SummarySchema { fields: vec![], time_index: None, diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index ae91de70..626e82be 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -907,7 +907,7 @@ pub fn bindable_intent(node: &QueryExpr) -> Option<&AggIntent> { /// method enumerates. /// /// `expr` must still be the [`bindable_intent`] shape for `implementation` to -/// have any effect; anything else falls back to [`crate::bind::logical`]. +/// have any effect; anything else falls back to [`crate::bind::keep_pre_asap`]. /// Only `expr`'s own top-level decision is forced — recursion into `expr`'s /// child goes back through [`crate::bind::select_and_bind`] (fresh candidate /// enumeration, not a forced pick), so choosing one candidate for a target @@ -935,12 +935,12 @@ fn construct_summary( } } } - crate::bind::logical(expr) + crate::bind::keep_pre_asap(expr) } /// Translate an [`Implementation`] into the `(family, needs a /// SummaryEstimate readout)` pair [`construct_summary_agg`] needs, or `None` -/// for `PassThrough` (the caller falls back to [`crate::bind::logical`]). +/// for `PassThrough` (the caller falls back to [`crate::bind::keep_pre_asap`]). /// /// Every family's partial state needs a readout to recover a value, except /// `ExactAggregate` — its partial state *is* the value already, so no @@ -1084,7 +1084,7 @@ fn readout(intent: &AggIntent, col: &ColumnRef, cost_model: &dyn CostModel) -> P /// Lift a pre-ASAP [`Schema`] to a [`SummarySchema`] with every column /// `SummaryFamilyType::Plain` — shared by [`construct_summary_agg`] and -/// [`crate::bind::logical`] (`pub(crate)` for that cross-module reuse). +/// [`crate::bind::keep_pre_asap`] (`pub(crate)` for that cross-module reuse). pub(crate) fn lift(schema: &Schema) -> SummarySchema { SummarySchema { fields: schema @@ -1797,7 +1797,7 @@ mod tests { &replacements[0].replacement, Replacement::Summary(node) if matches!( node.expr, - asap_types::post_asap::SummaryExpr::Logical(_) + asap_types::post_asap::SummaryExpr::KeepPreAsap(_) ) )); assert!(replacements[0].rationale.contains("only realization")); diff --git a/crates/devtools/src/bin/show_post_asap_ir.rs b/crates/devtools/src/bin/show_post_asap_ir.rs index ec50b8ca..bfc153a4 100644 --- a/crates/devtools/src/bin/show_post_asap_ir.rs +++ b/crates/devtools/src/bin/show_post_asap_ir.rs @@ -5,7 +5,7 @@ // `asap-aware-mapping` pre-ASAP → post-ASAP binding pass and prints the // resulting **post-ASAP IR** (the sketch-bound IR: `SummaryExpr`/`SummaryNode` // — the concrete `SummaryKind`/`SummaryParams` committed per aggregate, or -// `Logical` for whatever the pass left untouched). See `show_pre_asap_ir` +// `KeepPreAsap` for whatever the pass left untouched). See `show_pre_asap_ir` // for the sketch-agnostic IR one layer upstream. // // File format: one query per line, prefixed with "sql>" or "promql>". @@ -20,7 +20,7 @@ // `metrics(ts, service, region, latency, bytes)` catalog — the same table // used in cross_language.rs and topk_ir.rs. -use asap_aware_mapping::bind::logical; +use asap_aware_mapping::bind::keep_pre_asap; use asap_aware_mapping::{ Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG, }; @@ -51,7 +51,7 @@ fn bind(expr: &QueryExpr) -> Result, Stri replacement: Replacement::Summary(node), .. }) => Ok(node), - _ => logical(&root).map_err(|e| e.to_string()), + _ => keep_pre_asap(&root).map_err(|e| e.to_string()), } } diff --git a/crates/frontend-promql/tests/observability/promql_corpus.rs b/crates/frontend-promql/tests/observability/promql_corpus.rs index babde60a..23d3cd43 100644 --- a/crates/frontend-promql/tests/observability/promql_corpus.rs +++ b/crates/frontend-promql/tests/observability/promql_corpus.rs @@ -15,7 +15,7 @@ use std::rc::Rc; -use asap_aware_mapping::bind::{logical, ImplementError}; +use asap_aware_mapping::bind::{keep_pre_asap, ImplementError}; use asap_aware_mapping::{ Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG, }; @@ -42,7 +42,7 @@ fn bind(expr: &QueryExpr) -> Result, ImplementError> { replacement: Replacement::Summary(node), .. }) => Ok(node), - _ => logical(&root), + _ => keep_pre_asap(&root), } } @@ -92,7 +92,7 @@ fn tally(corpus: &str) -> Tally { struct BindTally { /// Root bound to `SummaryAgg`/`SummaryEstimate` — the pass did something. transformed: usize, - /// Root stayed `Logical` — the pass left the query untouched. + /// Root stayed `KeepPreAsap` — the pass left the query untouched. unchanged: usize, /// [`bind`] returned `Err` (schema derivation failed). errored: usize, @@ -105,7 +105,7 @@ fn bind_tally(corpus: &str, accuracy: AccuracyTarget) -> BindTally { continue; }; match bind(&tree) { - Ok(bound) if matches!(bound.expr, SummaryExpr::Logical(_)) => t.unchanged += 1, + Ok(bound) if matches!(bound.expr, SummaryExpr::KeepPreAsap(_)) => t.unchanged += 1, Ok(_) => t.transformed += 1, Err(_) => t.errored += 1, } diff --git a/crates/integration-tests/tests/l4_binding.rs b/crates/integration-tests/tests/l4_binding.rs index 470feb8b..576cf053 100644 --- a/crates/integration-tests/tests/l4_binding.rs +++ b/crates/integration-tests/tests/l4_binding.rs @@ -9,7 +9,7 @@ use std::rc::Rc; -use asap_aware_mapping::bind::{logical, ImplementError}; +use asap_aware_mapping::bind::{keep_pre_asap, ImplementError}; use asap_aware_mapping::{ Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG, }; @@ -40,7 +40,7 @@ fn bind(expr: &QueryExpr) -> Result, ImplementError> { replacement: Replacement::Summary(node), .. }) => Ok(node), - _ => logical(&root), + _ => keep_pre_asap(&root), } } @@ -59,7 +59,7 @@ fn dtype<'a>(schema: &'a SummarySchema, name: &str) -> &'a SummaryFamilyType { /// SummaryEstimate { query: Quantile{0.99} } → {quantile_0_99: Float64} /// └─ SummaryAgg { Kll{k:200}, col: SampleValue } → {quantile_0_99: Sketch(Kll, {k:200})} /// └─ SummaryAgg { Rate, col: SampleValue } → {ts, value: ExactAggregate(Rate), …} -/// └─ Logical(TimeRange{5m} → Scan) → {ts, value} +/// └─ KeepPreAsap(TimeRange{5m} → Scan) → {ts, value} /// ``` /// /// The nested tree exercises both realizations: the approximate quantile @@ -146,11 +146,11 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { ); // The leaf: unrewritten pass-through — TimeRange marker over the Scan. - let SummaryExpr::Logical(logical_leaf) = &leaf.expr else { - panic!("expected Logical leaf, got {:?}", leaf.expr); + let SummaryExpr::KeepPreAsap(kept_leaf) = &leaf.expr else { + panic!("expected KeepPreAsap leaf, got {:?}", leaf.expr); }; - let QueryExpr::TimeRange { range, child: scan } = logical_leaf.as_ref() else { - panic!("expected TimeRange leaf, got {logical_leaf:?}"); + let QueryExpr::TimeRange { range, child: scan } = kept_leaf.as_ref() else { + panic!("expected TimeRange leaf, got {kept_leaf:?}"); }; assert_eq!(range.as_secs(), 300); assert!(matches!(scan.as_ref(), QueryExpr::Scan { .. })); @@ -196,7 +196,7 @@ fn promql_exact_workload_binds_accumulators_not_sketches() { lower_promql("avg(http_requests_total)", AccuracyTarget::Exact).expect("lowering failed"); let root = bind(&l3).expect("binding failed"); assert!( - matches!(root.expr, SummaryExpr::Logical(_)), + matches!(root.expr, SummaryExpr::KeepPreAsap(_)), "avg has no mergeable accumulator — stays logical" ); } diff --git a/crates/integration-tests/tests/l4_binding_sql.rs b/crates/integration-tests/tests/l4_binding_sql.rs index 6cfa97ca..4289a27c 100644 --- a/crates/integration-tests/tests/l4_binding_sql.rs +++ b/crates/integration-tests/tests/l4_binding_sql.rs @@ -18,7 +18,7 @@ //! `bind.rs`'s module docs on the "logical parent subsumes bindable child" //! conservative fallback); a `Project` at the root is exactly such a logical //! parent, so feeding a raw `lower_sql` result straight into [`bind`] always -//! yields a whole-tree `SummaryExpr::Logical` — never a genuine sketch or +//! yields a whole-tree `SummaryExpr::KeepPreAsap` — never a genuine sketch or //! accumulator binding. //! //! The tests below extract the inner `Aggregate` node the same way this @@ -30,7 +30,7 @@ use std::rc::Rc; -use asap_aware_mapping::bind::{logical, ImplementError}; +use asap_aware_mapping::bind::{keep_pre_asap, ImplementError}; use asap_aware_mapping::{ Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG, }; @@ -61,7 +61,7 @@ fn bind(expr: &QueryExpr) -> Result, ImplementError> { replacement: Replacement::Summary(node), .. }) => Ok(node), - _ => logical(&root), + _ => keep_pre_asap(&root), } } @@ -130,7 +130,7 @@ async fn sql_full_query_root_stays_logical_under_the_identity_projection() { ); let root = bind(&l3).expect("binding failed"); assert!( - matches!(root.expr, SummaryExpr::Logical(ref e) if **e == l3), + matches!(root.expr, SummaryExpr::KeepPreAsap(ref e) if **e == l3), "bind does not look inside a Project to find a bindable Aggregate \ child, so the whole Project{{Aggregate}} tree stays logical" ); @@ -142,7 +142,7 @@ async fn sql_full_query_root_stays_logical_under_the_identity_projection() { /// ```text /// SummaryEstimate { query: Quantile{0.99} } → {…: Float64} /// └─ SummaryAgg { Kll{k:200}, col: metrics.latency } → {…: Sketch(Kll, {k:200})} -/// └─ Logical(Scan) → {ts, service, latency, bytes} +/// └─ KeepPreAsap(Scan) → {ts, service, latency, bytes} /// ``` /// /// The SQL counterpart of `l4_binding.rs`'s @@ -210,10 +210,10 @@ async fn sql_quantile_binds_kll_sketch_over_named_column() { SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) ); - let SummaryExpr::Logical(logical_leaf) = &child.expr else { - panic!("expected Logical leaf, got {:?}", child.expr); + let SummaryExpr::KeepPreAsap(kept_leaf) = &child.expr else { + panic!("expected KeepPreAsap leaf, got {:?}", child.expr); }; - assert!(matches!(logical_leaf.as_ref(), QueryExpr::Scan { .. })); + assert!(matches!(kept_leaf.as_ref(), QueryExpr::Scan { .. })); assert!( child .schema @@ -316,7 +316,7 @@ async fn sql_exact_workload_binds_accumulators_not_sketches() { let agg = inner_aggregate(&l3); let root = bind(agg).expect("binding failed"); assert!( - matches!(root.expr, SummaryExpr::Logical(_)), + matches!(root.expr, SummaryExpr::KeepPreAsap(_)), "avg has no mergeable accumulator — stays logical" ); } diff --git a/crates/types/src/post_asap/expr.rs b/crates/types/src/post_asap/expr.rs index d2025423..1c9958e0 100644 --- a/crates/types/src/post_asap/expr.rs +++ b/crates/types/src/post_asap/expr.rs @@ -23,16 +23,17 @@ pub struct SummaryNode { /// Sketch-bound IR produced by post-ASAP binding rules. Those rules /// selectively replace logical aggregates and joins in the pre-ASAP /// `QueryExpr` with their summary-bound counterparts; everything not -/// rewritten passes through as `Logical(Box)`. +/// rewritten passes through as `KeepPreAsap(Box)`. /// /// Traversing from the root node yields a DAG; shared sub-expressions appear /// as multiple `Rc` references to the same `SummaryNode`. #[derive(Debug, Clone)] pub enum SummaryExpr { - /// Any pre-ASAP node no binding rule rewrote (e.g. `Filter`, `Project`, - /// `Sort`). Output schema is the inner node's schema, lifted to - /// `SummarySchema` with all fields as `SummaryFamilyType::Plain`. - Logical(Box), + /// A pre-ASAP subtree kept as-is — no binding rule rewrote it into + /// post-ASAP form (e.g. `Filter`, `Project`, `Sort`). Output schema is + /// the inner node's schema, lifted to `SummarySchema` with all fields as + /// `SummaryFamilyType::Plain`. + KeepPreAsap(Box), /// Summary aggregation. Post-ASAP binding chose `family` — which /// summary family (exact accumulator, sketch, sample, wavelet, or diff --git a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md index 5adde4ff..6df08942 100644 --- a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md @@ -258,7 +258,7 @@ A custom cost model does not necessarily need to override every hook. The curren The crate has one source of truth for valid implementations, and deciding and constructing a candidate happen in the same step — not two steps bridged by a separately-named function: - `SketchFamilyStrategy::replacements(target)`, for a bindable `Aggregate`, calls `implementations_for_with(intent, cost_model)` (a `replacement.rs`-private function) to get every valid `Implementation`, ranked by preference and already sized to the target's own accuracy target — then, for each candidate, constructs its bound `SummaryNode` directly (child schema, summarized column, readout, recursion into the child) and returns it as a `ReplacementSubDAG`. It does not discard any candidate. -- A caller that needs one executable answer uses `.into_iter().next()` and handles the empty case. The crate's conservative fallback is `bind::logical`. +- A caller that needs one executable answer uses `.into_iter().next()` and handles the empty case. The crate's conservative fallback is `bind::keep_pre_asap`. In the [design document](../design_docs/asap_aware_mapping.md), each `ReplacementSubDAG` is a candidate. The complete `replacements()` result is the set of alternatives for one location in the plan. diff --git a/docs/user-guide/user-guide.md b/docs/user-guide/user-guide.md index f5d755f0..ebf27f63 100644 --- a/docs/user-guide/user-guide.md +++ b/docs/user-guide/user-guide.md @@ -140,7 +140,7 @@ let Some(ReplacementSubDAG { replacement: Replacement::Summary(post_asap), .. }) candidates.into_iter().next() else { // No candidate (e.g. the node isn't a bindable Aggregate) — fall back to - // `asap_aware_mapping::bind::logical(&root)`, the same conservative + // `asap_aware_mapping::bind::keep_pre_asap(&root)`, the same conservative // pass-through this crate's own dispatch uses. panic!("no candidate for this target"); }; @@ -166,7 +166,7 @@ root, since sharing needs one canonical decision to key on. Match on `post_asap.expr` (a `SummaryExpr`): -- `Logical(Box)` — this subtree wasn't rewritten; execute it exactly. +- `KeepPreAsap(Box)` — this subtree wasn't rewritten; execute it exactly. - `SummaryAgg { summary, params, .. }` — an exact accumulator (`summary.is_exact()`) or an approximate sketch, sized to the query's `AccuracyTarget`. - `SummaryEstimate { summary_input, query }` — wraps a sketch `SummaryAgg`; `query` is what to From 55077da43ddc77f9bc5bd839d403fca1c96c3834 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 24 Aug 2026 11:23:31 -0600 Subject: [PATCH 22/49] refactor(asap-aware-mapping): merge search.rs into replacement.rs, add CostModel::estimate_cost ## Why PR #263 (feat/cascades-search-252) built a Cascades/Volcano-style MEMO search engine (search.rs: PlanSpace/MemoGroup/RankedGroup/ search_workload/search_workload_with) but was paused for lacking a concrete use case. That use case now exists: replacement's "always return every candidate" principle should hold at whole-workload granularity too, and a flat Vec of fully-materialized candidate DAGs is combinatorially infeasible there (N independently-choosable sites -> 2^N plans) -- search.rs already built the right MEMO structure for this. Its base predates the implementation.rs->replacement.rs merge and the Logical->KeepPreAsap rename, so its content is manually re-integrated here rather than merged/cherry-picked. ## What - bind.rs shrinks: select_and_bind and keep_pre_asap (both single-target-scoped helpers with no workload-level state) move into replacement.rs, next to construct_summary/SketchFamilyStrategy which are their only real callers. bind.rs keeps only ImplementError, implement_workload/implement_workload_with (unchanged bodies -- these own genuine cross-root Rc-identity memoization state a per-target module has no business holding), and its own tests. select_and_bind is pub(crate) (bind.rs is its only other caller). keep_pre_asap stays pub and is re-exported from bind.rs (`pub use crate::replacement::keep_pre_asap;`) so `asap_aware_mapping::bind::keep_pre_asap` keeps resolving for existing external callers (devtools, frontend-promql tests, integration-tests) -- pub(crate) can't be re-exported wider than its own crate, so this was the only option that doesn't break the build. - search.rs's content (MAX_SEARCH_ITERATIONS, MemoGroup, PlanSpace, RankedGroup, is_duplicate_rewrite/is_duplicate_summary, rank_group, cse_preference, default_strategies[_with], search_workload[_with], site discovery) moves directly into replacement.rs -- no `mod search`, flattened into the same namespace as TargetSubDAG/ Replacement/ReplacementStrategy/SketchFamilyStrategy/ SharedSubtreeStrategy. crate::bind::bindable_intent and crate::bind::logical (renamed keep_pre_asap on this branch's tip) become local references; search.rs's own bind_one helper is unified with select_and_bind (`select_and_bind(target, cost_model).ok()`) since both did near-identical take-first-or-fall-back-to-keep_pre_asap work. The doc-link to `implementation::implementations_for_with` (module-private, per the earlier implementation.rs merge) drops its `[...]` brackets, matching cost_model.rs's existing style for doc-links to private items. asap_types::pre_asap::cse::structural_hash/HashCache go from pub(crate) to pub -- search.rs's own is_duplicate_rewrite doc already says "made pub for exactly this reuse", assuming a change that never actually landed on this branch's base; landing it now is what makes the merged code compile. search.rs's own #[cfg(test)] mod tests merges into replacement.rs's existing test module: its metric_scan/agg fixture helpers and `Reduction`/`Schema`/`AggIntent`/DefaultCostModel imports were identical (or made redundant by `use super::*`) to what the existing module already had, so those are dropped and the merged tests reuse the existing helpers directly. - CostModel::estimate_cost(&self, candidate: &ReplacementSubDAG, target: &TargetSubDAG<'_>) -> f64: a new trait method exposing an actual numeric cost per candidate (today PlanSpace::cost_sorted only exposes relative order), covering both Replacement::Summary (SketchFamilyStrategy) and Replacement::Rewrite (SharedSubtreeStrategy) shapes. Default body returns f64::NAN, documented as an explicit "not a real cost model" placeholder -- not a breaking change for existing CostModel implementors. DefaultCostModel overrides it, reusing (not duplicating) cse_recompute_cost/ cse_shared_maintenance_cost -- the same arithmetic already backing cse_share_decision: a Summary candidate costs cse_recompute_cost(target) + cse_shared_maintenance_cost(candidate's bound family); a Rewrite candidate costs cse_shared_maintenance_cost (if it's the "share" candidate, i.e. Rc::ptr_eq to target) or cse_recompute_cost * consumer_count (if it's "recompute independently"), recovering a representative bound SummaryNode via select_and_bind when needed. RankedGroup gains a `costs: Vec` field, aligned index-for-index with `candidates`, populated by PlanSpace::cost_sorted by calling estimate_cost per already-ranked candidate -- purely additive, does not change cost_sorted's existing ranking behavior/tests. - lib.rs: no `pub mod search`; PlanSpace/MemoGroup/RankedGroup/ search_workload/search_workload_with/default_strategies[_with]/ MAX_SEARCH_ITERATIONS fold into the existing `pub use replacement::{ ... }` block. `## Status`/`## Terminology` rewritten: the `replacement` bullet now covers decide+construct+search-across-a- workload as one module's job (the old "no search/ranking-across-a- whole-plan logic lives here" language is gone); a new Terminology row covers workload-wide search, reusing search.rs's own MEMO-groups framing; the `bind` bullet explains the shrink and why implement_workload/implement_workload_with still deserve their own module (genuine cross-root memoization state). - Developer guide: bind::keep_pre_asap/bind::select_and_bind references become replacement::keep_pre_asap/replacement:: select_and_bind; the "not yet the whole-plan Cascades/Volcano-style search" paragraph is rewritten to describe search_workload/ PlanSpace as implemented; the extension map gains rows for search_workload, PlanSpace::cost_sorted, and CostModel::estimate_cost. docs/design_docs/asap_aware_mapping.md was checked and left as-is -- it's a timeless design document with no "not yet implemented" language to update, and no literal "Pseudocode for Replacement Plan Searching" heading to annotate. ## Output $ cargo build --workspace --all-targets && cargo test --workspace \ && cargo fmt --all -- --check \ && cargo clippy --workspace --all-targets --all-features -- -D warnings (all clean; asap-aware-mapping: 69 passed, 0 failed) PR #263 (feat/cascades-search-252) is superseded by this merge -- its content now lives inside this branch. Left open for the user to close. Co-Authored-By: Claude Sonnet 5 --- crates/asap-aware-mapping/src/bind.rs | 84 +- crates/asap-aware-mapping/src/cost_model.rs | 206 ++- crates/asap-aware-mapping/src/lib.rs | 64 +- crates/asap-aware-mapping/src/replacement.rs | 1325 ++++++++++++++++- crates/types/src/pre_asap/cse.rs | 20 +- .../ASAP-aware-mapping-developer-guide.md | 17 +- 6 files changed, 1599 insertions(+), 117 deletions(-) diff --git a/crates/asap-aware-mapping/src/bind.rs b/crates/asap-aware-mapping/src/bind.rs index b4530f8f..8e524c96 100644 --- a/crates/asap-aware-mapping/src/bind.rs +++ b/crates/asap-aware-mapping/src/bind.rs @@ -20,8 +20,8 @@ //! the same `Rc` must resolve to the *same* canonical decision to //! be shareable at all — there is no meaningful "N candidates" answer to //! memoize against. So this one entry point keeps a rank-and-take-first -//! behavior internally (via the private [`select_and_bind`]), scoped to -//! workload-wide CSE memoization specifically. It is not a general +//! behavior internally (via [`crate::replacement::select_and_bind`]), scoped +//! to workload-wide CSE memoization specifically. It is not a general //! "bind me one tree" API — for a single target, go through //! `SketchFamilyStrategy` and decide what to keep yourself. //! @@ -35,17 +35,34 @@ //! conservative: multi-intent `Aggregate` nodes (SQL `SELECT SUM(a), AVG(b)`) //! and aggregates with a `HAVING` predicate (the filter would need the //! estimate first) stay logical — see [`crate::replacement::bindable_intent`]. +//! +//! ## Why this module is so small +//! +//! `select_and_bind`/`keep_pre_asap` themselves moved into +//! [`crate::replacement`] — both are single-target-scoped helpers with no +//! real workload-level state, the same category as everything else already +//! living there. [`keep_pre_asap`] is re-exported here (`pub use +//! crate::replacement::keep_pre_asap`) so `bind::keep_pre_asap` keeps +//! resolving for existing external callers that reach it through this +//! module's path; `select_and_bind` stays `pub(crate)` in `replacement.rs`, +//! reachable from this module (its only other crate-internal caller, +//! [`implement_workload_with`]) without a public re-export. What's left +//! here — [`ImplementError`], [`implement_workload`]/ +//! [`implement_workload_with`], and this module's own tests — genuinely +//! belongs in its own module: `implement_workload`/`implement_workload_with` +//! own cross-root `Rc`-identity memoization state a per-target module has +//! no business holding (see "Why `implement_workload`/`implement_workload_with` +//! are here" above). use std::rc::Rc; -use asap_types::post_asap::{SummaryExpr, SummaryNode}; +use asap_types::post_asap::SummaryNode; use asap_types::pre_asap::query_expr::{QueryExpr, QueryExprError}; use thiserror::Error; use crate::cost_model::{CostModel, CseCandidate, DefaultCostModel, ShareDecision}; -use crate::replacement::{ - Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG, -}; +pub use crate::replacement::keep_pre_asap; +use crate::replacement::select_and_bind; /// Errors from the pre-ASAP → post-ASAP binding pass. #[derive(Debug, Error)] @@ -55,44 +72,6 @@ pub enum ImplementError { Schema(#[from] QueryExprError), } -/// Rank-and-take-first selector, scoped to [`implement_workload_with`]'s -/// CSE memoization and to [`crate::replacement`]'s own recursion into a -/// node's child (see the module docs' "Why `implement_workload`/ -/// `implement_workload_with` are here") — **not** a general single-answer -/// API. `root` must already be the caller's own `Rc`, never fabricated per -/// call, so this never allocates beyond what the caller already held. -/// -/// `pub(crate)`: [`crate::replacement`]'s own construction helper calls this -/// to bind a target's child, so a nested aggregate gets its own independent -/// enumeration instead of inheriting the parent's forced candidate. -pub(crate) fn select_and_bind( - root: &Rc, - cost_model: &dyn CostModel, -) -> Result, ImplementError> { - let target = TargetSubDAG::new(root); - match SketchFamilyStrategy::new(cost_model) - .replacements(&target) - .into_iter() - .next() - { - Some(ReplacementSubDAG { - replacement: Replacement::Summary(node), - .. - }) => Ok(node), - Some(ReplacementSubDAG { - replacement: Replacement::Rewrite(_), - .. - }) => { - unreachable!("SketchFamilyStrategy never returns a Rewrite candidate") - } - // No candidate at all: `root` isn't `bindable_intent` shape (or its - // intent has no realization `implementations_for_with` can't - // produce — never happens, that match is exhaustive) — the same - // conservative fallback `SketchFamilyStrategy::matches` uses. - None => keep_pre_asap(root), - } -} - /// Bind a whole workload's worth of already-CSE'd roots /// ([`asap_types::pre_asap::cse::share_common_subtrees`]'s output), reusing /// one bound [`SummaryNode`] wherever two roots share the same `Rc` *and* @@ -184,25 +163,12 @@ pub fn implement_workload_with( .collect() } -/// Wrap an unrewritten pre-ASAP subtree, lifting its schema with every column -/// `SummaryFamilyType::Plain`. Public so a caller can fall back to this -/// explicitly — e.g. when `SketchFamilyStrategy::replacements()` returns no -/// candidate for a target, or a deployment wants to force a node its own -/// runtime can't actually implement — through the same fallback this -/// crate's own dispatch uses, without duplicating the schema-lift logic. -pub fn keep_pre_asap(expr: &QueryExpr) -> Result, ImplementError> { - let schema = expr.output_schema()?; - Ok(Rc::new(SummaryNode { - expr: SummaryExpr::KeepPreAsap(Box::new(expr.clone())), - schema: crate::replacement::lift(&schema), - })) -} - #[cfg(test)] mod tests { use super::*; use asap_types::post_asap::{ - ExactKind, ExactParams, SketchKind, SketchParams, SketchQuery, SummaryFamilyType, + ExactKind, ExactParams, SketchKind, SketchParams, SketchQuery, SummaryExpr, + SummaryFamilyType, }; use asap_types::pre_asap::agg_intent::{default_quantile, AggIntent}; use asap_types::pre_asap::expr_ir::{ColumnRef, CompareOpKind, ScalarValue}; diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index 1564e5ec..d7a6f4a3 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -44,6 +44,8 @@ //! that forces detection to stay cost-agnostic). [`bind::implement_workload_with`](crate::bind::implement_workload_with) //! is the caller. +use std::rc::Rc; + use asap_types::post_asap::{ SketchKind, SketchParams, SketchQuery, SummaryFamilyType, SummaryNode, }; @@ -51,7 +53,9 @@ use asap_types::pre_asap::agg_intent::AggIntent; use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::query_expr::QueryExpr; -use crate::replacement::Implementation; +use crate::replacement::{ + select_and_bind, Implementation, Replacement, ReplacementSubDAG, TargetSubDAG, +}; /// A CSE-detected, legality-gated shared subtree with two or more consumers /// — the unit [`CostModel::cse_share_decision`] decides over. Built by @@ -296,6 +300,45 @@ pub trait CostModel { ShareDecision::RecomputeIndependently } } + + /// Estimate a comparable, numeric cost for one already-constructed + /// [`ReplacementSubDAG`] candidate at `target` — a real `f64`, not just a + /// relative rank, meant for a caller that wants to *display* "candidate A + /// costs ≈ X, candidate B costs ≈ Y" (e.g. a DAG-visualization view built + /// on [`PlanSpace::cost_sorted`](crate::replacement::PlanSpace::cost_sorted)), + /// not just order candidates against each other — that ordering job + /// already belongs to [`rank_candidates`](Self::rank_candidates) (for a + /// [`SketchFamilyStrategy`](crate::replacement::SketchFamilyStrategy) + /// group) and [`cse_share_decision`](Self::cse_share_decision) (for a + /// [`SharedSubtreeStrategy`](crate::replacement::SharedSubtreeStrategy) + /// group). + /// + /// One method covers both candidate shapes this crate ships: + /// `candidate.replacement`'s [`Replacement::Summary`] arm (a + /// `SketchFamilyStrategy` candidate — the bound `SummaryNode` is right + /// there, nothing to reconstruct) and its [`Replacement::Rewrite`] arm + /// (a `SharedSubtreeStrategy` share-vs-recompute candidate — no bound + /// `SummaryNode` of its own, since sharing is a decision about a target + /// already bound some other way; a representative binding is recovered + /// from `target` itself). `target` is threaded through explicitly + /// (rather than only ever the target embedded in `candidate` — there + /// isn't one for a `Rewrite`) so both arms have the `consumer_count` + /// context a cost estimate needs to be meaningful. + /// + /// Default: **not a real cost model** — always returns `f64::NAN`. + /// `f64::partial_cmp` against `NAN` is always `None`, so a caller that + /// forgot to check whether its `CostModel` actually overrides this can't + /// silently treat the placeholder as a real comparison. A deployment + /// that wants numeric costs exposed should override this method; + /// [`DefaultCostModel`] does, reusing + /// [`cse_recompute_cost`](Self::cse_recompute_cost)/ + /// [`cse_shared_maintenance_cost`](Self::cse_shared_maintenance_cost) — + /// the same arithmetic that already backs `cse_share_decision` — rather + /// than inventing a second, drifting cost formula. + fn estimate_cost(&self, candidate: &ReplacementSubDAG, target: &TargetSubDAG<'_>) -> f64 { + let _ = (candidate, target); + f64::NAN + } } /// The default cost model: preserves [`summary_candidates`]'s built-in static @@ -308,6 +351,58 @@ impl CostModel for DefaultCostModel { fn rank_candidates(&self, _intent: &AggIntent, candidates: &[SketchKind]) -> Vec { candidates.to_vec() } + + /// Real numbers, reusing [`CostModel::cse_recompute_cost`]/ + /// [`CostModel::cse_shared_maintenance_cost`] — the same arithmetic + /// `cse_share_decision`'s default body already composes — rather than a + /// second formula: + /// + /// - [`Replacement::Summary`]: `cse_recompute_cost` (the one-time + /// structural cost of building `target` at all) plus + /// `cse_shared_maintenance_cost` of the candidate's own bound family + /// (a pricier family — a sketch over an exact accumulator, say — + /// costs more here, consistent with the per-family weighting + /// [`default_cse_shared_maintenance_cost`] already orders candidates + /// by). + /// - [`Replacement::Rewrite`]: recovers one representative bound + /// `SummaryNode` for `target` via `select_and_bind` (the same + /// rank-and-take-first helper `replacement::bind_one` reuses for the + /// identical need), then charges + /// `cse_shared_maintenance_cost` for the candidate that shares + /// `target`'s own `Rc` (`Rc::ptr_eq`), or `cse_recompute_cost * + /// consumer_count` for the one that doesn't — the same two terms + /// `cse_share_decision` already compares against each other. `NaN` + /// only if `target` itself can't be bound at all (schema derivation + /// failed) — never expected for a target that's already part of a + /// legitimate workload tree. + fn estimate_cost(&self, candidate: &ReplacementSubDAG, target: &TargetSubDAG<'_>) -> f64 { + let consumer_count = target.consumer_count.max(1); + match &candidate.replacement { + Replacement::Summary(node) => { + let cse = CseCandidate { + subtree: target.root, + bound_summary: node, + consumer_count, + }; + (self.cse_recompute_cost(&cse) + self.cse_shared_maintenance_cost(&cse)).0 + } + Replacement::Rewrite(rc) => { + let Ok(bound) = select_and_bind(target.root, self) else { + return f64::NAN; + }; + let cse = CseCandidate { + subtree: target.root, + bound_summary: &bound, + consumer_count, + }; + if Rc::ptr_eq(rc, target.root) { + self.cse_shared_maintenance_cost(&cse).0 + } else { + (self.cse_recompute_cost(&cse) * consumer_count).0 + } + } + } + } } #[cfg(test)] @@ -600,4 +695,113 @@ mod tests { ShareDecision::Share ); } + + // ── estimate_cost ──────────────────────────────────────────────────── + + /// The trait's default `estimate_cost` body is an explicit placeholder, + /// not a real cost model — a `CostModel` that only overrides + /// `rank_candidates` (the minimum required to implement the trait) must + /// still get `f64::NAN` back, never a value that looks like a real + /// estimate. + #[test] + fn estimate_cost_default_body_is_a_nan_placeholder() { + struct RankOnly; + impl CostModel for RankOnly { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchKind], + ) -> Vec { + candidates.to_vec() + } + } + + let root = Rc::new(scan()); + let target = TargetSubDAG::new(&root); + let candidate = ReplacementSubDAG { + replacement: Replacement::Summary(Rc::new(summary_node(SummaryFamilyType::Plain( + asap_types::pre_asap::DataType::Float64, + )))), + rationale: "whatever".into(), + }; + assert!(RankOnly.estimate_cost(&candidate, &target).is_nan()); + } + + /// `DefaultCostModel::estimate_cost` for a [`Replacement::Summary`] + /// candidate reuses [`default_cse_shared_maintenance_cost`]'s own + /// per-family ordering: a candidate bound to a cheap-to-maintain family + /// (an exact accumulator) must cost less than one bound to an + /// expensive-to-maintain family (a fitted statistical model), same + /// target either way — consistent with + /// `default_shared_maintenance_cost_orders_families_cheapest_to_priciest` + /// above. + #[test] + fn estimate_cost_for_summary_orders_candidates_by_family_cheapest_to_priciest() { + let root = Rc::new(scan()); + let target = TargetSubDAG::new(&root); + + let cheap = ReplacementSubDAG { + replacement: Replacement::Summary(Rc::new(summary_node( + SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + ))), + rationale: "exact accumulator".into(), + }; + let pricey = ReplacementSubDAG { + replacement: Replacement::Summary(Rc::new(summary_node(SummaryFamilyType::StatModel( + asap_types::post_asap::StatModelKind::Parametric, + asap_types::post_asap::StatModelParams::Parametric { + family: "gaussian_mixture".into(), + }, + )))), + rationale: "fitted statistical model".into(), + }; + + let cheap_cost = DefaultCostModel.estimate_cost(&cheap, &target); + let pricey_cost = DefaultCostModel.estimate_cost(&pricey, &target); + assert!( + cheap_cost.is_finite() && pricey_cost.is_finite(), + "cheap={cheap_cost}, pricey={pricey_cost}" + ); + assert!( + cheap_cost < pricey_cost, + "an ExactAggregate candidate should cost less than a StatModel one: \ + exact={cheap_cost}, stat_model={pricey_cost}" + ); + } + + /// `DefaultCostModel::estimate_cost` for a [`Replacement::Rewrite`] pair + /// (the `SharedSubtreeStrategy` share-vs-recompute shape) agrees with + /// what `cse_share_decision` would already pick for the same target: with + /// many consumers of a cheap-to-recompute leaf, the "share" candidate + /// (the target's own `Rc`) must cost less than the "recompute + /// independently" one (a fresh `Rc`) — mirrors + /// `cse_share_decision_shares_when_recompute_dominates_maintenance` + /// above, through `estimate_cost` instead of `cse_share_decision` + /// directly. + #[test] + fn estimate_cost_for_rewrite_prefers_sharing_when_recompute_dominates_maintenance() { + let target_root = Rc::new(scan()); + let target = TargetSubDAG::with_consumer_count(&target_root, 20); + + let share = ReplacementSubDAG { + replacement: Replacement::Rewrite(Rc::clone(&target_root)), + rationale: "build once and share".into(), + }; + let recompute = ReplacementSubDAG { + replacement: Replacement::Rewrite(Rc::new((*target_root).clone())), + rationale: "build independently".into(), + }; + + let share_cost = DefaultCostModel.estimate_cost(&share, &target); + let recompute_cost = DefaultCostModel.estimate_cost(&recompute, &target); + assert!( + share_cost.is_finite() && recompute_cost.is_finite(), + "share={share_cost}, recompute={recompute_cost}" + ); + assert!( + share_cost < recompute_cost, + "with 20 consumers of a cheap-to-recompute leaf, sharing should cost less: \ + share={share_cost}, recompute={recompute_cost}" + ); + } } diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 48322a7d..a505ca56 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -32,17 +32,32 @@ //! - [`replacement`] — the `TargetSubDAG`/`ReplacementSubDAG`/ //! `ReplacementStrategy` vocabulary `docs/design_docs/asap_aware_mapping.md` stubs out //! under "Key concepts (not yet implemented)", implemented for real (issue -//! #251, part of #33). One module, one step: -//! [`replacement::SketchFamilyStrategy::replacements`] both *decides* what -//! an `AggIntent` may become ([`replacement::implementations_for_with`], -//! exhaustive and ranked via a `CostModel`, sized to the `AccuracyTarget`) -//! and *constructs* each candidate's bound -//! [`SummaryNode`](asap_types::post_asap::SummaryNode) — every candidate -//! comes back, not just one. [`replacement::SharedSubtreeStrategy`] does -//! the analogous job for the build-independently-vs-build-once-and-share -//! choice at a CSE-detected shared subtree. No search/ranking-across-a- -//! whole-plan logic lives here — see that module's docs for what's -//! deliberately left to a future Cascades/Volcano-style search engine. +//! #251, part of #33) — plus, merged into the same module (issue #252, +//! part of #33), the workload-level search engine that decides *and* +//! constructs *and* searches across a whole workload's worth of targets, +//! all as one module's job. One module, three connected steps: +//! 1. [`replacement::SketchFamilyStrategy::replacements`] both *decides* +//! what an `AggIntent` may become ([`replacement::implementations_for_with`], +//! exhaustive and ranked via a `CostModel`, sized to the +//! `AccuracyTarget`) and *constructs* each candidate's bound +//! [`SummaryNode`](asap_types::post_asap::SummaryNode) — every +//! candidate comes back, not just one. +//! [`replacement::SharedSubtreeStrategy`] does the analogous job for +//! the build-independently-vs-build-once-and-share choice at a +//! CSE-detected shared subtree. +//! 2. [`replacement::search_workload`]/[`replacement::search_workload_with`] +//! *search* — discover every candidate site across a whole workload +//! (not just one target in isolation) and run both strategies above +//! against each one, to a fixpoint, without ever materializing a flat +//! `2^N`-sized candidate-plan list: [`replacement::PlanSpace`] holds +//! one Cascades-style [`replacement::MemoGroup`] per distinct site, +//! each carrying every alternative discovered for it. +//! 3. [`replacement::PlanSpace::cost_sorted`] is the final +//! `sorted_by(cost_model)` step, ranking each group's candidates +//! best-first via the same [`CostModel`](cost_model::CostModel) the +//! single-target steps above already consult — see +//! [`replacement`]'s own module docs for the full design (MEMO groups +//! vs. flat plans, dedup discipline, termination, cost-based ranking). //! - [`bind`] — has no "bind me one tree" entry point for a single target. //! [`replacement::SketchFamilyStrategy`] is the only public way to get //! bound output for a target, and it always returns *every* candidate; a @@ -55,13 +70,23 @@ //! onto one shared subtree bind to one shared `SummaryNode` too (issue //! #212, #222, #223) — memoization needs one canonical decision per //! shared root to key sharing on, so that entry point is the one place a -//! single-answer selection still lives. See the terminology section below -//! for why this crate's logical→physical step is named "implementation" -//! rather than "bind". +//! single-answer selection still lives. Everything else `bind` used to +//! hold ([`replacement::select_and_bind`], the single-target rank-and- +//! take-first helper, and [`replacement::keep_pre_asap`], the pass-through +//! fallback) moved into [`replacement`] itself — both are single-target- +//! scoped helpers with no real workload-level state, the same category as +//! everything else that module owns; only the genuine cross-root +//! `Rc`-identity memoization state above still needs its own module. See +//! the terminology section below for why this crate's logical→physical +//! step is named "implementation" rather than "bind". //! - [`cost_model`] — the [`CostModel`](cost_model::CostModel) trait every //! deployment's cost-based sketch selection plugs into (issues #6, #33). //! `asap-plan` itself only ships [`DefaultCostModel`](cost_model::DefaultCostModel), -//! which preserves [`replacement`]'s built-in static preference order. +//! which preserves [`replacement`]'s built-in static preference order and +//! — via [`CostModel::estimate_cost`](cost_model::CostModel::estimate_cost) +//! — exposes an actual numeric cost per candidate, not just a relative +//! rank, for a caller (e.g. a DAG-visualization view) that wants to show +//! "candidate A costs ≈ X" next to "candidate B costs ≈ Y". //! //! ## Terminology — "bind" already means three different things nearby; //! this crate's own logical→physical step is named "implementation" instead @@ -83,7 +108,8 @@ //! | **Bind #1** | name resolution | `ColumnRef` (a name) → `ColumnId` (a concrete schema column) — the classic RDBMS "Parse → **Bind** → Optimize" pipeline sense (e.g. SQL Server's query-processor terminology) | [`asap_types::pre_asap::binder::Binder`](https://docs.rs/asap-types) | //! | **Implementation** — `replacement::implementations_for_with` | pre-ASAP → post-ASAP, *one node* | enumerating every concrete physical realization (a sketch family, an exact accumulator, or pass-through) for one [`AggIntent`](asap_types::pre_asap::agg_intent::AggIntent) | [`replacement`] | //! | **Replacement** — [`replacement::SketchFamilyStrategy::replacements`] | pre-ASAP → post-ASAP, *one target, every candidate* | wrap each `implementations_for_with` candidate into its own bound [`SummaryNode`](asap_types::post_asap::SummaryNode), ranked — a caller wanting one answer takes the first entry itself | [`replacement`] | -//! | **`implement_workload`** — [`bind::implement_workload`] | pre-ASAP → post-ASAP, *whole workload* | walk every root of a `QueryWorkload`, keeping the first (`cost_model`-preferred) candidate per node, sharing one bound `SummaryNode` across roots CSE already collapsed onto one `Rc` — emits the complete post-ASAP `SummaryExpr`/`SummaryNode` DAG | [`bind`] | +//! | **Search** — [`replacement::search_workload`]/[`replacement::search_workload_with`] | pre-ASAP → post-ASAP, *whole workload, every candidate* | a Cascades/Volcano-style MEMO search: discover every candidate site across a whole workload (not just one target), run every registered `ReplacementStrategy` against each to a fixpoint, and dedup into a [`replacement::PlanSpace`] — one [`replacement::MemoGroup`] per distinct site holding every alternative discovered for it, never a flat `2^N`-sized list of whole candidate plans | [`replacement`] | +//! | **`implement_workload`** — [`bind::implement_workload`] | pre-ASAP → post-ASAP, *whole workload, one answer* | walk every root of a `QueryWorkload`, keeping the first (`cost_model`-preferred) candidate per node, sharing one bound `SummaryNode` across roots CSE already collapsed onto one `Rc` — emits the complete post-ASAP `SummaryExpr`/`SummaryNode` DAG | [`bind`] | //! | **Bind #2** (downstream, not in this crate) | post-ASAP → deployment placement | a *deployment's* own physical binder, additionally deciding **placement** (edge vs. backend, wire format, …) — a genuinely different, deployment-specific decision this crate doesn't model at all | e.g. `control_plane::sketch_algebra::rules::bind_*` (as of this writing; expected to fold into that deployment's cost-model layer rather than stay a separate "bind" concept) | //! //! A related question (tracked alongside issues #6/#33): whether this @@ -110,6 +136,8 @@ pub mod replacement; pub use bind::{implement_workload, implement_workload_with, ImplementError}; pub use cost_model::{CostModel, DefaultCostModel}; pub use replacement::{ - summary_candidates, Implementation, Matcher, Replacement, ReplacementStrategy, - ReplacementSubDAG, SharedSubtreeStrategy, SketchFamilyStrategy, TargetSubDAG, + default_strategies, default_strategies_with, search_workload, search_workload_with, + summary_candidates, Implementation, Matcher, MemoGroup, PlanSpace, RankedGroup, Replacement, + ReplacementStrategy, ReplacementSubDAG, SharedSubtreeStrategy, SketchFamilyStrategy, + TargetSubDAG, MAX_SEARCH_ITERATIONS, }; diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index 626e82be..7411bee4 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -20,7 +20,7 @@ //! mechanically turns the already-decided `(kind, params)` into a real //! [`SummaryNode`] — derives the child schema, resolves the summarized //! column, builds the readout query, recurses into the child (via -//! [`crate::bind::select_and_bind`], so a nested aggregate gets its own +//! [`select_and_bind`], so a nested aggregate gets its own //! independent enumeration, never the outer target's forced choice), and //! assembles the `SummaryAgg`/`SummaryEstimate` node. //! @@ -91,29 +91,172 @@ //! //! ## Non-goals (tracked separately, not attempted here) //! -//! - **No search/selection-across-a-whole-plan logic.** `docs/design_docs/asap_aware_mapping.md`'s -//! replacement-plan-searching pseudocode — trying every `ReplacementStrategy` -//! against every candidate plan, deduplicating, iterating to a fixpoint, -//! then ranking by a `CostModel` — is a Cascades/Volcano-style search -//! engine, tracked as a separate follow-up. Taking the first candidate off -//! [`SketchFamilyStrategy::replacements`] is a single-node stand-in for -//! that, not the real thing: it never compares whole candidate *plans*, -//! only one node's own candidates against each other via -//! `cost_model.rank_candidates`. -//! - **No workload-wide `TargetSubDAG` discovery pass.** Finding every -//! candidate node in a whole workload (walking every root, deduplicating by -//! `Rc` identity, computing real consumer counts) is exactly what PR #247's -//! `SharedSubexpressionRule`/`SketchApplicabilityRule` traversals already -//! do — reusable, but wiring it up to feed this module's strategies -//! automatically is part of the same future search engine, not this issue. -//! This module's own tests build `TargetSubDAG`s directly, the same -//! hand-rolled-fixture style `bind.rs`/`cost_model.rs`'s own tests already -//! use. //! - **[`implementations_for_with`]'s own outward-facing behavior is //! unchanged.** Same inputs still produce the same exhaustive, ranked //! list — only its home moved (from a separate `implementation` module //! into this one) and its own visibility dropped to module-private, since //! [`SketchFamilyStrategy`] is now its only caller. +//! +//! ## Workload-wide search — merged in from the former `search.rs` (issue #252, part of #33) +//! +//! This section used to carry two more "non-goals" bullets here — "no +//! search/selection-across-a-whole-plan logic" and "no workload-wide +//! `TargetSubDAG` discovery pass" — describing work deliberately left for a +//! future Cascades/Volcano-style search engine (PR #263, +//! `feat/cascades-search-252`, over the [`ReplacementStrategy`] extension +//! point above). That engine is [`PlanSpace`]/[`MemoGroup`]/ +//! [`search_workload`]/[`search_workload_with`] below, merged into this +//! module rather than kept as a separate `search` module — the same "one +//! module, one step" reasoning the top of this file already uses for +//! decide-and-build: searching *across* a whole workload's worth of +//! [`TargetSubDAG`]s is a natural continuation of deciding and building +//! replacements *for* one, not a different concern that deserves its own +//! file. What follows (through "Cost-based final selection" below) is that +//! engine's own design documentation, preserved from `search.rs`. +//! +//! ### The pseudocode, and the two things it deliberately leaves open +//! +//! ```text +//! candidate_plans = { input_workload_plan } +//! loop: +//! new_plans = {} +//! for plan in candidate_plans: +//! for site in plan.bindable_sites(): +//! for strategy in registered_strategies: +//! if strategy.matches(site): +//! for replacement in strategy.replacements(site): +//! new_plans += substitute(plan, site, replacement) +//! new_plans -= candidate_plans +//! candidate_plans += new_plans +//! until new_plans is empty +//! return candidate_plans.sorted_by(cost_model) +//! ``` +//! +//! Read literally, this enumerates whole *plans* — full copies of the +//! workload's tree, one per combination of per-site choices. A workload with +//! `N` independently-choosable sites would produce up to `2^N` flat plans, +//! each one duplicating every untouched sibling subtree. This module does +//! not do that: +//! +//! 1. **MEMO groups, not flat plans.** [`MemoGroup`] is this engine's +//! Cascades-style "group": one distinct [`TargetSubDAG`] (identified by +//! its own `Rc` pointer identity — the same currency +//! [`asap_types::pre_asap::cse::share_common_subtrees`] already +//! established across the workload) holding every +//! [`ReplacementSubDAG`] alternative discovered for it. [`PlanSpace`] is +//! a collection of these groups, keyed by site — a candidate "plan" is +//! never materialized as a distinct top-level `Rc` at all; +//! two logically-different overall choices at two different sites are +//! just two different entries in two different groups, sharing every +//! other node in the workload by construction (they *are* the same +//! `Rc`s — nothing was copied to make a second "plan"). +//! 2. **Dedup by structural hash + `PartialEq`, reusing `pre_asap::cse`'s own +//! discipline.** [`asap_types::pre_asap::cse::structural_hash`] (made +//! `pub` for exactly this reuse) is only ever a candidate-narrowing +//! filter; [`MemoGroup::add_candidate`]'s actual duplicate check is +//! `QueryExpr`'s derived `PartialEq` — the same "hash is a filter, +//! `PartialEq` is the decision, no exceptions" rule `cse.rs`'s own +//! "Correctness" section states and this module inherits rather than +//! reinvents. See [`is_duplicate_rewrite`] for the one deliberate +//! wrinkle this reuse needs (a `Rc`-identity case pure value equality +//! would get wrong). +//! +//! ### Where "for site in plan.bindable_sites()" comes from +//! +//! [`discover_sites`] is the workload-wide `TargetSubDAG` discovery pass +//! this section used to flag as explicitly *not* implemented ("no +//! workload-wide `TargetSubDAG` discovery pass is shipped either... wiring +//! it up automatically belongs to the same future search engine, not this +//! issue") — this is that future engine, so it's this module's job now. It +//! walks every workload root's whole DAG (the same **relational-skeleton** +//! operator-child scope `asap_types::pre_asap::cse::share_common_subtrees` +//! itself uses — see that module's "Algorithm" section), discovering one +//! site per distinct `Rc` and a *real* `consumer_count`: how many +//! operator-child positions anywhere in the workload reference that exact +//! `Rc`, not just how many of the workload's own top-level roots happen to +//! be it. This deliberately goes one step further than +//! [`crate::bind::implement_workload_with`]'s own consumer-count pass, +//! which only counts whole-root sharing (that function's own doc calls +//! widening this "future work" for binding) — a `SharedSubtreeStrategy` +//! candidate three levels under an unshared `Filter` is exactly as real a +//! search-space site as a shared whole root, so this module's discovery +//! can't stop at the top level the way binding's does. +//! +//! `discover_sites` duplicates (rather than reuses) this module's own +//! `#[cfg(test)]`-only `count_consumers` traversal (in the test module +//! below), which mirrors this exact shape for this module's own test +//! fixtures — that copy is intentionally test-only, so it isn't reachable +//! from this module's production code without either moving it into +//! shared, non-test-gated code or duplicating the (small, self-contained) +//! traversal here. Duplicating was judged simpler than restructuring a test +//! helper into shared production code for one caller. +//! +//! ### Termination +//! +//! Every discovered site is asked *once* per registered strategy, never +//! re-asked — [`search_workload_with`]'s loop processes each round's +//! frontier of not-yet-visited sites exactly one time each, so there is no +//! scenario where the same `(site, strategy)` pair is queried twice (the +//! `new_plans -= candidate_plans` dedup step the module-level pseudocode +//! describes is therefore never asked to recognize "the same candidate, +//! proposed again" as a special case — see [`MemoGroup::add_candidate`]'s +//! own doc on why that distinction matters for [`Replacement::Summary`] +//! specifically, where no real equality check exists to make it safely). +//! +//! What *can* grow the frontier is a candidate's own reachable structure: +//! after a site is processed, every [`Replacement::Rewrite`] candidate's +//! **children** (never the candidate's own top-level node — that value is +//! an alternative *for* the site just processed, not a new site of its own; +//! see [`discover_new_descendant_sites`]) are scanned for pointers not +//! already known, and any found become next round's frontier. Both shipped +//! strategies are idempotent in exactly this sense: [`SketchFamilyStrategy`] +//! produces terminal [`Replacement::Summary`] candidates (no `QueryExpr` +//! children to scan at all), and [`SharedSubtreeStrategy`]'s two +//! [`Replacement::Rewrite`] candidates both reuse the target's own +//! already-known child `Rc`s verbatim (`Rc::clone`/a shallow top-level +//! `.clone()` — see that strategy's own doc). So for both, the frontier is +//! always empty after round one: real workloads converge in exactly one +//! round, regardless of size. +//! +//! That said, a future strategy whose `Replacement::Rewrite` candidates +//! invent brand-new descendant structure every time they're computed (e.g. +//! internal state that fabricates a fresh child node on every call) could +//! in principle keep the frontier non-empty forever. Since this crate has +//! no cardinality/statistics estimation to bound anything by, +//! [`search_workload_with`] enforces a generous, documented round cap +//! ([`MAX_SEARCH_ITERATIONS`]) instead: exceeding it panics with a clear +//! message naming the actual cause, rather than hanging silently — a test +//! ([`tests::a_pathologically_growing_strategy_trips_the_iteration_cap`]) +//! pins that this guard actually fires, by using exactly that shape of +//! pathological strategy. +//! +//! ### Cost-based final selection — reusing `CostModel`, not a second interface +//! +//! [`PlanSpace::cost_sorted`] is the `sorted_by(cost_model)` step, and it +//! reuses this crate's existing [`CostModel`] trait rather than inventing a +//! second cost interface (`docs/design_docs/cse-cost-model-decision.md`, +//! issue #237, explicitly reasoned about *why* a narrow, direct cost +//! comparison was enough for the CSE share/recompute decision alone, and +//! flagged that a real search engine — this module — is where that stops +//! being the whole story; it isn't a contradiction of #237, it's the scope +//! change #237 itself named). Concretely, per [`MemoGroup`]: +//! +//! - A group whose candidates are the [`SharedSubtreeStrategy`] +//! share-vs-recompute pair is ranked by calling +//! [`CostModel::cse_share_decision`] — the exact comparison +//! [`crate::bind::implement_workload_with`] already uses for this same +//! decision — rather than re-deriving a competing comparison in this +//! module. +//! - A group whose candidates are [`SketchFamilyStrategy`]'s sketch-family +//! candidates is ranked via [`CostModel::rank_candidates`] (the same hook +//! `implementations_for_with` itself consults), applied to the +//! candidates' own [`SketchKind`]s. +//! - Any other shape (a single candidate, or a mix this module doesn't have +//! a defined comparison for) keeps discovery order — there is nothing to +//! rank, or no [`CostModel`] hook this module knows how to apply; it never +//! invents a comparison `CostModel` doesn't already define. + +use std::collections::HashMap; use asap_types::post_asap::{ ExactKind, ExactParams, SamplingKind, SamplingParams, SketchKind, SketchParams, @@ -121,14 +264,15 @@ use asap_types::post_asap::{ SummaryFamilyType, SummaryField, SummaryNode, SummarySchema, WaveletKind, WaveletParams, }; use asap_types::pre_asap::agg_intent::{agg_is_mergeable, AggIntent}; +use asap_types::pre_asap::cse::{share_common_subtrees, structural_hash, HashCache}; use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::query_expr::{QueryExpr, Reduction}; use asap_types::pre_asap::schema::Schema; use asap_types::types::AccuracyTarget; use std::rc::Rc; -use crate::bind::{select_and_bind, ImplementError}; -use crate::cost_model::{CostModel, DefaultCostModel}; +use crate::bind::ImplementError; +use crate::cost_model::{CostModel, CseCandidate, DefaultCostModel, ShareDecision}; /// A pre-ASAP sub-DAG a [`ReplacementStrategy`] knows how to replace. /// @@ -883,6 +1027,66 @@ fn describe_intent(intent: &AggIntent) -> String { } } +// ── select_and_bind / keep_pre_asap: rank-and-take-first, and its fallback ── + +/// Rank-and-take-first selector, scoped to +/// [`crate::bind::implement_workload_with`]'s CSE memoization and to this +/// module's own recursion into a node's child (via [`construct_summary_agg`]) +/// — **not** a general single-answer API. `root` must already be the +/// caller's own `Rc`, never fabricated per call, so this never allocates +/// beyond what the caller already held. +/// +/// `pub(crate)`: reachable both from this module's own construction helper +/// (so a nested aggregate gets its own independent enumeration instead of +/// inheriting the parent's forced candidate) and from `bind.rs`'s +/// [`crate::bind::implement_workload_with`] (workload-wide CSE memoization +/// needs one canonical decision per shared root to key sharing on — see +/// that function's own module docs). Every other caller goes through +/// [`SketchFamilyStrategy::replacements`] directly and decides for itself. +pub(crate) fn select_and_bind( + root: &Rc, + cost_model: &dyn CostModel, +) -> Result, ImplementError> { + let target = TargetSubDAG::new(root); + match SketchFamilyStrategy::new(cost_model) + .replacements(&target) + .into_iter() + .next() + { + Some(ReplacementSubDAG { + replacement: Replacement::Summary(node), + .. + }) => Ok(node), + Some(ReplacementSubDAG { + replacement: Replacement::Rewrite(_), + .. + }) => { + unreachable!("SketchFamilyStrategy never returns a Rewrite candidate") + } + // No candidate at all: `root` isn't `bindable_intent` shape (or its + // intent has no realization `implementations_for_with` can't + // produce — never happens, that match is exhaustive) — the same + // conservative fallback `SketchFamilyStrategy::matches` uses. + None => keep_pre_asap(root), + } +} + +/// Wrap an unrewritten pre-ASAP subtree, lifting its schema with every column +/// `SummaryFamilyType::Plain`. `pub` — re-exported as +/// [`crate::bind::keep_pre_asap`] for existing external callers that reach +/// it through that module's path — so a caller can fall back to this +/// explicitly — e.g. when `SketchFamilyStrategy::replacements()` returns no +/// candidate for a target, or a deployment wants to force a node its own +/// runtime can't actually implement — through the same fallback this +/// crate's own dispatch uses, without duplicating the schema-lift logic. +pub fn keep_pre_asap(expr: &QueryExpr) -> Result, ImplementError> { + let schema = expr.output_schema()?; + Ok(Rc::new(SummaryNode { + expr: SummaryExpr::KeepPreAsap(Box::new(expr.clone())), + schema: lift(&schema), + })) +} + // ── Construction: turn one already-decided Implementation into a SummaryNode ─ /// The bindable shape [`SketchFamilyStrategy`] targets: a single intent, no @@ -907,9 +1111,9 @@ pub fn bindable_intent(node: &QueryExpr) -> Option<&AggIntent> { /// method enumerates. /// /// `expr` must still be the [`bindable_intent`] shape for `implementation` to -/// have any effect; anything else falls back to [`crate::bind::keep_pre_asap`]. +/// have any effect; anything else falls back to [`keep_pre_asap`]. /// Only `expr`'s own top-level decision is forced — recursion into `expr`'s -/// child goes back through [`crate::bind::select_and_bind`] (fresh candidate +/// child goes back through [`select_and_bind`] (fresh candidate /// enumeration, not a forced pick), so choosing one candidate for a target /// never leaks into that target's own nested aggregates. fn construct_summary( @@ -935,12 +1139,12 @@ fn construct_summary( } } } - crate::bind::keep_pre_asap(expr) + keep_pre_asap(expr) } /// Translate an [`Implementation`] into the `(family, needs a /// SummaryEstimate readout)` pair [`construct_summary_agg`] needs, or `None` -/// for `PassThrough` (the caller falls back to [`crate::bind::keep_pre_asap`]). +/// for `PassThrough` (the caller falls back to [`keep_pre_asap`]). /// /// Every family's partial state needs a readout to recover a value, except /// `ExactAggregate` — its partial state *is* the value already, so no @@ -1084,8 +1288,8 @@ fn readout(intent: &AggIntent, col: &ColumnRef, cost_model: &dyn CostModel) -> P /// Lift a pre-ASAP [`Schema`] to a [`SummarySchema`] with every column /// `SummaryFamilyType::Plain` — shared by [`construct_summary_agg`] and -/// [`crate::bind::keep_pre_asap`] (`pub(crate)` for that cross-module reuse). -pub(crate) fn lift(schema: &Schema) -> SummarySchema { +/// [`keep_pre_asap`], both in this module. +fn lift(schema: &Schema) -> SummarySchema { SummarySchema { fields: schema .columns @@ -1158,6 +1362,624 @@ impl ReplacementStrategy for SharedSubtreeStrategy { } } +// ── Workload-wide search: MemoGroup / PlanSpace / search_workload ────────── +// +// Merged in from the former `search.rs` (issue #252, part of #33) — see this +// file's own top-level "Workload-wide search" doc section for the full +// design rationale. + +/// A generous, documented backstop against a hypothetically ill-behaved +/// future [`ReplacementStrategy`] (see the module docs' "Termination" +/// section) — not a bound either shipped strategy could ever approach. +/// [`SketchFamilyStrategy`] and [`SharedSubtreeStrategy`] both converge in +/// exactly 2 passes over a fixed site set, regardless of workload size. +pub const MAX_SEARCH_ITERATIONS: usize = 1_000; + +// ── MemoGroup ──────────────────────────────────────────────────────────── + +/// One Cascades-style MEMO group: a single distinct [`TargetSubDAG`] (its +/// own `target` `Rc`, keyed by pointer identity in +/// [`PlanSpace`]'s internal map — never re-derived by value) plus every +/// [`ReplacementSubDAG`] alternative any registered [`ReplacementStrategy`] +/// proposed for it. +/// +/// `candidates` is deliberately *not* required to be non-empty — a site no +/// registered strategy has an opinion on still gets a group (with an empty +/// candidate list), so [`PlanSpace`] always has exactly one group per +/// discovered site, not "one group per site something matched". +#[derive(Debug, Clone)] +pub struct MemoGroup { + /// The target sub-DAG this group is for. + pub target: Rc, + /// How many operator-child positions across the whole workload + /// reference this exact `Rc` — see [`discover_sites`]. + pub consumer_count: usize, + /// Every distinct alternative discovered for `target`, in discovery + /// order (not ranked — see [`PlanSpace::cost_sorted`] for the ranked + /// view). + pub candidates: Vec, +} + +impl MemoGroup { + fn new(target: Rc, consumer_count: usize) -> Self { + Self { + target, + consumer_count, + candidates: Vec::new(), + } + } + + /// Add `candidate` unless it's already present (see + /// [`is_duplicate_rewrite`]/[`is_duplicate_summary`] for what "already + /// present" means for each [`Replacement`] variant). Returns whether it + /// was actually added — [`search_workload_with`]'s fixpoint loop uses + /// this to detect when a pass made no progress. + fn add_candidate(&mut self, candidate: ReplacementSubDAG) -> bool { + let is_duplicate = self.candidates.iter().any(|existing| { + match (&existing.replacement, &candidate.replacement) { + (Replacement::Rewrite(existing_rc), Replacement::Rewrite(rc)) => { + is_duplicate_rewrite(existing_rc, rc, &self.target) + } + (Replacement::Summary(existing_node), Replacement::Summary(node)) => { + is_duplicate_summary(existing_node, node) + } + // A `Rewrite` and a `Summary` are never the same candidate — + // they're different `Replacement` variants entirely. + (Replacement::Rewrite(_), Replacement::Summary(_)) + | (Replacement::Summary(_), Replacement::Rewrite(_)) => false, + } + }); + if is_duplicate { + false + } else { + self.candidates.push(candidate); + true + } + } +} + +/// Are `existing` and `candidate` the same [`Replacement::Rewrite`] +/// candidate for a group targeting `target`? +/// +/// Structural (`QueryExpr`) value equality alone is *not* enough here: this +/// module's one shipped multi-candidate `Replacement::Rewrite` source, +/// [`SharedSubtreeStrategy`], deliberately returns **two** candidates that +/// are value-equal to each other (`build once and share` vs. `build +/// independently` — see that strategy's own doc) but represent genuinely +/// different physical choices, distinguished *only* by whether the +/// candidate's `Rc` is the group's own `target` `Rc` (share) or a freshly +/// allocated one (recompute independently) — this IR has no field that +/// records "materialized once and shared", so `Rc` identity against +/// `target` is the only signal that distinction exists in at all. Treating +/// those two as duplicates of each other via pure value equality would +/// silently collapse a real choice into one candidate — the "false-positive +/// dedup is a wrong answer, not a missed optimization" failure mode +/// `cse.rs`'s own "Correctness" section warns about, just one level up from +/// where that module states it. +/// +/// So: two candidates whose "is this the target's own `Rc`?" bit disagrees +/// are never duplicates of each other, full stop. Only when that bit +/// *agrees* does this fall through to the real dedup discipline — +/// [`structural_hash`] as a candidate-narrowing filter, `QueryExpr`'s +/// derived `PartialEq` as the actual decision — protecting against the +/// (currently hypothetical, since neither shipped strategy causes it) +/// case of the exact same alternative being proposed twice. A fresh +/// [`HashCache`] per call: this is a pairwise check between two candidates +/// for one group, not a bottom-up pass over a whole tree, so there is no +/// wider traversal to amortize the cache across the way `InternTable`'s own +/// use of `structural_hash` does. +fn is_duplicate_rewrite( + existing: &Rc, + candidate: &Rc, + target: &Rc, +) -> bool { + let existing_is_target = Rc::ptr_eq(existing, target); + let candidate_is_target = Rc::ptr_eq(candidate, target); + if existing_is_target != candidate_is_target { + return false; + } + let mut cache = HashCache::new(); + structural_hash(existing, &mut cache) == structural_hash(candidate, &mut cache) + && existing == candidate +} + +/// Are `existing` and `candidate` the same [`Replacement::Summary`] +/// candidate? +/// +/// [`SummaryNode`] derives neither `PartialEq` nor `Hash` (it embeds +/// `SketchParams`/`f64`-bearing accuracy targets deep inside `SummaryExpr`, +/// the same reason `QueryExpr` can't derive `Hash` either — see +/// [`structural_hash`]'s own doc). Per this module's inherited "hash is a +/// filter, `PartialEq` is the decision, no exceptions" rule, there is no +/// real equality check to back a dedup *decision* here — and skipping the +/// check is the only choice that rule permits: never merging two candidates +/// is harmless (at worst, a redundant entry in a group's candidate list), +/// while comparing by some proxy this module can't actually verify (e.g. +/// `Debug` text, or `ReplacementSubDAG::rationale` — documented elsewhere in +/// this crate as prose for a report, "not machine parsing") risks exactly +/// the false-positive merge the rule exists to prevent. Both strategies +/// shipped today already return a structurally distinct candidate for every +/// entry of one `replacements()` call, so this is future-proofing against a +/// hypothetical repeat call, not a gap either strategy's own tests exercise. +fn is_duplicate_summary(_existing: &Rc, _candidate: &Rc) -> bool { + false +} + +// ── PlanSpace ──────────────────────────────────────────────────────────── + +/// The deduped candidate space [`search_workload`]/[`search_workload_with`] +/// discover: one [`MemoGroup`] per distinct site in the (already-CSE'd) +/// workload, plus the workload's own post-CSE roots so a caller can still +/// map a `Root`'s `Id` back to the `Rc` whose group holds its +/// alternatives. +pub struct PlanSpace { + /// The workload's roots, after the one `share_common_subtrees` pass + /// [`search_workload_with`] runs up front — the same post-CSE roots + /// every site in `groups` was discovered from. + pub roots: Vec<(Id, Rc)>, + groups: HashMap<*const QueryExpr, MemoGroup>, + /// Discovery order — stable iteration for [`PlanSpace::groups`]/ + /// [`PlanSpace::cost_sorted`], since `HashMap` iteration order isn't. + order: Vec<*const QueryExpr>, +} + +impl PlanSpace { + /// Every discovered group, in discovery order. + pub fn groups(&self) -> impl Iterator { + self.order.iter().map(move |ptr| &self.groups[ptr]) + } + + /// How many distinct sites were discovered. + pub fn len(&self) -> usize { + self.groups.len() + } + + /// Whether no sites were discovered at all (an empty workload, or one + /// with no `QueryExpr` nodes reachable from any root — never true for a + /// non-empty `roots`, since every root is itself a site). + pub fn is_empty(&self) -> bool { + self.groups.is_empty() + } + + /// The group for `target`, if `target`'s own `Rc` is a discovered site + /// (i.e. `Rc::ptr_eq` to some node reachable from `roots`). + pub fn group_for(&self, target: &Rc) -> Option<&MemoGroup> { + self.groups.get(&Rc::as_ptr(target)) + } + + /// The `sorted_by(cost_model)` step: every group, each with its own + /// candidates ranked best-first under `cost_model` where this module + /// knows how (see the module docs' "Cost-based final selection" + /// section) — groups themselves stay in discovery order, since sites + /// are independent decision points, not alternatives competing with + /// each other. + /// + /// Ranking itself is decided entirely by [`rank_group`] before + /// [`RankedGroup::costs`] is ever computed — pairing each candidate with + /// [`CostModel::estimate_cost`]'s own number is an additive annotation + /// for a caller that wants to *display* a cost (e.g. a + /// DAG-visualization view), not a second ranking signal, so plugging in + /// a `CostModel` whose `estimate_cost` disagrees with its own + /// `rank_candidates`/`cse_share_decision` (a deployment bug, not + /// something this method tries to protect against) would show a + /// `RankedGroup` whose `costs` aren't monotonically non-decreasing — + /// `cost_sorted`'s own ordering guarantee is unaffected either way. + pub fn cost_sorted(&self, cost_model: &dyn CostModel) -> Vec> { + self.order + .iter() + .map(|ptr| { + let group = &self.groups[ptr]; + let candidates = rank_group(group, cost_model); + let target = TargetSubDAG::with_consumer_count(&group.target, group.consumer_count); + let costs = candidates + .iter() + .map(|c| cost_model.estimate_cost(c, &target)) + .collect(); + RankedGroup { + target: &group.target, + consumer_count: group.consumer_count, + candidates, + costs, + } + }) + .collect() + } +} + +/// One [`MemoGroup`]'s candidates, ranked best-first by +/// [`PlanSpace::cost_sorted`]. +#[derive(Debug)] +pub struct RankedGroup<'a> { + pub target: &'a Rc, + pub consumer_count: usize, + pub candidates: Vec<&'a ReplacementSubDAG>, + /// `costs[i]` is `candidates[i]`'s own [`CostModel::estimate_cost`] + /// estimate — aligned index-for-index with `candidates`, one number per + /// candidate, for a caller that wants an actual `f64` next to each + /// candidate (e.g. "candidate A costs ≈ X, candidate B costs ≈ Y") and + /// not just `candidates`' own relative order. `f64::NAN` throughout + /// unless `cost_model` overrides `estimate_cost` — see that method's own + /// doc. + pub costs: Vec, +} + +/// Rank `group`'s candidates best-first under `cost_model`, per the module +/// docs' "Cost-based final selection" section. Falls back to discovery +/// order whenever there's nothing to rank (0 or 1 candidates) or this +/// module doesn't have a defined `CostModel` comparison for the shape it +/// sees — it never invents one. +fn rank_group<'a>(group: &'a MemoGroup, cost_model: &dyn CostModel) -> Vec<&'a ReplacementSubDAG> { + let mut ranked: Vec<&ReplacementSubDAG> = group.candidates.iter().collect(); + if ranked.len() <= 1 { + return ranked; + } + + // Shape 1: a `SharedSubtreeStrategy` share-vs-recompute pair (every + // candidate is a `Rewrite`) — rank via `CostModel::cse_share_decision`, + // the same comparison `bind::implement_workload_with` already uses. + if ranked + .iter() + .all(|c| matches!(c.replacement, Replacement::Rewrite(_))) + { + if let Some(prefer_target) = cse_preference(group, cost_model) { + ranked.sort_by_key(|c| { + let is_target = matches!( + &c.replacement, + Replacement::Rewrite(rc) if Rc::ptr_eq(rc, &group.target) + ); + u8::from(is_target != prefer_target) + }); + } + return ranked; + } + + // Shape 2: `SketchFamilyStrategy`'s sketch-family candidates (every + // candidate is a `Summary` that realizes a `SketchKind`) — rank via + // `CostModel::rank_candidates`, the same hook `implementations_for_with` + // itself consults. + if let Some(intent) = bindable_intent(&group.target) { + let kinds: Option> = ranked + .iter() + .map(|c| match &c.replacement { + Replacement::Summary(node) => sketch_kind_of(node), + Replacement::Rewrite(_) => None, + }) + .collect(); + if let Some(kinds) = kinds { + let order = cost_model.rank_candidates(intent, &kinds); + ranked.sort_by_key(|c| { + let kind = match &c.replacement { + Replacement::Summary(node) => sketch_kind_of(node), + Replacement::Rewrite(_) => None, + }; + kind.and_then(|k| order.iter().position(|o| *o == k)) + .unwrap_or(usize::MAX) + }); + } + } + ranked +} + +/// For a group whose candidates are all [`Replacement::Rewrite`] (the +/// [`SharedSubtreeStrategy`] shape): does [`CostModel::cse_share_decision`] +/// prefer the candidate that shares `group.target`'s own `Rc` (`true`), or +/// the one that recomputes independently (`false`)? `None` when there's no +/// real comparison to make — fewer than 2 consumers (mirrors +/// [`SharedSubtreeStrategy::matches`]'s own gate), or `group.target` can't +/// actually be bound at all (no candidate and no logical fallback — never +/// expected in practice for a target that's already part of a legitimate +/// workload tree, but this degrades to "keep discovery order" rather than +/// panicking). +fn cse_preference(group: &MemoGroup, cost_model: &dyn CostModel) -> Option { + if group.consumer_count < 2 { + return None; + } + let bound = bind_one(&group.target, cost_model)?; + let candidate = CseCandidate { + subtree: &group.target, + bound_summary: &bound, + consumer_count: group.consumer_count, + }; + Some(match cost_model.cse_share_decision(&candidate) { + ShareDecision::Share => true, + ShareDecision::RecomputeIndependently => false, + }) +} + +/// [`cse_preference`] only needs one representative bound [`SummaryNode`] +/// for `target` (to build a [`CseCandidate`] for +/// [`CostModel::cse_share_decision`]), not the full ranked candidate list +/// [`SketchFamilyStrategy::replacements`] returns — so this just reuses +/// [`select_and_bind`], the same rank-and-take-first helper +/// `construct_summary_agg`'s own recursion and +/// `crate::bind::implement_workload_with` already use, wrapped to swallow +/// the (here, uninteresting) error into `None`. +fn bind_one(target: &Rc, cost_model: &dyn CostModel) -> Option> { + select_and_bind(target, cost_model).ok() +} + +/// The `SketchKind` a bound [`Replacement::Summary`] candidate ultimately +/// realizes, if any (`None` for an `ExactAggregate`/pass-through +/// `Summary` — nothing to rank against another `SketchKind`). +/// +/// Mirrors this module's own `#[cfg(test)]`-only `summary_family_kind` +/// helper (in the test module below), which does the identical +/// `SummaryEstimate`-unwrap-then-match for that module's own tests; that +/// copy is test-only, so this needs its own for real (non-test) ranking +/// code — the same "duplicate a small, self-contained traversal rather than +/// restructure a test helper" call this file's own top doc already makes +/// for [`discover_sites`]. +fn sketch_kind_of(node: &SummaryNode) -> Option { + match &node.expr { + SummaryExpr::SummaryEstimate { summary_input, .. } => sketch_kind_of(summary_input), + SummaryExpr::SummaryAgg { + family: SummaryFamilyType::Sketch(kind, _), + .. + } => Some(kind.clone()), + _ => None, + } +} + +// ── default_strategies ────────────────────────────────────────────────── + +/// The strategies [`search_workload`] runs, in the built-in +/// [`DefaultCostModel`] configuration — mirrors this module's own two +/// shipped [`ReplacementStrategy`] impls, the same way a hypothetical +/// `applicability::default_rules()` mirrors *its* module's own rule set. +/// Use [`default_strategies_with`] to plug in a deployment-specific +/// [`CostModel`] instead. +pub fn default_strategies() -> Vec> { + vec![ + Box::new(SketchFamilyStrategy::default_cost_model()), + Box::new(SharedSubtreeStrategy), + ] +} + +/// Like [`default_strategies`], but [`SketchFamilyStrategy`] ranks/binds via +/// `cost_model` instead of the built-in [`DefaultCostModel`] — the same +/// customization point [`SketchFamilyStrategy::new`] itself offers. +pub fn default_strategies_with<'a>( + cost_model: &'a dyn CostModel, +) -> Vec> { + vec![ + Box::new(SketchFamilyStrategy::new(cost_model)), + Box::new(SharedSubtreeStrategy), + ] +} + +// ── search_workload ────────────────────────────────────────────────────── + +/// Search a whole workload's pre-ASAP roots for every candidate replacement +/// [`default_strategies`] can find, deduped into a [`PlanSpace`]. Candidate +/// *generation* uses the built-in [`DefaultCostModel`] (via +/// [`default_strategies`], the same way [`SketchFamilyStrategy::default_cost_model`] +/// does); call [`PlanSpace::cost_sorted`] on the result for the final +/// `sorted_by(cost_model)` step. Use [`search_workload_with`] to plug in a +/// custom strategy set (e.g. built via [`default_strategies_with`] for a +/// deployment-specific [`CostModel`]). +pub fn search_workload(roots: Vec<(Id, Rc)>) -> PlanSpace { + search_workload_with(roots, &default_strategies()) +} + +/// Like [`search_workload`], but with an explicit `strategies` set (see +/// [`default_strategies_with`] to plug in a deployment-specific +/// [`CostModel`] for candidate generation). +/// +/// Runs [`share_common_subtrees`] once over `roots` first — the same +/// pattern a hypothetical `applicability::find_applicable_optimizations` +/// uses, so every strategy sees the same already-deduplicated tree +/// [`crate::bind::implement_workload`] would — then discovers every site +/// (see [`discover_sites`]) and runs the fixpoint loop the module docs +/// describe, capped at [`MAX_SEARCH_ITERATIONS`] passes (see the module +/// docs' "Termination" section). Deduping candidate plans this way needs no +/// [`CostModel`] at all — that only enters at two well-defined points: each +/// [`ReplacementStrategy`] in `strategies` may already carry its own (e.g. +/// [`SketchFamilyStrategy::new`]'s), and [`PlanSpace::cost_sorted`]'s final +/// ranking step takes one explicitly. +pub fn search_workload_with<'s, Id>( + roots: Vec<(Id, Rc)>, + strategies: &[Box], +) -> PlanSpace { + // `share_common_subtrees` wants owned `QueryExpr`s, not already-`Rc` + // roots — the same `Rc::try_unwrap`-with-clone-fallback pattern + // `asap_types::pre_asap::cse::intern_child` itself uses to recover an + // owned node without cloning in the common (uniquely-owned) case. + let owned_roots: Vec<(Id, QueryExpr)> = roots + .into_iter() + .map(|(id, rc)| { + let expr = Rc::try_unwrap(rc).unwrap_or_else(|shared| (*shared).clone()); + (id, expr) + }) + .collect(); + let cse_roots = share_common_subtrees(owned_roots); + + let mut order = Vec::new(); + let mut nodes = HashMap::new(); + let mut counts: HashMap<*const QueryExpr, usize> = HashMap::new(); + discover_sites(&cse_roots, &mut order, &mut nodes, &mut counts); + + let mut groups: HashMap<*const QueryExpr, MemoGroup> = HashMap::new(); + for ptr in &order { + groups.insert(*ptr, MemoGroup::new(Rc::clone(&nodes[ptr]), counts[ptr])); + } + + // Round-based frontier: every site is asked exactly once per strategy + // (never re-asked — see the module docs' "Termination" section on why + // that matters for `Replacement::Summary` dedup specifically). A round + // can grow the *next* round's frontier only by a candidate's own + // reachable children exposing a genuinely new, not-yet-known `Rc` — see + // `discover_new_descendant_sites`. + let mut frontier = order.clone(); + let mut rounds = 0usize; + while !frontier.is_empty() { + rounds += 1; + assert!( + rounds <= MAX_SEARCH_ITERATIONS, + "search_workload: fixpoint search did not converge within {MAX_SEARCH_ITERATIONS} \ + rounds — a registered ReplacementStrategy's Replacement::Rewrite candidates keep \ + exposing new, never-before-seen descendant structure every round. \ + SketchFamilyStrategy/SharedSubtreeStrategy never do this (see replacement.rs's \ + module docs' \"Termination\" section); check any custom strategies passed to \ + search_workload_with.", + ); + + let sites_before = order.len(); + for ptr in &frontier { + let (target, consumer_count) = { + let group = &groups[ptr]; + (Rc::clone(&group.target), group.consumer_count) + }; + let site = TargetSubDAG::with_consumer_count(&target, consumer_count); + + let mut proposed = Vec::new(); + for strategy in strategies { + if strategy.matches(&site) { + proposed.extend(strategy.replacements(&site)); + } + } + + for candidate in &proposed { + if let Replacement::Rewrite(rc) = &candidate.replacement { + discover_new_descendant_sites(rc, &mut order, &mut nodes, &mut counts); + } + } + + let group = groups + .get_mut(ptr) + .expect("every discovered site has a group"); + for candidate in proposed { + group.add_candidate(candidate); + } + } + + // Any pointer `discover_new_descendant_sites` appended to `order` + // this round is a genuinely new site — give it a group and process + // it next round. Sites already in `groups` are never revisited. + let new_sites = &order[sites_before..]; + for ptr in new_sites { + groups + .entry(*ptr) + .or_insert_with(|| MemoGroup::new(Rc::clone(&nodes[ptr]), counts[ptr])); + } + frontier = new_sites.to_vec(); + } + + PlanSpace { + roots: cse_roots, + groups, + order, + } +} + +// ── site discovery ─────────────────────────────────────────────────────── + +/// Walk every root's whole DAG, discovering one site per distinct `Rc` and +/// its real `consumer_count` — see the module docs' "Where 'for site in +/// plan.bindable_sites()' comes from" section for the full rationale. +fn discover_sites( + roots: &[(Id, Rc)], + order: &mut Vec<*const QueryExpr>, + nodes: &mut HashMap<*const QueryExpr, Rc>, + counts: &mut HashMap<*const QueryExpr, usize>, +) { + for (_, root) in roots { + walk(root, order, nodes, counts); + } +} + +/// Scan `candidate`'s **children** (deliberately never `candidate`'s own +/// top-level pointer — see the module docs' "Termination" section: a +/// [`Replacement::Rewrite`]'s value is an alternative *for* the site that +/// proposed it, never a new site of its own) for any `Rc` not already known, +/// appending each to `order`/`nodes`/`counts` so +/// [`search_workload_with`]'s next round processes it. A no-op when every +/// child is already known — the case both shipped strategies always produce +/// (see that section). +fn discover_new_descendant_sites( + candidate: &Rc, + order: &mut Vec<*const QueryExpr>, + nodes: &mut HashMap<*const QueryExpr, Rc>, + counts: &mut HashMap<*const QueryExpr, usize>, +) { + walk_children(candidate, order, nodes, counts); +} + +/// Visit `node`: count this occurrence, and — the first time this exact +/// `Rc` is seen — record it as a site and recurse into its children. +fn walk( + node: &Rc, + order: &mut Vec<*const QueryExpr>, + nodes: &mut HashMap<*const QueryExpr, Rc>, + counts: &mut HashMap<*const QueryExpr, usize>, +) { + let ptr = Rc::as_ptr(node); + let already_visited = counts.contains_key(&ptr); + *counts.entry(ptr).or_insert(0) += 1; + if !already_visited { + order.push(ptr); + nodes.insert(ptr, Rc::clone(node)); + walk_children(node, order, nodes, counts); + } +} + +/// `node`'s own **relational-skeleton** operator children — the same scope +/// `asap_types::pre_asap::cse::share_common_subtrees`/`rebuild_children` +/// itself uses (see that module's "Algorithm" section) and +/// `tests::count_consumers` mirrors for its own fixtures. Exhaustive over +/// every `QueryExpr` variant: a new variant fails to compile here until this +/// match is extended too. +fn walk_children( + node: &QueryExpr, + order: &mut Vec<*const QueryExpr>, + nodes: &mut HashMap<*const QueryExpr, Rc>, + counts: &mut HashMap<*const QueryExpr, usize>, +) { + use QueryExpr::*; + match node { + Scan { .. } | PromqlScalarBridge(_) | QueryTimestamp => {} + PromqlVectorFromScalar(c) | PromqlScalarFromVector(c) => walk(c, order, nodes, counts), + PromqlRelabel { child, .. } + | PromqlInfoEnrich { child, .. } + | PromqlSeriesSample { child, .. } + | Filter { child, .. } + | Project { child, .. } + | Aggregate { child, .. } + | Dedup { child, .. } + | PromqlSubquery { child, .. } + | TimeRange { child, .. } + | TimeShift { child, .. } + | SQLWindowFunc { child, .. } + | Sort { child, .. } + | Limit { child, .. } => walk(child, order, nodes, counts), + Concat { children } => { + for c in children { + walk_children(c, order, nodes, counts); + } + } + Join { left, right, .. } | SetOp { left, right, .. } => { + walk(left, order, nodes, counts); + walk(right, order, nodes, counts); + } + BinaryOp { lhs, rhs, .. } => { + walk(lhs, order, nodes, counts); + walk(rhs, order, nodes, counts); + } + Column(_) + | Literal(_) + | Compare { .. } + | BoolAnd(_) + | BoolOr(_) + | Not(_) + | IsNull(_) + | IsNotNull(_) + | Cast { .. } + | InList { .. } + | FunctionCall { .. } + | Arithmetic { .. } + | Case { .. } => {} + } +} + #[cfg(test)] mod tests { use super::*; @@ -1864,7 +2686,7 @@ mod tests { /// node's own decision — a nested aggregate underneath it still gets its /// own independent (`cost_model`-ranked) enumeration, not whatever the /// caller happened to pick for the outer target. This is the behavior - /// [`construct_summary`]'s recursion (via `crate::bind::select_and_bind`) + /// [`construct_summary`]'s recursion (via [`select_and_bind`]) /// gets for free: only the top node's `Implementation` is ever forced /// from outside; the child is always re-enumerated fresh. #[test] @@ -2083,4 +2905,451 @@ mod tests { assert!(SharedSubtreeStrategy.matches(&target)); assert_eq!(SharedSubtreeStrategy.replacements(&target).len(), 2); } + + // ── search_workload / PlanSpace / MemoGroup (merged from search.rs) ── + // + // Reuses this test module's own `metric_scan`/`agg` fixture helpers + // above (identical to `search.rs`'s own copies, which are dropped here + // to avoid a duplicate-definition collision now that both test modules + // share one file) and `count_consumers` above (which mirrors + // `discover_sites`' own real, non-test traversal for these fixtures). + + // ── discovery + MEMO shape ─────────────────────────────────────────── + + #[test] + fn single_bindable_aggregate_gets_a_group_with_every_sketch_candidate() { + let root = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); + let space = search_workload(vec![("q", root)]); + + // One group for the Aggregate, one for its Scan child. + assert_eq!(space.len(), 2); + + let agg_group = space + .groups() + .find(|g| matches!(g.target.as_ref(), QueryExpr::Aggregate { .. })) + .expect("an Aggregate group must be discovered"); + assert_eq!(agg_group.consumer_count, 1); + assert_eq!( + agg_group.candidates.len(), + 2, + "quantile has 2 summary_candidates entries: {:?}", + agg_group.candidates + ); + assert!(agg_group + .candidates + .iter() + .all(|c| matches!(c.replacement, Replacement::Summary(_)))); + + let scan_group = space + .groups() + .find(|g| matches!(g.target.as_ref(), QueryExpr::Scan { .. })) + .expect("a Scan group must be discovered"); + assert_eq!(scan_group.consumer_count, 1); + assert!( + scan_group.candidates.is_empty(), + "no strategy matches a bare Scan" + ); + } + + #[test] + fn cardinality_group_gets_all_three_candidates() { + let root = Rc::new(agg(vec![2], default_cardinality(), metric_scan(&["job"]))); + let space = search_workload(vec![("q", root)]); + let agg_group = space + .groups() + .find(|g| matches!(g.target.as_ref(), QueryExpr::Aggregate { .. })) + .unwrap(); + assert_eq!(agg_group.candidates.len(), 3); + } + + #[test] + fn shared_aggregate_across_two_roots_gets_both_strategies_candidates() { + // Two independently-built, structurally identical Sum aggregates: + // share_common_subtrees (run inside search_workload) collapses them + // onto one Rc with consumer_count 2, so this single group should + // carry SketchFamilyStrategy's one ExactAggregate candidate *and* + // SharedSubtreeStrategy's share-vs-recompute pair. + let a = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let b = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let space = search_workload(vec![("a", Rc::new(a)), ("b", Rc::new(b))]); + + // roots[0] and roots[1] must have merged onto the same Rc. + assert!(Rc::ptr_eq(&space.roots[0].1, &space.roots[1].1)); + + let group = space.group_for(&space.roots[0].1).unwrap(); + assert_eq!(group.consumer_count, 2); + assert_eq!( + group.candidates.len(), + 3, + "1 ExactAggregate Summary + 2 Rewrite (share/recompute): {:?}", + group.candidates + ); + + let summary_count = group + .candidates + .iter() + .filter(|c| matches!(c.replacement, Replacement::Summary(_))) + .count(); + let rewrite_count = group + .candidates + .iter() + .filter(|c| matches!(c.replacement, Replacement::Rewrite(_))) + .count(); + assert_eq!(summary_count, 1); + assert_eq!(rewrite_count, 2); + + // The two Rewrite candidates must NOT have collapsed into one + // (the "false-positive dedup" failure mode `is_duplicate_rewrite` + // exists to prevent). + let one_is_the_target = group.candidates.iter().any( + |c| matches!(&c.replacement, Replacement::Rewrite(rc) if Rc::ptr_eq(rc, &group.target)), + ); + let one_is_not = group.candidates.iter().any(|c| { + matches!(&c.replacement, Replacement::Rewrite(rc) if !Rc::ptr_eq(rc, &group.target)) + }); + assert!(one_is_the_target && one_is_not); + } + + #[test] + fn nested_shared_subtree_below_an_unshared_parent_is_still_discovered() { + // A shared grouped Aggregate nested under two *different*, + // unshared Filter parents — real consumer_count must come from + // walking the whole DAG, not just root-level pointer identity + // (bind::implement_workload_with's own consumer-count pass would + // miss this; this module's discover_sites must not). + use asap_types::pre_asap::expr_ir::ScalarValue; + use asap_types::pre_asap::query_expr::Predicate; + + let shared = Rc::new(agg( + vec![2], + AggIntent::Sum { col: None }, + metric_scan(&["job"]), + )); + // Different predicates so the two Filter *parents* stay distinct + // (don't themselves merge under CSE) — only their shared `child` + // should collapse onto one `Rc`. + let root_a = QueryExpr::Filter { + pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Int64(1)))), + child: Rc::clone(&shared), + }; + let root_b = QueryExpr::Filter { + pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Int64(2)))), + child: Rc::clone(&shared), + }; + + let space = search_workload(vec![("a", Rc::new(root_a)), ("b", Rc::new(root_b))]); + assert_eq!( + space.len(), + 4, + "2 distinct Filters + 1 shared Aggregate + 1 shared Scan" + ); + + // `share_common_subtrees` re-clones+re-interns anything that already + // had more than one owner going in (see `cse.rs`'s own doc on + // `intern_child`'s clone-fallback path) — so the post-CSE shared + // node is a *fresh* Rc, structurally equal to (but not the same + // pointer as) the pre-search `shared` variable. Recover it from the + // post-CSE root's own `child` field instead of the stale `shared` + // handle. + let QueryExpr::Filter { + child: post_cse_shared_a, + .. + } = space.roots[0].1.as_ref() + else { + panic!("expected a Filter root"); + }; + let QueryExpr::Filter { + child: post_cse_shared_b, + .. + } = space.roots[1].1.as_ref() + else { + panic!("expected a Filter root"); + }; + assert!( + Rc::ptr_eq(post_cse_shared_a, post_cse_shared_b), + "fixture sanity: the two Filters' children must still merge" + ); + let post_cse_shared = post_cse_shared_a; + let group = space + .group_for(post_cse_shared) + .expect("shared node must be a discovered site"); + assert_eq!(group.consumer_count, 2); + assert!( + SharedSubtreeStrategy.matches(&TargetSubDAG::with_consumer_count( + post_cse_shared, + group.consumer_count + )) + ); + } + + // ── dedup ──────────────────────────────────────────────────────────── + + #[test] + fn add_candidate_rejects_a_true_rewrite_duplicate() { + // SharedSubtreeStrategy's `Replacement::Rewrite` candidates are + // real `QueryExpr` values with `PartialEq`, so `add_candidate` can + // (and must) actually reject a genuine repeat — unlike the + // `Replacement::Summary` case (see the test below). + let root = Rc::new(agg( + vec![2], + AggIntent::Sum { col: None }, + metric_scan(&["job"]), + )); + let mut group = MemoGroup::new(Rc::clone(&root), 2); + let target = TargetSubDAG::with_consumer_count(&root, 2); + let mut inserted = 0; + for candidate in SharedSubtreeStrategy.replacements(&target) { + if group.add_candidate(candidate) { + inserted += 1; + } + } + assert_eq!(inserted, 2, "share + recompute-independently candidates"); + + // Re-adding the identical candidate list must add nothing new: the + // "share" candidate is literally the same Rc as before, and the + // "recompute independently" candidate is a fresh Rc but + // structurally identical value, both already covered by + // `is_duplicate_rewrite`. + let mut re_inserted = 0; + for candidate in SharedSubtreeStrategy.replacements(&target) { + if group.add_candidate(candidate) { + re_inserted += 1; + } + } + assert_eq!( + re_inserted, 0, + "re-proposing the same Rewrite candidates must not grow the group" + ); + assert_eq!(group.candidates.len(), 2); + } + + #[test] + fn add_candidate_never_dedups_summary_candidates() { + // Documented, deliberate consequence of `SummaryNode` deriving no + // `PartialEq` (see `is_duplicate_summary`'s own doc): re-proposing + // the same `Replacement::Summary` candidates DOES grow the group — + // this module refuses to guess at an equality check it can't back + // with a real `PartialEq`. `search_workload_with` never actually + // does this in practice (every site is asked exactly once — see the + // module docs' "Termination" section), so this test exists to pin + // the documented behavior, not to endorse calling `replacements` + // twice for the same site. + let root = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); + let mut group = MemoGroup::new(Rc::clone(&root), 1); + let strategy = SketchFamilyStrategy::default_cost_model(); + let target = TargetSubDAG::new(&root); + for candidate in strategy.replacements(&target) { + group.add_candidate(candidate); + } + assert_eq!(group.candidates.len(), 2); + + for candidate in strategy.replacements(&target) { + group.add_candidate(candidate); + } + assert_eq!( + group.candidates.len(), + 4, + "Summary candidates are never deduped by this module — see is_duplicate_summary" + ); + } + + #[test] + fn is_duplicate_rewrite_never_merges_share_with_recompute() { + let target = Rc::new(agg( + vec![2], + AggIntent::Sum { col: None }, + metric_scan(&["job"]), + )); + let share = Rc::clone(&target); + let recompute = Rc::new((*target).clone()); + assert!(!Rc::ptr_eq(&share, &recompute)); + assert_eq!( + *share, *recompute, + "fixture sanity: same value, different Rc" + ); + assert!(!is_duplicate_rewrite(&share, &recompute, &target)); + assert!(!is_duplicate_rewrite(&recompute, &share, &target)); + } + + #[test] + fn is_duplicate_rewrite_catches_a_real_repeat() { + let target = Rc::new(agg( + vec![2], + AggIntent::Sum { col: None }, + metric_scan(&["job"]), + )); + let first_recompute = Rc::new((*target).clone()); + let second_recompute = Rc::new((*target).clone()); + assert!(!Rc::ptr_eq(&first_recompute, &second_recompute)); + assert!(is_duplicate_rewrite( + &first_recompute, + &second_recompute, + &target + )); + } + + // ── cost-based ranking ─────────────────────────────────────────────── + + #[test] + fn cost_sorted_orders_shared_subtree_candidates_by_cse_share_decision() { + // Many consumers of a cheap-to-recompute, cheap-to-maintain exact + // accumulator: cse_share_decision should prefer Share (see + // cost_model.rs's own `cse_share_decision_shares_when_recompute_dominates_maintenance`). + let mut roots = Vec::new(); + let shared = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + for i in 0..20 { + roots.push((i, Rc::new(shared.clone()))); + } + let space = search_workload(roots); + let group = space.group_for(&space.roots[0].1).unwrap(); + assert_eq!(group.consumer_count, 20); + + let ranked = space.cost_sorted(&DefaultCostModel); + let ranked_group = ranked + .iter() + .find(|g| Rc::ptr_eq(g.target, &space.roots[0].1)) + .unwrap(); + let rewrites: Vec<&ReplacementSubDAG> = ranked_group + .candidates + .iter() + .filter(|c| matches!(c.replacement, Replacement::Rewrite(_))) + .copied() + .collect(); + assert_eq!(rewrites.len(), 2); + let first_shares_target = match &rewrites[0].replacement { + Replacement::Rewrite(rc) => Rc::ptr_eq(rc, &group.target), + Replacement::Summary(_) => false, + }; + assert!( + first_shares_target, + "with 20 cheap consumers, Share should rank first: {rewrites:?}" + ); + } + + #[test] + fn cost_sorted_orders_sketch_candidates_by_rank_candidates() { + struct PreferDDSketch; + impl CostModel for PreferDDSketch { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchKind], + ) -> Vec { + let mut v = candidates.to_vec(); + if let Some(pos) = v.iter().position(|k| *k == SketchKind::DDSketch) { + let dd = v.remove(pos); + v.insert(0, dd); + } + v + } + } + + let root = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); + let space = search_workload(vec![("q", root)]); + let ranked = space.cost_sorted(&PreferDDSketch); + let agg_group = ranked + .iter() + .find(|g| matches!(g.target.as_ref(), QueryExpr::Aggregate { .. })) + .unwrap(); + assert_eq!(agg_group.candidates.len(), 2); + let first_kind = match &agg_group.candidates[0].replacement { + Replacement::Summary(node) => sketch_kind_of(node), + Replacement::Rewrite(_) => None, + }; + assert_eq!(first_kind, Some(SketchKind::DDSketch)); + } + + /// [`RankedGroup::costs`] is a per-candidate annotation, aligned + /// index-for-index with `candidates` — each entry must equal what + /// calling [`CostModel::estimate_cost`] directly on that same candidate + /// and target produces, not some other (or stale) number. + #[test] + fn cost_sorted_pairs_each_candidate_with_its_own_estimate_cost() { + let root = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); + let space = search_workload(vec![("q", root)]); + let ranked = space.cost_sorted(&DefaultCostModel); + let agg_group = ranked + .iter() + .find(|g| matches!(g.target.as_ref(), QueryExpr::Aggregate { .. })) + .unwrap(); + assert_eq!( + agg_group.costs.len(), + agg_group.candidates.len(), + "costs must be aligned 1:1 with candidates" + ); + assert!(!agg_group.costs.is_empty()); + + let target = TargetSubDAG::with_consumer_count(agg_group.target, agg_group.consumer_count); + for (candidate, &cost) in agg_group.candidates.iter().zip(&agg_group.costs) { + assert_eq!( + cost, + DefaultCostModel.estimate_cost(candidate, &target), + "RankedGroup::costs must match calling CostModel::estimate_cost directly \ + for the same candidate/target" + ); + } + } + + // ── termination ────────────────────────────────────────────────────── + + #[test] + fn default_strategies_converge_without_hitting_the_iteration_cap() { + // A workload exercising both strategies at once; if this test + // completes at all, the fixpoint converged well under + // MAX_SEARCH_ITERATIONS (both strategies are idempotent — see the + // module docs — so this always converges in exactly 2 passes). + let a = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + let b = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + let space = search_workload(vec![("a", Rc::new(a)), ("b", Rc::new(b))]); + assert!(!space.is_empty()); + } + + /// A deliberately ill-behaved [`ReplacementStrategy`]: every call to + /// `replacements` wraps `target` in two `Filter` layers — the outer one + /// (ignored by site discovery — see [`discover_new_descendant_sites`]) + /// and an inner one carrying a monotonically-increasing counter, so the + /// inner layer is a **brand-new, never-before-seen `Rc` every call**. + /// Each round, `search_workload_with` discovers that inner layer as a + /// new site, processes it next round (this strategy matches + /// everything), and gets handed *another* fresh inner layer — the + /// frontier never empties, exactly the failure mode + /// [`MAX_SEARCH_ITERATIONS`] exists to catch. + struct AlwaysGrowingStrategy { + next: std::cell::Cell, + } + + impl ReplacementStrategy for AlwaysGrowingStrategy { + fn matches(&self, _target: &TargetSubDAG<'_>) -> bool { + true + } + + fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { + let n = self.next.get(); + self.next.set(n + 1); + use asap_types::pre_asap::expr_ir::ScalarValue; + use asap_types::pre_asap::query_expr::Predicate; + let fresh_inner_layer = QueryExpr::Filter { + pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Int64(n)))), + child: Rc::clone(target.root), + }; + let outer_wrapper = QueryExpr::Filter { + pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Boolean(true)))), + child: Rc::new(fresh_inner_layer), + }; + vec![ReplacementSubDAG { + replacement: Replacement::Rewrite(Rc::new(outer_wrapper)), + rationale: format!("pathological candidate #{n}"), + }] + } + } + + #[test] + #[should_panic(expected = "did not converge")] + fn a_pathologically_growing_strategy_trips_the_iteration_cap() { + let root = Rc::new(metric_scan(&["job"])); + let strategies: Vec> = vec![Box::new(AlwaysGrowingStrategy { + next: std::cell::Cell::new(0), + })]; + let _ = search_workload_with(vec![("q", root)], &strategies); + } } diff --git a/crates/types/src/pre_asap/cse.rs b/crates/types/src/pre_asap/cse.rs index dc139790..7639dc76 100644 --- a/crates/types/src/pre_asap/cse.rs +++ b/crates/types/src/pre_asap/cse.rs @@ -164,7 +164,14 @@ impl InternTable { /// letting it *persist* across every node in one bottom-up pass (as /// [`InternTable`] does via its own `hash_cache` field), rather than /// starting a new one per call. -pub(crate) type HashCache = HashMap<*const QueryExpr, u64>; +/// +/// `pub` (not `pub(crate)`) so `asap_aware_mapping`'s workload-search MEMO +/// engine (`replacement::is_duplicate_rewrite`) can reuse this exact +/// candidate-narrowing filter for its own dedup, instead of maintaining a +/// parallel reimplementation — the same "one real hash, reused everywhere +/// it's needed" rationale [`structural_hash`]'s own doc gives for +/// [`dag_export`](crate::dag_export)'s `pub(crate)` reuse. +pub type HashCache = HashMap<*const QueryExpr, u64>; /// Coarse structural hash used only to bucket [`InternTable::intern`]'s /// candidate search — never the actual sharing decision (`PartialEq` is). @@ -194,19 +201,24 @@ pub(crate) type HashCache = HashMap<*const QueryExpr, u64>; /// [`dag_node_count`]'s own DAG-vs-tree fix (issue #212/#223/#237's stage /// 4) in spirit, applied to hashing instead of counting. /// -/// `pub(crate)` (not private) so [`dag_export`](crate::dag_export) can call +/// `pub` (not private) so [`dag_export`](crate::dag_export) can call /// this exact function for its exported nodes' `hash` field instead of /// maintaining its own parallel reimplementation — issue #223 stage 3. That /// makes `tools/dag-viewer`'s "shared subtree" highlighting reflect this /// module's real hashing, not a lookalike computed a different way; see the /// module doc's "Landing plan" section. A NaN/infinite `f64` makes JSON /// serialization fail; falling back to a fixed hash just puts every such -/// node in one (larger, still `PartialEq`-disambiguated) bucket. +/// node in one (larger, still `PartialEq`-disambiguated) bucket. Made `pub` +/// (rather than staying `pub(crate)`) for one more reuse across the crate +/// boundary: `asap_aware_mapping`'s workload-search MEMO engine +/// (`replacement::is_duplicate_rewrite`) needs the identical +/// candidate-narrowing filter this module's own [`InternTable::intern`] +/// already uses, so it doesn't have to reinvent (and risk drifting from) it. /// /// Exhaustive over every `QueryExpr` variant, matching [`rebuild_children`] /// in which fields count as an operator child (must stay in sync — a new /// variant fails to compile in both places until both are extended). -pub(crate) fn structural_hash(node: &QueryExpr, cache: &mut HashCache) -> u64 { +pub fn structural_hash(node: &QueryExpr, cache: &mut HashCache) -> u64 { use QueryExpr::*; fn child_hash(child: &Rc, cache: &mut HashCache) -> u64 { diff --git a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md index 6df08942..78d8146e 100644 --- a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md @@ -8,9 +8,9 @@ This guide explains how to extend ASAP-aware mapping in the current codebase. Us - add a new kind of replacement without duplicating existing planner logic, - write the tests expected for a new extension. -The focus here is the **current code interfaces and their contracts**. For the higher-level motivation and future search design, see the separate design document, [`docs/design_docs/asap_aware_mapping.md`](../design_docs/asap_aware_mapping.md). +The focus here is the **current code interfaces and their contracts**. For the higher-level motivation and the replacement-plan-searching design this crate now implements, see the separate design document, [`docs/design_docs/asap_aware_mapping.md`](../design_docs/asap_aware_mapping.md). -Names such as `MyStrategy`, `MyCostModel`, and `PreferDDSketch` are illustrative; they do not ship with this crate. Samples that use real types—such as `SketchFamilyStrategy`, `SharedSubtreeStrategy`, and `implementations_for_with`—are copied from `replacement.rs`, `bind.rs`, or `cost_model.rs`. +Names such as `MyStrategy`, `MyCostModel`, and `PreferDDSketch` are illustrative; they do not ship with this crate. Samples that use real types—such as `SketchFamilyStrategy`, `SharedSubtreeStrategy`, `implementations_for_with`, `PlanSpace`, and `search_workload`—are copied from `replacement.rs`, `bind.rs`, or `cost_model.rs`. If you only need to find the right extension point, start with the [extension map](#18-current-extension-map). If you are implementing a strategy, read sections 1–5 first. @@ -258,15 +258,15 @@ A custom cost model does not necessarily need to override every hook. The curren The crate has one source of truth for valid implementations, and deciding and constructing a candidate happen in the same step — not two steps bridged by a separately-named function: - `SketchFamilyStrategy::replacements(target)`, for a bindable `Aggregate`, calls `implementations_for_with(intent, cost_model)` (a `replacement.rs`-private function) to get every valid `Implementation`, ranked by preference and already sized to the target's own accuracy target — then, for each candidate, constructs its bound `SummaryNode` directly (child schema, summarized column, readout, recursion into the child) and returns it as a `ReplacementSubDAG`. It does not discard any candidate. -- A caller that needs one executable answer uses `.into_iter().next()` and handles the empty case. The crate's conservative fallback is `bind::keep_pre_asap`. +- A caller that needs one executable answer uses `.into_iter().next()` and handles the empty case. The crate's conservative fallback is `replacement::keep_pre_asap`. In the [design document](../design_docs/asap_aware_mapping.md), each `ReplacementSubDAG` is a candidate. The complete `replacements()` result is the set of alternatives for one location in the plan. -**Tradeoff:** a single-target bind sizes and constructs every valid sketch candidate before the caller keeps the first one. This costs more than constructing only the preferred candidate, but it means there is exactly one place in the crate that decides what an `AggIntent` may become and exactly one place bound output comes from. Child nodes repeat selection independently (via `bind::select_and_bind`), so a choice at one target does not force choices in nested aggregates. +**Tradeoff:** a single-target bind sizes and constructs every valid sketch candidate before the caller keeps the first one. This costs more than constructing only the preferred candidate, but it means there is exactly one place in the crate that decides what an `AggIntent` may become and exactly one place bound output comes from. Child nodes repeat selection independently (via `replacement::select_and_bind`), so a choice at one target does not force choices in nested aggregates. -This is not yet the whole-plan Cascades/Volcano-style search described in the design document. Today, `cost_model.rank_candidates` ranks one node at a time. Whole-plan candidate generation and selection remain future work. +This crate now also implements the whole-plan Cascades/Volcano-style search the design document describes (issue #252, part of #33): `replacement::search_workload`/`search_workload_with` discover every candidate site across a whole workload and run every registered `ReplacementStrategy` against each to a fixpoint, deduping into a `PlanSpace` — one `MemoGroup` per distinct site, holding every alternative discovered for it. `PlanSpace::cost_sorted` is the final `sorted_by(cost_model)` step. See `replacement.rs`'s own module docs ("Workload-wide search") for the full design: MEMO groups instead of a flat `2^N`-sized plan list, dedup discipline, termination, and cost-based ranking. -**One exception:** `bind::implement_workload` and `implement_workload_with` select the first candidate internally. Workload-wide CSE memoizes by `Rc` pointer identity, so roots that share an `Rc` must use the same canonical decision. Other callers use `SketchFamilyStrategy::replacements()` directly. +**One exception:** `bind::implement_workload` and `implement_workload_with` select the first candidate internally. Workload-wide CSE memoizes by `Rc` pointer identity, so roots that share an `Rc` must use the same canonical decision. Other callers use `SketchFamilyStrategy::replacements()` (for one target) or `search_workload`/`search_workload_with` (for a whole workload's worth of candidates) directly. ### Replacement-strategy path @@ -575,7 +575,7 @@ fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { `implementations_for_with` already did the hard part — enumerating and sizing every candidate, ranked. This loop's only job is to hand each one to `construct_summary(expr, implementation, cost_model)` — a private helper in the same file, not a separately-named public bridge in a different module — which derives the child schema, resolves the summarized column, builds the readout, recurses into the child, and assembles the `SummaryNode`. No per-candidate sizing logic lives here: sizing already happened inside `implementations_for_with`. -Because only `expr`'s own top-level `Implementation` is ever supplied from outside — `construct_summary` recurses into `expr`'s child via `bind::select_and_bind`, a fresh internal selection over that child's own candidates, not a forced one — enumerating a candidate for one target never leaks into that target's own nested aggregates. (An earlier version of this strategy forced ranking via a `CostModel`-wrapping adapter for the whole recursive bind, which had exactly that leak as a latent bug; today's design doesn't have the problem, because nothing below the top node is ever forced.) +Because only `expr`'s own top-level `Implementation` is ever supplied from outside — `construct_summary` recurses into `expr`'s child via `replacement::select_and_bind`, a fresh internal selection over that child's own candidates, not a forced one — enumerating a candidate for one target never leaks into that target's own nested aggregates. (An earlier version of this strategy forced ranking via a `CostModel`-wrapping adapter for the whole recursive bind, which had exactly that leak as a latent bug; today's design doesn't have the problem, because nothing below the top node is ever forced.) This is the pattern to preserve when adding another strategy that needs to enumerate alternatives through an API that normally chooses one: get the exhaustive list from the same enumeration function the single-answer path uses, and construct each entry directly — never wrap the `CostModel` to trick a ranked-first path into producing what you want. @@ -1249,6 +1249,9 @@ Use this table to find the right place for a change. | Decide whether an available implementation satisfies a required one | `impl Matcher` | | Produce a normal (ranked-first) bound summary for one target | `SketchFamilyStrategy::replacements(...).into_iter().next()` | | Bind a whole workload's roots, sharing across CSE-collapsed roots | reuse `bind::implement_workload`/`implement_workload_with` | +| Search a whole workload for every candidate at every site (never pruning) | `replacement::search_workload`/`search_workload_with` | +| Get every candidate ranked best-first, across a whole workload | `PlanSpace::cost_sorted` | +| Get a real numeric cost per candidate, not just a relative rank | `CostModel::estimate_cost` | | Enumerate valid sketch kinds | reuse `replacement::summary_candidates` | | Build a target with no workload context | `TargetSubDAG::new` | | Build a target with known sharing context | `TargetSubDAG::with_consumer_count` | From c562f3c3e22d6e6c15df8b45d16ba7e950822921 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 24 Aug 2026 11:29:53 -0600 Subject: [PATCH 23/49] refactor(asap-aware-mapping): rename "site" -> "target"/TargetSubDAG in the merged search engine ## Why search.rs's own vocabulary (inherited from the design doc's old pseudocode, `plan.bindable_sites()`) used the word "site" throughout -- function names (discover_sites, discover_new_descendant_sites), doc-comment prose, and local variables -- for the exact same concept TargetSubDAG already names: a discovered site is immediately wrapped as `TargetSubDAG::with_consumer_count(&target, consumer_count)` to call a ReplacementStrategy. Same terminology-drift category as the earlier implementation/bind/Logical renames this session -- there shouldn't be two names for one concept. ## What - discover_sites -> discover_targets, discover_new_descendant_sites -> discover_new_descendant_targets (definitions and all call sites). - Local variables in search_workload_with's round loop: the TargetSubDAG-typed local (previously `site`) is now `target`; the Rc it wraps (previously also `target`, colliding) is renamed `root` to make room -- matches the root/target naming TargetSubDAG's own fields and this crate's other fixtures already use. `sites_before`/`new_sites` -> `targets_before`/`new_targets`. - Doc-comment prose describing the real implementation (module docs' "Where 'for site in plan.bindable_sites()' comes from" section -> "Where TargetSubDAG discovery comes from", MemoGroup's "one distinct site" -> "one distinct TargetSubDAG", PlanSpace/discovery/dedup/ termination prose throughout replacement.rs, plus the search-engine prose in lib.rs's Status/Terminology and the developer guide that was written earlier in this same session) all now say "target"/`TargetSubDAG` instead of "site". - The one place "site" stays: the module doc's literal quoted pseudocode block (`for site in plan.bindable_sites(): ...`) and one sentence explicitly introducing it as a historical quote -- that block is explicitly framed as what the design doc "has stubbed out since #206/#211", not current API, so it stays verbatim. - Left alone as unrelated: cost_model.rs's two pre-existing "site" uses (a physical "site count" in a list of example cost dimensions, and "a single use site" in cse_recompute_cost's doc) -- both predate this session and mean something else (a deployment's execution site / a CSE consumer location), not TargetSubDAG. ## Output $ cargo build --workspace --all-targets && cargo test --workspace \ && cargo fmt --all -- --check \ && cargo clippy --workspace --all-targets --all-features -- -D warnings (all clean; asap-aware-mapping: 69 passed, 0 failed) Co-Authored-By: Claude Sonnet 5 --- crates/asap-aware-mapping/src/lib.rs | 12 +- crates/asap-aware-mapping/src/replacement.rs | 186 +++++++++--------- .../ASAP-aware-mapping-developer-guide.md | 4 +- 3 files changed, 104 insertions(+), 98 deletions(-) diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index a505ca56..9b873410 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -46,11 +46,11 @@ //! the build-independently-vs-build-once-and-share choice at a //! CSE-detected shared subtree. //! 2. [`replacement::search_workload`]/[`replacement::search_workload_with`] -//! *search* — discover every candidate site across a whole workload -//! (not just one target in isolation) and run both strategies above -//! against each one, to a fixpoint, without ever materializing a flat -//! `2^N`-sized candidate-plan list: [`replacement::PlanSpace`] holds -//! one Cascades-style [`replacement::MemoGroup`] per distinct site, +//! *search* — discover every candidate `TargetSubDAG` across a whole +//! workload (not just one target in isolation) and run both strategies +//! above against each one, to a fixpoint, without ever materializing a +//! flat `2^N`-sized candidate-plan list: [`replacement::PlanSpace`] holds +//! one Cascades-style [`replacement::MemoGroup`] per distinct target, //! each carrying every alternative discovered for it. //! 3. [`replacement::PlanSpace::cost_sorted`] is the final //! `sorted_by(cost_model)` step, ranking each group's candidates @@ -108,7 +108,7 @@ //! | **Bind #1** | name resolution | `ColumnRef` (a name) → `ColumnId` (a concrete schema column) — the classic RDBMS "Parse → **Bind** → Optimize" pipeline sense (e.g. SQL Server's query-processor terminology) | [`asap_types::pre_asap::binder::Binder`](https://docs.rs/asap-types) | //! | **Implementation** — `replacement::implementations_for_with` | pre-ASAP → post-ASAP, *one node* | enumerating every concrete physical realization (a sketch family, an exact accumulator, or pass-through) for one [`AggIntent`](asap_types::pre_asap::agg_intent::AggIntent) | [`replacement`] | //! | **Replacement** — [`replacement::SketchFamilyStrategy::replacements`] | pre-ASAP → post-ASAP, *one target, every candidate* | wrap each `implementations_for_with` candidate into its own bound [`SummaryNode`](asap_types::post_asap::SummaryNode), ranked — a caller wanting one answer takes the first entry itself | [`replacement`] | -//! | **Search** — [`replacement::search_workload`]/[`replacement::search_workload_with`] | pre-ASAP → post-ASAP, *whole workload, every candidate* | a Cascades/Volcano-style MEMO search: discover every candidate site across a whole workload (not just one target), run every registered `ReplacementStrategy` against each to a fixpoint, and dedup into a [`replacement::PlanSpace`] — one [`replacement::MemoGroup`] per distinct site holding every alternative discovered for it, never a flat `2^N`-sized list of whole candidate plans | [`replacement`] | +//! | **Search** — [`replacement::search_workload`]/[`replacement::search_workload_with`] | pre-ASAP → post-ASAP, *whole workload, every candidate* | a Cascades/Volcano-style MEMO search: discover every candidate `TargetSubDAG` across a whole workload (not just one target in isolation), run every registered `ReplacementStrategy` against each to a fixpoint, and dedup into a [`replacement::PlanSpace`] — one [`replacement::MemoGroup`] per distinct `TargetSubDAG` holding every alternative discovered for it, never a flat `2^N`-sized list of whole candidate plans | [`replacement`] | //! | **`implement_workload`** — [`bind::implement_workload`] | pre-ASAP → post-ASAP, *whole workload, one answer* | walk every root of a `QueryWorkload`, keeping the first (`cost_model`-preferred) candidate per node, sharing one bound `SummaryNode` across roots CSE already collapsed onto one `Rc` — emits the complete post-ASAP `SummaryExpr`/`SummaryNode` DAG | [`bind`] | //! | **Bind #2** (downstream, not in this crate) | post-ASAP → deployment placement | a *deployment's* own physical binder, additionally deciding **placement** (edge vs. backend, wire format, …) — a genuinely different, deployment-specific decision this crate doesn't model at all | e.g. `control_plane::sketch_algebra::rules::bind_*` (as of this writing; expected to fold into that deployment's cost-model layer rather than stay a separate "bind" concept) | //! diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index 7411bee4..06fe9314 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -133,10 +133,10 @@ //! ``` //! //! Read literally, this enumerates whole *plans* — full copies of the -//! workload's tree, one per combination of per-site choices. A workload with -//! `N` independently-choosable sites would produce up to `2^N` flat plans, -//! each one duplicating every untouched sibling subtree. This module does -//! not do that: +//! workload's tree, one per combination of per-target choices. A workload +//! with `N` independently-choosable targets would produce up to `2^N` flat +//! plans, each one duplicating every untouched sibling subtree. This module +//! does not do that: //! //! 1. **MEMO groups, not flat plans.** [`MemoGroup`] is this engine's //! Cascades-style "group": one distinct [`TargetSubDAG`] (identified by @@ -144,12 +144,12 @@ //! [`asap_types::pre_asap::cse::share_common_subtrees`] already //! established across the workload) holding every //! [`ReplacementSubDAG`] alternative discovered for it. [`PlanSpace`] is -//! a collection of these groups, keyed by site — a candidate "plan" is -//! never materialized as a distinct top-level `Rc` at all; -//! two logically-different overall choices at two different sites are -//! just two different entries in two different groups, sharing every -//! other node in the workload by construction (they *are* the same -//! `Rc`s — nothing was copied to make a second "plan"). +//! a collection of these groups, keyed by `TargetSubDAG` — a candidate +//! "plan" is never materialized as a distinct top-level `Rc` +//! at all; two logically-different overall choices at two different +//! targets are just two different entries in two different groups, +//! sharing every other node in the workload by construction (they *are* +//! the same `Rc`s — nothing was copied to make a second "plan"). //! 2. **Dedup by structural hash + `PartialEq`, reusing `pre_asap::cse`'s own //! discipline.** [`asap_types::pre_asap::cse::structural_hash`] (made //! `pub` for exactly this reuse) is only ever a candidate-narrowing @@ -161,28 +161,31 @@ //! wrinkle this reuse needs (a `Rc`-identity case pure value equality //! would get wrong). //! -//! ### Where "for site in plan.bindable_sites()" comes from +//! ### Where `TargetSubDAG` discovery comes from //! -//! [`discover_sites`] is the workload-wide `TargetSubDAG` discovery pass +//! [`discover_targets`] is the workload-wide `TargetSubDAG` discovery pass //! this section used to flag as explicitly *not* implemented ("no //! workload-wide `TargetSubDAG` discovery pass is shipped either... wiring //! it up automatically belongs to the same future search engine, not this -//! issue") — this is that future engine, so it's this module's job now. It -//! walks every workload root's whole DAG (the same **relational-skeleton** -//! operator-child scope `asap_types::pre_asap::cse::share_common_subtrees` -//! itself uses — see that module's "Algorithm" section), discovering one -//! site per distinct `Rc` and a *real* `consumer_count`: how many -//! operator-child positions anywhere in the workload reference that exact -//! `Rc`, not just how many of the workload's own top-level roots happen to -//! be it. This deliberately goes one step further than +//! issue") — this is that future engine, so it's this module's job now, and +//! it's what the quoted pseudocode's `for site in plan.bindable_sites()` +//! line above stands for: every `TargetSubDAG` this pass discovers is one +//! iteration of that loop. It walks every workload root's whole DAG (the +//! same **relational-skeleton** operator-child scope +//! `asap_types::pre_asap::cse::share_common_subtrees` itself uses — see +//! that module's "Algorithm" section), discovering one `TargetSubDAG` per +//! distinct `Rc` and a *real* `consumer_count`: how many operator-child +//! positions anywhere in the workload reference that exact `Rc`, not just +//! how many of the workload's own top-level roots happen to be it. This +//! deliberately goes one step further than //! [`crate::bind::implement_workload_with`]'s own consumer-count pass, //! which only counts whole-root sharing (that function's own doc calls //! widening this "future work" for binding) — a `SharedSubtreeStrategy` //! candidate three levels under an unshared `Filter` is exactly as real a -//! search-space site as a shared whole root, so this module's discovery -//! can't stop at the top level the way binding's does. +//! target as a shared whole root, so this module's discovery can't stop at +//! the top level the way binding's does. //! -//! `discover_sites` duplicates (rather than reuses) this module's own +//! `discover_targets` duplicates (rather than reuses) this module's own //! `#[cfg(test)]`-only `count_consumers` traversal (in the test module //! below), which mirrors this exact shape for this module's own test //! fixtures — that copy is intentionally test-only, so it isn't reachable @@ -193,22 +196,23 @@ //! //! ### Termination //! -//! Every discovered site is asked *once* per registered strategy, never +//! Every discovered target is asked *once* per registered strategy, never //! re-asked — [`search_workload_with`]'s loop processes each round's -//! frontier of not-yet-visited sites exactly one time each, so there is no -//! scenario where the same `(site, strategy)` pair is queried twice (the -//! `new_plans -= candidate_plans` dedup step the module-level pseudocode -//! describes is therefore never asked to recognize "the same candidate, -//! proposed again" as a special case — see [`MemoGroup::add_candidate`]'s -//! own doc on why that distinction matters for [`Replacement::Summary`] -//! specifically, where no real equality check exists to make it safely). +//! frontier of not-yet-visited targets exactly one time each, so there is +//! no scenario where the same `(target, strategy)` pair is queried twice +//! (the `new_plans -= candidate_plans` dedup step the module-level +//! pseudocode describes is therefore never asked to recognize "the same +//! candidate, proposed again" as a special case — see +//! [`MemoGroup::add_candidate`]'s own doc on why that distinction matters +//! for [`Replacement::Summary`] specifically, where no real equality check +//! exists to make it safely). //! //! What *can* grow the frontier is a candidate's own reachable structure: -//! after a site is processed, every [`Replacement::Rewrite`] candidate's +//! after a target is processed, every [`Replacement::Rewrite`] candidate's //! **children** (never the candidate's own top-level node — that value is -//! an alternative *for* the site just processed, not a new site of its own; -//! see [`discover_new_descendant_sites`]) are scanned for pointers not -//! already known, and any found become next round's frontier. Both shipped +//! an alternative *for* the target just processed, not a new target of its +//! own; see [`discover_new_descendant_targets`]) are scanned for pointers +//! not already known, and any found become next round's frontier. Both shipped //! strategies are idempotent in exactly this sense: [`SketchFamilyStrategy`] //! produces terminal [`Replacement::Summary`] candidates (no `QueryExpr` //! children to scan at all), and [`SharedSubtreeStrategy`]'s two @@ -1372,7 +1376,7 @@ impl ReplacementStrategy for SharedSubtreeStrategy { /// future [`ReplacementStrategy`] (see the module docs' "Termination" /// section) — not a bound either shipped strategy could ever approach. /// [`SketchFamilyStrategy`] and [`SharedSubtreeStrategy`] both converge in -/// exactly 2 passes over a fixed site set, regardless of workload size. +/// exactly 2 passes over a fixed target set, regardless of workload size. pub const MAX_SEARCH_ITERATIONS: usize = 1_000; // ── MemoGroup ──────────────────────────────────────────────────────────── @@ -1383,16 +1387,17 @@ pub const MAX_SEARCH_ITERATIONS: usize = 1_000; /// [`ReplacementSubDAG`] alternative any registered [`ReplacementStrategy`] /// proposed for it. /// -/// `candidates` is deliberately *not* required to be non-empty — a site no -/// registered strategy has an opinion on still gets a group (with an empty -/// candidate list), so [`PlanSpace`] always has exactly one group per -/// discovered site, not "one group per site something matched". +/// `candidates` is deliberately *not* required to be non-empty — a +/// `TargetSubDAG` no registered strategy has an opinion on still gets a +/// group (with an empty candidate list), so [`PlanSpace`] always has +/// exactly one group per discovered `TargetSubDAG`, not "one group per +/// `TargetSubDAG` something matched". #[derive(Debug, Clone)] pub struct MemoGroup { /// The target sub-DAG this group is for. pub target: Rc, /// How many operator-child positions across the whole workload - /// reference this exact `Rc` — see [`discover_sites`]. + /// reference this exact `Rc` — see [`discover_targets`]. pub consumer_count: usize, /// Every distinct alternative discovered for `target`, in discovery /// order (not ranked — see [`PlanSpace::cost_sorted`] for the ranked @@ -1508,14 +1513,14 @@ fn is_duplicate_summary(_existing: &Rc, _candidate: &Rc` whose group holds its -/// alternatives. +/// discover: one [`MemoGroup`] per distinct `TargetSubDAG` in the +/// (already-CSE'd) workload, plus the workload's own post-CSE roots so a +/// caller can still map a `Root`'s `Id` back to the `Rc` whose +/// group holds its alternatives. pub struct PlanSpace { /// The workload's roots, after the one `share_common_subtrees` pass /// [`search_workload_with`] runs up front — the same post-CSE roots - /// every site in `groups` was discovered from. + /// every `TargetSubDAG` in `groups` was discovered from. pub roots: Vec<(Id, Rc)>, groups: HashMap<*const QueryExpr, MemoGroup>, /// Discovery order — stable iteration for [`PlanSpace::groups`]/ @@ -1529,20 +1534,21 @@ impl PlanSpace { self.order.iter().map(move |ptr| &self.groups[ptr]) } - /// How many distinct sites were discovered. + /// How many distinct targets were discovered. pub fn len(&self) -> usize { self.groups.len() } - /// Whether no sites were discovered at all (an empty workload, or one + /// Whether no targets were discovered at all (an empty workload, or one /// with no `QueryExpr` nodes reachable from any root — never true for a - /// non-empty `roots`, since every root is itself a site). + /// non-empty `roots`, since every root is itself a target). pub fn is_empty(&self) -> bool { self.groups.is_empty() } - /// The group for `target`, if `target`'s own `Rc` is a discovered site - /// (i.e. `Rc::ptr_eq` to some node reachable from `roots`). + /// The group for `target`, if `target`'s own `Rc` is a discovered + /// `TargetSubDAG` (i.e. `Rc::ptr_eq` to some node reachable from + /// `roots`). pub fn group_for(&self, target: &Rc) -> Option<&MemoGroup> { self.groups.get(&Rc::as_ptr(target)) } @@ -1550,7 +1556,7 @@ impl PlanSpace { /// The `sorted_by(cost_model)` step: every group, each with its own /// candidates ranked best-first under `cost_model` where this module /// knows how (see the module docs' "Cost-based final selection" - /// section) — groups themselves stay in discovery order, since sites + /// section) — groups themselves stay in discovery order, since targets /// are independent decision points, not alternatives competing with /// each other. /// @@ -1708,7 +1714,7 @@ fn bind_one(target: &Rc, cost_model: &dyn CostModel) -> Option Option { match &node.expr { SummaryExpr::SummaryEstimate { summary_input, .. } => sketch_kind_of(summary_input), @@ -1768,10 +1774,10 @@ pub fn search_workload(roots: Vec<(Id, Rc)>) -> PlanSpace { /// Runs [`share_common_subtrees`] once over `roots` first — the same /// pattern a hypothetical `applicability::find_applicable_optimizations` /// uses, so every strategy sees the same already-deduplicated tree -/// [`crate::bind::implement_workload`] would — then discovers every site -/// (see [`discover_sites`]) and runs the fixpoint loop the module docs -/// describe, capped at [`MAX_SEARCH_ITERATIONS`] passes (see the module -/// docs' "Termination" section). Deduping candidate plans this way needs no +/// [`crate::bind::implement_workload`] would — then discovers every +/// `TargetSubDAG` (see [`discover_targets`]) and runs the fixpoint loop the +/// module docs describe, capped at [`MAX_SEARCH_ITERATIONS`] passes (see the +/// module docs' "Termination" section). Deduping candidate plans this way needs no /// [`CostModel`] at all — that only enters at two well-defined points: each /// [`ReplacementStrategy`] in `strategies` may already carry its own (e.g. /// [`SketchFamilyStrategy::new`]'s), and [`PlanSpace::cost_sorted`]'s final @@ -1796,19 +1802,19 @@ pub fn search_workload_with<'s, Id>( let mut order = Vec::new(); let mut nodes = HashMap::new(); let mut counts: HashMap<*const QueryExpr, usize> = HashMap::new(); - discover_sites(&cse_roots, &mut order, &mut nodes, &mut counts); + discover_targets(&cse_roots, &mut order, &mut nodes, &mut counts); let mut groups: HashMap<*const QueryExpr, MemoGroup> = HashMap::new(); for ptr in &order { groups.insert(*ptr, MemoGroup::new(Rc::clone(&nodes[ptr]), counts[ptr])); } - // Round-based frontier: every site is asked exactly once per strategy + // Round-based frontier: every target is asked exactly once per strategy // (never re-asked — see the module docs' "Termination" section on why // that matters for `Replacement::Summary` dedup specifically). A round // can grow the *next* round's frontier only by a candidate's own // reachable children exposing a genuinely new, not-yet-known `Rc` — see - // `discover_new_descendant_sites`. + // `discover_new_descendant_targets`. let mut frontier = order.clone(); let mut rounds = 0usize; while !frontier.is_empty() { @@ -1823,45 +1829,45 @@ pub fn search_workload_with<'s, Id>( search_workload_with.", ); - let sites_before = order.len(); + let targets_before = order.len(); for ptr in &frontier { - let (target, consumer_count) = { + let (root, consumer_count) = { let group = &groups[ptr]; (Rc::clone(&group.target), group.consumer_count) }; - let site = TargetSubDAG::with_consumer_count(&target, consumer_count); + let target = TargetSubDAG::with_consumer_count(&root, consumer_count); let mut proposed = Vec::new(); for strategy in strategies { - if strategy.matches(&site) { - proposed.extend(strategy.replacements(&site)); + if strategy.matches(&target) { + proposed.extend(strategy.replacements(&target)); } } for candidate in &proposed { if let Replacement::Rewrite(rc) = &candidate.replacement { - discover_new_descendant_sites(rc, &mut order, &mut nodes, &mut counts); + discover_new_descendant_targets(rc, &mut order, &mut nodes, &mut counts); } } let group = groups .get_mut(ptr) - .expect("every discovered site has a group"); + .expect("every discovered target has a group"); for candidate in proposed { group.add_candidate(candidate); } } - // Any pointer `discover_new_descendant_sites` appended to `order` - // this round is a genuinely new site — give it a group and process - // it next round. Sites already in `groups` are never revisited. - let new_sites = &order[sites_before..]; - for ptr in new_sites { + // Any pointer `discover_new_descendant_targets` appended to `order` + // this round is a genuinely new target — give it a group and process + // it next round. Targets already in `groups` are never revisited. + let new_targets = &order[targets_before..]; + for ptr in new_targets { groups .entry(*ptr) .or_insert_with(|| MemoGroup::new(Rc::clone(&nodes[ptr]), counts[ptr])); } - frontier = new_sites.to_vec(); + frontier = new_targets.to_vec(); } PlanSpace { @@ -1871,12 +1877,12 @@ pub fn search_workload_with<'s, Id>( } } -// ── site discovery ─────────────────────────────────────────────────────── +// ── target discovery ───────────────────────────────────────────────────── -/// Walk every root's whole DAG, discovering one site per distinct `Rc` and -/// its real `consumer_count` — see the module docs' "Where 'for site in -/// plan.bindable_sites()' comes from" section for the full rationale. -fn discover_sites( +/// Walk every root's whole DAG, discovering one `TargetSubDAG` per distinct +/// `Rc` and its real `consumer_count` — see the module docs' "Where +/// `TargetSubDAG` discovery comes from" section for the full rationale. +fn discover_targets( roots: &[(Id, Rc)], order: &mut Vec<*const QueryExpr>, nodes: &mut HashMap<*const QueryExpr, Rc>, @@ -1889,13 +1895,13 @@ fn discover_sites( /// Scan `candidate`'s **children** (deliberately never `candidate`'s own /// top-level pointer — see the module docs' "Termination" section: a -/// [`Replacement::Rewrite`]'s value is an alternative *for* the site that -/// proposed it, never a new site of its own) for any `Rc` not already known, -/// appending each to `order`/`nodes`/`counts` so +/// [`Replacement::Rewrite`]'s value is an alternative *for* the target that +/// proposed it, never a new target of its own) for any `Rc` not already +/// known, appending each to `order`/`nodes`/`counts` so /// [`search_workload_with`]'s next round processes it. A no-op when every /// child is already known — the case both shipped strategies always produce /// (see that section). -fn discover_new_descendant_sites( +fn discover_new_descendant_targets( candidate: &Rc, order: &mut Vec<*const QueryExpr>, nodes: &mut HashMap<*const QueryExpr, Rc>, @@ -1905,7 +1911,7 @@ fn discover_new_descendant_sites( } /// Visit `node`: count this occurrence, and — the first time this exact -/// `Rc` is seen — record it as a site and recurse into its children. +/// `Rc` is seen — record it as a target and recurse into its children. fn walk( node: &Rc, order: &mut Vec<*const QueryExpr>, @@ -2912,7 +2918,7 @@ mod tests { // above (identical to `search.rs`'s own copies, which are dropped here // to avoid a duplicate-definition collision now that both test modules // share one file) and `count_consumers` above (which mirrors - // `discover_sites`' own real, non-test traversal for these fixtures). + // `discover_targets`' own real, non-test traversal for these fixtures). // ── discovery + MEMO shape ─────────────────────────────────────────── @@ -3016,7 +3022,7 @@ mod tests { // unshared Filter parents — real consumer_count must come from // walking the whole DAG, not just root-level pointer identity // (bind::implement_workload_with's own consumer-count pass would - // miss this; this module's discover_sites must not). + // miss this; this module's discover_targets must not). use asap_types::pre_asap::expr_ir::ScalarValue; use asap_types::pre_asap::query_expr::Predicate; @@ -3072,7 +3078,7 @@ mod tests { let post_cse_shared = post_cse_shared_a; let group = space .group_for(post_cse_shared) - .expect("shared node must be a discovered site"); + .expect("shared node must be a discovered target"); assert_eq!(group.consumer_count, 2); assert!( SharedSubtreeStrategy.matches(&TargetSubDAG::with_consumer_count( @@ -3130,10 +3136,10 @@ mod tests { // the same `Replacement::Summary` candidates DOES grow the group — // this module refuses to guess at an equality check it can't back // with a real `PartialEq`. `search_workload_with` never actually - // does this in practice (every site is asked exactly once — see the + // does this in practice (every target is asked exactly once — see the // module docs' "Termination" section), so this test exists to pin // the documented behavior, not to endorse calling `replacements` - // twice for the same site. + // twice for the same target. let root = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); let mut group = MemoGroup::new(Rc::clone(&root), 1); let strategy = SketchFamilyStrategy::default_cost_model(); @@ -3306,11 +3312,11 @@ mod tests { /// A deliberately ill-behaved [`ReplacementStrategy`]: every call to /// `replacements` wraps `target` in two `Filter` layers — the outer one - /// (ignored by site discovery — see [`discover_new_descendant_sites`]) + /// (ignored by target discovery — see [`discover_new_descendant_targets`]) /// and an inner one carrying a monotonically-increasing counter, so the /// inner layer is a **brand-new, never-before-seen `Rc` every call**. /// Each round, `search_workload_with` discovers that inner layer as a - /// new site, processes it next round (this strategy matches + /// new target, processes it next round (this strategy matches /// everything), and gets handed *another* fresh inner layer — the /// frontier never empties, exactly the failure mode /// [`MAX_SEARCH_ITERATIONS`] exists to catch. diff --git a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md index 78d8146e..c97f6326 100644 --- a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md @@ -264,7 +264,7 @@ In the [design document](../design_docs/asap_aware_mapping.md), each `Replacemen **Tradeoff:** a single-target bind sizes and constructs every valid sketch candidate before the caller keeps the first one. This costs more than constructing only the preferred candidate, but it means there is exactly one place in the crate that decides what an `AggIntent` may become and exactly one place bound output comes from. Child nodes repeat selection independently (via `replacement::select_and_bind`), so a choice at one target does not force choices in nested aggregates. -This crate now also implements the whole-plan Cascades/Volcano-style search the design document describes (issue #252, part of #33): `replacement::search_workload`/`search_workload_with` discover every candidate site across a whole workload and run every registered `ReplacementStrategy` against each to a fixpoint, deduping into a `PlanSpace` — one `MemoGroup` per distinct site, holding every alternative discovered for it. `PlanSpace::cost_sorted` is the final `sorted_by(cost_model)` step. See `replacement.rs`'s own module docs ("Workload-wide search") for the full design: MEMO groups instead of a flat `2^N`-sized plan list, dedup discipline, termination, and cost-based ranking. +This crate now also implements the whole-plan Cascades/Volcano-style search the design document describes (issue #252, part of #33): `replacement::search_workload`/`search_workload_with` discover every candidate `TargetSubDAG` across a whole workload and run every registered `ReplacementStrategy` against each to a fixpoint, deduping into a `PlanSpace` — one `MemoGroup` per distinct `TargetSubDAG`, holding every alternative discovered for it. `PlanSpace::cost_sorted` is the final `sorted_by(cost_model)` step. See `replacement.rs`'s own module docs ("Workload-wide search") for the full design: MEMO groups instead of a flat `2^N`-sized plan list, dedup discipline, termination, and cost-based ranking. **One exception:** `bind::implement_workload` and `implement_workload_with` select the first candidate internally. Workload-wide CSE memoizes by `Rc` pointer identity, so roots that share an `Rc` must use the same canonical decision. Other callers use `SketchFamilyStrategy::replacements()` (for one target) or `search_workload`/`search_workload_with` (for a whole workload's worth of candidates) directly. @@ -1249,7 +1249,7 @@ Use this table to find the right place for a change. | Decide whether an available implementation satisfies a required one | `impl Matcher` | | Produce a normal (ranked-first) bound summary for one target | `SketchFamilyStrategy::replacements(...).into_iter().next()` | | Bind a whole workload's roots, sharing across CSE-collapsed roots | reuse `bind::implement_workload`/`implement_workload_with` | -| Search a whole workload for every candidate at every site (never pruning) | `replacement::search_workload`/`search_workload_with` | +| Search a whole workload for every candidate at every `TargetSubDAG` (never pruning) | `replacement::search_workload`/`search_workload_with` | | Get every candidate ranked best-first, across a whole workload | `PlanSpace::cost_sorted` | | Get a real numeric cost per candidate, not just a relative rank | `CostModel::estimate_cost` | | Enumerate valid sketch kinds | reuse `replacement::summary_candidates` | From 51ae6e48c7414716f6bc43a3f2b6f2c1bf30b9ab Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 24 Aug 2026 11:52:22 -0600 Subject: [PATCH 24/49] refactor(asap-aware-mapping): delete bind.rs, retire "select_and_bind" (#251, part of #33) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why `asap-aware-mapping`'s output boundary is `PlanSpace` — every candidate `TargetSubDAG` replacement plus its cost, meant for a downstream DAG-visualization consumer. It does not own "commit to one final, physically-materialized, shared post-ASAP DAG for a whole workload" — that's a downstream deployment's job (picking *which* sketch and *where* to place it are a joint decision only a deployment can see the full picture for; this crate pre-deciding one in isolation, with no real consumer of that single materialized answer inside this crate, is out of scope). `bind.rs`'s `implement_workload`/`implement_workload_with` (keep first/cost-preferred candidate per node, memoized by `Rc` identity across roots so pre-ASAP CSE sharing survives into a single physical binding) was exactly that out-of-scope "commit to one answer" step, so it's deleted. ## What 1. Deleted `bind.rs` entirely (no replacement file, no rename). Removed `pub mod bind;` and its re-exports from `lib.rs`. `ImplementError` moved into `replacement.rs` (`pub`, since it's part of what a `ReplacementStrategy` implementor's construction path can realistically fail with) — still needed by `realize_child`/`keep_pre_asap`/the child-recursion path. `bind.rs`'s own test suite (16 tests exercising `select_and_bind`'s/`construct_summary_agg`'s schema-derivation behavior across various `AggIntent` shapes) moved into `replacement.rs`'s existing test module, adapted to call `realize_child` directly — this crate's own production logic, not `bind.rs`'s deleted orchestration. The one test specific to `implement_workload_with`'s CSE-decline memoization behavior has no equivalent left to test and was dropped (that comparison is already covered by `cost_sorted_orders_shared_subtree_candidates_by_cse_share_decision` and `cost_model.rs`'s own `cse_share_decision` tests). Rewrote `lib.rs`'s `## Status` section: two capabilities — `replacement::ReplacementStrategy` (per-target, exhaustive) and `replacement::search_workload`/`PlanSpace::cost_sorted` (workload-wide, every candidate + cost) — plus an explicit statement that picking/ materializing one final answer is a downstream deployment's job. Rewrote the `## Terminology` table: dropped the `implement_workload` row; the `Bind #2` (downstream placement) row now also covers "which candidate to commit to", since that's where the deleted step's behavior would live if a deployment still wants it. 2. Renamed the `pub(crate)` single-target rank-and-take-first helper `select_and_bind` → `realize_child` (word "bind" is retired from this crate's vocabulary — see `lib.rs`'s Terminology section). Also renamed its sibling `bind_one` (the representative-node helper `cse_preference` uses for a `CostModel::cse_share_decision` comparison) → `realize_one` for the same reason. Updated every call site and doc comment; behavior unchanged (rank via `SketchFamilyStrategy`, take first, `Rewrite` is unreachable, `None` falls back to `keep_pre_asap`). 3. Renamed "site" → "target"/`TargetSubDAG` throughout the merged search-engine content in `replacement.rs`: `discover_sites` → `discover_targets`, `discover_new_descendant_sites` → `discover_new_descendant_targets`, local variables/doc prose (`MemoGroup`'s "one distinct site", the "Where 'for site in plan.bindable_sites()' comes from" section heading, `PlanSpace`/ dedup/termination prose). The quoted historical pseudocode block itself (`for site in plan.bindable_sites()`) stays verbatim, framed explicitly as a quote. (This item was landed independently by a concurrent session as commit c562f3c while this work was in progress on the same shared branch — reconciled here via `git reset --hard origin/...` + manual reapply per the shared-branch protocol; this commit only carries items 1/2/4/5 on top of it.) 4. Rewrote `crates/integration-tests/tests/cse.rs`: it called the now- deleted `implement_workload` and asserted two bound `SummaryNode`s were `Rc::ptr_eq` — a workload-level "one materialized answer" claim that no longer has an API to make. Rewrote its three tests to call `replacement::search_workload` and assert on the resulting `PlanSpace` instead: the shared subtree collapses onto one `MemoGroup`/one shared `Rc`, mirroring `replacement.rs`'s own `shared_aggregate_across_two_roots_gets_both_strategies_candidates` test through the crate's public API. 5. Fixed every other `asap_aware_mapping::bind::*` import (devtools' `show_post_asap_ir`, `frontend-promql`'s corpus test, `integration-tests`' `l4_binding`/`l4_binding_sql`) to `asap_aware_mapping::replacement::*`, and updated stale doc references in `asap-types::pre_asap::cse`/`mod.rs` that pointed at `asap_aware_mapping::implement_workload`/`bind::implement_workload_with`. ## Verification $ cargo build --workspace --all-targets && cargo test --workspace \ && cargo fmt --all -- --check \ && cargo clippy --workspace --all-targets --all-features -- -D warnings (all clean; asap-aware-mapping: 68 passed, 0 failed; cse.rs: 3 passed) Co-Authored-By: Claude Sonnet 5 --- crates/asap-aware-mapping/src/bind.rs | 775 ------------------ crates/asap-aware-mapping/src/cost_model.rs | 21 +- crates/asap-aware-mapping/src/lib.rs | 107 ++- crates/asap-aware-mapping/src/replacement.rs | 679 +++++++++++++-- crates/devtools/src/bin/show_post_asap_ir.rs | 2 +- .../tests/observability/promql_corpus.rs | 2 +- crates/integration-tests/tests/cse.rs | 170 ++-- crates/integration-tests/tests/l4_binding.rs | 2 +- .../integration-tests/tests/l4_binding_sql.rs | 2 +- crates/types/src/pre_asap/cse.rs | 35 +- crates/types/src/pre_asap/mod.rs | 2 +- 11 files changed, 812 insertions(+), 985 deletions(-) delete mode 100644 crates/asap-aware-mapping/src/bind.rs diff --git a/crates/asap-aware-mapping/src/bind.rs b/crates/asap-aware-mapping/src/bind.rs deleted file mode 100644 index 8e524c96..00000000 --- a/crates/asap-aware-mapping/src/bind.rs +++ /dev/null @@ -1,775 +0,0 @@ -//! Workload-level pre-ASAP → post-ASAP binding orchestration (issue #98). -//! -//! This module does **not** expose a "bind me one tree" entry point for a -//! single target. [`crate::replacement::SketchFamilyStrategy`] is the only -//! public way to get bound output for one target — it always returns every -//! candidate [`ReplacementSubDAG`](crate::replacement::ReplacementSubDAG), -//! ranked; a caller that wants a single executable answer takes the first -//! entry itself (`.into_iter().next()`) and decides what to do if the list -//! is empty ([`keep_pre_asap`] is the same pass-through fallback this crate's own -//! dispatch would otherwise use — see `crate::replacement`'s own module docs -//! for how deciding *and* constructing each candidate both happen inside -//! that one strategy). -//! -//! What this module *does* provide is workload-wide orchestration: -//! -//! ## Why [`implement_workload`]/[`implement_workload_with`] are here -//! -//! Workload-level CSE sharing ([`asap_types::pre_asap::cse::share_common_subtrees`]) -//! memoizes on `Rc` pointer identity: two workload roots that collapsed onto -//! the same `Rc` must resolve to the *same* canonical decision to -//! be shareable at all — there is no meaningful "N candidates" answer to -//! memoize against. So this one entry point keeps a rank-and-take-first -//! behavior internally (via [`crate::replacement::select_and_bind`]), scoped -//! to workload-wide CSE memoization specifically. It is not a general -//! "bind me one tree" API — for a single target, go through -//! `SketchFamilyStrategy` and decide what to keep yourself. -//! -//! ## Conservative fallbacks -//! -//! [`SummaryExpr::KeepPreAsap`] boxes a whole pre-ASAP subtree — it has no -//! post-ASAP children — so a *logical* operator above a bindable aggregate -//! (`Filter`/`BinaryOp`/… over a quantile) subsumes the aggregate into the -//! logical wrapper unbound. Rewriting through logical parents is the -//! post-ASAP rule engine's job (#6/#33), not this pass's. Similarly -//! conservative: multi-intent `Aggregate` nodes (SQL `SELECT SUM(a), AVG(b)`) -//! and aggregates with a `HAVING` predicate (the filter would need the -//! estimate first) stay logical — see [`crate::replacement::bindable_intent`]. -//! -//! ## Why this module is so small -//! -//! `select_and_bind`/`keep_pre_asap` themselves moved into -//! [`crate::replacement`] — both are single-target-scoped helpers with no -//! real workload-level state, the same category as everything else already -//! living there. [`keep_pre_asap`] is re-exported here (`pub use -//! crate::replacement::keep_pre_asap`) so `bind::keep_pre_asap` keeps -//! resolving for existing external callers that reach it through this -//! module's path; `select_and_bind` stays `pub(crate)` in `replacement.rs`, -//! reachable from this module (its only other crate-internal caller, -//! [`implement_workload_with`]) without a public re-export. What's left -//! here — [`ImplementError`], [`implement_workload`]/ -//! [`implement_workload_with`], and this module's own tests — genuinely -//! belongs in its own module: `implement_workload`/`implement_workload_with` -//! own cross-root `Rc`-identity memoization state a per-target module has -//! no business holding (see "Why `implement_workload`/`implement_workload_with` -//! are here" above). - -use std::rc::Rc; - -use asap_types::post_asap::SummaryNode; -use asap_types::pre_asap::query_expr::{QueryExpr, QueryExprError}; -use thiserror::Error; - -use crate::cost_model::{CostModel, CseCandidate, DefaultCostModel, ShareDecision}; -pub use crate::replacement::keep_pre_asap; -use crate::replacement::select_and_bind; - -/// Errors from the pre-ASAP → post-ASAP binding pass. -#[derive(Debug, Error)] -pub enum ImplementError { - /// Schema derivation failed while lifting an edge to `SummarySchema`. - #[error("schema derivation failed during pre-ASAP → post-ASAP binding: {0}")] - Schema(#[from] QueryExprError), -} - -/// Bind a whole workload's worth of already-CSE'd roots -/// ([`asap_types::pre_asap::cse::share_common_subtrees`]'s output), reusing -/// one bound [`SummaryNode`] wherever two roots share the same `Rc` *and* -/// `cost_model` decides it's worth it (issue #212, #222, #223 stages 2 and -/// 4, #237). -/// -/// This is a real caller for `share_common_subtrees`, wired up deliberately: -/// the pass's own landing plan calls out that its predecessor -/// (`asap-plan::cse::dedupe_subtrees`) was deleted in #192 for being unwired -/// dead code, and lands this memoization alongside it so that never becomes -/// true again. -/// -/// The memo key is `Rc::as_ptr` — pointer identity, not a second -/// `PartialEq`/structural-equality pass. `share_common_subtrees` already made -/// the (non-negotiable, `PartialEq`-checked) sharing decision; this only -/// needs to recognize when it already bound the exact `Rc` a later root -/// hands back, and is deliberately *not* a general "does an available -/// `Implementation` satisfy this one" lookup — that subsumption question is -/// `asap_aware_mapping::replacement::Matcher`'s documented, deliberately-unfilled -/// job, not this one's. -/// -/// A first pass over `roots` counts each distinct `Rc` pointer's -/// true `consumer_count` across the whole workload, so the -/// [`CseCandidate`]/[`CostModel::cse_share_decision`] cost comparison (see -/// `docs/design_docs/cse-cost-model-decision.md`) sees the real total, not a running -/// count that grows as roots are processed left to right. The decision is -/// made once, the first time a shared pointer is bound, and cached alongside -/// the bound `SummaryNode` so every later occurrence of that same `Rc` -/// applies the same decision consistently — either every consumer reuses one -/// shared `SummaryNode`, or every consumer (including the first) binds -/// independently. -/// -/// Only whole-root sharing is memoized (matching two workload roots that are -/// themselves the same `Rc` after CSE) — a root is bound at most -/// once per distinct root pointer when the decision is -/// `Share`, but it still walks each such tree's own internal structure -/// fresh; a subtree shared only *below* two different roots' top level does -/// not additionally memoize inside that walk. Widening this to sub-root -/// memoization is future work. -pub fn implement_workload( - roots: Vec<(Id, Rc)>, -) -> Vec<(Id, Result, ImplementError>)> { - implement_workload_with(roots, &DefaultCostModel) -} - -/// Like [`implement_workload`], but ranks candidate summaries — and decides -/// CSE sharing — via `cost_model` instead of the built-in defaults (see -/// [`crate::cost_model`]). -pub fn implement_workload_with( - roots: Vec<(Id, Rc)>, - cost_model: &dyn CostModel, -) -> Vec<(Id, Result, ImplementError>)> { - let mut consumer_count: std::collections::HashMap<*const QueryExpr, usize> = - std::collections::HashMap::new(); - for (_, expr) in &roots { - *consumer_count.entry(Rc::as_ptr(expr)).or_insert(0) += 1; - } - - let mut memo: std::collections::HashMap<*const QueryExpr, (Rc, ShareDecision)> = - std::collections::HashMap::new(); - roots - .into_iter() - .map(|(id, expr)| { - let ptr = Rc::as_ptr(&expr); - let result = match memo.get(&ptr) { - Some((cached, ShareDecision::Share)) => Ok(Rc::clone(cached)), - Some((_, ShareDecision::RecomputeIndependently)) => { - select_and_bind(&expr, cost_model) - } - None => select_and_bind(&expr, cost_model).inspect(|node| { - let count = consumer_count[&ptr]; - let decision = if count > 1 { - let candidate = CseCandidate { - subtree: &expr, - bound_summary: node, - consumer_count: count, - }; - cost_model.cse_share_decision(&candidate) - } else { - // Only one consumer: nothing to compare against, and - // this branch is never consulted again for `ptr`. - ShareDecision::Share - }; - memo.insert(ptr, (Rc::clone(node), decision)); - }), - }; - (id, result) - }) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::post_asap::{ - ExactKind, ExactParams, SketchKind, SketchParams, SketchQuery, SummaryExpr, - SummaryFamilyType, - }; - use asap_types::pre_asap::agg_intent::{default_quantile, AggIntent}; - use asap_types::pre_asap::expr_ir::{ColumnRef, CompareOpKind, ScalarValue}; - use asap_types::pre_asap::query_expr::{Predicate, Reduction, Source}; - use asap_types::pre_asap::schema::{Column, DataType, Schema}; - use asap_types::types::AccuracyTarget; - use std::time::Duration; - - fn metric_scan(labels: &[&str]) -> QueryExpr { - let mut columns = vec![ - Column::new("ts", DataType::Timestamp, false), - Column::new("value", DataType::Float64, false), - ]; - columns.extend(labels.iter().map(|n| Column::new(*n, DataType::Utf8, true))); - QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index(columns, 0, vec![]), - } - } - - /// A cross-series reduction, grouped by `by` (possibly empty — a - /// genuine full reduction, never "no grouping concept"). - fn agg(by: Vec, intent: AggIntent, child: QueryExpr) -> QueryExpr { - QueryExpr::Aggregate { - reduction: Reduction::by(by), - measures: vec![intent], - output_names: vec![], - having: None, - child: Rc::new(child), - } - } - - /// A per-entity reduction: no grouping concept at all (issue #165). - fn agg_per_entity(intent: AggIntent, child: QueryExpr) -> QueryExpr { - QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![intent], - output_names: vec![], - having: None, - child: Rc::new(child), - } - } - - fn field<'a>( - schema: &'a asap_types::post_asap::SummarySchema, - name: &str, - ) -> &'a asap_types::post_asap::SummaryField { - schema - .fields - .iter() - .find(|f| f.name == name) - .unwrap_or_else(|| panic!("no field {name:?} in {schema:?}")) - } - - /// This module no longer exposes a single-answer public API — the tests - /// below use the same "rank via `cost_model`, keep the first candidate" - /// pattern `select_and_bind` already implements, since `bind.rs`'s own - /// tests are the one internal caller allowed to reach it directly. - /// An external caller doesn't have this shortcut — it goes through - /// `SketchFamilyStrategy::replacements()` itself (see the module docs). - fn bind_first( - expr: &QueryExpr, - cost_model: &dyn CostModel, - ) -> Result, ImplementError> { - select_and_bind(&Rc::new(expr.clone()), cost_model) - } - - fn bind(expr: &QueryExpr) -> Result, ImplementError> { - bind_first(expr, &DefaultCostModel) - } - - #[test] - fn quantile_binds_kll_wrapped_in_estimate() { - // quantile by (job) (m) at ε=0.01 → Estimate(Quantile) over - // SummaryAgg(Kll{k:200}) over KeepPreAsap(Scan). job = col 2. - let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); - let root = bind(&q).unwrap(); - - let SummaryExpr::SummaryEstimate { - summary_input, - query, - } = &root.expr - else { - panic!("expected SummaryEstimate root, got {:?}", root.expr); - }; - assert!(matches!(query, SketchQuery::Quantile { q } if *q == 0.99)); - // Estimate edge: plain row shape — group key + Float64 answer. - assert_eq!( - field(&root.schema, "quantile_0_99").dtype, - SummaryFamilyType::Plain(DataType::Float64) - ); - assert_eq!( - field(&root.schema, "job").dtype, - SummaryFamilyType::Plain(DataType::Utf8) - ); - - let SummaryExpr::SummaryAgg { - child, - family, - col, - reduction, - } = &summary_input.expr - else { - panic!("expected SummaryAgg, got {:?}", summary_input.expr); - }; - assert_eq!( - family, - &SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) - ); - assert_eq!(col, &ColumnRef::SampleValue); - assert_eq!(reduction, &Reduction::by(vec![2])); - // SummaryAgg edge: the state column carries the committed family. - assert_eq!( - field(&summary_input.schema, "quantile_0_99").dtype, - SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) - ); - assert!(matches!(child.expr, SummaryExpr::KeepPreAsap(ref e) - if matches!(**e, QueryExpr::Scan { .. }))); - } - - /// A deployment-supplied [`CostModel`] can override the default KLL - /// choice — `bind_first` (via `select_and_bind`) must actually consult it, - /// not just accept and ignore it (issue: cost model interface, see - /// `crate::cost_model`). - struct PreferDDSketch; - - impl CostModel for PreferDDSketch { - fn rank_candidates( - &self, - _intent: &AggIntent, - candidates: &[SketchKind], - ) -> Vec { - let mut v = candidates.to_vec(); - if let Some(pos) = v.iter().position(|k| *k == SketchKind::DDSketch) { - let ddsketch = v.remove(pos); - v.insert(0, ddsketch); - } - v - } - } - - #[test] - fn bind_with_custom_cost_model_overrides_default_summary_choice() { - let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); - - // Default: KLL (see `quantile_binds_kll_wrapped_in_estimate` above). - let default_root = bind(&q).unwrap(); - let SummaryExpr::SummaryEstimate { summary_input, .. } = &default_root.expr else { - panic!("expected SummaryEstimate root, got {:?}", default_root.expr); - }; - let SummaryExpr::SummaryAgg { family, .. } = &summary_input.expr else { - panic!("expected SummaryAgg, got {:?}", summary_input.expr); - }; - assert!(matches!( - family, - SummaryFamilyType::Sketch(SketchKind::Kll, _) - )); - - // With `PreferDDSketch`: DDSketch instead, same query. - let custom_root = bind_first(&q, &PreferDDSketch).unwrap(); - let SummaryExpr::SummaryEstimate { summary_input, .. } = &custom_root.expr else { - panic!("expected SummaryEstimate root, got {:?}", custom_root.expr); - }; - let SummaryExpr::SummaryAgg { family, .. } = &summary_input.expr else { - panic!("expected SummaryAgg, got {:?}", summary_input.expr); - }; - assert_eq!( - family, - &SummaryFamilyType::Sketch( - SketchKind::DDSketch, - SketchParams::DDSketch { alpha: 0.01 } - ) - ); - } - - /// A deployment-supplied `CostModel` can realize an `AggIntent::Extension` - /// intent as a real sketch instead of the default `PassThrough` (issue - /// #150) — `implementations_for_with` must consult `realize_extension` - /// for the `Extension` arm, and `readout` must consult - /// `readout_extension` to build its `SketchQuery` without panicking. - struct FrequencyCostModel; - - impl CostModel for FrequencyCostModel { - fn rank_candidates( - &self, - _intent: &AggIntent, - candidates: &[SketchKind], - ) -> Vec { - candidates.to_vec() - } - - fn realize_extension( - &self, - ext_kind: &str, - _payload: &serde_json::Value, - ) -> crate::replacement::Implementation { - if ext_kind == "frequency" { - crate::replacement::Implementation::Sketch { - kind: SketchKind::CountSketch, - params: SketchParams::CountSketch { - width: 256, - depth: 4, - }, - } - } else { - crate::replacement::Implementation::PassThrough - } - } - - fn readout_extension( - &self, - ext_kind: &str, - payload: &serde_json::Value, - _col: &ColumnRef, - ) -> SketchQuery { - assert_eq!(ext_kind, "frequency"); - let value = payload["item"].as_str().map(str::to_string); - SketchQuery::PointCount { - key: ColumnRef::Named("item".into()), - value, - } - } - } - - #[test] - fn extension_intent_stays_logical_by_default() { - // Without a CostModel overriding `realize_extension`, an - // `Extension` intent must stay `PassThrough` -- today's behavior, - // unchanged. - let intent = AggIntent::Extension { - ext_kind: "frequency".to_string(), - payload: serde_json::json!({ "item": "checkout" }), - }; - let q = agg(vec![], intent, metric_scan(&[])); - let root = bind(&q).unwrap(); - assert!(matches!(root.expr, SummaryExpr::KeepPreAsap(_))); - } - - #[test] - fn extension_intent_binds_via_custom_cost_model() { - let intent = AggIntent::Extension { - ext_kind: "frequency".to_string(), - payload: serde_json::json!({ "item": "checkout" }), - }; - let q = agg(vec![], intent, metric_scan(&[])); - let root = bind_first(&q, &FrequencyCostModel).unwrap(); - - let SummaryExpr::SummaryEstimate { - summary_input, - query, - } = &root.expr - else { - panic!("expected SummaryEstimate root, got {:?}", root.expr); - }; - assert!(matches!( - query, - SketchQuery::PointCount { key: ColumnRef::Named(k), value: Some(v) } - if k == "item" && v == "checkout" - )); - - let SummaryExpr::SummaryAgg { family, .. } = &summary_input.expr else { - panic!("expected SummaryAgg, got {:?}", summary_input.expr); - }; - assert_eq!( - family, - &SummaryFamilyType::Sketch( - SketchKind::CountSketch, - SketchParams::CountSketch { - width: 256, - depth: 4 - } - ) - ); - } - - #[test] - fn exact_sum_binds_accumulator_without_estimate() { - let q = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let root = bind(&q).unwrap(); - let SummaryExpr::SummaryAgg { family, .. } = &root.expr else { - panic!( - "expected bare SummaryAgg (no estimate), got {:?}", - root.expr - ); - }; - assert_eq!( - family, - &SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - ); - assert_eq!( - field(&root.schema, "sum").dtype, - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - ); - } - - #[test] - fn per_series_rate_keeps_labels_and_retypes_value() { - // rate(m[5m]) — per-series: every label survives; the sample value - // column becomes the Rate accumulator state. - let q = agg_per_entity( - AggIntent::Rate, - QueryExpr::TimeRange { - range: Duration::from_secs(300), - child: Rc::new(metric_scan(&["job"])), - }, - ); - let root = bind(&q).unwrap(); - let SummaryExpr::SummaryAgg { family, .. } = &root.expr else { - panic!("expected SummaryAgg, got {:?}", root.expr); - }; - assert_eq!( - family, - &SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) - ); - assert_eq!( - root.schema - .fields - .iter() - .map(|f| f.name.as_str()) - .collect::>(), - vec!["ts", "value", "job"], - ); - assert_eq!( - field(&root.schema, "value").dtype, - SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) - ); - assert_eq!(root.schema.time_index, Some(0)); - } - - /// Issue #163, case 1: a bare per-series range function (e.g. - /// `quantile_over_time(...)`) binds to `SummaryAgg { reduction: - /// PerEntity, .. }` — proving the pre-ASAP `Reduction` this crate - /// already computes (issue #165) is carried onto the post-ASAP node - /// verbatim, not flattened back into an ambiguous bare `Vec`. - #[test] - fn bare_per_series_aggregate_binds_summary_agg_with_per_entity_reduction() { - let q = agg_per_entity( - default_quantile(0.99), - QueryExpr::TimeRange { - range: Duration::from_secs(10), - child: Rc::new(metric_scan(&["job"])), - }, - ); - let root = bind(&q).unwrap(); - let SummaryExpr::SummaryEstimate { summary_input, .. } = &root.expr else { - panic!("expected estimate root, got {:?}", root.expr); - }; - let SummaryExpr::SummaryAgg { reduction, .. } = &summary_input.expr else { - panic!("expected SummaryAgg, got {:?}", summary_input.expr); - }; - assert_eq!(reduction, &Reduction::PerEntity); - } - - /// Issue #163, case 2: an aggregation operator explicitly invoked with - /// no `by(...)` (e.g. `count(hll_metric)`) binds to `SummaryAgg { - /// reduction: Reduce(vec![]), .. }` — byte-identical `by: []` to the - /// previous test at the old `Vec` shape; `reduction` is what - /// tells them apart now. - #[test] - fn explicit_empty_by_aggregate_binds_summary_agg_with_reduce_reduction() { - let intent = AggIntent::Cardinality { - col: None, - accuracy: AccuracyTarget::Epsilon(0.01), - }; - let q = agg(vec![], intent, metric_scan(&["job"])); - let root = bind(&q).unwrap(); - let SummaryExpr::SummaryEstimate { summary_input, .. } = &root.expr else { - panic!("expected estimate root, got {:?}", root.expr); - }; - let SummaryExpr::SummaryAgg { reduction, .. } = &summary_input.expr else { - panic!("expected SummaryAgg, got {:?}", summary_input.expr); - }; - assert_eq!(reduction, &Reduction::by(vec![])); - } - - #[test] - fn nested_aggregates_bind_per_node() { - // quantile(0.9, sum by (job) (m)) — the implementation decision - // fires per node over the nested tree: KLL over an exact Sum - // accumulator. - let inner = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let outer = agg(vec![], default_quantile(0.9), inner); - let root = bind(&outer).unwrap(); - - let SummaryExpr::SummaryEstimate { summary_input, .. } = &root.expr else { - panic!("expected estimate root, got {:?}", root.expr); - }; - let SummaryExpr::SummaryAgg { child, family, .. } = &summary_input.expr else { - panic!("expected outer SummaryAgg, got {:?}", summary_input.expr); - }; - assert!(matches!( - family, - SummaryFamilyType::Sketch(SketchKind::Kll, _) - )); - let SummaryExpr::SummaryAgg { - family: inner_family, - child: leaf, - .. - } = &child.expr - else { - panic!("expected inner SummaryAgg, got {:?}", child.expr); - }; - assert_eq!( - inner_family, - &SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - ); - assert!(matches!(leaf.expr, SummaryExpr::KeepPreAsap(_))); - } - - /// Issue #115: the summary is built over the intent's own input column. - /// Before `Cardinality`/`Quantile` carried `col`, `summarised_column` always - /// fell through to `ColumnRef::SampleValue`, so an HLL was built over the - /// wrong column for every SQL `COUNT(DISTINCT c)`. - #[test] - fn sketch_binds_the_intents_input_column() { - // `metric_scan(&["job"])` → columns [ts=0, value=1, job=2]. - let cases = [ - (Some(2), ColumnRef::Named("job".into())), - (Some(1), ColumnRef::Named("value".into())), - // PromQL convention: no column ⇒ the synthetic sample value. - (None, ColumnRef::SampleValue), - ]; - for (col, want) in cases { - let intent = AggIntent::Cardinality { - col, - accuracy: AccuracyTarget::Epsilon(0.01), - }; - let root = bind(&agg(vec![0], intent, metric_scan(&["job"]))).unwrap(); - let bound = find_summary_col(&root) - .unwrap_or_else(|| panic!("expected a SummaryAgg for col={col:?}")); - assert_eq!(bound, want, "wrong summarised column for col={col:?}"); - } - } - - /// The `col` of the first `SummaryAgg` in the tree. - fn find_summary_col(node: &SummaryNode) -> Option { - match &node.expr { - SummaryExpr::SummaryAgg { col, .. } => Some(col.clone()), - SummaryExpr::SummaryEstimate { summary_input, .. } => find_summary_col(summary_input), - _ => None, - } - } - - #[test] - fn pass_through_intents_stay_logical() { - // avg is exact but non-mergeable; histogram_quantile (classic - // buckets, #79) is never sketchable; exact quantile is exact by - // decree. All three stay whole logical subtrees. - for intent in [ - AggIntent::Avg { col: None }, - AggIntent::HistogramQuantile { q: 0.99 }, - AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::Exact, - }, - ] { - let q = agg(vec![2], intent.clone(), metric_scan(&["job"])); - let root = bind(&q).unwrap(); - assert!( - matches!(root.expr, SummaryExpr::KeepPreAsap(ref e) if **e == q), - "expected KeepPreAsap passthrough for {intent:?}" - ); - } - } - - #[test] - fn logical_parent_subsumes_bindable_child() { - // Filter over a bindable quantile: `KeepPreAsap` has no post-ASAP - // children, so the conservative fallback keeps the whole subtree - // logical. - let q = QueryExpr::Filter { - pred: Predicate(Rc::new(QueryExpr::Compare { - left: Rc::new(QueryExpr::Column(0)), - op: CompareOpKind::Gt, - right: Rc::new(QueryExpr::Literal(ScalarValue::Float64(0.5))), - })), - child: Rc::new(agg(vec![], default_quantile(0.99), metric_scan(&[]))), - }; - let root = bind(&q).unwrap(); - assert!(matches!(root.expr, SummaryExpr::KeepPreAsap(ref e) if **e == q)); - } - - #[test] - fn having_and_multi_intent_stay_logical() { - let mut q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); - if let QueryExpr::Aggregate { having, .. } = &mut q { - *having = Some(Predicate(Rc::new(QueryExpr::Literal( - ScalarValue::Boolean(true), - )))); - } - assert!(matches!( - bind(&q).unwrap().expr, - SummaryExpr::KeepPreAsap(_) - )); - - let multi = QueryExpr::Aggregate { - reduction: Reduction::by(vec![2]), - measures: vec![AggIntent::Sum { col: None }, AggIntent::Avg { col: None }], - output_names: vec![], - having: None, - child: Rc::new(metric_scan(&["job"])), - }; - assert!(matches!( - bind(&multi).unwrap().expr, - SummaryExpr::KeepPreAsap(_) - )); - } - - #[test] - fn topk_binds_cms_with_heap_and_topk_readout() { - let q = agg( - vec![2], - AggIntent::TopK { - k: 5, - accuracy: AccuracyTarget::Epsilon(0.01), - }, - metric_scan(&["job"]), - ); - let root = bind(&q).unwrap(); - let SummaryExpr::SummaryEstimate { - summary_input, - query, - } = &root.expr - else { - panic!("expected estimate root, got {:?}", root.expr); - }; - assert!(matches!(query, SketchQuery::TopK { k: 5 })); - assert!(matches!( - &summary_input.expr, - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::Sketch(SketchKind::CmsWithHeap, _), - .. - } - )); - } - - #[test] - fn sql_reducer_resolves_named_input_column() { - // SUM(bytes) over a tabular scan: `col` resolves positionally to the - // named column, not the PromQL sample value. - let scan = QueryExpr::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - predicates: vec![], - schema: Schema { - columns: vec![ - Column::new("host", DataType::Utf8, false), - Column::new("bytes", DataType::Int64, false), - ], - time_index: None, - unique_keys: vec![], - closed: true, - }, - }; - let q = agg(vec![0], AggIntent::Sum { col: Some(1) }, scan); - let root = bind(&q).unwrap(); - let SummaryExpr::SummaryAgg { col, .. } = &root.expr else { - panic!("expected SummaryAgg, got {:?}", root.expr); - }; - assert_eq!(col, &ColumnRef::Named("bytes".into())); - } - - /// Issue #237, #223 stage 4: a `CostModel` that declines CSE sharing - /// makes `implement_workload_with` bind each occurrence independently, - /// even though the two roots are the exact same `Rc` (as - /// `share_common_subtrees` would hand back for two identical workload - /// entries) — the opposite of `DefaultCostModel`'s unconditional-share - /// behavior pinned by `crates/integration-tests/tests/cse.rs`. - #[test] - fn implement_workload_with_recomputes_independently_when_cost_model_declines_sharing() { - struct NeverShareCse; - impl CostModel for NeverShareCse { - fn rank_candidates( - &self, - _intent: &AggIntent, - candidates: &[SketchKind], - ) -> Vec { - candidates.to_vec() - } - fn cse_share_decision(&self, _candidate: &CseCandidate) -> ShareDecision { - ShareDecision::RecomputeIndependently - } - } - - let shared = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); - let bound = implement_workload_with( - vec![("a", Rc::clone(&shared)), ("b", Rc::clone(&shared))], - &NeverShareCse, - ); - let [(_, ra), (_, rb)] = bound.as_slice() else { - panic!("expected 2 bound results"); - }; - let ra = ra.as_ref().expect("a failed to bind"); - let rb = rb.as_ref().expect("b failed to bind"); - assert!( - !Rc::ptr_eq(ra, rb), - "a CostModel that declines CSE sharing must bind each occurrence \ - independently, even for two roots that are the same Rc" - ); - } -} diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index d7a6f4a3..01651ea5 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -27,7 +27,7 @@ //! //! Every entry point that doesn't take an explicit `&dyn CostModel` //! ([`SketchFamilyStrategy::default_cost_model`](crate::replacement::SketchFamilyStrategy::default_cost_model), -//! [`implement_workload`](crate::bind::implement_workload)) runs against +//! [`search_workload`](crate::replacement::search_workload)) runs against //! [`DefaultCostModel`], so a deployment that never plugs in its own cost //! model keeps today's static-preference-order behavior exactly, byte for //! byte. @@ -41,8 +41,10 @@ //! cost comparison rather than a fixed rule. See //! `docs/design_docs/cse-cost-model-decision.md` for the full design discussion (why //! cost-based, why not a full plan-search engine, the layering constraint -//! that forces detection to stay cost-agnostic). [`bind::implement_workload_with`](crate::bind::implement_workload_with) -//! is the caller. +//! that forces detection to stay cost-agnostic). +//! [`PlanSpace::cost_sorted`](crate::replacement::PlanSpace::cost_sorted) +//! (via [`crate::replacement`]'s own `cse_preference`) and +//! [`DefaultCostModel::estimate_cost`] are this crate's own callers. use std::rc::Rc; @@ -54,13 +56,14 @@ use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::query_expr::QueryExpr; use crate::replacement::{ - select_and_bind, Implementation, Replacement, ReplacementSubDAG, TargetSubDAG, + realize_child, Implementation, Replacement, ReplacementSubDAG, TargetSubDAG, }; /// A CSE-detected, legality-gated shared subtree with two or more consumers /// — the unit [`CostModel::cse_share_decision`] decides over. Built by -/// [`bind::implement_workload_with`](crate::bind::implement_workload_with) -/// the first time it binds a subtree that +/// [`PlanSpace::cost_sorted`](crate::replacement::PlanSpace::cost_sorted) +/// (via [`crate::replacement`]'s own `cse_preference`) the first time it +/// needs a representative bound node for a subtree that /// [`asap_types::pre_asap::cse::share_common_subtrees`] already collapsed /// onto one `Rc` for two or more workload roots. See /// `docs/design_docs/cse-cost-model-decision.md`. @@ -230,7 +233,7 @@ pub trait CostModel { /// same `CostModel` realized as `Implementation::Sketch` via /// [`realize_extension`](Self::realize_extension). Only ever called /// when `realize_extension` returned `Sketch` for the same - /// `(ext_kind, payload)` — `bind::readout` has no other way to build a + /// `(ext_kind, payload)` — `replacement::readout` has no other way to build a /// `SketchQuery` for a shape core doesn't know. A deployment that /// overrides `realize_extension` to return `Sketch` for some /// `ext_kind` MUST also override this for that same `ext_kind`, or @@ -365,7 +368,7 @@ impl CostModel for DefaultCostModel { /// [`default_cse_shared_maintenance_cost`] already orders candidates /// by). /// - [`Replacement::Rewrite`]: recovers one representative bound - /// `SummaryNode` for `target` via `select_and_bind` (the same + /// `SummaryNode` for `target` via `realize_child` (the same /// rank-and-take-first helper `replacement::bind_one` reuses for the /// identical need), then charges /// `cse_shared_maintenance_cost` for the candidate that shares @@ -387,7 +390,7 @@ impl CostModel for DefaultCostModel { (self.cse_recompute_cost(&cse) + self.cse_shared_maintenance_cost(&cse)).0 } Replacement::Rewrite(rc) => { - let Ok(bound) = select_and_bind(target.root, self) else { + let Ok(bound) = realize_child(target.root, self) else { return f64::NAN; }; let cse = CseCandidate { diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 9b873410..1b02cb1f 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -29,56 +29,54 @@ //! //! ## Status //! -//! - [`replacement`] — the `TargetSubDAG`/`ReplacementSubDAG`/ +//! This crate has two replacement/search capabilities — and deliberately no +//! third one that commits to a single, final, physically-materialized +//! answer for a whole workload: +//! +//! - [`replacement::ReplacementStrategy`] — per-target, never prunes, +//! exhaustive. The `TargetSubDAG`/`ReplacementSubDAG`/ //! `ReplacementStrategy` vocabulary `docs/design_docs/asap_aware_mapping.md` stubs out //! under "Key concepts (not yet implemented)", implemented for real (issue -//! #251, part of #33) — plus, merged into the same module (issue #252, -//! part of #33), the workload-level search engine that decides *and* -//! constructs *and* searches across a whole workload's worth of targets, -//! all as one module's job. One module, three connected steps: -//! 1. [`replacement::SketchFamilyStrategy::replacements`] both *decides* -//! what an `AggIntent` may become ([`replacement::implementations_for_with`], -//! exhaustive and ranked via a `CostModel`, sized to the -//! `AccuracyTarget`) and *constructs* each candidate's bound -//! [`SummaryNode`](asap_types::post_asap::SummaryNode) — every -//! candidate comes back, not just one. -//! [`replacement::SharedSubtreeStrategy`] does the analogous job for -//! the build-independently-vs-build-once-and-share choice at a -//! CSE-detected shared subtree. -//! 2. [`replacement::search_workload`]/[`replacement::search_workload_with`] -//! *search* — discover every candidate `TargetSubDAG` across a whole -//! workload (not just one target in isolation) and run both strategies -//! above against each one, to a fixpoint, without ever materializing a -//! flat `2^N`-sized candidate-plan list: [`replacement::PlanSpace`] holds -//! one Cascades-style [`replacement::MemoGroup`] per distinct target, -//! each carrying every alternative discovered for it. -//! 3. [`replacement::PlanSpace::cost_sorted`] is the final -//! `sorted_by(cost_model)` step, ranking each group's candidates -//! best-first via the same [`CostModel`](cost_model::CostModel) the -//! single-target steps above already consult — see -//! [`replacement`]'s own module docs for the full design (MEMO groups -//! vs. flat plans, dedup discipline, termination, cost-based ranking). -//! - [`bind`] — has no "bind me one tree" entry point for a single target. -//! [`replacement::SketchFamilyStrategy`] is the only public way to get -//! bound output for a target, and it always returns *every* candidate; a -//! caller that wants one answer takes the first entry itself. What `bind` -//! provides is workload-wide orchestration: -//! [`bind::implement_workload`]/[`bind::implement_workload_with`], which -//! drive rank-and-take-first selection over a whole workload's roots, -//! memoized on `Rc` identity so two roots that -//! `asap_types::pre_asap::cse::share_common_subtrees` already collapsed -//! onto one shared subtree bind to one shared `SummaryNode` too (issue -//! #212, #222, #223) — memoization needs one canonical decision per -//! shared root to key sharing on, so that entry point is the one place a -//! single-answer selection still lives. Everything else `bind` used to -//! hold ([`replacement::select_and_bind`], the single-target rank-and- -//! take-first helper, and [`replacement::keep_pre_asap`], the pass-through -//! fallback) moved into [`replacement`] itself — both are single-target- -//! scoped helpers with no real workload-level state, the same category as -//! everything else that module owns; only the genuine cross-root -//! `Rc`-identity memoization state above still needs its own module. See -//! the terminology section below for why this crate's logical→physical -//! step is named "implementation" rather than "bind". +//! #251, part of #33). [`replacement::SketchFamilyStrategy::replacements`] +//! both *decides* what an `AggIntent` may become +//! ([`replacement::implementations_for_with`], exhaustive and ranked via a +//! `CostModel`, sized to the `AccuracyTarget`) and *constructs* each +//! candidate's bound [`SummaryNode`](asap_types::post_asap::SummaryNode) — +//! every candidate comes back, not just one. +//! [`replacement::SharedSubtreeStrategy`] does the analogous job for the +//! build-independently-vs-build-once-and-share choice at a CSE-detected +//! shared subtree. +//! - [`replacement::search_workload`]/[`replacement::PlanSpace::cost_sorted`] +//! — workload-wide, every candidate + cost, never materializes one +//! physical answer. Merged into the same module (issue #252, part of +//! #33): [`replacement::search_workload`]/[`replacement::search_workload_with`] +//! *search* — discover every candidate `TargetSubDAG` across a whole +//! workload (not just one target in isolation) and run every registered +//! strategy against each one, to a fixpoint, without ever materializing a +//! flat `2^N`-sized candidate-plan list: [`replacement::PlanSpace`] holds +//! one Cascades-style [`replacement::MemoGroup`] per distinct +//! `TargetSubDAG`, each carrying every alternative discovered for it. +//! [`replacement::PlanSpace::cost_sorted`] is the final +//! `sorted_by(cost_model)` step, ranking each group's candidates +//! best-first via the same [`CostModel`](cost_model::CostModel) the +//! single-target steps above already consult — see [`replacement`]'s own +//! module docs for the full design (MEMO groups vs. flat plans, dedup +//! discipline, termination, cost-based ranking). +//! +//! **Picking *which* candidate, and materializing one final answer, is a +//! downstream deployment's job, out of this crate's scope.** This crate's +//! output boundary is [`replacement::PlanSpace`]: every candidate +//! replacement plus its cost, meant for a downstream consumer (e.g. a +//! DAG-visualization view, or a deployment's own physical binder). Which +//! sketch to commit to *and* where to place it are a joint decision only a +//! deployment can see the full picture for — picking one in isolation, with +//! no real consumer of that single materialized answer inside this crate, +//! is out of scope. (A prior workload-wide "keep first/cost-preferred +//! candidate per node, memoized by `Rc` identity" entry point — +//! `bind::implement_workload`/`implement_workload_with` — used to live here +//! and was removed for exactly this reason; see the terminology table below +//! for where that "one answer" step now belongs, downstream.) +//! //! - [`cost_model`] — the [`CostModel`](cost_model::CostModel) trait every //! deployment's cost-based sketch selection plugs into (issues #6, #33). //! `asap-plan` itself only ships [`DefaultCostModel`](cost_model::DefaultCostModel), @@ -94,7 +92,7 @@ //! `asap-plan` and its downstream consumers (e.g. `ASAPQuery-backend`'s //! `control_plane`) independently reused the word "bind" for three //! *different*, layer-specific meanings — none of which is what this -//! crate's [`replacement`]/[`bind`] modules do. To avoid becoming a +//! crate's own [`replacement`] module does. To avoid becoming a //! fourth, colliding sense of the same word, this crate names its own //! logical intent → physical realization step after the term the //! query-optimization literature already uses for exactly that step: @@ -109,8 +107,7 @@ //! | **Implementation** — `replacement::implementations_for_with` | pre-ASAP → post-ASAP, *one node* | enumerating every concrete physical realization (a sketch family, an exact accumulator, or pass-through) for one [`AggIntent`](asap_types::pre_asap::agg_intent::AggIntent) | [`replacement`] | //! | **Replacement** — [`replacement::SketchFamilyStrategy::replacements`] | pre-ASAP → post-ASAP, *one target, every candidate* | wrap each `implementations_for_with` candidate into its own bound [`SummaryNode`](asap_types::post_asap::SummaryNode), ranked — a caller wanting one answer takes the first entry itself | [`replacement`] | //! | **Search** — [`replacement::search_workload`]/[`replacement::search_workload_with`] | pre-ASAP → post-ASAP, *whole workload, every candidate* | a Cascades/Volcano-style MEMO search: discover every candidate `TargetSubDAG` across a whole workload (not just one target in isolation), run every registered `ReplacementStrategy` against each to a fixpoint, and dedup into a [`replacement::PlanSpace`] — one [`replacement::MemoGroup`] per distinct `TargetSubDAG` holding every alternative discovered for it, never a flat `2^N`-sized list of whole candidate plans | [`replacement`] | -//! | **`implement_workload`** — [`bind::implement_workload`] | pre-ASAP → post-ASAP, *whole workload, one answer* | walk every root of a `QueryWorkload`, keeping the first (`cost_model`-preferred) candidate per node, sharing one bound `SummaryNode` across roots CSE already collapsed onto one `Rc` — emits the complete post-ASAP `SummaryExpr`/`SummaryNode` DAG | [`bind`] | -//! | **Bind #2** (downstream, not in this crate) | post-ASAP → deployment placement | a *deployment's* own physical binder, additionally deciding **placement** (edge vs. backend, wire format, …) — a genuinely different, deployment-specific decision this crate doesn't model at all | e.g. `control_plane::sketch_algebra::rules::bind_*` (as of this writing; expected to fold into that deployment's cost-model layer rather than stay a separate "bind" concept) | +//! | **Bind #2** (downstream, not in this crate) | post-ASAP → deployment placement | a *deployment's* own physical binder, deciding **which** candidate to commit to *and* **placement** (edge vs. backend, wire format, …) for a whole workload — a genuinely different, deployment-specific decision this crate doesn't model at all (this is also where a prior workload-wide "keep first/cost-preferred candidate per node" step, `bind::implement_workload`/`implement_workload_with`, would belong if a deployment still wants that exact behavior — it isn't shipped by this crate) | e.g. `control_plane::sketch_algebra::rules::bind_*` (as of this writing; expected to fold into that deployment's cost-model layer rather than stay a separate "bind" concept) | //! //! A related question (tracked alongside issues #6/#33): whether this //! crate should also own a **matching** predicate — "does an already @@ -129,15 +126,13 @@ //! `sketch_algebra::capability::Capability`/`is_satisfied_by` is the //! reference downstream implementation. -pub mod bind; pub mod cost_model; pub mod replacement; -pub use bind::{implement_workload, implement_workload_with, ImplementError}; pub use cost_model::{CostModel, DefaultCostModel}; pub use replacement::{ default_strategies, default_strategies_with, search_workload, search_workload_with, - summary_candidates, Implementation, Matcher, MemoGroup, PlanSpace, RankedGroup, Replacement, - ReplacementStrategy, ReplacementSubDAG, SharedSubtreeStrategy, SketchFamilyStrategy, - TargetSubDAG, MAX_SEARCH_ITERATIONS, + summary_candidates, ImplementError, Implementation, Matcher, MemoGroup, PlanSpace, RankedGroup, + Replacement, ReplacementStrategy, ReplacementSubDAG, SharedSubtreeStrategy, + SketchFamilyStrategy, TargetSubDAG, MAX_SEARCH_ITERATIONS, }; diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index 06fe9314..2b9bd893 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -20,7 +20,7 @@ //! mechanically turns the already-decided `(kind, params)` into a real //! [`SummaryNode`] — derives the child schema, resolves the summarized //! column, builds the readout query, recurses into the child (via -//! [`select_and_bind`], so a nested aggregate gets its own +//! [`realize_child`], so a nested aggregate gets its own //! independent enumeration, never the outer target's forced choice), and //! assembles the `SummaryAgg`/`SummaryEstimate` node. //! @@ -52,12 +52,17 @@ //! A caller that wants one executable answer takes the first //! (`cost_model`-preferred) entry off `replacements()` itself //! (`.into_iter().next()`) — that "keep the head" step lives entirely on the -//! calling side, not behind a second module-level entry point. `bind.rs`'s -//! own [`crate::bind::implement_workload`]/[`crate::bind::implement_workload_with`] -//! are the one place inside this crate that still perform that take-first -//! step internally, because workload-wide CSE memoization needs one -//! canonical decision per shared root to key sharing on (see `bind.rs`'s own -//! module docs). Every other caller goes through +//! calling side, not behind a second module-level entry point. This +//! module's own [`realize_child`] performs that take-first step +//! internally, but only for one, narrow, single-target purpose: recursing +//! into a node's child while constructing one concrete candidate (see +//! [`construct_summary_agg`]) and, symmetrically, recovering one +//! representative bound node for a [`CostModel::cse_share_decision`] +//! comparison (see [`realize_one`]) — never a workload-wide "commit to one +//! final answer" step. Committing to one physically-materialized answer for +//! a whole workload (previously `bind::implement_workload`/ +//! `implement_workload_with`) is out of this crate's scope — see the crate +//! doc's `## Status` section for why. Every other caller goes through //! `SketchFamilyStrategy::replacements` directly and decides for itself. //! //! This means an ordinary single-target bind sizes and fully constructs @@ -176,14 +181,10 @@ //! that module's "Algorithm" section), discovering one `TargetSubDAG` per //! distinct `Rc` and a *real* `consumer_count`: how many operator-child //! positions anywhere in the workload reference that exact `Rc`, not just -//! how many of the workload's own top-level roots happen to be it. This -//! deliberately goes one step further than -//! [`crate::bind::implement_workload_with`]'s own consumer-count pass, -//! which only counts whole-root sharing (that function's own doc calls -//! widening this "future work" for binding) — a `SharedSubtreeStrategy` -//! candidate three levels under an unshared `Filter` is exactly as real a -//! target as a shared whole root, so this module's discovery can't stop at -//! the top level the way binding's does. +//! how many of the workload's own top-level roots happen to be it — a +//! `SharedSubtreeStrategy` candidate three levels under an unshared +//! `Filter` is exactly as real a target as a shared whole root, so this +//! module's discovery can't stop at the top level. //! //! `discover_targets` duplicates (rather than reuses) this module's own //! `#[cfg(test)]`-only `count_consumers` traversal (in the test module @@ -247,10 +248,8 @@ //! //! - A group whose candidates are the [`SharedSubtreeStrategy`] //! share-vs-recompute pair is ranked by calling -//! [`CostModel::cse_share_decision`] — the exact comparison -//! [`crate::bind::implement_workload_with`] already uses for this same -//! decision — rather than re-deriving a competing comparison in this -//! module. +//! [`CostModel::cse_share_decision`] via this module's own +//! [`cse_preference`] — rather than re-deriving a competing comparison. //! - A group whose candidates are [`SketchFamilyStrategy`]'s sketch-family //! candidates is ranked via [`CostModel::rank_candidates`] (the same hook //! `implementations_for_with` itself consults), applied to the @@ -270,18 +269,31 @@ use asap_types::post_asap::{ use asap_types::pre_asap::agg_intent::{agg_is_mergeable, AggIntent}; use asap_types::pre_asap::cse::{share_common_subtrees, structural_hash, HashCache}; use asap_types::pre_asap::expr_ir::ColumnRef; -use asap_types::pre_asap::query_expr::{QueryExpr, Reduction}; +use asap_types::pre_asap::query_expr::{QueryExpr, QueryExprError, Reduction}; use asap_types::pre_asap::schema::Schema; use asap_types::types::AccuracyTarget; use std::rc::Rc; +use thiserror::Error; -use crate::bind::ImplementError; use crate::cost_model::{CostModel, CseCandidate, DefaultCostModel, ShareDecision}; +/// Errors from the pre-ASAP → post-ASAP replacement/construction path +/// ([`realize_child`] and [`keep_pre_asap`]). Moved here from the former +/// `bind.rs` (issue #251): this is what a [`ReplacementStrategy`] +/// implementor's own construction path can realistically fail with — +/// schema derivation over a pre-ASAP [`QueryExpr`] — not something specific +/// to workload-wide orchestration. +#[derive(Debug, Error)] +pub enum ImplementError { + /// Schema derivation failed while lifting an edge to `SummarySchema`. + #[error("schema derivation failed during pre-ASAP → post-ASAP binding: {0}")] + Schema(#[from] QueryExprError), +} + /// A pre-ASAP sub-DAG a [`ReplacementStrategy`] knows how to replace. /// /// `root` is a reference into the workload's own [`QueryExpr`] tree (an -/// `Rc`, the same currency [`crate::bind::implement_workload`] and +/// `Rc`, the same currency [`search_workload`] and /// `asap_types::pre_asap::cse::share_common_subtrees` already thread through /// this crate's public API — not a bare `&QueryExpr` — so a strategy that /// needs the node's own `Rc` identity, not just its shape, has it available @@ -1031,23 +1043,29 @@ fn describe_intent(intent: &AggIntent) -> String { } } -// ── select_and_bind / keep_pre_asap: rank-and-take-first, and its fallback ── - -/// Rank-and-take-first selector, scoped to -/// [`crate::bind::implement_workload_with`]'s CSE memoization and to this -/// module's own recursion into a node's child (via [`construct_summary_agg`]) -/// — **not** a general single-answer API. `root` must already be the -/// caller's own `Rc`, never fabricated per call, so this never allocates -/// beyond what the caller already held. +// ── realize_child / keep_pre_asap: rank-and-take-first, and its fallback ── + +/// Rank-and-take-first selector for a single [`QueryExpr`] node: enumerate +/// every candidate via [`SketchFamilyStrategy::replacements`], keep the +/// `cost_model`-preferred (first) one, and fall back to [`keep_pre_asap`] +/// when there's no candidate at all — **not** a general single-answer API +/// for a whole workload (that "commit to one final answer" step is a +/// downstream deployment's job, out of this crate's scope — see the crate +/// doc's `## Status` section). `root` must already be the caller's own +/// `Rc`, never fabricated per call, so this never allocates beyond what the +/// caller already held. /// -/// `pub(crate)`: reachable both from this module's own construction helper -/// (so a nested aggregate gets its own independent enumeration instead of -/// inheriting the parent's forced candidate) and from `bind.rs`'s -/// [`crate::bind::implement_workload_with`] (workload-wide CSE memoization -/// needs one canonical decision per shared root to key sharing on — see -/// that function's own module docs). Every other caller goes through +/// `pub(crate)`: reachable from this module's own construction helper +/// ([`construct_summary_agg`], so a nested aggregate gets its own +/// independent enumeration instead of inheriting the parent's forced +/// candidate), from this module's own [`realize_one`] (the representative +/// bound `SummaryNode` [`cse_preference`] needs for a +/// [`CostModel::cse_share_decision`] comparison), and from +/// [`crate::cost_model::DefaultCostModel::estimate_cost`] (the same +/// representative-node need, for a [`Replacement::Rewrite`] candidate's own +/// cost estimate). Every other caller goes through /// [`SketchFamilyStrategy::replacements`] directly and decides for itself. -pub(crate) fn select_and_bind( +pub(crate) fn realize_child( root: &Rc, cost_model: &dyn CostModel, ) -> Result, ImplementError> { @@ -1076,9 +1094,7 @@ pub(crate) fn select_and_bind( } /// Wrap an unrewritten pre-ASAP subtree, lifting its schema with every column -/// `SummaryFamilyType::Plain`. `pub` — re-exported as -/// [`crate::bind::keep_pre_asap`] for existing external callers that reach -/// it through that module's path — so a caller can fall back to this +/// `SummaryFamilyType::Plain`. `pub` so a caller can fall back to this /// explicitly — e.g. when `SketchFamilyStrategy::replacements()` returns no /// candidate for a target, or a deployment wants to force a node its own /// runtime can't actually implement — through the same fallback this @@ -1096,7 +1112,12 @@ pub fn keep_pre_asap(expr: &QueryExpr) -> Result, ImplementError /// The bindable shape [`SketchFamilyStrategy`] targets: a single intent, no /// `HAVING`. A multi-intent node (SQL `SELECT SUM(a), AVG(b)`), or one with a /// `HAVING` predicate (the filter would need the estimate first), stays -/// logical — see the module docs' "Conservative fallbacks" in `bind.rs`. +/// logical — conservative fallbacks: [`SummaryExpr::KeepPreAsap`] boxes a +/// whole pre-ASAP subtree with no post-ASAP children, so a *logical* +/// operator above a bindable aggregate (`Filter`/`BinaryOp`/… over a +/// quantile) subsumes the aggregate into the logical wrapper unbound too — +/// rewriting through logical parents is the post-ASAP rule engine's job +/// (#6/#33), not this pass's. pub fn bindable_intent(node: &QueryExpr) -> Option<&AggIntent> { if let QueryExpr::Aggregate { measures, having, .. @@ -1117,7 +1138,7 @@ pub fn bindable_intent(node: &QueryExpr) -> Option<&AggIntent> { /// `expr` must still be the [`bindable_intent`] shape for `implementation` to /// have any effect; anything else falls back to [`keep_pre_asap`]. /// Only `expr`'s own top-level decision is forced — recursion into `expr`'s -/// child goes back through [`select_and_bind`] (fresh candidate +/// child goes back through [`realize_child`] (fresh candidate /// enumeration, not a forced pick), so choosing one candidate for a target /// never leaks into that target's own nested aggregates. fn construct_summary( @@ -1209,7 +1230,7 @@ fn construct_summary_agg( // single place that decides this; nothing downstream re-derives it. let agg = Rc::new(SummaryNode { expr: SummaryExpr::SummaryAgg { - child: select_and_bind(child, cost_model)?, + child: realize_child(child, cost_model)?, family, col, reduction: reduction.clone(), @@ -1621,8 +1642,8 @@ fn rank_group<'a>(group: &'a MemoGroup, cost_model: &dyn CostModel) -> Vec<&'a R } // Shape 1: a `SharedSubtreeStrategy` share-vs-recompute pair (every - // candidate is a `Rewrite`) — rank via `CostModel::cse_share_decision`, - // the same comparison `bind::implement_workload_with` already uses. + // candidate is a `Rewrite`) — rank via `CostModel::cse_share_decision` + // (see `cse_preference` below). if ranked .iter() .all(|c| matches!(c.replacement, Replacement::Rewrite(_))) @@ -1680,7 +1701,7 @@ fn cse_preference(group: &MemoGroup, cost_model: &dyn CostModel) -> Option if group.consumer_count < 2 { return None; } - let bound = bind_one(&group.target, cost_model)?; + let bound = realize_one(&group.target, cost_model)?; let candidate = CseCandidate { subtree: &group.target, bound_summary: &bound, @@ -1696,12 +1717,12 @@ fn cse_preference(group: &MemoGroup, cost_model: &dyn CostModel) -> Option /// for `target` (to build a [`CseCandidate`] for /// [`CostModel::cse_share_decision`]), not the full ranked candidate list /// [`SketchFamilyStrategy::replacements`] returns — so this just reuses -/// [`select_and_bind`], the same rank-and-take-first helper +/// [`realize_child`], the same rank-and-take-first helper /// `construct_summary_agg`'s own recursion and -/// `crate::bind::implement_workload_with` already use, wrapped to swallow -/// the (here, uninteresting) error into `None`. -fn bind_one(target: &Rc, cost_model: &dyn CostModel) -> Option> { - select_and_bind(target, cost_model).ok() +/// [`crate::cost_model::DefaultCostModel::estimate_cost`] already use, +/// wrapped to swallow the (here, uninteresting) error into `None`. +fn realize_one(target: &Rc, cost_model: &dyn CostModel) -> Option> { + realize_child(target, cost_model).ok() } /// The `SketchKind` a bound [`Replacement::Summary`] candidate ultimately @@ -1773,11 +1794,11 @@ pub fn search_workload(roots: Vec<(Id, Rc)>) -> PlanSpace { /// /// Runs [`share_common_subtrees`] once over `roots` first — the same /// pattern a hypothetical `applicability::find_applicable_optimizations` -/// uses, so every strategy sees the same already-deduplicated tree -/// [`crate::bind::implement_workload`] would — then discovers every -/// `TargetSubDAG` (see [`discover_targets`]) and runs the fixpoint loop the -/// module docs describe, capped at [`MAX_SEARCH_ITERATIONS`] passes (see the -/// module docs' "Termination" section). Deduping candidate plans this way needs no +/// uses, so every strategy sees the same already-deduplicated tree — then +/// discovers every `TargetSubDAG` (see [`discover_targets`]) and runs the +/// fixpoint loop the module docs describe, capped at +/// [`MAX_SEARCH_ITERATIONS`] passes (see the module docs' "Termination" +/// section). Deduping candidate plans this way needs no /// [`CostModel`] at all — that only enters at two well-defined points: each /// [`ReplacementStrategy`] in `strategies` may already carry its own (e.g. /// [`SketchFamilyStrategy::new`]'s), and [`PlanSpace::cost_sorted`]'s final @@ -2692,7 +2713,7 @@ mod tests { /// node's own decision — a nested aggregate underneath it still gets its /// own independent (`cost_model`-ranked) enumeration, not whatever the /// caller happened to pick for the outer target. This is the behavior - /// [`construct_summary`]'s recursion (via [`select_and_bind`]) + /// [`construct_summary`]'s recursion (via [`realize_child`]) /// gets for free: only the top node's `Implementation` is ever forced /// from outside; the child is always re-enumerated fresh. #[test] @@ -3021,8 +3042,8 @@ mod tests { // A shared grouped Aggregate nested under two *different*, // unshared Filter parents — real consumer_count must come from // walking the whole DAG, not just root-level pointer identity - // (bind::implement_workload_with's own consumer-count pass would - // miss this; this module's discover_targets must not). + // (a naive whole-root-only consumer-count pass would miss this; + // this module's discover_targets must not). use asap_types::pre_asap::expr_ir::ScalarValue; use asap_types::pre_asap::query_expr::Predicate; @@ -3358,4 +3379,544 @@ mod tests { })]; let _ = search_workload_with(vec![("q", root)], &strategies); } + // ── realize_child / keep_pre_asap: end-to-end single-target realization ── + // + // Moved from the former `bind.rs` (issue #251): `bind.rs`'s own + // workload-wide orchestration (`implement_workload`/ + // `implement_workload_with`) was deleted as out of this crate's scope + // (see the crate doc's `## Status` section), but these tests exercise + // `construct_summary_agg`'s schema derivation end to end through + // `realize_child` — production logic that still lives in this module — + // so they move here rather than disappear. Unlike `bind.rs` (an + // external caller that had to reconstruct the rank-and-take-first + // pattern by hand since `realize_child` is `pub(crate)`), these tests + // call `realize_child` directly. + + fn agg_per_entity(intent: AggIntent, child: QueryExpr) -> QueryExpr { + QueryExpr::Aggregate { + reduction: ReductionTy::PerEntity, + measures: vec![intent], + output_names: vec![], + having: None, + child: Rc::new(child), + } + } + + fn field<'a>(schema: &'a SummarySchema, name: &str) -> &'a SummaryField { + schema + .fields + .iter() + .find(|f| f.name == name) + .unwrap_or_else(|| panic!("no field {name:?} in {schema:?}")) + } + + fn realize_first( + expr: &QueryExpr, + cost_model: &dyn CostModel, + ) -> Result, ImplementError> { + realize_child(&Rc::new(expr.clone()), cost_model) + } + + fn realize(expr: &QueryExpr) -> Result, ImplementError> { + realize_first(expr, &DefaultCostModel) + } + + #[test] + fn quantile_realizes_kll_wrapped_in_estimate() { + // quantile by (job) (m) at ε=0.01 → Estimate(Quantile) over + // SummaryAgg(Kll{k:200}) over KeepPreAsap(Scan). job = col 2. + let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + let root = realize(&q).unwrap(); + + let SummaryExpr::SummaryEstimate { + summary_input, + query, + } = &root.expr + else { + panic!("expected SummaryEstimate root, got {:?}", root.expr); + }; + assert!(matches!(query, PostAsapSketchQuery::Quantile { q } if *q == 0.99)); + // Estimate edge: plain row shape — group key + Float64 answer. + assert_eq!( + field(&root.schema, "quantile_0_99").dtype, + SummaryFamilyType::Plain(DataType::Float64) + ); + assert_eq!( + field(&root.schema, "job").dtype, + SummaryFamilyType::Plain(DataType::Utf8) + ); + + let SummaryExpr::SummaryAgg { + child, + family, + col, + reduction, + } = &summary_input.expr + else { + panic!("expected SummaryAgg, got {:?}", summary_input.expr); + }; + assert_eq!( + family, + &SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) + ); + assert_eq!(col, &ColumnRef::SampleValue); + assert_eq!(reduction, &ReductionTy::by(vec![2])); + // SummaryAgg edge: the state column carries the committed family. + assert_eq!( + field(&summary_input.schema, "quantile_0_99").dtype, + SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) + ); + assert!(matches!(child.expr, SummaryExpr::KeepPreAsap(ref e) + if matches!(**e, QueryExpr::Scan { .. }))); + } + + /// A deployment-supplied [`CostModel`] can override the default KLL + /// choice — `realize_first` (via `realize_child`) must actually consult + /// it, not just accept and ignore it (issue: cost model interface, see + /// `crate::cost_model`). + struct PreferDDSketchViaCostModel; + + impl CostModel for PreferDDSketchViaCostModel { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchKind], + ) -> Vec { + let mut v = candidates.to_vec(); + if let Some(pos) = v.iter().position(|k| *k == SketchKind::DDSketch) { + let ddsketch = v.remove(pos); + v.insert(0, ddsketch); + } + v + } + } + + #[test] + fn realize_with_custom_cost_model_overrides_default_summary_choice() { + let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + + // Default: KLL (see `quantile_realizes_kll_wrapped_in_estimate` above). + let default_root = realize(&q).unwrap(); + let SummaryExpr::SummaryEstimate { summary_input, .. } = &default_root.expr else { + panic!("expected SummaryEstimate root, got {:?}", default_root.expr); + }; + let SummaryExpr::SummaryAgg { family, .. } = &summary_input.expr else { + panic!("expected SummaryAgg, got {:?}", summary_input.expr); + }; + assert!(matches!( + family, + SummaryFamilyType::Sketch(SketchKind::Kll, _) + )); + + // With `PreferDDSketchViaCostModel`: DDSketch instead, same query. + let custom_root = realize_first(&q, &PreferDDSketchViaCostModel).unwrap(); + let SummaryExpr::SummaryEstimate { summary_input, .. } = &custom_root.expr else { + panic!("expected SummaryEstimate root, got {:?}", custom_root.expr); + }; + let SummaryExpr::SummaryAgg { family, .. } = &summary_input.expr else { + panic!("expected SummaryAgg, got {:?}", summary_input.expr); + }; + assert_eq!( + family, + &SummaryFamilyType::Sketch( + SketchKind::DDSketch, + SketchParams::DDSketch { alpha: 0.01 } + ) + ); + } + + /// A deployment-supplied `CostModel` can realize an `AggIntent::Extension` + /// intent as a real sketch instead of the default `PassThrough` (issue + /// #150) — `implementations_for_with` must consult `realize_extension` + /// for the `Extension` arm, and `readout` must consult + /// `readout_extension` to build its `SketchQuery` without panicking. + struct FrequencyCostModel; + + impl CostModel for FrequencyCostModel { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchKind], + ) -> Vec { + candidates.to_vec() + } + + fn realize_extension( + &self, + ext_kind: &str, + _payload: &serde_json::Value, + ) -> Implementation { + if ext_kind == "frequency" { + Implementation::Sketch { + kind: SketchKind::CountSketch, + params: SketchParams::CountSketch { + width: 256, + depth: 4, + }, + } + } else { + Implementation::PassThrough + } + } + + fn readout_extension( + &self, + ext_kind: &str, + payload: &serde_json::Value, + _col: &ColumnRef, + ) -> PostAsapSketchQuery { + assert_eq!(ext_kind, "frequency"); + let value = payload["item"].as_str().map(str::to_string); + PostAsapSketchQuery::PointCount { + key: ColumnRef::Named("item".into()), + value, + } + } + } + + #[test] + fn extension_intent_stays_logical_by_default() { + // Without a CostModel overriding `realize_extension`, an + // `Extension` intent must stay `PassThrough` -- today's behavior, + // unchanged. + let intent = AggIntent::Extension { + ext_kind: "frequency".to_string(), + payload: serde_json::json!({ "item": "checkout" }), + }; + let q = agg(vec![], intent, metric_scan(&[])); + let root = realize(&q).unwrap(); + assert!(matches!(root.expr, SummaryExpr::KeepPreAsap(_))); + } + + #[test] + fn extension_intent_realizes_via_custom_cost_model() { + let intent = AggIntent::Extension { + ext_kind: "frequency".to_string(), + payload: serde_json::json!({ "item": "checkout" }), + }; + let q = agg(vec![], intent, metric_scan(&[])); + let root = realize_first(&q, &FrequencyCostModel).unwrap(); + + let SummaryExpr::SummaryEstimate { + summary_input, + query, + } = &root.expr + else { + panic!("expected SummaryEstimate root, got {:?}", root.expr); + }; + assert!(matches!( + query, + PostAsapSketchQuery::PointCount { key: ColumnRef::Named(k), value: Some(v) } + if k == "item" && v == "checkout" + )); + + let SummaryExpr::SummaryAgg { family, .. } = &summary_input.expr else { + panic!("expected SummaryAgg, got {:?}", summary_input.expr); + }; + assert_eq!( + family, + &SummaryFamilyType::Sketch( + SketchKind::CountSketch, + SketchParams::CountSketch { + width: 256, + depth: 4 + } + ) + ); + } + + #[test] + fn exact_sum_realizes_accumulator_without_estimate() { + let q = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let root = realize(&q).unwrap(); + let SummaryExpr::SummaryAgg { family, .. } = &root.expr else { + panic!( + "expected bare SummaryAgg (no estimate), got {:?}", + root.expr + ); + }; + assert_eq!( + family, + &SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) + ); + assert_eq!( + field(&root.schema, "sum").dtype, + SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) + ); + } + + #[test] + fn per_series_rate_keeps_labels_and_retypes_value() { + // rate(m[5m]) — per-series: every label survives; the sample value + // column becomes the Rate accumulator state. + use std::time::Duration; + let q = agg_per_entity( + AggIntent::Rate, + QueryExpr::TimeRange { + range: Duration::from_secs(300), + child: Rc::new(metric_scan(&["job"])), + }, + ); + let root = realize(&q).unwrap(); + let SummaryExpr::SummaryAgg { family, .. } = &root.expr else { + panic!("expected SummaryAgg, got {:?}", root.expr); + }; + assert_eq!( + family, + &SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) + ); + assert_eq!( + root.schema + .fields + .iter() + .map(|f| f.name.as_str()) + .collect::>(), + vec!["ts", "value", "job"], + ); + assert_eq!( + field(&root.schema, "value").dtype, + SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) + ); + assert_eq!(root.schema.time_index, Some(0)); + } + + /// Issue #163, case 1: a bare per-series range function (e.g. + /// `quantile_over_time(...)`) realizes to `SummaryAgg { reduction: + /// PerEntity, .. }` — proving the pre-ASAP `Reduction` this crate + /// already computes (issue #165) is carried onto the post-ASAP node + /// verbatim, not flattened back into an ambiguous bare `Vec`. + #[test] + fn bare_per_series_aggregate_realizes_summary_agg_with_per_entity_reduction() { + use std::time::Duration; + let q = agg_per_entity( + default_quantile(0.99), + QueryExpr::TimeRange { + range: Duration::from_secs(10), + child: Rc::new(metric_scan(&["job"])), + }, + ); + let root = realize(&q).unwrap(); + let SummaryExpr::SummaryEstimate { summary_input, .. } = &root.expr else { + panic!("expected estimate root, got {:?}", root.expr); + }; + let SummaryExpr::SummaryAgg { reduction, .. } = &summary_input.expr else { + panic!("expected SummaryAgg, got {:?}", summary_input.expr); + }; + assert_eq!(reduction, &ReductionTy::PerEntity); + } + + /// Issue #163, case 2: an aggregation operator explicitly invoked with + /// no `by(...)` (e.g. `count(hll_metric)`) realizes to `SummaryAgg { + /// reduction: Reduce(vec![]), .. }` — byte-identical `by: []` to the + /// previous test at the old `Vec` shape; `reduction` is what + /// tells them apart now. + #[test] + fn explicit_empty_by_aggregate_realizes_summary_agg_with_reduce_reduction() { + let intent = AggIntent::Cardinality { + col: None, + accuracy: AccuracyTarget::Epsilon(0.01), + }; + let q = agg(vec![], intent, metric_scan(&["job"])); + let root = realize(&q).unwrap(); + let SummaryExpr::SummaryEstimate { summary_input, .. } = &root.expr else { + panic!("expected estimate root, got {:?}", root.expr); + }; + let SummaryExpr::SummaryAgg { reduction, .. } = &summary_input.expr else { + panic!("expected SummaryAgg, got {:?}", summary_input.expr); + }; + assert_eq!(reduction, &ReductionTy::by(vec![])); + } + + #[test] + fn nested_aggregates_realize_per_node() { + // quantile(0.9, sum by (job) (m)) — the implementation decision + // fires per node over the nested tree: KLL over an exact Sum + // accumulator. + let inner = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let outer = agg(vec![], default_quantile(0.9), inner); + let root = realize(&outer).unwrap(); + + let SummaryExpr::SummaryEstimate { summary_input, .. } = &root.expr else { + panic!("expected estimate root, got {:?}", root.expr); + }; + let SummaryExpr::SummaryAgg { child, family, .. } = &summary_input.expr else { + panic!("expected outer SummaryAgg, got {:?}", summary_input.expr); + }; + assert!(matches!( + family, + SummaryFamilyType::Sketch(SketchKind::Kll, _) + )); + let SummaryExpr::SummaryAgg { + family: inner_family, + child: leaf, + .. + } = &child.expr + else { + panic!("expected inner SummaryAgg, got {:?}", child.expr); + }; + assert_eq!( + inner_family, + &SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) + ); + assert!(matches!(leaf.expr, SummaryExpr::KeepPreAsap(_))); + } + + /// Issue #115: the summary is built over the intent's own input column. + /// Before `Cardinality`/`Quantile` carried `col`, `summarised_column` always + /// fell through to `ColumnRef::SampleValue`, so an HLL was built over the + /// wrong column for every SQL `COUNT(DISTINCT c)`. + #[test] + fn sketch_realizes_over_the_intents_input_column() { + // `metric_scan(&["job"])` → columns [ts=0, value=1, job=2]. + let cases = [ + (Some(2), ColumnRef::Named("job".into())), + (Some(1), ColumnRef::Named("value".into())), + // PromQL convention: no column ⇒ the synthetic sample value. + (None, ColumnRef::SampleValue), + ]; + for (col, want) in cases { + let intent = AggIntent::Cardinality { + col, + accuracy: AccuracyTarget::Epsilon(0.01), + }; + let root = realize(&agg(vec![0], intent, metric_scan(&["job"]))).unwrap(); + let bound = find_summary_col(&root) + .unwrap_or_else(|| panic!("expected a SummaryAgg for col={col:?}")); + assert_eq!(bound, want, "wrong summarised column for col={col:?}"); + } + } + + /// The `col` of the first `SummaryAgg` in the tree. + fn find_summary_col(node: &SummaryNode) -> Option { + match &node.expr { + SummaryExpr::SummaryAgg { col, .. } => Some(col.clone()), + SummaryExpr::SummaryEstimate { summary_input, .. } => find_summary_col(summary_input), + _ => None, + } + } + + #[test] + fn pass_through_intents_stay_logical() { + // avg is exact but non-mergeable; histogram_quantile (classic + // buckets, #79) is never sketchable; exact quantile is exact by + // decree. All three stay whole logical subtrees. + for intent in [ + AggIntent::Avg { col: None }, + AggIntent::HistogramQuantile { q: 0.99 }, + AggIntent::Quantile { + col: None, + q: 0.99, + accuracy: AccuracyTarget::Exact, + }, + ] { + let q = agg(vec![2], intent.clone(), metric_scan(&["job"])); + let root = realize(&q).unwrap(); + assert!( + matches!(root.expr, SummaryExpr::KeepPreAsap(ref e) if **e == q), + "expected KeepPreAsap passthrough for {intent:?}" + ); + } + } + + #[test] + fn logical_parent_subsumes_bindable_child() { + // Filter over a bindable quantile: `KeepPreAsap` has no post-ASAP + // children, so the conservative fallback keeps the whole subtree + // logical. + use asap_types::pre_asap::expr_ir::{CompareOpKind, ScalarValue}; + use asap_types::pre_asap::query_expr::Predicate; + let q = QueryExpr::Filter { + pred: Predicate(Rc::new(QueryExpr::Compare { + left: Rc::new(QueryExpr::Column(0)), + op: CompareOpKind::Gt, + right: Rc::new(QueryExpr::Literal(ScalarValue::Float64(0.5))), + })), + child: Rc::new(agg(vec![], default_quantile(0.99), metric_scan(&[]))), + }; + let root = realize(&q).unwrap(); + assert!(matches!(root.expr, SummaryExpr::KeepPreAsap(ref e) if **e == q)); + } + + #[test] + fn having_and_multi_intent_stay_logical() { + use asap_types::pre_asap::expr_ir::ScalarValue; + use asap_types::pre_asap::query_expr::Predicate; + let mut q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + if let QueryExpr::Aggregate { having, .. } = &mut q { + *having = Some(Predicate(Rc::new(QueryExpr::Literal( + ScalarValue::Boolean(true), + )))); + } + assert!(matches!( + realize(&q).unwrap().expr, + SummaryExpr::KeepPreAsap(_) + )); + + let multi = QueryExpr::Aggregate { + reduction: ReductionTy::by(vec![2]), + measures: vec![AggIntent::Sum { col: None }, AggIntent::Avg { col: None }], + output_names: vec![], + having: None, + child: Rc::new(metric_scan(&["job"])), + }; + assert!(matches!( + realize(&multi).unwrap().expr, + SummaryExpr::KeepPreAsap(_) + )); + } + + #[test] + fn topk_realizes_cms_with_heap_and_topk_readout() { + let q = agg( + vec![2], + AggIntent::TopK { + k: 5, + accuracy: AccuracyTarget::Epsilon(0.01), + }, + metric_scan(&["job"]), + ); + let root = realize(&q).unwrap(); + let SummaryExpr::SummaryEstimate { + summary_input, + query, + } = &root.expr + else { + panic!("expected estimate root, got {:?}", root.expr); + }; + assert!(matches!(query, PostAsapSketchQuery::TopK { k: 5 })); + assert!(matches!( + &summary_input.expr, + SummaryExpr::SummaryAgg { + family: SummaryFamilyType::Sketch(SketchKind::CmsWithHeap, _), + .. + } + )); + } + + #[test] + fn sql_reducer_resolves_named_input_column() { + // SUM(bytes) over a tabular scan: `col` resolves positionally to the + // named column, not the PromQL sample value. + let scan = QueryExpr::Scan { + source: Source::Table { + table_ref: "t".into(), + }, + predicates: vec![], + schema: SchemaTy { + columns: vec![ + Column::new("host", DataType::Utf8, false), + Column::new("bytes", DataType::Int64, false), + ], + time_index: None, + unique_keys: vec![], + closed: true, + }, + }; + let q = agg(vec![0], AggIntent::Sum { col: Some(1) }, scan); + let root = realize(&q).unwrap(); + let SummaryExpr::SummaryAgg { col, .. } = &root.expr else { + panic!("expected SummaryAgg, got {:?}", root.expr); + }; + assert_eq!(col, &ColumnRef::Named("bytes".into())); + } } diff --git a/crates/devtools/src/bin/show_post_asap_ir.rs b/crates/devtools/src/bin/show_post_asap_ir.rs index bfc153a4..5aa97bf4 100644 --- a/crates/devtools/src/bin/show_post_asap_ir.rs +++ b/crates/devtools/src/bin/show_post_asap_ir.rs @@ -20,7 +20,7 @@ // `metrics(ts, service, region, latency, bytes)` catalog — the same table // used in cross_language.rs and topk_ir.rs. -use asap_aware_mapping::bind::keep_pre_asap; +use asap_aware_mapping::replacement::keep_pre_asap; use asap_aware_mapping::{ Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG, }; diff --git a/crates/frontend-promql/tests/observability/promql_corpus.rs b/crates/frontend-promql/tests/observability/promql_corpus.rs index 23d3cd43..2aba0cd1 100644 --- a/crates/frontend-promql/tests/observability/promql_corpus.rs +++ b/crates/frontend-promql/tests/observability/promql_corpus.rs @@ -15,7 +15,7 @@ use std::rc::Rc; -use asap_aware_mapping::bind::{keep_pre_asap, ImplementError}; +use asap_aware_mapping::replacement::{keep_pre_asap, ImplementError}; use asap_aware_mapping::{ Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG, }; diff --git a/crates/integration-tests/tests/cse.rs b/crates/integration-tests/tests/cse.rs index 6d28e1a7..29bc2547 100644 --- a/crates/integration-tests/tests/cse.rs +++ b/crates/integration-tests/tests/cse.rs @@ -1,32 +1,44 @@ -//! End-to-end pre-ASAP CSE → implementation pin (issue #212, #222, #223). +//! End-to-end pre-ASAP CSE → workload-wide search pin (issue #212, #222, +//! #223). //! //! Drives the full staged pipeline this issue lands: two independently //! lowered `QueryExpr` trees → `share_common_subtrees` (stage 1, -//! `asap-types::pre_asap::cse`) → `implement_workload` (stage 2, -//! `asap-aware-mapping`) — and asserts the sharing that stage 1 decides -//! survives into stage 2's bound output as one genuinely shared -//! `Rc`, not just one shared `Rc`. This is the "real -//! caller" the issue's landing plan requires before `share_common_subtrees` -//! is allowed to exist at all (its predecessor, -//! `asap-plan::cse::dedupe_subtrees`, was deleted in #192 for being unwired -//! dead code). +//! `asap-types::pre_asap::cse`, run internally by `search_workload`) → +//! `search_workload` (stage 2, `asap-aware-mapping`) — and asserts the +//! sharing that stage 1 decides survives into stage 2's discovered +//! `PlanSpace` as one genuinely shared `MemoGroup`, not just one shared +//! `Rc`. This is the "real caller" the issue's landing plan +//! requires before `share_common_subtrees` is allowed to exist at all (its +//! predecessor, `asap-plan::cse::dedupe_subtrees`, was deleted in #192 for +//! being unwired dead code). +//! +//! Committing to one final, physically-materialized answer for a whole +//! workload (the former `implement_workload`/`implement_workload_with`, +//! which this test file used to drive instead of `search_workload`) is out +//! of `asap-aware-mapping`'s scope — see that crate's `lib.rs` `## Status` +//! section — so these tests assert on the discovered `PlanSpace` shape +//! directly, the same way `asap-aware-mapping::replacement`'s own +//! `shared_aggregate_across_two_roots_gets_both_strategies_candidates` test +//! does, just exercised through the crate's public API from this external +//! integration-test crate. use std::rc::Rc; -use asap_aware_mapping::implement_workload; +use asap_aware_mapping::{search_workload, Replacement}; use asap_frontend_promql::lower_promql; -use asap_types::pre_asap::cse::share_common_subtrees; use asap_types::pre_asap::query_expr::QueryExpr; use asap_types::types::AccuracyTarget; /// Two workload entries that happen to submit the exact same query (a /// realistic case — two dashboards, or a query fired both standalone and as /// part of a larger batch) collapse onto one shared `Rc` after -/// `share_common_subtrees`, and then onto one genuinely-shared, single build -/// `Rc` after `implement_workload` — no second structural- -/// equality pass at the post-ASAP layer needed for this kind of sharing. +/// `search_workload`'s internal `share_common_subtrees` pass, and onto one +/// genuinely-shared [`MemoGroup`](asap_aware_mapping::MemoGroup) — carrying +/// every candidate discovered for it exactly once, not once per root — no +/// second structural-equality pass at the post-ASAP layer needed for this +/// kind of sharing. #[test] -fn duplicate_workload_queries_share_one_bound_summary() { +fn duplicate_workload_queries_collapse_onto_one_memo_group() { // Grouped (`by (job)`), so the shared `Aggregate`'s output schema carries // a provable unique key — the legality gate `share_common_subtrees` // enforces (see `asap-types::pre_asap::cse`'s module doc) — and its @@ -45,69 +57,99 @@ fn duplicate_workload_queries_share_one_bound_summary() { "fixture sanity: identical query text lowers identically" ); - let shared = share_common_subtrees(vec![("a", a), ("b", b)]); - let [(_, qa), (_, qb)] = shared.as_slice() else { - panic!("expected 2 roots"); - }; + let space = search_workload(vec![("a", Rc::new(a)), ("b", Rc::new(b))]); + + // roots[0] and roots[1] must have merged onto the same Rc — the + // `share_common_subtrees` pass `search_workload` runs internally. assert!( - Rc::ptr_eq(qa, qb), - "share_common_subtrees must collapse the two identical roots onto one Rc" + Rc::ptr_eq(&space.roots[0].1, &space.roots[1].1), + "search_workload must collapse the two identical roots onto one Rc" ); - let bound = implement_workload(shared); - let [(_, ra), (_, rb)] = bound.as_slice() else { - panic!("expected 2 bound results"); - }; - let ra = ra.as_ref().expect("query a failed to bind"); - let rb = rb.as_ref().expect("query b failed to bind"); - assert!( - Rc::ptr_eq(ra, rb), - "implement_workload must reuse one bound SummaryNode for the shared Rc, \ - got two independently-built nodes: {ra:?} vs {rb:?}" + // The single shared root is one discovered TargetSubDAG, holding one + // MemoGroup with consumer_count 2 — SketchFamilyStrategy's one + // ExactAggregate candidate *and* SharedSubtreeStrategy's share-vs- + // recompute pair, exactly as `shared_aggregate_across_two_roots_gets_both_strategies_candidates` + // (asap-aware-mapping::replacement's own equivalent, internal test) + // pins for the same fixture shape. + let group = space + .group_for(&space.roots[0].1) + .expect("shared root must be a discovered target"); + assert_eq!(group.consumer_count, 2); + assert_eq!( + group.candidates.len(), + 3, + "1 ExactAggregate Summary + 2 Rewrite (share/recompute): {:?}", + group.candidates ); + + let summary_count = group + .candidates + .iter() + .filter(|c| matches!(c.replacement, Replacement::Summary(_))) + .count(); + let rewrite_count = group + .candidates + .iter() + .filter(|c| matches!(c.replacement, Replacement::Rewrite(_))) + .count(); + assert_eq!(summary_count, 1); + assert_eq!(rewrite_count, 2); + + // The two Rewrite candidates must NOT have collapsed into one (the + // "false-positive dedup" failure mode `is_duplicate_rewrite` exists to + // prevent): one shares the group's own target `Rc`, the other is a + // structurally-identical but independently-built `Rc`. + let one_is_the_target = group.candidates.iter().any( + |c| matches!(&c.replacement, Replacement::Rewrite(rc) if Rc::ptr_eq(rc, &group.target)), + ); + let one_is_not = group.candidates.iter().any( + |c| matches!(&c.replacement, Replacement::Rewrite(rc) if !Rc::ptr_eq(rc, &group.target)), + ); + assert!(one_is_the_target && one_is_not); } /// The negative control: two workload entries whose queries are NOT -/// structurally identical must not be conflated by `implement_workload`'s -/// `Rc::as_ptr` memoization — different `Rc` roots always bind -/// independently. +/// structurally identical must not be conflated — `search_workload` +/// discovers two independent roots, each its own `TargetSubDAG` with its +/// own `MemoGroup` and `consumer_count == 1`. #[test] -fn distinct_workload_queries_do_not_share_a_bound_summary() { +fn distinct_workload_queries_get_independent_memo_groups() { let a = lower_promql("sum by (job) (http_requests_total)", AccuracyTarget::Exact) .expect("query a failed to lower"); let b = lower_promql("sum by (job) (http_response_total)", AccuracyTarget::Exact) .expect("query b failed to lower"); assert_ne!(a, b, "fixture sanity: the two queries differ"); - let shared = share_common_subtrees(vec![("a", a), ("b", b)]); - let [(_, qa), (_, qb)] = shared.as_slice() else { - panic!("expected 2 roots"); - }; - assert!(!Rc::ptr_eq(qa, qb)); + let space = search_workload(vec![("a", Rc::new(a)), ("b", Rc::new(b))]); + assert!(!Rc::ptr_eq(&space.roots[0].1, &space.roots[1].1)); - let bound = implement_workload(shared); - let [(_, ra), (_, rb)] = bound.as_slice() else { - panic!("expected 2 bound results"); - }; - let ra = ra.as_ref().expect("query a failed to bind"); - let rb = rb.as_ref().expect("query b failed to bind"); + let group_a = space + .group_for(&space.roots[0].1) + .expect("root a must be a discovered target"); + let group_b = space + .group_for(&space.roots[1].1) + .expect("root b must be a discovered target"); assert!( - !Rc::ptr_eq(ra, rb), - "distinct queries must bind independently" + !Rc::ptr_eq(&group_a.target, &group_b.target), + "distinct queries must land in distinct MemoGroups" ); + assert_eq!(group_a.consumer_count, 1); + assert_eq!(group_b.consumer_count, 1); } /// Single-query CSE (a repeated sub-expression within one query) also -/// survives through `implement_workload`: the two grouped-`Aggregate` -/// branches of a `BinaryOp` collapse to one shared `Rc` in -/// `share_common_subtrees`, and to one shared bound `Rc` here. +/// survives through `search_workload`: the two grouped-`Aggregate` branches +/// of a `BinaryOp` collapse to one shared `Rc` in the internal +/// `share_common_subtrees` pass, and to one shared `MemoGroup` (with +/// `consumer_count == 2`, one per branch) here. #[test] -fn single_query_repeated_subexpression_shares_one_bound_summary() { +fn single_query_repeated_subexpression_shares_one_memo_group() { let query = "sum by (job) (http_requests_total) / sum by (job) (http_requests_total)"; let expr = lower_promql(query, AccuracyTarget::Exact).expect("query failed to lower"); - let shared = share_common_subtrees(vec![("q", expr)]); - let [(_, root)] = shared.as_slice() else { + let space = search_workload(vec![("q", Rc::new(expr))]); + let [(_, root)] = space.roots.as_slice() else { panic!("expected 1 root"); }; let QueryExpr::BinaryOp { lhs, rhs, .. } = root.as_ref() else { @@ -118,17 +160,15 @@ fn single_query_repeated_subexpression_shares_one_bound_summary() { "the two identical sum-by-job branches must collapse onto one Rc" ); - // Bind both branches (as if they were two roots of a tiny sub-workload) - // through the same memoized `implement_workload`, and confirm they bind - // to the identical `Rc`. - let bound = implement_workload(vec![("lhs", Rc::clone(lhs)), ("rhs", Rc::clone(rhs))]); - let [(_, ra), (_, rb)] = bound.as_slice() else { - panic!("expected 2 bound results"); - }; - let ra = ra.as_ref().expect("lhs failed to bind"); - let rb = rb.as_ref().expect("rhs failed to bind"); + let group = space + .group_for(lhs) + .expect("the shared branch must be a discovered target"); + assert_eq!( + group.consumer_count, 2, + "the shared branch is referenced from both BinaryOp operand positions" + ); assert!( - Rc::ptr_eq(ra, rb), - "the shared branch must bind to one shared SummaryNode" + !group.candidates.is_empty(), + "the shared branch must carry at least one candidate" ); } diff --git a/crates/integration-tests/tests/l4_binding.rs b/crates/integration-tests/tests/l4_binding.rs index 576cf053..b542e39f 100644 --- a/crates/integration-tests/tests/l4_binding.rs +++ b/crates/integration-tests/tests/l4_binding.rs @@ -9,7 +9,7 @@ use std::rc::Rc; -use asap_aware_mapping::bind::{keep_pre_asap, ImplementError}; +use asap_aware_mapping::replacement::{keep_pre_asap, ImplementError}; use asap_aware_mapping::{ Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG, }; diff --git a/crates/integration-tests/tests/l4_binding_sql.rs b/crates/integration-tests/tests/l4_binding_sql.rs index 4289a27c..322dc424 100644 --- a/crates/integration-tests/tests/l4_binding_sql.rs +++ b/crates/integration-tests/tests/l4_binding_sql.rs @@ -30,7 +30,7 @@ use std::rc::Rc; -use asap_aware_mapping::bind::{keep_pre_asap, ImplementError}; +use asap_aware_mapping::replacement::{keep_pre_asap, ImplementError}; use asap_aware_mapping::{ Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG, }; diff --git a/crates/types/src/pre_asap/cse.rs b/crates/types/src/pre_asap/cse.rs index 7639dc76..23c38199 100644 --- a/crates/types/src/pre_asap/cse.rs +++ b/crates/types/src/pre_asap/cse.rs @@ -10,8 +10,8 @@ //! `sql_lowering.rs`'s `median_is_the_same_intent_as_an_explicit_half_percentile` //! test). [`share_common_subtrees`] is the single entry point, run once per //! workload batch (or once per query — see "Single-query CSE" below) *after* -//! `resolve_root`, *before* `implement_workload` -//! ([`asap_aware_mapping::implement_workload`]). +//! `resolve_root`, *before* the pre-ASAP → post-ASAP replacement/search pass +//! (`asap_aware_mapping::replacement`). //! //! ## Algorithm: classic hash-consing / value-numbering //! @@ -81,20 +81,23 @@ //! ## Landing plan (issue #223) //! //! This module is stage 1 of a 4-stage plan. Stage 2 -//! ([`asap_aware_mapping::implement_workload`]) is a real caller, wired at -//! the same time so this never becomes unwired dead code again (the original -//! `asap-plan::cse::dedupe_subtrees` was deleted in #192 for exactly that). -//! Stage 3 — [`dag_export`](crate::dag_export) computing its per-node `hash` -//! by calling this module's [`structural_hash`] directly, instead of a -//! parallel reimplementation — is also done, so `tools/dag-viewer`'s -//! "shared subtree" highlighting now flags exactly the candidate pairs this -//! module's own `InternTable` would bucket together (still only a hash -//! match, not a guarantee of `share_common_subtrees`-actual sharing — see -//! `dag_export`'s module doc). Stage 4 (issue #237) is implemented in -//! `asap_aware_mapping::cost_model::CostModel::cse_share_decision` and its -//! caller, `asap_aware_mapping::bind::implement_workload_with` — a real, -//! Volcano/Cascades-style cost comparison over what this module detects, not -//! a fixed rule. See `docs/design_docs/cse-cost-model-decision.md`. This module's own +//! (`asap_aware_mapping::replacement::search_workload_with`, which runs +//! [`share_common_subtrees`] itself before searching) is a real caller, +//! wired at the same time so this never becomes unwired dead code again +//! (the original `asap-plan::cse::dedupe_subtrees` was deleted in #192 for +//! exactly that). Stage 3 — [`dag_export`](crate::dag_export) computing its +//! per-node `hash` by calling this module's [`structural_hash`] directly, +//! instead of a parallel reimplementation — is also done, so +//! `tools/dag-viewer`'s "shared subtree" highlighting now flags exactly the +//! candidate pairs this module's own `InternTable` would bucket together +//! (still only a hash match, not a guarantee of +//! `share_common_subtrees`-actual sharing — see `dag_export`'s module doc). +//! Stage 4 (issue #237) is implemented in +//! `asap_aware_mapping::cost_model::CostModel::cse_share_decision`, called +//! from `asap_aware_mapping::replacement::PlanSpace::cost_sorted` (via that +//! module's own `cse_preference`) — a real, Volcano/Cascades-style cost +//! comparison over what this module detects, not a fixed rule. See +//! `docs/design_docs/cse-cost-model-decision.md`. This module's own //! unconditional "share whenever legal" behavior is unchanged: detection //! stays cost-agnostic by construction (this crate cannot depend on //! `asap-aware-mapping`'s `CostModel`), and the cost-aware decision is diff --git a/crates/types/src/pre_asap/mod.rs b/crates/types/src/pre_asap/mod.rs index f7a18b21..177c43ba 100644 --- a/crates/types/src/pre_asap/mod.rs +++ b/crates/types/src/pre_asap/mod.rs @@ -23,7 +23,7 @@ //! - [`cse`] — workload-level structural common-subexpression elimination //! over an already-`resolve_root`'d tree (issue #212, #222, #223), run //! *after* `resolve_root` / `canonicalize` and *before* implementation -//! (`asap_aware_mapping::implement_workload`). +//! (`asap_aware_mapping::replacement`). //! //! Formerly the separate `asap-l2` crate; folded in here since //! `binder`/`column_resolution`/`canonicalize`/`resolve` have no From ca1b3ec5d1028c561083fdbdba950764e080eed2 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 24 Aug 2026 11:55:46 -0600 Subject: [PATCH 25/49] docs(asap-aware-mapping): fix remaining bind.rs/select_and_bind references The bind.rs deletion (51ae6e4) left three docs pointing at APIs that no longer exist: cse-cost-model-decision.md and user-guide.md still described bind::implement_workload_with as the CSE share-decision's home, and the developer guide still named bind::select_and_bind and an 'implement_workload keeps first candidate' exception that no longer applies now that this crate stops at PlanSpace (every candidate + cost) and doesn't commit to one physically-materialized answer. - cse-cost-model-decision.md: point at PlanSpace::cost_sorted/rank_group/ cse_preference as where cse_share_decision now hooks in; note the 'no search engine' framing predates the search.rs merge. - user-guide.md: point at search_workload/search_workload_with + PlanSpace instead of implement_workload/implement_workload_with; fix the bind::keep_pre_asap reference to replacement::keep_pre_asap. - developer guide: fix replacement::select_and_bind -> replacement::realize_child (renamed in 51ae6e4), rewrite the workload-selection 'one exception' paragraph, drop the stale bind::implement_workload extension-map row. Co-Authored-By: Claude Sonnet 5 --- docs/design_docs/cse-cost-model-decision.md | 61 ++++++++++++------- .../ASAP-aware-mapping-developer-guide.md | 9 ++- docs/user-guide/user-guide.md | 13 ++-- 3 files changed, 51 insertions(+), 32 deletions(-) diff --git a/docs/design_docs/cse-cost-model-decision.md b/docs/design_docs/cse-cost-model-decision.md index edc5731c..57265749 100644 --- a/docs/design_docs/cse-cost-model-decision.md +++ b/docs/design_docs/cse-cost-model-decision.md @@ -46,18 +46,23 @@ cost comparison does, without needing a separately hardcoded cheap-threshold carve-out — a cheap-to-recompute candidate naturally loses the comparison on its own. -Why this doesn't need full Volcano/Cascades-scale infrastructure: this repo -has no plan-enumeration/DP-search engine anywhere, and `CostModel` is -deliberately a narrow, single-shot ranking/sizing interface -(`rank_candidates`/`size_params`), not a cost-driven search engine. The -decision here is binary (share vs. don't, per already-detected candidate), -so a direct cost comparison captures the Volcano/Cascades *policy* — weigh -real costs, don't apply a fixed rule — without requiring a memo-based search -space this repo doesn't otherwise have. `implement_workload_with` still -computes the true `consumer_count` for each candidate via a whole-workload -pre-pass before deciding, rather than deciding on a running/partial count — -the decision is made once, from full knowledge of the workload's sharing -structure, the same way a real cost-based optimizer would. +Why this doesn't need its own, separate search infrastructure: **this +paragraph predates issue #252's MEMO-based search engine** (`PlanSpace`/ +`MemoGroup`, now living in `replacement.rs` — a real Volcano/Cascades-style +candidate space, since folded in from what was PR #263). That engine exists +for enumerating and ranking the *much* larger candidate space (every summary +family choice at every site, workload-wide). This specific decision — share +vs. don't, for one already-detected CSE candidate — still doesn't need to be +re-derived by that machinery: it's binary, so `PlanSpace::cost_sorted`'s +ranking step reuses one direct `CostModel::cse_share_decision` comparison per +group rather than a separate search, the same policy this section originally +argued for (weigh real costs, don't apply a fixed rule) — it's just now one +ranking step *inside* the bigger search engine's output, not a standalone +decision with no search engine around it at all. `search_workload_with`'s +target-discovery pass still computes the true `consumer_count` for each +candidate via a whole-workload traversal before any ranking happens — the +decision is made from full knowledge of the workload's sharing structure, the +same way a real cost-based optimizer would. ## Layering constraint @@ -70,13 +75,23 @@ gate) and the cost-aware decision is applied downstream, in ## Where it hooks in -[`bind::implement_workload_with`](../crates/asap-aware-mapping/src/bind.rs) -computes each shared subtree's true `consumer_count` across the whole -workload up front, then — the first time it binds that subtree — asks -`CostModel::cse_share_decision` once and caches the resulting `ShareDecision` -alongside the bound `SummaryNode`, so every later occurrence of the same -`Rc` consistently reuses the cached summary (`Share`) or rebinds -independently (`RecomputeIndependently`) per that one decision. +[`PlanSpace::cost_sorted`](../crates/asap-aware-mapping/src/replacement.rs) +is where this hooks in today. `search_workload_with` computes each shared +subtree's true `consumer_count` across the whole workload up front (the same +role `implement_workload_with`'s pre-pass used to play, before that function +was retired along with `bind.rs` — this crate no longer commits to one +physically-materialized answer at all; picking and building one final +`SummaryNode` per shared subtree is a downstream deployment's job, not this +crate's). For a `MemoGroup` whose candidates are a +[`SharedSubtreeStrategy`](../crates/asap-aware-mapping/src/replacement.rs) +share-vs-recompute pair, `cost_sorted`'s ranking step (`rank_group`/ +`cse_preference`) asks `CostModel::cse_share_decision` once per group — using +one representative bound `SummaryNode` built just for that comparison, not +cached anywhere — and sorts the pair so the preferred candidate (`Share` or +`RecomputeIndependently`) comes first. Both candidates are still returned; +ranking never drops one (see Rule 2 in `docs/design_docs/asap_aware_mapping.md`'s +mental-model section — a `CostModel` orders and parameterizes, it never +prunes). ## Defaults @@ -99,6 +114,8 @@ either or both, same as `size_params` already lets a deployment override ## Scope -This decision, and `cse_share_decision`'s wiring into `implement_workload_with`, -close out #223's stage 4 and #212's original "add CSE" tracking issue. Stage -3 (`dag_export::structural_hash` unification) landed separately in PR #244. +This decision, and `cse_share_decision`'s wiring into `PlanSpace::cost_sorted` +(originally into `implement_workload_with`, before `bind.rs` was retired — +see above), close out #223's stage 4 and #212's original "add CSE" tracking +issue. Stage 3 (`dag_export::structural_hash` unification) landed separately +in PR #244. diff --git a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md index c97f6326..55c9b16c 100644 --- a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md @@ -20,7 +20,7 @@ If you only need to find the right extension point, start with the [extension ma One term is central to this guide: -- **Implementation**: one valid realization of an `AggIntent`. It may be an approximate sketch, an exact mergeable accumulator, or a pass-through with no summary. `implementations_for_with` (a `replacement.rs`-private function) enumerates the valid implementations for one node and ranks them. It does not walk the plan or select a winner. `SketchFamilyStrategy::replacements()` is its only caller: for one target, it constructs every candidate's `SummaryNode` directly and returns all of them — deciding and constructing happen in the same step, not two steps bridged by a separately-named function in another module. A caller that needs one result takes the first candidate. Workload-wide helpers—`implement_workload` and `implement_workload_with`—perform that selection internally so shared `Rc` roots receive one consistent decision. +- **Implementation**: one valid realization of an `AggIntent`. It may be an approximate sketch, an exact mergeable accumulator, or a pass-through with no summary. `implementations_for_with` (a `replacement.rs`-private function) enumerates the valid implementations for one node and ranks them. It does not walk the plan or select a winner. `SketchFamilyStrategy::replacements()` is its only caller: for one target, it constructs every candidate's `SummaryNode` directly and returns all of them — deciding and constructing happen in the same step, not two steps bridged by a separately-named function in another module. A caller that needs one result takes the first candidate. At workload scale, `search_workload`/`search_workload_with` extend the same "never prune" contract across every `TargetSubDAG` in the whole workload (see §3) — this crate stops at that full candidate space plus cost; it does not itself commit to one final, physically-shared answer per site. That commitment (which candidate to build, where to place it) is a downstream deployment's call, not this crate's. In short: `ReplacementStrategy` enumerates, packages, and binds every candidate, and the caller selects when it needs a single executable answer. Section 3 shows the complete flow. @@ -262,11 +262,11 @@ The crate has one source of truth for valid implementations, and deciding and co In the [design document](../design_docs/asap_aware_mapping.md), each `ReplacementSubDAG` is a candidate. The complete `replacements()` result is the set of alternatives for one location in the plan. -**Tradeoff:** a single-target bind sizes and constructs every valid sketch candidate before the caller keeps the first one. This costs more than constructing only the preferred candidate, but it means there is exactly one place in the crate that decides what an `AggIntent` may become and exactly one place bound output comes from. Child nodes repeat selection independently (via `replacement::select_and_bind`), so a choice at one target does not force choices in nested aggregates. +**Tradeoff:** a single-target bind sizes and constructs every valid sketch candidate before the caller keeps the first one. This costs more than constructing only the preferred candidate, but it means there is exactly one place in the crate that decides what an `AggIntent` may become and exactly one place bound output comes from. Child nodes repeat selection independently (via `replacement::realize_child`), so a choice at one target does not force choices in nested aggregates. This crate now also implements the whole-plan Cascades/Volcano-style search the design document describes (issue #252, part of #33): `replacement::search_workload`/`search_workload_with` discover every candidate `TargetSubDAG` across a whole workload and run every registered `ReplacementStrategy` against each to a fixpoint, deduping into a `PlanSpace` — one `MemoGroup` per distinct `TargetSubDAG`, holding every alternative discovered for it. `PlanSpace::cost_sorted` is the final `sorted_by(cost_model)` step. See `replacement.rs`'s own module docs ("Workload-wide search") for the full design: MEMO groups instead of a flat `2^N`-sized plan list, dedup discipline, termination, and cost-based ranking. -**One exception:** `bind::implement_workload` and `implement_workload_with` select the first candidate internally. Workload-wide CSE memoizes by `Rc` pointer identity, so roots that share an `Rc` must use the same canonical decision. Other callers use `SketchFamilyStrategy::replacements()` (for one target) or `search_workload`/`search_workload_with` (for a whole workload's worth of candidates) directly. +**No workload-wide single-answer selection lives in this crate any more.** An earlier version had `bind::implement_workload`/`implement_workload_with`, which picked and memoized one candidate per shared `Rc` root so CSE-collapsed roots got one consistent, physically-shared bound `SummaryNode`. That was removed: committing to one final answer per site — and physically materializing it — is a downstream deployment's call (it needs to weigh placement too, which this crate can't see), not something this crate should pre-decide with no real consumer of that single answer. Callers now get every candidate, ranked with cost, from `search_workload`/`search_workload_with`'s `PlanSpace`, and make the final pick themselves. The two remaining `.into_iter().next()`-shaped helpers in `replacement.rs` (`realize_child`, used for a single candidate's own child recursion during construction; `realize_one`, used internally by `cost_sorted`'s ranking step to get one representative bound node for a cost comparison) are narrow, single-target implementation details — neither is a "here's the workload's answer" entry point. ### Replacement-strategy path @@ -575,7 +575,7 @@ fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { `implementations_for_with` already did the hard part — enumerating and sizing every candidate, ranked. This loop's only job is to hand each one to `construct_summary(expr, implementation, cost_model)` — a private helper in the same file, not a separately-named public bridge in a different module — which derives the child schema, resolves the summarized column, builds the readout, recurses into the child, and assembles the `SummaryNode`. No per-candidate sizing logic lives here: sizing already happened inside `implementations_for_with`. -Because only `expr`'s own top-level `Implementation` is ever supplied from outside — `construct_summary` recurses into `expr`'s child via `replacement::select_and_bind`, a fresh internal selection over that child's own candidates, not a forced one — enumerating a candidate for one target never leaks into that target's own nested aggregates. (An earlier version of this strategy forced ranking via a `CostModel`-wrapping adapter for the whole recursive bind, which had exactly that leak as a latent bug; today's design doesn't have the problem, because nothing below the top node is ever forced.) +Because only `expr`'s own top-level `Implementation` is ever supplied from outside — `construct_summary` recurses into `expr`'s child via `replacement::realize_child`, a fresh internal selection over that child's own candidates, not a forced one — enumerating a candidate for one target never leaks into that target's own nested aggregates. (An earlier version of this strategy forced ranking via a `CostModel`-wrapping adapter for the whole recursive bind, which had exactly that leak as a latent bug; today's design doesn't have the problem, because nothing below the top node is ever forced.) This is the pattern to preserve when adding another strategy that needs to enumerate alternatives through an API that normally chooses one: get the exhaustive list from the same enumeration function the single-answer path uses, and construct each entry directly — never wrap the `CostModel` to trick a ranked-first path into producing what you want. @@ -1248,7 +1248,6 @@ Use this table to find the right place for a change. | Change current share/recompute choice | `CostModel::cse_share_decision` | | Decide whether an available implementation satisfies a required one | `impl Matcher` | | Produce a normal (ranked-first) bound summary for one target | `SketchFamilyStrategy::replacements(...).into_iter().next()` | -| Bind a whole workload's roots, sharing across CSE-collapsed roots | reuse `bind::implement_workload`/`implement_workload_with` | | Search a whole workload for every candidate at every `TargetSubDAG` (never pruning) | `replacement::search_workload`/`search_workload_with` | | Get every candidate ranked best-first, across a whole workload | `PlanSpace::cost_sorted` | | Get a real numeric cost per candidate, not just a relative rank | `CostModel::estimate_cost` | diff --git a/docs/user-guide/user-guide.md b/docs/user-guide/user-guide.md index ebf27f63..6af62c53 100644 --- a/docs/user-guide/user-guide.md +++ b/docs/user-guide/user-guide.md @@ -140,7 +140,7 @@ let Some(ReplacementSubDAG { replacement: Replacement::Summary(post_asap), .. }) candidates.into_iter().next() else { // No candidate (e.g. the node isn't a bindable Aggregate) — fall back to - // `asap_aware_mapping::bind::keep_pre_asap(&root)`, the same conservative + // `asap_aware_mapping::replacement::keep_pre_asap(&root)`, the same conservative // pass-through this crate's own dispatch uses. panic!("no candidate for this target"); }; @@ -157,10 +157,13 @@ built-in static preference order (`DefaultCostModel` — what `default_cost_mode See the `CostModel` trait doc in `crates/asap-aware-mapping/src/cost_model.rs` for its overridable hooks (`rank_candidates`, `size_params`, `realize_extension`, …). -To bind every root of a whole workload at once — sharing one bound `SummaryNode` across roots CSE -already collapsed onto the same `Rc` — use `asap_aware_mapping::implement_workload`/ -`implement_workload_with` instead; those two still keep only the cost-model-preferred candidate per -root, since sharing needs one canonical decision to key on. +To see every root of a whole workload at once — including the candidates CSE-shared subtrees get +(a shared subtree's `MemoGroup` carries both the "share" and "recompute independently" options, +ranked by `CostModel::cse_share_decision`) — use `asap_aware_mapping::search_workload`/ +`search_workload_with` instead; unlike the single-target path above, these return every discovered +site's full candidate list (a `PlanSpace`), not one picked winner. Committing to one final, +physically-materialized `SummaryNode` per shared subtree is out of this crate's scope — that's a +downstream deployment's call, once it also knows where each candidate would be placed. ### Reading the result From f6311e21484fa38fa51837586988efd8101ce341 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 24 Aug 2026 12:12:10 -0600 Subject: [PATCH 26/49] refactor(types): split SketchKind into category (SketchKind) + algorithm (SketchAlgorithm) (#266, part of #33) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Absorbs PR #266 (refactor/sketch-kind-algorithm-nesting) into this branch — #266 was last rebased on a much older tip and this branch's own code has since changed enormously (implementation.rs/bind.rs merged into replacement.rs, SummaryExpr::Logical -> KeepPreAsap, search.rs/PlanSpace merged in, bind.rs deleted), so #266's diff was read and manually reapplied against the current surface rather than merged/rebased/cherry-picked. Restructures the sketch type system into a real three-level nesting — family -> kind -> algorithm — instead of a flat (algorithm, params) pair: SummaryFamilyType::Sketch(SketchKind) SketchKind::Quantile(SketchAlgorithm, SketchParams) // + Cardinality/Frequency/TopK Implementation::Sketch(SketchKind) // mirrors SummaryFamilyType's shape - `SketchAlgorithm` — the old flat `SketchKind` enum (Kll/Cms/Hll/DDSketch/ CmsWithHeap/Kmv/Theta/CountSketch/CountSketchWithHeap), renamed verbatim — the actual concrete algorithm choice. - `SketchKind` — new nested enum (Quantile/Cardinality/Frequency/TopK), each variant carrying the committed `(SketchAlgorithm, SketchParams)` pair — a `SketchKind` value always already names which concrete algorithm, never "some quantile sketch, algorithm TBD". `SketchKind::new(algorithm, params)` is the one place a pair gets classified into its category; `.algorithm()`/ `.params()` extract the pair back out regardless of variant. - `SummaryFamilyType::Sketch(SketchKind, SketchParams)` collapses to `Sketch(SketchKind)`; `Implementation::Sketch { kind, params }` becomes the tuple variant `Sketch(SketchKind)`. - `SketchFamilyStrategy` -> `SketchAlgorithmStrategy` — it only ever enumerates algorithms within an already-fixed family+kind, never chooses between families/kinds (unbuilt future work, no ReplacementStrategy does that yet). `ForceSketchKind` no longer exists on this branch (already deleted during this session's earlier ReplacementStrategy/binding-unification work), so its rename to `ForceSketchAlgorithm` doesn't apply. - `CostModel::rank_candidates`/`size_params` now take/return `SketchAlgorithm` (they operate one level below `SketchKind`, on the concrete algorithm choice within an already-fixed kind); `sketch_kind_of` (the PlanSpace ranking helper added when search.rs merged into replacement.rs) likewise extracts `.algorithm()` for `CostModel::rank_candidates` comparisons. Updated every construction/destructuring site across `replacement.rs`, `cost_model.rs`, `lib.rs`, `crates/devtools`, `crates/frontend-promql`'s observability corpus test, and `crates/integration-tests`' l4_binding(_sql) and cse tests — not just `asap-aware-mapping`, per #266's own scope but against the current, much larger merged surface. Dev guide gets the "Family, kind, and algorithm" section from #266 plus the same family/kind/algorithm terminology pass across the rest of the guide and the user guide. Verified: cargo build/test/fmt/clippy --workspace all clean. PR #266 (refactor/sketch-kind-algorithm-nesting) is now superseded by this commit and should be closed. Co-Authored-By: Claude Sonnet 5 --- crates/asap-aware-mapping/src/cost_model.rs | 63 +-- crates/asap-aware-mapping/src/lib.rs | 8 +- crates/asap-aware-mapping/src/replacement.rs | 436 ++++++++++-------- crates/devtools/src/bin/show_post_asap_ir.rs | 6 +- .../tests/observability/promql_corpus.rs | 6 +- crates/integration-tests/tests/cse.rs | 2 +- crates/integration-tests/tests/l4_binding.rs | 22 +- .../integration-tests/tests/l4_binding_sql.rs | 27 +- crates/types/src/post_asap/mod.rs | 17 +- .../post_asap/query_time/error_estimation.rs | 4 +- crates/types/src/post_asap/schema.rs | 9 +- crates/types/src/post_asap/sketch.rs | 82 +++- .../ASAP-aware-mapping-developer-guide.md | 123 +++-- docs/user-guide/user-guide.md | 10 +- 14 files changed, 488 insertions(+), 327 deletions(-) diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index 01651ea5..d5eea93e 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -13,7 +13,7 @@ //! //! This trait is scoped to the approximate-**sketch** family specifically //! ([`CostModel::rank_candidates`]/[`size_params`](CostModel::size_params) -//! take/return [`SketchKind`]/[`SketchParams`]) — `asap_sketch` also has +//! take/return [`SketchAlgorithm`]/[`SketchParams`]) — `asap_sketch` also has //! sibling families for sampling-based, wavelet-transform, and fitted //! statistical-model summaries //! ([`asap_types::post_asap::SamplingKind`]/…/[`asap_types::post_asap::StatModelKind`]), @@ -26,7 +26,7 @@ //! than overloading these ones across incompatible `Kind`/`Params` types. //! //! Every entry point that doesn't take an explicit `&dyn CostModel` -//! ([`SketchFamilyStrategy::default_cost_model`](crate::replacement::SketchFamilyStrategy::default_cost_model), +//! ([`SketchAlgorithmStrategy::default_cost_model`](crate::replacement::SketchAlgorithmStrategy::default_cost_model), //! [`search_workload`](crate::replacement::search_workload)) runs against //! [`DefaultCostModel`], so a deployment that never plugs in its own cost //! model keeps today's static-preference-order behavior exactly, byte for @@ -49,7 +49,7 @@ use std::rc::Rc; use asap_types::post_asap::{ - SketchKind, SketchParams, SketchQuery, SummaryFamilyType, SummaryNode, + SketchAlgorithm, SketchParams, SketchQuery, SummaryFamilyType, SummaryNode, }; use asap_types::pre_asap::agg_intent::AggIntent; use asap_types::pre_asap::expr_ir::ColumnRef; @@ -188,13 +188,17 @@ pub trait CostModel { /// /// Implementations MAY reorder freely and MAY drop entries that aren't /// available in their deployment, but MUST NOT invent a candidate that - /// wasn't in the input — an unknown [`SketchKind`] has no + /// wasn't in the input — an unknown [`SketchAlgorithm`] has no /// [`SketchParams`](asap_types::post_asap::SketchParams) sizing logic in /// `replacement::implementations_for_with` and binding it will panic. /// Returning an empty `Vec` means "no candidate is acceptable"; /// `implementations_for_with` treats that the same as `candidates` /// having been empty to begin with. - fn rank_candidates(&self, intent: &AggIntent, candidates: &[SketchKind]) -> Vec; + fn rank_candidates( + &self, + intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec; /// Size [`SketchParams`] for `kind` (one of the candidates /// [`rank_candidates`](Self::rank_candidates) put first) under the @@ -210,7 +214,7 @@ pub trait CostModel { /// not resize them, can leave this method unimplemented. fn size_params( &self, - kind: SketchKind, + kind: SketchAlgorithm, intent: &AggIntent, eps: f64, delta: f64, @@ -311,14 +315,14 @@ pub trait CostModel { /// on [`PlanSpace::cost_sorted`](crate::replacement::PlanSpace::cost_sorted)), /// not just order candidates against each other — that ordering job /// already belongs to [`rank_candidates`](Self::rank_candidates) (for a - /// [`SketchFamilyStrategy`](crate::replacement::SketchFamilyStrategy) + /// [`SketchAlgorithmStrategy`](crate::replacement::SketchAlgorithmStrategy) /// group) and [`cse_share_decision`](Self::cse_share_decision) (for a /// [`SharedSubtreeStrategy`](crate::replacement::SharedSubtreeStrategy) /// group). /// /// One method covers both candidate shapes this crate ships: /// `candidate.replacement`'s [`Replacement::Summary`] arm (a - /// `SketchFamilyStrategy` candidate — the bound `SummaryNode` is right + /// `SketchAlgorithmStrategy` candidate — the bound `SummaryNode` is right /// there, nothing to reconstruct) and its [`Replacement::Rewrite`] arm /// (a `SharedSubtreeStrategy` share-vs-recompute candidate — no bound /// `SummaryNode` of its own, since sharing is a decision about a target @@ -351,7 +355,11 @@ pub trait CostModel { pub struct DefaultCostModel; impl CostModel for DefaultCostModel { - fn rank_candidates(&self, _intent: &AggIntent, candidates: &[SketchKind]) -> Vec { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { candidates.to_vec() } @@ -430,8 +438,8 @@ mod tests { fn rank_candidates( &self, _intent: &AggIntent, - candidates: &[SketchKind], - ) -> Vec { + candidates: &[SketchAlgorithm], + ) -> Vec { let mut v = candidates.to_vec(); v.reverse(); v @@ -453,8 +461,8 @@ mod tests { fn size_params_default_body_matches_default_size_params() { let intent = default_cardinality(); assert_eq!( - AlwaysPreferLast.size_params(SketchKind::Hll, &intent, 0.01, 0.01), - crate::replacement::default_size_params(SketchKind::Hll, &intent, 0.01, 0.01), + AlwaysPreferLast.size_params(SketchAlgorithm::Hll, &intent, 0.01, 0.01), + crate::replacement::default_size_params(SketchAlgorithm::Hll, &intent, 0.01, 0.01), ); } @@ -468,20 +476,20 @@ mod tests { fn rank_candidates( &self, _intent: &AggIntent, - candidates: &[SketchKind], - ) -> Vec { + candidates: &[SketchAlgorithm], + ) -> Vec { candidates.to_vec() } fn size_params( &self, - kind: SketchKind, + kind: SketchAlgorithm, intent: &AggIntent, eps: f64, delta: f64, ) -> SketchParams { match kind { - SketchKind::Kll => { + SketchAlgorithm::Kll => { let k = if eps >= 0.01 { 200 } else { 2048 }; SketchParams::Kll { k } } @@ -496,19 +504,21 @@ mod tests { let intent = default_quantile(0.99); assert_eq!( - DiscreteKllRungs.size_params(SketchKind::Kll, &intent, 0.001, 0.01), + DiscreteKllRungs.size_params(SketchAlgorithm::Kll, &intent, 0.001, 0.01), SketchParams::Kll { k: 2048 }, ); // Untouched kinds still fall through to the default formula. assert_eq!( - DiscreteKllRungs.size_params(SketchKind::Hll, &intent, 0.01, 0.01), - crate::replacement::default_size_params(SketchKind::Hll, &intent, 0.01, 0.01), + DiscreteKllRungs.size_params(SketchAlgorithm::Hll, &intent, 0.01, 0.01), + crate::replacement::default_size_params(SketchAlgorithm::Hll, &intent, 0.01, 0.01), ); } // ── CSE sharing (issue #237, #223 stage 4) ────────────────────────── - use asap_types::post_asap::{ExactKind, ExactParams, SummaryExpr, SummaryField, SummarySchema}; + use asap_types::post_asap::{ + ExactKind, ExactParams, SketchKind, SummaryExpr, SummaryField, SummarySchema, + }; use asap_types::pre_asap::query_expr::Source; use asap_types::pre_asap::schema::{Column, DataType, Schema}; @@ -613,8 +623,7 @@ mod tests { ExactParams::Sum, )); let sketch = default_cse_shared_maintenance_cost(&SummaryFamilyType::Sketch( - SketchKind::Hll, - SketchParams::Hll { precision: 12 }, + SketchKind::Cardinality(SketchAlgorithm::Hll, SketchParams::Hll { precision: 12 }), )); assert!( exact < sketch, @@ -670,8 +679,8 @@ mod tests { fn rank_candidates( &self, _intent: &AggIntent, - candidates: &[SketchKind], - ) -> Vec { + candidates: &[SketchAlgorithm], + ) -> Vec { candidates.to_vec() } fn cse_recompute_cost(&self, _candidate: &CseCandidate) -> Cost { @@ -713,8 +722,8 @@ mod tests { fn rank_candidates( &self, _intent: &AggIntent, - candidates: &[SketchKind], - ) -> Vec { + candidates: &[SketchAlgorithm], + ) -> Vec { candidates.to_vec() } } diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 1b02cb1f..4fde19e6 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -8,7 +8,7 @@ //! **Common sub-expression elimination (CSE) is not this crate's job.** //! Detection is a primary pass over the pre-ASAP `QueryExpr` IR itself //! (`asap_types::pre_asap`, design tracked in issue #223), run before a -//! tree ever reaches [`replacement::SketchFamilyStrategy`] — see issue #222 +//! tree ever reaches [`replacement::SketchAlgorithmStrategy`] — see issue #222 //! for why (batch query optimization needs to see shared work across a //! `QueryWorkload` before summary binding, not after). This crate may //! eventually run a second, narrower CSE pass of its own over an @@ -37,7 +37,7 @@ //! exhaustive. The `TargetSubDAG`/`ReplacementSubDAG`/ //! `ReplacementStrategy` vocabulary `docs/design_docs/asap_aware_mapping.md` stubs out //! under "Key concepts (not yet implemented)", implemented for real (issue -//! #251, part of #33). [`replacement::SketchFamilyStrategy::replacements`] +//! #251, part of #33). [`replacement::SketchAlgorithmStrategy::replacements`] //! both *decides* what an `AggIntent` may become //! ([`replacement::implementations_for_with`], exhaustive and ranked via a //! `CostModel`, sized to the `AccuracyTarget`) and *constructs* each @@ -105,7 +105,7 @@ //! | **Parse** | parse | text (PromQL/SQL) → AST | `asap-frontend-promql` / `asap-frontend-sql` | //! | **Bind #1** | name resolution | `ColumnRef` (a name) → `ColumnId` (a concrete schema column) — the classic RDBMS "Parse → **Bind** → Optimize" pipeline sense (e.g. SQL Server's query-processor terminology) | [`asap_types::pre_asap::binder::Binder`](https://docs.rs/asap-types) | //! | **Implementation** — `replacement::implementations_for_with` | pre-ASAP → post-ASAP, *one node* | enumerating every concrete physical realization (a sketch family, an exact accumulator, or pass-through) for one [`AggIntent`](asap_types::pre_asap::agg_intent::AggIntent) | [`replacement`] | -//! | **Replacement** — [`replacement::SketchFamilyStrategy::replacements`] | pre-ASAP → post-ASAP, *one target, every candidate* | wrap each `implementations_for_with` candidate into its own bound [`SummaryNode`](asap_types::post_asap::SummaryNode), ranked — a caller wanting one answer takes the first entry itself | [`replacement`] | +//! | **Replacement** — [`replacement::SketchAlgorithmStrategy::replacements`] | pre-ASAP → post-ASAP, *one target, every candidate* | wrap each `implementations_for_with` candidate into its own bound [`SummaryNode`](asap_types::post_asap::SummaryNode), ranked — a caller wanting one answer takes the first entry itself | [`replacement`] | //! | **Search** — [`replacement::search_workload`]/[`replacement::search_workload_with`] | pre-ASAP → post-ASAP, *whole workload, every candidate* | a Cascades/Volcano-style MEMO search: discover every candidate `TargetSubDAG` across a whole workload (not just one target in isolation), run every registered `ReplacementStrategy` against each to a fixpoint, and dedup into a [`replacement::PlanSpace`] — one [`replacement::MemoGroup`] per distinct `TargetSubDAG` holding every alternative discovered for it, never a flat `2^N`-sized list of whole candidate plans | [`replacement`] | //! | **Bind #2** (downstream, not in this crate) | post-ASAP → deployment placement | a *deployment's* own physical binder, deciding **which** candidate to commit to *and* **placement** (edge vs. backend, wire format, …) for a whole workload — a genuinely different, deployment-specific decision this crate doesn't model at all (this is also where a prior workload-wide "keep first/cost-preferred candidate per node" step, `bind::implement_workload`/`implement_workload_with`, would belong if a deployment still wants that exact behavior — it isn't shipped by this crate) | e.g. `control_plane::sketch_algebra::rules::bind_*` (as of this writing; expected to fold into that deployment's cost-model layer rather than stay a separate "bind" concept) | //! @@ -134,5 +134,5 @@ pub use replacement::{ default_strategies, default_strategies_with, search_workload, search_workload_with, summary_candidates, ImplementError, Implementation, Matcher, MemoGroup, PlanSpace, RankedGroup, Replacement, ReplacementStrategy, ReplacementSubDAG, SharedSubtreeStrategy, - SketchFamilyStrategy, TargetSubDAG, MAX_SEARCH_ITERATIONS, + SketchAlgorithmStrategy, TargetSubDAG, MAX_SEARCH_ITERATIONS, }; diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index 2b9bd893..d83f2a1d 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -3,9 +3,9 @@ //! under "Key concepts (not yet implemented)", implemented for real (issue //! #251, part of #33). //! -//! ## One step, not two: `SketchFamilyStrategy::replacements()` decides *and* builds +//! ## One step, not two: `SketchAlgorithmStrategy::replacements()` decides *and* builds //! -//! For a bindable `Aggregate`, `SketchFamilyStrategy::replacements()` is the +//! For a bindable `Aggregate`, `SketchAlgorithmStrategy::replacements()` is the //! single place this crate both decides what an `AggIntent` may become and //! turns each of those candidates into a real, executable //! [`ReplacementSubDAG`]: @@ -63,7 +63,7 @@ //! a whole workload (previously `bind::implement_workload`/ //! `implement_workload_with`) is out of this crate's scope — see the crate //! doc's `## Status` section for why. Every other caller goes through -//! `SketchFamilyStrategy::replacements` directly and decides for itself. +//! `SketchAlgorithmStrategy::replacements` directly and decides for itself. //! //! This means an ordinary single-target bind sizes and fully constructs //! *every* sketch candidate at every sketch-capable node (not just the one a @@ -73,7 +73,7 @@ //! //! ## The two strategies, and why these two //! -//! - [`SketchFamilyStrategy`] wraps [`implementations_for_with`]'s exhaustive, +//! - [`SketchAlgorithmStrategy`] wraps [`implementations_for_with`]'s exhaustive, //! ranked list directly: for the same bindable-`Aggregate` shape this crate //! binds (single intent, no `HAVING`), every entry becomes its own bound //! candidate. @@ -100,7 +100,7 @@ //! unchanged.** Same inputs still produce the same exhaustive, ranked //! list — only its home moved (from a separate `implementation` module //! into this one) and its own visibility dropped to module-private, since -//! [`SketchFamilyStrategy`] is now its only caller. +//! [`SketchAlgorithmStrategy`] is now its only caller. //! //! ## Workload-wide search — merged in from the former `search.rs` (issue #252, part of #33) //! @@ -214,7 +214,7 @@ //! an alternative *for* the target just processed, not a new target of its //! own; see [`discover_new_descendant_targets`]) are scanned for pointers //! not already known, and any found become next round's frontier. Both shipped -//! strategies are idempotent in exactly this sense: [`SketchFamilyStrategy`] +//! strategies are idempotent in exactly this sense: [`SketchAlgorithmStrategy`] //! produces terminal [`Replacement::Summary`] candidates (no `QueryExpr` //! children to scan at all), and [`SharedSubtreeStrategy`]'s two //! [`Replacement::Rewrite`] candidates both reuse the target's own @@ -250,10 +250,10 @@ //! share-vs-recompute pair is ranked by calling //! [`CostModel::cse_share_decision`] via this module's own //! [`cse_preference`] — rather than re-deriving a competing comparison. -//! - A group whose candidates are [`SketchFamilyStrategy`]'s sketch-family +//! - A group whose candidates are [`SketchAlgorithmStrategy`]'s sketch-family //! candidates is ranked via [`CostModel::rank_candidates`] (the same hook //! `implementations_for_with` itself consults), applied to the -//! candidates' own [`SketchKind`]s. +//! candidates' own [`SketchAlgorithm`]s. //! - Any other shape (a single candidate, or a mix this module doesn't have //! a defined comparison for) keeps discovery order — there is nothing to //! rank, or no [`CostModel`] hook this module knows how to apply; it never @@ -262,8 +262,8 @@ use std::collections::HashMap; use asap_types::post_asap::{ - ExactKind, ExactParams, SamplingKind, SamplingParams, SketchKind, SketchParams, - SketchQuery as PostAsapSketchQuery, StatModelKind, StatModelParams, SummaryExpr, + ExactKind, ExactParams, SamplingKind, SamplingParams, SketchAlgorithm, SketchKind, + SketchParams, SketchQuery as PostAsapSketchQuery, StatModelKind, StatModelParams, SummaryExpr, SummaryFamilyType, SummaryField, SummaryNode, SummarySchema, WaveletKind, WaveletParams, }; use asap_types::pre_asap::agg_intent::{agg_is_mergeable, AggIntent}; @@ -303,7 +303,7 @@ pub enum ImplementError { /// reference this exact `Rc` — 1 for an ordinary single-use node, 2+ when the /// caller already ran `share_common_subtrees` and found this subtree shared. /// A strategy that only cares about `root`'s own shape (e.g. -/// [`SketchFamilyStrategy`]) can ignore it entirely; [`SharedSubtreeStrategy`] +/// [`SketchAlgorithmStrategy`]) can ignore it entirely; [`SharedSubtreeStrategy`] /// is the one strategy that consults it. Computing a *real* consumer count /// across a whole workload is a traversal this module deliberately does not /// own (see the module docs' "Non-goals") — [`TargetSubDAG::new`] defaults it @@ -373,7 +373,7 @@ pub struct ReplacementSubDAG { /// of this trait or any existing strategy required. /// /// `replacements` is only meaningful when `matches` would return `true` for -/// the same target; both [`SketchFamilyStrategy`] and [`SharedSubtreeStrategy`] +/// the same target; both [`SketchAlgorithmStrategy`] and [`SharedSubtreeStrategy`] /// return an empty `Vec` rather than panicking when called on a target they /// don't match, so a caller that skips the `matches` check first still gets a /// safe (merely uninformative) answer instead of a crash. @@ -399,7 +399,7 @@ pub trait ReplacementStrategy { /// [`implementations_for_with`] is where every valid realization gets /// enumerated, exhaustive and ranked (most-preferred first) — this crate has /// no separate function that computes just "the one" `Implementation` -/// independently of that list. [`SketchFamilyStrategy`] is the sole +/// independently of that list. [`SketchAlgorithmStrategy`] is the sole /// consumer: it wraps every entry of this list into its own bound /// [`SummaryNode`] and returns all of them, ranked — a caller wanting a /// single answer keeps the first one itself (see the module docs above). @@ -414,11 +414,11 @@ pub enum Implementation { params: ExactParams, }, /// An approximate sketch sized to the intent's [`AccuracyTarget`]. - /// Needs a `SummaryEstimate` readout to recover a value. - Sketch { - kind: SketchKind, - params: SketchParams, - }, + /// Needs a `SummaryEstimate` readout to recover a value. Already + /// classified into its [`SketchKind`] category (`SketchKind::new` + /// having been called) — construction always goes through that + /// classifier, never this variant directly. + Sketch(SketchKind), /// A sampling-based summary (a retained row subset). Needs a /// `SummaryEstimate` readout. Not chosen by any core `AggIntent` /// dispatch today — see the module docs. @@ -490,17 +490,24 @@ pub trait Matcher { pub const DEFAULT_DELTA: f64 = 0.01; /// The sketch kinds that can serve an intent, most-preferred first. -/// This is the `AggIntent → SketchKind` map of issue #98; +/// This is the `AggIntent → SketchAlgorithm` map of issue #98; /// [`implementations_for_with`] sizes and ranks every entry via `cost_model`. /// Listed here so the candidate set has one home. -pub fn summary_candidates(intent: &AggIntent) -> &'static [SketchKind] { +pub fn summary_candidates(intent: &AggIntent) -> &'static [SketchAlgorithm] { match intent { - AggIntent::Quantile { .. } => &[SketchKind::Kll, SketchKind::DDSketch], - AggIntent::Cardinality { .. } => &[SketchKind::Hll, SketchKind::Theta, SketchKind::Kmv], + AggIntent::Quantile { .. } => &[SketchAlgorithm::Kll, SketchAlgorithm::DDSketch], + AggIntent::Cardinality { .. } => &[ + SketchAlgorithm::Hll, + SketchAlgorithm::Theta, + SketchAlgorithm::Kmv, + ], // Count-Sketch-with-heap is CMS-with-heap's balanced/zero-mean-error // alternative for the same heavy-hitter shape. - AggIntent::TopK { .. } => &[SketchKind::CmsWithHeap, SketchKind::CountSketchWithHeap], - AggIntent::Count { .. } => &[SketchKind::Cms, SketchKind::CountSketch], + AggIntent::TopK { .. } => &[ + SketchAlgorithm::CmsWithHeap, + SketchAlgorithm::CountSketchWithHeap, + ], + AggIntent::Count { .. } => &[SketchAlgorithm::Cms, SketchAlgorithm::CountSketch], _ => &[], } } @@ -525,7 +532,7 @@ pub fn accuracy_target(intent: &AggIntent) -> Option<&AccuracyTarget> { /// (most-preferred first via `cost_model`) — the *only* place this crate /// decides what an `AggIntent` may become. Nothing in this crate computes /// "the one" `Implementation` independently of this list: -/// [`SketchFamilyStrategy`] keeps every entry as a candidate, and a caller +/// [`SketchAlgorithmStrategy`] keeps every entry as a candidate, and a caller /// that wants a single executable answer takes the head of *that* strategy's /// output itself. /// @@ -533,7 +540,7 @@ pub fn accuracy_target(intent: &AggIntent) -> Option<&AccuracyTarget> { /// explicit realization is a compile error, and the coverage-matrix test pins /// each variant's category. /// -/// Module-private: [`SketchFamilyStrategy::replacements`] is the only +/// Module-private: [`SketchAlgorithmStrategy::replacements`] is the only /// caller — a caller outside this module has no use for the bare /// `Implementation` list on its own, only for the bound /// [`ReplacementSubDAG`]s that strategy produces from it. @@ -680,9 +687,9 @@ fn sketch_implementations( let ranked = cost_model.rank_candidates(intent, summary_candidates(intent)); ranked .into_iter() - .map(|kind| { - let params = cost_model.size_params(kind.clone(), intent, eps, delta); - Implementation::Sketch { kind, params } + .map(|algorithm| { + let params = cost_model.size_params(algorithm.clone(), intent, eps, delta); + Implementation::Sketch(SketchKind::new(algorithm, params)) }) .collect() } @@ -698,21 +705,21 @@ fn sketch_implementations( /// range. A non-positive ε saturates to the clamp maximum (tightest /// allowed). pub fn default_size_params( - kind: SketchKind, + kind: SketchAlgorithm, intent: &AggIntent, eps: f64, delta: f64, ) -> SketchParams { match kind { - SketchKind::Kll => SketchParams::Kll { k: kll_k(eps) }, - SketchKind::Cms => SketchParams::Cms { + SketchAlgorithm::Kll => SketchParams::Kll { k: kll_k(eps) }, + SketchAlgorithm::Cms => SketchParams::Cms { width: cms_width(eps), depth: cms_depth(delta), }, - SketchKind::Hll => SketchParams::Hll { + SketchAlgorithm::Hll => SketchParams::Hll { precision: hll_precision(eps), }, - SketchKind::CmsWithHeap => { + SketchAlgorithm::CmsWithHeap => { let k = match intent { AggIntent::TopK { k, .. } => *k, _ => unreachable!("CmsWithHeap is only a TopK candidate"), @@ -726,20 +733,20 @@ pub fn default_size_params( // Non-preferred candidates (DDSketch / Theta / Kmv / CountSketch / // CountSketchWithHeap) are only reachable once a cost model picks // them; sized here so that wiring is local. - SketchKind::DDSketch => SketchParams::DDSketch { alpha: eps }, - SketchKind::Theta => SketchParams::Theta { k: kmv_k(eps) }, - SketchKind::Kmv => SketchParams::Kmv { k: kmv_k(eps) }, + SketchAlgorithm::DDSketch => SketchParams::DDSketch { alpha: eps }, + SketchAlgorithm::Theta => SketchParams::Theta { k: kmv_k(eps) }, + SketchAlgorithm::Kmv => SketchParams::Kmv { k: kmv_k(eps) }, // Count-Sketch is CMS's balanced/zero-mean-error alternative — // same (width, depth) shape, sized the same way for now (a // Count-Sketch-specific bound uses an L2-norm error guarantee // rather than CMS's L1-norm one; this is a placeholder pending // that refinement, same status as the other non-preferred // candidates above). - SketchKind::CountSketch => SketchParams::CountSketch { + SketchAlgorithm::CountSketch => SketchParams::CountSketch { width: cms_width(eps), depth: cms_depth(delta), }, - SketchKind::CountSketchWithHeap => { + SketchAlgorithm::CountSketchWithHeap => { let k = match intent { AggIntent::TopK { k, .. } => *k, _ => unreachable!("CountSketchWithHeap is only a TopK candidate"), @@ -810,7 +817,7 @@ pub struct ExpectedCaseSizing { /// stated assumption, not a full redesign of the depth/width tradeoff /// space, so depth relaxation is left as explicit future scope. /// -/// For every `SketchKind` outside the CMS family, this is identical to +/// For every `SketchAlgorithm` outside the CMS family, this is identical to /// [`default_size_params`] — `width_relaxation` only ever touches the /// [`cms_width`]-sized formulas this issue is about. /// @@ -818,7 +825,7 @@ pub struct ExpectedCaseSizing { /// function's existence — this is a separate, additive entry point, never /// called from [`default_size_params`] or [`implementations_for_with`]. pub fn posterior_aware_size_params( - kind: SketchKind, + kind: SketchAlgorithm, intent: &AggIntent, eps: f64, delta: f64, @@ -833,11 +840,11 @@ pub fn posterior_aware_size_params( saturating_ceil(base as f64 * f, 2, base) }; match kind { - SketchKind::Cms => SketchParams::Cms { + SketchAlgorithm::Cms => SketchParams::Cms { width: relaxed_width(eps), depth: cms_depth(delta), }, - SketchKind::CmsWithHeap => { + SketchAlgorithm::CmsWithHeap => { let k = match intent { AggIntent::TopK { k, .. } => *k, _ => unreachable!("CmsWithHeap is only a TopK candidate"), @@ -848,11 +855,11 @@ pub fn posterior_aware_size_params( heap_size: k as u32, } } - SketchKind::CountSketch => SketchParams::CountSketch { + SketchAlgorithm::CountSketch => SketchParams::CountSketch { width: relaxed_width(eps), depth: cms_depth(delta), }, - SketchKind::CountSketchWithHeap => { + SketchAlgorithm::CountSketchWithHeap => { let k = match intent { AggIntent::TopK { k, .. } => *k, _ => unreachable!("CountSketchWithHeap is only a TopK candidate"), @@ -866,14 +873,14 @@ pub fn posterior_aware_size_params( // Every other kind is untouched by this issue's CMS-specific // relaxation — defer to the existing formula verbatim. Spelled out // exhaustively, matching `default_size_params`'s own match, rather - // than a wildcard arm: a future `SketchKind` variant then fails to + // than a wildcard arm: a future `SketchAlgorithm` variant then fails to // compile *here* too, instead of silently inheriting worst-case // sizing with no signal that this function never considered it. - SketchKind::Kll => default_size_params(kind, intent, eps, delta), - SketchKind::Hll => default_size_params(kind, intent, eps, delta), - SketchKind::DDSketch => default_size_params(kind, intent, eps, delta), - SketchKind::Theta => default_size_params(kind, intent, eps, delta), - SketchKind::Kmv => default_size_params(kind, intent, eps, delta), + SketchAlgorithm::Kll => default_size_params(kind, intent, eps, delta), + SketchAlgorithm::Hll => default_size_params(kind, intent, eps, delta), + SketchAlgorithm::DDSketch => default_size_params(kind, intent, eps, delta), + SketchAlgorithm::Theta => default_size_params(kind, intent, eps, delta), + SketchAlgorithm::Kmv => default_size_params(kind, intent, eps, delta), } } @@ -920,9 +927,9 @@ fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { (x.ceil() as u32).clamp(lo, hi) } -// ── SketchFamilyStrategy ───────────────────────────────────────────────── +// ── SketchAlgorithmStrategy ───────────────────────────────────────────────── -/// A single static instance so [`SketchFamilyStrategy::default_cost_model`] +/// A single static instance so [`SketchAlgorithmStrategy::default_cost_model`] /// can hand out a `&'static dyn CostModel` without heap-allocating one — /// `DefaultCostModel` is a unit struct with no state, so one instance serves /// every caller (same pattern `applicability::SketchApplicabilityRule` uses). @@ -934,14 +941,14 @@ static DEFAULT_COST_MODEL: DefaultCostModel = DefaultCostModel; /// /// Ranked (only to *order the enumeration*, never to drop a candidate) via a /// [`CostModel`] — [`DefaultCostModel`] unless constructed with -/// [`SketchFamilyStrategy::new`] — so a deployment-specific cost model's +/// [`SketchAlgorithmStrategy::new`] — so a deployment-specific cost model's /// other hooks (`size_params`, `realize_extension`, `readout_extension`) are /// still consulted while binding each candidate. -pub struct SketchFamilyStrategy<'a> { +pub struct SketchAlgorithmStrategy<'a> { cost_model: &'a dyn CostModel, } -impl SketchFamilyStrategy<'static> { +impl SketchAlgorithmStrategy<'static> { /// A strategy that ranks/binds via the built-in [`DefaultCostModel`] — /// what a deployment gets with no custom cost model plugged in. pub fn default_cost_model() -> Self { @@ -951,7 +958,7 @@ impl SketchFamilyStrategy<'static> { } } -impl<'a> SketchFamilyStrategy<'a> { +impl<'a> SketchAlgorithmStrategy<'a> { /// A strategy that ranks/binds via `cost_model` instead of the built-in /// static preference order — the same customization point /// [`implementations_for_with`] already offers. @@ -960,7 +967,7 @@ impl<'a> SketchFamilyStrategy<'a> { } } -impl ReplacementStrategy for SketchFamilyStrategy<'_> { +impl ReplacementStrategy for SketchAlgorithmStrategy<'_> { fn matches(&self, target: &TargetSubDAG<'_>) -> bool { bindable_intent(target.root).is_some() } @@ -992,10 +999,11 @@ impl ReplacementStrategy for SketchFamilyStrategy<'_> { /// [`ReplacementSubDAG::rationale`] text. fn describe_implementation(intent: &AggIntent, implementation: &Implementation) -> String { match implementation { - Implementation::Sketch { kind, .. } => format!( - "{} realizes as a {kind:?} sketch — one of summary_candidates' \ + Implementation::Sketch(kind) => format!( + "{} realizes as a {:?} sketch — one of summary_candidates' \ alternatives for this intent (asap_aware_mapping::replacement::implementations_for_with)", - describe_intent(intent) + describe_intent(intent), + kind.algorithm() ), Implementation::ExactAggregate { kind, .. } => format!( "{} realizes as an exact {kind:?} accumulator — the only realization \ @@ -1046,7 +1054,7 @@ fn describe_intent(intent: &AggIntent) -> String { // ── realize_child / keep_pre_asap: rank-and-take-first, and its fallback ── /// Rank-and-take-first selector for a single [`QueryExpr`] node: enumerate -/// every candidate via [`SketchFamilyStrategy::replacements`], keep the +/// every candidate via [`SketchAlgorithmStrategy::replacements`], keep the /// `cost_model`-preferred (first) one, and fall back to [`keep_pre_asap`] /// when there's no candidate at all — **not** a general single-answer API /// for a whole workload (that "commit to one final answer" step is a @@ -1064,13 +1072,13 @@ fn describe_intent(intent: &AggIntent) -> String { /// [`crate::cost_model::DefaultCostModel::estimate_cost`] (the same /// representative-node need, for a [`Replacement::Rewrite`] candidate's own /// cost estimate). Every other caller goes through -/// [`SketchFamilyStrategy::replacements`] directly and decides for itself. +/// [`SketchAlgorithmStrategy::replacements`] directly and decides for itself. pub(crate) fn realize_child( root: &Rc, cost_model: &dyn CostModel, ) -> Result, ImplementError> { let target = TargetSubDAG::new(root); - match SketchFamilyStrategy::new(cost_model) + match SketchAlgorithmStrategy::new(cost_model) .replacements(&target) .into_iter() .next() @@ -1083,19 +1091,19 @@ pub(crate) fn realize_child( replacement: Replacement::Rewrite(_), .. }) => { - unreachable!("SketchFamilyStrategy never returns a Rewrite candidate") + unreachable!("SketchAlgorithmStrategy never returns a Rewrite candidate") } // No candidate at all: `root` isn't `bindable_intent` shape (or its // intent has no realization `implementations_for_with` can't // produce — never happens, that match is exhaustive) — the same - // conservative fallback `SketchFamilyStrategy::matches` uses. + // conservative fallback `SketchAlgorithmStrategy::matches` uses. None => keep_pre_asap(root), } } /// Wrap an unrewritten pre-ASAP subtree, lifting its schema with every column /// `SummaryFamilyType::Plain`. `pub` so a caller can fall back to this -/// explicitly — e.g. when `SketchFamilyStrategy::replacements()` returns no +/// explicitly — e.g. when `SketchAlgorithmStrategy::replacements()` returns no /// candidate for a target, or a deployment wants to force a node its own /// runtime can't actually implement — through the same fallback this /// crate's own dispatch uses, without duplicating the schema-lift logic. @@ -1109,7 +1117,7 @@ pub fn keep_pre_asap(expr: &QueryExpr) -> Result, ImplementError // ── Construction: turn one already-decided Implementation into a SummaryNode ─ -/// The bindable shape [`SketchFamilyStrategy`] targets: a single intent, no +/// The bindable shape [`SketchAlgorithmStrategy`] targets: a single intent, no /// `HAVING`. A multi-intent node (SQL `SELECT SUM(a), AVG(b)`), or one with a /// `HAVING` predicate (the filter would need the estimate first), stays /// logical — conservative fallbacks: [`SummaryExpr::KeepPreAsap`] boxes a @@ -1132,7 +1140,7 @@ pub fn bindable_intent(node: &QueryExpr) -> Option<&AggIntent> { /// Construct `expr`'s [`ReplacementSubDAG`] payload for one already-decided /// [`Implementation`] of its top intent — the mechanical half of -/// [`SketchFamilyStrategy::replacements`], called once per candidate that +/// [`SketchAlgorithmStrategy::replacements`], called once per candidate that /// method enumerates. /// /// `expr` must still be the [`bindable_intent`] shape for `implementation` to @@ -1179,7 +1187,7 @@ fn summary_family(implementation: Implementation) -> Option<(SummaryFamilyType, Implementation::ExactAggregate { kind, params } => { (SummaryFamilyType::ExactAggregate(kind, params), false) } - Implementation::Sketch { kind, params } => (SummaryFamilyType::Sketch(kind, params), true), + Implementation::Sketch(kind) => (SummaryFamilyType::Sketch(kind), true), Implementation::Sample { kind, params } => (SummaryFamilyType::Sample(kind, params), true), Implementation::Wavelet { kind, params } => { (SummaryFamilyType::Wavelet(kind, params), true) @@ -1396,7 +1404,7 @@ impl ReplacementStrategy for SharedSubtreeStrategy { /// A generous, documented backstop against a hypothetically ill-behaved /// future [`ReplacementStrategy`] (see the module docs' "Termination" /// section) — not a bound either shipped strategy could ever approach. -/// [`SketchFamilyStrategy`] and [`SharedSubtreeStrategy`] both converge in +/// [`SketchAlgorithmStrategy`] and [`SharedSubtreeStrategy`] both converge in /// exactly 2 passes over a fixed target set, regardless of workload size. pub const MAX_SEARCH_ITERATIONS: usize = 1_000; @@ -1660,12 +1668,12 @@ fn rank_group<'a>(group: &'a MemoGroup, cost_model: &dyn CostModel) -> Vec<&'a R return ranked; } - // Shape 2: `SketchFamilyStrategy`'s sketch-family candidates (every - // candidate is a `Summary` that realizes a `SketchKind`) — rank via + // Shape 2: `SketchAlgorithmStrategy`'s sketch-family candidates (every + // candidate is a `Summary` that realizes a `SketchAlgorithm`) — rank via // `CostModel::rank_candidates`, the same hook `implementations_for_with` // itself consults. if let Some(intent) = bindable_intent(&group.target) { - let kinds: Option> = ranked + let kinds: Option> = ranked .iter() .map(|c| match &c.replacement { Replacement::Summary(node) => sketch_kind_of(node), @@ -1716,7 +1724,7 @@ fn cse_preference(group: &MemoGroup, cost_model: &dyn CostModel) -> Option /// [`cse_preference`] only needs one representative bound [`SummaryNode`] /// for `target` (to build a [`CseCandidate`] for /// [`CostModel::cse_share_decision`]), not the full ranked candidate list -/// [`SketchFamilyStrategy::replacements`] returns — so this just reuses +/// [`SketchAlgorithmStrategy::replacements`] returns — so this just reuses /// [`realize_child`], the same rank-and-take-first helper /// `construct_summary_agg`'s own recursion and /// [`crate::cost_model::DefaultCostModel::estimate_cost`] already use, @@ -1725,24 +1733,24 @@ fn realize_one(target: &Rc, cost_model: &dyn CostModel) -> Option Option { +fn sketch_kind_of(node: &SummaryNode) -> Option { match &node.expr { SummaryExpr::SummaryEstimate { summary_input, .. } => sketch_kind_of(summary_input), SummaryExpr::SummaryAgg { - family: SummaryFamilyType::Sketch(kind, _), + family: SummaryFamilyType::Sketch(kind), .. - } => Some(kind.clone()), + } => Some(kind.algorithm().clone()), _ => None, } } @@ -1757,19 +1765,19 @@ fn sketch_kind_of(node: &SummaryNode) -> Option { /// [`CostModel`] instead. pub fn default_strategies() -> Vec> { vec![ - Box::new(SketchFamilyStrategy::default_cost_model()), + Box::new(SketchAlgorithmStrategy::default_cost_model()), Box::new(SharedSubtreeStrategy), ] } -/// Like [`default_strategies`], but [`SketchFamilyStrategy`] ranks/binds via +/// Like [`default_strategies`], but [`SketchAlgorithmStrategy`] ranks/binds via /// `cost_model` instead of the built-in [`DefaultCostModel`] — the same -/// customization point [`SketchFamilyStrategy::new`] itself offers. +/// customization point [`SketchAlgorithmStrategy::new`] itself offers. pub fn default_strategies_with<'a>( cost_model: &'a dyn CostModel, ) -> Vec> { vec![ - Box::new(SketchFamilyStrategy::new(cost_model)), + Box::new(SketchAlgorithmStrategy::new(cost_model)), Box::new(SharedSubtreeStrategy), ] } @@ -1779,7 +1787,7 @@ pub fn default_strategies_with<'a>( /// Search a whole workload's pre-ASAP roots for every candidate replacement /// [`default_strategies`] can find, deduped into a [`PlanSpace`]. Candidate /// *generation* uses the built-in [`DefaultCostModel`] (via -/// [`default_strategies`], the same way [`SketchFamilyStrategy::default_cost_model`] +/// [`default_strategies`], the same way [`SketchAlgorithmStrategy::default_cost_model`] /// does); call [`PlanSpace::cost_sorted`] on the result for the final /// `sorted_by(cost_model)` step. Use [`search_workload_with`] to plug in a /// custom strategy set (e.g. built via [`default_strategies_with`] for a @@ -1801,7 +1809,7 @@ pub fn search_workload(roots: Vec<(Id, Rc)>) -> PlanSpace { /// section). Deduping candidate plans this way needs no /// [`CostModel`] at all — that only enters at two well-defined points: each /// [`ReplacementStrategy`] in `strategies` may already carry its own (e.g. -/// [`SketchFamilyStrategy::new`]'s), and [`PlanSpace::cost_sorted`]'s final +/// [`SketchAlgorithmStrategy::new`]'s), and [`PlanSpace::cost_sorted`]'s final /// ranking step takes one explicitly. pub fn search_workload_with<'s, Id>( roots: Vec<(Id, Rc)>, @@ -1845,7 +1853,7 @@ pub fn search_workload_with<'s, Id>( "search_workload: fixpoint search did not converge within {MAX_SEARCH_ITERATIONS} \ rounds — a registered ReplacementStrategy's Replacement::Rewrite candidates keep \ exposing new, never-before-seen descendant structure every round. \ - SketchFamilyStrategy/SharedSubtreeStrategy never do this (see replacement.rs's \ + SketchAlgorithmStrategy/SharedSubtreeStrategy never do this (see replacement.rs's \ module docs' \"Termination\" section); check any custom strategies passed to \ search_workload_with.", ); @@ -2037,7 +2045,7 @@ mod tests { /// Shorthand for asserting the realization *category*. #[derive(Debug, PartialEq)] enum Cat { - Sketch(SketchKind), + Sketch(SketchAlgorithm), Acc(ExactKind), Pass, } @@ -2045,7 +2053,7 @@ mod tests { fn cat(intent: &AggIntent) -> Cat { match preferred(intent) { Implementation::ExactAggregate { kind, .. } => Cat::Acc(kind), - Implementation::Sketch { kind, .. } => Cat::Sketch(kind), + Implementation::Sketch(kind) => Cat::Sketch(kind.algorithm().clone()), Implementation::PassThrough => Cat::Pass, other => { panic!("this coverage matrix expects only Exact/Sketch/PassThrough, got {other:?}") @@ -2063,7 +2071,7 @@ mod tests { use AggIntent as A; use Cat::*; use ExactKind as E; - use SketchKind as K; + use SketchAlgorithm as K; let matrix: Vec<(A, Cat)> = vec![ // approximate-capable, at an ε target → sketch (default_quantile(0.99), Sketch(K::Kll)), @@ -2210,10 +2218,10 @@ mod tests { let approx = default_quantile(0.99); // ε = 0.01 assert_eq!( preferred(&approx), - Implementation::Sketch { - kind: SketchKind::Kll, - params: SketchParams::Kll { k: 200 }, // design.md worked example - } + Implementation::Sketch(SketchKind::Quantile( + SketchAlgorithm::Kll, + SketchParams::Kll { k: 200 }, // design.md worked example + )) ); let looser = AggIntent::Quantile { @@ -2223,10 +2231,10 @@ mod tests { }; assert_eq!( preferred(&looser), - Implementation::Sketch { - kind: SketchKind::Kll, - params: SketchParams::Kll { k: 40 }, // ⌈2/0.05⌉ - } + Implementation::Sketch(SketchKind::Quantile( + SketchAlgorithm::Kll, + SketchParams::Kll { k: 40 }, // ⌈2/0.05⌉ + )) ); } @@ -2236,10 +2244,10 @@ mod tests { // sizing must invert it back exactly. assert_eq!( preferred(&default_cardinality()), - Implementation::Sketch { - kind: SketchKind::Hll, - params: SketchParams::Hll { precision: 14 }, - } + Implementation::Sketch(SketchKind::Cardinality( + SketchAlgorithm::Hll, + SketchParams::Hll { precision: 14 }, + )) ); } @@ -2253,13 +2261,13 @@ mod tests { }; assert_eq!( preferred(&intent), - Implementation::Sketch { - kind: SketchKind::Cms, - params: SketchParams::Cms { + Implementation::Sketch(SketchKind::Frequency( + SketchAlgorithm::Cms, + SketchParams::Cms { width: 2719, depth: 7 }, // ⌈e/0.001⌉, ⌈ln 1000⌉ - } + )) ); // Epsilon-only falls back to DEFAULT_DELTA → depth 5. let intent = AggIntent::Count { @@ -2267,13 +2275,13 @@ mod tests { }; assert_eq!( preferred(&intent), - Implementation::Sketch { - kind: SketchKind::Cms, - params: SketchParams::Cms { + Implementation::Sketch(SketchKind::Frequency( + SketchAlgorithm::Cms, + SketchParams::Cms { width: 2719, depth: 5 }, - } + )) ); } @@ -2284,15 +2292,14 @@ mod tests { accuracy: eps(0.01), }; match preferred(&intent) { - Implementation::Sketch { - kind: SketchKind::CmsWithHeap, - params: - SketchParams::CmsWithHeap { - width, - depth, - heap_size, - }, - } => { + Implementation::Sketch(SketchKind::TopK( + SketchAlgorithm::CmsWithHeap, + SketchParams::CmsWithHeap { + width, + depth, + heap_size, + }, + )) => { assert_eq!(heap_size, 25); assert_eq!(width, 272); // ⌈e/0.01⌉ assert_eq!(depth, 5); @@ -2305,24 +2312,31 @@ mod tests { fn candidate_lists_match_the_issue_map() { assert_eq!( summary_candidates(&default_quantile(0.5)), - &[SketchKind::Kll, SketchKind::DDSketch] + &[SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] ); assert_eq!( summary_candidates(&default_cardinality()), - &[SketchKind::Hll, SketchKind::Theta, SketchKind::Kmv] + &[ + SketchAlgorithm::Hll, + SketchAlgorithm::Theta, + SketchAlgorithm::Kmv + ] ); assert_eq!( summary_candidates(&AggIntent::TopK { k: 5, accuracy: eps(0.01) }), - &[SketchKind::CmsWithHeap, SketchKind::CountSketchWithHeap] + &[ + SketchAlgorithm::CmsWithHeap, + SketchAlgorithm::CountSketchWithHeap + ] ); assert_eq!( summary_candidates(&AggIntent::Count { accuracy: eps(0.01) }), - &[SketchKind::Cms, SketchKind::CountSketch] + &[SketchAlgorithm::Cms, SketchAlgorithm::CountSketch] ); assert!(summary_candidates(&AggIntent::Rate).is_empty()); } @@ -2332,15 +2346,15 @@ mod tests { // Quantile's candidate list is [Kll, DDSketch] — implementations_for_with // must return both, ranked with the DefaultCostModel's preferred // (Kll) first. - let kinds: Vec = + let kinds: Vec = implementations_for_with(&default_quantile(0.99), &DefaultCostModel) .into_iter() .map(|implementation| match implementation { - Implementation::Sketch { kind, .. } => kind, + Implementation::Sketch(kind) => kind.algorithm().clone(), other => panic!("expected Sketch, got {other:?}"), }) .collect(); - assert_eq!(kinds, vec![SketchKind::Kll, SketchKind::DDSketch]); + assert_eq!(kinds, vec![SketchAlgorithm::Kll, SketchAlgorithm::DDSketch]); } #[test] @@ -2352,10 +2366,10 @@ mod tests { }; assert_eq!( preferred(&intent), - Implementation::Sketch { - kind: SketchKind::Kll, - params: SketchParams::Kll { k: 65_535 }, - } + Implementation::Sketch(SketchKind::Quantile( + SketchAlgorithm::Kll, + SketchParams::Kll { k: 65_535 }, + )) ); } @@ -2368,9 +2382,9 @@ mod tests { #[test] fn posterior_aware_sizing_shrinks_width_under_stated_assumption() { let intent = count_intent(0.01); - let worst_case = default_size_params(SketchKind::Cms, &intent, 0.01, 0.01); + let worst_case = default_size_params(SketchAlgorithm::Cms, &intent, 0.01, 0.01); let relaxed = posterior_aware_size_params( - SketchKind::Cms, + SketchAlgorithm::Cms, &intent, 0.01, 0.01, @@ -2404,9 +2418,9 @@ mod tests { // width_relaxation = 1.0 must reproduce default_size_params exactly // — the "no risk taken" boundary. let intent = count_intent(0.01); - let worst_case = default_size_params(SketchKind::Cms, &intent, 0.01, 0.01); + let worst_case = default_size_params(SketchAlgorithm::Cms, &intent, 0.01, 0.01); let relaxed = posterior_aware_size_params( - SketchKind::Cms, + SketchAlgorithm::Cms, &intent, 0.01, 0.01, @@ -2420,10 +2434,10 @@ mod tests { #[test] fn posterior_aware_sizing_invalid_relaxation_falls_back_to_worst_case() { let intent = count_intent(0.01); - let worst_case = default_size_params(SketchKind::Cms, &intent, 0.01, 0.01); + let worst_case = default_size_params(SketchAlgorithm::Cms, &intent, 0.01, 0.01); for bad in [0.0, -0.5, 1.5, f64::NAN, f64::INFINITY] { let relaxed = posterior_aware_size_params( - SketchKind::Cms, + SketchAlgorithm::Cms, &intent, 0.01, 0.01, @@ -2450,7 +2464,7 @@ mod tests { // CountSketch assert_eq!( posterior_aware_size_params( - SketchKind::CountSketch, + SketchAlgorithm::CountSketch, &count_intent(0.01), 0.01, 0.01, @@ -2463,7 +2477,7 @@ mod tests { ); // CmsWithHeap / CountSketchWithHeap carry k through untouched. match posterior_aware_size_params( - SketchKind::CmsWithHeap, + SketchAlgorithm::CmsWithHeap, &cms_heap_intent, 0.01, 0.01, @@ -2491,8 +2505,8 @@ mod tests { width_relaxation: 0.1, }; assert_eq!( - posterior_aware_size_params(SketchKind::Kll, &intent, 0.01, 0.01, assumption), - default_size_params(SketchKind::Kll, &intent, 0.01, 0.01), + posterior_aware_size_params(SketchAlgorithm::Kll, &intent, 0.01, 0.01, assumption), + default_size_params(SketchAlgorithm::Kll, &intent, 0.01, 0.01), ); } @@ -2502,7 +2516,7 @@ mod tests { // existing callers must be untouched by adding // posterior_aware_size_params alongside it. assert_eq!( - default_size_params(SketchKind::Cms, &count_intent(0.001), 0.001, 0.001), + default_size_params(SketchAlgorithm::Cms, &count_intent(0.001), 0.001, 0.001), SketchParams::Cms { width: 2719, depth: 7 @@ -2510,7 +2524,7 @@ mod tests { ); } - // ── SketchFamilyStrategy / SharedSubtreeStrategy fixtures ─────────── + // ── SketchAlgorithmStrategy / SharedSubtreeStrategy fixtures ─────────── fn metric_scan(labels: &[&str]) -> QueryExpr { let mut columns = vec![ @@ -2535,18 +2549,18 @@ mod tests { } } - // ── SketchFamilyStrategy ───────────────────────────────────────────── + // ── SketchAlgorithmStrategy ───────────────────────────────────────────── #[test] fn matches_a_bindable_aggregate() { let q = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); let target = TargetSubDAG::new(&q); - assert!(SketchFamilyStrategy::default_cost_model().matches(&target)); + assert!(SketchAlgorithmStrategy::default_cost_model().matches(&target)); } #[test] fn does_not_match_a_multi_intent_or_having_aggregate() { - let strategy = SketchFamilyStrategy::default_cost_model(); + let strategy = SketchAlgorithmStrategy::default_cost_model(); let multi = Rc::new(QueryExpr::Aggregate { reduction: ReductionTy::by(vec![2]), @@ -2575,8 +2589,8 @@ mod tests { fn does_not_match_a_non_aggregate_node() { let scan = Rc::new(metric_scan(&["job"])); let target = TargetSubDAG::new(&scan); - assert!(!SketchFamilyStrategy::default_cost_model().matches(&target)); - assert!(SketchFamilyStrategy::default_cost_model() + assert!(!SketchAlgorithmStrategy::default_cost_model().matches(&target)); + assert!(SketchAlgorithmStrategy::default_cost_model() .replacements(&target) .is_empty()); } @@ -2588,22 +2602,22 @@ mod tests { // not just Kll (the CostModel-ranked head implementations_for_with commits to). let q = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); let target = TargetSubDAG::new(&q); - let replacements = SketchFamilyStrategy::default_cost_model().replacements(&target); + let replacements = SketchAlgorithmStrategy::default_cost_model().replacements(&target); assert_eq!( replacements.len(), 2, "expected 2 candidates, got {replacements:?}" ); - let kinds: Vec = replacements + let kinds: Vec = replacements .iter() .map(|r| match &r.replacement { - Replacement::Summary(node) => summary_family_kind(node), + Replacement::Summary(node) => summary_family_algorithm(node), Replacement::Rewrite(_) => panic!("expected a Summary replacement"), }) .collect(); - assert!(kinds.contains(&SketchKind::Kll), "{kinds:?}"); - assert!(kinds.contains(&SketchKind::DDSketch), "{kinds:?}"); + assert!(kinds.contains(&SketchAlgorithm::Kll), "{kinds:?}"); + assert!(kinds.contains(&SketchAlgorithm::DDSketch), "{kinds:?}"); assert!( replacements.iter().all(|r| !r.rationale.is_empty()), "every candidate must carry a rationale" @@ -2614,17 +2628,21 @@ mod tests { fn cardinality_enumerates_all_three_summary_candidates() { let q = Rc::new(agg(vec![2], default_cardinality(), metric_scan(&["job"]))); let target = TargetSubDAG::new(&q); - let replacements = SketchFamilyStrategy::default_cost_model().replacements(&target); - let kinds: Vec = replacements + let replacements = SketchAlgorithmStrategy::default_cost_model().replacements(&target); + let kinds: Vec = replacements .iter() .map(|r| match &r.replacement { - Replacement::Summary(node) => summary_family_kind(node), + Replacement::Summary(node) => summary_family_algorithm(node), Replacement::Rewrite(_) => panic!("expected a Summary replacement"), }) .collect(); assert_eq!( kinds, - vec![SketchKind::Hll, SketchKind::Theta, SketchKind::Kmv], + vec![ + SketchAlgorithm::Hll, + SketchAlgorithm::Theta, + SketchAlgorithm::Kmv + ], "expected every summary_candidates entry for Cardinality" ); } @@ -2640,7 +2658,7 @@ mod tests { }; let q = Rc::new(agg(vec![2], intent, metric_scan(&["job"]))); let target = TargetSubDAG::new(&q); - let replacements = SketchFamilyStrategy::default_cost_model().replacements(&target); + let replacements = SketchAlgorithmStrategy::default_cost_model().replacements(&target); assert_eq!(replacements.len(), 1, "{replacements:?}"); assert!(matches!( &replacements[0].replacement, @@ -2660,7 +2678,7 @@ mod tests { metric_scan(&["job"]), )); let target = TargetSubDAG::new(&q); - let replacements = SketchFamilyStrategy::default_cost_model().replacements(&target); + let replacements = SketchAlgorithmStrategy::default_cost_model().replacements(&target); assert_eq!(replacements.len(), 1, "{replacements:?}"); assert!(matches!( &replacements[0].replacement, @@ -2680,10 +2698,10 @@ mod tests { fn rank_candidates( &self, _intent: &AggIntent, - candidates: &[SketchKind], - ) -> Vec { + candidates: &[SketchAlgorithm], + ) -> Vec { let mut v = candidates.to_vec(); - if let Some(pos) = v.iter().position(|k| *k == SketchKind::DDSketch) { + if let Some(pos) = v.iter().position(|k| *k == SketchAlgorithm::DDSketch) { let dd = v.remove(pos); v.insert(0, dd); } @@ -2696,16 +2714,16 @@ mod tests { let q = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); let target = TargetSubDAG::new(&q); let custom = PreferDDSketch; - let replacements = SketchFamilyStrategy::new(&custom).replacements(&target); - let kinds: Vec = replacements + let replacements = SketchAlgorithmStrategy::new(&custom).replacements(&target); + let kinds: Vec = replacements .iter() .map(|r| match &r.replacement { - Replacement::Summary(node) => summary_family_kind(node), + Replacement::Summary(node) => summary_family_algorithm(node), Replacement::Rewrite(_) => panic!("expected a Summary replacement"), }) .collect(); - assert!(kinds.contains(&SketchKind::Kll)); - assert!(kinds.contains(&SketchKind::DDSketch)); + assert!(kinds.contains(&SketchAlgorithm::Kll)); + assert!(kinds.contains(&SketchAlgorithm::DDSketch)); assert_eq!(kinds.len(), 2); } @@ -2723,21 +2741,21 @@ mod tests { let inner = agg(vec![2], default_quantile(0.5), metric_scan(&["job"])); let outer = Rc::new(agg(vec![], default_quantile(0.99), inner)); let target = TargetSubDAG::new(&outer); - let replacements = SketchFamilyStrategy::default_cost_model().replacements(&target); + let replacements = SketchAlgorithmStrategy::default_cost_model().replacements(&target); let ddsketch = replacements .iter() .find(|r| { matches!(&r.replacement, Replacement::Summary(node) - if summary_family_kind(node) == SketchKind::DDSketch) + if summary_family_algorithm(node) == SketchAlgorithm::DDSketch) }) .expect("the outer target's DDSketch candidate must be present"); let Replacement::Summary(node) = &ddsketch.replacement else { unreachable!("filtered on Replacement::Summary above"); }; assert_eq!( - summary_family_kind(node), - SketchKind::DDSketch, + summary_family_algorithm(node), + SketchAlgorithm::DDSketch, "the outer (target) node must be the DDSketch candidate" ); @@ -2750,22 +2768,23 @@ mod tests { panic!("expected SummaryAgg, got {:?}", summary_input.expr); }; assert_eq!( - summary_family_kind(child), - SketchKind::Kll, + summary_family_algorithm(child), + SketchAlgorithm::Kll, "the nested inner aggregate must still get the cost-model-ranked \ default (Kll), not inherit the outer target's DDSketch candidate" ); } - /// The `SummaryFamilyType`'s `SketchKind`, from the top `SummaryAgg` - /// reachable under a (possibly `SummaryEstimate`-wrapped) bound root. - fn summary_family_kind(node: &SummaryNode) -> SketchKind { + /// The `SummaryFamilyType`'s committed `SketchAlgorithm`, from the top + /// `SummaryAgg` reachable under a (possibly `SummaryEstimate`-wrapped) + /// bound root. + fn summary_family_algorithm(node: &SummaryNode) -> SketchAlgorithm { match &node.expr { asap_types::post_asap::SummaryExpr::SummaryEstimate { summary_input, .. } => { - summary_family_kind(summary_input) + summary_family_algorithm(summary_input) } asap_types::post_asap::SummaryExpr::SummaryAgg { family, .. } => match family { - asap_types::post_asap::SummaryFamilyType::Sketch(kind, _) => kind.clone(), + asap_types::post_asap::SummaryFamilyType::Sketch(kind) => kind.algorithm().clone(), other => panic!("expected a Sketch family, got {other:?}"), }, other => panic!("expected SummaryAgg/SummaryEstimate, got {other:?}"), @@ -2994,7 +3013,7 @@ mod tests { // Two independently-built, structurally identical Sum aggregates: // share_common_subtrees (run inside search_workload) collapses them // onto one Rc with consumer_count 2, so this single group should - // carry SketchFamilyStrategy's one ExactAggregate candidate *and* + // carry SketchAlgorithmStrategy's one ExactAggregate candidate *and* // SharedSubtreeStrategy's share-vs-recompute pair. let a = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); let b = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); @@ -3163,7 +3182,7 @@ mod tests { // twice for the same target. let root = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); let mut group = MemoGroup::new(Rc::clone(&root), 1); - let strategy = SketchFamilyStrategy::default_cost_model(); + let strategy = SketchAlgorithmStrategy::default_cost_model(); let target = TargetSubDAG::new(&root); for candidate in strategy.replacements(&target) { group.add_candidate(candidate); @@ -3260,10 +3279,10 @@ mod tests { fn rank_candidates( &self, _intent: &AggIntent, - candidates: &[SketchKind], - ) -> Vec { + candidates: &[SketchAlgorithm], + ) -> Vec { let mut v = candidates.to_vec(); - if let Some(pos) = v.iter().position(|k| *k == SketchKind::DDSketch) { + if let Some(pos) = v.iter().position(|k| *k == SketchAlgorithm::DDSketch) { let dd = v.remove(pos); v.insert(0, dd); } @@ -3283,7 +3302,7 @@ mod tests { Replacement::Summary(node) => sketch_kind_of(node), Replacement::Rewrite(_) => None, }; - assert_eq!(first_kind, Some(SketchKind::DDSketch)); + assert_eq!(first_kind, Some(SketchAlgorithm::DDSketch)); } /// [`RankedGroup::costs`] is a per-candidate annotation, aligned @@ -3457,14 +3476,20 @@ mod tests { }; assert_eq!( family, - &SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) + &SummaryFamilyType::Sketch(SketchKind::Quantile( + SketchAlgorithm::Kll, + SketchParams::Kll { k: 200 } + )) ); assert_eq!(col, &ColumnRef::SampleValue); assert_eq!(reduction, &ReductionTy::by(vec![2])); // SummaryAgg edge: the state column carries the committed family. assert_eq!( field(&summary_input.schema, "quantile_0_99").dtype, - SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) + SummaryFamilyType::Sketch(SketchKind::Quantile( + SketchAlgorithm::Kll, + SketchParams::Kll { k: 200 } + )) ); assert!(matches!(child.expr, SummaryExpr::KeepPreAsap(ref e) if matches!(**e, QueryExpr::Scan { .. }))); @@ -3480,10 +3505,10 @@ mod tests { fn rank_candidates( &self, _intent: &AggIntent, - candidates: &[SketchKind], - ) -> Vec { + candidates: &[SketchAlgorithm], + ) -> Vec { let mut v = candidates.to_vec(); - if let Some(pos) = v.iter().position(|k| *k == SketchKind::DDSketch) { + if let Some(pos) = v.iter().position(|k| *k == SketchAlgorithm::DDSketch) { let ddsketch = v.remove(pos); v.insert(0, ddsketch); } @@ -3505,7 +3530,7 @@ mod tests { }; assert!(matches!( family, - SummaryFamilyType::Sketch(SketchKind::Kll, _) + SummaryFamilyType::Sketch(SketchKind::Quantile(SketchAlgorithm::Kll, _)) )); // With `PreferDDSketchViaCostModel`: DDSketch instead, same query. @@ -3518,10 +3543,10 @@ mod tests { }; assert_eq!( family, - &SummaryFamilyType::Sketch( - SketchKind::DDSketch, + &SummaryFamilyType::Sketch(SketchKind::Quantile( + SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha: 0.01 } - ) + )) ); } @@ -3536,8 +3561,8 @@ mod tests { fn rank_candidates( &self, _intent: &AggIntent, - candidates: &[SketchKind], - ) -> Vec { + candidates: &[SketchAlgorithm], + ) -> Vec { candidates.to_vec() } @@ -3547,13 +3572,13 @@ mod tests { _payload: &serde_json::Value, ) -> Implementation { if ext_kind == "frequency" { - Implementation::Sketch { - kind: SketchKind::CountSketch, - params: SketchParams::CountSketch { + Implementation::Sketch(SketchKind::Frequency( + SketchAlgorithm::CountSketch, + SketchParams::CountSketch { width: 256, depth: 4, }, - } + )) } else { Implementation::PassThrough } @@ -3615,13 +3640,13 @@ mod tests { }; assert_eq!( family, - &SummaryFamilyType::Sketch( - SketchKind::CountSketch, + &SummaryFamilyType::Sketch(SketchKind::Frequency( + SketchAlgorithm::CountSketch, SketchParams::CountSketch { width: 256, depth: 4 } - ) + )) ); } @@ -3744,7 +3769,7 @@ mod tests { }; assert!(matches!( family, - SummaryFamilyType::Sketch(SketchKind::Kll, _) + SummaryFamilyType::Sketch(SketchKind::Quantile(SketchAlgorithm::Kll, _)) )); let SummaryExpr::SummaryAgg { family: inner_family, @@ -3887,7 +3912,10 @@ mod tests { assert!(matches!( &summary_input.expr, SummaryExpr::SummaryAgg { - family: SummaryFamilyType::Sketch(SketchKind::CmsWithHeap, _), + family: SummaryFamilyType::Sketch(SketchKind::TopK( + SketchAlgorithm::CmsWithHeap, + _ + )), .. } )); diff --git a/crates/devtools/src/bin/show_post_asap_ir.rs b/crates/devtools/src/bin/show_post_asap_ir.rs index 5aa97bf4..2e49b573 100644 --- a/crates/devtools/src/bin/show_post_asap_ir.rs +++ b/crates/devtools/src/bin/show_post_asap_ir.rs @@ -22,7 +22,7 @@ use asap_aware_mapping::replacement::keep_pre_asap; use asap_aware_mapping::{ - Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG, + Replacement, ReplacementStrategy, ReplacementSubDAG, SketchAlgorithmStrategy, TargetSubDAG, }; use asap_devtools::{lower_promql, lower_sql, SqlCatalog}; use asap_types::pre_asap::query_expr::QueryExpr; @@ -34,7 +34,7 @@ use std::rc::Rc; const ACCURACY: AccuracyTarget = AccuracyTarget::Epsilon(0.01); /// `asap-aware-mapping` has no "bind me one tree" public API any more — -/// `SketchFamilyStrategy::replacements` always returns every candidate, and +/// `SketchAlgorithmStrategy::replacements` always returns every candidate, and /// a caller decides what to keep. This debug tool just wants one /// representative binding per query, so it takes the first /// (`cost_model`-preferred) candidate the same way a production caller @@ -42,7 +42,7 @@ const ACCURACY: AccuracyTarget = AccuracyTarget::Epsilon(0.01); fn bind(expr: &QueryExpr) -> Result, String> { let root = Rc::new(expr.clone()); let target = TargetSubDAG::new(&root); - match SketchFamilyStrategy::default_cost_model() + match SketchAlgorithmStrategy::default_cost_model() .replacements(&target) .into_iter() .next() diff --git a/crates/frontend-promql/tests/observability/promql_corpus.rs b/crates/frontend-promql/tests/observability/promql_corpus.rs index 2aba0cd1..99599952 100644 --- a/crates/frontend-promql/tests/observability/promql_corpus.rs +++ b/crates/frontend-promql/tests/observability/promql_corpus.rs @@ -17,7 +17,7 @@ use std::rc::Rc; use asap_aware_mapping::replacement::{keep_pre_asap, ImplementError}; use asap_aware_mapping::{ - Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG, + Replacement, ReplacementStrategy, ReplacementSubDAG, SketchAlgorithmStrategy, TargetSubDAG, }; use asap_frontend_promql::{lower_promql, PromqlError as LoweringError}; use asap_types::post_asap::{SummaryExpr, SummaryNode}; @@ -25,7 +25,7 @@ use asap_types::pre_asap::query_expr::QueryExpr; use asap_types::types::AccuracyTarget; /// This crate has no "bind me one tree" public API any more — -/// `SketchFamilyStrategy::replacements` always returns every candidate, and +/// `SketchAlgorithmStrategy::replacements` always returns every candidate, and /// a caller decides what to keep. This test-only helper reproduces the /// take-the-first-(`cost_model`-preferred)-candidate pattern so [`bind_tally`] /// gets one representative `Result` per query, matching what a totality @@ -33,7 +33,7 @@ use asap_types::types::AccuracyTarget; fn bind(expr: &QueryExpr) -> Result, ImplementError> { let root = Rc::new(expr.clone()); let target = TargetSubDAG::new(&root); - match SketchFamilyStrategy::default_cost_model() + match SketchAlgorithmStrategy::default_cost_model() .replacements(&target) .into_iter() .next() diff --git a/crates/integration-tests/tests/cse.rs b/crates/integration-tests/tests/cse.rs index 29bc2547..c42226f8 100644 --- a/crates/integration-tests/tests/cse.rs +++ b/crates/integration-tests/tests/cse.rs @@ -67,7 +67,7 @@ fn duplicate_workload_queries_collapse_onto_one_memo_group() { ); // The single shared root is one discovered TargetSubDAG, holding one - // MemoGroup with consumer_count 2 — SketchFamilyStrategy's one + // MemoGroup with consumer_count 2 — SketchAlgorithmStrategy's one // ExactAggregate candidate *and* SharedSubtreeStrategy's share-vs- // recompute pair, exactly as `shared_aggregate_across_two_roots_gets_both_strategies_candidates` // (asap-aware-mapping::replacement's own equivalent, internal test) diff --git a/crates/integration-tests/tests/l4_binding.rs b/crates/integration-tests/tests/l4_binding.rs index b542e39f..f6bb2ab2 100644 --- a/crates/integration-tests/tests/l4_binding.rs +++ b/crates/integration-tests/tests/l4_binding.rs @@ -2,7 +2,7 @@ //! //! Drives the full pipeline — PromQL text → pre-ASAP `QueryExpr` //! (`lower_promql`) → post-ASAP `SummaryExpr` DAG (via -//! `SketchFamilyStrategy::replacements`, see [`bind`] below) — and pins the +//! `SketchAlgorithmStrategy::replacements`, see [`bind`] below) — and pins the //! summary-bound shape node by node, including the family `(Kind, Params)` //! committed on each edge's schema. This is the design doc's §"L4 — sketch //! algebra" worked example, running for real. @@ -11,12 +11,12 @@ use std::rc::Rc; use asap_aware_mapping::replacement::{keep_pre_asap, ImplementError}; use asap_aware_mapping::{ - Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG, + Replacement, ReplacementStrategy, ReplacementSubDAG, SketchAlgorithmStrategy, TargetSubDAG, }; use asap_frontend_promql::lower_promql; use asap_types::post_asap::{ - ExactKind, ExactParams, SketchKind, SketchParams, SketchQuery, SummaryExpr, SummaryFamilyType, - SummaryNode, SummarySchema, + ExactKind, ExactParams, SketchAlgorithm, SketchKind, SketchParams, SketchQuery, SummaryExpr, + SummaryFamilyType, SummaryNode, SummarySchema, }; use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::query_expr::{QueryExpr, Reduction}; @@ -24,14 +24,14 @@ use asap_types::pre_asap::schema::DataType; use asap_types::types::AccuracyTarget; /// This crate has no "bind me one tree" public API any more — -/// `SketchFamilyStrategy::replacements` always returns every candidate, and +/// `SketchAlgorithmStrategy::replacements` always returns every candidate, and /// a caller decides what to keep. This test-only helper reproduces the /// take-the-first-(`cost_model`-preferred)-candidate pattern so the /// single-answer pins below don't all repeat it by hand. fn bind(expr: &QueryExpr) -> Result, ImplementError> { let root = Rc::new(expr.clone()); let target = TargetSubDAG::new(&root); - match SketchFamilyStrategy::default_cost_model() + match SketchAlgorithmStrategy::default_cost_model() .replacements(&target) .into_iter() .next() @@ -105,7 +105,10 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { }; assert_eq!( family, - &SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) + &SummaryFamilyType::Sketch(SketchKind::Quantile( + SketchAlgorithm::Kll, + SketchParams::Kll { k: 200 } + )) ); assert_eq!(col, &ColumnRef::SampleValue); assert_eq!( @@ -115,7 +118,10 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { ); assert_eq!( dtype(&summary_input.schema, "quantile_0_99"), - &SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) + &SummaryFamilyType::Sketch(SketchKind::Quantile( + SketchAlgorithm::Kll, + SketchParams::Kll { k: 200 } + )) ); // The rate: exact counter-reset-aware accumulator, per-series (labels diff --git a/crates/integration-tests/tests/l4_binding_sql.rs b/crates/integration-tests/tests/l4_binding_sql.rs index 322dc424..456a184a 100644 --- a/crates/integration-tests/tests/l4_binding_sql.rs +++ b/crates/integration-tests/tests/l4_binding_sql.rs @@ -1,7 +1,7 @@ //! End-to-end SQL query-string → post-ASAP IR pin (issue #191). //! //! The SQL counterpart of `l4_binding.rs`: drives SQL text — `lower_sql` -//! (text → pre-ASAP `QueryExpr`) → `SketchFamilyStrategy::replacements` +//! (text → pre-ASAP `QueryExpr`) → `SketchAlgorithmStrategy::replacements` //! (pre-ASAP → post-ASAP `SummaryExpr`, see [`bind`] below) — and pins the //! resulting sketch-vs-exact-accumulator binding shape node by node, the way //! `l4_binding.rs` does for PromQL. @@ -32,12 +32,12 @@ use std::rc::Rc; use asap_aware_mapping::replacement::{keep_pre_asap, ImplementError}; use asap_aware_mapping::{ - Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG, + Replacement, ReplacementStrategy, ReplacementSubDAG, SketchAlgorithmStrategy, TargetSubDAG, }; use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::post_asap::{ - ExactKind, ExactParams, SketchKind, SketchParams, SketchQuery, SummaryExpr, SummaryFamilyType, - SummaryNode, SummarySchema, + ExactKind, ExactParams, SketchAlgorithm, SketchKind, SketchParams, SketchQuery, SummaryExpr, + SummaryFamilyType, SummaryNode, SummarySchema, }; use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::query_expr::{QueryExpr, Reduction}; @@ -45,14 +45,14 @@ use asap_types::pre_asap::schema::{Column, DataType, Schema}; use asap_types::types::AccuracyTarget; /// This crate has no "bind me one tree" public API any more — -/// `SketchFamilyStrategy::replacements` always returns every candidate, and +/// `SketchAlgorithmStrategy::replacements` always returns every candidate, and /// a caller decides what to keep. This test-only helper reproduces the /// take-the-first-(`cost_model`-preferred)-candidate pattern so the /// single-answer pins below don't all repeat it by hand. fn bind(expr: &QueryExpr) -> Result, ImplementError> { let root = Rc::new(expr.clone()); let target = TargetSubDAG::new(&root); - match SketchFamilyStrategy::default_cost_model() + match SketchAlgorithmStrategy::default_cost_model() .replacements(&target) .into_iter() .next() @@ -190,7 +190,10 @@ async fn sql_quantile_binds_kll_sketch_over_named_column() { }; assert_eq!( family, - &SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) + &SummaryFamilyType::Sketch(SketchKind::Quantile( + SketchAlgorithm::Kll, + SketchParams::Kll { k: 200 } + )) ); assert_eq!( col, @@ -207,7 +210,10 @@ async fn sql_quantile_binds_kll_sketch_over_named_column() { ); assert_eq!( summary_input.schema.fields[0].dtype, - SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) + SummaryFamilyType::Sketch(SketchKind::Quantile( + SketchAlgorithm::Kll, + SketchParams::Kll { k: 200 } + )) ); let SummaryExpr::KeepPreAsap(kept_leaf) = &child.expr else { @@ -265,7 +271,10 @@ async fn sql_count_distinct_binds_hll_sketch_over_named_column() { }; assert_eq!( family, - &SummaryFamilyType::Sketch(SketchKind::Hll, SketchParams::Hll { precision: 14 }) + &SummaryFamilyType::Sketch(SketchKind::Cardinality( + SketchAlgorithm::Hll, + SketchParams::Hll { precision: 14 } + )) ); assert_eq!( col, diff --git a/crates/types/src/post_asap/mod.rs b/crates/types/src/post_asap/mod.rs index d449dc96..6ca27740 100644 --- a/crates/types/src/post_asap/mod.rs +++ b/crates/types/src/post_asap/mod.rs @@ -3,14 +3,19 @@ //! //! Where [`crate::pre_asap`] carries *intent* only ("compute a //! quantile to ε accuracy"), this module is the summary-bound IR: the -//! summary family, kind, and parameters are committed (one `(Kind, Params)` -//! pair per family — [`sketch::ExactKind`]/[`sketch::ExactParams`], -//! [`sketch::SketchKind`]/[`sketch::SketchParams`], +//! summary family, kind/algorithm, and parameters are committed (one +//! `(Kind, Params)` pair per family — [`sketch::ExactKind`]/[`sketch::ExactParams`], //! [`sketch::SamplingKind`]/[`sketch::SamplingParams`], //! [`sketch::WaveletKind`]/[`sketch::WaveletParams`], //! [`sketch::StatModelKind`]/[`sketch::StatModelParams`]), and //! [`expr::SummaryNode`] / [`expr::SummaryExpr`] describe the summary -//! computation. +//! computation. The `Sketch` family is the one exception to that +//! one-pair-per-family shape: it nests a third level, [`sketch::SketchKind`] +//! (quantile/cardinality/frequency/top-k), which itself carries the +//! committed [`sketch::SketchAlgorithm`] and [`sketch::SketchParams`] — +//! `SummaryFamilyType::Sketch(SketchKind)`, not a flat `(kind, params)` pair +//! — because `Sketch` is the one family with more than one algorithm per +//! purpose today; no other family needs that extra level yet. pub mod expr; pub mod query_time; @@ -24,6 +29,6 @@ pub use query_time::{ }; pub use schema::{SummaryFamilyType, SummaryField, SummarySchema}; pub use sketch::{ - ExactKind, ExactParams, SamplingKind, SamplingParams, SketchKind, SketchParams, SketchQuery, - StatModelKind, StatModelParams, WaveletKind, WaveletParams, + ExactKind, ExactParams, SamplingKind, SamplingParams, SketchAlgorithm, SketchKind, + SketchParams, SketchQuery, StatModelKind, StatModelParams, WaveletKind, WaveletParams, }; diff --git a/crates/types/src/post_asap/query_time/error_estimation.rs b/crates/types/src/post_asap/query_time/error_estimation.rs index c343214a..2bd9e603 100644 --- a/crates/types/src/post_asap/query_time/error_estimation.rs +++ b/crates/types/src/post_asap/query_time/error_estimation.rs @@ -46,7 +46,7 @@ //! plan-time sizing. As of this module landing, **this repository has no //! vendored CMS/CountSketch/CU-Sketch runtime and no counter-array data //! structure anywhere** — confirmed by inspecting -//! `crates/types/src/post_asap/sketch.rs` (`SketchKind`, `SketchParams`, +//! `crates/types/src/post_asap/sketch.rs` (`SketchAlgorithm`, `SketchParams`, //! this module's neighbors) and `crates/asap-aware-mapping/src/implementation.rs` //! (`default_size_params`, `cms_width`, `cms_depth`): both are purely //! planning-time sizing metadata. There is no `A[row][col]` counter matrix @@ -116,7 +116,7 @@ pub fn cu_sketch_posterior_error_bound(row: &[u64], rows: u32, delta: f64) -> Op /// /// Count-Sketch counters are signed (each row's hash also picks a random /// sign, so a flow's contribution can subtract as well as add — "balanced/ -/// zero-mean-error" per `SketchKind::CountSketch`'s own doc), and a flow's +/// zero-mean-error" per `SketchAlgorithm::CountSketch`'s own doc), and a flow's /// size is estimated as the *median*, not the minimum, of its `r` per-row /// counters. That breaks Algorithm 1/2's simple `p = δ^(1/r)` derivation (a /// union bound over "any row is bad"): the median needs *more than half* the diff --git a/crates/types/src/post_asap/schema.rs b/crates/types/src/post_asap/schema.rs index 20599137..0f51b42b 100644 --- a/crates/types/src/post_asap/schema.rs +++ b/crates/types/src/post_asap/schema.rs @@ -1,5 +1,5 @@ use super::sketch::{ - ExactKind, ExactParams, SamplingKind, SamplingParams, SketchKind, SketchParams, StatModelKind, + ExactKind, ExactParams, SamplingKind, SamplingParams, SketchKind, StatModelKind, StatModelParams, WaveletKind, WaveletParams, }; use crate::pre_asap::DataType; @@ -27,8 +27,11 @@ pub enum SummaryFamilyType { /// `Increase`) — the partial state *is* the value; no readout needed. ExactAggregate(ExactKind, ExactParams), /// Approximate sketch state (KLL/CMS/HLL/…), read out via a - /// `SummaryEstimate`. - Sketch(SketchKind, SketchParams), + /// `SummaryEstimate`. A [`SketchKind`] already carries the concrete + /// algorithm and params committed to, not just its category — a bound + /// node needs to know it's specifically KLL, not merely "some quantile + /// sketch". + Sketch(SketchKind), /// Sampling-based summary state (a retained row subset). Sample(SamplingKind, SamplingParams), /// Wavelet-transform summary state (a coefficient vector). diff --git a/crates/types/src/post_asap/sketch.rs b/crates/types/src/post_asap/sketch.rs index 88d358bc..b0458780 100644 --- a/crates/types/src/post_asap/sketch.rs +++ b/crates/types/src/post_asap/sketch.rs @@ -34,10 +34,12 @@ pub enum ExactParams { // ── Approximate sketches ───────────────────────────────────────────────────── -/// An approximate, mergeable sketch family — bounded error, sized by its -/// [`SketchParams`]. +/// A specific sketch algorithm — bounded error, sized by its +/// [`SketchParams`]. Each algorithm belongs to exactly one [`SketchKind`] +/// category (e.g. `Kll` and `DDSketch` both realize `SketchKind::Quantile`); +/// [`SketchKind::new`] is where that classification is made. #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum SketchKind { +pub enum SketchAlgorithm { /// KLL quantile sketch (mergeable, ε-accurate rank queries). Kll, /// Count-Min Sketch (mergeable, (ε,δ)-accurate frequency queries). @@ -61,8 +63,8 @@ pub enum SketchKind { CountSketchWithHeap, } -/// Concrete, catalog-validated parameters for a specific [`SketchKind`] -/// instance. The variant must correspond to the associated `SketchKind`; +/// Concrete, catalog-validated parameters for a specific [`SketchAlgorithm`] +/// instance. The variant must correspond to the associated `SketchAlgorithm`; /// mismatches are caught at post-ASAP bind time, before any later, /// deployment-specific stage ever sees the plan. #[derive(Debug, Clone, PartialEq)] @@ -102,6 +104,76 @@ pub enum SketchParams { }, } +/// A committed sketch choice: which *category* of query shape it answers — +/// quantile-style, cardinality-style, frequency-style, or heavy-hitter/ +/// top-k-style estimation — together with the concrete [`SketchAlgorithm`] +/// and [`SketchParams`] realizing it. Sits between +/// [`SummaryFamilyType::Sketch`](super::schema::SummaryFamilyType::Sketch) +/// (the `Sketch` family as a whole, sibling to `Sample`/`Wavelet`/ +/// `StatModel`) and the bare algorithm: `Kll` vs. `DDSketch` is a choice +/// *within* `Quantile`, not a choice *of* `SketchKind` — every `Quantile` +/// value already carries which of the two (and its params) was picked. +/// +/// [`SketchKind::new`] is the one place `(SketchAlgorithm, SketchParams)` +/// pairs get classified into a category; construct through it rather than +/// naming a variant directly, so a new algorithm can't drift out of sync +/// with its category. See `implementation::summary_candidates` for the +/// `AggIntent -> [SketchAlgorithm]` candidate list this ultimately groups. +#[derive(Debug, Clone, PartialEq)] +pub enum SketchKind { + /// Approximate rank/percentile queries (e.g. p99 latency). + Quantile(SketchAlgorithm, SketchParams), + /// Approximate distinct-element counting. + Cardinality(SketchAlgorithm, SketchParams), + /// Approximate point/frequency counting (e.g. per-key event counts). + Frequency(SketchAlgorithm, SketchParams), + /// Approximate heavy-hitter / top-k queries. + TopK(SketchAlgorithm, SketchParams), +} + +impl SketchKind { + /// Classify `(algorithm, params)` into its `SketchKind` category. The + /// one place that mapping is made — every other piece of this crate + /// that needs to know an algorithm's category goes through this rather + /// than re-deriving it. + pub fn new(algorithm: SketchAlgorithm, params: SketchParams) -> Self { + match algorithm { + SketchAlgorithm::Kll | SketchAlgorithm::DDSketch => { + SketchKind::Quantile(algorithm, params) + } + SketchAlgorithm::Hll | SketchAlgorithm::Theta | SketchAlgorithm::Kmv => { + SketchKind::Cardinality(algorithm, params) + } + SketchAlgorithm::Cms | SketchAlgorithm::CountSketch => { + SketchKind::Frequency(algorithm, params) + } + SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap => { + SketchKind::TopK(algorithm, params) + } + } + } + + /// The algorithm this kind committed to, regardless of category. + pub fn algorithm(&self) -> &SketchAlgorithm { + match self { + SketchKind::Quantile(a, _) + | SketchKind::Cardinality(a, _) + | SketchKind::Frequency(a, _) + | SketchKind::TopK(a, _) => a, + } + } + + /// The parameters this kind committed to, regardless of category. + pub fn params(&self) -> &SketchParams { + match self { + SketchKind::Quantile(_, p) + | SketchKind::Cardinality(_, p) + | SketchKind::Frequency(_, p) + | SketchKind::TopK(_, p) => p, + } + } +} + // ── Sampling summaries ─────────────────────────────────────────────────────── /// A sampling-based summary family — retains an actual (weighted) subset of diff --git a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md index 55c9b16c..ad584585 100644 --- a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md @@ -10,7 +10,7 @@ This guide explains how to extend ASAP-aware mapping in the current codebase. Us The focus here is the **current code interfaces and their contracts**. For the higher-level motivation and the replacement-plan-searching design this crate now implements, see the separate design document, [`docs/design_docs/asap_aware_mapping.md`](../design_docs/asap_aware_mapping.md). -Names such as `MyStrategy`, `MyCostModel`, and `PreferDDSketch` are illustrative; they do not ship with this crate. Samples that use real types—such as `SketchFamilyStrategy`, `SharedSubtreeStrategy`, `implementations_for_with`, `PlanSpace`, and `search_workload`—are copied from `replacement.rs`, `bind.rs`, or `cost_model.rs`. +Names such as `MyStrategy`, `MyCostModel`, and `PreferDDSketch` are illustrative; they do not ship with this crate. Samples that use real types—such as `SketchAlgorithmStrategy`, `SharedSubtreeStrategy`, `implementations_for_with`, `PlanSpace`, and `search_workload`—are copied from `replacement.rs`, `bind.rs`, or `cost_model.rs`. If you only need to find the right extension point, start with the [extension map](#18-current-extension-map). If you are implementing a strategy, read sections 1–5 first. @@ -20,7 +20,7 @@ If you only need to find the right extension point, start with the [extension ma One term is central to this guide: -- **Implementation**: one valid realization of an `AggIntent`. It may be an approximate sketch, an exact mergeable accumulator, or a pass-through with no summary. `implementations_for_with` (a `replacement.rs`-private function) enumerates the valid implementations for one node and ranks them. It does not walk the plan or select a winner. `SketchFamilyStrategy::replacements()` is its only caller: for one target, it constructs every candidate's `SummaryNode` directly and returns all of them — deciding and constructing happen in the same step, not two steps bridged by a separately-named function in another module. A caller that needs one result takes the first candidate. At workload scale, `search_workload`/`search_workload_with` extend the same "never prune" contract across every `TargetSubDAG` in the whole workload (see §3) — this crate stops at that full candidate space plus cost; it does not itself commit to one final, physically-shared answer per site. That commitment (which candidate to build, where to place it) is a downstream deployment's call, not this crate's. +- **Implementation**: one valid realization of an `AggIntent`. It may be an approximate sketch, an exact mergeable accumulator, or a pass-through with no summary. `implementations_for_with` (a `replacement.rs`-private function) enumerates the valid implementations for one node and ranks them. It does not walk the plan or select a winner. `SketchAlgorithmStrategy::replacements()` is its only caller: for one target, it constructs every candidate's `SummaryNode` directly and returns all of them — deciding and constructing happen in the same step, not two steps bridged by a separately-named function in another module. A caller that needs one result takes the first candidate. At workload scale, `search_workload`/`search_workload_with` extend the same "never prune" contract across every `TargetSubDAG` in the whole workload (see §3) — this crate stops at that full candidate space plus cost; it does not itself commit to one final, physically-shared answer per site. That commitment (which candidate to build, where to place it) is a downstream deployment's call, not this crate's. In short: `ReplacementStrategy` enumerates, packages, and binds every candidate, and the caller selects when it needs a single executable answer. Section 3 shows the complete flow. @@ -183,7 +183,7 @@ The crate cannot hardcode real deployment costs: `asap-plan` depends on `asap_ir | Hook | Use it to | Default? | |---|---|---| -| `rank_candidates` | Order valid sketch families | No | +| `rank_candidates` | Order valid sketch algorithms | No | | `size_params` | Convert an accuracy target into sketch parameters | Yes | | `realize_extension` | Map a custom intent to an implementation | Yes | | `readout_extension` | Query a custom extension summary | Panics until paired with a custom realization | @@ -194,13 +194,13 @@ The crate cannot hardcode real deployment costs: `asap-plan` depends on `asap_ir - **`rank_candidates`** — order the sketch candidates for one `AggIntent`, best first. This is the only required hook. ```rust - fn rank_candidates(&self, intent: &AggIntent, candidates: &[SketchKind]) -> Vec; + fn rank_candidates(&self, intent: &AggIntent, candidates: &[SketchAlgorithm]) -> Vec; ``` -- **`size_params`** — choose parameters, such as sketch capacity, for an already-selected `SketchKind` and accuracy target `(eps, delta)`. It is separate from ranking so a deployment can customize sizing without changing family selection. The default is `replacement::default_size_params`. +- **`size_params`** — choose parameters, such as sketch capacity, for an already-selected `SketchAlgorithm` and accuracy target `(eps, delta)`. It is separate from ranking so a deployment can customize sizing without changing family selection. The default is `replacement::default_size_params`. ```rust - fn size_params(&self, kind: SketchKind, intent: &AggIntent, eps: f64, delta: f64) -> SketchParams; + fn size_params(&self, kind: SketchAlgorithm, intent: &AggIntent, eps: f64, delta: f64) -> SketchParams; ``` - **`realize_extension`** — map a deployment-defined `AggIntent::Extension` to a post-ASAP `Implementation`. The default is `Implementation::PassThrough`. @@ -210,7 +210,7 @@ The crate cannot hardcode real deployment costs: `asap-plan` depends on `asap_ir ```rust fn realize_extension(&self, ext_kind: &str, _payload: &serde_json::Value) -> Implementation { if ext_kind == "frequency" { - Implementation::Sketch { kind: SketchKind::CountSketch, params: /* ... */ } + Implementation::Sketch(SketchKind::Frequency(SketchAlgorithm::CountSketch, /* params */)) } else { Implementation::PassThrough // fall back to the default for anything else } @@ -253,11 +253,40 @@ A custom cost model does not necessarily need to override every hook. The curren --- +### Family, kind, and algorithm + +Three levels sit below "summary" in this crate's type vocabulary, and `Sketch` is the only family with all three: + +| Level | Type | Example | +| --- | --- | --- | +| **family** | `SummaryFamilyType` | `Sketch`, `Sample`, `Wavelet`, `StatModel`, `ExactAggregate` | +| **kind** | `SketchKind` (only inside `Sketch`) | `Quantile`, `Cardinality`, `Frequency`, `TopK` | +| **algorithm** | `SketchAlgorithm` (nested inside a `SketchKind`) | `Kll` / `DDSketch` (both `Quantile`); `Hll` / `Theta` / `Kmv` (all `Cardinality`) | + +A `SketchKind` isn't just a category tag — every value already carries the committed algorithm and its params: + +```rust +pub enum SketchKind { + Quantile(SketchAlgorithm, SketchParams), + Cardinality(SketchAlgorithm, SketchParams), + Frequency(SketchAlgorithm, SketchParams), + TopK(SketchAlgorithm, SketchParams), +} +``` + +`SketchKind::new(algorithm, params)` is the one place an `(algorithm, params)` pair gets classified into its category — construct through it rather than naming a variant directly, so a new algorithm can't drift out of sync with its category. `.algorithm()`/`.params()` pull the committed pair back out regardless of which category variant it's in. + +Where this matters in practice: `CostModel::rank_candidates`/`size_params`, `implementation::implementations_for_with`, and `SketchAlgorithmStrategy`'s candidate enumeration all operate one level down, at **algorithm** — `summary_candidates(intent)` returns a list of `SketchAlgorithm`s (`[Kll, DDSketch]` for a `Quantile` intent), never a bare `SketchKind` with nothing chosen underneath it. `SketchKind` only shows up once an algorithm has actually been picked and sized — on `Implementation::Sketch(SketchKind)` and `SummaryFamilyType::Sketch(SketchKind)`, both single-field wrapping the already-committed kind. + +No other family needs this extra level today — `Sample`/`Wavelet`/`StatModel` are each a flat `(Kind, Params)` pair, same shape `Sketch` used to be before this split. `Sketch` grew a third level because it's the one family with more than one algorithm per purpose (KLL vs. DDSketch both answer `Quantile`). + +--- + ## 3. How the current pieces fit together The crate has one source of truth for valid implementations, and deciding and constructing a candidate happen in the same step — not two steps bridged by a separately-named function: -- `SketchFamilyStrategy::replacements(target)`, for a bindable `Aggregate`, calls `implementations_for_with(intent, cost_model)` (a `replacement.rs`-private function) to get every valid `Implementation`, ranked by preference and already sized to the target's own accuracy target — then, for each candidate, constructs its bound `SummaryNode` directly (child schema, summarized column, readout, recursion into the child) and returns it as a `ReplacementSubDAG`. It does not discard any candidate. +- `SketchAlgorithmStrategy::replacements(target)`, for a bindable `Aggregate`, calls `implementations_for_with(intent, cost_model)` (a `replacement.rs`-private function) to get every valid `Implementation`, ranked by preference and already sized to the target's own accuracy target — then, for each candidate, constructs its bound `SummaryNode` directly (child schema, summarized column, readout, recursion into the child) and returns it as a `ReplacementSubDAG`. It does not discard any candidate. - A caller that needs one executable answer uses `.into_iter().next()` and handles the empty case. The crate's conservative fallback is `replacement::keep_pre_asap`. In the [design document](../design_docs/asap_aware_mapping.md), each `ReplacementSubDAG` is a candidate. The complete `replacements()` result is the set of alternatives for one location in the plan. @@ -278,7 +307,7 @@ ReplacementStrategy::matches(...) | v ReplacementStrategy::replacements(...) - | (SketchFamilyStrategy: decide via implementations_for_with, + | (SketchAlgorithmStrategy: decide via implementations_for_with, | then construct each candidate's SummaryNode, in one method) +---------------------------------------------+ | | | @@ -289,7 +318,7 @@ The important rule is: > Strategies should reuse existing decision and binding logic where possible instead of reimplementing it. -`SketchFamilyStrategy` follows this literally: `implementations_for_with` is the single source of truth for what an `AggIntent` may become, and every candidate it produces gets constructed the same way. +`SketchAlgorithmStrategy` follows this literally: `implementations_for_with` is the single source of truth for what an `AggIntent` may become, and every candidate it produces gets constructed the same way. --- @@ -332,7 +361,7 @@ There are four decisions to make. `matches` should contain the minimum structural and semantic checks needed to determine whether the strategy applies. -For example, `SketchFamilyStrategy` only matches the aggregate shape that the existing binder can actually bind: +For example, `SketchAlgorithmStrategy` only matches the aggregate shape that the existing binder can actually bind: - the node is an `Aggregate`, - it has one aggregation intent, @@ -479,7 +508,7 @@ If another module already knows how to determine whether something is legal or h Do not create a second implementation of the same semantics inside the strategy. -The existing `SketchFamilyStrategy` is the model to follow: it reuses `implementation.rs`'s existing candidate list and the existing binder. +The existing `SketchAlgorithmStrategy` is the model to follow: it reuses `implementation.rs`'s existing candidate list and the existing binder. --- @@ -503,22 +532,22 @@ If your transformation requires context not currently represented in `TargetSubD --- -## 6. Example: current `SketchFamilyStrategy` +## 6. Example: current `SketchAlgorithmStrategy` -`SketchFamilyStrategy` is the reference implementation for a strategy that produces bound summaries. +`SketchAlgorithmStrategy` is the reference implementation for a strategy that produces bound summaries. Construction: ```rust let strategy = - SketchFamilyStrategy::default_cost_model(); + SketchAlgorithmStrategy::default_cost_model(); ``` or with a custom cost model: ```rust let model = MyCostModel; // illustrative -let strategy = SketchFamilyStrategy::new(&model); +let strategy = SketchAlgorithmStrategy::new(&model); ``` The strategy matches bindable aggregate nodes. @@ -550,7 +579,7 @@ For an approximate quantile, the current candidate list includes both KLL and DD ### How each candidate actually gets constructed -`SketchFamilyStrategy`'s whole `replacements()` body is one loop: +`SketchAlgorithmStrategy`'s whole `replacements()` body is one loop: ```rust fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { @@ -645,13 +674,13 @@ impl CostModel for PreferDDSketch { fn rank_candidates( &self, _intent: &AggIntent, - candidates: &[SketchKind], - ) -> Vec { + candidates: &[SketchAlgorithm], + ) -> Vec { let mut ranked = candidates.to_vec(); if let Some(pos) = ranked.iter().position( - |k| *k == SketchKind::DDSketch + |k| *k == SketchAlgorithm::DDSketch ) { let dd = ranked.remove(pos); @@ -669,13 +698,13 @@ Then inject it into code that accepts a `&dyn CostModel`: let model = PreferDDSketch; let strategy = - SketchFamilyStrategy::new(&model); + SketchAlgorithmStrategy::new(&model); let replacements = strategy.replacements(&target); ``` -Important: changing `rank_candidates` changes the preferred ordering, but `SketchFamilyStrategy` still enumerates every valid sketch candidate. +Important: changing `rank_candidates` changes the preferred ordering, but `SketchAlgorithmStrategy` still enumerates every valid sketch candidate. A custom cost model should not change which alternatives are semantically legal. @@ -687,7 +716,7 @@ Use this as a practical guide. ### `rank_candidates` -Use when you want to change the preference among valid sketch families. +Use when you want to change the preference among valid sketch algorithms. Example: @@ -702,26 +731,26 @@ Signature: fn rank_candidates( &self, intent: &AggIntent, - candidates: &[SketchKind], -) -> Vec; + candidates: &[SketchAlgorithm], +) -> Vec; ``` The returned vector should rank candidates from most to least preferred. -It should rank candidates that were supplied to it rather than invent unrelated sketch kinds. +It should rank candidates that were supplied to it rather than invent unrelated sketch algorithms. --- ### `size_params` -Use when the sketch family is already known and you want to choose its parameters from an accuracy target. +Use when the sketch algorithm is already known and you want to choose its parameters from an accuracy target. Signature: ```rust fn size_params( &self, - kind: SketchKind, + kind: SketchAlgorithm, intent: &AggIntent, eps: f64, delta: f64, @@ -737,7 +766,7 @@ Typical uses include: Conceptually: ```text -SketchKind + AggIntent + accuracy target +SketchAlgorithm + AggIntent + accuracy target | v SketchParams @@ -893,13 +922,13 @@ That turns a cost decision into a legality decision and prevents later global pl --- -## 12. Adding a new sketch family +## 12. Adding a new sketch algorithm -A new sketch family generally touches more than `ReplacementStrategy`. +A new sketch algorithm generally touches more than `ReplacementStrategy`. -The strategy should not maintain its own private list of sketch kinds. +The strategy should not maintain its own private list of sketch algorithms. -`SketchFamilyStrategy` obtains sketch alternatives through `implementation.rs`'s existing interface: +`SketchAlgorithmStrategy` obtains sketch alternatives through `implementation.rs`'s existing interface: ```rust summary_candidates(intent) @@ -907,7 +936,7 @@ summary_candidates(intent) and binds them through the normal binder. -Therefore, when adding a new built-in sketch family, the intended flow is: +Therefore, when adding a new built-in sketch algorithm, the intended flow is: ```text 1. Teach `implementation.rs` that the sketch is a valid candidate @@ -915,15 +944,15 @@ Therefore, when adding a new built-in sketch family, the intended flow is: 2. Teach the cost model how to rank and size it. -3. Ensure the binder can realize the sketch family. +3. Ensure the binder can realize the algorithm. -4. SketchFamilyStrategy will then enumerate it through the +4. SketchAlgorithmStrategy will then enumerate it through the existing candidate/binding path. ``` This keeps one source of truth for sketch applicability. -Do not special-case the new sketch inside `SketchFamilyStrategy` unless the strategy itself needs fundamentally new behavior. +Do not special-case the new sketch inside `SketchAlgorithmStrategy` unless the strategy itself needs fundamentally new behavior. --- @@ -934,7 +963,7 @@ The basic calling pattern is: ```rust let target = TargetSubDAG::new(&root); let strategy = - SketchFamilyStrategy::default_cost_model(); + SketchAlgorithmStrategy::default_cost_model(); if strategy.matches(&target) { let candidates = @@ -1097,13 +1126,13 @@ For ranking: let ranked = model.rank_candidates( &intent, - &[SketchKind::Kll, - SketchKind::DDSketch], + &[SketchAlgorithm::Kll, + SketchAlgorithm::DDSketch], ); assert_eq!( ranked[0], - SketchKind::DDSketch + SketchAlgorithm::DDSketch ); ``` @@ -1113,7 +1142,7 @@ For example: ```rust let strategy = - SketchFamilyStrategy::new(&model); + SketchAlgorithmStrategy::new(&model); let replacements = strategy.replacements(&target); @@ -1156,7 +1185,7 @@ fn replacements(...) -> Vec { ### Mistake: maintaining a second sketch-applicability table -If `implementation.rs` already defines which sketch families satisfy an `AggIntent`, reuse that source. +If `implementation.rs` already defines which sketch algorithms satisfy an `AggIntent`, reuse that source. Otherwise the binder and replacement strategy can silently disagree. @@ -1224,7 +1253,7 @@ When adding a new cost model: - [ ] Use extension hooks for extension-defined implementations/readouts. - [ ] Use CSE hooks for recompute-vs.-sharing costs. - [ ] Test the hook directly. -- [ ] Test integration through a consumer such as `SketchFamilyStrategy`. +- [ ] Test integration through a consumer such as `SketchAlgorithmStrategy`. - [ ] Verify that changing cost preferences does not silently remove valid replacement candidates. --- @@ -1239,7 +1268,7 @@ Use this table to find the right place for a change. | Add a new replacement for an existing target shape | `ReplacementStrategy::replacements` | | Change when a strategy applies | `ReplacementStrategy::matches` | | Add a new built-in sketch candidate | `implementation.rs`'s summary-candidate mapping | -| Prefer one sketch family over another | `CostModel::rank_candidates` | +| Prefer one sketch algorithm over another | `CostModel::rank_candidates` | | Change sketch sizing for an accuracy target | `CostModel::size_params` | | Add extension-defined implementation behavior | `CostModel::realize_extension` | | Add extension-defined readout behavior | `CostModel::readout_extension` | @@ -1247,10 +1276,10 @@ Use this table to find the right place for a change. | Change shared-maintenance cost | `CostModel::cse_shared_maintenance_cost` | | Change current share/recompute choice | `CostModel::cse_share_decision` | | Decide whether an available implementation satisfies a required one | `impl Matcher` | -| Produce a normal (ranked-first) bound summary for one target | `SketchFamilyStrategy::replacements(...).into_iter().next()` | +| Produce a normal (ranked-first) bound summary for one target | `SketchAlgorithmStrategy::replacements(...).into_iter().next()` | | Search a whole workload for every candidate at every `TargetSubDAG` (never pruning) | `replacement::search_workload`/`search_workload_with` | | Get every candidate ranked best-first, across a whole workload | `PlanSpace::cost_sorted` | | Get a real numeric cost per candidate, not just a relative rank | `CostModel::estimate_cost` | -| Enumerate valid sketch kinds | reuse `replacement::summary_candidates` | +| Enumerate valid sketch algorithms | reuse `replacement::summary_candidates` | | Build a target with no workload context | `TargetSubDAG::new` | | Build a target with known sharing context | `TargetSubDAG::with_consumer_count` | diff --git a/docs/user-guide/user-guide.md b/docs/user-guide/user-guide.md index 6af62c53..4a20e815 100644 --- a/docs/user-guide/user-guide.md +++ b/docs/user-guide/user-guide.md @@ -124,16 +124,16 @@ allowed, `Epsilon(e)` / `EpsilonDelta{epsilon, delta}` otherwise. Feed the `QueryExpr` to `asap-aware-mapping`. This crate depends only on `asap-types`, never on a front end, so it's agnostic to which language produced the tree. There is no "bind me one tree" -entry point: `SketchFamilyStrategy::replacements()` always returns every valid candidate for a +entry point: `SketchAlgorithmStrategy::replacements()` always returns every valid candidate for a target, ranked, and you take the one you want. ```rust -use asap_aware_mapping::{Replacement, ReplacementStrategy, ReplacementSubDAG, SketchFamilyStrategy, TargetSubDAG}; +use asap_aware_mapping::{Replacement, ReplacementStrategy, ReplacementSubDAG, SketchAlgorithmStrategy, TargetSubDAG}; use std::rc::Rc; let root = Rc::new(pre_asap); let target = TargetSubDAG::new(&root); -let candidates = SketchFamilyStrategy::default_cost_model().replacements(&target); +let candidates = SketchAlgorithmStrategy::default_cost_model().replacements(&target); // Take the cost-model-preferred candidate — the common case. let Some(ReplacementSubDAG { replacement: Replacement::Summary(post_asap), .. }) = @@ -148,10 +148,10 @@ else { ``` `implementations_for_with(&AggIntent, &dyn CostModel) -> Vec` is the lower-level -enumeration `SketchFamilyStrategy` wraps, if you only need the per-node decision (sketch, exact +enumeration `SketchAlgorithmStrategy` wraps, if you only need the per-node decision (sketch, exact accumulator, or pass-through) without binding it into a `SummaryNode`. -`SketchFamilyStrategy::new(&dyn CostModel)` (vs. `default_cost_model()`) is the extension point for +`SketchAlgorithmStrategy::new(&dyn CostModel)` (vs. `default_cost_model()`) is the extension point for a deployment that wants its own candidate ranking or parameter sizing instead of this crate's built-in static preference order (`DefaultCostModel` — what `default_cost_model()` uses). See the `CostModel` trait doc in `crates/asap-aware-mapping/src/cost_model.rs` for its overridable From 5504a2f8c2348bf10220b23faf7d8b009a7c1cd5 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 24 Aug 2026 12:28:13 -0600 Subject: [PATCH 27/49] feat(asap-aware-mapping): rebuild applicability.rs as a view over PlanSpace (#257, part of #33) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Absorbs PR #264 (feat/applicability-view-257) into this branch, re-derived against the current tip rather than merged/rebased/cherry-picked (#264's base predates the SketchFamilyStrategy->SketchAlgorithmStrategy rename, the search.rs->replacement.rs merge, and the "site"->"target"/TargetSubDAG rename). Replaces the deleted PR #247 rule-based traversal with a pure view over replacement::PlanSpace: a TargetSubDAG is applicability-worthy exactly when its candidate list contains something beyond the trivial, no-op realization. - OptimizationKind::SketchApproximation — the candidate list contains a Replacement::Summary that realizes SummaryFamilyType::Sketch(..). - OptimizationKind::CommonSubexpressionReuse — consumer_count >= 2 and the candidate list contains SharedSubtreeStrategy's "build once and share" candidate. Each finding's `reason` is the matching candidate's own ReplacementSubDAG::rationale verbatim — no new prose. No new applicability-specific extension-point trait: a new finding needs a new impl ReplacementStrategy wired into default_strategies/default_strategies_with regardless, since that's the only way a candidate reaches PlanSpace at all. Kept as its own module (crates/asap-aware-mapping/src/applicability.rs, `pub mod applicability;` in lib.rs) rather than flattened into replacement.rs: unlike implementation.rs/search.rs, this is a reporting/presentation transformation (internal PlanSpace -> external ApplicabilityFinding shape for a downstream DAG-visualization consumer), a genuinely different kind of concern from generating or ranking candidates. Also fixes now-stale cross-references in replacement.rs that pointed at the deleted PR #247 ApplicabilityRule/SharedSubexpressionRule/register_site traversal, and a stale reference in the developer guide (section 4.2) to the same deleted trait. Adds an "Applicability reporting" section (§19) and an extension-map row to docs/developer_docs/ASAP-aware-mapping-developer-guide.md; docs/design_docs/asap_aware_mapping.md already carries a conceptual "Applicability Reporting" section and, matching this session's convention of keeping that document implementation-agnostic, is left as-is. cargo build --workspace --all-targets, cargo test --workspace, cargo fmt --all -- --check, and cargo clippy --workspace --all-targets --all-features -- -D warnings all pass clean. PR #264 (feat/applicability-view-257) and PR #263 (feat/cascades-search-252) are both now superseded by this branch; flagged for the user to close. Co-Authored-By: Claude Sonnet 5 --- .../asap-aware-mapping/src/applicability.rs | 740 ++++++++++++++++++ crates/asap-aware-mapping/src/lib.rs | 15 + crates/asap-aware-mapping/src/replacement.rs | 64 +- .../ASAP-aware-mapping-developer-guide.md | 50 +- 4 files changed, 839 insertions(+), 30 deletions(-) create mode 100644 crates/asap-aware-mapping/src/applicability.rs diff --git a/crates/asap-aware-mapping/src/applicability.rs b/crates/asap-aware-mapping/src/applicability.rs new file mode 100644 index 00000000..bd891c75 --- /dev/null +++ b/crates/asap-aware-mapping/src/applicability.rs @@ -0,0 +1,740 @@ +//! Optimization-applicability *reporting* over a workload's pre-ASAP query +//! roots (issue #33: "Add logic to detect which optimizations are applicable +//! to a query workload"; this module: issue #257). +//! +//! ## The reframing: applicability *is* "this `TargetSubDAG`'s candidate +//! list is non-trivial" +//! +//! Earlier (PR #247, superseded by this module — see "What this replaces" +//! below), "is optimization X applicable here?" was a yes/no fact each rule +//! re-derived by walking the tree itself. That made sense before there was +//! any other structure to consult. But [`crate::replacement::search_workload`] +//! (issue #252) now *already* computes, for every +//! [`TargetSubDAG`](crate::replacement::TargetSubDAG) in the workload, every +//! semantically valid [`crate::replacement::ReplacementSubDAG`] a registered +//! [`ReplacementStrategy`] can propose — a [`PlanSpace`] of [`MemoGroup`]s. A +//! rule re-deriving the same yes/no fact from scratch would be answering a +//! question the search already answered, via a second, independently +//! maintained traversal that has to keep agreeing with the first one. +//! +//! Once that candidate space exists, "which optimizations are applicable" +//! collapses into a single question this module asks of *that* data instead: +//! **for a given `TargetSubDAG`, does its candidate list contain anything +//! other than the trivial, no-op realization?** A `TargetSubDAG` whose only +//! candidate is "the one thing `SketchAlgorithmStrategy` would have committed +//! to anyway, with no alternative" has no optimization to report — that +//! candidate isn't an *opportunity*, it's just the target's existing shape +//! reflected back. A `TargetSubDAG` with more than one candidate (several +//! sketch families to choose between), or one candidate that is itself a +//! genuine alternative to the status quo (share this already-shared subtree +//! instead of recomputing it at every consumer), *is* an applicability +//! finding — [`find_applicable_optimizations`] and +//! [`find_applicable_optimizations_with`] just translate [`PlanSpace`]'s +//! [`MemoGroup`]s into that shape: +//! +//! - [`OptimizationKind::SketchApproximation`] — the `TargetSubDAG`'s +//! candidate list contains at least one [`Replacement::Summary`] that +//! actually realizes a sketch family (`SummaryFamilyType::Sketch`), i.e. +//! [`SketchAlgorithmStrategy`] found something to offer beyond whatever +//! exact/pass-through candidate [`crate::replacement`]'s own +//! `implementations_for_with` would have committed to on its own. +//! - [`OptimizationKind::CommonSubexpressionReuse`] — the `TargetSubDAG` +//! has two or more consumers *and* its candidate list contains the +//! [`SharedSubtreeStrategy`] "build once and share" candidate (the one +//! whose `Rc` is the group's own `target`) — i.e. sharing this subtree +//! instead of recomputing it independently is a real, reported choice, not +//! just an accident of how the workload happened to be built. +//! +//! Each finding's `reason` is literally the matching candidate's own +//! [`crate::replacement::ReplacementSubDAG::rationale`] (joined, if more than one candidate +//! qualifies) — this module invents no new prose to explain *why* a +//! candidate is valid; that explanation already exists on the candidate a +//! [`ReplacementStrategy`] produced, and repeating it here (rather than +//! re-describing the same fact in different words) keeps exactly one place +//! that has to be right about "why is this a valid alternative". +//! +//! ## What this replaces, and what carries over unmodified +//! +//! [`ApplicabilityFinding`] and [`OptimizationKind`] keep PR #247's original +//! shape and contract — a struct/enum pair meant for a downstream +//! DAG-visualization consumer, `#[non_exhaustive]` discipline (only an +//! optimization backed by a real, registered [`ReplacementStrategy`] gets a +//! variant; see the catalog table below for everything still deliberately +//! unrepresented). So do the two top-level entry points, +//! [`find_applicable_optimizations`] and [`find_applicable_optimizations_with`] +//! — same "workload roots in, findings out" contract, mirroring +//! [`crate::replacement::search_workload`]/[`crate::replacement::search_workload_with`]'s +//! own signature shape. Only the *data source* changed: this module now +//! calls those two functions and translates the result, rather than running +//! its own rules and their supporting traversal over the tree a second time. +//! All of that old traversal is deleted, not kept alongside the new +//! implementation — see "Two guarantees the old traversal made, re-verified" +//! below for the two properties it's important that deletion didn't quietly +//! lose. +//! +//! ## Why a second, applicability-specific rule trait doesn't exist here +//! +//! PR #247 gave this module its own extension-point trait, `ApplicabilityRule` +//! (`fn optimization(&self) -> OptimizationKind` + `fn evaluate(&self, roots) +//! -> Vec`), the same shape [`crate::cost_model::CostModel`] +//! and [`crate::replacement::Matcher`] use elsewhere in this crate. Once +//! findings are a *view* over [`PlanSpace`] rather than an independent +//! computation, that trait would be a second extension point answering a +//! question [`ReplacementStrategy`] (issue #251) already answers: "does this +//! `TargetSubDAG` have an alternative worth reporting, and why". A caller who +//! wants a new optimization represented as a finding needs a new +//! `impl ReplacementStrategy` wired into +//! [`crate::replacement::search_workload_with`]'s strategy set *regardless* +//! (that's the only way its candidates end up in the [`PlanSpace`] this +//! module reads) — adding an `ApplicabilityRule` too would mean maintaining +//! two extension points for the same new capability, one of which (the rule) +//! would just be re-describing candidates the other (the strategy) already +//! produced. So this module ships no extension-point trait of its own: +//! [`ReplacementStrategy`] already *is* that extension point, one layer +//! down, and [`find_applicable_optimizations_with`]'s own `strategies` +//! parameter is where a caller plugs in a custom one (or a custom +//! `CostModel`, via [`crate::replacement::SketchAlgorithmStrategy::new`]) — the identical spot +//! [`crate::replacement::search_workload_with`] itself exposes. +//! +//! ## Two guarantees the old traversal made, re-verified against the new one +//! +//! 1. **A finding is reported at the maximal `TargetSubDAG`, never once more +//! per subsumed descendant.** [`crate::replacement`]'s own +//! `discover_targets` (used by [`crate::replacement::search_workload_with`], +//! and so by this module) walks every workload root's whole DAG but only +//! *recurses into a node's children the first time that node's `Rc` is +//! seen*; every subsequent occurrence still counts towards +//! `consumer_count`, but never triggers a second descent. A node nested +//! under an already-discovered shared ancestor therefore only becomes its +//! own `TargetSubDAG` if something *outside* that ancestor also +//! references it — identical to PR #247's own discovery pass, which +//! reported "the highest point sharing starts," not a finding at every +//! subsumed level below it. Same guarantee, same mechanism, just living in +//! [`crate::replacement`] now instead of here. +//! 2. **A node reachable via more than one path is one finding, not one per +//! path.** [`MemoGroup`]s are keyed by `Rc` pointer identity in +//! [`PlanSpace`]'s internal map — there is exactly one group per distinct +//! `Rc`, full stop, so a shared `Aggregate` reached via two different +//! `BinaryOp` branches (or two different workload roots) is exactly one +//! group, hence at most one [`OptimizationKind::SketchApproximation`] +//! finding, no matter how many paths reach it. +//! [`tests::a_shared_sketchable_aggregate_is_reported_only_once`] pins +//! this directly. +//! +//! ## One thing [`PlanSpace`] doesn't carry that this module still needs: +//! human-readable `location` text +//! +//! [`MemoGroup`]/[`PlanSpace`] deliberately track only `Rc` +//! pointer identity — the currency the search itself needs — not +//! caller-facing prose. [`ApplicabilityFinding::location`] is prose (a +//! breadcrumb like `root "dash_a" > lhs`), so this module keeps one small, +//! self-contained walk of its own, [`collect_locations`], whose *only* job +//! is turning "this `Rc`" into "the human-readable place(s) it occurs" for a +//! finding already decided by [`PlanSpace`]. This is not a reincarnation of +//! the deleted rule traversal: it makes no applicability decision (it runs +//! the same regardless of what any strategy found), and duplicating this +//! small, self-contained shape rather than threading location strings +//! through [`crate::replacement`]'s own `discover_targets` matches the same +//! call that module's own docs already make for its (test-only) +//! `count_consumers` counterpart — see [`crate::replacement`]'s "Where +//! `TargetSubDAG` discovery comes from" section. +//! +//! ## Catalog primitives deliberately left as future work +//! +//! The internal catalog +//! (`ProjectASAP/internal-docs/catalog_of_optimizations.md`) lists several +//! primitives with **no [`ReplacementStrategy`] implementation anywhere in +//! this codebase today**. Faking a variant for one of them would report a +//! finding this codebase cannot back with a real candidate, so none of the +//! below get an [`OptimizationKind`] variant yet — each gets one once a real +//! strategy exists and is wired into [`crate::replacement::default_strategies`]: +//! +//! | Catalog entry | Status | Where a future `OptimizationKind` would come from | +//! |---|---|---| +//! | Semantic-equivalent rewriting (e.g. `avg` → `sum`/`count`) | No `ReplacementStrategy` implementation in this codebase yet (tracked separately, issue #253) | Once wired into `default_strategies()`: any `Replacement::Rewrite` candidate that strategy proposes | +//! | Roll-ups (fine-to-coarse group-by reuse) | No `ReplacementStrategy` implementation in this codebase yet (tracked separately, issue #254) | Once wired into `default_strategies()`: any `Replacement::Rewrite` candidate that strategy proposes | +//! | Wavelets/OMP | Params type exists (`WaveletKind`/`WaveletParams`), reachable only via a deployment `CostModel::realize_extension` (no core `AggIntent` dispatch picks it) | A `ReplacementStrategy` that inspects a deployment's own `CostModel`, once some intent shape actually maps to `Implementation::Wavelet` | +//! | Sampling | Same story as Wavelets: `SamplingKind`/`SamplingParams` exist, unreachable from core dispatch | Same hook as Wavelets, for `Implementation::Sample` | +//! | Deep generative compression | No representation at all — no `Implementation`/`SummaryFamilyType` variant | Needs a new summary family added to `asap_types::post_asap` first | +//! | Approximation frameworks for windows | No representation — `TimeRange`/`PromqlSubquery` windows are always evaluated exactly | Would key off those node types once an approximate-window operator exists | +//! | Function decomposition | No representation anywhere | No hook point identified yet | +//! | Continuous distributed monitoring | No representation — `RepeatingEntry`/`RepetitionInterval` in `asap_types::workload` describe *that* a query repeats, not any monitoring-specific decomposition | Would likely key off `RepeatingEntry` once such logic exists | +//! | Incremental computation across time | No representation — nothing carries state across repeated evaluations of a `RepeatingEntry` today | Would key off `RepeatingEntry` + `TimeShift`/`TimeRange` once incremental state-carry exists | +//! | Delta encoding | No representation — `AggIntent::Delta`/`IDelta` are PromQL *value*-difference semantics, not a wire/storage delta-encoding optimization | Would plug into a future deployment-side wire/storage encoding decision (post-ASAP), not this crate's IR-level dispatch | +//! +//! [`ReplacementStrategy`]: crate::replacement::ReplacementStrategy +//! [`ReplacementSubDAG`]: crate::replacement::ReplacementSubDAG +//! [`Replacement`]: crate::replacement::Replacement +//! [`Replacement::Summary`]: crate::replacement::Replacement::Summary +//! [`Replacement::Rewrite`]: crate::replacement::Replacement::Rewrite +//! [`SketchAlgorithmStrategy`]: crate::replacement::SketchAlgorithmStrategy +//! [`SharedSubtreeStrategy`]: crate::replacement::SharedSubtreeStrategy +//! [`PlanSpace`]: crate::replacement::PlanSpace +//! [`MemoGroup`]: crate::replacement::MemoGroup + +use std::collections::{HashMap, HashSet}; +use std::fmt::Display; +use std::rc::Rc; + +use asap_types::post_asap::{SummaryExpr, SummaryFamilyType, SummaryNode}; +use asap_types::pre_asap::query_expr::QueryExpr; + +use crate::replacement::{self, MemoGroup, PlanSpace, Replacement, ReplacementStrategy}; + +/// Which optimization an [`ApplicabilityFinding`] is about. +/// +/// `#[non_exhaustive]`: only optimizations with a real [`ReplacementStrategy`] +/// behind them get a variant (see the module docs' "Catalog primitives +/// deliberately left as future work" table for everything else in the +/// catalog). +/// +/// [`ReplacementStrategy`]: crate::replacement::ReplacementStrategy +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum OptimizationKind { + /// A `TargetSubDAG`'s candidate list contains at least one + /// [`Replacement::Summary`] that realizes a sketch family — + /// [`crate::replacement::SketchAlgorithmStrategy`] found a genuine sketch + /// alternative for this `Aggregate`, beyond whatever exact/pass-through + /// candidate `crate::replacement`'s own `implementations_for_with` would + /// have committed to on its own. + SketchApproximation, + /// A `TargetSubDAG` has two or more consumers *and* its candidate list + /// contains [`crate::replacement::SharedSubtreeStrategy`]'s "build once + /// and share" candidate — the catalog's cross-statistic / cross-metrics / + /// cross-subpopulation reuse entries, all the same underlying structural + /// fact. + CommonSubexpressionReuse, +} + +/// One positive applicability result: `optimization` is applicable at +/// `location` (a human-readable breadcrumb into the workload — e.g. +/// `root "dashboard_p99"` or `root "ratio" > lhs`), for `reason` +/// (human-readable, meant for a report/log, not machine parsing — literally +/// the matching candidate's own [`crate::replacement::ReplacementSubDAG::rationale`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ApplicabilityFinding { + pub optimization: OptimizationKind, + pub location: String, + pub reason: String, +} + +/// Determine which known optimizations are applicable to a workload's +/// pre-ASAP query roots, using [`crate::replacement::default_strategies`]. +/// +/// `roots` — like [`crate::replacement::search_workload`]'s own `Id` type +/// parameter — is caller-chosen: a `QueryWorkload` entry's own key, an index, +/// a query name. It only needs [`Display`], since a finding's `location` is +/// prose, not a structured key back to the caller. +/// +/// Internally runs [`crate::replacement::search_workload`] to build the +/// candidate-plan space, then reads findings off it — see the module docs' +/// "The reframing" section for what that translation actually checks. +pub fn find_applicable_optimizations( + roots: Vec<(Id, QueryExpr)>, +) -> Vec { + find_applicable_optimizations_with(roots, &replacement::default_strategies()) +} + +/// Like [`find_applicable_optimizations`], but searches with `strategies` +/// instead of [`crate::replacement::default_strategies`] — the extension +/// point for a deployment-specific [`ReplacementStrategy`], or a custom +/// `CostModel` plugged into +/// [`crate::replacement::SketchAlgorithmStrategy::new`] (e.g. via +/// [`crate::replacement::default_strategies_with`]). +/// +/// [`ReplacementStrategy`]: crate::replacement::ReplacementStrategy +pub fn find_applicable_optimizations_with<'s, Id: Display>( + roots: Vec<(Id, QueryExpr)>, + strategies: &[Box], +) -> Vec { + let ided: Vec<(String, Rc)> = roots + .into_iter() + .map(|(id, expr)| (id.to_string(), Rc::new(expr))) + .collect(); + let space = replacement::search_workload_with(ided, strategies); + findings_from_plan_space(&space) +} + +/// Translate every discovered [`MemoGroup`] in `space` into zero, one, or two +/// [`ApplicabilityFinding`]s (a `TargetSubDAG` can be both sketch-approximable +/// *and* shared — the two optimizations are independent axes, not mutually +/// exclusive). +/// +/// `space`'s own `Id` is always `String` here: [`find_applicable_optimizations_with`] +/// already converted the caller's `Id: Display` into a `String` (via +/// `to_string()`) before calling [`crate::replacement::search_workload_with`], +/// so this function (and [`collect_locations`], which formats `id` with +/// [`std::fmt::Debug`] for the breadcrumb text) doesn't need its own generic +/// `Id` bound. +fn findings_from_plan_space(space: &PlanSpace) -> Vec { + let locations = collect_locations(&space.roots); + let mut findings = Vec::new(); + for group in space.groups() { + let location = locations + .get(&Rc::as_ptr(&group.target)) + .map(|locs| locs.join(", ")) + .unwrap_or_default(); + + if let Some(reason) = sketch_finding_reason(group) { + findings.push(ApplicabilityFinding { + optimization: OptimizationKind::SketchApproximation, + location: location.clone(), + reason, + }); + } + if let Some(reason) = shared_subexpr_finding_reason(group) { + findings.push(ApplicabilityFinding { + optimization: OptimizationKind::CommonSubexpressionReuse, + location, + reason, + }); + } + } + findings +} + +/// Does `group`'s candidate list contain a genuine sketch-family realization? +/// If so, the finding's `reason` is every such candidate's own `rationale`, +/// joined — this module does not invent new prose to restate why a candidate +/// is valid. +fn sketch_finding_reason(group: &MemoGroup) -> Option { + let reasons: Vec<&str> = group + .candidates + .iter() + .filter( + |c| matches!(&c.replacement, Replacement::Summary(node) if is_sketch_realization(node)), + ) + .map(|c| c.rationale.as_str()) + .collect(); + if reasons.is_empty() { + None + } else { + Some(reasons.join("; ")) + } +} + +/// Does `group` have two or more consumers *and* a "build once and share" +/// candidate (the [`Replacement::Rewrite`] whose `Rc` is the group's own +/// `target`) in its candidate list? If so, the finding's `reason` is that +/// candidate's own `rationale`. +fn shared_subexpr_finding_reason(group: &MemoGroup) -> Option { + if group.consumer_count < 2 { + return None; + } + group + .candidates + .iter() + .find( + |c| matches!(&c.replacement, Replacement::Rewrite(rc) if Rc::ptr_eq(rc, &group.target)), + ) + .map(|c| c.rationale.clone()) +} + +/// Does `node` (unwrapping any `SummaryEstimate` layer, the same shape +/// [`crate::replacement`]'s own private `sketch_kind_of` unwraps) ultimately +/// realize a [`SummaryFamilyType::Sketch`] family? This module only needs the +/// yes/no fact (a candidate's own `rationale` already names the specific +/// `SketchKind`/`SketchAlgorithm` for a finding's `reason` text), so unlike +/// `replacement.rs`'s counterpart this returns `bool`, not the kind itself. +fn is_sketch_realization(node: &SummaryNode) -> bool { + match &node.expr { + SummaryExpr::SummaryEstimate { summary_input, .. } => is_sketch_realization(summary_input), + SummaryExpr::SummaryAgg { family, .. } => matches!(family, SummaryFamilyType::Sketch(..)), + _ => false, + } +} + +// ── location breadcrumbs ───────────────────────────────────────────────── + +/// Build `location` text for every distinct `TargetSubDAG` reachable from +/// `roots` — see the module docs' "One thing `PlanSpace` doesn't carry" +/// section for why this module needs its own small walk for this. Returns +/// every breadcrumb path that reaches a given `Rc`, not just the first: a +/// shared node referenced from two workload roots (or two branches of one +/// root) needs both breadcrumbs in its finding's `location`, not just one. +fn collect_locations(roots: &[(String, Rc)]) -> HashMap<*const QueryExpr, Vec> { + let mut locations: HashMap<*const QueryExpr, Vec> = HashMap::new(); + let mut children_walked: HashSet<*const QueryExpr> = HashSet::new(); + for (id, root) in roots { + visit( + root, + format!("root {id:?}"), + &mut locations, + &mut children_walked, + ); + } + locations +} + +/// Record `label` as one of `node`'s breadcrumbs, then — only the first time +/// this exact `Rc` is seen — recurse into its children. Every subsequent +/// visit still records its own `label` (a genuinely different path reaching +/// the same shared node), it just doesn't re-walk that node's children a +/// second time — the same "walk once, but every occurrence still counts" +/// split `crate::replacement`'s own target-discovery `walk` makes, applied +/// here to text instead of a consumer count. +fn visit( + node: &Rc, + label: String, + locations: &mut HashMap<*const QueryExpr, Vec>, + children_walked: &mut HashSet<*const QueryExpr>, +) { + let ptr = Rc::as_ptr(node); + locations.entry(ptr).or_default().push(label.clone()); + if children_walked.insert(ptr) { + visit_children(node, &label, locations, children_walked); + } +} + +/// `node`'s own **relational-skeleton** operator children — the same scope +/// `crate::replacement`'s own target-discovery `walk_children` (and +/// `asap_types::pre_asap::cse::share_common_subtrees`'s `rebuild_children`) +/// use. Exhaustive over every `QueryExpr` variant: a new variant fails to +/// compile here until this match is extended too. +fn visit_children( + node: &QueryExpr, + label: &str, + locations: &mut HashMap<*const QueryExpr, Vec>, + children_walked: &mut HashSet<*const QueryExpr>, +) { + use QueryExpr::*; + match node { + Scan { .. } | PromqlScalarBridge(_) | QueryTimestamp => {} + PromqlVectorFromScalar(c) | PromqlScalarFromVector(c) => { + visit(c, format!("{label} > child"), locations, children_walked) + } + PromqlRelabel { child, .. } + | PromqlInfoEnrich { child, .. } + | PromqlSeriesSample { child, .. } + | Filter { child, .. } + | Project { child, .. } + | Aggregate { child, .. } + | Dedup { child, .. } + | PromqlSubquery { child, .. } + | TimeRange { child, .. } + | TimeShift { child, .. } + | SQLWindowFunc { child, .. } + | Sort { child, .. } + | Limit { child, .. } => visit( + child, + format!("{label} > child"), + locations, + children_walked, + ), + Concat { children } => { + for (i, c) in children.iter().enumerate() { + visit_children( + c, + &format!("{label} > concat[{i}]"), + locations, + children_walked, + ); + } + } + Join { left, right, .. } | SetOp { left, right, .. } => { + visit(left, format!("{label} > left"), locations, children_walked); + visit( + right, + format!("{label} > right"), + locations, + children_walked, + ); + } + BinaryOp { lhs, rhs, .. } => { + visit(lhs, format!("{label} > lhs"), locations, children_walked); + visit(rhs, format!("{label} > rhs"), locations, children_walked); + } + Column(_) + | Literal(_) + | Compare { .. } + | BoolAnd(_) + | BoolOr(_) + | Not(_) + | IsNull(_) + | IsNotNull(_) + | Cast { .. } + | InList { .. } + | FunctionCall { .. } + | Arithmetic { .. } + | Case { .. } => {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + use asap_types::pre_asap::agg_intent::{default_quantile, AggIntent}; + use asap_types::pre_asap::query_expr::{Reduction, Source}; + use asap_types::pre_asap::schema::{Column, DataType, Schema}; + use asap_types::types::AccuracyTarget; + + fn metric_scan(labels: &[&str]) -> QueryExpr { + let mut columns = vec![ + Column::new("ts", DataType::Timestamp, false), + Column::new("value", DataType::Float64, false), + ]; + columns.extend(labels.iter().map(|n| Column::new(*n, DataType::Utf8, true))); + QueryExpr::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index(columns, 0, vec![]), + } + } + + fn agg(by: Vec, intent: AggIntent, child: QueryExpr) -> QueryExpr { + QueryExpr::Aggregate { + reduction: Reduction::by(by), + measures: vec![intent], + output_names: vec![], + having: None, + child: Rc::new(child), + } + } + + // ── SketchApproximation ────────────────────────────────────────────── + + #[test] + fn approximate_quantile_is_a_sketch_applicability_finding() { + let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + let findings = find_applicable_optimizations(vec![("dashboard_p99", q)]); + let sketch: Vec<_> = findings + .iter() + .filter(|f| f.optimization == OptimizationKind::SketchApproximation) + .collect(); + assert_eq!( + sketch.len(), + 1, + "expected one sketch finding, got {findings:?}" + ); + assert!(sketch[0].location.contains("dashboard_p99")); + assert!(sketch[0].reason.to_lowercase().contains("kll")); + } + + #[test] + fn exact_quantile_is_not_a_sketch_applicability_finding() { + let q = agg( + vec![2], + AggIntent::Quantile { + col: None, + q: 0.99, + accuracy: AccuracyTarget::Exact, + }, + metric_scan(&["job"]), + ); + let findings = find_applicable_optimizations(vec![("exact_p99", q)]); + assert!( + findings + .iter() + .all(|f| f.optimization != OptimizationKind::SketchApproximation), + "an Exact accuracy target must not report sketch-applicability, got {findings:?}" + ); + } + + #[test] + fn nested_aggregate_still_finds_the_inner_sketchable_node() { + // avg(quantile(0.9, sum by (job) (m))) shaped test isn't representable + // (avg is PassThrough, not a wrapper we recurse through structurally + // the way an Aggregate's own child is) — instead nest a sketchable + // quantile under an exact sum, the same nesting replacement.rs's own + // nested_aggregates_bind_per_node test uses. + let inner = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let outer = agg(vec![], default_quantile(0.9), inner); + let findings = find_applicable_optimizations(vec![("q", outer)]); + let sketch_count = findings + .iter() + .filter(|f| f.optimization == OptimizationKind::SketchApproximation) + .count(); + assert_eq!( + sketch_count, 1, + "expected the outer quantile only, got {findings:?}" + ); + } + + #[test] + fn pass_through_intent_reports_no_sketch_finding() { + let q = agg(vec![2], AggIntent::Avg { col: None }, metric_scan(&["job"])); + let findings = find_applicable_optimizations(vec![("avg_latency", q)]); + assert!(findings + .iter() + .all(|f| f.optimization != OptimizationKind::SketchApproximation)); + } + + /// A sketch-applicable `Aggregate` reachable via two paths that CSE + /// collapses onto one `Rc` — the same `median(x) == median(x)` shape + /// `pre_asap::cse`'s own `single_query_shares_its_own_repeated_subtree` + /// test uses — must be reported once, not once per path: it is exactly + /// one [`crate::replacement::MemoGroup`], keyed by `Rc` pointer identity, + /// not one per path that reaches it. + #[test] + fn a_shared_sketchable_aggregate_is_reported_only_once() { + let quantile = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + let root = QueryExpr::BinaryOp { + op: asap_types::pre_asap::query_expr::BinaryOpKind::Compare( + asap_types::pre_asap::expr_ir::CompareOpKind::Eq, + ), + lhs: Rc::new(quantile.clone()), + rhs: Rc::new(quantile), + vector_match: None, + }; + let findings = find_applicable_optimizations(vec![("ratio", root)]); + let sketch: Vec<_> = findings + .iter() + .filter(|f| f.optimization == OptimizationKind::SketchApproximation) + .collect(); + assert_eq!( + sketch.len(), + 1, + "a single shared Aggregate must produce one finding, not one \ + per path that reaches it: got {findings:?}" + ); + } + + // ── CommonSubexpressionReuse ───────────────────────────────────────── + + #[test] + fn two_roots_with_the_same_grouped_aggregate_share_a_reuse_finding() { + // Grouped (`by (job)`), so the shared `Aggregate`'s output schema + // carries a provable unique key — share_common_subtrees's legality + // gate — and identical across both roots, so it is shareable. + let a = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let b = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let findings = find_applicable_optimizations(vec![("dash_a", a), ("dash_b", b)]); + let reuse: Vec<_> = findings + .iter() + .filter(|f| f.optimization == OptimizationKind::CommonSubexpressionReuse) + .collect(); + assert_eq!( + reuse.len(), + 1, + "expected one reuse finding, got {findings:?}" + ); + assert!(reuse[0].location.contains("dash_a")); + assert!(reuse[0].location.contains("dash_b")); + } + + #[test] + fn distinct_queries_report_no_reuse_finding() { + let a = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let b = agg( + vec![2], + AggIntent::Sum { col: None }, + metric_scan(&["route"]), + ); + let findings = find_applicable_optimizations(vec![("dash_a", a), ("dash_b", b)]); + assert!( + findings + .iter() + .all(|f| f.optimization != OptimizationKind::CommonSubexpressionReuse), + "structurally different queries must not report reuse, got {findings:?}" + ); + } + + #[test] + fn ungrouped_identical_aggregates_are_not_shareable_so_no_finding() { + // Empty `by`: no provable unique key — share_common_subtrees never + // hoists these, so consumer_count stays 1 for each and this module + // must not report a finding either. + let a = agg(vec![], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let b = agg(vec![], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let findings = find_applicable_optimizations(vec![("a", a), ("b", b)]); + assert!(findings + .iter() + .all(|f| f.optimization != OptimizationKind::CommonSubexpressionReuse)); + } + + #[test] + fn single_query_repeated_subexpression_is_a_reuse_finding() { + // The same shared branch appearing twice within one query (an `a/a` + // shape) — single-query CSE. + let branch = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let q = QueryExpr::BinaryOp { + op: asap_types::pre_asap::query_expr::BinaryOpKind::Arithmetic( + asap_types::pre_asap::expr_ir::ArithmeticOpKind::Div, + ), + lhs: Rc::new(branch.clone()), + rhs: Rc::new(branch), + vector_match: None, + }; + let findings = find_applicable_optimizations(vec![("ratio", q)]); + let reuse: Vec<_> = findings + .iter() + .filter(|f| f.optimization == OptimizationKind::CommonSubexpressionReuse) + .collect(); + assert_eq!( + reuse.len(), + 1, + "expected one reuse finding, got {findings:?}" + ); + assert!(reuse[0].location.contains("lhs")); + assert!(reuse[0].location.contains("rhs")); + } + + /// A shared node nested three levels under two *different*, unshared + /// `Filter` parents (mirrors `crate::replacement::tests:: + /// nested_shared_subtree_below_an_unshared_parent_is_still_discovered`) + /// must still be exactly one finding — the maximal-`TargetSubDAG` + /// guarantee the module docs describe, now provided by + /// `crate::replacement`'s own target discovery rather than this module's + /// (deleted) traversal. + #[test] + fn a_deeply_shared_subtree_under_different_parents_is_reported_once() { + use asap_types::pre_asap::expr_ir::ScalarValue; + use asap_types::pre_asap::query_expr::Predicate; + + let shared = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let root_a = QueryExpr::Filter { + pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Int64(1)))), + child: Rc::new(shared.clone()), + }; + let root_b = QueryExpr::Filter { + pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Int64(2)))), + child: Rc::new(shared), + }; + let findings = find_applicable_optimizations(vec![("a", root_a), ("b", root_b)]); + let reuse: Vec<_> = findings + .iter() + .filter(|f| f.optimization == OptimizationKind::CommonSubexpressionReuse) + .collect(); + assert_eq!( + reuse.len(), + 1, + "a node shared under two different parents must be one finding, got {findings:?}" + ); + assert!(reuse[0].location.contains('a')); + assert!(reuse[0].location.contains('b')); + } + + // ── Custom strategy set / cost model plumbing ─────────────────────── + + struct AlwaysDDSketch; + impl crate::cost_model::CostModel for AlwaysDDSketch { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[asap_types::post_asap::SketchAlgorithm], + ) -> Vec { + let mut v = candidates.to_vec(); + if let Some(pos) = v + .iter() + .position(|k| *k == asap_types::post_asap::SketchAlgorithm::DDSketch) + { + let dd = v.remove(pos); + v.insert(0, dd); + } + v + } + } + + #[test] + fn custom_cost_model_changes_the_reported_sketch_kind() { + let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + let custom_model = AlwaysDDSketch; + let strategies: Vec> = vec![Box::new( + crate::replacement::SketchAlgorithmStrategy::new(&custom_model), + )]; + let findings = find_applicable_optimizations_with(vec![("q", q)], &strategies); + assert_eq!(findings.len(), 1); + assert!(findings[0].reason.to_lowercase().contains("ddsketch")); + } +} diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 4fde19e6..c3bf43bb 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -85,6 +85,16 @@ //! — exposes an actual numeric cost per candidate, not just a relative //! rank, for a caller (e.g. a DAG-visualization view) that wants to show //! "candidate A costs ≈ X" next to "candidate B costs ≈ Y". +//! - [`applicability`] — a reporting *view* over [`replacement`]'s +//! candidate-plan space (issue #257, part of #33): translates every +//! discovered `TargetSubDAG` with a non-trivial candidate list into an +//! [`applicability::ApplicabilityFinding`] ("which optimizations are +//! applicable, where, and why"), meant for the same downstream consumer +//! (e.g. a DAG-visualization view) the crate doc's `## Status` section +//! above already names for [`replacement::PlanSpace`] itself. Superseded PR +//! #247's own rule-based traversal, which re-walked the tree once per +//! optimization before [`replacement::search_workload`] existed to read +//! from instead — see that module's docs for the full reframing. //! //! ## Terminology — "bind" already means three different things nearby; //! this crate's own logical→physical step is named "implementation" instead @@ -126,9 +136,14 @@ //! `sketch_algebra::capability::Capability`/`is_satisfied_by` is the //! reference downstream implementation. +pub mod applicability; pub mod cost_model; pub mod replacement; +pub use applicability::{ + find_applicable_optimizations, find_applicable_optimizations_with, ApplicabilityFinding, + OptimizationKind, +}; pub use cost_model::{CostModel, DefaultCostModel}; pub use replacement::{ default_strategies, default_strategies_with, search_workload, search_workload_with, diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index d83f2a1d..9019ab03 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -42,12 +42,15 @@ //! equivalent) — see [`Replacement`] — plus a human-readable `rationale`. //! - [`ReplacementStrategy`] — `matches` + `replacements`, the same //! extension-point shape [`CostModel`] and [`Matcher`] already use in this -//! crate (and the same shape issue #33's applicability-rule framework, PR -//! #247, uses for its own `ApplicabilityRule`): a new replacement source is -//! a new `impl ReplacementStrategy`, not a restructuring of this trait or of -//! any existing strategy. `replacements` is **exhaustive, not ranked, not -//! filtered** — reporting "every valid candidate" is core's job; picking -//! the best one is left to the caller. +//! crate: a new replacement source is a new `impl ReplacementStrategy`, not +//! a restructuring of this trait or of any existing strategy. `replacements` +//! is **exhaustive, not ranked, not filtered** — reporting "every valid +//! candidate" is core's job; picking the best one is left to the caller. +//! [`crate::applicability`] (issue #257) is this trait's own downstream +//! consumer, not a second extension point: it reports "which optimizations +//! are applicable" as a pure view over the candidates strategies registered +//! here already produced, rather than re-deriving applicability with a rule +//! of its own. //! //! A caller that wants one executable answer takes the first //! (`cost_model`-preferred) entry off `replacements()` itself @@ -81,10 +84,9 @@ //! `asap_types::pre_asap::cse::share_common_subtrees`'s sharing decision. //! Wherever a [`TargetSubDAG`] already has two or more consumers (i.e. //! `share_common_subtrees` already collapsed two or more workload -//! locations onto the same `Rc` — PR #247's own -//! `SharedSubexpressionRule` traversal/dedup logic, over in the -//! applicability-rule framework, does the identical discovery; this -//! module's own tests reuse the same dedup logic to build realistic +//! locations onto the same `Rc` — [`discover_targets`] below +//! does the identical workload-wide discovery for [`search_workload_with`]; +//! this module's own tests reuse the same dedup logic to build realistic //! fixtures), it reports the two-way candidate CSE's own detection pass //! deliberately declines to pick between on its own: build once and share //! the already-interned subtree, or build it independently at each @@ -355,8 +357,8 @@ pub enum Replacement { /// One candidate replacement for a [`TargetSubDAG`], plus a human-readable /// `rationale` explaining why it's a valid candidate (meant for a /// report/log/debugging a search engine's choices, not machine parsing — -/// the same role an `ApplicabilityFinding`'s `reason` field plays for -/// applicability findings, over in PR #247's applicability-rule framework). +/// [`crate::applicability::ApplicabilityFinding::reason`] literally reuses +/// this same string rather than inventing new prose of its own. #[derive(Debug, Clone)] pub struct ReplacementSubDAG { pub replacement: Replacement, @@ -932,7 +934,7 @@ fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { /// A single static instance so [`SketchAlgorithmStrategy::default_cost_model`] /// can hand out a `&'static dyn CostModel` without heap-allocating one — /// `DefaultCostModel` is a unit struct with no state, so one instance serves -/// every caller (same pattern `applicability::SketchApplicabilityRule` uses). +/// every caller. static DEFAULT_COST_MODEL: DefaultCostModel = DefaultCostModel; /// Wraps [`implementations_for_with`]'s exhaustive, ranked list directly: for @@ -1039,8 +1041,10 @@ fn describe_implementation(intent: &AggIntent, implementation: &Implementation) /// this crate's other `AggIntent` matches, e.g. [`implementations_for_with`]'s) /// — this is prose for a rationale string, not a decision, so an unlisted /// variant just falls back to its `Debug` tag rather than forcing every -/// future intent to be named here too (same rationale, and same shape, as -/// `applicability`'s own private `describe_intent`). +/// future intent to be named here too. [`crate::applicability`] needs no +/// counterpart of its own: it reads a candidate's `rationale` — built from +/// this text — straight off [`ReplacementSubDAG`], rather than re-describing +/// the same intent a second time. fn describe_intent(intent: &AggIntent) -> String { match intent { AggIntent::Quantile { q, .. } => format!("quantile(q={q})"), @@ -1346,9 +1350,9 @@ fn lift(schema: &Schema) -> SummarySchema { /// This strategy does not decide sharing itself, nor does it discover which /// nodes are shared — by the time a caller builds a `TargetSubDAG` with /// `consumer_count >= 2`, `share_common_subtrees` has already made that -/// (legality-gated, `PartialEq`-checked) call; PR #247's -/// `SharedSubexpressionRule` traversal discovers real consumer counts across -/// a workload the same way (this module's own tests reuse the identical +/// (legality-gated, `PartialEq`-checked) call; [`discover_targets`] below +/// discovers real consumer counts across a workload the same way for +/// [`search_workload_with`] (this module's own tests reuse the identical /// dedup logic to build realistic fixtures — see the module docs' /// "Non-goals" on why that traversal isn't itself part of this strategy). /// This strategy only reframes "two or more consumers already share this @@ -1759,9 +1763,11 @@ fn sketch_kind_of(node: &SummaryNode) -> Option { /// The strategies [`search_workload`] runs, in the built-in /// [`DefaultCostModel`] configuration — mirrors this module's own two -/// shipped [`ReplacementStrategy`] impls, the same way a hypothetical -/// `applicability::default_rules()` mirrors *its* module's own rule set. -/// Use [`default_strategies_with`] to plug in a deployment-specific +/// shipped [`ReplacementStrategy`] impls. +/// [`crate::applicability::find_applicable_optimizations`] (issue #257) uses +/// this same set (via [`search_workload`]) rather than keeping a second, +/// applicability-specific list to stay in sync with. Use +/// [`default_strategies_with`] to plug in a deployment-specific /// [`CostModel`] instead. pub fn default_strategies() -> Vec> { vec![ @@ -1800,10 +1806,11 @@ pub fn search_workload(roots: Vec<(Id, Rc)>) -> PlanSpace { /// [`default_strategies_with`] to plug in a deployment-specific /// [`CostModel`] for candidate generation). /// -/// Runs [`share_common_subtrees`] once over `roots` first — the same -/// pattern a hypothetical `applicability::find_applicable_optimizations` -/// uses, so every strategy sees the same already-deduplicated tree — then -/// discovers every `TargetSubDAG` (see [`discover_targets`]) and runs the +/// Runs [`share_common_subtrees`] once over `roots` first — so every +/// strategy (and, transitively, every +/// [`crate::applicability::ApplicabilityFinding`] a caller reads off the +/// result) sees the same already-deduplicated tree — then discovers every +/// `TargetSubDAG` (see [`discover_targets`]) and runs the /// fixpoint loop the module docs describe, capped at /// [`MAX_SEARCH_ITERATIONS`] passes (see the module docs' "Termination" /// section). Deduping candidate plans this way needs no @@ -2857,10 +2864,9 @@ mod tests { assert!(replacements[1].rationale.contains('3')); } - /// Builds realistic multi-consumer `TargetSubDAG`s the same way - /// `applicability::SharedSubexpressionRule`'s `register_site`/ - /// `walk_rc_children` traversal does: dedup by `Rc::as_ptr`, walking - /// only the relational-skeleton operator children + /// Builds realistic multi-consumer `TargetSubDAG`s the same way this + /// module's own [`discover_targets`]/`walk` does: dedup by `Rc::as_ptr`, + /// walking only the relational-skeleton operator children /// `asap_types::pre_asap::cse::share_common_subtrees` itself scopes to, /// so a shared node nested below another shared node is only ever /// counted at the highest (maximal) point sharing starts. Test-only: diff --git a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md index ad584585..832eb108 100644 --- a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md @@ -383,7 +383,7 @@ is enough for the current shared-subtree strategy. Keep `matches` cheap and unsurprising. -It should answer *whether the strategy applies here* in the plain English sense — not `applicability.rs`'s formal `ApplicabilityRule`/`ApplicabilityFinding` concept (issue #247, a separate reporting layer covered in the design doc's "Applicability reporting" section). `matches` isn't that machinery and doesn't need to produce anything it consumes; it should also not perform ranking or choose a winner. +It should answer *whether the strategy applies here* in the plain English sense — not `applicability.rs`'s formal `ApplicabilityFinding` concept (issue #257, a separate reporting layer over this trait's own output — see §19). `matches` isn't that machinery and doesn't need to produce anything it consumes; it should also not perform ranking or choose a winner. --- @@ -1283,3 +1283,51 @@ Use this table to find the right place for a change. | Enumerate valid sketch algorithms | reuse `replacement::summary_candidates` | | Build a target with no workload context | `TargetSubDAG::new` | | Build a target with known sharing context | `TargetSubDAG::with_consumer_count` | +| Report which optimizations are applicable, where, and why | `applicability::find_applicable_optimizations`/`find_applicable_optimizations_with` | +| Add a new kind of applicability finding | new `impl ReplacementStrategy`, wired into `default_strategies`/`default_strategies_with` — not a new applicability-specific trait, see §19 | + +--- + +## 19. Applicability reporting (`applicability.rs`) + +`applicability::find_applicable_optimizations`/`find_applicable_optimizations_with` answer a different question than everything above: not "what could this target become" (`ReplacementStrategy::replacements`) but "which of the optimizations already discovered are worth telling a user about, and why." It is a **reporting view over `PlanSpace`**, not a second search or a second rule engine. + +### The rule + +> A `TargetSubDAG` is applicability-worthy exactly when its `PlanSpace` candidate list contains something beyond the trivial, no-op realization. + +Concretely, `applicability.rs` reads two shapes off each `MemoGroup`: + +- `OptimizationKind::SketchApproximation` — the group's candidates include a `Replacement::Summary` that actually realizes `SummaryFamilyType::Sketch(..)`, i.e. `SketchAlgorithmStrategy` found a real sketch alternative, not just an exact/pass-through candidate. +- `OptimizationKind::CommonSubexpressionReuse` — `consumer_count >= 2` and the group's candidates include `SharedSubtreeStrategy`'s "build once and share" candidate (the `Replacement::Rewrite` whose `Rc` is the group's own `target`). + +Each `ApplicabilityFinding::reason` is copied verbatim from the matching candidate's own `ReplacementSubDAG::rationale`. Nothing in `applicability.rs` re-explains why a candidate is valid; that explanation already exists exactly once, on the candidate itself. + +### Why there is no `ApplicabilityRule` trait + +An earlier version of this module (superseded, PR #247) had its own extension-point trait for adding a new optimization to the report. It is gone. Once findings are read off `PlanSpace`, a new applicability finding needs a new `impl ReplacementStrategy` wired into `default_strategies`/`default_strategies_with` regardless — that is the only way a new kind of candidate reaches the `PlanSpace` this module reads. A second, applicability-specific extension point would just be a second place to register the same thing. `find_applicable_optimizations_with`'s own `strategies: &[Box]` parameter is where a caller plugs in something custom — the same customization point `search_workload_with` itself exposes. + +### What it still owns: `location` text + +`PlanSpace`/`MemoGroup` track `Rc` pointer identity, not human-readable breadcrumbs. `applicability.rs` keeps one small, self-contained traversal, `collect_locations`, whose only job is turning "this `Rc`" into prose like `root "dash_a" > lhs` for `ApplicabilityFinding::location`. It makes no applicability decision — it runs identically regardless of what any strategy found. + +### Using it + +```rust +use asap_aware_mapping::{find_applicable_optimizations, OptimizationKind}; + +let findings = find_applicable_optimizations(vec![("dashboard_p99", query)]); +for finding in &findings { + match finding.optimization { + OptimizationKind::SketchApproximation => { /* ... */ } + OptimizationKind::CommonSubexpressionReuse => { /* ... */ } + _ => { /* OptimizationKind is #[non_exhaustive] */ } + } +} +``` + +To plug in a deployment-specific strategy or `CostModel`, use `find_applicable_optimizations_with` with a strategy set built the same way `default_strategies_with` builds one — see §8 and §11. + +### Adding a new applicability finding + +There is no separate checklist here: follow [§4](#4-adding-a-new-replacementstrategy) to add the new `ReplacementStrategy`, wire it into `default_strategies`/`default_strategies_with` (§18), add an `OptimizationKind` variant, and extend `findings_from_plan_space` to recognize the new candidate shape. If the new strategy's `matches`/`replacements` are already correct and tested, the applicability finding falls out of the existing `PlanSpace` translation with no new discovery logic required. From d19ce87c2db582bff9bba755f05e8e86df8dcbf7 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 24 Aug 2026 12:51:56 -0600 Subject: [PATCH 28/49] docs(asap-aware-mapping): add PlanSpace/MemoGroup/RankedGroup to the glossary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Section 2 documented TargetSubDAG/Replacement/ReplacementSubDAG/ ReplacementStrategy/CostModel but not the whole-workload types the search.rs merge added — add them as a glossary entry in the same style, including how cost_sorted dispatches by candidate shape. Co-Authored-By: Claude Sonnet 5 --- .../ASAP-aware-mapping-developer-guide.md | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md index 832eb108..11b35d6c 100644 --- a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md @@ -253,6 +253,35 @@ A custom cost model does not necessarily need to override every hook. The curren --- +### `PlanSpace` / `MemoGroup` / `RankedGroup` — the whole-workload view + +`ReplacementStrategy` answers "what are the candidates for this one target?" `PlanSpace` answers the same question for every target in a whole workload at once, without materializing `2^N` fully-copied plans for `N` independently-choosable sites. + +```rust +// replacement.rs + +// One MemoGroup per distinct TargetSubDAG in the whole workload — +// never a flat list of fully-materialized plans. +pub struct MemoGroup { + pub target: Rc, + pub consumer_count: usize, + pub candidates: Vec, // every alternative, unranked +} + +pub struct RankedGroup<'a> { + pub target: &'a Rc, + pub consumer_count: usize, + pub candidates: Vec<&'a ReplacementSubDAG>, // same candidates, ranked + pub costs: Vec, // costs[i] <-> candidates[i] +} +``` + +`search_workload(roots)` runs the shared-subtree pass once, discovers every target across every root's whole DAG (not just root-level sharing — a `SharedSubtreeStrategy` candidate three levels under an unshared `Filter` is exactly as real a site as a shared whole root), and asks every registered strategy to a fixpoint. Two logically different candidates at two different targets are never copied into two separate plans — they're two entries in two different `MemoGroup`s, sharing every other node in the workload by construction. + +`PlanSpace::cost_sorted(cost_model)` is the one ranking step: for each group, it dispatches by candidate shape — a same-shape `Rewrite` pair (a `SharedSubtreeStrategy` share/recompute choice) goes through `CostModel::cse_share_decision`; a same-shape run of `Summary` candidates realizing sketches (a `SketchAlgorithmStrategy` choice) goes through `CostModel::rank_candidates`; every candidate also gets a real number from `CostModel::estimate_cost`, aligned index-for-index in `costs`. Groups whose candidates don't fit either shape (a lone candidate, or a mix — e.g. one target where both strategies fired at once) keep discovery order for the un-rankable part. Count in, count out — nothing is ever dropped to produce a ranking. + +--- + ### Family, kind, and algorithm Three levels sit below "summary" in this crate's type vocabulary, and `Sketch` is the only family with all three: From d459a6216cabd8cc32d3f71767a1a1fd3380c246 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 24 Aug 2026 13:07:38 -0600 Subject: [PATCH 29/49] refactor(asap-aware-mapping): reframe applicability.rs as explanation of a replacement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit applicability.rs already reused a matching candidate's own `rationale` to explain why a Replacement exists at a given target — it wasn't really answering "is optimization X applicable, yes/no". Rename the module and its API to match what it actually does: - applicability.rs -> explanation.rs - ApplicabilityFinding -> ReplacementExplanation (field `optimization` renamed to `kind`) - OptimizationKind -> ExplanationKind (variants unchanged) - find_applicable_optimizations[_with] -> explain_replacements[_with] Add `node_hash: u64` to ReplacementExplanation, computed via asap_types::pre_asap::cse::structural_hash on the MemoGroup's `target` — the same HashCache-backed function replacement.rs's own dedup logic and asap_types::dag_export::DagNode::hash already use, so a downstream consumer (issue #257's follow-up in dag_export) can correlate an explanation with the exact DagNode it's about by comparing hashes, no string-matching required. Updates every internal cross-reference (lib.rs's module docs/pub use, replacement.rs's doc comments, the developer guide's §19 and extension-map table, the design doc's "Applicability Reporting" section, renamed to "Explaining a Replacement") and every test. Co-Authored-By: Claude Sonnet 5 --- .../src/{applicability.rs => explanation.rs} | 176 ++++++++++++------ crates/asap-aware-mapping/src/lib.rs | 25 +-- crates/asap-aware-mapping/src/replacement.rs | 20 +- docs/design_docs/asap_aware_mapping.md | 12 +- .../ASAP-aware-mapping-developer-guide.md | 50 ++--- 5 files changed, 169 insertions(+), 114 deletions(-) rename crates/asap-aware-mapping/src/{applicability.rs => explanation.rs} (81%) diff --git a/crates/asap-aware-mapping/src/applicability.rs b/crates/asap-aware-mapping/src/explanation.rs similarity index 81% rename from crates/asap-aware-mapping/src/applicability.rs rename to crates/asap-aware-mapping/src/explanation.rs index bd891c75..bffb08d3 100644 --- a/crates/asap-aware-mapping/src/applicability.rs +++ b/crates/asap-aware-mapping/src/explanation.rs @@ -1,8 +1,18 @@ -//! Optimization-applicability *reporting* over a workload's pre-ASAP query -//! roots (issue #33: "Add logic to detect which optimizations are applicable -//! to a query workload"; this module: issue #257). +//! This crate's **explanation of a replacement**: for a `TargetSubDAG` that +//! [`crate::replacement::search_workload`] found something to say about, why +//! does that candidate exist? (issue #33: "Add logic to detect which +//! optimizations are applicable to a query workload"; this module: issue +//! #257.) //! -//! ## The reframing: applicability *is* "this `TargetSubDAG`'s candidate +//! This module does not answer "is optimization X applicable here, yes or +//! no" — that framing implies a classifier deciding admissibility from +//! scratch. What it actually does is narrower and more mechanical: reuse a +//! matching candidate's own [`crate::replacement::ReplacementSubDAG::rationale`] +//! to explain, in the candidate's own words, why a [`Replacement`] exists at +//! a given target. No new prose is invented here; see "The reframing" below +//! for exactly what's being reused and why. +//! +//! ## The reframing: an explanation *is* "this `TargetSubDAG`'s candidate //! list is non-trivial" //! //! Earlier (PR #247, superseded by this module — see "What this replaces" @@ -28,17 +38,17 @@ //! sketch families to choose between), or one candidate that is itself a //! genuine alternative to the status quo (share this already-shared subtree //! instead of recomputing it at every consumer), *is* an applicability -//! finding — [`find_applicable_optimizations`] and -//! [`find_applicable_optimizations_with`] just translate [`PlanSpace`]'s +//! finding — [`explain_replacements`] and +//! [`explain_replacements_with`] just translate [`PlanSpace`]'s //! [`MemoGroup`]s into that shape: //! -//! - [`OptimizationKind::SketchApproximation`] — the `TargetSubDAG`'s +//! - [`ExplanationKind::SketchApproximation`] — the `TargetSubDAG`'s //! candidate list contains at least one [`Replacement::Summary`] that //! actually realizes a sketch family (`SummaryFamilyType::Sketch`), i.e. //! [`SketchAlgorithmStrategy`] found something to offer beyond whatever //! exact/pass-through candidate [`crate::replacement`]'s own //! `implementations_for_with` would have committed to on its own. -//! - [`OptimizationKind::CommonSubexpressionReuse`] — the `TargetSubDAG` +//! - [`ExplanationKind::CommonSubexpressionReuse`] — the `TargetSubDAG` //! has two or more consumers *and* its candidate list contains the //! [`SharedSubtreeStrategy`] "build once and share" candidate (the one //! whose `Rc` is the group's own `target`) — i.e. sharing this subtree @@ -55,13 +65,13 @@ //! //! ## What this replaces, and what carries over unmodified //! -//! [`ApplicabilityFinding`] and [`OptimizationKind`] keep PR #247's original +//! [`ReplacementExplanation`] and [`ExplanationKind`] keep PR #247's original //! shape and contract — a struct/enum pair meant for a downstream //! DAG-visualization consumer, `#[non_exhaustive]` discipline (only an //! optimization backed by a real, registered [`ReplacementStrategy`] gets a //! variant; see the catalog table below for everything still deliberately //! unrepresented). So do the two top-level entry points, -//! [`find_applicable_optimizations`] and [`find_applicable_optimizations_with`] +//! [`explain_replacements`] and [`explain_replacements_with`] //! — same "workload roots in, findings out" contract, mirroring //! [`crate::replacement::search_workload`]/[`crate::replacement::search_workload_with`]'s //! own signature shape. Only the *data source* changed: this module now @@ -75,8 +85,8 @@ //! ## Why a second, applicability-specific rule trait doesn't exist here //! //! PR #247 gave this module its own extension-point trait, `ApplicabilityRule` -//! (`fn optimization(&self) -> OptimizationKind` + `fn evaluate(&self, roots) -//! -> Vec`), the same shape [`crate::cost_model::CostModel`] +//! (`fn optimization(&self) -> ExplanationKind` + `fn evaluate(&self, roots) +//! -> Vec`), the same shape [`crate::cost_model::CostModel`] //! and [`crate::replacement::Matcher`] use elsewhere in this crate. Once //! findings are a *view* over [`PlanSpace`] rather than an independent //! computation, that trait would be a second extension point answering a @@ -91,7 +101,7 @@ //! would just be re-describing candidates the other (the strategy) already //! produced. So this module ships no extension-point trait of its own: //! [`ReplacementStrategy`] already *is* that extension point, one layer -//! down, and [`find_applicable_optimizations_with`]'s own `strategies` +//! down, and [`explain_replacements_with`]'s own `strategies` //! parameter is where a caller plugs in a custom one (or a custom //! `CostModel`, via [`crate::replacement::SketchAlgorithmStrategy::new`]) — the identical spot //! [`crate::replacement::search_workload_with`] itself exposes. @@ -116,7 +126,7 @@ //! [`PlanSpace`]'s internal map — there is exactly one group per distinct //! `Rc`, full stop, so a shared `Aggregate` reached via two different //! `BinaryOp` branches (or two different workload roots) is exactly one -//! group, hence at most one [`OptimizationKind::SketchApproximation`] +//! group, hence at most one [`ExplanationKind::SketchApproximation`] //! finding, no matter how many paths reach it. //! [`tests::a_shared_sketchable_aggregate_is_reported_only_once`] pins //! this directly. @@ -126,7 +136,7 @@ //! //! [`MemoGroup`]/[`PlanSpace`] deliberately track only `Rc` //! pointer identity — the currency the search itself needs — not -//! caller-facing prose. [`ApplicabilityFinding::location`] is prose (a +//! caller-facing prose. [`ReplacementExplanation::location`] is prose (a //! breadcrumb like `root "dash_a" > lhs`), so this module keeps one small, //! self-contained walk of its own, [`collect_locations`], whose *only* job //! is turning "this `Rc`" into "the human-readable place(s) it occurs" for a @@ -146,10 +156,10 @@ //! primitives with **no [`ReplacementStrategy`] implementation anywhere in //! this codebase today**. Faking a variant for one of them would report a //! finding this codebase cannot back with a real candidate, so none of the -//! below get an [`OptimizationKind`] variant yet — each gets one once a real +//! below get an [`ExplanationKind`] variant yet — each gets one once a real //! strategy exists and is wired into [`crate::replacement::default_strategies`]: //! -//! | Catalog entry | Status | Where a future `OptimizationKind` would come from | +//! | Catalog entry | Status | Where a future `ExplanationKind` would come from | //! |---|---|---| //! | Semantic-equivalent rewriting (e.g. `avg` → `sum`/`count`) | No `ReplacementStrategy` implementation in this codebase yet (tracked separately, issue #253) | Once wired into `default_strategies()`: any `Replacement::Rewrite` candidate that strategy proposes | //! | Roll-ups (fine-to-coarse group-by reuse) | No `ReplacementStrategy` implementation in this codebase yet (tracked separately, issue #254) | Once wired into `default_strategies()`: any `Replacement::Rewrite` candidate that strategy proposes | @@ -177,11 +187,12 @@ use std::fmt::Display; use std::rc::Rc; use asap_types::post_asap::{SummaryExpr, SummaryFamilyType, SummaryNode}; +use asap_types::pre_asap::cse::{structural_hash, HashCache}; use asap_types::pre_asap::query_expr::QueryExpr; use crate::replacement::{self, MemoGroup, PlanSpace, Replacement, ReplacementStrategy}; -/// Which optimization an [`ApplicabilityFinding`] is about. +/// Which kind of replacement a [`ReplacementExplanation`] is about. /// /// `#[non_exhaustive]`: only optimizations with a real [`ReplacementStrategy`] /// behind them get a variant (see the module docs' "Catalog primitives @@ -191,7 +202,7 @@ use crate::replacement::{self, MemoGroup, PlanSpace, Replacement, ReplacementStr /// [`ReplacementStrategy`]: crate::replacement::ReplacementStrategy #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[non_exhaustive] -pub enum OptimizationKind { +pub enum ExplanationKind { /// A `TargetSubDAG`'s candidate list contains at least one /// [`Replacement::Summary`] that realizes a sketch family — /// [`crate::replacement::SketchAlgorithmStrategy`] found a genuine sketch @@ -207,20 +218,30 @@ pub enum OptimizationKind { CommonSubexpressionReuse, } -/// One positive applicability result: `optimization` is applicable at -/// `location` (a human-readable breadcrumb into the workload — e.g. -/// `root "dashboard_p99"` or `root "ratio" > lhs`), for `reason` -/// (human-readable, meant for a report/log, not machine parsing — literally -/// the matching candidate's own [`crate::replacement::ReplacementSubDAG::rationale`]). +/// Why a [`Replacement`] of `kind` exists at `location` (a human-readable +/// breadcrumb into the workload — e.g. `root "dashboard_p99"` or +/// `root "ratio" > lhs`): `reason` (human-readable, meant for a report/log, +/// not machine parsing — literally the matching candidate's own +/// [`crate::replacement::ReplacementSubDAG::rationale`]). +/// +/// `node_hash` is [`structural_hash`](asap_types::pre_asap::cse::structural_hash) +/// of the `TargetSubDAG`'s own `target` subtree — the same function, on the +/// same `Rc` shape, that [`asap_types::dag_export::DagNode::hash`] +/// is computed with. A downstream consumer that independently exported the +/// same `QueryExpr` (e.g. via `asap_types::dag_export::export`) can match +/// this explanation to the exact `DagNode` it's about by comparing hashes — +/// no string-matching or path-guessing against `location` required. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct ApplicabilityFinding { - pub optimization: OptimizationKind, +pub struct ReplacementExplanation { + pub kind: ExplanationKind, pub location: String, pub reason: String, + pub node_hash: u64, } -/// Determine which known optimizations are applicable to a workload's -/// pre-ASAP query roots, using [`crate::replacement::default_strategies`]. +/// Explain every replacement [`crate::replacement::search_workload`] finds +/// across a workload's pre-ASAP query roots, using +/// [`crate::replacement::default_strategies`]. /// /// `roots` — like [`crate::replacement::search_workload`]'s own `Id` type /// parameter — is caller-chosen: a `QueryWorkload` entry's own key, an index, @@ -230,13 +251,13 @@ pub struct ApplicabilityFinding { /// Internally runs [`crate::replacement::search_workload`] to build the /// candidate-plan space, then reads findings off it — see the module docs' /// "The reframing" section for what that translation actually checks. -pub fn find_applicable_optimizations( +pub fn explain_replacements( roots: Vec<(Id, QueryExpr)>, -) -> Vec { - find_applicable_optimizations_with(roots, &replacement::default_strategies()) +) -> Vec { + explain_replacements_with(roots, &replacement::default_strategies()) } -/// Like [`find_applicable_optimizations`], but searches with `strategies` +/// Like [`explain_replacements`], but searches with `strategies` /// instead of [`crate::replacement::default_strategies`] — the extension /// point for a deployment-specific [`ReplacementStrategy`], or a custom /// `CostModel` plugged into @@ -244,10 +265,10 @@ pub fn find_applicable_optimizations( /// [`crate::replacement::default_strategies_with`]). /// /// [`ReplacementStrategy`]: crate::replacement::ReplacementStrategy -pub fn find_applicable_optimizations_with<'s, Id: Display>( +pub fn explain_replacements_with<'s, Id: Display>( roots: Vec<(Id, QueryExpr)>, strategies: &[Box], -) -> Vec { +) -> Vec { let ided: Vec<(String, Rc)> = roots .into_iter() .map(|(id, expr)| (id.to_string(), Rc::new(expr))) @@ -257,37 +278,45 @@ pub fn find_applicable_optimizations_with<'s, Id: Display>( } /// Translate every discovered [`MemoGroup`] in `space` into zero, one, or two -/// [`ApplicabilityFinding`]s (a `TargetSubDAG` can be both sketch-approximable +/// [`ReplacementExplanation`]s (a `TargetSubDAG` can be both sketch-approximable /// *and* shared — the two optimizations are independent axes, not mutually /// exclusive). /// -/// `space`'s own `Id` is always `String` here: [`find_applicable_optimizations_with`] +/// `space`'s own `Id` is always `String` here: [`explain_replacements_with`] /// already converted the caller's `Id: Display` into a `String` (via /// `to_string()`) before calling [`crate::replacement::search_workload_with`], /// so this function (and [`collect_locations`], which formats `id` with /// [`std::fmt::Debug`] for the breadcrumb text) doesn't need its own generic /// `Id` bound. -fn findings_from_plan_space(space: &PlanSpace) -> Vec { +fn findings_from_plan_space(space: &PlanSpace) -> Vec { let locations = collect_locations(&space.roots); + // One cache for the whole pass, mirroring `dag_export::export`'s own + // `HashCache` reuse — this is a bottom-up pass over every discovered + // group, so amortizing the cache across groups (rather than resetting it + // per group) is real, not just a micro-optimization. + let mut hash_cache = HashCache::new(); let mut findings = Vec::new(); for group in space.groups() { let location = locations .get(&Rc::as_ptr(&group.target)) .map(|locs| locs.join(", ")) .unwrap_or_default(); + let node_hash = structural_hash(&group.target, &mut hash_cache); if let Some(reason) = sketch_finding_reason(group) { - findings.push(ApplicabilityFinding { - optimization: OptimizationKind::SketchApproximation, + findings.push(ReplacementExplanation { + kind: ExplanationKind::SketchApproximation, location: location.clone(), reason, + node_hash, }); } if let Some(reason) = shared_subexpr_finding_reason(group) { - findings.push(ApplicabilityFinding { - optimization: OptimizationKind::CommonSubexpressionReuse, + findings.push(ReplacementExplanation { + kind: ExplanationKind::CommonSubexpressionReuse, location, reason, + node_hash, }); } } @@ -497,10 +526,10 @@ mod tests { #[test] fn approximate_quantile_is_a_sketch_applicability_finding() { let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); - let findings = find_applicable_optimizations(vec![("dashboard_p99", q)]); + let findings = explain_replacements(vec![("dashboard_p99", q)]); let sketch: Vec<_> = findings .iter() - .filter(|f| f.optimization == OptimizationKind::SketchApproximation) + .filter(|f| f.kind == ExplanationKind::SketchApproximation) .collect(); assert_eq!( sketch.len(), @@ -511,6 +540,29 @@ mod tests { assert!(sketch[0].reason.to_lowercase().contains("kll")); } + /// `node_hash` must be the literal `structural_hash` a downstream + /// consumer would compute over the *same* `QueryExpr` subtree via + /// `asap_types::dag_export::export` — the whole point of carrying it is + /// that two independent exports of the same tree agree, with no + /// string-matching against `location` required. + #[test] + fn node_hash_matches_dag_export_hash_for_the_same_subtree() { + let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); + let graph = asap_types::dag_export::export(&q); + let expected_hash = graph.nodes[graph.root as usize].hash; + + let findings = explain_replacements(vec![("dashboard_p99", q)]); + let sketch = findings + .iter() + .find(|f| f.kind == ExplanationKind::SketchApproximation) + .expect("expected a sketch finding"); + assert_eq!( + sketch.node_hash, expected_hash, + "ReplacementExplanation::node_hash must match dag_export's DagNode::hash \ + for the same QueryExpr subtree" + ); + } + #[test] fn exact_quantile_is_not_a_sketch_applicability_finding() { let q = agg( @@ -522,11 +574,11 @@ mod tests { }, metric_scan(&["job"]), ); - let findings = find_applicable_optimizations(vec![("exact_p99", q)]); + let findings = explain_replacements(vec![("exact_p99", q)]); assert!( findings .iter() - .all(|f| f.optimization != OptimizationKind::SketchApproximation), + .all(|f| f.kind != ExplanationKind::SketchApproximation), "an Exact accuracy target must not report sketch-applicability, got {findings:?}" ); } @@ -540,10 +592,10 @@ mod tests { // nested_aggregates_bind_per_node test uses. let inner = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); let outer = agg(vec![], default_quantile(0.9), inner); - let findings = find_applicable_optimizations(vec![("q", outer)]); + let findings = explain_replacements(vec![("q", outer)]); let sketch_count = findings .iter() - .filter(|f| f.optimization == OptimizationKind::SketchApproximation) + .filter(|f| f.kind == ExplanationKind::SketchApproximation) .count(); assert_eq!( sketch_count, 1, @@ -554,10 +606,10 @@ mod tests { #[test] fn pass_through_intent_reports_no_sketch_finding() { let q = agg(vec![2], AggIntent::Avg { col: None }, metric_scan(&["job"])); - let findings = find_applicable_optimizations(vec![("avg_latency", q)]); + let findings = explain_replacements(vec![("avg_latency", q)]); assert!(findings .iter() - .all(|f| f.optimization != OptimizationKind::SketchApproximation)); + .all(|f| f.kind != ExplanationKind::SketchApproximation)); } /// A sketch-applicable `Aggregate` reachable via two paths that CSE @@ -577,10 +629,10 @@ mod tests { rhs: Rc::new(quantile), vector_match: None, }; - let findings = find_applicable_optimizations(vec![("ratio", root)]); + let findings = explain_replacements(vec![("ratio", root)]); let sketch: Vec<_> = findings .iter() - .filter(|f| f.optimization == OptimizationKind::SketchApproximation) + .filter(|f| f.kind == ExplanationKind::SketchApproximation) .collect(); assert_eq!( sketch.len(), @@ -599,10 +651,10 @@ mod tests { // gate — and identical across both roots, so it is shareable. let a = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); let b = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let findings = find_applicable_optimizations(vec![("dash_a", a), ("dash_b", b)]); + let findings = explain_replacements(vec![("dash_a", a), ("dash_b", b)]); let reuse: Vec<_> = findings .iter() - .filter(|f| f.optimization == OptimizationKind::CommonSubexpressionReuse) + .filter(|f| f.kind == ExplanationKind::CommonSubexpressionReuse) .collect(); assert_eq!( reuse.len(), @@ -621,11 +673,11 @@ mod tests { AggIntent::Sum { col: None }, metric_scan(&["route"]), ); - let findings = find_applicable_optimizations(vec![("dash_a", a), ("dash_b", b)]); + let findings = explain_replacements(vec![("dash_a", a), ("dash_b", b)]); assert!( findings .iter() - .all(|f| f.optimization != OptimizationKind::CommonSubexpressionReuse), + .all(|f| f.kind != ExplanationKind::CommonSubexpressionReuse), "structurally different queries must not report reuse, got {findings:?}" ); } @@ -637,10 +689,10 @@ mod tests { // must not report a finding either. let a = agg(vec![], AggIntent::Sum { col: None }, metric_scan(&["job"])); let b = agg(vec![], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let findings = find_applicable_optimizations(vec![("a", a), ("b", b)]); + let findings = explain_replacements(vec![("a", a), ("b", b)]); assert!(findings .iter() - .all(|f| f.optimization != OptimizationKind::CommonSubexpressionReuse)); + .all(|f| f.kind != ExplanationKind::CommonSubexpressionReuse)); } #[test] @@ -656,10 +708,10 @@ mod tests { rhs: Rc::new(branch), vector_match: None, }; - let findings = find_applicable_optimizations(vec![("ratio", q)]); + let findings = explain_replacements(vec![("ratio", q)]); let reuse: Vec<_> = findings .iter() - .filter(|f| f.optimization == OptimizationKind::CommonSubexpressionReuse) + .filter(|f| f.kind == ExplanationKind::CommonSubexpressionReuse) .collect(); assert_eq!( reuse.len(), @@ -691,10 +743,10 @@ mod tests { pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Int64(2)))), child: Rc::new(shared), }; - let findings = find_applicable_optimizations(vec![("a", root_a), ("b", root_b)]); + let findings = explain_replacements(vec![("a", root_a), ("b", root_b)]); let reuse: Vec<_> = findings .iter() - .filter(|f| f.optimization == OptimizationKind::CommonSubexpressionReuse) + .filter(|f| f.kind == ExplanationKind::CommonSubexpressionReuse) .collect(); assert_eq!( reuse.len(), @@ -733,7 +785,7 @@ mod tests { let strategies: Vec> = vec![Box::new( crate::replacement::SketchAlgorithmStrategy::new(&custom_model), )]; - let findings = find_applicable_optimizations_with(vec![("q", q)], &strategies); + let findings = explain_replacements_with(vec![("q", q)], &strategies); assert_eq!(findings.len(), 1); assert!(findings[0].reason.to_lowercase().contains("ddsketch")); } diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index c3bf43bb..ac4ce3eb 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -85,13 +85,15 @@ //! — exposes an actual numeric cost per candidate, not just a relative //! rank, for a caller (e.g. a DAG-visualization view) that wants to show //! "candidate A costs ≈ X" next to "candidate B costs ≈ Y". -//! - [`applicability`] — a reporting *view* over [`replacement`]'s -//! candidate-plan space (issue #257, part of #33): translates every -//! discovered `TargetSubDAG` with a non-trivial candidate list into an -//! [`applicability::ApplicabilityFinding`] ("which optimizations are -//! applicable, where, and why"), meant for the same downstream consumer -//! (e.g. a DAG-visualization view) the crate doc's `## Status` section -//! above already names for [`replacement::PlanSpace`] itself. Superseded PR +//! - [`explanation`] — this crate's explanation of a replacement: a +//! reporting *view* over [`replacement`]'s candidate-plan space (issue +//! #257, part of #33) that translates every discovered `TargetSubDAG` with +//! a non-trivial candidate list into an +//! [`explanation::ReplacementExplanation`] (why a replacement exists, +//! where, reusing the candidate's own rationale rather than inventing new +//! prose), meant for the same downstream consumer (e.g. a +//! DAG-visualization view) the crate doc's `## Status` section above +//! already names for [`replacement::PlanSpace`] itself. Superseded PR //! #247's own rule-based traversal, which re-walked the tree once per //! optimization before [`replacement::search_workload`] existed to read //! from instead — see that module's docs for the full reframing. @@ -136,15 +138,14 @@ //! `sketch_algebra::capability::Capability`/`is_satisfied_by` is the //! reference downstream implementation. -pub mod applicability; pub mod cost_model; +pub mod explanation; pub mod replacement; -pub use applicability::{ - find_applicable_optimizations, find_applicable_optimizations_with, ApplicabilityFinding, - OptimizationKind, -}; pub use cost_model::{CostModel, DefaultCostModel}; +pub use explanation::{ + explain_replacements, explain_replacements_with, ExplanationKind, ReplacementExplanation, +}; pub use replacement::{ default_strategies, default_strategies_with, search_workload, search_workload_with, summary_candidates, ImplementError, Implementation, Matcher, MemoGroup, PlanSpace, RankedGroup, diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index 9019ab03..216a56c2 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -46,10 +46,10 @@ //! a restructuring of this trait or of any existing strategy. `replacements` //! is **exhaustive, not ranked, not filtered** — reporting "every valid //! candidate" is core's job; picking the best one is left to the caller. -//! [`crate::applicability`] (issue #257) is this trait's own downstream -//! consumer, not a second extension point: it reports "which optimizations -//! are applicable" as a pure view over the candidates strategies registered -//! here already produced, rather than re-deriving applicability with a rule +//! [`crate::explanation`] (issue #257) is this trait's own downstream +//! consumer, not a second extension point: it explains why a replacement +//! exists as a pure view over the candidates strategies registered here +//! already produced, rather than re-deriving that explanation with a rule //! of its own. //! //! A caller that wants one executable answer takes the first @@ -357,7 +357,7 @@ pub enum Replacement { /// One candidate replacement for a [`TargetSubDAG`], plus a human-readable /// `rationale` explaining why it's a valid candidate (meant for a /// report/log/debugging a search engine's choices, not machine parsing — -/// [`crate::applicability::ApplicabilityFinding::reason`] literally reuses +/// [`crate::explanation::ReplacementExplanation::reason`] literally reuses /// this same string rather than inventing new prose of its own. #[derive(Debug, Clone)] pub struct ReplacementSubDAG { @@ -1041,7 +1041,7 @@ fn describe_implementation(intent: &AggIntent, implementation: &Implementation) /// this crate's other `AggIntent` matches, e.g. [`implementations_for_with`]'s) /// — this is prose for a rationale string, not a decision, so an unlisted /// variant just falls back to its `Debug` tag rather than forcing every -/// future intent to be named here too. [`crate::applicability`] needs no +/// future intent to be named here too. [`crate::explanation`] needs no /// counterpart of its own: it reads a candidate's `rationale` — built from /// this text — straight off [`ReplacementSubDAG`], rather than re-describing /// the same intent a second time. @@ -1764,9 +1764,9 @@ fn sketch_kind_of(node: &SummaryNode) -> Option { /// The strategies [`search_workload`] runs, in the built-in /// [`DefaultCostModel`] configuration — mirrors this module's own two /// shipped [`ReplacementStrategy`] impls. -/// [`crate::applicability::find_applicable_optimizations`] (issue #257) uses +/// [`crate::explanation::explain_replacements`] (issue #257) uses /// this same set (via [`search_workload`]) rather than keeping a second, -/// applicability-specific list to stay in sync with. Use +/// explanation-specific list to stay in sync with. Use /// [`default_strategies_with`] to plug in a deployment-specific /// [`CostModel`] instead. pub fn default_strategies() -> Vec> { @@ -1808,7 +1808,7 @@ pub fn search_workload(roots: Vec<(Id, Rc)>) -> PlanSpace { /// /// Runs [`share_common_subtrees`] once over `roots` first — so every /// strategy (and, transitively, every -/// [`crate::applicability::ApplicabilityFinding`] a caller reads off the +/// [`crate::explanation::ReplacementExplanation`] a caller reads off the /// result) sees the same already-deduplicated tree — then discovers every /// `TargetSubDAG` (see [`discover_targets`]) and runs the /// fixpoint loop the module docs describe, capped at @@ -2938,7 +2938,7 @@ mod tests { #[test] fn realistic_cse_output_produces_a_two_consumer_target() { // Two workload roots that `share_common_subtrees` collapses onto one - // Rc (mirrors `applicability`'s and `cse`'s own fixtures): a grouped + // Rc (mirrors `explanation`'s and `cse`'s own fixtures): a grouped // Sum aggregate over the same scan, built independently at each root. let a = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); let b = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); diff --git a/docs/design_docs/asap_aware_mapping.md b/docs/design_docs/asap_aware_mapping.md index 7c63e941..821c5b58 100644 --- a/docs/design_docs/asap_aware_mapping.md +++ b/docs/design_docs/asap_aware_mapping.md @@ -689,14 +689,14 @@ The legality and cost of this transformation depend on: --- -# Applicability Reporting +# Explaining a Replacement ASAP-aware mapping should make its discovered opportunities visible to users and other components. -For each relevant region of the plan, the planner should be able to report: +For each relevant region of the plan, the planner should be able to explain: -- which transformations are applicable, -- what alternatives each transformation introduces, +- which replacements exist, +- what alternatives each replacement introduces, - what conditions make those alternatives legal, - and which parts of the plan they affect. @@ -705,7 +705,7 @@ For example: ```text Aggregation: Quantile(latency, 0.99) -Applicable alternatives: +Available alternatives: - exact quantile - KLL - DDSketch @@ -715,7 +715,7 @@ Additional opportunities: - reuse finer-grained aggregation through roll-up ``` -Applicability is therefore a **view of the candidate search space**, not a second optimization engine. +Implemented as `asap-aware-mapping`'s `explanation` module (`explain_replacements`/`explain_replacements_with`, issue #257): a **view of the candidate search space**, not a second optimization engine — each explanation's reason is the matching candidate's own rationale, not new prose invented by the reporting layer. This keeps explanations consistent with the plans the optimizer can actually produce. diff --git a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md index 11b35d6c..74060651 100644 --- a/docs/developer_docs/ASAP-aware-mapping-developer-guide.md +++ b/docs/developer_docs/ASAP-aware-mapping-developer-guide.md @@ -412,7 +412,7 @@ is enough for the current shared-subtree strategy. Keep `matches` cheap and unsurprising. -It should answer *whether the strategy applies here* in the plain English sense — not `applicability.rs`'s formal `ApplicabilityFinding` concept (issue #257, a separate reporting layer over this trait's own output — see §19). `matches` isn't that machinery and doesn't need to produce anything it consumes; it should also not perform ranking or choose a winner. +It should answer *whether the strategy applies here* in the plain English sense — not `explanation.rs`'s formal `ReplacementExplanation` concept (issue #257, a separate reporting layer over this trait's own output — see §19). `matches` isn't that machinery and doesn't need to produce anything it consumes; it should also not perform ranking or choose a winner. --- @@ -1312,51 +1312,53 @@ Use this table to find the right place for a change. | Enumerate valid sketch algorithms | reuse `replacement::summary_candidates` | | Build a target with no workload context | `TargetSubDAG::new` | | Build a target with known sharing context | `TargetSubDAG::with_consumer_count` | -| Report which optimizations are applicable, where, and why | `applicability::find_applicable_optimizations`/`find_applicable_optimizations_with` | -| Add a new kind of applicability finding | new `impl ReplacementStrategy`, wired into `default_strategies`/`default_strategies_with` — not a new applicability-specific trait, see §19 | +| Explain why a replacement exists, where, and why | `explanation::explain_replacements`/`explain_replacements_with` | +| Add a new kind of replacement explanation | new `impl ReplacementStrategy`, wired into `default_strategies`/`default_strategies_with` — not a new explanation-specific trait, see §19 | --- -## 19. Applicability reporting (`applicability.rs`) +## 19. Explaining a replacement (`explanation.rs`) -`applicability::find_applicable_optimizations`/`find_applicable_optimizations_with` answer a different question than everything above: not "what could this target become" (`ReplacementStrategy::replacements`) but "which of the optimizations already discovered are worth telling a user about, and why." It is a **reporting view over `PlanSpace`**, not a second search or a second rule engine. +`explanation::explain_replacements`/`explain_replacements_with` answer a different question than everything above: not "what could this target become" (`ReplacementStrategy::replacements`) but "why does the replacement already discovered for this target exist, and where." It is a **reporting view over `PlanSpace`**, not a second search or a second rule engine — this crate's *explanation of a replacement*, not an applicability classifier deciding admissibility from scratch. ### The rule -> A `TargetSubDAG` is applicability-worthy exactly when its `PlanSpace` candidate list contains something beyond the trivial, no-op realization. +> A `TargetSubDAG` is worth explaining exactly when its `PlanSpace` candidate list contains something beyond the trivial, no-op realization. -Concretely, `applicability.rs` reads two shapes off each `MemoGroup`: +Concretely, `explanation.rs` reads two shapes off each `MemoGroup`: -- `OptimizationKind::SketchApproximation` — the group's candidates include a `Replacement::Summary` that actually realizes `SummaryFamilyType::Sketch(..)`, i.e. `SketchAlgorithmStrategy` found a real sketch alternative, not just an exact/pass-through candidate. -- `OptimizationKind::CommonSubexpressionReuse` — `consumer_count >= 2` and the group's candidates include `SharedSubtreeStrategy`'s "build once and share" candidate (the `Replacement::Rewrite` whose `Rc` is the group's own `target`). +- `ExplanationKind::SketchApproximation` — the group's candidates include a `Replacement::Summary` that actually realizes `SummaryFamilyType::Sketch(..)`, i.e. `SketchAlgorithmStrategy` found a real sketch alternative, not just an exact/pass-through candidate. +- `ExplanationKind::CommonSubexpressionReuse` — `consumer_count >= 2` and the group's candidates include `SharedSubtreeStrategy`'s "build once and share" candidate (the `Replacement::Rewrite` whose `Rc` is the group's own `target`). -Each `ApplicabilityFinding::reason` is copied verbatim from the matching candidate's own `ReplacementSubDAG::rationale`. Nothing in `applicability.rs` re-explains why a candidate is valid; that explanation already exists exactly once, on the candidate itself. +Each `ReplacementExplanation::reason` is copied verbatim from the matching candidate's own `ReplacementSubDAG::rationale`. Nothing in `explanation.rs` re-explains why a candidate is valid; that explanation already exists exactly once, on the candidate itself. -### Why there is no `ApplicabilityRule` trait +`ReplacementExplanation` also carries `node_hash: u64` — `asap_types::pre_asap::cse::structural_hash` of the `TargetSubDAG`'s own `target` subtree, the identical function (and identical `Rc` input shape) `asap_types::dag_export::DagNode::hash` is computed with. A downstream consumer that independently exported the same `QueryExpr` (e.g. `tools/dag-viewer`'s `dag_export` devtools binary) can match an explanation to the exact `DagNode` it's about by comparing hashes, with no string-matching or path-guessing against `location` required — see `crates/devtools/src/bin/dag_export.rs` for the reference consumer. -An earlier version of this module (superseded, PR #247) had its own extension-point trait for adding a new optimization to the report. It is gone. Once findings are read off `PlanSpace`, a new applicability finding needs a new `impl ReplacementStrategy` wired into `default_strategies`/`default_strategies_with` regardless — that is the only way a new kind of candidate reaches the `PlanSpace` this module reads. A second, applicability-specific extension point would just be a second place to register the same thing. `find_applicable_optimizations_with`'s own `strategies: &[Box]` parameter is where a caller plugs in something custom — the same customization point `search_workload_with` itself exposes. +### Why there is no `ExplanationRule` trait + +An earlier version of this module (superseded, PR #247, under the name `applicability.rs`) had its own extension-point trait for adding a new optimization to the report. It is gone. Once explanations are read off `PlanSpace`, a new explanation needs a new `impl ReplacementStrategy` wired into `default_strategies`/`default_strategies_with` regardless — that is the only way a new kind of candidate reaches the `PlanSpace` this module reads. A second, explanation-specific extension point would just be a second place to register the same thing. `explain_replacements_with`'s own `strategies: &[Box]` parameter is where a caller plugs in something custom — the same customization point `search_workload_with` itself exposes. ### What it still owns: `location` text -`PlanSpace`/`MemoGroup` track `Rc` pointer identity, not human-readable breadcrumbs. `applicability.rs` keeps one small, self-contained traversal, `collect_locations`, whose only job is turning "this `Rc`" into prose like `root "dash_a" > lhs` for `ApplicabilityFinding::location`. It makes no applicability decision — it runs identically regardless of what any strategy found. +`PlanSpace`/`MemoGroup` track `Rc` pointer identity, not human-readable breadcrumbs. `explanation.rs` keeps one small, self-contained traversal, `collect_locations`, whose only job is turning "this `Rc`" into prose like `root "dash_a" > lhs` for `ReplacementExplanation::location`. It makes no explanation decision — it runs identically regardless of what any strategy found. ### Using it ```rust -use asap_aware_mapping::{find_applicable_optimizations, OptimizationKind}; - -let findings = find_applicable_optimizations(vec![("dashboard_p99", query)]); -for finding in &findings { - match finding.optimization { - OptimizationKind::SketchApproximation => { /* ... */ } - OptimizationKind::CommonSubexpressionReuse => { /* ... */ } - _ => { /* OptimizationKind is #[non_exhaustive] */ } +use asap_aware_mapping::{explain_replacements, ExplanationKind}; + +let explanations = explain_replacements(vec![("dashboard_p99", query)]); +for explanation in &explanations { + match explanation.kind { + ExplanationKind::SketchApproximation => { /* ... */ } + ExplanationKind::CommonSubexpressionReuse => { /* ... */ } + _ => { /* ExplanationKind is #[non_exhaustive] */ } } } ``` -To plug in a deployment-specific strategy or `CostModel`, use `find_applicable_optimizations_with` with a strategy set built the same way `default_strategies_with` builds one — see §8 and §11. +To plug in a deployment-specific strategy or `CostModel`, use `explain_replacements_with` with a strategy set built the same way `default_strategies_with` builds one — see §8 and §11. -### Adding a new applicability finding +### Adding a new kind of replacement explanation -There is no separate checklist here: follow [§4](#4-adding-a-new-replacementstrategy) to add the new `ReplacementStrategy`, wire it into `default_strategies`/`default_strategies_with` (§18), add an `OptimizationKind` variant, and extend `findings_from_plan_space` to recognize the new candidate shape. If the new strategy's `matches`/`replacements` are already correct and tested, the applicability finding falls out of the existing `PlanSpace` translation with no new discovery logic required. +There is no separate checklist here: follow [§4](#4-adding-a-new-replacementstrategy) to add the new `ReplacementStrategy`, wire it into `default_strategies`/`default_strategies_with` (§18), add an `ExplanationKind` variant, and extend `findings_from_plan_space` to recognize the new candidate shape. If the new strategy's `matches`/`replacements` are already correct and tested, the explanation falls out of the existing `PlanSpace` translation with no new discovery logic required. From fedf0ed34c9071bbb89db7d40c7ab14db4aeac07 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 24 Aug 2026 13:07:49 -0600 Subject: [PATCH 30/49] feat(dag_export): wire replacement explanations into the DAG exporter and viewer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give asap_types::dag_export::DagNode a generic, crate-agnostic `notes: Vec` field (kind/reason strings; empty and omitted from JSON by default) — a layering seam so a higher crate can annotate an already-exported graph without asap_types knowing anything about "replacements". asap_types never populates it itself. The devtools dag_export binary is the first populator: after exporting each query's QueryExpr, it also calls asap_aware_mapping::explain_replacements on the same tree and, for every ReplacementExplanation returned, matches its node_hash against each DagNode's hash (both sides compute the identical structural_hash) and pushes a DagNote onto every match. A miss logs to stderr rather than panicking (exploratory tooling). Add an `--epsilon ` flag to the binary (default preserves the existing AccuracyTarget::Exact behavior) so a caller can actually exercise the SketchApproximation path — without it every AggIntent lowers exact and no sketch alternative ever exists to report. tools/dag-viewer: nodes with a non-empty `notes` array get a small colored badge (color keyed to ExplanationKind, bottom-right corner so it never collides with the existing root badge), and clicking the node shows each note's kind/reason in the side panel next to the existing detail/root/shared-subtree sections. README.md documents the new `notes` field and `--epsilon`; dag.example.json (regenerated via generate-sample.sh, now passing --epsilon 0.01) demonstrates a populated notes array out of the box. Co-Authored-By: Claude Sonnet 5 --- crates/devtools/src/bin/dag_export.rs | 93 +++++++++++++++++++++------ crates/types/src/dag_export.rs | 59 +++++++++++++++++ tools/dag-viewer/README.md | 58 +++++++++++++++++ tools/dag-viewer/dag.example.json | 50 ++++++++------ tools/dag-viewer/generate-sample.sh | 5 ++ tools/dag-viewer/index.html | 66 ++++++++++++++++++- tools/dag-viewer/node-style.js | 45 +++++++++++++ 7 files changed, 334 insertions(+), 42 deletions(-) diff --git a/crates/devtools/src/bin/dag_export.rs b/crates/devtools/src/bin/dag_export.rs index 7c71eb68..41e232d6 100644 --- a/crates/devtools/src/bin/dag_export.rs +++ b/crates/devtools/src/bin/dag_export.rs @@ -8,12 +8,23 @@ // cargo run -p asap-lower --bin dag_export -- --sql "..." --name q1 > /tmp/dag.json // // `--name` is optional; an unnamed query defaults to `q` (1-indexed). +// +// `--epsilon ` is optional and applies to every query in the run: it +// lowers with `AccuracyTarget::Epsilon()` instead of the default +// `AccuracyTarget::Exact`. Without it, every `AggIntent` lowers exact and +// `asap_aware_mapping::SketchAlgorithmStrategy` never has a genuine sketch +// alternative to report — so no node ever picks up a `SketchApproximation` +// note. Pass it to actually exercise that path, e.g.: +// cargo run -p asap-lower --bin dag_export -- \ +// --epsilon 0.01 --sql "SELECT quantile(0.99, latency) FROM metrics" --name p99 -use asap_devtools::{lower_promql, lower_sql, SqlCatalog}; -use asap_types::dag_export::{self, NamedGraph, WorkloadGraph}; +use asap_types::dag_export::{self, DagGraph, DagNote, NamedGraph, WorkloadGraph}; +use asap_types::pre_asap::query_expr::QueryExpr; use asap_types::pre_asap::schema::{Column, DataType, Schema}; use asap_types::types::AccuracyTarget; +use asap_devtools::{lower_promql, lower_sql, SqlCatalog}; + enum Lang { Sql, PromQl, @@ -45,11 +56,13 @@ fn catalog() -> SqlCatalog { } /// Parses `--sql "" --name "