From 2ee38d15cb3a0cc9d75c37cf3a8d1602aae0ebdb Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 18 Jul 2026 10:58:25 -0600 Subject: [PATCH 01/11] feat(control_plane): Phase 2 step 1 -- merge schema.rs onto asap-ir Column/ColumnId/DataType/Schema/CseError/cse_reuse_is_legal are now re-exported from asap_ir::intent_algebra::schema instead of defined locally. Unlike AggIntent's merge (Phase 1b), this needed no boundary- conversion layer: fresh diff showed asap_ir's version is a purely additive, backward-compatible superset of the pre-merge local type -- - Column gains `table: Option` (SQL join qualifier) + Column::new()/with_table() constructors. - Schema gains `closed: bool` (schema-on-read completeness flag) + column_id_qualified(). - DataType is byte-identical, no changes. Both new fields are #[serde(default)], confirmed backward-compatible by asap_ir's own tests. This is what made a full swap the right call here instead of Phase 1b's boundary-conversion approach for Column/DataType -- deleted agg_intent.rs's to_asap_column/from_asap_column/to_asap_dtype/ from_asap_dtype helpers, no longer needed once there's only one Column/DataType type. Blast radius was much smaller than the migration plan's stale ~38-site estimate: 15 real Column{}/Schema{} struct-literal construction sites across 9 files needed the new field added (table: None / closed: false) -- most of the original grep hits were field references or doc comments, not constructions. Verified: full workspace builds clean; control_plane's 820-test suite passes unchanged (same 1 pre-existing failure as #392, confirmed unrelated). Remaining Phase 2 work (docs/migration-plan-backend-plan.md): expr_ir.rs, query_expr.rs/relational.rs fresh diff, binder.rs/column_resolution.rs fresh diff, cse.rs move to L4, lower.rs, PromQL/SQL frontend retarget. --- .../src/intent_algebra/agg_intent.rs | 52 +-- control_plane/src/intent_algebra/binder.rs | 7 + .../src/intent_algebra/column_resolution.rs | 7 + control_plane/src/intent_algebra/cse.rs | 1 + .../src/intent_algebra/query_expr.rs | 3 + control_plane/src/intent_algebra/schema.rs | 364 ++---------------- control_plane/src/optimizer/cost/mod.rs | 1 + control_plane/src/optimizer/rules/mod.rs | 2 + .../src/physical/colored_dag/allocator.rs | 2 + .../src/physical/colored_dag/tests.rs | 3 + .../src/sketch_algebra/physical_expr.rs | 3 + .../sketch_algebra/rules/bind_archive_only.rs | 3 + control_plane/src/sketch_algebra/tests.rs | 1 + 13 files changed, 77 insertions(+), 372 deletions(-) diff --git a/control_plane/src/intent_algebra/agg_intent.rs b/control_plane/src/intent_algebra/agg_intent.rs index d84c145e..902c15f3 100644 --- a/control_plane/src/intent_algebra/agg_intent.rs +++ b/control_plane/src/intent_algebra/agg_intent.rs @@ -8,6 +8,12 @@ //! ASAPController has no tagged releases yet) and holds only what's //! genuinely control_plane-specific: //! +//! (Phase 2: `Column`/`DataType` are also re-exported from `asap_ir` now +//! — `schema.rs` got the full swap, not a boundary conversion, since +//! asap_ir's version turned out to be a purely additive, backward- +//! compatible superset. `output_column` below no longer needs a +//! conversion layer as a result.) +//! //! - **`frequency()` / `as_frequency()`** — control_plane's standalone //! point-frequency-via-CMS query (`count(*) WHERE key = k`), carried //! through the shared `AggIntent` as an `Extension` rather than a @@ -43,7 +49,6 @@ //! in scope, not off the intent. use asap_ir::intent_algebra::agg_accuracy as asap_agg_accuracy; -use asap_ir::intent_algebra::schema::{Column as AsapColumn, DataType as AsapDataType}; pub use asap_ir::intent_algebra::{ agg_is_exact, agg_is_mergeable, default_cardinality, default_quantile, is_frequency_heavy_hitter, ranking_measure, AggIntent, MathFunc, RankingMeasure, TimeFunc, @@ -54,43 +59,6 @@ use crate::types_v2::AccuracyTarget; const FREQUENCY_EXT_KIND: &str = "frequency"; -/// `control_plane::Column`/`DataType` and `asap_ir::Column`/`DataType` -/// are structurally identical but not the same type — merging `schema.rs` -/// itself is Phase 2 scope (it cascades into `Schema`/`QueryExpr`, used -/// pervasively; ~38 `Column{}` literals across this repo). Convert at -/// this boundary instead of widening this change. -fn to_asap_dtype(dt: &DataType) -> AsapDataType { - match dt { - DataType::Int64 => AsapDataType::Int64, - DataType::Float64 => AsapDataType::Float64, - DataType::Utf8 => AsapDataType::Utf8, - DataType::Bool => AsapDataType::Bool, - DataType::Timestamp => AsapDataType::Timestamp, - } -} - -fn from_asap_dtype(dt: &AsapDataType) -> DataType { - match dt { - AsapDataType::Int64 => DataType::Int64, - AsapDataType::Float64 => DataType::Float64, - AsapDataType::Utf8 => DataType::Utf8, - AsapDataType::Bool => DataType::Bool, - AsapDataType::Timestamp => DataType::Timestamp, - } -} - -fn to_asap_column(c: &Column) -> AsapColumn { - AsapColumn::new(c.name.clone(), to_asap_dtype(&c.dtype), c.nullable) -} - -fn from_asap_column(c: AsapColumn) -> Column { - Column { - name: c.name, - dtype: from_asap_dtype(&c.dtype), - nullable: c.nullable, - } -} - /// Construct control_plane's point-frequency-via-CMS intent. See module /// docs for why this is an `Extension`, not a shared first-class variant. pub fn frequency(accuracy: AccuracyTarget) -> AggIntent { @@ -190,11 +158,7 @@ pub fn archive_only(intent: &AggIntent) -> bool { /// `"frequency"` — that's control_plane-only knowledge). pub fn output_column(intent: &AggIntent, input: &Column) -> Column { if as_frequency(intent).is_some() { - return Column { - name: "frequency".into(), - dtype: DataType::Int64, - nullable: false, - }; + return Column::new("frequency", DataType::Int64, false); } - from_asap_column(intent.output_column(&to_asap_column(input))) + intent.output_column(input) } diff --git a/control_plane/src/intent_algebra/binder.rs b/control_plane/src/intent_algebra/binder.rs index 794e93e4..1f1f7495 100644 --- a/control_plane/src/intent_algebra/binder.rs +++ b/control_plane/src/intent_algebra/binder.rs @@ -137,6 +137,7 @@ impl Binder { name, dtype: DataType::Utf8, // labels / group keys are strings nullable: true, + table: None, }); } } @@ -146,6 +147,7 @@ impl Binder { columns, time_index, unique_keys: Vec::new(), + closed: false, } } } @@ -157,11 +159,13 @@ fn default_leaf_columns() -> Vec { name: "ts".into(), dtype: DataType::Timestamp, nullable: false, + table: None, }, Column { name: "value".into(), dtype: DataType::Float64, nullable: false, + table: None, }, ] } @@ -269,16 +273,19 @@ mod tests { name: "ts".into(), dtype: DataType::Timestamp, nullable: false, + table: None, }, Column { name: "value".into(), dtype: DataType::Float64, nullable: false, + table: None, }, Column { name: "datacenter".into(), dtype: DataType::Utf8, nullable: false, + table: None, }, ]) } else { diff --git a/control_plane/src/intent_algebra/column_resolution.rs b/control_plane/src/intent_algebra/column_resolution.rs index 89357c7c..744abc47 100644 --- a/control_plane/src/intent_algebra/column_resolution.rs +++ b/control_plane/src/intent_algebra/column_resolution.rs @@ -91,11 +91,13 @@ pub fn infer_source_schema(_metric_or_table_name: &str) -> Schema { name: "ts".into(), dtype: DataType::Timestamp, nullable: false, + table: None, }, Column { name: "value".into(), dtype: DataType::Float64, nullable: false, + table: None, }, ], 0, @@ -265,6 +267,7 @@ pub fn output_schema_for_aggregate(input: &Schema, by: &[ColumnId], aggs: &[AggI name: "value".into(), dtype: DataType::Float64, nullable: false, + table: None, }); for intent in aggs { out_cols.push(crate::intent_algebra::output_column(intent, &probe)); @@ -280,6 +283,7 @@ pub fn output_schema_for_aggregate(input: &Schema, by: &[ColumnId], aggs: &[AggI columns: out_cols, time_index: None, unique_keys, + closed: false, } } @@ -402,11 +406,13 @@ mod tests { name: "host".into(), dtype: DataType::Utf8, nullable: false, + table: None, }); input.columns.push(Column { name: "region".into(), dtype: DataType::Utf8, nullable: false, + table: None, }); // Group by host, region (positions 2 and 3). let by = vec![2usize, 3usize]; @@ -439,6 +445,7 @@ mod tests { name: "host".into(), dtype: DataType::Utf8, nullable: false, + table: None, }); let ids = resolve_named_keys(&["host".to_string()], &s).unwrap(); assert_eq!(ids, vec![2usize]); diff --git a/control_plane/src/intent_algebra/cse.rs b/control_plane/src/intent_algebra/cse.rs index 0863d6ac..62c91465 100644 --- a/control_plane/src/intent_algebra/cse.rs +++ b/control_plane/src/intent_algebra/cse.rs @@ -170,6 +170,7 @@ mod tests { name: name.into(), dtype, nullable: false, + table: None, } } diff --git a/control_plane/src/intent_algebra/query_expr.rs b/control_plane/src/intent_algebra/query_expr.rs index cb3aed3d..c54ccacb 100644 --- a/control_plane/src/intent_algebra/query_expr.rs +++ b/control_plane/src/intent_algebra/query_expr.rs @@ -560,6 +560,7 @@ impl QueryExpr { name: "value".into(), dtype: DataType::Float64, nullable: false, + table: None, }); for intent in aggs { out_cols.push(crate::intent_algebra::output_column(intent, &probe)); @@ -578,6 +579,7 @@ impl QueryExpr { columns: out_cols, time_index: None, unique_keys, + closed: false, }) } QueryExpr::LetBinding { name, expr, child } => { @@ -794,6 +796,7 @@ mod tests { name: name.into(), dtype, nullable: false, + table: None, } } diff --git a/control_plane/src/intent_algebra/schema.rs b/control_plane/src/intent_algebra/schema.rs index 0a895cce..75d2d989 100644 --- a/control_plane/src/intent_algebra/schema.rs +++ b/control_plane/src/intent_algebra/schema.rs @@ -1,332 +1,40 @@ //! Layer 3 schema flow — every L3 edge carries a typed `Schema`. //! -//! Per `control_plane/docs/design.md` §6 "Schema flow — every L3 edge carries -//! a typed schema". The DAG is type-checked: a node's output schema is a -//! function of its inputs and parameters and is verifiable independently -//! of the surrounding context. +//! ## Phase 2 (docs/migration-plan-backend-plan.md) //! -//! `Schema::unique_keys` is the load-bearing field for the workload-level -//! CSE pass (`design.md` §6 "DAG, not tree" + the batched-queries example -//! around line ~1284). Two `QueryExpr::Ref` consumers can share a producer -//! only when its output schema is provably stable across reads — the -//! unique-key metadata is what lets the deduper assert that. +//! `Column` / `ColumnId` / `DataType` / `Schema` / `CseError` / +//! `cse_reuse_is_legal` are no longer defined in this repo — re-exported +//! from `asap_ir::intent_algebra::schema`. Unlike `AggIntent`'s merge +//! (Phase 1b), this one needed no boundary-conversion layer: asap_ir's +//! version is a pure additive superset of control_plane's pre-merge +//! version — //! -//! Single-query plans don't read this field; it lives here so the metadata -//! is available the moment workload-aware planning lands without requiring -//! an L3-wide schema change. - -#![allow(dead_code)] - -use serde::{Deserialize, Serialize}; - -/// Index into [`Schema::columns`] used everywhere a column position is -/// referenced (group-by keys, unique-key sets, the time axis index). -/// -/// Aliased to `usize` to match `design.md`'s `Vec>` for -/// `unique_keys`. Kept as a named type so downstream code can pattern on -/// the intent ("this is a column position, not just any number"). -pub type ColumnId = usize; - -/// One column in a [`Schema`]. Mirrors `design.md` §6 `Field` — -/// `name + dtype + nullable`. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct Column { - /// Column name as it appears in the producer's output. PromQL leaves - /// produce label-name + the synthetic `value` / `timestamp` columns; - /// SQL leaves carry their `information_schema` names. - pub name: String, - /// Column data type. Kept narrow at L3 (`Int64` / `Float64` / `Utf8` - /// / `Bool` / `Timestamp`); `Sketch(...)` is an L4-only addition per - /// design.md §6.4 and is intentionally absent here. - pub dtype: DataType, - /// Whether NULL values are allowed in this column. PromQL value - /// columns are non-nullable; SQL columns inherit their DDL nullability. - pub nullable: bool, -} - -/// L3 column data types. Deliberately narrow: no sketch state at this -/// layer (see `design.md` §6.4 for the L4 `DataType::Sketch(...)` -/// extension). -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum DataType { - /// 64-bit signed integer. Counter columns, group-cardinality outputs. - Int64, - /// 64-bit IEEE-754 float. Quantile / Avg / Sum-over-floats output. - Float64, - /// UTF-8 string. PromQL label values, SQL `VARCHAR` / `TEXT`. - Utf8, - /// Boolean — predicate output, `unless` / `and` / `or` PromQL ops. - Bool, - /// Wall-clock timestamp. PromQL leaves carry exactly one of these - /// (the `time_index` column); SQL leaves may or may not. - Timestamp, -} - -/// Per-edge L3 schema. Flowing between any two L3 operators, on every -/// node's input and output. -/// -/// `unique_keys` is metadata for reuse-aware planning: each inner `Vec` -/// is a set of column indices that together uniquely identify rows. The -/// outer `Vec` allows multiple unique-key sets (primary key + another -/// unique constraint). Populated by per-node input/output spec — -/// `Aggregate { by, .. }` emits `unique_keys = [by]`; `Distinct { cols }` -/// adds `cols`; most other nodes pass through. -/// -/// **Consumed by**: workload-level CSE (`CostModel::workload_cost` in the -/// design, not yet shipped). The single-query path, the `Bind*` rules, -/// push-down, and L5 emitters do not read this field. -#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] -pub struct Schema { - /// Columns flowing on this edge, in positional order. - pub columns: Vec, - /// Index into `columns` for the time axis, if any. PromQL leaves - /// always carry one; SQL leaves may or may not. - #[serde(default)] - pub time_index: Option, - /// Unique-key sets — each inner vec is a tuple of column indices - /// that together uniquely identifies a row. Empty `Vec` means - /// "no provable unique constraint" (the conservative default). - #[serde(default)] - pub unique_keys: Vec>, -} - -impl Schema { - /// Construct a `Schema` from columns alone — no time index, no - /// unique-key constraint. Used by `Scan` over a tabular source - /// when the catalog supplies no primary-key metadata. - pub fn new(columns: Vec) -> Self { - Self { - columns, - time_index: None, - unique_keys: Vec::new(), - } - } - - /// Construct a `Scan`-style schema with explicit `time_index` + - /// inferred unique keys (e.g. PromQL leaves: `[time_index, label_set]`). - pub fn with_time_index( - columns: Vec, - time_index: ColumnId, - unique_keys: Vec>, - ) -> Self { - Self { - columns, - time_index: Some(time_index), - unique_keys, - } - } - - /// Look up a column by name. `None` if not present. - pub fn column_id(&self, name: &str) -> Option { - self.columns.iter().position(|c| c.name == name) - } - - /// Whether this schema has *any* provable unique key. The CSE pass - /// reads this to decide whether two `Ref` consumers can safely share - /// a producer (see `design.md` §6 line ~1284 + the unit test in - /// `tests::cse_substitution_legal_only_with_unique_keys`). - pub fn has_unique_key(&self) -> bool { - !self.unique_keys.is_empty() - } - - /// Append `cols` as an additional unique-key set if not already present. - /// Used by `Distinct { cols }` per design.md §6 schema-flow table: - /// "the input schema with `unique_keys` tightened to include `cols`". - pub fn add_unique_key(&mut self, cols: Vec) { - if !self.unique_keys.contains(&cols) { - self.unique_keys.push(cols); - } - } -} - -// ── CSE legality (the load-bearing consumer of `unique_keys`) ──────────────── -// -// Phase F per `control_plane/docs/design.md` §6 Schema flow + the batched- -// queries example (§6 line ~1320): -// -// "CSE legality leans on `Schema::unique_keys` (§6 Schema flow): two -// `QueryExpr::Ref` consumers can share a producer only when its -// output schema is provably stable across reads — the unique-key -// metadata is what lets the deduper assert that without re-running -// the producer's logic." -// -// `cse_reuse_is_legal` is the gatekeeper. The workload-level CSE pass -// (`intent_algebra::cse::dedupe_subtrees`) consults it before emitting -// a `LetBinding` to share a producer between ≥2 `Ref` consumers. - -use thiserror::Error; - -/// Errors returned by [`cse_reuse_is_legal`] when shared-producer reuse -/// would violate the design's stability invariant. -#[derive(Debug, Error, PartialEq, Eq)] -pub enum CseError { - /// Producer schema lacks any `unique_keys` set — row identity is - /// not provably stable across reads, so two `Ref` consumers cannot - /// safely share it. The deduper falls back to per-consumer - /// recomputation. Per design.md §6 line ~1356. - #[error( - "shared-producer CSE refused: producer schema has no unique_keys \ - (design.md §6 schema-flow — without a provable unique key the \ - deduper cannot assert row identity across reads)" - )] - NoUniqueKeys, - /// Trivially-callable case: only one consumer means no reuse to - /// gate. Returned so the caller can short-circuit instead of - /// emitting a degenerate `LetBinding`. - #[error("CSE not applicable: {0} consumer(s) — need ≥ 2 for shared-producer reuse")] - InsufficientConsumers(usize), -} - -/// Two `QueryExpr::Ref` nodes can share a producer (same `LetBinding`) -/// only when the producer's output schema has stable per-row identity — -/// i.e. `Schema::unique_keys` is non-empty. This is the gatekeeper: -/// returns `Ok(())` if shared-producer reuse is legal, otherwise `Err`. -/// -/// Per design.md §6 line ~1356 — `unique_keys` is what makes CSE -/// provably correct. The deduper consults this before emitting a -/// `LetBinding`, and `CostModel::workload_cost` only credits a shared -/// binding when this gate has fired green. -/// -/// `consumer_count` is the number of `QueryExpr::Ref { name }` sites the -/// deduper has identified for the candidate binding. Single-consumer -/// cases short-circuit with `InsufficientConsumers` — a `LetBinding` -/// with one `Ref` is just a no-op alias and shouldn't be hoisted. -pub fn cse_reuse_is_legal(producer_schema: &Schema, consumer_count: usize) -> Result<(), CseError> { - if consumer_count < 2 { - return Err(CseError::InsufficientConsumers(consumer_count)); - } - if !producer_schema.has_unique_key() { - return Err(CseError::NoUniqueKeys); - } - Ok(()) -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - fn col(name: &str, dtype: DataType) -> Column { - Column { - name: name.into(), - dtype, - nullable: false, - } - } - - /// `cse_reuse_is_legal` accepts a producer schema with at least one - /// `unique_keys` set + ≥2 consumers. This is the design.md §6 - /// "load-bearing" green path. - #[test] - fn cse_reuse_legal_when_unique_keys_set() { - let producer = Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("service", DataType::Utf8), - col("value", DataType::Float64), - ], - 0, - vec![vec![0, 1]], - ); - assert_eq!(cse_reuse_is_legal(&producer, 2), Ok(())); - assert_eq!(cse_reuse_is_legal(&producer, 5), Ok(())); - } - - /// Schema without `unique_keys` is the conservative-default case — - /// the deduper must refuse to share it. Pins design.md §6 line - /// ~1356 ("Without it, the deduper has to be conservative and reuse - /// drops on the floor"). - #[test] - fn cse_reuse_illegal_when_unique_keys_empty() { - let producer = Schema::new(vec![col("a", DataType::Int64), col("b", DataType::Float64)]); - assert_eq!( - cse_reuse_is_legal(&producer, 2), - Err(CseError::NoUniqueKeys) - ); - } - - /// Single-consumer case is short-circuited — no `LetBinding` should - /// be emitted for one `Ref` because there's no reuse to credit. - #[test] - fn cse_reuse_rejects_single_consumer() { - let producer = Schema::with_time_index( - vec![col("ts", DataType::Timestamp), col("v", DataType::Float64)], - 0, - vec![vec![0]], - ); - assert_eq!( - cse_reuse_is_legal(&producer, 1), - Err(CseError::InsufficientConsumers(1)) - ); - assert_eq!( - cse_reuse_is_legal(&producer, 0), - Err(CseError::InsufficientConsumers(0)) - ); - } - - /// Empty `unique_keys` rejection takes precedence over the consumer - /// count check only when both pass — but here we verify the - /// insufficient-consumers branch fires first (a defensive ordering - /// so callers see the clearer error when they get the call wrong). - #[test] - fn cse_reuse_consumer_check_precedes_unique_key_check() { - let producer = Schema::new(vec![col("a", DataType::Int64)]); - // Both conditions fail; consumer check is reported. - assert_eq!( - cse_reuse_is_legal(&producer, 1), - Err(CseError::InsufficientConsumers(1)) - ); - } - - #[test] - fn schema_new_has_no_time_or_unique_key() { - let s = Schema::new(vec![col("k", DataType::Utf8), col("v", DataType::Float64)]); - assert!(s.time_index.is_none()); - assert!(!s.has_unique_key()); - assert_eq!(s.column_id("k"), Some(0)); - assert_eq!(s.column_id("v"), Some(1)); - assert_eq!(s.column_id("missing"), None); - } - - #[test] - fn schema_with_time_index_populates_metadata() { - let s = Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("service", DataType::Utf8), - col("value", DataType::Float64), - ], - 0, - vec![vec![0, 1]], - ); - assert_eq!(s.time_index, Some(0)); - assert!(s.has_unique_key()); - assert_eq!(s.unique_keys, vec![vec![0, 1]]); - } - - #[test] - fn add_unique_key_dedupes() { - let mut s = Schema::new(vec![col("a", DataType::Utf8), col("b", DataType::Utf8)]); - s.add_unique_key(vec![0]); - s.add_unique_key(vec![0]); - s.add_unique_key(vec![0, 1]); - assert_eq!(s.unique_keys, vec![vec![0], vec![0, 1]]); - } - - #[test] - fn schema_serde_roundtrip() { - let s = Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("value", DataType::Float64), - ], - 0, - vec![vec![0]], - ); - let json = serde_json::to_string(&s).unwrap(); - let back: Schema = serde_json::from_str(&json).unwrap(); - assert_eq!(s, back); - } -} +//! - `Column` gains `table: Option` (table/alias qualifier for +//! SQL `t.col` disambiguation across joins) plus `Column::new()` / +//! `Column::with_table()` constructors. +//! - `Schema` gains `closed: bool` (schema-on-read completeness flag, +//! Apache Calcite `DynamicRecordType`-style) plus +//! `column_id_qualified()`. +//! +//! Both new fields are `#[serde(default)]`, confirmed backward-compatible +//! by asap_ir's own tests (`schema_closed_defaults_to_open_when_absent`, +//! `column_table_defaults_to_none_when_absent`) — no wire-format break, +//! unlike `AccuracyTarget`'s tag-shape change in Phase 1b. This is what +//! made a full swap the obvious call here instead of converting at a +//! boundary: `agg_intent.rs`'s `to_asap_column`/`from_asap_column`/ +//! `to_asap_dtype`/`from_asap_dtype` helpers from Phase 1b are deleted — +//! no longer needed once there's only one `Column`/`DataType` type. +//! +//! `DataType` itself was already byte-identical between the two repos; +//! no changes there at all. +//! +//! Blast radius from the swap: every `Column { .. }` / `Schema { .. }` +//! struct literal across this repo (~38 `Column{}` sites) needs the new +//! field addressed — either via the `table`/`closed` field explicitly, +//! or (preferred where the call site doesn't care) `Column::new(..)` / +//! `Schema::new(..)` / `Schema::with_time_index(..)`, which default the +//! new fields the same way the pre-merge constructors did. + +pub use asap_ir::intent_algebra::schema::{ + cse_reuse_is_legal, Column, ColumnId, CseError, DataType, Schema, +}; diff --git a/control_plane/src/optimizer/cost/mod.rs b/control_plane/src/optimizer/cost/mod.rs index ad96858d..ec399451 100644 --- a/control_plane/src/optimizer/cost/mod.rs +++ b/control_plane/src/optimizer/cost/mod.rs @@ -745,6 +745,7 @@ mod workload_cost_tests { name: name.into(), dtype, nullable: false, + table: None, } } diff --git a/control_plane/src/optimizer/rules/mod.rs b/control_plane/src/optimizer/rules/mod.rs index 9235a2f6..c1c40b7c 100644 --- a/control_plane/src/optimizer/rules/mod.rs +++ b/control_plane/src/optimizer/rules/mod.rs @@ -218,11 +218,13 @@ pub fn bind_workload_typed(w: &QueryWorkload) -> Option) -> QueryExpr { name: "ts".into(), dtype: DataType::Timestamp, nullable: false, + table: None, }, Column { name: "service".into(), dtype: DataType::Utf8, nullable: false, + table: None, }, Column { name: "value".into(), dtype: DataType::Float64, nullable: false, + table: None, }, ], 0, diff --git a/control_plane/src/sketch_algebra/physical_expr.rs b/control_plane/src/sketch_algebra/physical_expr.rs index 22d78252..0be8cf18 100644 --- a/control_plane/src/sketch_algebra/physical_expr.rs +++ b/control_plane/src/sketch_algebra/physical_expr.rs @@ -269,16 +269,19 @@ mod tests { name: "ts".into(), dtype: DataType::Timestamp, nullable: false, + table: None, }, Column { name: "service".into(), dtype: DataType::Utf8, nullable: false, + table: None, }, Column { name: "value".into(), dtype: DataType::Float64, nullable: false, + table: None, }, ], 0, diff --git a/control_plane/src/sketch_algebra/rules/bind_archive_only.rs b/control_plane/src/sketch_algebra/rules/bind_archive_only.rs index 1747d7dc..ed9776e4 100644 --- a/control_plane/src/sketch_algebra/rules/bind_archive_only.rs +++ b/control_plane/src/sketch_algebra/rules/bind_archive_only.rs @@ -86,16 +86,19 @@ mod tests { name: "ts".into(), dtype: DataType::Timestamp, nullable: false, + table: None, }, Column { name: "service".into(), dtype: DataType::Utf8, nullable: false, + table: None, }, Column { name: "value".into(), dtype: DataType::Float64, nullable: false, + table: None, }, ], 0, diff --git a/control_plane/src/sketch_algebra/tests.rs b/control_plane/src/sketch_algebra/tests.rs index 46f8ea96..e7e0391a 100644 --- a/control_plane/src/sketch_algebra/tests.rs +++ b/control_plane/src/sketch_algebra/tests.rs @@ -21,6 +21,7 @@ fn col(name: &str, dtype: DataType) -> Column { name: name.into(), dtype, nullable: false, + table: None, } } From 79ab894d12658d2e3d6f59e6e332c1d29885a5d6 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 18 Jul 2026 11:05:17 -0600 Subject: [PATCH 02/11] feat(control_plane): Phase 2 step 2 -- add expr_ir.rs (unused until next step) New file, re-exported from asap_ir::intent_algebra::expr_ir. Per the D2 decision in ASAPController's intent-algebra-reconciliation.md: one generic Expr scalar IR shared across L2 (Expr) and L3 (Expr), replacing control_plane's separate ad hoc Predicate (query_expr.rs) and ScalarExpr (relational.rs) types. Deliberately not re-exported into the crate::intent_algebra::* top-level surface yet -- query_expr::ColumnRef already claims that name, and nothing constructs Expr until the query_expr.rs/relational.rs merge (next step) retargets Predicate/ScalarExpr onto L3Expr/L2Expr. This step only makes the type available; verified inert (cargo build clean, full test suite unchanged, 820 passed). --- control_plane/src/intent_algebra/expr_ir.rs | 36 +++++++++++++++++++++ control_plane/src/intent_algebra/mod.rs | 6 ++++ 2 files changed, 42 insertions(+) create mode 100644 control_plane/src/intent_algebra/expr_ir.rs diff --git a/control_plane/src/intent_algebra/expr_ir.rs b/control_plane/src/intent_algebra/expr_ir.rs new file mode 100644 index 00000000..41736d68 --- /dev/null +++ b/control_plane/src/intent_algebra/expr_ir.rs @@ -0,0 +1,36 @@ +//! Language-independent scalar expression IR. +//! +//! ## Phase 2 step 2 (docs/migration-plan-backend-plan.md) +//! +//! New file, re-exported from ASAPController's `asap-ir` crate. This is +//! the D2 decision from `ASAPController/docs/intent-algebra-reconciliation.md` +//! (already validated by ASAPController's own evolution — control_plane +//! had no equivalent generic-over-column-type scalar IR before this): +//! `Predicate` and `HavingPredicate` in `query_expr.rs` are currently ad +//! hoc, control_plane-only types; `relational.rs`'s `ScalarExpr` is a +//! separate, L2-only scalar type. Once `query_expr.rs`/`relational.rs` +//! are merged (next Phase 2 step), both collapse onto this one +//! `Expr` — `L2Expr = Expr` for the front-end-emitted L2 +//! tree, `L3Expr = Expr` for the canonical positional L3 tree. +//! Nothing in this repo constructs `Expr` yet — that lands with the +//! `query_expr.rs`/`relational.rs` merge, not here. This step only makes +//! the type available. +//! +//! One generic [`Expr`] spans the lowering boundary; the two layers are +//! aliases that differ only in the column-reference type `C`: +//! +//! - [`L2Expr`] = `Expr` — name-based. The per-language front ends +//! emit it (PromQL label matchers, SQL `WHERE` / projection / sort-key +//! expressions) on the Layer-2 `relational` tree. +//! - [`L3Expr`] = `Expr` — **positional**. The canonical L3 +//! `query_expr` tree carries it; the converter resolves every `ColumnRef` +//! against the in-scope schema to produce it, so L3 column identity is +//! unambiguous (no name collisions across a join). +//! +//! `Expr` shares the scalar/operator vocabulary +//! ([`L3Scalar`], [`CompareOp`], [`ArithOp`]) — the **union** of what the two +//! front ends need: PromQL contributes `Regex` / `NotRegex` (`=~` / `!~`); SQL +//! contributes arithmetic, `CASE`, `IN`, `CAST`, `IS [NOT] NULL`, scalar +//! function calls, and the `LIKE` / `ILIKE` comparison family. + +pub use asap_ir::intent_algebra::{ArithOp, ColumnRef, CompareOp, Expr, L2Expr, L3Expr, L3Scalar}; diff --git a/control_plane/src/intent_algebra/mod.rs b/control_plane/src/intent_algebra/mod.rs index 533c2df5..c3c675a9 100644 --- a/control_plane/src/intent_algebra/mod.rs +++ b/control_plane/src/intent_algebra/mod.rs @@ -73,6 +73,12 @@ pub mod agg_intent; pub mod cse; +// Not yet re-exported into the crate::intent_algebra::* top-level surface +// below -- `query_expr::ColumnRef` already claims that name, and this +// module is unused until the query_expr.rs/relational.rs merge (next +// Phase 2 step) retargets `Predicate`/`ScalarExpr` onto `L3Expr`/`L2Expr`. +// Reachable today only via the full `intent_algebra::expr_ir::` path. +pub mod expr_ir; pub mod query_expr; pub mod schema; From e7dd67b014721a40885e1a9a0b059c375b09ab15 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 19 Jul 2026 07:17:24 -0600 Subject: [PATCH 03/11] feat(control_plane): Phase 2 step 3 -- merge query_expr.rs/relational.rs onto asap-ir QueryExpr, Predicate, and their supporting types are no longer defined locally -- re-exported from asap_ir::intent_algebra. asap_ir's version is a real superset (~22 variants vs. the pre-merge 16); its output_schema_in is adopted via the inherent method rather than reimplemented. Three representational differences turned out not to be missing capability, just differently structured: - Predicate is now L3Expr (struct Predicate(pub L3Expr)); Between desugars to Compare(Ge) AND Compare(Le). - Partition doesn't exist in asap_ir -- folded into Aggregate.by: GroupKeys at construction time (intent_algebra::lower), eliminating the R11 PartitionElim rule and every Partition-specific arm. - ScalarSubquery is rejected at construction time rather than lowered: L3Expr::Column is strictly positional, so the old name-based Predicate::Column(ColumnRef::Named) hack for referencing a hoisted LetBinding has no equivalent, and neither this repo's parsers nor ASAPController's own L2 lowering construct it today. R7 SubqueryDecorrelation is deleted as a result (dead code -- its target shape is now built directly at construction time). Also fixes a real regression the merge surfaced: the Binder only collected column names from GROUP BY / TopK / Partition keys, since the pre-merge Predicate was name-based and needed no positional resolution. The new positional L3Expr::Column requires every predicate-referenced name to already be in scope, so Binder::collect_referenced_columns now also walks Filter/Aggregate.having/Join.pred/Project scalar trees. Preserves two pre-existing, deliberately-tested behaviors that a naive delegation to AggFunc::to_sketch_op() would have dropped: avg_over_time still approximates as a p50 quantile sketch, and Rate/Increase/Delta still collapse onto AggIntent::Sum (disambiguated via the separate outer_fn field) rather than adopting asap_ir's dedicated intents, per asap_tier_analysis's existing outer_fn dispatch design. Co-Authored-By: Claude Sonnet 5 --- control_plane/src/asap_tier_analysis.rs | 11 +- control_plane/src/deployment_model.rs | 15 +- control_plane/src/intent_algebra/binder.rs | 68 +- control_plane/src/intent_algebra/cse.rs | 57 +- control_plane/src/intent_algebra/lower.rs | 697 +++++---- control_plane/src/intent_algebra/mod.rs | 18 +- .../src/intent_algebra/query_expr.rs | 1295 ++--------------- .../src/intent_algebra/relational.rs | 27 +- control_plane/src/optimizer/cost/mod.rs | 61 +- control_plane/src/optimizer/engine.rs | 271 ++-- control_plane/src/optimizer/rules/mod.rs | 18 +- control_plane/src/optimizer/trait_def.rs | 10 +- control_plane/src/physical/allocator.rs | 158 +- .../src/physical/colored_dag/allocator.rs | 24 +- control_plane/src/physical/colored_dag/dag.rs | 2 +- .../src/physical/colored_dag/emitter.rs | 39 +- .../src/physical/colored_dag/tests.rs | 70 +- control_plane/src/physical/plan.rs | 2 +- control_plane/src/physical/planner.rs | 124 +- control_plane/src/physical/window_fusion.rs | 7 +- control_plane/src/query_parser/mod.rs | 103 +- control_plane/src/query_parser/promql.rs | 52 +- control_plane/src/sketch_algebra/lower.rs | 8 +- .../src/sketch_algebra/physical_expr.rs | 60 +- .../sketch_algebra/rules/bind_archive_only.rs | 63 +- .../sketch_algebra/rules/bind_exact_agg.rs | 23 +- control_plane/src/sketch_algebra/tests.rs | 63 +- 27 files changed, 1345 insertions(+), 2001 deletions(-) diff --git a/control_plane/src/asap_tier_analysis.rs b/control_plane/src/asap_tier_analysis.rs index 1b85bc67..21ab4449 100644 --- a/control_plane/src/asap_tier_analysis.rs +++ b/control_plane/src/asap_tier_analysis.rs @@ -319,10 +319,11 @@ fn collect_agg_intents(expr: &QueryExpr, out: &mut Vec) { QueryExpr::Scan { .. } | QueryExpr::Ref { .. } => {} // A-variants lifted in Batch 2 of the relational migration. They // carry no AggIntent themselves — recurse into their children to - // find Aggregates further down the tree. + // find Aggregates further down the tree. `Partition` no longer + // exists in the canonical IR — its keys fold into `Aggregate.by` + // at construction time (`intent_algebra::lower`). QueryExpr::Filter { child, .. } | QueryExpr::Project { child, .. } - | QueryExpr::Partition { child, .. } | QueryExpr::Distinct { child, .. } | QueryExpr::Sort { child, .. } | QueryExpr::Limit { child, .. } @@ -342,6 +343,12 @@ fn collect_agg_intents(expr: &QueryExpr, out: &mut Vec) { collect_agg_intents(left, out); collect_agg_intents(right, out); } + // The PromQL-surface superset (Scalar/EvalTime/VectorFromScalar/ + // ScalarFromVector/Relabel/InfoJoin/Sample/TimeRange/TimeShift/ + // WindowFunc) isn't constructed by this parser today; the + // single-child wrappers among them carry no `AggIntent` either + // way, so a no-op default is safe. + _ => {} } } diff --git a/control_plane/src/deployment_model.rs b/control_plane/src/deployment_model.rs index 47f159c1..f2df9fc8 100644 --- a/control_plane/src/deployment_model.rs +++ b/control_plane/src/deployment_model.rs @@ -199,13 +199,14 @@ mod tests { let id = DeploymentModelId::asaplifecycle(); assert!(reg.contains(&id)); let m = reg.lookup(&id).expect("asaplifecycle must be registered"); - // The default rule set carries the 11 engine rules. (R6 - // HistogramQuantileFusion was retired in Step γ5 — see - // `optimizer::engine` module note.) + // The default rule set carries the 9 engine rules. (R6 + // HistogramQuantileFusion was retired in Step γ5; R7 + // SubqueryDecorrelation and R11 PartitionElim were retired in + // the `asap_ir` merge — see `optimizer::engine` module note.) assert_eq!( m.rules.len(), - 11, - "asaplifecycle should ship the 11 engine rules" + 9, + "asaplifecycle should ship the 9 engine rules" ); // Emitter set carries the three demo emitters. assert!(m.emitters.has("opamp_edge_yaml")); @@ -223,7 +224,9 @@ mod tests { assert!(cats.contains(&RuleCategory::Fusion)); assert!(cats.contains(&RuleCategory::Elim)); assert!(cats.contains(&RuleCategory::Cse)); - assert!(cats.contains(&RuleCategory::Decorrelate)); + // `RuleCategory::Decorrelate` has no producer left — R7 + // SubqueryDecorrelation was retired in the `asap_ir` merge (see + // `optimizer::engine` module note). } #[test] diff --git a/control_plane/src/intent_algebra/binder.rs b/control_plane/src/intent_algebra/binder.rs index 1f1f7495..4be693a7 100644 --- a/control_plane/src/intent_algebra/binder.rs +++ b/control_plane/src/intent_algebra/binder.rs @@ -170,20 +170,39 @@ fn default_leaf_columns() -> Vec { ] } -/// Walk the legacy tree and collect every distinct group-key name the +/// Walk the legacy tree and collect every distinct column name the /// legacy → canonical converter resolves positionally: `Aggregate.keys`, -/// `TopK.by`, and `Partition.keys`. Sorted + de-duplicated for a stable, -/// deterministic column order. +/// `TopK.by`, `Partition.keys`, and every `ScalarExpr::Column(name)` +/// reachable from a `Filter.pred` / `Aggregate.having` / `Join.pred` / +/// `Project` item — since the canonical `Predicate(L3Expr)` is fully +/// positional (`L3Expr::Column(ColumnId)`, no name-based fallback), +/// `convert_scalar` requires every referenced name to already be in the +/// schema. Sorted + de-duplicated for a stable, deterministic column +/// order. /// /// `AggItem.col` (the statistic's *input* column) is deliberately not /// collected — the converter never resolves it positionally; it only -/// ever resolves group-by keys. +/// ever resolves group-by keys and predicate/projection columns. fn collect_referenced_columns(tree: &LQueryExpr) -> Vec { let mut out: Vec = Vec::new(); tree.walk(&mut |node| match node { - LQueryExpr::Aggregate { keys, .. } => out.extend(keys.iter().cloned()), + LQueryExpr::Aggregate { keys, having, .. } => { + out.extend(keys.iter().cloned()); + if let Some(pred) = having { + collect_columns_from_scalar(pred, &mut out); + } + } LQueryExpr::TopK { by, .. } => out.extend(by.iter().cloned()), LQueryExpr::Partition { keys, .. } => out.extend(keys.keys().iter().cloned()), + LQueryExpr::Filter { pred, .. } => collect_columns_from_scalar(pred, &mut out), + LQueryExpr::Join { + pred: Some(pred), .. + } => collect_columns_from_scalar(pred, &mut out), + LQueryExpr::Project { cols, .. } => { + for item in cols { + collect_columns_from_scalar(&item.expr, &mut out); + } + } _ => {} }); out.sort(); @@ -191,6 +210,45 @@ fn collect_referenced_columns(tree: &LQueryExpr) -> Vec { out } +/// Recursively collect every `ScalarExpr::Column(name)` reachable from +/// `expr`. Does not descend into `ScalarSubquery`'s inner `QueryExpr` — +/// `lower::convert_scalar` rejects `ScalarSubquery` outright (see its +/// module doc), so there is no positional resolution to satisfy inside +/// one. +fn collect_columns_from_scalar( + expr: &crate::intent_algebra::relational::ScalarExpr, + out: &mut Vec, +) { + use crate::intent_algebra::relational::ScalarExpr as SE; + match expr { + SE::Column(name) => out.push(name.clone()), + SE::Literal(_) | SE::ScalarSubquery(_) => {} + SE::BinaryOp { lhs, rhs, .. } => { + collect_columns_from_scalar(lhs, out); + collect_columns_from_scalar(rhs, out); + } + SE::FunctionCall { args, .. } => { + for a in args { + collect_columns_from_scalar(a, out); + } + } + SE::InList { expr, list, .. } => { + collect_columns_from_scalar(expr, out); + for a in list { + collect_columns_from_scalar(a, out); + } + } + SE::Between { + expr, low, high, .. + } => { + collect_columns_from_scalar(expr, out); + collect_columns_from_scalar(low, out); + collect_columns_from_scalar(high, out); + } + SE::IsNull { expr, .. } => collect_columns_from_scalar(expr, out), + } +} + // ── Tests ───────────────────────────────────────────────────────────────────── #[cfg(test)] diff --git a/control_plane/src/intent_algebra/cse.rs b/control_plane/src/intent_algebra/cse.rs index 62c91465..545223f9 100644 --- a/control_plane/src/intent_algebra/cse.rs +++ b/control_plane/src/intent_algebra/cse.rs @@ -133,14 +133,20 @@ pub fn dedupe_subtrees(roots: Vec<(QueryId, QueryExpr)>) -> CseWorkloadPlan { QueryExpr::Aggregate { by, aggs, + output_names, having, child, } if *child == shared_expr => QueryExpr::Aggregate { by, aggs, + output_names, having, child: Box::new(QueryExpr::Ref { - name: binding_name.clone(), + // `QueryExpr::Ref.name` is `asap_ir`'s `BindingName` + // (positional-workload-agnostic) — distinct from this + // module's own `types_v2::BindingName` (CSE-plan-level + // binding identity). Convert at the boundary. + name: asap_ir::intent_algebra::BindingName::new(binding_name.0.clone()), }), }, other => other, @@ -175,23 +181,27 @@ mod tests { } fn ts_scan() -> QueryExpr { + let schema = Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("service", DataType::Utf8), + col("value", DataType::Float64), + ], + 0, + vec![vec![0, 1]], + ); + let lf = LabelFilter { + label: "service".into(), + equals: "api".into(), + }; + let pred = crate::intent_algebra::label_filter_to_predicate(&lf, &schema) + .expect("service column present in schema"); QueryExpr::Scan { source: Source::TimeSeries { metric: "http_request_duration_seconds".into(), }, - label_filters: vec![LabelFilter { - label: "service".into(), - equals: "api".into(), - }], - schema: Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("service", DataType::Utf8), - col("value", DataType::Float64), - ], - 0, - vec![vec![0, 1]], - ), + predicates: vec![pred], + schema, } } @@ -216,12 +226,13 @@ mod tests { #[test] fn dedupe_subtrees_single_root_passthrough() { let q = QueryExpr::Aggregate { - by: vec![1], + by: vec![1].into(), aggs: vec![AggIntent::Quantile { col: None, q: 0.99, accuracy: AccuracyTarget::Epsilon(0.01), }], + output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), }; @@ -237,22 +248,24 @@ mod tests { #[test] fn dedupe_subtrees_basic() { let q1 = QueryExpr::Aggregate { - by: vec![1], + by: vec![1].into(), aggs: vec![AggIntent::Quantile { col: None, q: 0.99, accuracy: AccuracyTarget::Epsilon(0.01), }], + output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), }; let q2 = QueryExpr::Aggregate { - by: vec![1], + by: vec![1].into(), aggs: vec![AggIntent::Quantile { col: None, q: 0.95, accuracy: AccuracyTarget::Epsilon(0.01), }], + output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), }; @@ -269,7 +282,7 @@ mod tests { QueryExpr::Aggregate { child, .. } => assert_eq!( **child, QueryExpr::Ref { - name: BindingName::new("shared_0"), + name: asap_ir::intent_algebra::BindingName::new("shared_0"), }, "Aggregate child should be a Ref to the hoisted binding" ), @@ -283,8 +296,9 @@ mod tests { #[test] fn dedupe_subtrees_no_shared_subexpr() { let q1 = QueryExpr::Aggregate { - by: vec![], + by: vec![].into(), aggs: vec![AggIntent::Sum { col: None }], + output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), }; @@ -294,7 +308,7 @@ mod tests { source: Source::TimeSeries { metric: "different_metric".into(), }, - label_filters: vec![], + predicates: vec![], schema: Schema::with_time_index( vec![ col("ts", DataType::Timestamp), @@ -306,8 +320,9 @@ mod tests { ), }; let q2 = QueryExpr::Aggregate { - by: vec![], + by: vec![].into(), aggs: vec![AggIntent::Max { col: None }], + output_names: Vec::new(), having: None, child: Box::new(QueryExpr::Window { kind: WindowKind::Sliding, diff --git a/control_plane/src/intent_algebra/lower.rs b/control_plane/src/intent_algebra/lower.rs index 76333c85..87ca3482 100644 --- a/control_plane/src/intent_algebra/lower.rs +++ b/control_plane/src/intent_algebra/lower.rs @@ -5,33 +5,87 @@ //! *whole* canonical `query_expr::QueryExpr` tree. This is the single //! entry the parse path routes through — [`convert_root`]. //! +//! ## Phase 2 step 3 (docs/migration-plan-backend-plan.md) +//! +//! Two shape changes since the canonical `QueryExpr` merged onto +//! `asap_ir` (see `query_expr.rs`'s module docs for the full rationale): +//! +//! - **Predicates** translate to `L3Expr` (via `Predicate`, `expr_ir.rs`) +//! instead of this repo's old 8-variant `Predicate` enum. `Between` +//! desugars via `query_expr::between`. **`ScalarSubquery` is rejected** +//! ([`ConvertError::UnsupportedScalarSubquery`]) rather than lowered: +//! `asap_ir`'s `L3Expr` has no slot for "reference the value bound by +//! an enclosing `LetBinding`" (the old `Predicate::Column(ColumnRef:: +//! Named(subq_name))` was itself a hack the canonical positional +//! `L3Expr::Column(ColumnId)` can't reproduce without inventing a +//! sentinel id space nothing downstream knows about). ASAPController's +//! own L2→L3 lowering (`crates/l2/src/lower.rs`) has no correlated- +//! subquery construct either — its `Ref`/`LetBinding` are documented as +//! "Reserved: no front end emits yet." This repo's own front ends never +//! construct `ScalarExpr::ScalarSubquery` either (grep-verified: the +//! only non-test, non-definition sites were `lower.rs` itself and the +//! now-dead `optimizer/engine.rs` R7 rule), so rejecting it is a no-op +//! for every real query today. `optimizer/engine.rs`'s R7 +//! `SubqueryDecorrelation` is deleted rather than ported — it pattern- +//! matched the removed `Predicate::BinaryOp` / `Predicate::ScalarSubquery` +//! / `Predicate::Column(ColumnRef::Named)` variants directly and has no +//! construction site left to fire against. Real correlated-subquery +//! support is follow-up work once `asap_ir` grows a representation for +//! it. +//! - **`GROUP BY` keys attach directly to `Aggregate.by: GroupKeys`** +//! instead of wrapping the result in a `Partition` node (removed from +//! `asap_ir`'s `QueryExpr` — folded into `GroupKeys`'s `by`/`without` +//! distinction). The single-statistic fusion arm resolves `keys` once +//! and threads `GroupKeys` into every fused node (including both +//! siblings of a `StdDev`/`Variance` `Merge` fan-out) instead of +//! wrapping the finished shape afterward. A standalone legacy +//! `LQueryExpr::Partition` (not part of the fusion arm) folds its keys +//! into the nearest `Aggregate` its converted subtree contains, via +//! [`fold_partition_keys`]. +//! +//! `having` is `Option` (real typed HAVING) directly — no +//! translation needed beyond running the HAVING expression through +//! [`convert_scalar`] like any other predicate. +//! //! ## Variant mapping //! //! | relational `QueryExpr` | canonical `QueryExpr` | //! |---|---| -//! | `Source(spec)` | `Scan { TimeSeries, label_filters: [], schema }` | +//! | `Source(spec)` | `Scan { TimeSeries, predicates: [], schema }` | //! | `Ref(name)` | `Ref { name }` | //! | `Filter` | `Filter` (pred via [`convert_scalar`]) | //! | `Project` | `Project` (each item's expr via [`convert_scalar`])| -//! | `Aggregate` (multi-agg / HAVING / `Custom` / un-grouped `COUNT(*)`) | plain `Aggregate` (keys→by, AggFunc→AggIntent) | -//! | `Aggregate` (single sketchable) | *fuses* — `Window`-input → `Window { Aggregate }`, `GROUP BY` → wrapping `Partition`, `StdDev`/`Variance` → `Merge` of sibling quantile aggregates. See the `Aggregate` arm. | +//! | `Aggregate` | single agg + no HAVING → *fuses* (see below); otherwise plain `Aggregate` (keys→by: GroupKeys, AggFunc→AggIntent via [`agg_func_to_intents`]) | //! | `Window` | `Window` (slide → Sliding else Tumbling) | -//! | `Partition` | `Partition` | +//! | `Partition` | keys folded into the nearest `Aggregate.by` inside the converted subtree, via [`fold_partition_keys`] | //! | `Distinct` | `Distinct` | //! | `TopK` | `Aggregate { aggs: [AggIntent::TopK] }` (HeavyHitter)| //! | `Merge` | `Merge` | //! | `Join` | `Join` (None pred → `Literal(Bool(true))`) | //! | `SetOp` | `SetOp` | -//! | `Sort` | `Sort` | +//! | `Sort` | `Sort` (keys pass through — `relational::SortKey` already re-exports the canonical, `L3Expr`-based type) | //! | `Limit` | `Limit` | //! | `LetBinding` | `LetBinding` (relational `body` → canonical `child`)| //! | `PromQLSubquery` | `Subquery` | -//! | `BinaryOp` | `BinaryOp` | +//! | `BinaryOp` | `BinaryOp` (`op` passes straight through — `relational::BinaryOpKind` is a re-export of the canonical type, not a separate flat enum) | //! -//! The single-statistic sketchable `Aggregate` fusion (`Window`-swap, -//! `Partition` wrap, `StdDev` / `Variance` fan-out) is done directly in -//! canonical terms inside the [`convert`] `Aggregate` arm — there is no -//! intermediate sketch-fused L2-or-L3 IR. +//! A single-statistic `Aggregate` (exactly one `AggItem`, no `HAVING`) +//! fuses directly into canonical shape rather than staying a plain +//! `Aggregate` wrapping the untouched child: +//! * input is a `Window` → emit `Window { Aggregate { by } } }` (the +//! window-defines-sketch-lifecycle shape); +//! * otherwise → emit `Aggregate { by }`; +//! * `StdDev` / `Variance` fan out into a `Merge` of two sibling +//! quantile aggregates, both carrying the same `by` (Step α F1 +//! strategy) — the only `AggFunc`s [`agg_func_to_intents`] maps to +//! more than one `AggIntent`; +//! * `GROUP BY` keys resolve once and thread into every fused node's +//! `by: GroupKeys` directly. +//! `AggFunc::Custom` produces no canonical intent regardless of arity — +//! [`agg_func_to_intents`] returns empty, which raises +//! [`ConvertError::NoCanonicalIntent`] once execution reaches the plain +//! multi-agg path (single-agg-with-empty-intents falls through to it +//! rather than being special-cased inline). //! //! ## Schema threading //! @@ -46,61 +100,43 @@ #![allow(dead_code)] -use std::time::Duration; - -use thiserror::Error; +use asap_ir::intent_algebra::BindingName; use crate::intent_algebra::agg_intent::AggIntent; use crate::intent_algebra::binder::Binder; -use crate::intent_algebra::column_resolution::{resolve_named_keys, ResolveError}; +use crate::intent_algebra::column_resolution::{ + resolve_column_ref, resolve_column_refs, resolve_named_keys, ResolveError, +}; use crate::intent_algebra::query_expr::{ - from_legacy_scalar, ColumnRef as CColumnRef, HavingPredicate, LiteralValue, - PartitionKeys as CPartitionKeys, Predicate, ProjectItem as CProjectItem, - QueryExpr as CQueryExpr, QueryExprError, Source, WindowKind as CWindowKind, + between, ArithOp, BinaryOpKind, CompareOp, GroupKeys, L3Scalar, Predicate, + ProjectItem as CProjectItem, QueryExpr as CQueryExpr, Source, WindowKind as CWindowKind, }; use crate::intent_algebra::relational::{ AggFunc, ColumnRef as LColumnRef, PartitionKeys as LPartitionKeys, QueryExpr as LQueryExpr, ScalarExpr as LScalarExpr, }; use crate::intent_algebra::schema::Schema; -use crate::types_v2::{AccuracyTarget, BindingName}; +use crate::intent_algebra::L3Expr; +use crate::types_v2::AccuracyTarget; /// Errors produced while converting a legacy `QueryExpr` to canonical. -/// -/// `PartialEq` is not derived because [`QueryExprError`] (carried by -/// `Scalar`) does not derive it — callers compare via -/// `matches!(err, ConvertError::Variant { .. })`. -#[derive(Debug, Error)] +#[derive(Debug, thiserror::Error)] pub enum ConvertError { - /// A column reference (`Aggregate` key, `Partition` / `TopK` key) - /// did not resolve against the inherited schema. + /// A column reference (`Aggregate` key, `Distinct` column, scalar + /// `Column` leaf) did not resolve against the inherited schema. #[error("column resolution failed: {0}")] Resolve(#[from] ResolveError), /// An `AggItem.func` has no canonical `AggIntent` equivalent — only /// `AggFunc::Custom(_)` triggers this today. #[error("AggItem `{alias}` uses non-canonical func ({func_dbg}) — no AggIntent equivalent")] NoCanonicalIntent { alias: String, func_dbg: String }, - /// A legacy `ScalarExpr` leaf failed to translate. Unreachable in - /// practice — `convert_scalar` handles every variant — but kept as a - /// typed boundary around [`from_legacy_scalar`]. - #[error("scalar translation failed: {0}")] - Scalar(QueryExprError), + /// A `ScalarExpr::ScalarSubquery` was encountered. See the module doc + /// for why this is rejected rather than lowered. + #[error("scalar subqueries are not supported by the canonical IR yet")] + UnsupportedScalarSubquery, } /// Lower a legacy Layer-2 `QueryExpr` tree to the canonical L3 IR. -/// -/// A single recursive walk ([`convert`]): every Layer-2 relational node -/// maps onto its canonical counterpart, and the single-statistic -/// sketchable `Aggregate` fuses (window-swap, `Partition` wrap, `StdDev` -/// fan-out) directly in canonical terms — see the [`convert`] `Aggregate` -/// arm. There is no intermediate legacy Layer-3 IR. -/// -/// The inherited schema comes from the [`Binder`] — the explicit L3 -/// name-resolution pass — which builds the complete, self-contained -/// schema every `ColumnId` indexes into. Because the Binder guarantees -/// every referenced name is in scope, the per-arm positional resolution -/// below (`resolve_column_ref` / `resolve_named_keys`) is **total**: it -/// cannot raise `ConvertError::Resolve` on a well-formed legacy tree. pub fn convert_root(legacy: &LQueryExpr) -> Result { let schema = Binder::new().bind(legacy); convert(legacy, &schema) @@ -115,7 +151,7 @@ pub fn convert(legacy: &LQueryExpr, schema: &Schema) -> Result Result { - let cols = cols + LQueryExpr::Project { cols, input } => CQueryExpr::Project { + cols: cols .iter() .map(|pi| { Ok(CProjectItem { alias: pi.alias.clone(), - expr: convert_scalar(&pi.expr, schema)?, + expr: convert_scalar(&pi.expr, schema)?.0, }) }) - .collect::, ConvertError>>()?; - CQueryExpr::Project { - cols, - child: Box::new(convert(input, schema)?), - } - } + .collect::, ConvertError>>()?, + qualifier: None, + child: Box::new(convert(input, schema)?), + }, LQueryExpr::Aggregate { keys, @@ -152,109 +186,72 @@ pub fn convert(legacy: &LQueryExpr, schema: &Schema) -> Result { - // A single-statistic sketchable aggregate *fuses* — this is the - // former `legacy_lower::lower_aggregate` step, done directly in + let by: GroupKeys = resolve_named_keys(keys, schema)?.into(); + + // A single-statistic aggregate *fuses* — this is the former + // `legacy_lower::lower_aggregate` step, done directly in // canonical terms rather than via an intermediate legacy L3 - // node: - // * input is a `Window` → emit `Window { Aggregate { by: [] } }` - // (the window-defines-sketch-lifecycle shape); - // * otherwise → emit `Aggregate { by: [] }`; - // * `StdDev` / `Variance` fan out into a `Merge` of two - // sibling quantile aggregates (Step α F1 strategy); - // * `GROUP BY` keys wrap the result in a `Partition`. - // - // Matching the *raw* `input` for `Window` is exact for every - // tree the parsers emit: they never nest an `Aggregate` - // directly over a `Window` directly over another sketchable - // `Aggregate`, the only shape where the raw vs. sketch-lowered - // input would differ. - // - // Multi-agg, `HAVING`-bearing, `Custom`, and un-grouped - // `COUNT(*)` aggregates fall through to the plain canonical - // `Aggregate` below. + // node. `Custom` (empty `intents`) falls through to the plain + // multi-agg path below, which raises `NoCanonicalIntent` + // uniformly for every arity. if aggs.len() == 1 && having.is_none() { let item = &aggs[0]; - let ungrouped_count = matches!(item.func, AggFunc::Count) && keys.is_empty(); - if !ungrouped_count { - let intents = agg_func_to_intents(&item.func); - if !intents.is_empty() { - let nodes: Vec = match input.as_ref() { - LQueryExpr::Window { - duration, - slide, - input: win_input, - } => { - let kind = if slide.is_some() { - CWindowKind::Sliding - } else { - CWindowKind::Tumbling - }; - let win_child = convert(win_input, schema)?; - intents - .into_iter() - .map(|intent| CQueryExpr::Window { - kind: kind.clone(), - size: *duration, - slide: *slide, - child: Box::new(CQueryExpr::Aggregate { - by: Vec::new(), - aggs: vec![intent], - having: None, - child: Box::new(win_child.clone()), - }), - }) - .collect() - } - other => { - let child = convert(other, schema)?; - intents - .into_iter() - .map(|intent| CQueryExpr::Aggregate { - by: Vec::new(), + let intents = agg_func_to_intents(&item.func, !keys.is_empty()); + if !intents.is_empty() { + let nodes: Vec = match input.as_ref() { + LQueryExpr::Window { + duration, + slide, + input: win_input, + } => { + let kind = if slide.is_some() { + CWindowKind::Sliding + } else { + CWindowKind::Tumbling + }; + let win_child = convert(win_input, schema)?; + intents + .into_iter() + .map(|intent| CQueryExpr::Window { + kind: kind.clone(), + size: *duration, + slide: *slide, + child: Box::new(CQueryExpr::Aggregate { + by: by.clone(), aggs: vec![intent], + output_names: Vec::new(), having: None, - child: Box::new(child.clone()), - }) - .collect() - } - }; - let sketch = if nodes.len() == 1 { - nodes.into_iter().next().unwrap() - } else { - CQueryExpr::Merge { children: nodes } - }; - return Ok(if keys.is_empty() { - sketch - } else { - CQueryExpr::Partition { - keys: CPartitionKeys::By(keys.clone()), - child: Box::new(sketch), - } - }); - } - // `intents` empty → `Custom` func; fall through to the - // plain path, which raises `NoCanonicalIntent`. + child: Box::new(win_child.clone()), + }), + }) + .collect() + } + other => { + let child = convert(other, schema)?; + intents + .into_iter() + .map(|intent| CQueryExpr::Aggregate { + by: by.clone(), + aggs: vec![intent], + output_names: Vec::new(), + having: None, + child: Box::new(child.clone()), + }) + .collect() + } + }; + return Ok(if nodes.len() == 1 { + nodes.into_iter().next().unwrap() + } else { + CQueryExpr::Merge { children: nodes } + }); } } - // Plain canonical `Aggregate`: multi-agg, `HAVING`, `Custom`, - // or the un-grouped `COUNT(*)` exact-row-count case. - let by = resolve_named_keys(keys, schema)?; + // Plain canonical `Aggregate`: multi-agg, `HAVING`, or `Custom`. let mut intents: Vec = Vec::with_capacity(aggs.len()); for item in aggs { - // Faithful mapping for un-grouped `COUNT(*)`: no GROUP BY - // means an exact row count, no sketch benefit. - // `agg_func_to_intents` is the *sketch* map and would pick - // `Frequency` — that loses the "this is exact" decision. Map - // it to `Count { Exact }` so downstream (`capability_for`, - // the `QeCollector`) sees it as exact. - if matches!(item.func, AggFunc::Count) && keys.is_empty() { - intents.push(AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }); - continue; - } - let mapped = agg_func_to_intents(&item.func); + let mapped = agg_func_to_intents(&item.func, !keys.is_empty()); if mapped.is_empty() { return Err(ConvertError::NoCanonicalIntent { alias: item.alias.clone(), @@ -263,16 +260,14 @@ pub fn convert(legacy: &LQueryExpr, schema: &Schema) -> Result None, - Some(se) => Some(HavingPredicate(format!( - "{:?}", - convert_scalar(se, schema)? - ))), - }; + let having = having + .as_ref() + .map(|se| convert_scalar(se, schema)) + .transpose()?; CQueryExpr::Aggregate { by, aggs: intents, + output_names: Vec::new(), having, child: Box::new(convert(input, schema)?), } @@ -293,16 +288,17 @@ pub fn convert(legacy: &LQueryExpr, schema: &Schema) -> Result CQueryExpr::Partition { - keys: match keys { - LPartitionKeys::By(k) => CPartitionKeys::By(k.clone()), - LPartitionKeys::Without(k) => CPartitionKeys::Without(k.clone()), - }, - child: Box::new(convert(input, schema)?), - }, + LQueryExpr::Partition { keys, input } => { + let by: GroupKeys = match keys { + LPartitionKeys::By(k) => resolve_named_keys(k, schema)?.into(), + LPartitionKeys::Without(k) => GroupKeys::without(resolve_named_keys(k, schema)?), + }; + let converted = convert(input, schema)?; + fold_partition_keys(converted, by) + } LQueryExpr::Distinct { cols, input } => CQueryExpr::Distinct { - cols: cols.iter().map(convert_column_ref).collect(), + cols: resolve_column_refs(cols, schema)?, child: Box::new(convert(input, schema)?), }, @@ -310,13 +306,14 @@ pub fn convert(legacy: &LQueryExpr, schema: &Schema) -> Result Result CQueryExpr::Join { kind: kind.clone(), - // Canonical `Join` requires a predicate; a legacy `None` pred - // is a CROSS JOIN — model it as the tautology `true`. pred: match pred { Some(se) => convert_scalar(se, schema)?, - None => Predicate::Literal(LiteralValue::Bool(true)), + // Canonical `Join` requires a predicate; a legacy `None` + // pred is a CROSS JOIN — model it as the tautology `true`. + None => Predicate(L3Expr::Literal(L3Scalar::Boolean(true))), }, left: Box::new(convert(left, schema)?), right: Box::new(convert(right, schema)?), @@ -359,8 +356,8 @@ pub fn convert(legacy: &LQueryExpr, schema: &Schema) -> Result CQueryExpr::Sort { - // `SortKey` is the single canonical type (deduped in PR 1). keys: keys.clone(), + partition_by: GroupKeys::none(), child: Box::new(convert(input, schema)?), }, @@ -393,8 +390,6 @@ pub fn convert(legacy: &LQueryExpr, schema: &Schema) -> Result CQueryExpr::BinaryOp { - // `BinaryOpKind` / `VectorMatch` are the single canonical - // types (deduped in PR 1). op: op.clone(), lhs: Box::new(convert(lhs, schema)?), rhs: Box::new(convert(rhs, schema)?), @@ -403,99 +398,175 @@ pub fn convert(legacy: &LQueryExpr, schema: &Schema) -> Result CQueryExpr { + match qe { + CQueryExpr::Aggregate { + aggs, + output_names, + having, + child, + .. + } => CQueryExpr::Aggregate { + by, + aggs, + output_names, + having, + child, + }, + CQueryExpr::Window { + kind, + size, + slide, + child, + } => CQueryExpr::Window { + kind, + size, + slide, + child: Box::new(fold_partition_keys(*child, by)), + }, + CQueryExpr::Merge { children } => CQueryExpr::Merge { + children: children + .into_iter() + .map(|c| fold_partition_keys(c, by.clone())) + .collect(), + }, + other => { + debug_assert!( + false, + "Partition over a non-aggregate shape has no GroupKeys home: {other:?}" + ); + other + } + } +} + /// Translate a legacy `ScalarExpr` to a canonical `Predicate`, recursing -/// through every composite variant so a nested `ScalarSubquery` (which -/// carries a legacy `QueryExpr` sub-tree) can be converted via [`convert`]. -/// `from_legacy_scalar` alone cannot do this — it has no converter to -/// recurse with — which is why `ScalarSubquery` was the one arm it defers. +/// through every composite variant. pub fn convert_scalar(se: &LScalarExpr, schema: &Schema) -> Result { match se { - LScalarExpr::ScalarSubquery(inner) => { - Ok(Predicate::ScalarSubquery(Box::new(convert(inner, schema)?))) + LScalarExpr::ScalarSubquery(_) => Err(ConvertError::UnsupportedScalarSubquery), + LScalarExpr::BinaryOp { op, lhs, rhs } => { + let l = convert_scalar(lhs, schema)?.0; + let r = convert_scalar(rhs, schema)?.0; + Ok(binary_scalar_op(op, l, r)) } - LScalarExpr::BinaryOp { op, lhs, rhs } => Ok(Predicate::BinaryOp { - op: op.clone(), - lhs: Box::new(convert_scalar(lhs, schema)?), - rhs: Box::new(convert_scalar(rhs, schema)?), - }), - LScalarExpr::IsNull { expr, negated } => Ok(Predicate::IsNull { - expr: Box::new(convert_scalar(expr, schema)?), - negated: *negated, - }), - LScalarExpr::FunctionCall { name, args } => Ok(Predicate::FunctionCall { - name: name.clone(), - args: args + LScalarExpr::IsNull { expr, negated } => { + let inner = Box::new(convert_scalar(expr, schema)?.0); + Ok(Predicate(if *negated { + L3Expr::IsNotNull(inner) + } else { + L3Expr::IsNull(inner) + })) + } + LScalarExpr::FunctionCall { name, args } => { + let exprs = args .iter() - .map(|a| convert_scalar(a, schema)) - .collect::, _>>()?, - }), + .map(|a| Ok(convert_scalar(a, schema)?.0)) + .collect::, ConvertError>>()?; + Ok(Predicate(L3Expr::FunctionCall { + name: name.clone(), + args: exprs, + })) + } LScalarExpr::InList { expr, list, negated, - } => Ok(Predicate::InList { - expr: Box::new(convert_scalar(expr, schema)?), - list: list + } => { + let e = convert_scalar(expr, schema)?.0; + let list_exprs = list .iter() - .map(|a| convert_scalar(a, schema)) - .collect::, _>>()?, - negated: *negated, - }), + .map(|a| Ok(convert_scalar(a, schema)?.0)) + .collect::, ConvertError>>()?; + Ok(Predicate(L3Expr::InList { + expr: Box::new(e), + list: list_exprs, + negated: *negated, + })) + } LScalarExpr::Between { expr, low, high, negated, - } => Ok(Predicate::Between { - expr: Box::new(convert_scalar(expr, schema)?), - low: Box::new(convert_scalar(low, schema)?), - high: Box::new(convert_scalar(high, schema)?), - negated: *negated, - }), - // `Column` / `Literal` are subquery-free leaves — `from_legacy_scalar` - // translates them directly (and cannot fail for these arms). - LScalarExpr::Column(_) | LScalarExpr::Literal(_) => { - from_legacy_scalar(se).map_err(ConvertError::Scalar) + } => { + let e = convert_scalar(expr, schema)?.0; + let l = convert_scalar(low, schema)?.0; + let h = convert_scalar(high, schema)?.0; + Ok(Predicate(between(e, l, h, *negated))) + } + LScalarExpr::Column(name) => { + let id = resolve_column_ref(&LColumnRef::Named(name.clone()), schema)?; + Ok(Predicate(L3Expr::Column(id))) } + LScalarExpr::Literal(lit) => Ok(Predicate(literal_from_legacy(lit))), } } -/// Legacy `ColumnRef` → canonical `ColumnRef`. The two enums have -/// identical variants; the canonical one is what L3 nodes carry. -fn convert_column_ref(c: &LColumnRef) -> CColumnRef { - match c { - LColumnRef::Named(n) => CColumnRef::Named(n.clone()), - LColumnRef::SampleValue => CColumnRef::SampleValue, - LColumnRef::Wildcard => CColumnRef::Wildcard, - } +/// Translate a legacy `ScalarExpr::BinaryOp`'s operator + converted +/// operands into the corresponding `L3Expr`. `op` is already the +/// canonical `BinaryOpKind` (`relational::BinaryOpKind` re-exports the +/// same type `query_expr::BinaryOp.op` carries — no separate flat legacy +/// enum exists), so this is a structural unwrap, not a value mapping. +fn binary_scalar_op(op: &BinaryOpKind, l: L3Expr, r: L3Expr) -> Predicate { + let e = match op { + BinaryOpKind::Arith(arith_op) => L3Expr::Arith { + op: arith_op.clone(), + left: Box::new(l), + right: Box::new(r), + }, + BinaryOpKind::Compare(cmp_op) => L3Expr::Compare { + left: Box::new(l), + op: cmp_op.clone(), + right: Box::new(r), + }, + BinaryOpKind::And => L3Expr::BoolAnd(vec![l, r]), + BinaryOpKind::Or => L3Expr::BoolOr(vec![l, r]), + // `Unless` / `Pow` / `Atan2` are PromQL vector-set / power ops with + // no scalar-predicate counterpart; neither parser constructs a + // `ScalarExpr::BinaryOp` with one of these (they only appear on + // `QueryExpr::BinaryOp`, passed straight through in `convert`). + // Defensive fallback rather than a panic on unreachable input. + BinaryOpKind::Unless | BinaryOpKind::Pow | BinaryOpKind::Atan2 => { + L3Expr::Literal(L3Scalar::Boolean(true)) + } + }; + Predicate(e) +} + +fn literal_from_legacy(lit: &crate::intent_algebra::relational::LiteralValue) -> L3Expr { + use crate::intent_algebra::relational::LiteralValue as L; + L3Expr::Literal(match lit { + L::Null => L3Scalar::Null, + L::Bool(b) => L3Scalar::Boolean(*b), + L::Int(i) => L3Scalar::Int64(*i), + L::Float(f) => L3Scalar::Float64(*f), + L::Str(s) => L3Scalar::Utf8(s.clone()), + // No L3Scalar counterpart -- fold to nanosecond count, matching + // the pre-merge `from_legacy_scalar`'s documented behavior. + L::Duration(d) => L3Scalar::Int64(d.as_nanos() as i64), + }) } // ── AggFunc → AggIntent sketch mapping ─────────────────────────────────────── /// Map an [`AggFunc`] to the canonical [`AggIntent`]s the `convert` -/// `Aggregate` arm fuses on. Empty for non-sketchable functions -/// (`Custom`); one intent for single-statistic functions; two for the -/// `StdDev` / `Variance` fan-out (the caller wraps the pair in a `Merge` -/// of sibling sketch aggregates). -fn agg_func_to_intents(func: &AggFunc) -> Vec { - use crate::intent_algebra::relational::{ - default_cardinality, default_frequency, default_quantile, - }; +/// `Aggregate` arm fuses on. `grouped` is `!keys.is_empty()` at the call +/// site. Empty for non-canonical functions (`Custom` only); one intent +/// for every ordinary function (delegates to [`AggFunc::to_sketch_op`]); +/// two for the `StdDev` / `Variance` fan-out (the caller wraps the pair +/// in a `Merge` of sibling sketch aggregates, the one case `to_sketch_op` +/// can't express since it returns a single `Option`). +fn agg_func_to_intents(func: &AggFunc, grouped: bool) -> Vec { match func { - AggFunc::Quantile(phi) => vec![default_quantile(*phi)], - AggFunc::CountDistinct => vec![default_cardinality()], - AggFunc::HeavyHitters { .. } => vec![default_frequency()], - AggFunc::Frequency => vec![default_frequency()], - AggFunc::Count => vec![default_frequency()], - AggFunc::Avg => vec![AggIntent::Quantile { - col: None, - q: 0.5, - accuracy: AccuracyTarget::Epsilon(0.01), - }], - AggFunc::Min => vec![AggIntent::Min { col: None }], - AggFunc::Max => vec![AggIntent::Max { col: None }], - // StdDev / Variance: legacy carried two quantiles in a single - // Quantile intent; Step α F1 fans them out into two siblings. AggFunc::StdDev { .. } | AggFunc::Variance { .. } => vec![ AggIntent::Quantile { col: None, @@ -508,10 +579,42 @@ fn agg_func_to_intents(func: &AggFunc) -> Vec { accuracy: AccuracyTarget::Epsilon(0.01), }, ], - AggFunc::Sum | AggFunc::Rate | AggFunc::Increase | AggFunc::Delta => { - vec![AggIntent::Sum { col: None }] - } - AggFunc::Custom(_) => vec![], + // `Rate` / `Increase` / `Delta` map onto `AggIntent::Sum`, not + // the dedicated `AggIntent::Rate` / `Increase` / `Delta` + // variants `to_sketch_op()` would otherwise reach for — the + // `asap_tier_analysis` engine dispatch deliberately collapses + // all of `rate` / `irate` / `increase` / `sum_over_time` onto + // one `Capability::ExactAgg(Sum)` and disambiguates via the + // separate typed `outer_fn` field instead (see + // `asap_tier_analysis::tests::rate_and_sum_over_time_share_ + // capability_but_differ_on_outer_fn` and the surrounding + // "outer_fn — rate vs plain disambiguation" test block, which + // documents this as the deliberate replacement for a retired + // raw-PromQL re-parser). Using the dedicated intents here would + // fragment that dispatch. + AggFunc::Rate | AggFunc::Increase | AggFunc::Delta => vec![AggIntent::Sum { col: None }], + // `Avg` approximates as the p50 (median) quantile sketch rather + // than `to_sketch_op()`'s literal `AggIntent::Avg` (exact, + // non-mergeable) — matches `Min`/`Max`'s boundary-quantile + // treatment (`AggIntent::Min` ~ q=0.0, `Max` ~ q=1.0) and is what + // `query_parser::QeCollector::collect_op` (which has no `Avg` + // arm of its own) relies on to classify `avg_over_time` as + // `AggType::Quantile` with `quantiles: [0.5]`. + AggFunc::Avg => vec![AggIntent::Quantile { + col: None, + q: 0.5, + accuracy: AccuracyTarget::Epsilon(0.01), + }], + // A *grouped* `Count` is `count_over_time(...) by (...)` (or the + // PromQL `topk` bridge's synthetic `Count` — see + // `query_parser::promql::build_windowed_agg`'s "Count-with- + // GROUP-BY → Frequency" comment) — structurally per-series and + // sketchable, so it takes the same `Frequency`/CMS path as + // `AggFunc::Frequency`/`HeavyHitters`. An *ungrouped* `Count` is + // the SQL `COUNT(*)` exact-row-count case and keeps + // `to_sketch_op()`'s literal `AggIntent::Count{Exact}`. + AggFunc::Count if grouped => vec![crate::intent_algebra::default_frequency()], + other => other.to_sketch_op().into_iter().collect(), } } @@ -521,8 +624,9 @@ fn agg_func_to_intents(func: &AggFunc) -> Vec { mod tests { use super::*; use crate::intent_algebra::relational::{ - AggFunc, AggItem, ColumnRef as LColumnRef, ProjectItem as LProjectItem, SourceSpec, + AggItem, ColumnRef as LColumnRef, ProjectItem as LProjectItem, SourceSpec, }; + use std::time::Duration; fn src(name: &str) -> LQueryExpr { LQueryExpr::Source(SourceSpec { name: name.into() }) @@ -543,13 +647,13 @@ mod tests { match c { CQueryExpr::Scan { source, - label_filters, + predicates, schema, } => { assert!( matches!(source, Source::TimeSeries { metric } if metric == "http_requests_total") ); - assert!(label_filters.is_empty()); + assert!(predicates.is_empty()); assert_eq!(schema.columns.len(), 2); // ts, value } other => panic!("expected Scan, got {other:?}"), @@ -620,9 +724,7 @@ mod tests { } #[test] - fn single_sketchable_aggregate_folds_to_canonical_aggregate() { - // A single-statistic sketchable `Aggregate` over a non-`Window` - // input folds to a canonical `Aggregate { by: [], aggs: [intent] }`. + fn single_aggregate_folds_to_canonical_aggregate() { let legacy = LQueryExpr::Aggregate { keys: vec![], aggs: vec![agg_item("s", AggFunc::Sum)], @@ -638,11 +740,27 @@ mod tests { } } + #[test] + fn ungrouped_count_is_exact() { + let legacy = LQueryExpr::Aggregate { + keys: vec![], + aggs: vec![agg_item("n", AggFunc::Count)], + having: None, + input: Box::new(src("m")), + }; + match convert_root(&legacy).unwrap() { + CQueryExpr::Aggregate { aggs, .. } => assert!(matches!( + aggs.as_slice(), + [AggIntent::Count { + accuracy: AccuracyTarget::Exact + }] + )), + other => panic!("expected Aggregate, got {other:?}"), + } + } + #[test] fn aggregate_target_column_is_not_a_group_by_key() { - // The `AggItem.col` (the statistic's input column) is *not* a - // GROUP BY key — only `Aggregate.keys` is. A `Named` agg-target - // column therefore leaves the canonical `by` empty. let legacy = LQueryExpr::Aggregate { keys: vec![], aggs: vec![AggItem { @@ -661,10 +779,7 @@ mod tests { } #[test] - fn single_sketchable_aggregate_over_window_folds_to_window_over_aggregate() { - // A single-statistic sketchable `Aggregate` whose input is a - // `Window` folds to the `Window { Aggregate { by: [] } }` shape — - // the window-defines-sketch-lifecycle form. + fn single_aggregate_over_window_folds_to_window_over_aggregate() { let legacy = LQueryExpr::Aggregate { keys: vec![], aggs: vec![agg_item("q", AggFunc::Quantile(0.99))], @@ -689,11 +804,36 @@ mod tests { } } + #[test] + fn stddev_fans_out_into_merge_of_quantile_siblings() { + let legacy = LQueryExpr::Aggregate { + keys: vec![], + aggs: vec![agg_item("sd", AggFunc::StdDev { population: false })], + having: None, + input: Box::new(src("m")), + }; + match convert_root(&legacy).unwrap() { + CQueryExpr::Merge { children } => { + assert_eq!(children.len(), 2); + for c in &children { + assert!(matches!( + c, + CQueryExpr::Aggregate { + aggs, + .. + } if matches!(aggs.as_slice(), [AggIntent::Quantile { .. }]) + )); + } + } + other => panic!("expected Merge, got {other:?}"), + } + } + #[test] fn topk_folds_into_aggregate_with_topk_intent() { let legacy = LQueryExpr::TopK { k: 5, - by: vec![], + by: vec![].into(), input: Box::new(src("m")), }; match convert_root(&legacy).unwrap() { @@ -720,23 +860,17 @@ mod tests { } #[test] - fn filter_pred_with_scalar_subquery_recurses() { - // Filter { pred: ScalarSubquery(Ref("cte")), Source } — the - // ScalarSubquery arm `from_legacy_scalar` defers is handled here - // by recursing through `convert`. + fn filter_pred_with_scalar_subquery_is_rejected() { + // ScalarSubquery has no canonical L3Expr representation (see the + // module doc) -- convert_scalar rejects it rather than guessing. let legacy = LQueryExpr::Filter { pred: LScalarExpr::ScalarSubquery(Box::new(LQueryExpr::Ref("cte".into()))), input: Box::new(src("m")), }; - match convert_root(&legacy).unwrap() { - CQueryExpr::Filter { pred, .. } => match pred { - Predicate::ScalarSubquery(inner) => { - assert!(matches!(*inner, CQueryExpr::Ref { name } if name.as_str() == "cte")); - } - other => panic!("expected ScalarSubquery, got {other:?}"), - }, - other => panic!("expected Filter, got {other:?}"), - } + assert!(matches!( + convert_root(&legacy).unwrap_err(), + ConvertError::UnsupportedScalarSubquery + )); } #[test] @@ -752,7 +886,7 @@ mod tests { CQueryExpr::Project { cols, .. } => { assert_eq!(cols.len(), 1); assert_eq!(cols[0].alias.as_deref(), Some("v")); - assert!(matches!(&cols[0].expr, Predicate::Column(_))); + assert!(matches!(cols[0].expr, L3Expr::Column(_))); } other => panic!("expected Project, got {other:?}"), } @@ -761,7 +895,7 @@ mod tests { #[test] fn binary_op_converts_both_sides() { let legacy = LQueryExpr::BinaryOp { - op: crate::intent_algebra::query_expr::BinaryOpKind::Add, + op: BinaryOpKind::Arith(ArithOp::Add), lhs: Box::new(src("a")), rhs: Box::new(src("b")), vector_match: None, @@ -775,17 +909,37 @@ mod tests { } } + #[test] + fn scalar_binary_op_translates_arith_and_compare() { + let legacy = LScalarExpr::BinaryOp { + op: BinaryOpKind::Compare(CompareOp::Gt), + lhs: Box::new(LScalarExpr::Column("value".into())), + rhs: Box::new(LScalarExpr::Literal( + crate::intent_algebra::relational::LiteralValue::Float(1.0), + )), + }; + let schema = crate::intent_algebra::column_resolution::infer_source_schema("m"); + let pred = convert_scalar(&legacy, &schema).unwrap(); + assert!(matches!( + pred.0, + L3Expr::Compare { + op: CompareOp::Gt, + .. + } + )); + } + #[test] fn cross_join_none_pred_becomes_true_literal() { let legacy = LQueryExpr::Join { - kind: crate::intent_algebra::query_expr::JoinKind::Cross, + kind: crate::intent_algebra::relational::JoinKind::Cross, pred: None, left: Box::new(src("a")), right: Box::new(src("b")), }; match convert_root(&legacy).unwrap() { CQueryExpr::Join { pred, .. } => { - assert!(matches!(pred, Predicate::Literal(LiteralValue::Bool(true)))); + assert!(matches!(pred.0, L3Expr::Literal(L3Scalar::Boolean(true)))); } other => panic!("expected Join, got {other:?}"), } @@ -831,7 +985,6 @@ mod tests { }), }; let c = convert_root(&legacy).unwrap(); - // Walk down and assert the spine survived. let CQueryExpr::Sort { child, .. } = c else { panic!("expected Sort") }; diff --git a/control_plane/src/intent_algebra/mod.rs b/control_plane/src/intent_algebra/mod.rs index c3c675a9..2043ae36 100644 --- a/control_plane/src/intent_algebra/mod.rs +++ b/control_plane/src/intent_algebra/mod.rs @@ -73,11 +73,10 @@ pub mod agg_intent; pub mod cse; -// Not yet re-exported into the crate::intent_algebra::* top-level surface -// below -- `query_expr::ColumnRef` already claims that name, and this -// module is unused until the query_expr.rs/relational.rs merge (next -// Phase 2 step) retargets `Predicate`/`ScalarExpr` onto `L3Expr`/`L2Expr`. -// Reachable today only via the full `intent_algebra::expr_ir::` path. +// `L2Expr` isn't used yet -- `relational.rs` (L2) still builds its own +// `ScalarExpr`, not `L2Expr`; that's the next Phase 2 step. `L3Expr` is +// used now: `query_expr.rs`'s `Predicate` is `L3Expr`-based as of this +// merge. pub mod expr_ir; pub mod query_expr; pub mod schema; @@ -113,11 +112,12 @@ pub use agg_intent::{ ranking_measure, AggIntent, MathFunc, RankingMeasure, TimeFunc, }; pub use cse::{dedupe_subtrees, CseWorkloadPlan}; +pub use expr_ir::{ArithOp, ColumnRef, CompareOp, Expr, L2Expr, L3Expr, L3Scalar}; pub use query_expr::{ - from_legacy_scalar, BinaryOpKind, BindingScope, ColumnRef, GroupSide, HavingPredicate, - JoinKind, LabelFilter, LiteralValue, PartitionKeys, Predicate, ProjectItem, QueryExpr, - QueryExprError, SetOpKind, SortKey, Source, VectorGrouping, VectorMatch, VectorMatchKind, - WindowKind, + aggregate_output_schema, between, conjoin, label_filter_to_predicate, AtModifier, BinaryOpKind, + BindingScope, DataModel, GroupKeys, GroupSide, InfoMatcher, JoinKind, LabelFilter, Predicate, + ProjectItem, QueryExpr, QueryExprError, SampleKind, SetOpKind, SortKey, Source, TimeShift, + VectorGrouping, VectorMatch, VectorMatchKind, WindowFuncKind, WindowKind, }; pub use schema::{cse_reuse_is_legal, Column, ColumnId, CseError, DataType, Schema}; diff --git a/control_plane/src/intent_algebra/query_expr.rs b/control_plane/src/intent_algebra/query_expr.rs index c54ccacb..ed787243 100644 --- a/control_plane/src/intent_algebra/query_expr.rs +++ b/control_plane/src/intent_algebra/query_expr.rs @@ -1,1192 +1,129 @@ //! Layer 3 IR — `QueryExpr` DAG (intent-only, language-orthogonal, //! deployment-independent). //! -//! Per `control_plane/docs/design.md` §6 "`core::intent_algebra` — Layer 3" -//! (around line ~269). Pure intent at this layer: no language-specific -//! operators (no `HistogramQuantile`, no `PromQLSubquery`), no sketch -//! types, no sketch parameters, no physical operator choice. +//! ## Phase 2 step 3 (docs/migration-plan-backend-plan.md) //! -//! Single-root tree per query. Multi-root DAGs (cross-query CSE fan-in) -//! live one level above in `WorkloadPlan` (`types_v2::WorkloadPlan`). -//! Within-query CTE / let-binding fan-in *is* expressible here via -//! [`QueryExpr::LetBinding`] + [`QueryExpr::Ref`]. +//! `QueryExpr` and its supporting types are no longer defined in this +//! repo — re-exported from `asap_ir::intent_algebra`. `asap_ir`'s version +//! is a real superset (~22 variants vs. this repo's pre-merge 16: +//! `EvalTime`, `VectorFromScalar`/`ScalarFromVector`, `Relabel`, +//! `InfoJoin`, `Sample`, `TimeRange`, `TimeShift`, `WindowFunc` are new, +//! each backing real PromQL surface from issues #40-#118), and its +//! `output_schema_in` is a complete, already-tested implementation for +//! every variant — adopted via the inherent method, not reimplemented. //! -//! Variant set. Phase B shipped `Scan`, `Window`, `Aggregate`, -//! `LetBinding`, `Ref`. Batch 2 of the relational migration adds the ten -//! "A-classified" structurally-canonical variants from `design.md` §6: -//! `Filter`, `Project`, `Partition`, `Distinct`, `Merge`, `Join`, -//! `SetOp`, `Sort`, `Limit`, `BinaryOp`. Step γ7 adds `Subquery` (the -//! canonical counterpart of `relational::PromQLSubquery`). `WindowFunc` -//! remains deferred until the planner grows a consumer for it. The shape -//! defined here is forward-compatible — adding more variants is purely -//! additive. +//! Three representational differences turned out, on inspection, not to +//! be missing capability — `asap_ir` represents the same things more +//! consolidated, just under different names: //! -//! Single-input variants here use `child:` (matching the existing -//! `Window`, `Aggregate`, `LetBinding` shape). Legacy `input:` survives in -//! `relational::QueryExpr` until its consumers redirect through here. - -#![allow(dead_code)] - -use std::collections::HashMap; -use std::time::Duration; - -use serde::{Deserialize, Serialize}; -use thiserror::Error; - -use crate::intent_algebra::agg_intent::AggIntent; -use crate::intent_algebra::schema::{Column, ColumnId, DataType, Schema}; -use crate::types_v2::BindingName; - -/// Errors produced when the L3 IR is constructed or its output schema is -/// derived. Surfaced by the lowering function and any caller that walks -/// the DAG. -#[derive(Debug, Error)] -pub enum QueryExprError { - /// `Ref(name)` did not resolve against any in-scope `LetBinding`. - #[error("unresolved ref: {0}")] - UnresolvedRef(String), - /// `Aggregate { by, .. }` referenced a column position that is not - /// in the input schema. Caught at schema-derivation time per the - /// design's locally-checkable invariant. - #[error("by-column id {0} out of range (input has {1} columns)")] - InvalidGroupByColumn(ColumnId, usize), - /// `Window` requires a `time_index` field on its input schema — - /// `design.md` §6 schema-flow table. - #[error("Window requires a time_index on input schema")] - WindowMissingTimeIndex, - /// `Merge` requires at least one child to derive its output schema. - #[error("Merge requires at least one child")] - EmptyMerge, - /// A legacy `ScalarExpr` variant has no canonical `Predicate` counterpart - /// yet. Surfaces from [`from_legacy_scalar`] for `ScalarSubquery` only — - /// it carries a legacy `QueryExpr` sub-tree that needs the tree converter. - #[error("legacy ScalarExpr variant `{0}` is not yet representable in canonical Predicate")] - UnsupportedLegacyScalar(&'static str), -} - -/// Streaming / time-window kind. PromQL `[5m]` is `Sliding`; SQL `TUMBLE` -/// is `Tumbling`; PromQL has no native `Session` window so it stays -/// unused for the DC + PromQL scope of this PR. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum WindowKind { - Tumbling, - Sliding, - Session, -} - -/// Source of a `Scan`. Phase B ships `TimeSeries` (the only shape DC + -/// PromQL needs); `Table` is sketched out so future deployment models -/// (asap-fusion / OLAP) plug in without an enum-shape rev. Recursive -/// `Source::Join` is design.md §6 line ~378 territory and stays out of -/// scope for now. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum Source { - /// PromQL / DC lifecycle leaf — a metric stream identified by name + - /// optional label filters; produces `(timestamp, value, *labels)` - /// columns. - TimeSeries { metric: String }, - /// Tabular leaf — reserved for asap-fusion. Carries the table name; - /// columns ride on the supplied `Schema`. - Table { table_ref: String }, -} - -/// Equality label filter on a `Scan`. PromQL `{service="api"}` → one of -/// these; richer match operators (`!=`, `=~`, `!~`) live in `Filter`'s -/// generic predicate per design.md §6 line ~296 and are deferred to the -/// follow-up phase that adds the `Filter` variant. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +//! - **`Predicate`** is now `L3Expr` (`expr_ir.rs`, landed previously) — +//! `struct Predicate(pub L3Expr)`, not this repo's own 8-variant enum. +//! Six of the eight variants map directly. `Between` desugars to +//! `Expr::BoolAnd([Compare(Ge), Compare(Le)])` (verified: one real +//! construction site, nothing downstream pattern-matches on the shape). +//! `ScalarSubquery(Box)` has no `Expr` slot — it's +//! restructured to a `QueryExpr`-level `LetBinding` wrap instead of an +//! embedded predicate node, in `lower.rs`'s `convert_scalar` (see that +//! file). This is exactly the shape `optimizer/engine.rs`'s R7 +//! `SubqueryDecorrelation` rule already produces as a *post-hoc* +//! rewrite — doing it at construction time makes R7 dead code, deleted +//! there. +//! - **`HavingPredicate`** (this repo's opaque `String` wrapper) is gone; +//! `Aggregate.having` is `Option` — real typed HAVING +//! instead of a formatted debug string. Its one real construction site +//! (`lower.rs`) was already just `format!(...)`-ing a placeholder, not +//! real HAVING evaluation. +//! - **`Partition { keys, child }`** doesn't exist in `asap_ir` — folded +//! into `Aggregate.by: GroupKeys`, which already carries `by(...)` vs. +//! `without(...)` directly (`{keys: Vec, without: bool}`). +//! Verified this repo's own `promql.rs` already builds `Aggregate.by` +//! directly for the `count` case, skipping `Partition` entirely — the +//! `sum`/`avg`/etc. path was the inconsistent one. Verified downstream: +//! `physical/allocator.rs` labels `Partition` `ExecutionMode::Passthrough` +//! with rationale `"Partition for GROUP BY at Agent"`, and +//! `physical/planner.rs` converts it straight into +//! `PhysicalOp::HashAggregate { keys }` — nothing stage/sharding-specific +//! despite the old doc comment's framing. `promql.rs`'s +//! `apply_qe_partition` is deleted; grouping keys attach to the nearest +//! `Aggregate.by` directly instead. +//! +//! `LiteralValue` (this repo's narrow 5-variant literal enum) is replaced +//! by `L3Scalar` (same five cases, `expr_ir.rs`). `ColumnRef` here +//! (this repo's `Named`/`SampleValue`/`Wildcard`) is gone — L3 is fully +//! positional (`ColumnId`) in `asap_ir`; the name-based form only exists +//! at L2 now (`expr_ir::ColumnRef`, used by `relational.rs`, not yet +//! merged). `LabelFilter` stays — PromQL-ergonomic sugar for building a +//! `Scan`'s `predicates: Vec`, converted via +//! `label_filter_to_predicate` below (`asap_ir`'s `Scan` takes typed +//! `Predicate`s, not a separate label-filter list). + +use asap_ir::intent_algebra::schema::ColumnId; +pub use asap_ir::intent_algebra::{ + aggregate_output_schema, AtModifier, BinaryOpKind, BindingScope, DataModel, GroupKeys, + GroupSide, InfoMatcher, JoinKind, Predicate, ProjectItem, QueryExpr, QueryExprError, + SampleKind, SetOpKind, SortKey, Source, TimeShift, VectorGrouping, VectorMatch, + VectorMatchKind, WindowFuncKind, WindowKind, +}; +pub use asap_ir::intent_algebra::{ArithOp, ColumnRef, CompareOp, Expr, L3Scalar}; + +use crate::intent_algebra::schema::Schema; +use crate::intent_algebra::L3Expr; + +/// Equality label filter on a `Scan`. PromQL `{service="api"}` — kept as +/// ergonomic sugar for the parser; converted to a typed `Predicate` via +/// [`label_filter_to_predicate`] when building the `Scan` node itself. +/// Richer match operators (`!=`, `=~`, `!~`) go through +/// [`CompareOp::Regex`]/[`CompareOp::NotRegex`] the same way. +#[derive(Debug, Clone, PartialEq, Eq)] pub struct LabelFilter { pub label: String, pub equals: String, } -/// Optional HAVING-style predicate on `Aggregate`. Modeled as an opaque -/// expression string at L3 — Phase B doesn't have a typed predicate IR -/// yet; adding one is a separate PR (would also introduce the `Filter` -/// variant per design.md §6). -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct HavingPredicate(pub String); - -// ── Supporting types lifted from relational ───────────────────────────────── -// -// Per Batch 2 of the relational migration: these are structural copies of -// the legacy supporting enums so the canonical [`QueryExpr`] variants below -// can reference them without rooting the canonical IR in `relational`. -// Field shapes mirror `design.md` §6. - -/// Which column / field a sketch / projection / DISTINCT operation targets. -/// Survives at L3 as a name-keyed alias (design.md §6.1) even though -/// canonical schema uses positional [`ColumnId`] — intents like -/// `AggIntent::TopK { by: Vec }` consume it. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ColumnRef { - /// Explicit column name (SQL: `AVG(price)` → `Named("price")`). - Named(String), - /// The implicit metric sample value (PromQL — always the series value). - SampleValue, - /// All rows / COUNT(*). - Wildcard, -} - -/// Partition-key spec — `by (k1, k2, …)` or `without (k1, k2, …)`. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum PartitionKeys { - /// `by (k1, k2, ...)` — explicit key list. - By(Vec), - /// `without (k1, k2, ...)` — complement; resolved against schema at plan time. - Without(Vec), -} - -impl PartitionKeys { - pub fn keys(&self) -> &[String] { - match self { - PartitionKeys::By(k) | PartitionKeys::Without(k) => k, - } - } - - pub fn is_empty(&self) -> bool { - self.keys().is_empty() - } -} - -/// Binary operator kinds — used in both [`Predicate::BinaryOp`] (scalar -/// composition) and [`QueryExpr::BinaryOp`] (PromQL instant-vector -/// arithmetic between two relational sub-expressions). -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum BinaryOpKind { - // Arithmetic - Add, - Sub, - Mul, - Div, - Mod, - Pow, - // Comparison - Eq, - Ne, - Lt, - Le, - Gt, - Ge, - // Logical - And, - Or, - // Bitwise - BitAnd, - BitOr, - BitXor, - // String / pattern - Concat, - Like, - NotLike, - Regex, - NotRegex, - // PromQL-specific - Unless, - Atan2, -} - -impl std::fmt::Display for BinaryOpKind { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let s = match self { - BinaryOpKind::Add => "+", - BinaryOpKind::Sub => "-", - BinaryOpKind::Mul => "*", - BinaryOpKind::Div => "/", - BinaryOpKind::Mod => "%", - BinaryOpKind::Pow => "^", - BinaryOpKind::Eq => "=", - BinaryOpKind::Ne => "!=", - BinaryOpKind::Lt => "<", - BinaryOpKind::Le => "<=", - BinaryOpKind::Gt => ">", - BinaryOpKind::Ge => ">=", - BinaryOpKind::And => "AND", - BinaryOpKind::Or => "OR", - BinaryOpKind::BitAnd => "&", - BinaryOpKind::BitOr => "|", - BinaryOpKind::BitXor => "XOR", - BinaryOpKind::Concat => "||", - BinaryOpKind::Like => "LIKE", - BinaryOpKind::NotLike => "NOT LIKE", - BinaryOpKind::Regex => "=~", - BinaryOpKind::NotRegex => "!~", - BinaryOpKind::Unless => "unless", - BinaryOpKind::Atan2 => "atan2", - }; - write!(f, "{s}") - } -} - -/// JOIN variant. design.md §6 lists Inner / LeftOuter / RightOuter / -/// FullOuter / Cross / Semi / AntiSemi. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum JoinKind { - Inner, - LeftOuter, - RightOuter, - FullOuter, - Cross, - /// Semi-join: return only left rows that have a match (WHERE EXISTS). - Semi, - /// Anti-join: return only left rows that have no match (WHERE NOT EXISTS). - AntiSemi, -} - -/// Set-operation variant — UNION / INTERSECT / EXCEPT. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum SetOpKind { - Union, - Intersect, - Except, -} - -/// ORDER BY sort key. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct SortKey { - pub col: String, - pub desc: bool, - /// NULLS FIRST / NULLS LAST (None → database default). - #[serde(default)] - pub nulls_first: Option, -} - -/// PromQL vector matching semantics (`on (…)` / `ignoring (…)` plus -/// `group_left` / `group_right`). -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct VectorMatch { - pub kind: VectorMatchKind, - pub labels: Vec, - #[serde(default)] - pub grouping: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum VectorMatchKind { - On, - Ignoring, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct VectorGrouping { - pub side: GroupSide, - pub labels: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum GroupSide { - Left, - Right, -} - -/// Scalar literal value. Subset of values used by the canonical -/// [`Predicate`] — extended literal kinds (durations, intervals) stay in -/// `relational::LiteralValue` until the E-variants migrate. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum LiteralValue { - Null, - Bool(bool), - Int(i64), - Float(f64), - Str(String), -} - -/// One item in a SELECT projection list. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct ProjectItem { - /// Output column name (SQL `AS alias`; None → use expression name). - #[serde(default)] - pub alias: Option, - /// Projected expression. Modeled as a [`Predicate`] for the four - /// covered scalar shapes (column / literal / binary op / is-null); - /// the legacy E-variants (`FunctionCall`, `ScalarSubquery`, `InList`, - /// `Between`) stay in `relational::ScalarExpr` and live in - /// [`ProjectItem::raw_expr`] until they migrate. - pub expr: Predicate, -} - -// ── Typed Predicate ────────────────────────────────────────────────────────── - -/// Typed scalar predicate — the canonical counterpart of -/// `relational::ScalarExpr`. Covers all eight legacy scalar shapes: -/// column / literal / binary-op / is-null (the structurally clean four) -/// plus `FunctionCall` / `InList` / `Between` / `ScalarSubquery` (the -/// E-variants). `ScalarSubquery` carries a canonical [`QueryExpr`] — -/// translating a legacy `ScalarSubquery` requires the legacy→canonical -/// tree converter, so [`from_legacy_scalar`] still defers that one arm. -/// -/// See [`from_legacy_scalar`] for the migration helper. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum Predicate { - /// Column reference (by name). - Column(ColumnRef), - /// Constant literal. - Literal(LiteralValue), - /// Binary operator (=, !=, <, <=, >, >=, AND, OR, …). - BinaryOp { - op: BinaryOpKind, - lhs: Box, - rhs: Box, - }, - /// IS NULL / IS NOT NULL. - IsNull { expr: Box, negated: bool }, - /// Named scalar function call (`ABS(x)`, `DATE_TRUNC('hour', ts)`). - FunctionCall { name: String, args: Vec }, - /// Scalar sub-query (`SELECT MAX(price) FROM orders`). - ScalarSubquery(Box), - /// `expr IN (v1, v2, …)` / `NOT IN (…)`. - InList { - expr: Box, - list: Vec, - negated: bool, - }, - /// `expr BETWEEN low AND high` / `NOT BETWEEN …`. - Between { - expr: Box, - low: Box, - high: Box, - negated: bool, - }, -} - -/// L3 algebra node. See module doc for the variant subset rationale. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "node", rename_all = "snake_case")] -pub enum QueryExpr { - /// Outermost leaf — a metric stream / table read. `schema` is the - /// authoritative output of this scan, supplied by the lowering pass - /// (which consults the source / DB schema catalog at L1→L2 time). - Scan { - source: Source, - #[serde(default)] - label_filters: Vec, - schema: Schema, - }, - /// Streaming / time-window. Defines the lifecycle (flush / reset - /// bounds) of any aggregate in its sub-tree. - Window { - kind: WindowKind, - size: Duration, - #[serde(default)] - slide: Option, - child: Box, - }, - /// γ + α — GROUP BY + aggregate intents. `by` are positional - /// references into `child.output_schema().columns`; `aggs` carry - /// `AggIntent` (no sketch types — that's L4). - Aggregate { - by: Vec, - aggs: Vec, - #[serde(default)] - having: Option, - child: Box, - }, - /// SQL `WITH name AS (expr) SELECT ... FROM name` / PromQL recording- - /// rule binding. Names a sub-expression; references via `Ref(name)`. - /// Output schema = `child`'s output schema. - LetBinding { - name: BindingName, - expr: Box, - child: Box, - }, - /// Reference a `LetBinding` by name. Resolved at plan time; output - /// schema = the named binding's expression's output schema. - Ref { name: BindingName }, - - // ── A-classified variants lifted in Batch 2 of the relational migration ── - // - // Each is a structural copy of the legacy variant of the same name in - // `relational::QueryExpr`. Single-input variants here use `child:` to - // match the existing canonical `Window`/`Aggregate`/`LetBinding` shape, - // whereas legacy spells them `input:`. Consumers that haven't migrated - // yet keep using `relational::QueryExpr::*` — the legacy variants stay - // in place until the consumer-side redirect lands in subsequent batches. - /// σ — row-level filter (WHERE / PromQL label matchers). Uses the new - /// typed [`Predicate`] (only Column / Literal / BinaryOp / IsNull at L3 - /// for now; `FunctionCall` / `ScalarSubquery` / `InList` / `Between` - /// stay in `relational::ScalarExpr` until their own batch). - Filter { - pred: Predicate, - child: Box, - }, - - /// π — column projection (SELECT list). - Project { - cols: Vec, - child: Box, - }, - - /// Partition the stream by key tuple (`GROUP BY` / PromQL `by (dims)`). - /// Logical-only marker — carries a sharding hint for L5's stage allocator. - Partition { - keys: PartitionKeys, - child: Box, - }, - - /// δ — SQL `DISTINCT` / row deduplication on `cols`. - Distinct { - cols: Vec, - child: Box, - }, - - /// ⊕ — union of sub-results from independent stages or shards (the - /// exact-merge case). Sketch unions live in `PhysicalExpr`, not here. - Merge { children: Vec }, - - /// Logical join. L4 picks the physical alternative - /// (`HashJoin` / `SortMergeJoin` / `SketchJoin`). - Join { - kind: JoinKind, - pred: Predicate, - left: Box, - right: Box, - }, - - /// UNION / INTERSECT / EXCEPT, with or without ALL. - SetOp { - kind: SetOpKind, - all: bool, - left: Box, - right: Box, - }, - - /// Generic ORDER BY — survives L3 for non-heavy-hitter cases - /// (`ORDER BY name LIMIT 10`, `ORDER BY ts DESC LIMIT 1`). The heavy- - /// hitter shape (`ORDER BY count DESC LIMIT k`, PromQL `topk(k, …)`) - /// produces [`AggIntent::TopK`] rather than `Sort + Limit`. - Sort { - keys: Vec, - child: Box, - }, - - /// `LIMIT n OFFSET k`. Generic case only — see `Sort`'s doc-comment. - Limit { - n: usize, - offset: usize, - child: Box, - }, - - /// Arithmetic / comparison / boolean composition between two relational - /// sub-expressions (PromQL `+`, `/`, `and`, `or`, `unless`; SQL boolean - /// composition between sub-relations). - BinaryOp { - op: BinaryOpKind, - lhs: Box, - rhs: Box, - #[serde(default)] - vector_match: Option, - }, - - /// PromQL sub-query (`[range:resolution]`) — re-evaluates `child` - /// at `resolution`-spaced steps across the trailing `range` window, - /// producing a range-vector the enclosing function consumes. The - /// canonical counterpart of `relational::QueryExpr::PromQLSubquery`. - /// Logical pass-through for schema flow — the range/resolution are a - /// sampling hint the L5 precompute stage reads, not a schema transform. - Subquery { - range: Duration, - #[serde(default)] - resolution: Option, - child: Box, - }, -} - -impl QueryExpr { - /// Compute the output schema of this node. Walks the tree, resolving - /// `Ref` against `LetBinding`s in scope. Errors propagate per - /// [`QueryExprError`]. - /// - /// Callers that want the schema of the *root* of a single query call - /// `expr.output_schema(&BindingScope::default())`. - pub fn output_schema(&self) -> Result { - self.output_schema_in(&BindingScope::default()) - } - - /// Variant of [`Self::output_schema`] that takes an explicit binding - /// scope. Used internally during DAG walks; exposed for callers that - /// pre-populate bindings from a workload-level container. - pub fn output_schema_in(&self, scope: &BindingScope) -> Result { - match self { - QueryExpr::Scan { schema, .. } => Ok(schema.clone()), - QueryExpr::Window { child, .. } => { - let in_schema = child.output_schema_in(scope)?; - if in_schema.time_index.is_none() { - return Err(QueryExprError::WindowMissingTimeIndex); - } - // Window propagates row identity → carries unique_keys - // verbatim. Synthetic `window_id` / `window_start/end` - // columns (design.md §6 schema-flow row 6) are deferred - // until the planner consumes them. - Ok(in_schema) - } - QueryExpr::Aggregate { - by, aggs, child, .. - } => { - let in_schema = child.output_schema_in(scope)?; - // by-column ids must be in range. - let mut out_cols: Vec = Vec::with_capacity(by.len() + aggs.len()); - for &id in by { - let c = - in_schema - .columns - .get(id) - .ok_or(QueryExprError::InvalidGroupByColumn( - id, - in_schema.columns.len(), - ))?; - out_cols.push(c.clone()); - } - // One new column per intent. PromQL convention: - // intent applied to the synthetic `value` column when - // present; otherwise to the first non-grouped column. - let value_col_idx = in_schema - .column_id("value") - .or_else(|| (0..in_schema.columns.len()).find(|i| !by.contains(i))); - let probe = value_col_idx - .and_then(|i| in_schema.columns.get(i)) - .cloned() - .unwrap_or(Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - table: None, - }); - for intent in aggs { - out_cols.push(crate::intent_algebra::output_column(intent, &probe)); - } - // Output unique_keys = [by]. The group-by column tuple - // is unique in the output by construction (design.md §6 - // schema-flow table). - let unique_keys = if by.is_empty() { - Vec::new() - } else { - vec![(0..by.len()).collect()] - }; - // Aggregate strips the time axis — output is one row per - // group, not one row per timestamp. - Ok(Schema { - columns: out_cols, - time_index: None, - unique_keys, - closed: false, - }) - } - QueryExpr::LetBinding { name, expr, child } => { - // Bind `name` to `expr`'s output schema, then evaluate - // `child` in the extended scope. - let bound = expr.output_schema_in(scope)?; - let extended = scope.with(name.clone(), bound); - child.output_schema_in(&extended) - } - QueryExpr::Ref { name } => scope - .lookup(name) - .cloned() - .ok_or_else(|| QueryExprError::UnresolvedRef(name.as_str().into())), - - // ── A-variants — schema-flow per design.md §6 ──────────────── - // Filter / Partition / Sort / Limit / Subquery pass the child's - // schema through unchanged. - QueryExpr::Filter { child, .. } - | QueryExpr::Partition { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::Subquery { child, .. } => child.output_schema_in(scope), - - // Project: schema-flow says "the input schema projected to - // `cols`". Phase-B placeholder — return the child schema until - // the planner consumes Project's column-mapping output. - QueryExpr::Project { child, .. } => child.output_schema_in(scope), - - // Distinct: tighten unique_keys with the named cols. Schema is - // otherwise pass-through (design.md §6 schema-flow table). - QueryExpr::Distinct { cols, child } => { - let in_schema = child.output_schema_in(scope)?; - let mut out = in_schema.clone(); - let mut key_ids: Vec = Vec::with_capacity(cols.len()); - for c in cols { - if let ColumnRef::Named(name) = c { - if let Some(id) = in_schema.column_id(name) { - key_ids.push(id); - } - } - } - if !key_ids.is_empty() { - out.add_unique_key(key_ids); - } - Ok(out) - } - - // Merge / SetOp / Join / BinaryOp: take the left/first child's - // schema as the representative (design.md §6 schema-flow). - // Full union-compatibility checks land when the type-checker - // consumes them; structural lift only here. - QueryExpr::Merge { children } => children - .first() - .ok_or(QueryExprError::EmptyMerge) - .and_then(|c| c.output_schema_in(scope)), - QueryExpr::SetOp { left, .. } | QueryExpr::Join { left, .. } => { - left.output_schema_in(scope) - } - QueryExpr::BinaryOp { lhs, .. } => lhs.output_schema_in(scope), - } - } -} - -/// Lexical scope for `LetBinding` / `Ref` resolution. A persistent map -/// from binding name to the bound expression's output schema. Built -/// during the schema-derivation walk; the caller usually starts with -/// [`BindingScope::default()`]. -#[derive(Debug, Default, Clone)] -pub struct BindingScope { - bindings: HashMap, -} - -impl BindingScope { - /// Empty scope — no in-scope bindings. - pub fn new() -> Self { - Self::default() - } - - /// Return a new scope with `name` bound to `schema`. The original - /// scope is left unchanged (functional style — keeps recursion - /// shadow semantics correct). - pub fn with(&self, name: BindingName, schema: Schema) -> Self { - let mut bindings = self.bindings.clone(); - bindings.insert(name.as_str().into(), schema); - Self { bindings } - } - - /// Look up `name` in the current scope. `None` if unbound. - pub fn lookup(&self, name: &BindingName) -> Option<&Schema> { - self.bindings.get(name.as_str()) - } -} - -// ── relational::ScalarExpr → canonical Predicate translation ──────────────── - -/// Translate a [`relational::ScalarExpr`](crate::intent_algebra::relational::ScalarExpr) -/// into the canonical typed [`Predicate`]. All scalar shapes translate -/// except `ScalarSubquery`, which carries a legacy `QueryExpr` sub-tree: -/// that arm still returns [`QueryExprError::UnsupportedLegacyScalar`] until -/// the legacy→canonical tree converter lands and can recurse into it. -/// -/// `LiteralValue::Duration` is folded into a `Predicate::Literal(Int)` -/// carrying the nanosecond count, because the canonical [`LiteralValue`] -/// is deliberately narrower than the legacy spelling (no `Duration` literal -/// at this layer — see design.md §6 schema-flow `DataType` list). -pub fn from_legacy_scalar( - se: &crate::intent_algebra::relational::ScalarExpr, -) -> Result { - use crate::intent_algebra::relational as l; - match se { - l::ScalarExpr::Column(name) => Ok(Predicate::Column(ColumnRef::Named(name.clone()))), - l::ScalarExpr::Literal(lit) => Ok(Predicate::Literal(literal_from_legacy(lit))), - l::ScalarExpr::BinaryOp { op, lhs, rhs } => Ok(Predicate::BinaryOp { - op: binary_op_from_legacy(op), - lhs: Box::new(from_legacy_scalar(lhs)?), - rhs: Box::new(from_legacy_scalar(rhs)?), - }), - l::ScalarExpr::IsNull { expr, negated } => Ok(Predicate::IsNull { - expr: Box::new(from_legacy_scalar(expr)?), - negated: *negated, - }), - l::ScalarExpr::FunctionCall { name, args } => Ok(Predicate::FunctionCall { - name: name.clone(), - args: args - .iter() - .map(from_legacy_scalar) - .collect::, _>>()?, - }), - l::ScalarExpr::InList { - expr, - list, - negated, - } => Ok(Predicate::InList { - expr: Box::new(from_legacy_scalar(expr)?), - list: list - .iter() - .map(from_legacy_scalar) - .collect::, _>>()?, - negated: *negated, - }), - l::ScalarExpr::Between { - expr, - low, - high, - negated, - } => Ok(Predicate::Between { - expr: Box::new(from_legacy_scalar(expr)?), - low: Box::new(from_legacy_scalar(low)?), - high: Box::new(from_legacy_scalar(high)?), - negated: *negated, - }), - // `ScalarSubquery` carries a legacy `QueryExpr` sub-tree — needs the - // legacy→canonical tree converter to recurse. Deferred to that PR. - l::ScalarExpr::ScalarSubquery(_) => { - Err(QueryExprError::UnsupportedLegacyScalar("ScalarSubquery")) - } - } -} - -fn literal_from_legacy(lit: &crate::intent_algebra::relational::LiteralValue) -> LiteralValue { - use crate::intent_algebra::relational as l; - match lit { - l::LiteralValue::Null => LiteralValue::Null, - l::LiteralValue::Bool(b) => LiteralValue::Bool(*b), - l::LiteralValue::Int(i) => LiteralValue::Int(*i), - l::LiteralValue::Float(f) => LiteralValue::Float(*f), - l::LiteralValue::Str(s) => LiteralValue::Str(s.clone()), - // Durations fold to nanoseconds-as-Int — canonical LiteralValue - // has no Duration variant (narrower by design). - l::LiteralValue::Duration(d) => LiteralValue::Int(d.as_nanos() as i64), - } -} - -fn binary_op_from_legacy(op: &crate::intent_algebra::relational::BinaryOpKind) -> BinaryOpKind { - use crate::intent_algebra::relational as l; - match op { - l::BinaryOpKind::Add => BinaryOpKind::Add, - l::BinaryOpKind::Sub => BinaryOpKind::Sub, - l::BinaryOpKind::Mul => BinaryOpKind::Mul, - l::BinaryOpKind::Div => BinaryOpKind::Div, - l::BinaryOpKind::Mod => BinaryOpKind::Mod, - l::BinaryOpKind::Pow => BinaryOpKind::Pow, - l::BinaryOpKind::Eq => BinaryOpKind::Eq, - l::BinaryOpKind::Ne => BinaryOpKind::Ne, - l::BinaryOpKind::Lt => BinaryOpKind::Lt, - l::BinaryOpKind::Le => BinaryOpKind::Le, - l::BinaryOpKind::Gt => BinaryOpKind::Gt, - l::BinaryOpKind::Ge => BinaryOpKind::Ge, - l::BinaryOpKind::And => BinaryOpKind::And, - l::BinaryOpKind::Or => BinaryOpKind::Or, - l::BinaryOpKind::BitAnd => BinaryOpKind::BitAnd, - l::BinaryOpKind::BitOr => BinaryOpKind::BitOr, - l::BinaryOpKind::BitXor => BinaryOpKind::BitXor, - l::BinaryOpKind::Concat => BinaryOpKind::Concat, - l::BinaryOpKind::Like => BinaryOpKind::Like, - l::BinaryOpKind::NotLike => BinaryOpKind::NotLike, - l::BinaryOpKind::Regex => BinaryOpKind::Regex, - l::BinaryOpKind::NotRegex => BinaryOpKind::NotRegex, - l::BinaryOpKind::Unless => BinaryOpKind::Unless, - l::BinaryOpKind::Atan2 => BinaryOpKind::Atan2, - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::intent_algebra::schema::{Column, DataType}; - use crate::types_v2::AccuracyTarget; - - fn col(name: &str, dtype: DataType) -> Column { - Column { - name: name.into(), - dtype, - nullable: false, - table: None, - } - } - - fn ts_scan() -> QueryExpr { - QueryExpr::Scan { - source: Source::TimeSeries { - metric: "http_request_duration_seconds".into(), - }, - label_filters: vec![LabelFilter { - label: "service".into(), - equals: "api".into(), - }], - schema: Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("service", DataType::Utf8), - col("value", DataType::Float64), - ], - 0, - vec![vec![0, 1]], - ), - } - } - - #[test] - fn query_expr_simple_aggregate() { - let expr = QueryExpr::Aggregate { - by: vec![1], // service - aggs: vec![AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.01), - }], - having: None, - child: Box::new(QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, - child: Box::new(ts_scan()), - }), - }; - let schema = expr.output_schema().unwrap(); - // Output: [service, quantile_0_99] - assert_eq!(schema.columns.len(), 2); - assert_eq!(schema.columns[0].name, "service"); - assert_eq!(schema.columns[1].name, "quantile_0_99"); - // unique_keys = [by] — the by columns project to positions [0..by.len()) - // in the output schema. - assert_eq!(schema.unique_keys, vec![vec![0]]); - // Aggregate strips the time axis. - assert!(schema.time_index.is_none()); - } - - #[test] - fn query_expr_let_binding_ref() { - // LetBinding{name="w", expr=Window over Scan, - // child=Aggregate{ child=Ref{"w"} }} - let expr = QueryExpr::LetBinding { - name: BindingName::new("w"), - expr: Box::new(QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, - child: Box::new(ts_scan()), - }), - child: Box::new(QueryExpr::Aggregate { - by: vec![1], - aggs: vec![AggIntent::Max { col: None }], - having: None, - child: Box::new(QueryExpr::Ref { - name: BindingName::new("w"), - }), - }), - }; - let schema = expr.output_schema().unwrap(); - assert_eq!(schema.columns[0].name, "service"); - assert_eq!(schema.columns[1].name, "max"); - assert_eq!(schema.unique_keys, vec![vec![0]]); - } - - #[test] - fn query_expr_unresolved_ref_errors() { - let expr = QueryExpr::Ref { - name: BindingName::new("nope"), - }; - let err = expr.output_schema().unwrap_err(); - assert!(matches!(err, QueryExprError::UnresolvedRef(s) if s == "nope")); - } - - #[test] - fn query_expr_window_requires_time_index() { - let bad_scan = QueryExpr::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - label_filters: vec![], - // Tabular scan with no time index. - schema: Schema::new(vec![col("a", DataType::Int64)]), - }; - let expr = QueryExpr::Window { - kind: WindowKind::Tumbling, - size: Duration::from_secs(60), - slide: None, - child: Box::new(bad_scan), - }; - let err = expr.output_schema().unwrap_err(); - assert!(matches!(err, QueryExprError::WindowMissingTimeIndex)); - } - - #[test] - fn query_expr_aggregate_invalid_by_column() { - let expr = QueryExpr::Aggregate { - by: vec![99], - aggs: vec![AggIntent::Sum { col: None }], - having: None, - child: Box::new(ts_scan()), - }; - let err = expr.output_schema().unwrap_err(); - assert!(matches!(err, QueryExprError::InvalidGroupByColumn(99, _))); - } - - #[test] - fn query_expr_serde_roundtrip() { - let expr = QueryExpr::Aggregate { - by: vec![1], - aggs: vec![AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.01), - }], - having: None, - child: Box::new(ts_scan()), - }; - let json = serde_json::to_string(&expr).unwrap(); - let back: QueryExpr = serde_json::from_str(&json).unwrap(); - assert_eq!(expr, back); - } - - /// `unique_keys` is the load-bearing CSE-legality hook (design.md §6 - /// line ~1284). Two `Ref` consumers can share a producer iff the - /// producer's output schema has at least one provable unique-key set; - /// without it, the deduper has to be conservative and reuse drops on - /// the floor. - #[test] - fn cse_substitution_legal_only_with_unique_keys() { - // Producer 1: Scan → Window → Aggregate. Aggregate produces - // `unique_keys = [by]` (a provable unique key). Two `Ref` - // consumers can legally share this. - let producer_with_uk = QueryExpr::Aggregate { - by: vec![1], - aggs: vec![AggIntent::Sum { col: None }], - having: None, - child: Box::new(QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, - child: Box::new(ts_scan()), - }), - }; - let s1 = producer_with_uk.output_schema().unwrap(); - assert!( - s1.has_unique_key(), - "Aggregate must emit unique_keys = [by] per design.md §6 schema-flow" - ); - - // Producer 2: bare Scan with NO unique key declared. CSE deduper - // would have to refuse to share this without further proof. - let producer_without_uk = QueryExpr::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - label_filters: vec![], - schema: Schema::new(vec![col("a", DataType::Int64)]), - }; - let s2 = producer_without_uk.output_schema().unwrap(); - assert!( - !s2.has_unique_key(), - "no unique_keys → CSE deduper must conservatively refuse to share" - ); - - // The asymmetry is the design's claim: unique_keys is what makes - // CSE substitution legal. Encoded here as a unit invariant so - // downstream rewrites of the schema-flow rules can't silently - // break it. - assert_ne!(s1.has_unique_key(), s2.has_unique_key()); - } - - // ── A-variant lift (Batch 2) ───────────────────────────────────────── - - #[test] - fn filter_passes_child_schema_through() { - let expr = QueryExpr::Filter { - pred: Predicate::Literal(LiteralValue::Bool(true)), - child: Box::new(ts_scan()), - }; - let s = expr.output_schema().unwrap(); - // Filter is row-level — schema unchanged. - assert_eq!(s.columns.len(), 3); - assert_eq!(s.columns[2].name, "value"); - assert!(s.time_index.is_some()); - } - - #[test] - fn distinct_tightens_unique_keys() { - // Scan over a tabular source with no unique keys, then DISTINCT on - // `a`. Output schema should now have unique_keys=[[0]]. - let scan = QueryExpr::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - label_filters: vec![], - schema: Schema::new(vec![col("a", DataType::Int64), col("b", DataType::Utf8)]), - }; - let expr = QueryExpr::Distinct { - cols: vec![ColumnRef::Named("a".into())], - child: Box::new(scan), - }; - let s = expr.output_schema().unwrap(); - assert!(s.has_unique_key()); - assert_eq!(s.unique_keys, vec![vec![0]]); - } - - #[test] - fn merge_uses_first_child_schema_or_errors_empty() { - let scan = ts_scan(); - let expr = QueryExpr::Merge { - children: vec![scan.clone(), scan.clone()], - }; - let s = expr.output_schema().unwrap(); - assert_eq!(s.columns.len(), 3); - - let empty = QueryExpr::Merge { children: vec![] }; - let err = empty.output_schema().unwrap_err(); - assert!(matches!(err, QueryExprError::EmptyMerge)); - } - - #[test] - fn limit_and_sort_pass_schema_through() { - let expr = QueryExpr::Limit { - n: 10, - offset: 0, - child: Box::new(QueryExpr::Sort { - keys: vec![SortKey { - col: "ts".into(), - desc: false, - nulls_first: None, - }], - child: Box::new(ts_scan()), - }), - }; - let s = expr.output_schema().unwrap(); - assert_eq!(s.columns.len(), 3); - } - - #[test] - fn binary_op_uses_lhs_schema() { - let expr = QueryExpr::BinaryOp { - op: BinaryOpKind::Add, - lhs: Box::new(ts_scan()), - rhs: Box::new(ts_scan()), - vector_match: None, - }; - let s = expr.output_schema().unwrap(); - assert_eq!(s.columns.len(), 3); - } - - #[test] - fn join_uses_left_child_schema() { - let expr = QueryExpr::Join { - kind: JoinKind::Inner, - pred: Predicate::Literal(LiteralValue::Bool(true)), - left: Box::new(ts_scan()), - right: Box::new(ts_scan()), - }; - let s = expr.output_schema().unwrap(); - assert_eq!(s.columns.len(), 3); - } - - #[test] - fn a_variant_serde_roundtrip_filter() { - let expr = QueryExpr::Filter { - pred: Predicate::BinaryOp { - op: BinaryOpKind::Eq, - lhs: Box::new(Predicate::Column(ColumnRef::Named("service".into()))), - rhs: Box::new(Predicate::Literal(LiteralValue::Str("api".into()))), - }, - child: Box::new(ts_scan()), - }; - let json = serde_json::to_string(&expr).unwrap(); - let back: QueryExpr = serde_json::from_str(&json).unwrap(); - assert_eq!(expr, back); - } - - // ── from_legacy_scalar → Predicate translation ─────────────────────── - - #[test] - fn from_legacy_scalar_column() { - use crate::intent_algebra::relational as l; - let s = l::ScalarExpr::Column("foo".into()); - let p = from_legacy_scalar(&s).unwrap(); - assert!(matches!( - p, - Predicate::Column(ColumnRef::Named(ref n)) if n == "foo" - )); - } - - #[test] - fn from_legacy_scalar_literal_bool() { - use crate::intent_algebra::relational as l; - let s = l::ScalarExpr::Literal(l::LiteralValue::Bool(true)); - let p = from_legacy_scalar(&s).unwrap(); - assert!(matches!(p, Predicate::Literal(LiteralValue::Bool(true)))); - } - - #[test] - fn from_legacy_scalar_binary_op_and_is_null() { - use crate::intent_algebra::relational as l; - let s = l::ScalarExpr::BinaryOp { - op: l::BinaryOpKind::Eq, - lhs: Box::new(l::ScalarExpr::Column("a".into())), - rhs: Box::new(l::ScalarExpr::Literal(l::LiteralValue::Int(1))), - }; - let p = from_legacy_scalar(&s).unwrap(); - assert!(matches!( - p, - Predicate::BinaryOp { - op: BinaryOpKind::Eq, - .. - } - )); - - let s2 = l::ScalarExpr::IsNull { - expr: Box::new(l::ScalarExpr::Column("c".into())), - negated: true, - }; - let p2 = from_legacy_scalar(&s2).unwrap(); - assert!(matches!(p2, Predicate::IsNull { negated: true, .. })); - } - - #[test] - fn from_legacy_scalar_e_variants_translate() { - use crate::intent_algebra::relational as l; - - let f = l::ScalarExpr::FunctionCall { - name: "abs".into(), - args: vec![l::ScalarExpr::Column("x".into())], - }; - match from_legacy_scalar(&f).unwrap() { - Predicate::FunctionCall { name, args } => { - assert_eq!(name, "abs"); - assert_eq!(args.len(), 1); - assert!(matches!(&args[0], Predicate::Column(ColumnRef::Named(n)) if n == "x")); - } - other => panic!("expected FunctionCall, got {other:?}"), - } - - let il = l::ScalarExpr::InList { - expr: Box::new(l::ScalarExpr::Column("x".into())), - list: vec![l::ScalarExpr::Literal(l::LiteralValue::Int(1))], - negated: false, - }; - match from_legacy_scalar(&il).unwrap() { - Predicate::InList { list, negated, .. } => { - assert_eq!(list.len(), 1); - assert!(!negated); - } - other => panic!("expected InList, got {other:?}"), - } - - let bt = l::ScalarExpr::Between { - expr: Box::new(l::ScalarExpr::Column("x".into())), - low: Box::new(l::ScalarExpr::Literal(l::LiteralValue::Int(0))), - high: Box::new(l::ScalarExpr::Literal(l::LiteralValue::Int(10))), - negated: true, - }; - assert!(matches!( - from_legacy_scalar(&bt).unwrap(), - Predicate::Between { negated: true, .. } - )); - } - - #[test] - fn from_legacy_scalar_subquery_still_deferred() { - use crate::intent_algebra::relational as l; - // `ScalarSubquery` carries a legacy `QueryExpr` sub-tree — needs the - // legacy→canonical tree converter, so it still errors for now. - let sq = l::ScalarExpr::ScalarSubquery(Box::new(l::QueryExpr::Ref("cte".into()))); - assert!(matches!( - from_legacy_scalar(&sq).unwrap_err(), - QueryExprError::UnsupportedLegacyScalar("ScalarSubquery") - )); +/// Convert a name-based `LabelFilter` into a positional `Predicate` +/// against `schema`. `None` if `label` isn't in `schema` (the caller's +/// binder pass is expected to have already added every referenced label +/// to the schema; this is a defensive fallback, not the primary path). +pub fn label_filter_to_predicate(lf: &LabelFilter, schema: &Schema) -> Option { + let id = schema.column_id(&lf.label)?; + Some(Predicate(L3Expr::Compare { + left: Box::new(L3Expr::Column(id)), + op: CompareOp::Eq, + right: Box::new(L3Expr::Literal(L3Scalar::Utf8(lf.equals.clone()))), + })) +} + +/// Conjoin `predicates` into a single `Predicate` (`BoolAnd`), or `None` +/// if the list is empty. `asap_ir`'s `Scan.predicates` is a `Vec`, not a +/// single tree, so most callers won't need this — provided for the few +/// call sites that want one combined predicate (e.g. `Filter.pred`). +pub fn conjoin(predicates: Vec) -> Option { + let mut exprs: Vec = predicates.into_iter().map(|p| p.0).collect(); + match exprs.len() { + 0 => None, + 1 => Some(Predicate(exprs.remove(0))), + _ => Some(Predicate(L3Expr::BoolAnd(exprs))), + } +} + +/// `expr BETWEEN low AND high` (`NOT BETWEEN` when `negated`), desugared +/// to `Compare(expr >= low) AND Compare(expr <= high)` (De Morgan's for +/// the negated form). No `Expr` variant models `BETWEEN` directly — +/// this is the one real construction site's replacement (`lower.rs`). +pub fn between(expr: L3Expr, low: L3Expr, high: L3Expr, negated: bool) -> L3Expr { + let ge = L3Expr::Compare { + left: Box::new(expr.clone()), + op: CompareOp::Ge, + right: Box::new(low), + }; + let le = L3Expr::Compare { + left: Box::new(expr), + op: CompareOp::Le, + right: Box::new(high), + }; + if negated { + L3Expr::Not(Box::new(L3Expr::BoolAnd(vec![ge, le]))) + } else { + L3Expr::BoolAnd(vec![ge, le]) } } diff --git a/control_plane/src/intent_algebra/relational.rs b/control_plane/src/intent_algebra/relational.rs index bc776ba8..5688f8ab 100644 --- a/control_plane/src/intent_algebra/relational.rs +++ b/control_plane/src/intent_algebra/relational.rs @@ -27,6 +27,7 @@ use std::time::Duration; +use crate::intent_algebra::query_expr::{ArithOp, CompareOp}; use crate::types_v2::AccuracyTarget; // ── AggIntent harmonization ────────────────────────────────────────────────── @@ -653,14 +654,14 @@ fn scalar_from_predicate(p: &Predicate) -> ScalarExpr { FilterVal::Null => ScalarExpr::Literal(LiteralValue::Null), }; match &p.op { - FilterOp::Eq => bin(BinaryOpKind::Eq, col, val), - FilterOp::Ne => bin(BinaryOpKind::Ne, col, val), - FilterOp::Lt => bin(BinaryOpKind::Lt, col, val), - FilterOp::Le => bin(BinaryOpKind::Le, col, val), - FilterOp::Gt => bin(BinaryOpKind::Gt, col, val), - FilterOp::Ge => bin(BinaryOpKind::Ge, col, val), - FilterOp::Like => bin(BinaryOpKind::Like, col, val), - FilterOp::NotLike => bin(BinaryOpKind::NotLike, col, val), + FilterOp::Eq => bin(BinaryOpKind::Compare(CompareOp::Eq), col, val), + FilterOp::Ne => bin(BinaryOpKind::Compare(CompareOp::Ne), col, val), + FilterOp::Lt => bin(BinaryOpKind::Compare(CompareOp::Lt), col, val), + FilterOp::Le => bin(BinaryOpKind::Compare(CompareOp::Le), col, val), + FilterOp::Gt => bin(BinaryOpKind::Compare(CompareOp::Gt), col, val), + FilterOp::Ge => bin(BinaryOpKind::Compare(CompareOp::Ge), col, val), + FilterOp::Like => bin(BinaryOpKind::Compare(CompareOp::Like), col, val), + FilterOp::NotLike => bin(BinaryOpKind::Compare(CompareOp::NotLike), col, val), FilterOp::IsNull => ScalarExpr::IsNull { expr: Box::new(col), negated: false, @@ -670,12 +671,12 @@ fn scalar_from_predicate(p: &Predicate) -> ScalarExpr { negated: true, }, FilterOp::Regex(r) => bin( - BinaryOpKind::Regex, + BinaryOpKind::Compare(CompareOp::Regex), col, ScalarExpr::Literal(LiteralValue::Str(r.clone())), ), FilterOp::NotRegex(r) => bin( - BinaryOpKind::NotRegex, + BinaryOpKind::Compare(CompareOp::NotRegex), col, ScalarExpr::Literal(LiteralValue::Str(r.clone())), ), @@ -834,10 +835,10 @@ mod tests { #[test] fn binary_op_kind_display() { - assert_eq!(BinaryOpKind::Add.to_string(), "+"); + assert_eq!(BinaryOpKind::Arith(ArithOp::Add).to_string(), "+"); assert_eq!(BinaryOpKind::And.to_string(), "AND"); - assert_eq!(BinaryOpKind::Regex.to_string(), "=~"); - assert_eq!(BinaryOpKind::NotRegex.to_string(), "!~"); + assert_eq!(BinaryOpKind::Compare(CompareOp::Regex).to_string(), "=~"); + assert_eq!(BinaryOpKind::Compare(CompareOp::NotRegex).to_string(), "!~"); assert_eq!(BinaryOpKind::Unless.to_string(), "unless"); } diff --git a/control_plane/src/optimizer/cost/mod.rs b/control_plane/src/optimizer/cost/mod.rs index ec399451..20f070f3 100644 --- a/control_plane/src/optimizer/cost/mod.rs +++ b/control_plane/src/optimizer/cost/mod.rs @@ -435,7 +435,12 @@ pub fn workload_cost(plan: &WorkloadCostPlan<'_>) -> Result { @@ -618,7 +622,6 @@ fn walk_children_zero_cost_standalone( match expr { QueryExpr::Filter { child, .. } | QueryExpr::Project { child, .. } - | QueryExpr::Partition { child, .. } | QueryExpr::Distinct { child, .. } | QueryExpr::Sort { child, .. } | QueryExpr::Limit { child, .. } => { @@ -750,23 +753,27 @@ mod workload_cost_tests { } fn ts_scan() -> QueryExpr { + let schema = Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("service", DataType::Utf8), + col("value", DataType::Float64), + ], + 0, + vec![vec![0, 1]], + ); + let lf = LabelFilter { + label: "service".into(), + equals: "api".into(), + }; + let pred = crate::intent_algebra::label_filter_to_predicate(&lf, &schema) + .expect("service column present in schema"); QueryExpr::Scan { source: Source::TimeSeries { metric: "http_request_duration_seconds".into(), }, - label_filters: vec![LabelFilter { - label: "service".into(), - equals: "api".into(), - }], - schema: Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("service", DataType::Utf8), - col("value", DataType::Float64), - ], - 0, - vec![vec![0, 1]], - ), + predicates: vec![pred], + schema, } } @@ -782,12 +789,13 @@ mod workload_cost_tests { /// Wrap `child` in `Aggregate { by: [], aggs: [Quantile{q}] }`. fn quantile_root(q: f64, child: QueryExpr) -> QueryExpr { QueryExpr::Aggregate { - by: vec![], + by: vec![].into(), aggs: vec![AggIntent::Quantile { col: None, q, accuracy: AccuracyTarget::Epsilon(0.01), }], + output_names: Vec::new(), having: None, child: Box::new(child), } @@ -796,8 +804,9 @@ mod workload_cost_tests { /// Wrap `child` in `Aggregate { by: [], aggs: [Max] }`. fn max_root(child: QueryExpr) -> QueryExpr { QueryExpr::Aggregate { - by: vec![], + by: vec![].into(), aggs: vec![AggIntent::Max { col: None }], + output_names: Vec::new(), having: None, child: Box::new(child), } @@ -856,13 +865,13 @@ mod workload_cost_tests { let q1 = quantile_root( 0.99, QueryExpr::Ref { - name: BindingName::new("w"), + name: asap_ir::intent_algebra::BindingName::new("w"), }, ); let q2 = quantile_root( 0.95, QueryExpr::Ref { - name: BindingName::new("w"), + name: asap_ir::intent_algebra::BindingName::new("w"), }, ); @@ -905,13 +914,13 @@ mod workload_cost_tests { let q1 = quantile_root( 0.99, QueryExpr::Ref { - name: BindingName::new("w"), + name: asap_ir::intent_algebra::BindingName::new("w"), }, ); let q2 = quantile_root( 0.95, QueryExpr::Ref { - name: BindingName::new("w"), + name: asap_ir::intent_algebra::BindingName::new("w"), }, ); // q3 builds its own scan + window — no shared producer. @@ -948,17 +957,17 @@ mod workload_cost_tests { let q1 = quantile_root( 0.99, QueryExpr::Ref { - name: BindingName::new("w"), + name: asap_ir::intent_algebra::BindingName::new("w"), }, ); let q2 = quantile_root( 0.95, QueryExpr::Ref { - name: BindingName::new("w"), + name: asap_ir::intent_algebra::BindingName::new("w"), }, ); let q3 = max_root(QueryExpr::Ref { - name: BindingName::new("w"), + name: asap_ir::intent_algebra::BindingName::new("w"), }); let plan = WorkloadCostPlan { bindings: vec![(BindingName::new("w"), &shared)], @@ -1006,7 +1015,7 @@ mod workload_cost_tests { let q = quantile_root( 0.99, QueryExpr::Ref { - name: BindingName::new("missing"), + name: asap_ir::intent_algebra::BindingName::new("missing"), }, ); let plan = WorkloadCostPlan { diff --git a/control_plane/src/optimizer/engine.rs b/control_plane/src/optimizer/engine.rs index 603f651a..3ebbb9cc 100644 --- a/control_plane/src/optimizer/engine.rs +++ b/control_plane/src/optimizer/engine.rs @@ -9,17 +9,17 @@ //! //! | Rule | Name | Description | //! |------|------|-------------| -//! | R1 | `PredicatePushDown` | Push `Filter` below `Window`, `Partition`, `Sort` | +//! | R1 | `PredicatePushDown` | Push `Filter` below `Window`, `Sort` | //! | R2 | `MergeLifting` | Lift a mergeable single-intent `Aggregate` above `Merge` | //! | R3 | `HLLDedupElim` | Eliminate `Distinct` before a cardinality `Aggregate` | //! | R4 | `FilterWindowSwap` | Swap `Filter` below `Window` to reduce window input size | //! | R5 | `TopKFusion` | Fuse `Limit(Sort DESC)` into a `TopK`-intent `Aggregate` | //! | R6 | _(retired)_ | `HistogramQuantileFusion` retired in Step γ5 | -//! | R7 | `SubqueryDecorrelation` | Hoist a `ScalarSubquery` predicate to a `LetBinding` | +//! | R7 | _(retired)_ | `SubqueryDecorrelation` retired in the `asap_ir` merge — see `intent_algebra::lower` module docs | //! | R8 | `CommonSubexprElim` | Extract identical `Scan` sub-trees into `LetBinding`s | //! | R9 | `HydraConversion` | (disabled — Step γ TODO; see struct docs) | //! | R10| `WindowMerge` | Merge adjacent identical `Window` nodes | -//! | R11| `PartitionElim` | Remove `Partition` with empty key list | +//! | R11| _(retired)_ | `PartitionElim` retired — `Partition` no longer exists in the canonical IR; its keys fold into `Aggregate.by` at construction time | //! | R12| `SetOpFusion` | Fuse `SetOp(Union, Merge, Merge)` into a single `Merge` | //! //! Step γ7: the optimizer consumes and produces the canonical @@ -30,13 +30,15 @@ use std::collections::HashMap; +use asap_ir::intent_algebra::BindingName; + use crate::intent_algebra::agg_intent::AggIntent; -use crate::intent_algebra::query_expr::{ColumnRef, Predicate, QueryExpr, SetOpKind, Source}; +use crate::intent_algebra::query_expr::{GroupKeys, QueryExpr, SetOpKind, Source}; use crate::intent_algebra::relational::{agg_is_exact, agg_is_mergeable}; use crate::optimizer::cost::sketch_capability::{ default_capability_table, load_capability_overrides, SketchCapability, }; -use crate::types_v2::{AccuracyTarget, BindingName}; +use crate::types_v2::AccuracyTarget; // ── Cost model interface ────────────────────────────────────────────────────── @@ -223,7 +225,6 @@ impl CostModel for DefaultCostModel { } QueryExpr::Merge { children } => 1.0 / (children.len().max(1) as f64), QueryExpr::Filter { .. } => 0.5, - QueryExpr::Partition { .. } => 0.8, // partition adds overhead QueryExpr::Distinct { .. } => 0.9, _ => 1.0, }; @@ -262,9 +263,7 @@ impl CostModel for DefaultCostModel { } // Multi-intent / HAVING aggregate → exact original DB. QueryExpr::Aggregate { .. } => &dc.original_db, - QueryExpr::Partition { .. } - | QueryExpr::Merge { .. } - | QueryExpr::Distinct { .. } => &dc.backend_collector, + QueryExpr::Merge { .. } | QueryExpr::Distinct { .. } => &dc.backend_collector, QueryExpr::BinaryOp { .. } | QueryExpr::Subquery { .. } => &dc.backend_db, _ => &dc.agent, }; @@ -330,8 +329,12 @@ pub trait RewriteRule: Send + Sync { /// Push `Filter` nodes as deep as possible — reduces data volume early. /// /// * `Filter(p, Window(…, e))` → `Window(…, Filter(p, e))` -/// * `Filter(p, Partition(k, e))` → `Partition(k, Filter(p, e))` /// * `Filter(p, Sort(k, e))` → `Sort(k, Filter(p, e))` +/// +/// No `Partition` case: folded into `Aggregate.by` at construction time +/// (`intent_algebra::lower`), so `Filter` never wraps a bare grouping +/// marker anymore — pushing a `Filter` below an `Aggregate` would change +/// which rows get aggregated, so that's not a safe pushdown. pub struct PredicatePushDown; impl RewriteRule for PredicatePushDown { @@ -354,14 +357,14 @@ impl RewriteRule for PredicatePushDown { slide, child: Box::new(QueryExpr::Filter { pred, child: inner }), }), - // Filter below Partition - QueryExpr::Partition { keys, child: inner } => Some(QueryExpr::Partition { - keys, - child: Box::new(QueryExpr::Filter { pred, child: inner }), - }), // Filter below Sort (safe when pred references input columns only) - QueryExpr::Sort { keys, child: inner } => Some(QueryExpr::Sort { + QueryExpr::Sort { keys, + partition_by, + child: inner, + } => Some(QueryExpr::Sort { + keys, + partition_by, child: Box::new(QueryExpr::Filter { pred, child: inner }), }), // Not applicable — reconstruct @@ -397,6 +400,7 @@ impl RewriteRule for MergeLifting { ref aggs, ref having, ref child, + .. } if aggs.len() == 1 && having.is_none() && agg_is_mergeable(&aggs[0]) => { if let QueryExpr::Merge { children } = child.as_ref() { let new_children: Vec = children @@ -404,6 +408,7 @@ impl RewriteRule for MergeLifting { .map(|branch| QueryExpr::Aggregate { by: by.clone(), aggs: aggs.clone(), + output_names: Vec::new(), having: None, child: Box::new(branch.clone()), }) @@ -437,6 +442,7 @@ impl RewriteRule for HLLDedupElim { QueryExpr::Aggregate { by, aggs, + output_names, having: None, child, } if aggs.len() == 1 && matches!(&aggs[0], AggIntent::Cardinality { .. }) => { @@ -444,6 +450,7 @@ impl RewriteRule for HLLDedupElim { return Some(QueryExpr::Aggregate { by, aggs, + output_names, having: None, child: inner, }); @@ -509,15 +516,19 @@ impl RewriteRule for TopKFusion { offset: 0, child, } => { - if let QueryExpr::Sort { keys, child: inner } = *child { + if let QueryExpr::Sort { + keys, child: inner, .. + } = *child + { // Only fuse when all keys are DESC (top-k semantics). - if !keys.is_empty() && keys.iter().all(|k| k.desc) { + if !keys.is_empty() && keys.iter().all(|k| !k.ascending) { return Some(QueryExpr::Aggregate { - by: vec![], + by: GroupKeys::none(), aggs: vec![AggIntent::TopK { k: n, accuracy: AccuracyTarget::Epsilon(0.05), }], + output_names: Vec::new(), having: None, child: inner, }); @@ -537,70 +548,13 @@ impl RewriteRule for TopKFusion { // `Aggregate{Quantile(φ)}`. The fusion rule that used to merge a // `HistogramQuantile` wrapper with an inner DDSketch is no longer needed. -// ── R7: SubqueryDecorrelation ───────────────────────────────────────────────── - -/// Hoist a `ScalarSubquery` predicate into a `LetBinding` so the subquery -/// is evaluated once rather than once per row. -/// -/// Handles the simple case: a `Filter` whose `Predicate::BinaryOp` has a -/// `ScalarSubquery` on one side. -pub struct SubqueryDecorrelation; - -impl RewriteRule for SubqueryDecorrelation { - fn name(&self) -> &'static str { - "SubqueryDecorrelation" - } - - fn try_rewrite(&self, expr: QueryExpr, _model: &dyn CostModel) -> Option { - match expr { - QueryExpr::Filter { pred, child } => { - if let Some((name, sq_expr, new_pred)) = extract_scalar_subquery(pred) { - return Some(QueryExpr::LetBinding { - name: BindingName::new(name), - expr: Box::new(sq_expr), - child: Box::new(QueryExpr::Filter { - pred: new_pred, - child, - }), - }); - } - None - } - _ => None, - } - } -} - -/// If `pred` contains a `ScalarSubquery`, extract it as -/// `(binding_name, subquery_expr, pred_with_ref)`. -fn extract_scalar_subquery(pred: Predicate) -> Option<(String, QueryExpr, Predicate)> { - match pred { - Predicate::BinaryOp { op, lhs, rhs } => { - // Check lhs - if let Predicate::ScalarSubquery(sq) = *lhs { - let name = "__subq_0".to_string(); - let new_pred = Predicate::BinaryOp { - op, - lhs: Box::new(Predicate::Column(ColumnRef::Named(name.clone()))), - rhs, - }; - return Some((name, *sq, new_pred)); - } - // Check rhs - if let Predicate::ScalarSubquery(sq) = *rhs { - let name = "__subq_0".to_string(); - let new_pred = Predicate::BinaryOp { - op, - lhs, - rhs: Box::new(Predicate::Column(ColumnRef::Named(name.clone()))), - }; - return Some((name, *sq, new_pred)); - } - None - } - _ => None, - } -} +// R7 (retired): `SubqueryDecorrelation` used to hoist a `ScalarSubquery` +// predicate into a `LetBinding`, pattern-matching the old `Predicate` +// enum's `BinaryOp` / `ScalarSubquery` / `Column(ColumnRef::Named(_))` +// variants directly. The canonical `Predicate(L3Expr)` merge (see +// `intent_algebra::lower` module docs) rejects `ScalarSubquery` at +// construction time instead of lowering it, so this rule has no +// construction site left to fire against — deleted rather than ported. // ── R8: CommonSubexprElim ───────────────────────────────────────────────────── @@ -739,26 +693,11 @@ impl RewriteRule for WindowMerge { } } -// ── R11: PartitionElim ──────────────────────────────────────────────────────── - -/// Remove `Partition` with an empty key list — equivalent to a global -/// aggregation with no GROUP BY. -/// -/// `Partition([], e)` → `e` -pub struct PartitionElim; - -impl RewriteRule for PartitionElim { - fn name(&self) -> &'static str { - "PartitionElim" - } - - fn try_rewrite(&self, expr: QueryExpr, _model: &dyn CostModel) -> Option { - match expr { - QueryExpr::Partition { keys, child } if keys.is_empty() => Some(*child), - _ => None, - } - } -} +// R11 (retired): `PartitionElim` removed a `Partition` node with an empty +// key list. `Partition` no longer exists in the canonical IR — its keys +// fold into `Aggregate.by: GroupKeys` at construction time +// (`intent_algebra::lower`), so an empty-keys `Partition` never gets +// built in the first place; nothing left for this rule to eliminate. // ── R12: SetOpFusion ────────────────────────────────────────────────────────── @@ -906,11 +845,16 @@ impl QueryOptimizer { c, ) } - QueryExpr::Project { cols, child } => { + QueryExpr::Project { + cols, + qualifier, + child, + } => { let (new_child, c) = recurse!(child); ( QueryExpr::Project { cols, + qualifier, child: new_child, }, c, @@ -919,6 +863,7 @@ impl QueryOptimizer { QueryExpr::Aggregate { by, aggs, + output_names, having, child, } => { @@ -927,6 +872,7 @@ impl QueryOptimizer { QueryExpr::Aggregate { by, aggs, + output_names, having, child: new_child, }, @@ -950,16 +896,6 @@ impl QueryOptimizer { c, ) } - QueryExpr::Partition { keys, child } => { - let (new_child, c) = recurse!(child); - ( - QueryExpr::Partition { - keys, - child: new_child, - }, - c, - ) - } QueryExpr::Distinct { cols, child } => { let (new_child, c) = recurse!(child); ( @@ -970,11 +906,16 @@ impl QueryOptimizer { c, ) } - QueryExpr::Sort { keys, child } => { + QueryExpr::Sort { + keys, + partition_by, + child, + } => { let (new_child, c) = recurse!(child); ( QueryExpr::Sort { keys, + partition_by, child: new_child, }, c, @@ -1082,6 +1023,18 @@ impl QueryOptimizer { ce || cb, ) } + // `asap_ir`'s `QueryExpr` superset (Scalar / EvalTime / + // VectorFromScalar / ScalarFromVector / Relabel / InfoJoin / + // Sample / TimeRange / TimeShift / WindowFunc) — PromQL + // surface no rule here targets yet. Treated as opaque leaves: + // returned unchanged rather than recursed into, so a rewrite + // opportunity nested inside one of these is missed rather + // than mishandled. Not a correctness issue (the optimizer is + // a pure best-effort rewrite pass — leaving a subtree + // unrewritten is always semantically safe), just a + // known coverage gap to close when a rule needs to see + // inside them. + other => (other, false), } } } @@ -1093,13 +1046,13 @@ fn default_rules() -> Vec> { Box::new(FilterWindowSwap), Box::new(HLLDedupElim), Box::new(WindowMerge), - Box::new(PartitionElim), Box::new(TopKFusion), // R6 (HistogramQuantileFusion) retired in Step γ5. Box::new(MergeLifting), Box::new(SetOpFusion), Box::new(HydraConversion), - Box::new(SubqueryDecorrelation), + // R7 (SubqueryDecorrelation) and R11 (PartitionElim) retired — see + // the doc-table notes above. Box::new(CommonSubexprElim), ] } @@ -1114,13 +1067,13 @@ pub fn default_rules_as_optimizer_rules() -> Vec &'static str { - ::name(self) - } - fn category(&self) -> RuleCategory { - RuleCategory::Elim - } -} impl OptimizerRule for TopKFusion { fn name(&self) -> &'static str { ::name(self) @@ -1206,14 +1151,6 @@ impl OptimizerRule for HydraConversion { RuleCategory::Fusion } } -impl OptimizerRule for SubqueryDecorrelation { - fn name(&self) -> &'static str { - ::name(self) - } - fn category(&self) -> RuleCategory { - RuleCategory::Decorrelate - } -} impl OptimizerRule for CommonSubexprElim { fn name(&self) -> &'static str { ::name(self) @@ -1228,9 +1165,10 @@ impl OptimizerRule for CommonSubexprElim { #[cfg(test)] mod tests { use super::*; + use crate::intent_algebra::query_expr::Predicate; use crate::intent_algebra::relational::{default_cardinality, default_quantile}; use crate::intent_algebra::{ - BinaryOpKind, LiteralValue, PartitionKeys, Schema, SortKey, Source, WindowKind, + BinaryOpKind, L3Expr, L3Scalar, Schema, SortKey, Source, WindowKind, }; use std::time::Duration; @@ -1240,7 +1178,7 @@ mod tests { source: Source::TimeSeries { metric: name.into(), }, - label_filters: vec![], + predicates: vec![], schema: Schema::default(), } } @@ -1249,15 +1187,16 @@ mod tests { /// the legacy `SketchAgg`. fn sketch_agg(intent: AggIntent, child: QueryExpr) -> QueryExpr { QueryExpr::Aggregate { - by: vec![], + by: GroupKeys::none(), aggs: vec![intent], + output_names: Vec::new(), having: None, child: Box::new(child), } } fn true_pred() -> Predicate { - Predicate::Literal(LiteralValue::Bool(true)) + Predicate(L3Expr::Literal(L3Scalar::Boolean(true))) } fn opt() -> QueryOptimizer { @@ -1265,15 +1204,18 @@ mod tests { } /// Recursively check whether the tree contains any `Distinct` node. + /// Only walks the variants these tests actually build (`Scan`/`Ref`, + /// `Filter`/`Project`/`Aggregate`/`Window`/`Sort`/`Limit`/`Subquery`, + /// `Merge`, `Join`/`SetOp`/`BinaryOp`, `LetBinding`, `Distinct`) — the + /// PromQL-only leaves (`Scalar`, `EvalTime`, `Relabel`, …) never appear + /// in a tree these helpers construct. fn contains_distinct(qe: &QueryExpr) -> bool { match qe { QueryExpr::Distinct { .. } => true, - QueryExpr::Scan { .. } | QueryExpr::Ref { .. } => false, QueryExpr::Filter { child, .. } | QueryExpr::Project { child, .. } | QueryExpr::Aggregate { child, .. } | QueryExpr::Window { child, .. } - | QueryExpr::Partition { child, .. } | QueryExpr::Sort { child, .. } | QueryExpr::Limit { child, .. } | QueryExpr::Subquery { child, .. } => contains_distinct(child), @@ -1288,6 +1230,7 @@ mod tests { QueryExpr::LetBinding { expr, child, .. } => { contains_distinct(expr) || contains_distinct(child) } + _ => false, } } @@ -1313,16 +1256,21 @@ mod tests { } #[test] - fn r1_pushes_filter_below_partition() { + fn r1_pushes_filter_below_sort() { let expr = QueryExpr::Filter { pred: true_pred(), - child: Box::new(QueryExpr::Partition { - keys: PartitionKeys::By(vec!["host".into()]), + child: Box::new(QueryExpr::Sort { + keys: vec![SortKey { + expr: L3Expr::Column(0), + ascending: true, + nulls_first: false, + }], + partition_by: GroupKeys::none(), child: Box::new(scan("cpu")), }), }; let (result, _) = opt().optimize(expr); - assert!(matches!(&result, QueryExpr::Partition { child, .. } + assert!(matches!(&result, QueryExpr::Sort { child, .. } if matches!(child.as_ref(), QueryExpr::Filter { .. }))); } @@ -1333,7 +1281,7 @@ mod tests { let expr = sketch_agg( default_cardinality(), QueryExpr::Distinct { - cols: vec![ColumnRef::Named("user_id".into())], + cols: vec![0], child: Box::new(scan("events")), }, ); @@ -1353,10 +1301,11 @@ mod tests { offset: 0, child: Box::new(QueryExpr::Sort { keys: vec![SortKey { - col: "count".into(), - desc: true, - nulls_first: None, + expr: L3Expr::Column(0), + ascending: false, + nulls_first: false, }], + partition_by: GroupKeys::none(), child: Box::new(scan("events")), }), }; @@ -1375,10 +1324,11 @@ mod tests { offset: 0, child: Box::new(QueryExpr::Sort { keys: vec![SortKey { - col: "ts".into(), - desc: false, - nulls_first: None, + expr: L3Expr::Column(0), + ascending: true, + nulls_first: false, }], + partition_by: GroupKeys::none(), child: Box::new(scan("events")), }), }; @@ -1413,21 +1363,6 @@ mod tests { ); } - // ── R11: PartitionElim ──────────────────────────────────────────────────── - - #[test] - fn r11_removes_empty_partition() { - let expr = QueryExpr::Partition { - keys: PartitionKeys::By(vec![]), - child: Box::new(scan("m")), - }; - let (result, _) = opt().optimize(expr); - assert!( - matches!(&result, QueryExpr::Scan { .. }), - "empty Partition should be eliminated: {result:?}" - ); - } - // ── Fixed-point convergence ─────────────────────────────────────────────── #[test] @@ -1451,7 +1386,7 @@ mod tests { child: Box::new(sketch_agg( default_cardinality(), QueryExpr::Distinct { - cols: vec![ColumnRef::Named("uid".into())], + cols: vec![0], child: Box::new(scan("events")), }, )), diff --git a/control_plane/src/optimizer/rules/mod.rs b/control_plane/src/optimizer/rules/mod.rs index c1c40b7c..b2cdbea6 100644 --- a/control_plane/src/optimizer/rules/mod.rs +++ b/control_plane/src/optimizer/rules/mod.rs @@ -204,14 +204,13 @@ pub fn bind_workload_typed(w: &QueryWorkload) -> Option Option { - let child = self.alloc_node(*child, budget); - PlanNode { - expr: QueryExpr::Partition { - keys, - child: Box::new(child.expr.clone()), - }, - stage: PipelineStage::Agent, - mode: ExecutionMode::Passthrough, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale: "Partition for GROUP BY at Agent".into(), - ..Default::default() - }, - children: vec![child], - } - } - QueryExpr::Distinct { cols, child } => { let child = self.alloc_node(*child, budget); PlanNode { @@ -222,6 +204,7 @@ impl SketchAllocator { QueryExpr::Aggregate { by, aggs, + output_names, having, child, } => { @@ -233,6 +216,7 @@ impl SketchAllocator { expr: QueryExpr::Aggregate { by, aggs, + output_names, having, child: Box::new(child.expr.clone()), }, @@ -255,7 +239,7 @@ impl SketchAllocator { } // Single non-TopK intent → budget-driven sketch agg. let child = self.alloc_node(*child, budget); - return self.alloc_sketch_agg(by, aggs, child, budget); + return self.alloc_sketch_agg(by, aggs, output_names, child, budget); } // General multi-intent / HAVING aggregate → Db (exact). let child = self.alloc_node(*child, budget); @@ -264,6 +248,7 @@ impl SketchAllocator { expr: QueryExpr::Aggregate { by, aggs, + output_names, having, child: Box::new(child.expr.clone()), }, @@ -308,11 +293,16 @@ impl SketchAllocator { } // ── Exact / relational — Db ─────────────────────────────────── - QueryExpr::Project { cols, child } => { + QueryExpr::Project { + cols, + qualifier, + child, + } => { let child = self.alloc_node(*child, budget); PlanNode { expr: QueryExpr::Project { cols, + qualifier, child: Box::new(child.expr.clone()), }, stage: PipelineStage::Db, @@ -326,11 +316,16 @@ impl SketchAllocator { } } - QueryExpr::Sort { keys, child } => { + QueryExpr::Sort { + keys, + partition_by, + child, + } => { let child = self.alloc_node(*child, budget); PlanNode { expr: QueryExpr::Sort { keys, + partition_by, child: Box::new(child.expr.clone()), }, stage: PipelineStage::Db, @@ -508,6 +503,101 @@ impl SketchAllocator { children: vec![expr_node, body_node], } } + + // `asap_ir`'s PromQL-surface superset (Scalar / EvalTime / + // VectorFromScalar / ScalarFromVector / Relabel / InfoJoin / + // Sample / TimeRange / TimeShift / WindowFunc) — not yet + // targeted by a dedicated allocation rule in this deployment. + // Leaves stay informational Agent leaves; every other new + // variant wraps exactly one child, recursed into and staged + // as an Agent passthrough (mirroring `Filter`/`Window` above) + // until a real rule is written for them. + QueryExpr::Scalar(_) | QueryExpr::EvalTime => { + PlanNode::leaf(expr, PipelineStage::Agent, ExecutionMode::Passthrough) + } + QueryExpr::VectorFromScalar(child) => { + self.alloc_passthrough_child(child, "VectorFromScalar", budget, |c| { + QueryExpr::VectorFromScalar(Box::new(c)) + }) + } + QueryExpr::ScalarFromVector(child) => { + self.alloc_passthrough_child(child, "ScalarFromVector", budget, |c| { + QueryExpr::ScalarFromVector(Box::new(c)) + }) + } + QueryExpr::Relabel { dst, value, child } => { + self.alloc_passthrough_child(child, "Relabel", budget, |c| QueryExpr::Relabel { + dst, + value, + child: Box::new(c), + }) + } + QueryExpr::InfoJoin { selector, child } => { + self.alloc_passthrough_child(child, "InfoJoin", budget, |c| QueryExpr::InfoJoin { + selector, + child: Box::new(c), + }) + } + QueryExpr::Sample { by, kind, child } => { + self.alloc_passthrough_child(child, "Sample", budget, |c| QueryExpr::Sample { + by, + kind, + child: Box::new(c), + }) + } + QueryExpr::TimeRange { range, child } => { + self.alloc_passthrough_child(child, "TimeRange", budget, |c| QueryExpr::TimeRange { + range, + child: Box::new(c), + }) + } + QueryExpr::TimeShift { shift, child } => { + self.alloc_passthrough_child(child, "TimeShift", budget, |c| QueryExpr::TimeShift { + shift, + child: Box::new(c), + }) + } + QueryExpr::WindowFunc { + func, + args, + partition_by, + order_by, + output_name, + child, + } => self.alloc_passthrough_child(child, "WindowFunc", budget, |c| { + QueryExpr::WindowFunc { + func, + args, + partition_by, + order_by, + output_name, + child: Box::new(c), + } + }), + } + } + + /// Shared body for the single-child PromQL-surface passthrough arms: + /// recurse into `child`, rebuild the node via `rebuild`, and stage it + /// as an informational Agent passthrough carrying the child's cost. + fn alloc_passthrough_child( + &self, + child: Box, + label: &'static str, + budget: &mut BudgetState, + rebuild: impl FnOnce(QueryExpr) -> QueryExpr, + ) -> PlanNode { + let child_node = self.alloc_node(*child, budget); + PlanNode { + expr: rebuild(child_node.expr.clone()), + stage: PipelineStage::Agent, + mode: ExecutionMode::Passthrough, + cost: child_node.cost.clone(), + annotation: NodeAnnotation { + rationale: format!("{label} at Agent (informational passthrough)"), + ..Default::default() + }, + children: vec![child_node], } } @@ -519,8 +609,9 @@ impl SketchAllocator { /// `aggs.len() == 1` and that the single intent is not `TopK`. fn alloc_sketch_agg( &self, - by: Vec, + by: GroupKeys, aggs: Vec, + output_names: Vec, child: PlanNode, budget: &mut BudgetState, ) -> PlanNode { @@ -532,6 +623,7 @@ impl SketchAllocator { expr: QueryExpr::Aggregate { by, aggs, + output_names, having: None, child: Box::new(child.expr.clone()), }, @@ -555,6 +647,7 @@ impl SketchAllocator { expr: QueryExpr::Aggregate { by, aggs, + output_names, having: None, child: Box::new(child.expr.clone()), }, @@ -584,6 +677,7 @@ impl SketchAllocator { expr: QueryExpr::Aggregate { by, aggs, + output_names, having: None, child: Box::new(child.expr.clone()), }, @@ -611,6 +705,7 @@ impl SketchAllocator { expr: QueryExpr::Aggregate { by, aggs, + output_names, having: None, child: Box::new(child.expr.clone()), }, @@ -638,6 +733,7 @@ impl SketchAllocator { expr: QueryExpr::Aggregate { by, aggs, + output_names, having: None, child: Box::new(child.expr.clone()), }, @@ -728,7 +824,7 @@ mod tests { use crate::intent_algebra::relational::{ default_cardinality, default_frequency, default_quantile, }; - use crate::intent_algebra::{JoinKind, LiteralValue, Predicate, QueryExpr, Schema, Source}; + use crate::intent_algebra::{JoinKind, L3Expr, L3Scalar, Predicate, QueryExpr, Schema, Source}; use crate::physical::plan::{ExecutionMode, PipelineStage}; use crate::types::{SketchType, StageResourceBudgets}; use crate::types_v2::AccuracyTarget; @@ -740,7 +836,7 @@ mod tests { source: Source::TimeSeries { metric: name.into(), }, - label_filters: vec![], + predicates: vec![], schema: Schema::default(), } } @@ -749,8 +845,9 @@ mod tests { /// `Scan` — the canonical shape the legacy `SketchAgg` folded into. fn agg(intent: AggIntent) -> QueryExpr { QueryExpr::Aggregate { - by: vec![], + by: vec![].into(), aggs: vec![intent], + output_names: Vec::new(), having: None, child: Box::new(scan("m")), } @@ -793,7 +890,7 @@ mod tests { #[test] fn filter_at_agent() { let expr = QueryExpr::Filter { - pred: Predicate::Literal(LiteralValue::Bool(true)), + pred: Predicate(L3Expr::Literal(L3Scalar::Boolean(true))), child: Box::new(scan("m")), }; let node = alloc(unlimited(), expr); @@ -896,7 +993,7 @@ mod tests { fn join_goes_to_db() { let expr = QueryExpr::Join { kind: JoinKind::Inner, - pred: Predicate::Literal(LiteralValue::Bool(true)), + pred: Predicate(L3Expr::Literal(L3Scalar::Boolean(true))), left: Box::new(scan("orders")), right: Box::new(scan("items")), }; @@ -909,8 +1006,9 @@ mod tests { #[test] fn multi_intent_aggregate_goes_to_db() { let expr = QueryExpr::Aggregate { - by: vec![], + by: vec![].into(), aggs: vec![AggIntent::Sum { col: None }, AggIntent::Min { col: None }], + output_names: Vec::new(), having: None, child: Box::new(scan("m")), }; @@ -944,7 +1042,7 @@ mod tests { #[test] fn let_binding_inherits_body_stage() { let expr = QueryExpr::LetBinding { - name: crate::types_v2::BindingName::new("base"), + name: asap_ir::intent_algebra::BindingName::new("base"), expr: Box::new(scan("cpu")), child: Box::new(agg(AggIntent::TopK { k: 5, diff --git a/control_plane/src/physical/colored_dag/allocator.rs b/control_plane/src/physical/colored_dag/allocator.rs index 8807bd10..abc06f6d 100644 --- a/control_plane/src/physical/colored_dag/allocator.rs +++ b/control_plane/src/physical/colored_dag/allocator.rs @@ -240,19 +240,21 @@ impl ThreeStageWalker { // No colored-DAG consumer constructs them today; conservatively // route to the Edge stage (matches the per-row Scan/Window // policy) so the build is total. The proper stage-placement - // rules for Filter/Project/Partition/Distinct/Merge/Join/SetOp/ - // Sort/Limit/BinaryOp land alongside their consumers in - // follow-up batches. - QE::Filter { .. } - | QE::Project { .. } - | QE::Partition { .. } - | QE::Distinct { .. } - | QE::Sort { .. } - | QE::Limit { .. } - | QE::Subquery { .. } => Ok(StageId::Edge), + // rules for Filter/Project/Distinct/Sort/Limit/BinaryOp (and, + // since the `asap_ir` merge, the PromQL-surface superset — + // Scalar/EvalTime/VectorFromScalar/ScalarFromVector/Relabel/ + // InfoJoin/Sample/TimeRange/TimeShift/WindowFunc, also + // unconstructed here today) land alongside their consumers in + // follow-up batches. `Partition` no longer exists in the + // canonical IR — its keys fold into `Aggregate.by` at + // construction time (`intent_algebra::lower`). QE::Merge { .. } | QE::Join { .. } | QE::SetOp { .. } | QE::BinaryOp { .. } => { Ok(StageId::Backend) } + // Filter/Project/Distinct/Sort/Limit/Subquery, plus the + // PromQL-surface superset unconstructed here today, all fall + // through to this Edge default. + _ => Ok(StageId::Edge), } } } @@ -282,7 +284,7 @@ mod tests { source: Source::TimeSeries { metric: "http_request_duration_seconds".into(), }, - label_filters: vec![], + predicates: vec![], schema: Schema::with_time_index( vec![ Column { diff --git a/control_plane/src/physical/colored_dag/dag.rs b/control_plane/src/physical/colored_dag/dag.rs index 2b946502..4f3bfed9 100644 --- a/control_plane/src/physical/colored_dag/dag.rs +++ b/control_plane/src/physical/colored_dag/dag.rs @@ -151,7 +151,7 @@ mod tests { fn dummy_logical() -> PhysicalExpr { PhysicalExpr::Logical(QueryExpr::Ref { - name: crate::types_v2::BindingName::new("dummy"), + name: asap_ir::intent_algebra::BindingName::new("dummy"), }) } diff --git a/control_plane/src/physical/colored_dag/emitter.rs b/control_plane/src/physical/colored_dag/emitter.rs index 12147e21..2b8f6213 100644 --- a/control_plane/src/physical/colored_dag/emitter.rs +++ b/control_plane/src/physical/colored_dag/emitter.rs @@ -982,18 +982,36 @@ fn extract_edge_facts(qe: &crate::intent_algebra::QueryExpr, edge: &mut EdgeStag match qe { QE::Scan { source, - label_filters, - .. + predicates, + schema, } => { if let Source::TimeSeries { metric } = source { if edge.source_metric.is_none() { edge.source_metric = Some(metric.clone()); } } - for f in label_filters { - let pair = (f.label.clone(), f.equals.clone()); - if !edge.label_filters.contains(&pair) { - edge.label_filters.push(pair); + // Canonical `Scan.predicates` carries equality label filters + // as typed `Predicate(L3Expr::Compare{Column, Eq, + // Literal(Utf8)})` trees — resolve each `Column` id back to + // its name via the Scan's own schema. + use crate::intent_algebra::{CompareOp, L3Expr, L3Scalar}; + for p in predicates { + if let L3Expr::Compare { + left, + op: CompareOp::Eq, + right, + } = &p.0 + { + if let (L3Expr::Column(id), L3Expr::Literal(L3Scalar::Utf8(v))) = + (left.as_ref(), right.as_ref()) + { + if let Some(col) = schema.columns.get(*id) { + let pair = (col.name.clone(), v.clone()); + if !edge.label_filters.contains(&pair) { + edge.label_filters.push(pair); + } + } + } } } } @@ -1017,7 +1035,6 @@ fn extract_edge_facts(qe: &crate::intent_algebra::QueryExpr, edge: &mut EdgeStag // filters, window size) from any leaves below. QE::Filter { child, .. } | QE::Project { child, .. } - | QE::Partition { child, .. } | QE::Distinct { child, .. } | QE::Sort { child, .. } | QE::Limit { child, .. } @@ -1037,6 +1054,14 @@ fn extract_edge_facts(qe: &crate::intent_algebra::QueryExpr, edge: &mut EdgeStag extract_edge_facts(left, edge); extract_edge_facts(right, edge); } + // `Partition` no longer exists in the canonical IR (its keys + // fold into `Aggregate.by` at construction time). The PromQL- + // surface superset (Scalar/EvalTime/VectorFromScalar/ + // ScalarFromVector/Relabel/InfoJoin/Sample/TimeRange/TimeShift/ + // WindowFunc) isn't constructed here today; a no-op default is + // safe since none of the single-child ones carry edge facts this + // extractor cares about. + _ => {} } } diff --git a/control_plane/src/physical/colored_dag/tests.rs b/control_plane/src/physical/colored_dag/tests.rs index 56720d67..d74c3eb9 100644 --- a/control_plane/src/physical/colored_dag/tests.rs +++ b/control_plane/src/physical/colored_dag/tests.rs @@ -21,42 +21,48 @@ use asap_sketch::{SummaryKind, SummaryParams}; // ── Test fixtures ───────────────────────────────────────────────────────────── fn ts_scan(metric: &str, label: Option<(&str, &str)>) -> QueryExpr { + let schema = Schema::with_time_index( + vec![ + Column { + name: "ts".into(), + dtype: DataType::Timestamp, + nullable: false, + table: None, + }, + Column { + name: "service".into(), + dtype: DataType::Utf8, + nullable: false, + table: None, + }, + Column { + name: "value".into(), + dtype: DataType::Float64, + nullable: false, + table: None, + }, + ], + 0, + vec![vec![0, 1]], + ); + let predicates = label + .map(|(k, v)| { + let lf = LabelFilter { + label: k.into(), + equals: v.into(), + }; + vec![ + crate::intent_algebra::label_filter_to_predicate(&lf, &schema) + .expect("label column present in schema"), + ] + }) + .unwrap_or_default(); QueryExpr::Scan { source: Source::TimeSeries { metric: metric.into(), }, - label_filters: label - .map(|(k, v)| { - vec![LabelFilter { - label: k.into(), - equals: v.into(), - }] - }) - .unwrap_or_default(), - schema: Schema::with_time_index( - vec![ - Column { - name: "ts".into(), - dtype: DataType::Timestamp, - nullable: false, - table: None, - }, - Column { - name: "service".into(), - dtype: DataType::Utf8, - nullable: false, - table: None, - }, - Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - table: None, - }, - ], - 0, - vec![vec![0, 1]], - ), + predicates, + schema, } } diff --git a/control_plane/src/physical/plan.rs b/control_plane/src/physical/plan.rs index 2ef7033b..2383b1be 100644 --- a/control_plane/src/physical/plan.rs +++ b/control_plane/src/physical/plan.rs @@ -286,7 +286,7 @@ mod tests { source: Source::TimeSeries { metric: name.into(), }, - label_filters: vec![], + predicates: vec![], schema: Schema::default(), } } diff --git a/control_plane/src/physical/planner.rs b/control_plane/src/physical/planner.rs index 8c757752..65fec222 100644 --- a/control_plane/src/physical/planner.rs +++ b/control_plane/src/physical/planner.rs @@ -23,7 +23,8 @@ use std::time::Duration; use crate::intent_algebra::agg_intent::AggIntent; -use crate::intent_algebra::query_expr::{ColumnRef, QueryExpr}; +use crate::intent_algebra::query_expr::QueryExpr; +use crate::intent_algebra::schema::ColumnId; use crate::physical::sketch_catalog; use crate::physical::window_fusion::{fused_sketch_decision, recognize_windowed_sketch}; use crate::types::{SketchParams, SketchType}; @@ -293,6 +294,7 @@ fn plan_node(expr: &QueryExpr, config: &PhysicalPlannerConfig) -> PhysicalNode { aggs, having, child, + .. } => { if aggs.len() == 1 && having.is_none() { if let AggIntent::TopK { k, .. } = &aggs[0] { @@ -345,21 +347,13 @@ fn plan_node(expr: &QueryExpr, config: &PhysicalPlannerConfig) -> PhysicalNode { node } - // ── Partition / Merge: Backend stage ──────────────────────── - QueryExpr::Partition { keys, child } => { - let child = plan_node(child, config); - let mut node = PhysicalNode { - op: PhysicalOp::HashAggregate { - keys: keys.keys().to_vec(), - }, - placement: Placement::BackendCollector, - cost: PhysicalCost::default(), - children: vec![child], - }; - insert_exchange_if_needed(&mut node); - node - } - + // ── Merge: Backend stage ───────────────────────────────────── + // + // `Partition` no longer exists in the canonical IR — its keys + // fold into `Aggregate.by` at construction time + // (`intent_algebra::lower`), so the `HashAggregate { keys }` this + // arm used to build now comes straight out of the `Aggregate` + // arm above. QueryExpr::Merge { children } => { let children: Vec = children.iter().map(|c| plan_node(c, config)).collect(); @@ -445,6 +439,35 @@ fn plan_node(expr: &QueryExpr, config: &PhysicalPlannerConfig) -> PhysicalNode { // ── LetBinding ────────────────────────────────────────────── QueryExpr::LetBinding { child, .. } => plan_node(child, config), + + // The PromQL-surface superset (Scalar/EvalTime/VectorFromScalar/ + // ScalarFromVector/Relabel/InfoJoin/Sample/TimeRange/TimeShift/ + // WindowFunc) isn't constructed by this parser today. `Scalar` / + // `EvalTime` are leaves; every other new variant wraps exactly + // one child — inherit its placement, mirroring the Sort/Limit/ + // Project arm above, until a dedicated physical op is written. + QueryExpr::Scalar(_) | QueryExpr::EvalTime => PhysicalNode { + op: PhysicalOp::Passthrough, + placement: Placement::QueryEngine, + cost: PhysicalCost::default(), + children: vec![], + }, + QueryExpr::VectorFromScalar(child) + | QueryExpr::ScalarFromVector(child) + | QueryExpr::Relabel { child, .. } + | QueryExpr::InfoJoin { child, .. } + | QueryExpr::Sample { child, .. } + | QueryExpr::TimeRange { child, .. } + | QueryExpr::TimeShift { child, .. } + | QueryExpr::WindowFunc { child, .. } => { + let child = plan_node(child, config); + PhysicalNode { + op: PhysicalOp::Passthrough, + placement: child.placement.clone(), + cost: PhysicalCost::default(), + children: vec![child], + } + } } } @@ -476,17 +499,15 @@ pub(crate) fn decide_sketch_placement( /// Render a `Distinct { cols }` column tuple into a human-readable display /// string for the `Filter { pred }` rationale. `Distinct { cols: [] }` is -/// whole-row SQL DISTINCT and prints as `*`. -fn display_distinct_cols(cols: &[ColumnRef]) -> String { +/// whole-row SQL DISTINCT and prints as `*`. Canonical `cols` are +/// positional `ColumnId`s (no name at this layer — `plan_node` has no +/// schema in scope to resolve one), so each renders as `col#`. +fn display_distinct_cols(cols: &[ColumnId]) -> String { if cols.is_empty() { return "*".into(); } cols.iter() - .map(|c| match c { - ColumnRef::Named(s) => s.clone(), - ColumnRef::SampleValue => "@value".into(), - ColumnRef::Wildcard => "*".into(), - }) + .map(|id| format!("col#{id}")) .collect::>() .join(", ") } @@ -583,7 +604,7 @@ mod tests { source: Source::TimeSeries { metric: name.into(), }, - label_filters: vec![], + predicates: vec![], schema: Schema::default(), } } @@ -592,8 +613,9 @@ mod tests { /// canonical fold of the legacy `SketchAgg`. fn sketch_agg(intent: AggIntent, metric: &str) -> QueryExpr { QueryExpr::Aggregate { - by: vec![], + by: vec![].into(), aggs: vec![intent], + output_names: Vec::new(), having: None, child: Box::new(scan(metric)), } @@ -673,11 +695,12 @@ mod tests { #[test] fn plan_topk_at_query_engine() { let expr = QueryExpr::Aggregate { - by: vec![], + by: vec![].into(), aggs: vec![AggIntent::TopK { k: 10, accuracy: AccuracyTarget::Epsilon(0.05), }], + output_names: Vec::new(), having: None, child: Box::new(sketch_agg(default_frequency(), "m")), }; @@ -691,11 +714,12 @@ mod tests { // TopK(QueryEngine) wrapping a sketch Aggregate(Agent) → Exchange // between them. let expr = QueryExpr::Aggregate { - by: vec![], + by: vec![].into(), aggs: vec![AggIntent::TopK { k: 5, accuracy: AccuracyTarget::Epsilon(0.05), }], + output_names: Vec::new(), having: None, child: Box::new(sketch_agg(default_frequency(), "m")), }; @@ -706,23 +730,23 @@ mod tests { ); } - #[test] - fn plan_partition_at_backend() { - let expr = QueryExpr::Partition { - keys: crate::intent_algebra::PartitionKeys::By(vec!["region".into()]), - child: Box::new(sketch_agg(default_cardinality(), "m")), - }; - let node = plan(&expr, &default_config()); - assert_eq!(node.placement, Placement::BackendCollector); - } + // `plan_partition_at_backend` (a standalone `Partition` node always + // placing at `BackendCollector`) is removed: `Partition` no longer + // exists in the canonical IR — its keys fold into `Aggregate.by` at + // construction time (`intent_algebra::lower`), and a grouped + // single-intent `Aggregate` goes through the same budget-driven + // Agent→Backend→Precompute path as an ungrouped one (`by` isn't a + // placement input), so there is no direct equivalent assertion to + // make here. #[test] fn plan_multi_intent_aggregate_at_query_engine() { // Multi-intent Aggregate → exact HashAggregate at QueryEngine // (no single sketch serves multiple intents). let expr = QueryExpr::Aggregate { - by: vec![], + by: vec![].into(), aggs: vec![AggIntent::Sum { col: None }, AggIntent::Min { col: None }], + output_names: Vec::new(), having: None, child: Box::new(scan("trades")), }; @@ -733,18 +757,20 @@ mod tests { #[test] fn plan_full_pipeline_has_multiple_stages() { - // TopK(Partition(Window(Aggregate(Scan)))) + // TopK(Merge(Window(Aggregate(Scan)))) — `windowed_agg` fuses to + // Agent, `Merge` (the `Frequency` intent's sketch fan-in) sits at + // Backend, and the outer `TopK` intent runs at QueryEngine. // Should span: Agent → Backend → QueryEngine let expr = QueryExpr::Aggregate { - by: vec![], + by: vec![].into(), aggs: vec![AggIntent::TopK { k: 10, accuracy: AccuracyTarget::Epsilon(0.05), }], + output_names: Vec::new(), having: None, - child: Box::new(QueryExpr::Partition { - keys: crate::intent_algebra::PartitionKeys::By(vec!["svc".into()]), - child: Box::new(windowed_agg(default_frequency(), 60, "requests")), + child: Box::new(QueryExpr::Merge { + children: vec![windowed_agg(default_frequency(), 60, "requests")], }), }; let node = plan(&expr, &default_config()); @@ -761,9 +787,19 @@ mod tests { placements.contains(&Placement::QueryEngine), "should have QueryEngine: {placements:?}" ); + // Was `>= 2` when `Merge`'s child used to be a `Partition` node + // (which called `insert_exchange_if_needed` itself, contributing + // a second exchange on top of the outer TopK Aggregate's own). + // `Merge` doesn't call `insert_exchange_if_needed` on its + // children — it accepts heterogeneously-placed children by + // design (see its arm above) — so with `Partition` gone (its + // keys fold into `Aggregate.by` at construction time) this shape + // now has exactly one exchange-inserting boundary: the outer + // TopK Aggregate against its `Merge` child. The 3-stage span + // above is still the real invariant this test protects. assert!( - node.exchange_count() >= 2, - "should have ≥2 exchanges: {}", + node.exchange_count() >= 1, + "should have >=1 exchange: {}", node.exchange_count() ); } diff --git a/control_plane/src/physical/window_fusion.rs b/control_plane/src/physical/window_fusion.rs index 46758857..e8bc024e 100644 --- a/control_plane/src/physical/window_fusion.rs +++ b/control_plane/src/physical/window_fusion.rs @@ -78,6 +78,7 @@ pub fn recognize_windowed_sketch(expr: &QueryExpr) -> Option Option<&crate::intent_algebra::Schema> { QueryExpr::Filter { child, .. } | QueryExpr::Window { child, .. } | QueryExpr::Aggregate { child, .. } - | QueryExpr::Partition { child, .. } | QueryExpr::Distinct { child, .. } | QueryExpr::Project { child, .. } | QueryExpr::Sort { child, .. } @@ -184,7 +182,11 @@ fn root_scan_schema(qe: &QueryExpr) -> Option<&crate::intent_algebra::Schema> { QueryExpr::LetBinding { expr, child, .. } => { root_scan_schema(expr).or_else(|| root_scan_schema(child)) } - QueryExpr::Ref { .. } => None, + // `Ref` has no reachable `Scan` without a `LetBinding` scope, and + // the PromQL-surface superset (Scalar/EvalTime/VectorFromScalar/ + // ScalarFromVector/Relabel/InfoJoin/Sample/TimeRange/TimeShift/ + // WindowFunc) isn't constructed by this parser today. + _ => None, } } @@ -208,8 +210,8 @@ impl QeCollector { match expr { QueryExpr::Scan { source, - label_filters, - .. + predicates, + schema: scan_schema, } => { if self.metric_name.is_none() { self.metric_name = Some(match source { @@ -217,16 +219,20 @@ impl QeCollector { Source::Table { table_ref } => table_ref.clone(), }); } - // Canonical `Scan` carries equality label filters inline. - for lf in label_filters { - self.label_filters - .entry(lf.label.clone()) - .or_insert_with(|| lf.equals.clone()); + // Canonical `Scan.predicates` carries equality label + // filters as typed `Predicate(L3Expr::Compare{Column, Eq, + // Literal(Utf8)})` trees (the shape + // `query_expr::label_filter_to_predicate` builds) — resolve + // each `Column` id back to its name via the Scan's own + // schema to recover the flat name/value map this legacy + // `ParsedQuery` output still wants. + for p in predicates { + collect_filters_from_expr(&p.0, Some(scan_schema), &mut self.label_filters); } } QueryExpr::Filter { pred, child } => { // Extract equality label filters from the predicate tree. - collect_filters_from_scalar(pred, &mut self.label_filters); + collect_filters_from_scalar(pred, schema, &mut self.label_filters); self.visit(child, schema); } QueryExpr::Window { size, child, .. } => { @@ -235,14 +241,6 @@ impl QeCollector { } self.visit(child, schema); } - QueryExpr::Partition { keys, child } => { - for k in keys.keys() { - if !self.group_by_labels.contains(k) { - self.group_by_labels.push(k.clone()); - } - } - self.visit(child, schema); - } QueryExpr::Aggregate { by, aggs, child, .. } => { @@ -294,7 +292,13 @@ impl QeCollector { self.visit(expr, schema); self.visit(child, schema); } + // `Ref` has no reachable `Scan` without a `LetBinding` scope + // to resolve it against, and the PromQL-surface superset + // (Scalar/EvalTime/VectorFromScalar/ScalarFromVector/Relabel/ + // InfoJoin/Sample/TimeRange/TimeShift/WindowFunc) isn't + // constructed by this parser today. QueryExpr::Ref { .. } => {} + _ => {} } } @@ -374,28 +378,41 @@ impl QeCollector { } } -fn collect_filters_from_scalar(pred: &Predicate, out: &mut HashMap) { - match pred { - Predicate::BinaryOp { - op: BinaryOpKind::Eq, - lhs, - rhs, +fn collect_filters_from_scalar( + pred: &Predicate, + schema: Option<&crate::intent_algebra::Schema>, + out: &mut HashMap, +) { + collect_filters_from_expr(&pred.0, schema, out); +} + +/// Recover a flat name/value equality map from a canonical `L3Expr` +/// predicate tree — `Column(id) == Literal(Utf8(v))` conjuncts, `id` +/// resolved back to a name via `schema` (positional `Column` carries no +/// name of its own, unlike the pre-merge name-based `Predicate::Column`). +fn collect_filters_from_expr( + expr: &L3Expr, + schema: Option<&crate::intent_algebra::Schema>, + out: &mut HashMap, +) { + match expr { + L3Expr::Compare { + left, + op: CompareOp::Eq, + right, } => { - if let ( - Predicate::Column(ColumnRef::Named(col)), - Predicate::Literal(LiteralValue::Str(v)), - ) = (lhs.as_ref(), rhs.as_ref()) + if let (L3Expr::Column(id), L3Expr::Literal(L3Scalar::Utf8(v))) = + (left.as_ref(), right.as_ref()) { - out.insert(col.clone(), v.clone()); + if let Some(col) = schema.and_then(|s| s.columns.get(*id)) { + out.entry(col.name.clone()).or_insert_with(|| v.clone()); + } } } - Predicate::BinaryOp { - op: BinaryOpKind::And, - lhs, - rhs, - } => { - collect_filters_from_scalar(lhs, out); - collect_filters_from_scalar(rhs, out); + L3Expr::BoolAnd(parts) => { + for p in parts { + collect_filters_from_expr(p, schema, out); + } } _ => {} } @@ -574,12 +591,14 @@ mod doc_verify_all { ) .unwrap(); // The legacy `TopK` folds to a canonical `Aggregate` carrying an - // `AggIntent::TopK`, over the `Partition { Window { Aggregate } }` - // the grouped windowed frequency sketch lowers to. + // `AggIntent::TopK`, over the `Window { Aggregate { by } } }` the + // grouped windowed frequency sketch lowers to — the group-by key + // folds straight into the inner `Aggregate.by: GroupKeys` (no + // `Partition` wrap; that node doesn't exist in the canonical IR). match &expr { QueryExpr::Aggregate { aggs, child, .. } => { assert!(matches!(aggs.as_slice(), [AggIntent::TopK { k: 10, .. }])); - assert!(matches!(child.as_ref(), QueryExpr::Partition { .. })); + assert!(matches!(child.as_ref(), QueryExpr::Window { .. })); } other => panic!("expected Aggregate with TopK intent, got {other:?}"), } diff --git a/control_plane/src/query_parser/promql.rs b/control_plane/src/query_parser/promql.rs index 8aa1eb69..44813f08 100644 --- a/control_plane/src/query_parser/promql.rs +++ b/control_plane/src/query_parser/promql.rs @@ -184,6 +184,7 @@ use crate::intent_algebra::relational::{ PartitionKeys as QePartitionKeys, QueryExpr, SourceSpec as QeSourceSpec, VectorGrouping, VectorMatch, VectorMatchKind, }; +use crate::intent_algebra::{ArithOp, CompareOp}; use promql_parser::parser::{token::TokenType, BinaryExpr, VectorMatchCardinality}; /// Parse a PromQL expression string directly into an optimised [`QueryExpr`]. @@ -481,23 +482,23 @@ fn promql_token_to_binop(tok: TokenType) -> BinaryOpKind { // token::T_* are u8 constants; TokenType wraps them as TokenType(u8). let id = tok.id(); match id { - token::T_ADD => BinaryOpKind::Add, - token::T_SUB => BinaryOpKind::Sub, - token::T_MUL => BinaryOpKind::Mul, - token::T_DIV => BinaryOpKind::Div, - token::T_MOD => BinaryOpKind::Mod, + token::T_ADD => BinaryOpKind::Arith(ArithOp::Add), + token::T_SUB => BinaryOpKind::Arith(ArithOp::Sub), + token::T_MUL => BinaryOpKind::Arith(ArithOp::Mul), + token::T_DIV => BinaryOpKind::Arith(ArithOp::Div), + token::T_MOD => BinaryOpKind::Arith(ArithOp::Mod), token::T_POW => BinaryOpKind::Pow, - token::T_EQLC => BinaryOpKind::Eq, - token::T_NEQ => BinaryOpKind::Ne, - token::T_LSS => BinaryOpKind::Lt, - token::T_LTE => BinaryOpKind::Le, - token::T_GTR => BinaryOpKind::Gt, - token::T_GTE => BinaryOpKind::Ge, + token::T_EQLC => BinaryOpKind::Compare(CompareOp::Eq), + token::T_NEQ => BinaryOpKind::Compare(CompareOp::Ne), + token::T_LSS => BinaryOpKind::Compare(CompareOp::Lt), + token::T_LTE => BinaryOpKind::Compare(CompareOp::Le), + token::T_GTR => BinaryOpKind::Compare(CompareOp::Gt), + token::T_GTE => BinaryOpKind::Compare(CompareOp::Ge), token::T_LAND => BinaryOpKind::And, token::T_LOR => BinaryOpKind::Or, token::T_LUNLESS => BinaryOpKind::Unless, token::T_ATAN2 => BinaryOpKind::Atan2, - _ => BinaryOpKind::Add, // unknown — default to add + _ => BinaryOpKind::Arith(ArithOp::Add), // unknown — default to add } } @@ -586,8 +587,10 @@ fn build_qe_aggregate( having: None, input: Box::new(windowed), }; - // Don't wrap with Partition separately — keys are already in the Aggregate. - // The lowering pass will create the Partition node when it lowers the Aggregate. + // Don't wrap with Partition separately — keys are already in the + // Aggregate. The lowering pass folds them straight into the + // canonical `Aggregate.by: GroupKeys` (no `Partition` node exists in + // the canonical IR). agg } @@ -596,6 +599,7 @@ fn apply_qe_filters(input: QueryExpr, filters: Vec) -> QueryExpr { input } else { use crate::intent_algebra::relational::{BinaryOpKind, LiteralValue, ScalarExpr}; + use crate::intent_algebra::CompareOp; let pred = filters .iter() .fold(ScalarExpr::Literal(LiteralValue::Bool(true)), |acc, p| { @@ -608,52 +612,52 @@ fn apply_qe_filters(input: QueryExpr, filters: Vec) -> QueryExpr { }; let this = match &p.op { FilterOp::Eq => ScalarExpr::BinaryOp { - op: BinaryOpKind::Eq, + op: BinaryOpKind::Compare(CompareOp::Eq), lhs: Box::new(col), rhs: Box::new(val), }, FilterOp::Ne => ScalarExpr::BinaryOp { - op: BinaryOpKind::Ne, + op: BinaryOpKind::Compare(CompareOp::Ne), lhs: Box::new(col), rhs: Box::new(val), }, FilterOp::Lt => ScalarExpr::BinaryOp { - op: BinaryOpKind::Lt, + op: BinaryOpKind::Compare(CompareOp::Lt), lhs: Box::new(col), rhs: Box::new(val), }, FilterOp::Le => ScalarExpr::BinaryOp { - op: BinaryOpKind::Le, + op: BinaryOpKind::Compare(CompareOp::Le), lhs: Box::new(col), rhs: Box::new(val), }, FilterOp::Gt => ScalarExpr::BinaryOp { - op: BinaryOpKind::Gt, + op: BinaryOpKind::Compare(CompareOp::Gt), lhs: Box::new(col), rhs: Box::new(val), }, FilterOp::Ge => ScalarExpr::BinaryOp { - op: BinaryOpKind::Ge, + op: BinaryOpKind::Compare(CompareOp::Ge), lhs: Box::new(col), rhs: Box::new(val), }, FilterOp::Regex(r) => ScalarExpr::BinaryOp { - op: BinaryOpKind::Regex, + op: BinaryOpKind::Compare(CompareOp::Regex), lhs: Box::new(col), rhs: Box::new(ScalarExpr::Literal(LiteralValue::Str(r.clone()))), }, FilterOp::NotRegex(r) => ScalarExpr::BinaryOp { - op: BinaryOpKind::NotRegex, + op: BinaryOpKind::Compare(CompareOp::NotRegex), lhs: Box::new(col), rhs: Box::new(ScalarExpr::Literal(LiteralValue::Str(r.clone()))), }, FilterOp::Like => ScalarExpr::BinaryOp { - op: BinaryOpKind::Like, + op: BinaryOpKind::Compare(CompareOp::Like), lhs: Box::new(col), rhs: Box::new(val), }, FilterOp::NotLike => ScalarExpr::BinaryOp { - op: BinaryOpKind::NotLike, + op: BinaryOpKind::Compare(CompareOp::NotLike), lhs: Box::new(col), rhs: Box::new(val), }, diff --git a/control_plane/src/sketch_algebra/lower.rs b/control_plane/src/sketch_algebra/lower.rs index 2aec4ad6..c8f6b9de 100644 --- a/control_plane/src/sketch_algebra/lower.rs +++ b/control_plane/src/sketch_algebra/lower.rs @@ -67,11 +67,13 @@ fn bind_recursive(expr: &QueryExpr, accuracy: &AccuracyTarget) -> PhysicalExpr { // when relevant, otherwise we wrap the L3 sub-tree in `Logical`. match expr { QueryExpr::LetBinding { name, expr, child } => PhysicalExpr::LetBinding { - name: name.clone(), + name: crate::types_v2::BindingName::new(name.as_str()), expr: Box::new(bind_recursive(expr, accuracy)), child: Box::new(bind_recursive(child, accuracy)), }, - QueryExpr::Ref { name } => PhysicalExpr::Ref { name: name.clone() }, + QueryExpr::Ref { name } => PhysicalExpr::Ref { + name: crate::types_v2::BindingName::new(name.as_str()), + }, // The canonical L3 IR places `Window` *above* a single-statistic // sketchable `Aggregate` (`lower`'s window-swap). // The `Bind*` rules match `Aggregate` with the window as its @@ -89,6 +91,7 @@ fn bind_recursive(expr: &QueryExpr, accuracy: &AccuracyTarget) -> PhysicalExpr { let QueryExpr::Aggregate { by, aggs, + output_names, having, child: agg_child, } = child.as_ref() @@ -98,6 +101,7 @@ fn bind_recursive(expr: &QueryExpr, accuracy: &AccuracyTarget) -> PhysicalExpr { let pushed = QueryExpr::Aggregate { by: by.clone(), aggs: aggs.clone(), + output_names: output_names.clone(), having: having.clone(), child: Box::new(QueryExpr::Window { kind: kind.clone(), diff --git a/control_plane/src/sketch_algebra/physical_expr.rs b/control_plane/src/sketch_algebra/physical_expr.rs index 0be8cf18..7d072124 100644 --- a/control_plane/src/sketch_algebra/physical_expr.rs +++ b/control_plane/src/sketch_algebra/physical_expr.rs @@ -255,38 +255,42 @@ mod tests { use std::time::Duration; fn ts_scan() -> QueryExpr { + let schema = Schema::with_time_index( + vec![ + Column { + name: "ts".into(), + dtype: DataType::Timestamp, + nullable: false, + table: None, + }, + Column { + name: "service".into(), + dtype: DataType::Utf8, + nullable: false, + table: None, + }, + Column { + name: "value".into(), + dtype: DataType::Float64, + nullable: false, + table: None, + }, + ], + 0, + vec![vec![0, 1]], + ); + let lf = LabelFilter { + label: "service".into(), + equals: "api".into(), + }; + let pred = crate::intent_algebra::label_filter_to_predicate(&lf, &schema) + .expect("service column present in schema"); QueryExpr::Scan { source: Source::TimeSeries { metric: "http_request_duration_seconds".into(), }, - label_filters: vec![LabelFilter { - label: "service".into(), - equals: "api".into(), - }], - schema: Schema::with_time_index( - vec![ - Column { - name: "ts".into(), - dtype: DataType::Timestamp, - nullable: false, - table: None, - }, - Column { - name: "service".into(), - dtype: DataType::Utf8, - nullable: false, - table: None, - }, - Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - table: None, - }, - ], - 0, - vec![vec![0, 1]], - ), + predicates: vec![pred], + schema, } } diff --git a/control_plane/src/sketch_algebra/rules/bind_archive_only.rs b/control_plane/src/sketch_algebra/rules/bind_archive_only.rs index ed9776e4..c03d14aa 100644 --- a/control_plane/src/sketch_algebra/rules/bind_archive_only.rs +++ b/control_plane/src/sketch_algebra/rules/bind_archive_only.rs @@ -72,38 +72,42 @@ mod tests { use std::time::Duration; fn ts_scan() -> QueryExpr { + let schema = Schema::with_time_index( + vec![ + Column { + name: "ts".into(), + dtype: DataType::Timestamp, + nullable: false, + table: None, + }, + Column { + name: "service".into(), + dtype: DataType::Utf8, + nullable: false, + table: None, + }, + Column { + name: "value".into(), + dtype: DataType::Float64, + nullable: false, + table: None, + }, + ], + 0, + vec![vec![0, 1]], + ); + let lf = LabelFilter { + label: "service".into(), + equals: "api".into(), + }; + let pred = crate::intent_algebra::label_filter_to_predicate(&lf, &schema) + .expect("service column present in schema"); QueryExpr::Scan { source: Source::TimeSeries { metric: "http_request_duration_seconds_bucket".into(), }, - label_filters: vec![LabelFilter { - label: "service".into(), - equals: "api".into(), - }], - schema: Schema::with_time_index( - vec![ - Column { - name: "ts".into(), - dtype: DataType::Timestamp, - nullable: false, - table: None, - }, - Column { - name: "service".into(), - dtype: DataType::Utf8, - nullable: false, - table: None, - }, - Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - table: None, - }, - ], - 0, - vec![vec![0, 1]], - ), + predicates: vec![pred], + schema, } } @@ -118,8 +122,9 @@ mod tests { fn agg_with(intent: AggIntent) -> QueryExpr { QueryExpr::Aggregate { - by: vec![], + by: vec![].into(), aggs: vec![intent], + output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), } diff --git a/control_plane/src/sketch_algebra/rules/bind_exact_agg.rs b/control_plane/src/sketch_algebra/rules/bind_exact_agg.rs index 026fb98f..2af99235 100644 --- a/control_plane/src/sketch_algebra/rules/bind_exact_agg.rs +++ b/control_plane/src/sketch_algebra/rules/bind_exact_agg.rs @@ -136,17 +136,18 @@ impl Rule for BindExactAgg { #[cfg(test)] mod tests { use super::*; - use crate::intent_algebra::{LabelFilter, Schema, Source}; + use crate::intent_algebra::{Schema, Source}; fn scan(metric: &str) -> QueryExpr { + // `Schema::default()` has no columns, so a "service" label filter + // could never resolve to a `Predicate` here anyway + // (`label_filter_to_predicate` would return `None`) — dropped + // rather than built-and-discarded. QueryExpr::Scan { source: Source::TimeSeries { metric: metric.into(), }, - label_filters: vec![LabelFilter { - label: "service".into(), - equals: "api".into(), - }], + predicates: Vec::new(), schema: Schema::default(), } } @@ -155,7 +156,8 @@ mod tests { QueryExpr::Aggregate { aggs: vec![intent], child: Box::new(scan(metric)), - by: vec![], + by: vec![].into(), + output_names: Vec::new(), having: None, } } @@ -173,7 +175,8 @@ mod tests { slide: None, child: Box::new(scan(metric)), }), - by: vec![], + by: vec![].into(), + output_names: Vec::new(), having: None, } } @@ -272,7 +275,8 @@ mod tests { QueryExpr::Aggregate { aggs: vec![intent], child: Box::new(scan(metric)), - by, + by: by.into(), + output_names: Vec::new(), having: None, } } @@ -291,7 +295,8 @@ mod tests { slide: None, child: Box::new(scan(metric)), }), - by, + by: by.into(), + output_names: Vec::new(), having: None, } } diff --git a/control_plane/src/sketch_algebra/tests.rs b/control_plane/src/sketch_algebra/tests.rs index e7e0391a..9a206528 100644 --- a/control_plane/src/sketch_algebra/tests.rs +++ b/control_plane/src/sketch_algebra/tests.rs @@ -26,23 +26,27 @@ fn col(name: &str, dtype: DataType) -> Column { } fn ts_scan() -> QueryExpr { + let schema = Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("service", DataType::Utf8), + col("value", DataType::Float64), + ], + 0, + vec![vec![0, 1]], + ); + let lf = LabelFilter { + label: "service".into(), + equals: "api".into(), + }; + let pred = crate::intent_algebra::label_filter_to_predicate(&lf, &schema) + .expect("service column present in schema"); QueryExpr::Scan { source: Source::TimeSeries { metric: "http_request_duration_seconds".into(), }, - label_filters: vec![LabelFilter { - label: "service".into(), - equals: "api".into(), - }], - schema: Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("service", DataType::Utf8), - col("value", DataType::Float64), - ], - 0, - vec![vec![0, 1]], - ), + predicates: vec![pred], + schema, } } @@ -57,12 +61,13 @@ fn windowed_scan() -> QueryExpr { fn agg_quantile(q: f64, accuracy: AccuracyTarget) -> QueryExpr { QueryExpr::Aggregate { - by: vec![], + by: crate::intent_algebra::GroupKeys::none(), aggs: vec![AggIntent::Quantile { col: None, q, accuracy, }], + output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), } @@ -158,8 +163,9 @@ fn bind_picks_ddsketch_over_kll_when_eps_explicit() { /// Build an `Aggregate{TopK{k, accuracy}}` over the windowed scan. fn agg_topk(k: usize, accuracy: AccuracyTarget) -> QueryExpr { QueryExpr::Aggregate { - by: vec![], + by: vec![].into(), aggs: vec![AggIntent::TopK { k, accuracy }], + output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), } @@ -275,11 +281,12 @@ fn bind_cms_topk_picks_cost_min_meeting_sla() { #[test] fn bind_hll_cardinality_basic() { let expr = QueryExpr::Aggregate { - by: vec![], + by: vec![].into(), aggs: vec![AggIntent::Cardinality { col: None, accuracy: AccuracyTarget::Epsilon(0.01), }], + output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), }; @@ -319,8 +326,9 @@ fn sum_now_binds_to_exact_agg_after_pr_6_followup() { // `PhysicalExpr::ExactAgg { agg_type: Sum, .. }` so the ASAP-tier // exact-aggregation path can serve the intent. let expr = QueryExpr::Aggregate { - by: vec![], + by: vec![].into(), aggs: vec![AggIntent::Sum { col: None }], + output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), }; @@ -362,12 +370,13 @@ fn bind_exact_accuracy_disables_quantile_binding() { #[test] fn phase_b_pattern_only_temporal_quantile_binds_to_sketch() { let expr = QueryExpr::Aggregate { - by: vec![], + by: vec![].into(), aggs: vec![AggIntent::Quantile { col: None, q: 0.99, accuracy: AccuracyTarget::Epsilon(0.01), }], + output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), }; @@ -399,8 +408,9 @@ fn phase_b_pattern_only_temporal_quantile_binds_to_sketch() { #[test] fn phase_b_pattern_only_temporal_sum_binds_to_exact_agg() { let expr = QueryExpr::Aggregate { - by: vec![], + by: vec![].into(), aggs: vec![AggIntent::Sum { col: None }], + output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), }; @@ -422,8 +432,9 @@ fn phase_b_pattern_only_temporal_sum_binds_to_exact_agg() { #[test] fn phase_b_pattern_only_spatial_aggregate_binds_to_multiple_sum() { let expr = QueryExpr::Aggregate { - by: vec![1], // service column + by: vec![1].into(), // service column aggs: vec![AggIntent::Sum { col: None }], + output_names: Vec::new(), having: None, child: Box::new(ts_scan()), }; @@ -442,8 +453,9 @@ fn phase_b_pattern_only_spatial_aggregate_binds_to_multiple_sum() { #[test] fn phase_b_pattern_temporal_and_spatial_combined_binds_to_multiple_increase() { let expr = QueryExpr::Aggregate { - by: vec![1], + by: vec![1].into(), aggs: vec![AggIntent::Rate], + output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), }; @@ -476,8 +488,9 @@ fn phase_b_pattern_archive_only_routes_to_archive() { "Phase β intent must flag archive" ); let expr = QueryExpr::Aggregate { - by: vec![], + by: vec![].into(), aggs: vec![intent.clone()], + output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), }; @@ -724,8 +737,9 @@ fn phase_b_e2e_topk_well_formed() { fn phase_b_e2e_archive_only_e2e_binding() { let intent = AggIntent::Absent; let expr = QueryExpr::Aggregate { - by: vec![], + by: vec![].into(), aggs: vec![intent.clone()], + output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), }; @@ -766,8 +780,9 @@ fn phase_b_archive_only_intents_round_trip_through_binder() { ]; for intent in intents { let expr = QueryExpr::Aggregate { - by: vec![], + by: vec![].into(), aggs: vec![intent.clone()], + output_names: Vec::new(), having: None, child: Box::new(windowed_scan()), }; From b46afbdf1c080b1976eabe97c144c4ab95151468 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 19 Jul 2026 08:26:57 -0600 Subject: [PATCH 04/11] feat(control_plane): Phase 2 step 4 -- merge relational.rs (L2) onto asap-l2 relational.rs's QueryExpr/AggFunc/AggItem/SourceSpec/L2ProjectItem/ L2SortKey are no longer defined locally -- re-exported from the new asap-l2 crate (same git dependency/pinned commit as asap-ir). binder.rs and column_resolution.rs are likewise re-exports of asap_l2's versions, which are a strict superset of the ones they replace (broader column-reference collection, resolve_expr resolves a whole L2Expr tree in one generic pass instead of the old hand-rolled ScalarExpr walk). Also swaps control_plane's promql-parser dependency from crates.io 0.8 to the private ProjectASAP/promql-parser fork (pinned commit, matching what ASAPController's own frontend-promql already depends on) so the two repos parse PromQL identically going forward. asap_l2's own AggFunc->AggIntent mapping doesn't match two behaviors a real, tested consumer (asap_tier_analysis's outer_fn dispatch) still depends on -- avg_over_time as a p50 quantile-sketch approximation, and Rate/Increase/Delta collapsing onto AggIntent::Sum rather than adopting the dedicated intents asap_l2 would produce -- so lower.rs stays control_plane's own converter (not a re-export of asap_l2::lower) specifically to preserve that dispatch. Reconciling it to consume the dedicated intents directly is deliberately deferred to the PromQL frontend semantic-retarget step, alongside properly distinguishing changes/resets/delta/idelta/deriv/predict_linear (currently still bucketed together, matching pre-merge behavior exactly). AggFunc::Frequency and AggFunc::Custom(String) (control_plane-only extensions with no asap_l2 equivalent) are preserved as behavior rather than vocabulary: promql.rs now constructs plain AggFunc::Count for count_over_time (matching asap_l2's own frontend), and lower.rs recovers the Frequency-sketch routing via a grouped-or-windowed-Count trigger. Custom is dropped outright -- zero real construction sites. Partition (both at L2 and L3) is gone -- asap_l2's Aggregate carries `without: bool` directly, so control_plane's own Partition-folding logic (both in lower.rs and promql.rs's group-by handling) collapses into the same "fold keys into the nearest Aggregate" shape one layer up. Co-Authored-By: Claude Sonnet 5 --- control_plane/Cargo.toml | 11 +- control_plane/src/intent_algebra/binder.rs | 375 +------ .../src/intent_algebra/column_resolution.rs | 486 +-------- control_plane/src/intent_algebra/lower.rs | 913 ++++++++--------- control_plane/src/intent_algebra/mod.rs | 2 +- .../src/intent_algebra/relational.rs | 963 ++---------------- control_plane/src/physical/window_fusion.rs | 13 +- control_plane/src/query_parser/promql.rs | 391 ++++--- 8 files changed, 762 insertions(+), 2392 deletions(-) diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index 354dd32d..639cfaeb 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -24,7 +24,12 @@ thiserror = "1" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } chrono = { version = "0.4", features = ["serde"] } -promql-parser = "0.8" +# Private mirror of GreptimeTeam/promql-parser (Apache-2.0), matching +# ASAPController's frontend-promql -- carries local patches (limitk / +# limit_ratio, a batch of experimental Prometheus functions) upstream +# crates.io 0.8 doesn't have. Pinned to a commit, not the `asap` branch, +# for the same reproducibility reason as the asap-ir pin below. +promql-parser = { git = "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/ProjectASAP/promql-parser", rev = "c51beafb361af4cc95ed62ae377862c660ceb757" } prost = "0.13" bytes = "1" zstd = "0.13" @@ -44,7 +49,11 @@ asap_types.workspace = true # scratchpad/artifacts/enum-unification-plan.md) -- WindowKind needs # Copy/Default/Hash/Display/FromStr + snake_case serde for # asap_types::enums::WindowKind to replace the backend's own WindowType. +# Also satisfies everything Phase 2 (#395) needs -- asap-l2 (Step 4), +# asap-plan/asap-sketch + Implementation::is_satisfied_by (Steps 8-9) -- +# 7fcaf91 is a strict descendant of every rev that PR pinned along the way. asap-ir = { git = "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/ProjectASAP/ASAPController", rev = "7fcaf914d87e71407c3a6d7ccac613b867f9c11b" } +asap-l2 = { git = "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/ProjectASAP/ASAPController", rev = "7fcaf914d87e71407c3a6d7ccac613b867f9c11b" } asap-sketch = { git = "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/ProjectASAP/ASAPController", rev = "7fcaf914d87e71407c3a6d7ccac613b867f9c11b" } asap-plan = { git = "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/ProjectASAP/ASAPController", rev = "7fcaf914d87e71407c3a6d7ccac613b867f9c11b" } diff --git a/control_plane/src/intent_algebra/binder.rs b/control_plane/src/intent_algebra/binder.rs index 4be693a7..aa97b74a 100644 --- a/control_plane/src/intent_algebra/binder.rs +++ b/control_plane/src/intent_algebra/binder.rs @@ -1,363 +1,16 @@ //! The L3 **Binder** — name resolution as an explicit pass. //! -//! ## Why this exists -//! -//! Mature query engines all have one explicit boundary where symbolic -//! column / table *names* are resolved against a schema source, and -//! everything downstream of that boundary is fully resolved: -//! -//! - **ClickHouse** — `QueryAnalyzer` rewrites `IdentifierNode` → -//! `ColumnNode` against the real `StorageSnapshot`. -//! - **Trino** — `Analyzer` produces an `Analysis` side-table, resolving -//! identifiers against the catalog `Metadata`; the planner then uses -//! opaque, plan-local `Symbol`s. -//! - **RisingWave** — `Binder` resolves names to positional -//! `InputRef { index, data_type }` against its `Catalog`. -//! -//! Our canonical L3 IR (`query_expr::QueryExpr`) already commits to -//! positional column identity — `Aggregate.by: Vec` — exactly -//! like RisingWave's `InputRef`. What was missing was the *pass* that -//! produces it: resolution was smeared into `lower::convert` -//! with a hardcoded synthesized `(ts, value)` schema, so any query -//! referencing a real label / column name (`price`, `host`, …) errored. -//! -//! This module is that missing pass. [`Binder::bind`] produces the -//! complete, self-contained [`Schema`] every `ColumnId` in the converted -//! canonical tree indexes into — the IR's own "RelationType" (Trino) / -//! bind scope (RisingWave). The converter then becomes purely -//! *structural*: it threads the Binder's schema, and positional -//! resolution downstream is **total** — it never errors. -//! -//! ## The `SchemaCatalog` seam -//! -//! design.md §6 ("three distinct metadata sources") already names the -//! abstraction: the DB / source schema is "exposed through a -//! `SchemaCatalog` interface." [`SchemaCatalog`] is that interface. -//! -//! The default [`UsageDerivedCatalog`] knows nothing — every schema is -//! derived purely from what the query itself references. That is the -//! honest state for the observability domain: metric label sets are -//! open-ended and data-dependent, there is no closed catalog to resolve -//! against (unlike a SQL database's `information_schema`). A -//! registry-backed `SchemaCatalog` is future work — and crucially, the -//! `Binder` pass itself does not change when it lands; only the catalog -//! impl swaps. -//! -//! ## Where it sits -//! -//! Today the Binder runs at the L2→L3 (legacy → canonical) conversion -//! boundary — `lower::convert_root` calls it. Once the -//! legacy IR is retired it moves into the `core::lower` L1→L2→L3 passes -//! proper (the `lower_*(ast, schema)` signatures in design.md §6). - -use crate::intent_algebra::relational::QueryExpr as LQueryExpr; -use crate::intent_algebra::schema::{Column, DataType, Schema}; - -/// The DB / source-schema metadata source from design.md §6 "three -/// distinct metadata sources" — resolves a source (metric / table) name -/// to its known columns. -pub trait SchemaCatalog { - /// Columns known for `source`. `None` when the source is unknown to - /// this catalog — the [`Binder`] then falls back to a usage-derived - /// column set. - fn columns_for(&self, source: &str) -> Option>; -} - -/// The default catalog: knows nothing. Every schema the [`Binder`] -/// produces is derived purely from the query's own usage. -/// -/// This is the honest state for the observability domain — see the -/// module doc. A registry-backed `SchemaCatalog` is the future; the -/// `Binder` pass does not change when it lands. -pub struct UsageDerivedCatalog; - -impl SchemaCatalog for UsageDerivedCatalog { - fn columns_for(&self, _source: &str) -> Option> { - None - } -} - -/// The L3 Binder — the explicit name-resolution pass. See the module doc. -pub struct Binder { - catalog: C, -} - -impl Default for Binder { - fn default() -> Self { - Self::new() - } -} - -impl Binder { - /// A Binder with the default usage-derived catalog. - pub fn new() -> Self { - Self { - catalog: UsageDerivedCatalog, - } - } -} - -impl Binder { - /// A Binder backed by an explicit [`SchemaCatalog`]. - pub fn with_catalog(catalog: C) -> Self { - Self { catalog } - } - - /// Resolve the complete [`Schema`] in scope for a query rooted at - /// `tree`. - /// - /// The result contains the time axis, the synthetic `value` column, - /// and one column per distinct name referenced anywhere in the tree - /// — so positional `ColumnId` resolution downstream - /// (`resolve_column_ref` / `resolve_named_keys`) is **total** and - /// never errors. This is the IR's own self-contained "RelationType". - pub fn bind(&self, tree: &LQueryExpr) -> Schema { - // Base columns: from the catalog if it knows the source, else the - // conventional PromQL leaf shape `(ts, value)`. - let mut columns: Vec = tree - .source_name() - .and_then(|name| self.catalog.columns_for(name)) - .unwrap_or_else(default_leaf_columns); - - // Ensure the (ts, value) floor is present — the canonical lowering - // resolves `ColumnRef::SampleValue` against a column literally - // named `value`, and `Window` requires a time index. - for floor in default_leaf_columns() { - if !columns.iter().any(|c| c.name == floor.name) { - columns.push(floor); - } - } - - // Append one column per referenced-but-unknown name. These are - // group-by keys / sketch-target columns the converter resolves - // positionally; with them in the schema, resolution cannot fail. - for name in collect_referenced_columns(tree) { - if !columns.iter().any(|c| c.name == name) { - columns.push(Column { - name, - dtype: DataType::Utf8, // labels / group keys are strings - nullable: true, - table: None, - }); - } - } - - let time_index = columns.iter().position(|c| c.name == "ts"); - Schema { - columns, - time_index, - unique_keys: Vec::new(), - closed: false, - } - } -} - -/// The conventional PromQL leaf column shape: `(ts: Timestamp, value: Float64)`. -fn default_leaf_columns() -> Vec { - vec![ - Column { - name: "ts".into(), - dtype: DataType::Timestamp, - nullable: false, - table: None, - }, - Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - table: None, - }, - ] -} - -/// Walk the legacy tree and collect every distinct column name the -/// legacy → canonical converter resolves positionally: `Aggregate.keys`, -/// `TopK.by`, `Partition.keys`, and every `ScalarExpr::Column(name)` -/// reachable from a `Filter.pred` / `Aggregate.having` / `Join.pred` / -/// `Project` item — since the canonical `Predicate(L3Expr)` is fully -/// positional (`L3Expr::Column(ColumnId)`, no name-based fallback), -/// `convert_scalar` requires every referenced name to already be in the -/// schema. Sorted + de-duplicated for a stable, deterministic column -/// order. -/// -/// `AggItem.col` (the statistic's *input* column) is deliberately not -/// collected — the converter never resolves it positionally; it only -/// ever resolves group-by keys and predicate/projection columns. -fn collect_referenced_columns(tree: &LQueryExpr) -> Vec { - let mut out: Vec = Vec::new(); - tree.walk(&mut |node| match node { - LQueryExpr::Aggregate { keys, having, .. } => { - out.extend(keys.iter().cloned()); - if let Some(pred) = having { - collect_columns_from_scalar(pred, &mut out); - } - } - LQueryExpr::TopK { by, .. } => out.extend(by.iter().cloned()), - LQueryExpr::Partition { keys, .. } => out.extend(keys.keys().iter().cloned()), - LQueryExpr::Filter { pred, .. } => collect_columns_from_scalar(pred, &mut out), - LQueryExpr::Join { - pred: Some(pred), .. - } => collect_columns_from_scalar(pred, &mut out), - LQueryExpr::Project { cols, .. } => { - for item in cols { - collect_columns_from_scalar(&item.expr, &mut out); - } - } - _ => {} - }); - out.sort(); - out.dedup(); - out -} - -/// Recursively collect every `ScalarExpr::Column(name)` reachable from -/// `expr`. Does not descend into `ScalarSubquery`'s inner `QueryExpr` — -/// `lower::convert_scalar` rejects `ScalarSubquery` outright (see its -/// module doc), so there is no positional resolution to satisfy inside -/// one. -fn collect_columns_from_scalar( - expr: &crate::intent_algebra::relational::ScalarExpr, - out: &mut Vec, -) { - use crate::intent_algebra::relational::ScalarExpr as SE; - match expr { - SE::Column(name) => out.push(name.clone()), - SE::Literal(_) | SE::ScalarSubquery(_) => {} - SE::BinaryOp { lhs, rhs, .. } => { - collect_columns_from_scalar(lhs, out); - collect_columns_from_scalar(rhs, out); - } - SE::FunctionCall { args, .. } => { - for a in args { - collect_columns_from_scalar(a, out); - } - } - SE::InList { expr, list, .. } => { - collect_columns_from_scalar(expr, out); - for a in list { - collect_columns_from_scalar(a, out); - } - } - SE::Between { - expr, low, high, .. - } => { - collect_columns_from_scalar(expr, out); - collect_columns_from_scalar(low, out); - collect_columns_from_scalar(high, out); - } - SE::IsNull { expr, .. } => collect_columns_from_scalar(expr, out), - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::intent_algebra::relational::{ - AggFunc, AggItem, ColumnRef as LColumnRef, PartitionKeys, QueryExpr as LQueryExpr, - SourceSpec, - }; - - fn src(name: &str) -> LQueryExpr { - LQueryExpr::Source(SourceSpec { name: name.into() }) - } - - #[test] - fn bare_source_yields_ts_value_floor() { - let schema = Binder::new().bind(&src("m")); - assert_eq!(schema.columns.len(), 2); - assert_eq!(schema.columns[0].name, "ts"); - assert_eq!(schema.columns[1].name, "value"); - assert_eq!(schema.time_index, Some(0)); - } - - #[test] - fn aggregate_keys_and_partition_keys_land_in_schema() { - // Aggregate { keys: ["region"] } and Partition { By(["host"]) }. - let tree = LQueryExpr::Partition { - keys: PartitionKeys::By(vec!["host".into()]), - input: Box::new(LQueryExpr::Aggregate { - keys: vec!["region".into()], - aggs: vec![AggItem { - alias: "c".into(), - func: AggFunc::Count, - col: LColumnRef::Wildcard, - distinct: false, - }], - having: None, - input: Box::new(src("hits")), - }), - }; - let schema = Binder::new().bind(&tree); - assert!(schema.column_id("region").is_some()); - assert!(schema.column_id("host").is_some()); - } - - #[test] - fn topk_by_keys_land_in_schema() { - let tree = LQueryExpr::TopK { - k: 10, - by: vec!["symbol".into(), "exchange".into()], - input: Box::new(src("m")), - }; - let schema = Binder::new().bind(&tree); - assert!(schema.column_id("symbol").is_some()); - assert!(schema.column_id("exchange").is_some()); - } - - #[test] - fn referenced_names_are_deduplicated() { - // Same name referenced twice → one column. - let tree = LQueryExpr::Aggregate { - keys: vec!["region".into(), "region".into()], - aggs: vec![], - having: None, - input: Box::new(src("m")), - }; - let schema = Binder::new().bind(&tree); - let region_cols = schema.columns.iter().filter(|c| c.name == "region").count(); - assert_eq!(region_cols, 1); - } - - #[test] - fn custom_catalog_supplies_base_columns() { - struct FixedCatalog; - impl SchemaCatalog for FixedCatalog { - fn columns_for(&self, source: &str) -> Option> { - if source == "known_metric" { - Some(vec![ - Column { - name: "ts".into(), - dtype: DataType::Timestamp, - nullable: false, - table: None, - }, - Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - table: None, - }, - Column { - name: "datacenter".into(), - dtype: DataType::Utf8, - nullable: false, - table: None, - }, - ]) - } else { - None - } - } - } - let tree = src("known_metric"); - let schema = Binder::with_catalog(FixedCatalog).bind(&tree); - // `datacenter` came from the catalog, not usage-synthesis — and it - // is non-nullable, unlike a usage-derived column. - let dc = schema - .column_id("datacenter") - .and_then(|id| schema.columns.get(id)); - assert!(matches!(dc, Some(c) if !c.nullable)); - } -} +//! ## Phase 2 step 4 (docs/migration-plan-backend-plan.md) +//! +//! `Binder`, `SchemaCatalog`, `UsageDerivedCatalog` are no longer defined +//! in this repo — re-exported from `asap_l2::binder`. `asap_l2`'s version +//! is a strict superset of this repo's pre-merge one: it additionally +//! walks `Sort.keys` / `Sort.partition_by` / `Relabel.value` for +//! referenced column names (this repo's version only covered +//! `Filter.pred` / `Aggregate.having` / `Join.pred` / `Project` — added +//! in Phase 2 step 3 to fix a real resolution regression that step's own +//! `Predicate` merge surfaced), and adds +//! [`Binder::bind_with_inherited`] for the `BinaryOp`-side-rebinding case +//! (issue #52) — a capability this repo's Binder never had. + +pub use asap_l2::binder::{Binder, SchemaCatalog, UsageDerivedCatalog}; diff --git a/control_plane/src/intent_algebra/column_resolution.rs b/control_plane/src/intent_algebra/column_resolution.rs index 744abc47..fbb1459f 100644 --- a/control_plane/src/intent_algebra/column_resolution.rs +++ b/control_plane/src/intent_algebra/column_resolution.rs @@ -1,465 +1,29 @@ -//! Schema-driven column resolution for the legacy `QueryExpr` IR -//! (Step β of the relational migration). +//! Schema-driven column resolution for the Layer-2 `relational` IR. //! -//! Step α (PR #138) replaced the legacy `AggIntent` enum with the canonical -//! [`crate::intent_algebra::agg_intent::AggIntent`]. The legacy IR still -//! uses [`crate::intent_algebra::relational::ColumnRef::Named(String)`] for -//! column references; the canonical IR uses positional -//! [`crate::intent_algebra::schema::ColumnId`] resolved against a per-node -//! [`crate::intent_algebra::schema::Schema`]. +//! ## Phase 2 step 4 (docs/migration-plan-backend-plan.md) //! -//! Step β plumbs `Schema` through every consumer that walks the legacy -//! tree so Step γ can migrate variant-by-variant to positional column ids -//! without first having to acquire a Schema everywhere. +//! `ResolveError`, `infer_source_schema`, `infer_schema_for_root`, +//! `resolve_column_ref`, `resolve_column_refs`, `resolve_group_keys_promql`, +//! `resolve_expr`, `output_schema_for_aggregate` are no longer defined in +//! this repo — re-exported from `asap_l2::column_resolution`. //! -//! ## Threading approach +//! Two capabilities this repo's pre-merge version didn't have: //! -//! Approach **(c)** per the migration spec: each traversal function takes a -//! `parent_schema: &Schema` parameter (the schema in scope at that node). -//! The root walker derives the base schema via [`infer_schema_for_root`] -//! (which walks to the outermost `Source` leaf and synthesises a default -//! schema for it) and threads it downward. -//! -//! Legacy operators here are mostly pass-through with respect to schema — -//! they don't add columns, they just filter / partition / sort / window -//! the same rows. The few that DO transform the schema (`Aggregate`, -//! `Project`, `Distinct`) are flagged as Step γ TODOs; Step γ will -//! migrate them onto the canonical `QueryExpr::output_schema_in` path so -//! the schema is locally derivable from the variant fields. -//! -//! ## Why a synthesized default -//! -//! The relational migration plan's "Synthesise from metric name: -//! `(ts, value, *labels)`" decision applies here. There is no -//! `SchemaCatalog` in the controller today, so the source leaf has to -//! produce a schema purely from the metric / table name. This module -//! ships the minimal time-series schema that's correct for the DC + PromQL -//! use case (the only consumers that exist in tree today). Tabular -//! sources (`SourceSpec::Table { table_ref }`) inherit the same shape as -//! a placeholder; a real catalog lookup is a follow-up Step γ TODO. - -use thiserror::Error; - -use crate::intent_algebra::agg_intent::AggIntent; -use crate::intent_algebra::relational::{ColumnRef, QueryExpr, SourceSpec}; -use crate::intent_algebra::schema::{Column, ColumnId, DataType, Schema}; - -/// Errors returned by [`resolve_column_ref`] / [`resolve_column_refs`]. -#[derive(Debug, Error, PartialEq, Eq)] -pub enum ResolveError { - /// `ColumnRef::Named(name)` did not match any column in the supplied - /// schema. Common cause: the schema was synthesized from a metric name - /// but the name being resolved is a free-form label (label sets aren't - /// representable in the canonical `Schema` today — see the module - /// doc-comment). - #[error( - "column `{name}` not found in schema (have: {available:?}) \ - — Step γ TODO: label-set resolution against a real catalog" - )] - NotFound { - name: String, - available: Vec, - }, - /// `ColumnRef::SampleValue` was resolved against a schema that has no - /// `value` column. The synthesized time-series schema always has one, - /// so this only fires for tabular sources without the convention. - #[error("ColumnRef::SampleValue has no `value` column in schema (have: {available:?})")] - NoSampleValue { available: Vec }, - /// `ColumnRef::Wildcard` cannot be resolved to a single - /// [`ColumnId`] — by definition it refers to every row, not a specific - /// column. Callers that hit this branch should special-case `Wildcard` - /// instead of asking for a positional id. - #[error("ColumnRef::Wildcard cannot be resolved to a single ColumnId")] - WildcardNotPositional, -} - -/// Synthesize a default time-series schema for a metric / table source. -/// -/// The shape is the conventional PromQL leaf: `(ts: Timestamp, value: -/// Float64)`. Open-set labels are intentionally omitted — the canonical -/// [`Schema`] model uses a positional `Vec` and has no -/// representation for "any number of label columns whose names are -/// data-dependent." That's tracked as a Step γ TODO at the module level. -/// -/// Per the relational migration plan's "Synthesise from metric name: -/// `(ts, value, *labels)`" decision — minus the `*labels` part the -/// canonical schema model can't express today. -pub fn infer_source_schema(_metric_or_table_name: &str) -> Schema { - Schema::with_time_index( - vec![ - Column { - name: "ts".into(), - dtype: DataType::Timestamp, - nullable: false, - table: None, - }, - Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - table: None, - }, - ], - 0, - // No provable unique key without a catalog — leave empty so the - // CSE gatekeeper (`cse_reuse_is_legal`) conservatively refuses to - // share a producer across queries until a real catalog ships. - Vec::new(), - ) -} - -/// Walk the legacy `QueryExpr` tree to find the outermost (left-most) -/// `Source` leaf and synthesise its default schema via -/// [`infer_source_schema`]. This is the entry-point Schema that consumer -/// walkers thread downward through the tree. -/// -/// Falls back to an empty `Schema` when the root has no `Source` leaf -/// (e.g. a bare `Ref(name)` that resolves outside the supplied tree). -/// Callers that need real Ref resolution should pre-resolve via a -/// `BindingScope` analogue — for Step β the empty fallback is fine -/// because consumers only read the schema in pass-through paths. -pub fn infer_schema_for_root(expr: &QueryExpr) -> Schema { - match expr.source_name() { - Some(name) => infer_source_schema(name), - None => Schema::default(), - } -} - -/// Resolve a single [`ColumnRef`] against a schema, returning the -/// positional [`ColumnId`]. Step γ consumers will call this at the point -/// where they need a `ColumnId` instead of a `String` — for now the -/// helper exists so the migration is mechanical at that point. -/// -/// Resolution rules (mirror canonical -/// [`crate::intent_algebra::query_expr::ColumnRef`] semantics where they -/// already exist, e.g. `Distinct { cols }` in `output_schema_in`): -/// -/// * `ColumnRef::Named(name)` → `schema.column_id(name)`. -/// * `ColumnRef::SampleValue` → `schema.column_id("value")` (PromQL -/// convention; the canonical lowering emits a column literally named -/// `value` for time-series scans). -/// * `ColumnRef::Wildcard` → [`ResolveError::WildcardNotPositional`] -/// (callers must special-case). -pub fn resolve_column_ref(col: &ColumnRef, schema: &Schema) -> Result { - match col { - ColumnRef::Named(name) => schema - .column_id(name) - .ok_or_else(|| ResolveError::NotFound { - name: name.clone(), - available: schema.columns.iter().map(|c| c.name.clone()).collect(), - }), - ColumnRef::SampleValue => { - schema - .column_id("value") - .ok_or_else(|| ResolveError::NoSampleValue { - available: schema.columns.iter().map(|c| c.name.clone()).collect(), - }) - } - ColumnRef::Wildcard => Err(ResolveError::WildcardNotPositional), - } -} - -/// Slice-flavoured [`resolve_column_ref`]: resolves every entry, -/// short-circuiting on the first error. Used by `Distinct { cols }` and -/// the future `Aggregate { by }` migration when keys move from -/// `Vec` to `Vec`. -pub fn resolve_column_refs( - cols: &[ColumnRef], - schema: &Schema, -) -> Result, ResolveError> { - cols.iter().map(|c| resolve_column_ref(c, schema)).collect() -} - -/// Slice-flavoured variant of [`resolve_column_ref`] over a list of -/// `Vec` GROUP BY keys (the shape carried by -/// `relational::QueryExpr::Aggregate.keys`). Mirrors -/// [`resolve_column_refs`] but skips the `ColumnRef::Named` wrapping — -/// the legacy `Aggregate.keys` field is already a `Vec`. -/// -/// Used by `lower::convert` to translate a legacy -/// `Aggregate.keys: Vec` into the canonical `by: Vec`. -pub fn resolve_named_keys(keys: &[String], schema: &Schema) -> Result, ResolveError> { - keys.iter() - .map(|name| { - schema - .column_id(name) - .ok_or_else(|| ResolveError::NotFound { - name: name.clone(), - available: schema.columns.iter().map(|c| c.name.clone()).collect(), - }) - }) - .collect() -} - -// ── Aggregate schema transformation (Step γ1) ──────────────────────────────── - -/// Output schema produced by `QueryExpr::Aggregate { by, aggs, child, .. }` -/// when its `child` carries `input` as its output schema. -/// -/// Mirrors the canonical -/// [`crate::intent_algebra::query_expr::QueryExpr::output_schema_in`] -/// implementation for the `Aggregate` arm — extracted here so consumers -/// descending into a still-legacy `Aggregate.input` can derive the right -/// schema for the child without first having to translate the whole -/// subtree to canonical. -/// -/// Schema-flow rules (per `design.md` §6 schema-flow table, mirrored -/// in `query_expr.rs::output_schema_in`): -/// -/// * Output columns = `by` columns (preserved positionally) followed by -/// one new column per `aggs` entry, named + typed via -/// [`AggIntent::output_column`]. -/// * Time axis is stripped — `Aggregate` produces one row per group, not -/// one row per timestamp. -/// * `unique_keys` = `[by]` when `by` is non-empty (the group-by tuple is -/// unique by construction); empty when `by` is empty (single global row). -/// -/// `by` ids are silently clamped to in-range — out-of-range ids are -/// dropped from the output. The canonical -/// [`crate::intent_algebra::query_expr::QueryExpr::output_schema_in`] -/// surfaces them as -/// [`crate::intent_algebra::query_expr::QueryExprError::InvalidGroupByColumn`]; -/// the legacy bridge here can't error-type its callers without breaking -/// the Step β plumbing signature, so we drop instead. (Callers that need -/// the strict check should resolve `by` ids upstream via -/// [`resolve_named_keys`] which DOES surface `NotFound`.) -/// -/// # Example -/// -/// ```ignore -/// use control_plane::intent_algebra::{ -/// column_resolution::{infer_source_schema, output_schema_for_aggregate}, -/// agg_intent::AggIntent, -/// schema::{Column, DataType, Schema}, -/// }; -/// // Input: a PromQL scan schema (ts, value). -/// let input = infer_source_schema("http_requests_total"); -/// // Aggregate by [] (global) with [Count, Sum]. -/// let by: Vec = vec![]; -/// let aggs = vec![ -/// AggIntent::Count { accuracy: types_v2::AccuracyTarget::Exact }, -/// AggIntent::Sum { col: None }, -/// ]; -/// let output = output_schema_for_aggregate(&input, &by, &aggs); -/// assert_eq!(output.columns.len(), 2); // count + sum -/// assert!(output.time_index.is_none()); // time axis stripped -/// assert!(output.unique_keys.is_empty()); // global agg → no UK -/// ``` -pub fn output_schema_for_aggregate(input: &Schema, by: &[ColumnId], aggs: &[AggIntent]) -> Schema { - let mut out_cols: Vec = Vec::with_capacity(by.len() + aggs.len()); - // GROUP BY columns flow through positionally. - for &id in by { - if let Some(c) = input.columns.get(id) { - out_cols.push(c.clone()); - } - // out-of-range: silently drop — see doc-comment above. - } - // One new column per intent. PromQL convention: intent applied to the - // synthetic `value` column when present; otherwise to the first - // non-grouped column. Mirrors canonical query_expr.rs::output_schema_in. - let value_col_idx = input - .column_id("value") - .or_else(|| (0..input.columns.len()).find(|i| !by.contains(i))); - let probe = value_col_idx - .and_then(|i| input.columns.get(i)) - .cloned() - .unwrap_or(Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - table: None, - }); - for intent in aggs { - out_cols.push(crate::intent_algebra::output_column(intent, &probe)); - } - // Output unique_keys = [by] when by is non-empty; empty (global) → no UK. - let unique_keys = if by.is_empty() { - Vec::new() - } else { - vec![(0..by.len()).collect()] - }; - // Aggregate strips the time axis — output is one row per group. - Schema { - columns: out_cols, - time_index: None, - unique_keys, - closed: false, - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::intent_algebra::relational::SourceSpec; - - fn src(name: &str) -> QueryExpr { - QueryExpr::Source(SourceSpec { name: name.into() }) - } - - #[test] - fn source_schema_has_ts_and_value() { - let s = infer_source_schema("http_requests_total"); - assert_eq!(s.columns.len(), 2); - assert_eq!(s.columns[0].name, "ts"); - assert_eq!(s.columns[1].name, "value"); - assert_eq!(s.time_index, Some(0)); - // No provable unique key without a real catalog — conservative. - assert!(!s.has_unique_key()); - } - - #[test] - fn root_schema_via_walk() { - let expr = QueryExpr::Filter { - pred: crate::intent_algebra::relational::ScalarExpr::Literal( - crate::intent_algebra::relational::LiteralValue::Bool(true), - ), - input: Box::new(src("cpu_usage")), - }; - let s = infer_schema_for_root(&expr); - assert_eq!(s.columns.len(), 2); - assert_eq!(s.columns[1].name, "value"); - } - - #[test] - fn resolve_named_against_value_column() { - let s = infer_source_schema("m"); - let col = ColumnRef::Named("value".into()); - assert_eq!(resolve_column_ref(&col, &s), Ok(1)); - } - - #[test] - fn resolve_sample_value() { - let s = infer_source_schema("m"); - assert_eq!(resolve_column_ref(&ColumnRef::SampleValue, &s), Ok(1)); - } - - #[test] - fn resolve_wildcard_errors() { - let s = infer_source_schema("m"); - assert_eq!( - resolve_column_ref(&ColumnRef::Wildcard, &s), - Err(ResolveError::WildcardNotPositional) - ); - } - - #[test] - fn resolve_unknown_name_errors() { - let s = infer_source_schema("m"); - let err = resolve_column_ref(&ColumnRef::Named("host".into()), &s).unwrap_err(); - assert!(matches!(err, ResolveError::NotFound { .. })); - } - - #[test] - fn resolve_slice_short_circuits() { - let s = infer_source_schema("m"); - let cols = vec![ - ColumnRef::Named("value".into()), - ColumnRef::Named("not_there".into()), - ]; - let r = resolve_column_refs(&cols, &s); - assert!(matches!(r, Err(ResolveError::NotFound { .. }))); - } - - #[test] - fn root_schema_falls_back_when_no_source() { - let expr = QueryExpr::Ref("dangling".into()); - let s = infer_schema_for_root(&expr); - assert_eq!(s.columns.len(), 0); - } - - // ── output_schema_for_aggregate (Step γ1) ────────────────────────────── - - #[test] - fn output_schema_for_aggregate_global_count() { - use crate::types_v2::AccuracyTarget; - let input = infer_source_schema("m"); - let by: Vec = vec![]; - let aggs = vec![AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }]; - let out = output_schema_for_aggregate(&input, &by, &aggs); - assert_eq!(out.columns.len(), 1); - assert_eq!(out.columns[0].name, "count"); - assert!(out.time_index.is_none()); - assert!(out.unique_keys.is_empty()); - } - - #[test] - fn output_schema_for_aggregate_strips_time_axis() { - use crate::types_v2::AccuracyTarget; - let input = infer_source_schema("m"); - assert!(input.time_index.is_some()); - let aggs = vec![AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }]; - let out = output_schema_for_aggregate(&input, &[], &aggs); - assert!(out.time_index.is_none()); - } - - #[test] - fn output_schema_for_aggregate_preserves_by_columns_and_unique_keys() { - // Build an input schema with two extra label columns. - let mut input = infer_source_schema("m"); - input.columns.push(Column { - name: "host".into(), - dtype: DataType::Utf8, - nullable: false, - table: None, - }); - input.columns.push(Column { - name: "region".into(), - dtype: DataType::Utf8, - nullable: false, - table: None, - }); - // Group by host, region (positions 2 and 3). - let by = vec![2usize, 3usize]; - let aggs = vec![AggIntent::Sum { col: None }]; - let out = output_schema_for_aggregate(&input, &by, &aggs); - // Output columns: host, region, sum. - assert_eq!(out.columns.len(), 3); - assert_eq!(out.columns[0].name, "host"); - assert_eq!(out.columns[1].name, "region"); - assert_eq!(out.columns[2].name, "sum"); - // unique_keys = [[0, 1]] (the by tuple is unique by construction). - assert_eq!(out.unique_keys, vec![vec![0, 1]]); - } - - #[test] - fn output_schema_for_aggregate_drops_out_of_range_by_ids() { - let input = infer_source_schema("m"); - // schema only has columns 0..=1; ask for by=[5] which is out of range. - let aggs = vec![AggIntent::Sum { col: None }]; - let out = output_schema_for_aggregate(&input, &[5usize], &aggs); - // The out-of-range by id is silently dropped; output has only the agg. - assert_eq!(out.columns.len(), 1); - assert_eq!(out.columns[0].name, "sum"); - } - - #[test] - fn resolve_named_keys_resolves_present_columns() { - let mut s = infer_source_schema("m"); - s.columns.push(Column { - name: "host".into(), - dtype: DataType::Utf8, - nullable: false, - table: None, - }); - let ids = resolve_named_keys(&["host".to_string()], &s).unwrap(); - assert_eq!(ids, vec![2usize]); - } - - #[test] - fn resolve_named_keys_surfaces_not_found() { - let s = infer_source_schema("m"); - let err = resolve_named_keys(&["missing".to_string()], &s).unwrap_err(); - assert!(matches!(err, ResolveError::NotFound { .. })); - } -} - -// Silence "field never read" on `SourceSpec` when this module is the only -// consumer in some configurations. -#[allow(dead_code)] -const _: fn(&SourceSpec) -> &str = |s| s.name.as_str(); +//! - [`resolve_expr`] resolves a whole `L2Expr` tree (name-based) into an +//! `L3Expr` (positional) in one pass — this repo's `ScalarExpr` is gone +//! (see `relational.rs`'s module doc), so `lower.rs`'s `convert_scalar` +//! now calls this directly instead of hand-matching every `Expr` +//! variant itself. +//! - [`resolve_group_keys_promql`] encodes PromQL's absent-label grouping +//! semantics (issue #53): a GROUP BY key not present in a **closed** +//! schema is provably absent from every row, so it's dropped rather +//! than rejected. `resolve_named_keys`, this repo's old strict-only +//! equivalent, is gone — `lower.rs` calls this instead for `Aggregate` +//! / `Partition` keys. Every schema `lower.rs` builds today is open +//! (`Schema::closed == false`, per `Binder`'s PromQL-only usage-derived +//! catalog), so this degrades to the old strict behavior in practice — +//! ready for when a closed (SQL) schema starts flowing through. +pub use asap_l2::column_resolution::{ + infer_schema_for_root, infer_source_schema, output_schema_for_aggregate, resolve_column_ref, + resolve_column_refs, resolve_expr, resolve_group_keys_promql, ResolveError, +}; diff --git a/control_plane/src/intent_algebra/lower.rs b/control_plane/src/intent_algebra/lower.rs index 87ca3482..c729d1b6 100644 --- a/control_plane/src/intent_algebra/lower.rs +++ b/control_plane/src/intent_algebra/lower.rs @@ -5,87 +5,63 @@ //! *whole* canonical `query_expr::QueryExpr` tree. This is the single //! entry the parse path routes through — [`convert_root`]. //! -//! ## Phase 2 step 3 (docs/migration-plan-backend-plan.md) +//! ## Phase 2 step 4 (docs/migration-plan-backend-plan.md) //! -//! Two shape changes since the canonical `QueryExpr` merged onto -//! `asap_ir` (see `query_expr.rs`'s module docs for the full rationale): +//! Now that `relational.rs` (L2) itself merged onto `asap_l2` (see that +//! file's module doc), this converter is control_plane's *own* — not a +//! re-export of `asap_l2::lower::convert_root` — for one deliberate +//! reason: `asap_l2`'s `AggFunc`→`AggIntent` mapping +//! (`agg_func_to_intent` in its `lower.rs`) produces literal +//! `AggIntent::Avg` / `Rate` / `Increase` for those `AggFunc`s, whereas +//! this repo needs the pre-merge behavior a real, tested consumer +//! (`asap_tier_analysis`'s `outer_fn` dispatch) still depends on: +//! `avg_over_time` → a p50 quantile-sketch approximation, and +//! `Rate`/`Increase`/`Delta` → `AggIntent::Sum` (disambiguated via the +//! separate `outer_fn` field, not by intent shape). Reconciling that +//! dispatch to consume the dedicated intents directly is real, +//! deliberately-deferred follow-up work (tracked for the PromQL-frontend +//! semantic-retarget step), not a byproduct of this type merge. Every +//! *structural* piece below (scalar resolution, schema threading, the +//! `GroupKeys` shape) is unchanged from `asap_l2`'s own converter — +//! only the `Aggregate`/`AggFunc` handling is control_plane-specific. //! -//! - **Predicates** translate to `L3Expr` (via `Predicate`, `expr_ir.rs`) -//! instead of this repo's old 8-variant `Predicate` enum. `Between` -//! desugars via `query_expr::between`. **`ScalarSubquery` is rejected** -//! ([`ConvertError::UnsupportedScalarSubquery`]) rather than lowered: -//! `asap_ir`'s `L3Expr` has no slot for "reference the value bound by -//! an enclosing `LetBinding`" (the old `Predicate::Column(ColumnRef:: -//! Named(subq_name))` was itself a hack the canonical positional -//! `L3Expr::Column(ColumnId)` can't reproduce without inventing a -//! sentinel id space nothing downstream knows about). ASAPController's -//! own L2→L3 lowering (`crates/l2/src/lower.rs`) has no correlated- -//! subquery construct either — its `Ref`/`LetBinding` are documented as -//! "Reserved: no front end emits yet." This repo's own front ends never -//! construct `ScalarExpr::ScalarSubquery` either (grep-verified: the -//! only non-test, non-definition sites were `lower.rs` itself and the -//! now-dead `optimizer/engine.rs` R7 rule), so rejecting it is a no-op -//! for every real query today. `optimizer/engine.rs`'s R7 -//! `SubqueryDecorrelation` is deleted rather than ported — it pattern- -//! matched the removed `Predicate::BinaryOp` / `Predicate::ScalarSubquery` -//! / `Predicate::Column(ColumnRef::Named)` variants directly and has no -//! construction site left to fire against. Real correlated-subquery -//! support is follow-up work once `asap_ir` grows a representation for -//! it. -//! - **`GROUP BY` keys attach directly to `Aggregate.by: GroupKeys`** -//! instead of wrapping the result in a `Partition` node (removed from -//! `asap_ir`'s `QueryExpr` — folded into `GroupKeys`'s `by`/`without` -//! distinction). The single-statistic fusion arm resolves `keys` once -//! and threads `GroupKeys` into every fused node (including both -//! siblings of a `StdDev`/`Variance` `Merge` fan-out) instead of -//! wrapping the finished shape afterward. A standalone legacy -//! `LQueryExpr::Partition` (not part of the fusion arm) folds its keys -//! into the nearest `Aggregate` its converted subtree contains, via -//! [`fold_partition_keys`]. +//! Two consequences of adopting `asap_l2::relational::QueryExpr`: //! -//! `having` is `Option` (real typed HAVING) directly — no -//! translation needed beyond running the HAVING expression through -//! [`convert_scalar`] like any other predicate. +//! - **No more hand-rolled scalar conversion.** `ScalarExpr` is gone — +//! every scalar position carries the shared `L2Expr` directly (same +//! generic `Expr` the canonical tree's `L3Expr` is), so +//! `column_resolution::resolve_expr` does the whole +//! name→position resolution in one generic pass. `between()`, +//! `binary_scalar_op`, and `literal_from_legacy` (all present before +//! this step) are gone with it — nothing left for them to do. +//! - **No more `Partition`, at L2 or L3.** `Aggregate` carries +//! `without: bool` directly (`asap_l2`'s own step-3-equivalent design +//! choice) — `fold_partition_keys` and the standalone +//! `LQueryExpr::Partition` arm (both present before this step) are +//! gone with it. //! -//! ## Variant mapping +//! `ScalarSubquery` no longer exists as a concept at all — `asap_l2`'s +//! `relational::QueryExpr` has no such variant (its `Ref`/`LetBinding` +//! are "Reserved: no front end emits yet", same as this repo's own +//! pre-merge state) — so `ConvertError::UnsupportedScalarSubquery` (this +//! step's predecessor) has nothing left to reject; removed. //! -//! | relational `QueryExpr` | canonical `QueryExpr` | -//! |---|---| -//! | `Source(spec)` | `Scan { TimeSeries, predicates: [], schema }` | -//! | `Ref(name)` | `Ref { name }` | -//! | `Filter` | `Filter` (pred via [`convert_scalar`]) | -//! | `Project` | `Project` (each item's expr via [`convert_scalar`])| -//! | `Aggregate` | single agg + no HAVING → *fuses* (see below); otherwise plain `Aggregate` (keys→by: GroupKeys, AggFunc→AggIntent via [`agg_func_to_intents`]) | -//! | `Window` | `Window` (slide → Sliding else Tumbling) | -//! | `Partition` | keys folded into the nearest `Aggregate.by` inside the converted subtree, via [`fold_partition_keys`] | -//! | `Distinct` | `Distinct` | -//! | `TopK` | `Aggregate { aggs: [AggIntent::TopK] }` (HeavyHitter)| -//! | `Merge` | `Merge` | -//! | `Join` | `Join` (None pred → `Literal(Bool(true))`) | -//! | `SetOp` | `SetOp` | -//! | `Sort` | `Sort` (keys pass through — `relational::SortKey` already re-exports the canonical, `L3Expr`-based type) | -//! | `Limit` | `Limit` | -//! | `LetBinding` | `LetBinding` (relational `body` → canonical `child`)| -//! | `PromQLSubquery` | `Subquery` | -//! | `BinaryOp` | `BinaryOp` (`op` passes straight through — `relational::BinaryOpKind` is a re-export of the canonical type, not a separate flat enum) | +//! ## `Frequency` preservation (see `relational.rs`'s module doc) //! -//! A single-statistic `Aggregate` (exactly one `AggItem`, no `HAVING`) -//! fuses directly into canonical shape rather than staying a plain -//! `Aggregate` wrapping the untouched child: -//! * input is a `Window` → emit `Window { Aggregate { by } } }` (the -//! window-defines-sketch-lifecycle shape); -//! * otherwise → emit `Aggregate { by }`; -//! * `StdDev` / `Variance` fan out into a `Merge` of two sibling -//! quantile aggregates, both carrying the same `by` (Step α F1 -//! strategy) — the only `AggFunc`s [`agg_func_to_intents`] maps to -//! more than one `AggIntent`; -//! * `GROUP BY` keys resolve once and thread into every fused node's -//! `by: GroupKeys` directly. -//! `AggFunc::Custom` produces no canonical intent regardless of arity — -//! [`agg_func_to_intents`] returns empty, which raises -//! [`ConvertError::NoCanonicalIntent`] once execution reaches the plain -//! multi-agg path (single-agg-with-empty-intents falls through to it -//! rather than being special-cased inline). +//! `count_over_time(...)` (and the PromQL `topk` bridge's synthetic +//! inner count) must still route through a CMS/CountSketch-family +//! `AggIntent::Extension` rather than an exact `AggIntent::Count`, but +//! `AggFunc::Frequency` no longer exists as a distinct variant to key +//! off of — the frontend now constructs plain `AggFunc::Count` for both +//! cases (matching `asap_l2`'s own frontend-promql, which leaves the +//! sketch-vs-exact choice to L4). The trigger `agg_func_to_intents` uses +//! instead: **`Count` is a `Frequency` candidate when its `Aggregate` is +//! grouped (`by`/`without` non-trivial) OR its input is an `_over_time` +//! `Window`** — precisely the two shapes `query_parser::promql` +//! constructs a windowed `Count` from (the topk bridge is grouped by the +//! topk's own `by` keys; a bare `count_over_time(...)` is windowed but +//! typically ungrouped). An un-windowed, ungrouped `Count` (SQL +//! `COUNT(*)`, not in scope for this PromQL-only step) stays exact. //! //! ## Schema threading //! @@ -100,21 +76,19 @@ #![allow(dead_code)] +use asap_ir::intent_algebra::query_expr::InfoMatcher; use asap_ir::intent_algebra::BindingName; use crate::intent_algebra::agg_intent::AggIntent; use crate::intent_algebra::binder::Binder; use crate::intent_algebra::column_resolution::{ - resolve_column_ref, resolve_column_refs, resolve_named_keys, ResolveError, + resolve_column_refs, resolve_expr, resolve_group_keys_promql, ResolveError, }; use crate::intent_algebra::query_expr::{ - between, ArithOp, BinaryOpKind, CompareOp, GroupKeys, L3Scalar, Predicate, - ProjectItem as CProjectItem, QueryExpr as CQueryExpr, Source, WindowKind as CWindowKind, -}; -use crate::intent_algebra::relational::{ - AggFunc, ColumnRef as LColumnRef, PartitionKeys as LPartitionKeys, QueryExpr as LQueryExpr, - ScalarExpr as LScalarExpr, + GroupKeys, L3Scalar, Predicate, ProjectItem as CProjectItem, QueryExpr as CQueryExpr, + QueryExprError, SortKey as CSortKey, Source, WindowKind as CWindowKind, }; +use crate::intent_algebra::relational::{AggFunc, QueryExpr as LQueryExpr}; use crate::intent_algebra::schema::Schema; use crate::intent_algebra::L3Expr; use crate::types_v2::AccuracyTarget; @@ -126,14 +100,21 @@ pub enum ConvertError { /// `Column` leaf) did not resolve against the inherited schema. #[error("column resolution failed: {0}")] Resolve(#[from] ResolveError), - /// An `AggItem.func` has no canonical `AggIntent` equivalent — only - /// `AggFunc::Custom(_)` triggers this today. - #[error("AggItem `{alias}` uses non-canonical func ({func_dbg}) — no AggIntent equivalent")] - NoCanonicalIntent { alias: String, func_dbg: String }, - /// A `ScalarExpr::ScalarSubquery` was encountered. See the module doc - /// for why this is rejected rather than lowered. - #[error("scalar subqueries are not supported by the canonical IR yet")] - UnsupportedScalarSubquery, + /// An `AggItem.func` has no canonical `AggIntent` equivalent. No + /// `AggFunc` variant reaches this today (every one maps to + /// something) — kept as the escape hatch's shape for a future + /// extension point, matching `asap_l2`'s own error surface. + #[error("AggItem `{alias:?}` uses non-canonical func ({func_dbg}) — no AggIntent equivalent")] + NoCanonicalIntent { + alias: Option, + func_dbg: String, + }, + /// Schema derivation over a converted subtree failed (surfaced by + /// `column_resolution::output_schema_for_aggregate` callers, not by + /// `convert` itself today — kept for API-shape parity with + /// `asap_l2::lower::ConvertError`). + #[error("schema derivation failed: {0}")] + Schema(#[from] QueryExprError), } /// Lower a legacy Layer-2 `QueryExpr` tree to the canonical L3 IR. @@ -147,14 +128,57 @@ pub fn convert_root(legacy: &LQueryExpr) -> Result { /// see the module doc on schema threading. pub fn convert(legacy: &LQueryExpr, schema: &Schema) -> Result { Ok(match legacy { - LQueryExpr::Source(spec) => CQueryExpr::Scan { - source: Source::TimeSeries { - metric: spec.name.clone(), - }, - predicates: Vec::new(), - // Carry the Binder's complete schema — the same self-contained - // scope every `ColumnId` in this tree resolves against. - schema: schema.clone(), + LQueryExpr::Source(spec) => { + let scan = match &spec.schema { + Some(sql_schema) => CQueryExpr::Scan { + source: Source::Table { + table_ref: spec.name.clone(), + }, + predicates: Vec::new(), + schema: sql_schema.clone(), + }, + None => CQueryExpr::Scan { + source: Source::TimeSeries { + metric: spec.name.clone(), + }, + predicates: Vec::new(), + // Carry the Binder's complete schema — the same + // self-contained scope every `ColumnId` in this tree + // resolves against. + schema: schema.clone(), + }, + }; + if spec.shift.is_identity() { + scan + } else { + CQueryExpr::TimeShift { + shift: spec.shift, + child: Box::new(scan), + } + } + } + + LQueryExpr::Scalar(v) => CQueryExpr::Scalar(*v), + LQueryExpr::EvalTime => CQueryExpr::EvalTime, + LQueryExpr::VectorFromScalar(input) => { + CQueryExpr::VectorFromScalar(Box::new(convert(input, schema)?)) + } + LQueryExpr::ScalarFromVector(input) => { + CQueryExpr::ScalarFromVector(Box::new(convert(input, schema)?)) + } + LQueryExpr::Relabel { dst, value, input } => CQueryExpr::Relabel { + dst: dst.clone(), + value: resolve_expr(value, schema)?, + child: Box::new(convert(input, schema)?), + }, + LQueryExpr::Sample { keys, kind, input } => CQueryExpr::Sample { + by: resolve_column_refs(keys, schema)?.into(), + kind: *kind, + child: Box::new(convert(input, schema)?), + }, + LQueryExpr::InfoJoin { selector, input } => CQueryExpr::InfoJoin { + selector: selector.clone(), + child: Box::new(convert(input, schema)?), }, LQueryExpr::Ref(name) => CQueryExpr::Ref { @@ -162,41 +186,54 @@ pub fn convert(legacy: &LQueryExpr, schema: &Schema) -> Result CQueryExpr::Filter { - pred: convert_scalar(pred, schema)?, + pred: Predicate(resolve_expr(pred, schema)?), child: Box::new(convert(input, schema)?), }, - LQueryExpr::Project { cols, input } => CQueryExpr::Project { + LQueryExpr::Project { + cols, + qualifier, + input, + } => CQueryExpr::Project { cols: cols .iter() - .map(|pi| { + .map(|item| { Ok(CProjectItem { - alias: pi.alias.clone(), - expr: convert_scalar(&pi.expr, schema)?.0, + alias: item.alias.clone(), + expr: resolve_expr(&item.expr, schema)?, }) }) .collect::, ConvertError>>()?, - qualifier: None, + qualifier: qualifier.clone(), child: Box::new(convert(input, schema)?), }, LQueryExpr::Aggregate { keys, + without, aggs, having, input, } => { - let by: GroupKeys = resolve_named_keys(keys, schema)?.into(); + let by: GroupKeys = if *without { + GroupKeys::without(resolve_column_refs(keys, schema)?) + } else { + resolve_group_keys_promql(keys, schema)?.into() + }; + // See the module doc's "Frequency preservation" section — a + // grouped-or-windowed `Count` is a `Frequency` candidate. + let windowed = matches!(input.as_ref(), LQueryExpr::Window { .. }); + let frequency_trigger = !by.is_empty() || windowed; - // A single-statistic aggregate *fuses* — this is the former - // `legacy_lower::lower_aggregate` step, done directly in + // A single-statistic aggregate *fuses* — done directly in // canonical terms rather than via an intermediate legacy L3 - // node. `Custom` (empty `intents`) falls through to the plain - // multi-agg path below, which raises `NoCanonicalIntent` - // uniformly for every arity. + // node. Empty `intents` is structurally unreachable (every + // `AggFunc` maps to something), so it always falls through + // to the plain path below rather than being special-cased + // inline. if aggs.len() == 1 && having.is_none() { let item = &aggs[0]; - let intents = agg_func_to_intents(&item.func, !keys.is_empty()); + let intents = agg_func_to_intents(&item.func, frequency_trigger); if !intents.is_empty() { let nodes: Vec = match input.as_ref() { LQueryExpr::Window { @@ -248,10 +285,11 @@ pub fn convert(legacy: &LQueryExpr, schema: &Schema) -> Result = Vec::with_capacity(aggs.len()); for item in aggs { - let mapped = agg_func_to_intents(&item.func, !keys.is_empty()); + let mapped = agg_func_to_intents(&item.func, frequency_trigger); if mapped.is_empty() { return Err(ConvertError::NoCanonicalIntent { alias: item.alias.clone(), @@ -262,7 +300,7 @@ pub fn convert(legacy: &LQueryExpr, schema: &Schema) -> Result Result { - let by: GroupKeys = match keys { - LPartitionKeys::By(k) => resolve_named_keys(k, schema)?.into(), - LPartitionKeys::Without(k) => GroupKeys::without(resolve_named_keys(k, schema)?), - }; - let converted = convert(input, schema)?; - fold_partition_keys(converted, by) - } - LQueryExpr::Distinct { cols, input } => CQueryExpr::Distinct { cols: resolve_column_refs(cols, schema)?, child: Box::new(convert(input, schema)?), @@ -306,7 +335,7 @@ pub fn convert(legacy: &LQueryExpr, schema: &Schema) -> Result Result CQueryExpr::Join { kind: kind.clone(), pred: match pred { - Some(se) => convert_scalar(se, schema)?, + Some(p) => Predicate(resolve_expr(p, schema)?), // Canonical `Join` requires a predicate; a legacy `None` // pred is a CROSS JOIN — model it as the tautology `true`. None => Predicate(L3Expr::Literal(L3Scalar::Boolean(true))), @@ -355,9 +384,22 @@ pub fn convert(legacy: &LQueryExpr, schema: &Schema) -> Result CQueryExpr::Sort { - keys: keys.clone(), - partition_by: GroupKeys::none(), + LQueryExpr::Sort { + keys, + partition_by, + input, + } => CQueryExpr::Sort { + keys: keys + .iter() + .map(|k| { + Ok(CSortKey { + expr: resolve_expr(&k.expr, schema)?, + ascending: k.ascending, + nulls_first: k.nulls_first, + }) + }) + .collect::, ConvertError>>()?, + partition_by: resolve_column_refs(partition_by, schema)?.into(), child: Box::new(convert(input, schema)?), }, @@ -384,6 +426,34 @@ pub fn convert(legacy: &LQueryExpr, schema: &Schema) -> Result CQueryExpr::WindowFunc { + func: func.clone(), + args: args + .iter() + .map(|a| resolve_expr(a, schema)) + .collect::, ResolveError>>()?, + partition_by: resolve_column_refs(partition_by, schema)?.into(), + order_by: order_by + .iter() + .map(|k| { + Ok(CSortKey { + expr: resolve_expr(&k.expr, schema)?, + ascending: k.ascending, + nulls_first: k.nulls_first, + }) + }) + .collect::, ConvertError>>()?, + output_name: output_name.clone(), + child: Box::new(convert(input, schema)?), + }, + LQueryExpr::BinaryOp { op, lhs, @@ -398,223 +468,103 @@ pub fn convert(legacy: &LQueryExpr, schema: &Schema) -> Result CQueryExpr { - match qe { - CQueryExpr::Aggregate { - aggs, - output_names, - having, - child, - .. - } => CQueryExpr::Aggregate { - by, - aggs, - output_names, - having, - child, - }, - CQueryExpr::Window { - kind, - size, - slide, - child, - } => CQueryExpr::Window { - kind, - size, - slide, - child: Box::new(fold_partition_keys(*child, by)), - }, - CQueryExpr::Merge { children } => CQueryExpr::Merge { - children: children - .into_iter() - .map(|c| fold_partition_keys(c, by.clone())) - .collect(), - }, - other => { - debug_assert!( - false, - "Partition over a non-aggregate shape has no GroupKeys home: {other:?}" - ); - other - } - } -} - -/// Translate a legacy `ScalarExpr` to a canonical `Predicate`, recursing -/// through every composite variant. -pub fn convert_scalar(se: &LScalarExpr, schema: &Schema) -> Result { - match se { - LScalarExpr::ScalarSubquery(_) => Err(ConvertError::UnsupportedScalarSubquery), - LScalarExpr::BinaryOp { op, lhs, rhs } => { - let l = convert_scalar(lhs, schema)?.0; - let r = convert_scalar(rhs, schema)?.0; - Ok(binary_scalar_op(op, l, r)) - } - LScalarExpr::IsNull { expr, negated } => { - let inner = Box::new(convert_scalar(expr, schema)?.0); - Ok(Predicate(if *negated { - L3Expr::IsNotNull(inner) - } else { - L3Expr::IsNull(inner) - })) - } - LScalarExpr::FunctionCall { name, args } => { - let exprs = args - .iter() - .map(|a| Ok(convert_scalar(a, schema)?.0)) - .collect::, ConvertError>>()?; - Ok(Predicate(L3Expr::FunctionCall { - name: name.clone(), - args: exprs, - })) - } - LScalarExpr::InList { - expr, - list, - negated, - } => { - let e = convert_scalar(expr, schema)?.0; - let list_exprs = list - .iter() - .map(|a| Ok(convert_scalar(a, schema)?.0)) - .collect::, ConvertError>>()?; - Ok(Predicate(L3Expr::InList { - expr: Box::new(e), - list: list_exprs, - negated: *negated, - })) - } - LScalarExpr::Between { - expr, - low, - high, - negated, - } => { - let e = convert_scalar(expr, schema)?.0; - let l = convert_scalar(low, schema)?.0; - let h = convert_scalar(high, schema)?.0; - Ok(Predicate(between(e, l, h, *negated))) - } - LScalarExpr::Column(name) => { - let id = resolve_column_ref(&LColumnRef::Named(name.clone()), schema)?; - Ok(Predicate(L3Expr::Column(id))) - } - LScalarExpr::Literal(lit) => Ok(Predicate(literal_from_legacy(lit))), - } -} - -/// Translate a legacy `ScalarExpr::BinaryOp`'s operator + converted -/// operands into the corresponding `L3Expr`. `op` is already the -/// canonical `BinaryOpKind` (`relational::BinaryOpKind` re-exports the -/// same type `query_expr::BinaryOp.op` carries — no separate flat legacy -/// enum exists), so this is a structural unwrap, not a value mapping. -fn binary_scalar_op(op: &BinaryOpKind, l: L3Expr, r: L3Expr) -> Predicate { - let e = match op { - BinaryOpKind::Arith(arith_op) => L3Expr::Arith { - op: arith_op.clone(), - left: Box::new(l), - right: Box::new(r), - }, - BinaryOpKind::Compare(cmp_op) => L3Expr::Compare { - left: Box::new(l), - op: cmp_op.clone(), - right: Box::new(r), - }, - BinaryOpKind::And => L3Expr::BoolAnd(vec![l, r]), - BinaryOpKind::Or => L3Expr::BoolOr(vec![l, r]), - // `Unless` / `Pow` / `Atan2` are PromQL vector-set / power ops with - // no scalar-predicate counterpart; neither parser constructs a - // `ScalarExpr::BinaryOp` with one of these (they only appear on - // `QueryExpr::BinaryOp`, passed straight through in `convert`). - // Defensive fallback rather than a panic on unreachable input. - BinaryOpKind::Unless | BinaryOpKind::Pow | BinaryOpKind::Atan2 => { - L3Expr::Literal(L3Scalar::Boolean(true)) - } - }; - Predicate(e) -} - -fn literal_from_legacy(lit: &crate::intent_algebra::relational::LiteralValue) -> L3Expr { - use crate::intent_algebra::relational::LiteralValue as L; - L3Expr::Literal(match lit { - L::Null => L3Scalar::Null, - L::Bool(b) => L3Scalar::Boolean(*b), - L::Int(i) => L3Scalar::Int64(*i), - L::Float(f) => L3Scalar::Float64(*f), - L::Str(s) => L3Scalar::Utf8(s.clone()), - // No L3Scalar counterpart -- fold to nanosecond count, matching - // the pre-merge `from_legacy_scalar`'s documented behavior. - L::Duration(d) => L3Scalar::Int64(d.as_nanos() as i64), - }) -} - // ── AggFunc → AggIntent sketch mapping ─────────────────────────────────────── /// Map an [`AggFunc`] to the canonical [`AggIntent`]s the `convert` -/// `Aggregate` arm fuses on. `grouped` is `!keys.is_empty()` at the call -/// site. Empty for non-canonical functions (`Custom` only); one intent -/// for every ordinary function (delegates to [`AggFunc::to_sketch_op`]); -/// two for the `StdDev` / `Variance` fan-out (the caller wraps the pair -/// in a `Merge` of sibling sketch aggregates, the one case `to_sketch_op` -/// can't express since it returns a single `Option`). -fn agg_func_to_intents(func: &AggFunc, grouped: bool) -> Vec { +/// `Aggregate` arm fuses on. `frequency_trigger` is `true` when the +/// enclosing `Aggregate` is grouped or windowed — see the module doc's +/// "Frequency preservation" section; it only affects the `Count` arm. +/// One intent for every ordinary function; two for the `StdDev` / +/// `Variance` fan-out (the caller wraps the pair in a `Merge` of sibling +/// sketch aggregates). +fn agg_func_to_intents(func: &AggFunc, frequency_trigger: bool) -> Vec { + let q = |q: f64| AggIntent::Quantile { + col: None, + q, + accuracy: AccuracyTarget::Epsilon(0.01), + }; match func { - AggFunc::StdDev { .. } | AggFunc::Variance { .. } => vec![ - AggIntent::Quantile { - col: None, - q: 0.25, - accuracy: AccuracyTarget::Epsilon(0.01), - }, - AggIntent::Quantile { - col: None, - q: 0.75, - accuracy: AccuracyTarget::Epsilon(0.01), - }, - ], - // `Rate` / `Increase` / `Delta` map onto `AggIntent::Sum`, not - // the dedicated `AggIntent::Rate` / `Increase` / `Delta` - // variants `to_sketch_op()` would otherwise reach for — the - // `asap_tier_analysis` engine dispatch deliberately collapses - // all of `rate` / `irate` / `increase` / `sum_over_time` onto - // one `Capability::ExactAgg(Sum)` and disambiguates via the - // separate typed `outer_fn` field instead (see - // `asap_tier_analysis::tests::rate_and_sum_over_time_share_ - // capability_but_differ_on_outer_fn` and the surrounding - // "outer_fn — rate vs plain disambiguation" test block, which - // documents this as the deliberate replacement for a retired - // raw-PromQL re-parser). Using the dedicated intents here would - // fragment that dispatch. - AggFunc::Rate | AggFunc::Increase | AggFunc::Delta => vec![AggIntent::Sum { col: None }], + AggFunc::Count if frequency_trigger => vec![crate::intent_algebra::default_frequency()], + AggFunc::Count => vec![AggIntent::Count { + accuracy: AccuracyTarget::Exact, + }], + AggFunc::Sum => vec![AggIntent::Sum { col: None }], // `Avg` approximates as the p50 (median) quantile sketch rather - // than `to_sketch_op()`'s literal `AggIntent::Avg` (exact, - // non-mergeable) — matches `Min`/`Max`'s boundary-quantile - // treatment (`AggIntent::Min` ~ q=0.0, `Max` ~ q=1.0) and is what + // than a literal (exact, non-mergeable) `AggIntent::Avg` — + // matches `Min`/`Max`'s boundary-quantile treatment and is what // `query_parser::QeCollector::collect_op` (which has no `Avg` // arm of its own) relies on to classify `avg_over_time` as // `AggType::Quantile` with `quantiles: [0.5]`. - AggFunc::Avg => vec![AggIntent::Quantile { - col: None, - q: 0.5, - accuracy: AccuracyTarget::Epsilon(0.01), + AggFunc::Avg => vec![q(0.5)], + AggFunc::Min => vec![AggIntent::Min { col: None }], + AggFunc::Max => vec![AggIntent::Max { col: None }], + AggFunc::StdDev { .. } | AggFunc::Variance { .. } => vec![q(0.25), q(0.75)], + AggFunc::Quantile(phi) => vec![q(*phi)], + AggFunc::CountDistinct => vec![crate::intent_algebra::default_cardinality()], + AggFunc::HeavyHitters { .. } => vec![crate::intent_algebra::default_frequency()], + // `Rate` / `Increase` / `Delta` map onto `AggIntent::Sum`, not the + // dedicated `AggIntent::Rate` / `Increase` / `Delta` variants — + // the `asap_tier_analysis` engine dispatch deliberately collapses + // all of `rate` / `irate` / `increase` / `sum_over_time` onto one + // `Capability::ExactAgg(Sum)` and disambiguates via the separate + // typed `outer_fn` field instead (see + // `asap_tier_analysis::tests::rate_and_sum_over_time_share_ + // capability_but_differ_on_outer_fn` and the surrounding + // "outer_fn — rate vs plain disambiguation" test block). Using + // the dedicated intents here would fragment that dispatch. + // `Changes`/`Resets` also collapse onto `Count` for the same + // reason (this repo's pre-`asap_l2`-merge `AggFunc` never + // distinguished them from `Count` either) -- and everything else + // in the "counter-derivative range functions" family collapses + // onto `Delta`, matching this repo's pre-merge behavior exactly. + // Properly distinguishing all of these is deferred to the + // PromQL-frontend semantic-retarget step, alongside the + // `outer_fn` reconciliation above. + AggFunc::Rate { .. } | AggFunc::Increase { .. } => vec![AggIntent::Sum { col: None }], + AggFunc::Changes | AggFunc::Resets => { + agg_func_to_intents(&AggFunc::Count, frequency_trigger) + } + AggFunc::Delta + | AggFunc::IDelta + | AggFunc::Deriv + | AggFunc::PredictLinear { .. } + | AggFunc::DoubleExpSmoothing { .. } => vec![AggIntent::Sum { col: None }], + // Every remaining `AggFunc` (native-histogram accessors, + // math/trig, presence, time/calendar, `Group`/`CountValues`, + // the extended range-vector reducers) has no pre-`asap_l2`-merge + // equivalent in this repo's PromQL surface at all -- promql.rs + // doesn't construct any of them today (`walk_call_to_op`'s + // exhaustive function-name table has no arm reaching them), so + // there's no existing behavior to preserve. Map each directly + // onto its like-named `AggIntent` (all archive-only per + // `agg_intent::archive_only`, so this is inert until a real + // caller constructs one). + AggFunc::HistogramCount => vec![AggIntent::HistogramCount], + AggFunc::HistogramSum => vec![AggIntent::HistogramSum], + AggFunc::HistogramAvg => vec![AggIntent::HistogramAvg], + AggFunc::HistogramStdDev => vec![AggIntent::HistogramStdDev], + AggFunc::HistogramStdVar => vec![AggIntent::HistogramStdVar], + AggFunc::HistogramFraction { lower, upper } => vec![AggIntent::HistogramFraction { + lower: *lower, + upper: *upper, }], - // A *grouped* `Count` is `count_over_time(...) by (...)` (or the - // PromQL `topk` bridge's synthetic `Count` — see - // `query_parser::promql::build_windowed_agg`'s "Count-with- - // GROUP-BY → Frequency" comment) — structurally per-series and - // sketchable, so it takes the same `Frequency`/CMS path as - // `AggFunc::Frequency`/`HeavyHitters`. An *ungrouped* `Count` is - // the SQL `COUNT(*)` exact-row-count case and keeps - // `to_sketch_op()`'s literal `AggIntent::Count{Exact}`. - AggFunc::Count if grouped => vec![crate::intent_algebra::default_frequency()], - other => other.to_sketch_op().into_iter().collect(), + AggFunc::HistogramQuantile(phi) => vec![AggIntent::HistogramQuantile { q: *phi }], + AggFunc::Math(f) => vec![AggIntent::Math(f.clone())], + AggFunc::Absent => vec![AggIntent::Absent], + AggFunc::AbsentOverTime => vec![AggIntent::AbsentOverTime], + AggFunc::PresentOverTime => vec![AggIntent::PresentOverTime], + AggFunc::TimeFn(f) => vec![AggIntent::TimeFn(f.clone())], + AggFunc::Group => vec![AggIntent::Group], + AggFunc::CountValues { label } => vec![AggIntent::CountValues { + label: label.clone(), + }], + AggFunc::LastOverTime => vec![AggIntent::LastOverTime], + AggFunc::FirstOverTime => vec![AggIntent::FirstOverTime], + AggFunc::MadOverTime => vec![AggIntent::MadOverTime], + AggFunc::TsOfMinOverTime => vec![AggIntent::TsOfMinOverTime], + AggFunc::TsOfMaxOverTime => vec![AggIntent::TsOfMaxOverTime], + AggFunc::TsOfFirstOverTime => vec![AggIntent::TsOfFirstOverTime], + AggFunc::TsOfLastOverTime => vec![AggIntent::TsOfLastOverTime], } } @@ -623,21 +573,33 @@ fn agg_func_to_intents(func: &AggFunc, grouped: bool) -> Vec { #[cfg(test)] mod tests { use super::*; - use crate::intent_algebra::relational::{ - AggItem, ColumnRef as LColumnRef, ProjectItem as LProjectItem, SourceSpec, - }; + use crate::intent_algebra::relational::{AggItem, ColumnRef as LColumnRef, SourceSpec}; use std::time::Duration; fn src(name: &str) -> LQueryExpr { - LQueryExpr::Source(SourceSpec { name: name.into() }) + LQueryExpr::Source(SourceSpec::new(name)) } fn agg_item(alias: &str, func: AggFunc) -> AggItem { AggItem { - alias: alias.into(), + alias: Some(alias.into()), func, col: LColumnRef::SampleValue, - distinct: false, + } + } + + fn agg( + keys: Vec, + without: bool, + aggs: Vec, + input: LQueryExpr, + ) -> LQueryExpr { + LQueryExpr::Aggregate { + keys, + without, + aggs, + having: None, + input: Box::new(input), } } @@ -678,16 +640,15 @@ mod tests { #[test] fn window_over_aggregate_full_tree() { - // Window { Aggregate { keys: [], aggs: [Sum], Source } } let legacy = LQueryExpr::Window { duration: Duration::from_secs(300), slide: None, - input: Box::new(LQueryExpr::Aggregate { - keys: vec![], - aggs: vec![agg_item("s", AggFunc::Sum)], - having: None, - input: Box::new(src("m")), - }), + input: Box::new(agg( + vec![], + false, + vec![agg_item("s", AggFunc::Sum)], + src("m"), + )), }; match convert_root(&legacy).unwrap() { CQueryExpr::Window { @@ -707,30 +668,9 @@ mod tests { } } - #[test] - fn sliding_window_carries_slide() { - let legacy = LQueryExpr::Window { - duration: Duration::from_secs(600), - slide: Some(Duration::from_secs(60)), - input: Box::new(src("m")), - }; - match convert_root(&legacy).unwrap() { - CQueryExpr::Window { kind, slide, .. } => { - assert_eq!(kind, CWindowKind::Sliding); - assert_eq!(slide, Some(Duration::from_secs(60))); - } - other => panic!("expected Window, got {other:?}"), - } - } - #[test] fn single_aggregate_folds_to_canonical_aggregate() { - let legacy = LQueryExpr::Aggregate { - keys: vec![], - aggs: vec![agg_item("s", AggFunc::Sum)], - having: None, - input: Box::new(src("m")), - }; + let legacy = agg(vec![], false, vec![agg_item("s", AggFunc::Sum)], src("m")); match convert_root(&legacy).unwrap() { CQueryExpr::Aggregate { by, aggs, .. } => { assert!(by.is_empty(), "no GROUP BY → empty `by`: {by:?}"); @@ -741,13 +681,8 @@ mod tests { } #[test] - fn ungrouped_count_is_exact() { - let legacy = LQueryExpr::Aggregate { - keys: vec![], - aggs: vec![agg_item("n", AggFunc::Count)], - having: None, - input: Box::new(src("m")), - }; + fn ungrouped_unwindowed_count_is_exact() { + let legacy = agg(vec![], false, vec![agg_item("n", AggFunc::Count)], src("m")); match convert_root(&legacy).unwrap() { CQueryExpr::Aggregate { aggs, .. } => assert!(matches!( aggs.as_slice(), @@ -759,19 +694,60 @@ mod tests { } } + #[test] + fn windowed_count_is_frequency() { + // Mirrors `count_over_time(m[5m])`: no GROUP BY, but windowed. + let legacy = agg( + vec![], + false, + vec![agg_item("n", AggFunc::Count)], + LQueryExpr::Window { + duration: Duration::from_secs(300), + slide: None, + input: Box::new(src("m")), + }, + ); + match convert_root(&legacy).unwrap() { + CQueryExpr::Window { child, .. } => match *child { + CQueryExpr::Aggregate { aggs, .. } => { + assert!(matches!(aggs.as_slice(), [AggIntent::Extension { .. }])); + assert!(crate::intent_algebra::as_frequency(&aggs[0]).is_some()); + } + other => panic!("expected Aggregate, got {other:?}"), + }, + other => panic!("expected Window, got {other:?}"), + } + } + + #[test] + fn grouped_unwindowed_count_is_frequency() { + // Mirrors the PromQL `topk` bridge's synthetic grouped Count. + let legacy = agg( + vec![LColumnRef::Named("symbol".into())], + false, + vec![agg_item("n", AggFunc::Count)], + src("m"), + ); + match convert_root(&legacy).unwrap() { + CQueryExpr::Aggregate { aggs, .. } => { + assert!(crate::intent_algebra::as_frequency(&aggs[0]).is_some()); + } + other => panic!("expected Aggregate, got {other:?}"), + } + } + #[test] fn aggregate_target_column_is_not_a_group_by_key() { - let legacy = LQueryExpr::Aggregate { - keys: vec![], - aggs: vec![AggItem { - alias: "s".into(), + let legacy = agg( + vec![], + false, + vec![AggItem { + alias: Some("s".into()), func: AggFunc::Sum, col: LColumnRef::Named("price".into()), - distinct: false, }], - having: None, - input: Box::new(src("trades")), - }; + src("trades"), + ); match convert_root(&legacy).unwrap() { CQueryExpr::Aggregate { by, .. } => assert!(by.is_empty()), other => panic!("expected Aggregate, got {other:?}"), @@ -780,16 +756,16 @@ mod tests { #[test] fn single_aggregate_over_window_folds_to_window_over_aggregate() { - let legacy = LQueryExpr::Aggregate { - keys: vec![], - aggs: vec![agg_item("q", AggFunc::Quantile(0.99))], - having: None, - input: Box::new(LQueryExpr::Window { + let legacy = agg( + vec![], + false, + vec![agg_item("q", AggFunc::Quantile(0.99))], + LQueryExpr::Window { duration: Duration::from_secs(300), slide: None, input: Box::new(src("m")), - }), - }; + }, + ); match convert_root(&legacy).unwrap() { CQueryExpr::Window { kind, child, .. } => { assert_eq!(kind, CWindowKind::Tumbling); @@ -806,22 +782,20 @@ mod tests { #[test] fn stddev_fans_out_into_merge_of_quantile_siblings() { - let legacy = LQueryExpr::Aggregate { - keys: vec![], - aggs: vec![agg_item("sd", AggFunc::StdDev { population: false })], - having: None, - input: Box::new(src("m")), - }; + let legacy = agg( + vec![], + false, + vec![agg_item("sd", AggFunc::StdDev { population: false })], + src("m"), + ); match convert_root(&legacy).unwrap() { CQueryExpr::Merge { children } => { assert_eq!(children.len(), 2); for c in &children { assert!(matches!( c, - CQueryExpr::Aggregate { - aggs, - .. - } if matches!(aggs.as_slice(), [AggIntent::Quantile { .. }]) + CQueryExpr::Aggregate { aggs, .. } + if matches!(aggs.as_slice(), [AggIntent::Quantile { .. }]) )); } } @@ -829,11 +803,31 @@ mod tests { } } + #[test] + fn rate_and_increase_map_to_sum() { + for func in [ + AggFunc::Rate { + window: Duration::from_secs(300), + }, + AggFunc::Increase { + window: Duration::from_secs(300), + }, + ] { + let legacy = agg(vec![], false, vec![agg_item("r", func)], src("m")); + match convert_root(&legacy).unwrap() { + CQueryExpr::Aggregate { aggs, .. } => { + assert!(matches!(aggs.as_slice(), [AggIntent::Sum { col: None }])) + } + other => panic!("expected Aggregate, got {other:?}"), + } + } + } + #[test] fn topk_folds_into_aggregate_with_topk_intent() { let legacy = LQueryExpr::TopK { k: 5, - by: vec![].into(), + by: vec![], input: Box::new(src("m")), }; match convert_root(&legacy).unwrap() { @@ -845,41 +839,17 @@ mod tests { } } - #[test] - fn custom_agg_func_errors() { - let legacy = LQueryExpr::Aggregate { - keys: vec![], - aggs: vec![agg_item("u", AggFunc::Custom("my_udf".into()))], - having: None, - input: Box::new(src("m")), - }; - assert!(matches!( - convert_root(&legacy).unwrap_err(), - ConvertError::NoCanonicalIntent { .. } - )); - } - - #[test] - fn filter_pred_with_scalar_subquery_is_rejected() { - // ScalarSubquery has no canonical L3Expr representation (see the - // module doc) -- convert_scalar rejects it rather than guessing. - let legacy = LQueryExpr::Filter { - pred: LScalarExpr::ScalarSubquery(Box::new(LQueryExpr::Ref("cte".into()))), - input: Box::new(src("m")), - }; - assert!(matches!( - convert_root(&legacy).unwrap_err(), - ConvertError::UnsupportedScalarSubquery - )); - } - #[test] fn project_translates_each_item_expr() { + use crate::intent_algebra::relational::L2ProjectItem; + use crate::intent_algebra::L2Expr; + let legacy = LQueryExpr::Project { - cols: vec![LProjectItem { + cols: vec![L2ProjectItem { alias: Some("v".into()), - expr: LScalarExpr::Column("value".into()), + expr: L2Expr::Column(LColumnRef::SampleValue), }], + qualifier: None, input: Box::new(src("m")), }; match convert_root(&legacy).unwrap() { @@ -892,43 +862,6 @@ mod tests { } } - #[test] - fn binary_op_converts_both_sides() { - let legacy = LQueryExpr::BinaryOp { - op: BinaryOpKind::Arith(ArithOp::Add), - lhs: Box::new(src("a")), - rhs: Box::new(src("b")), - vector_match: None, - }; - match convert_root(&legacy).unwrap() { - CQueryExpr::BinaryOp { lhs, rhs, .. } => { - assert!(matches!(*lhs, CQueryExpr::Scan { .. })); - assert!(matches!(*rhs, CQueryExpr::Scan { .. })); - } - other => panic!("expected BinaryOp, got {other:?}"), - } - } - - #[test] - fn scalar_binary_op_translates_arith_and_compare() { - let legacy = LScalarExpr::BinaryOp { - op: BinaryOpKind::Compare(CompareOp::Gt), - lhs: Box::new(LScalarExpr::Column("value".into())), - rhs: Box::new(LScalarExpr::Literal( - crate::intent_algebra::relational::LiteralValue::Float(1.0), - )), - }; - let schema = crate::intent_algebra::column_resolution::infer_source_schema("m"); - let pred = convert_scalar(&legacy, &schema).unwrap(); - assert!(matches!( - pred.0, - L3Expr::Compare { - op: CompareOp::Gt, - .. - } - )); - } - #[test] fn cross_join_none_pred_becomes_true_literal() { let legacy = LQueryExpr::Join { @@ -962,42 +895,4 @@ mod tests { other => panic!("expected Subquery, got {other:?}"), } } - - #[test] - fn nested_tree_converts_recursively() { - // Sort { Limit { Filter { Window { Source } } } } — exercises a - // deep pass-through chain in one shot. - let legacy = LQueryExpr::Sort { - keys: vec![], - input: Box::new(LQueryExpr::Limit { - n: 10, - offset: 0, - input: Box::new(LQueryExpr::Filter { - pred: LScalarExpr::Literal( - crate::intent_algebra::relational::LiteralValue::Bool(true), - ), - input: Box::new(LQueryExpr::Window { - duration: Duration::from_secs(60), - slide: None, - input: Box::new(src("m")), - }), - }), - }), - }; - let c = convert_root(&legacy).unwrap(); - let CQueryExpr::Sort { child, .. } = c else { - panic!("expected Sort") - }; - let CQueryExpr::Limit { n, child, .. } = *child else { - panic!("expected Limit") - }; - assert_eq!(n, 10); - let CQueryExpr::Filter { child, .. } = *child else { - panic!("expected Filter") - }; - let CQueryExpr::Window { child, .. } = *child else { - panic!("expected Window") - }; - assert!(matches!(*child, CQueryExpr::Scan { .. })); - } } diff --git a/control_plane/src/intent_algebra/mod.rs b/control_plane/src/intent_algebra/mod.rs index 2043ae36..45bff7cf 100644 --- a/control_plane/src/intent_algebra/mod.rs +++ b/control_plane/src/intent_algebra/mod.rs @@ -128,5 +128,5 @@ pub use schema::{cse_reuse_is_legal, Column, ColumnId, CseError, DataType, Schem // at the point where they need a positional `ColumnId`. pub use column_resolution::{ infer_schema_for_root, infer_source_schema, output_schema_for_aggregate, resolve_column_ref, - resolve_column_refs, resolve_named_keys, ResolveError, + resolve_column_refs, resolve_expr, resolve_group_keys_promql, ResolveError, }; diff --git a/control_plane/src/intent_algebra/relational.rs b/control_plane/src/intent_algebra/relational.rs index 5688f8ab..627cc89c 100644 --- a/control_plane/src/intent_algebra/relational.rs +++ b/control_plane/src/intent_algebra/relational.rs @@ -2,169 +2,94 @@ //! `query_parser` front ends emit, before lowering to the canonical L3 //! `intent_algebra::query_expr` types. //! -//! This module defines two mutually recursive expression types: +//! ## Phase 2 step 4 (docs/migration-plan-backend-plan.md) //! -//! * [`QueryExpr`] — *relational* operators. Each node takes zero or more -//! relations as input and produces a relation. Maps to SQL's FROM / GROUP BY -//! / JOIN / UNION layer and PromQL's binary / sub-query layer. +//! `QueryExpr`, `AggFunc`, `AggItem`, `SourceSpec`, `L2ProjectItem`, +//! `L2SortKey` are no longer defined in this repo — re-exported from +//! `asap_l2::relational` (the `asap-l2` crate, same git dependency / +//! pinned commit as `asap-ir`). `asap_l2`'s L2 is a real superset: every +//! node has a positional-name-based (`L2Expr` / `ColumnRef`) twin of its +//! canonical L3 counterpart (`Scalar`, `EvalTime`, `VectorFromScalar`, +//! `ScalarFromVector`, `Relabel`, `Sample`, `InfoJoin`, `WindowFunc` are +//! new), and `ScalarExpr` (this repo's own 8-variant scalar enum) is gone +//! — every scalar position (`Filter.pred`, `Aggregate.having`, +//! `Join.pred`, `Project` items) now carries the shared `L2Expr` +//! (`expr_ir::Expr`, merged in Phase 2 step 2) directly, same +//! as the canonical L3 tree carries `L3Expr`. //! -//! * [`ScalarExpr`] — *scalar* operators. Each node computes a single value -//! from a row. Used for WHERE predicates, SELECT projections, HAVING -//! conditions, and JOIN conditions. +//! `Partition` doesn't exist at L2 either — `Aggregate` carries +//! `without: bool` directly (PromQL `without(...)` vs `by(...)`), matching +//! how L3's `GroupKeys` already modeled this after the `query_expr.rs` +//! merge (step 3). `PartitionKeys` is gone as a result. //! -//! # Stage vocabulary +//! `AggFunc::Frequency` and `AggFunc::Custom(String)` — this repo's own +//! extensions, with no `asap_l2::relational::AggFunc` equivalent — are +//! **not** ported forward as enum variants (`AggFunc` is a foreign type; +//! Rust's orphan rules forbid adding variants to it from this crate). +//! Per the tie-break rule (adopt ASAPController's version; genuinely +//! control_plane-only functionality gets ported onto it, not used to +//! keep this repo's copy), both are preserved as *behavior* in +//! `lower.rs` instead of as *vocabulary*: //! -//! Once the [`crate::physical::allocator::SketchAllocator`] annotates the tree, -//! every node carries a [`PipelineStage`](crate::physical::plan::PipelineStage) -//! tag that says where the work executes: -//! -//! | Stage | Component | -//! |-----------|----------------------------| -//! | Agent | OTel Collector at the SDK | -//! | Backend | Central merge collector | -//! | Precompute| ASAPQuery engine | -//! | Db | Backend OLAP / exact store | - -use std::time::Duration; - -use crate::intent_algebra::query_expr::{ArithOp, CompareOp}; -use crate::types_v2::AccuracyTarget; - -// ── AggIntent harmonization ────────────────────────────────────────────────── -// -// The `AggIntent` / `ExactAgg` enums that historically lived here have -// been deleted in favor of the canonical `intent_algebra::agg_intent::AggIntent` -// vocabulary. The translation table is documented in the migration spec; in -// short: -// -// Legacy Quantile { quantiles, accuracy } → fan-out into multiple -// canonical Quantile { q, accuracy } -// siblings (callers wrap them -// in a Merge node). -// Legacy Cardinality { accuracy: f64 } → canonical Cardinality -// { accuracy: AccuracyTarget } -// Legacy Frequency { accuracy: f64 } → canonical Frequency -// { accuracy: AccuracyTarget } -// Legacy Extrema { min: true, max: false } → canonical Min -// Legacy Extrema { min: false, max: true } → canonical Max -// Legacy Extrema { min: true, max: true } → fan-out into Min + Max siblings. -// Legacy Extrema { min: false, max: false } → translation error. -// Legacy Exact(Sum) → canonical Sum -// Legacy Exact(Count) → canonical Count { accuracy: AccuracyTarget::Exact } -// Legacy Exact(Avg) → canonical Avg -// Legacy Exact(Min) → canonical Min -// Legacy Exact(Max) → canonical Max -// Legacy PerPartition { inner, keys } → recurse on inner, then wrap in the -// `PerPartitionWrap` shape below. -// (PerPartition structural collapse -// to `Aggregate { by, aggs }` is -// Step γ's job.) -// -// PerPartition semantics ride on `PerPartitionWrap` (a thin legacy-only -// wrapper consumed by the `physical::sketch_catalog` per-partition sizing -// helpers). Free helpers (`agg_is_mergeable`, `agg_is_exact`, etc.) mirror -// what the old `AggIntent::method()` API used to provide so the migration -// is a typed search-and-replace rather than a semantic rewrite. +//! - **`Frequency`** (control_plane's per-series sample-count +//! estimation, routing to CMS/CountSketch): `query_parser::promql` now +//! constructs `AggFunc::Count` for `count_over_time(...)` (matching +//! `asap_l2`'s own frontend-promql, which also uses bare `Count` and +//! leaves the sketch-vs-exact choice to L4's `Bind*` rules for the +//! `TopK`-over-`Count` heavy-hitter shape). `lower.rs`'s +//! `agg_func_to_intents` recovers control_plane's original "windowed +//! Count → Frequency sketch, non-windowed Count → exact" split — the +//! `_over_time` window wrapper is exactly the same discriminator +//! `promql.rs`'s own comment documented for the pre-merge +//! `AggFunc::Frequency` construction, so this is a vocabulary +//! substitution, not a behavior change. See `lower.rs`'s module doc for +//! the "grouped OR windowed" precise trigger. +//! - **`Custom(String)`** (arbitrary named aggregate / UDA extension +//! point): zero real construction sites in this repo (grep-verified — +//! `AggFunc::Custom` appeared only in `lower.rs`'s own dead-code +//! handling and one unit test). Dropped outright rather than ported; +//! `lower.rs`'s `agg_func_to_intents` still returns +//! `ConvertError::NoCanonicalIntent` for any `AggFunc` variant it can't +//! map (structurally unreachable today since every `asap_l2::AggFunc` +//! variant maps to something), preserving the escape hatch's shape for +//! when a real extension need arises. + +pub use asap_ir::intent_algebra::expr_ir::{ColumnRef, L2Expr}; +pub use asap_ir::intent_algebra::query_expr::{ + BinaryOpKind, GroupSide, JoinKind, SampleKind, SetOpKind, VectorGrouping, VectorMatch, + VectorMatchKind, WindowFuncKind, +}; +pub use asap_l2::relational::{AggFunc, AggItem, L2ProjectItem, L2SortKey, QueryExpr, SourceSpec}; /// Canonical L3 aggregation intent. Re-exported here so existing -/// `relational::AggIntent` references keep working — the type is now the +/// `relational::AggIntent` references keep working — the type is the /// single canonical [`crate::intent_algebra::agg_intent::AggIntent`]. pub use crate::intent_algebra::agg_intent::AggIntent; -/// Per-partition wrapper that historically lived on the legacy `AggIntent` -/// enum as a `PerPartition { inner, keys }` variant. Canonical L3 represents -/// this shape via `QueryExpr::Aggregate { by: keys, aggs: [inner] }`; this -/// wrapper survives only as the input type for the -/// `physical::sketch_catalog` per-partition sizing helpers. -#[derive(Debug, Clone, PartialEq)] -pub struct PerPartitionWrap { - pub inner: AggIntent, - pub keys: Vec, -} - -// ── Shared sketch / predicate types ─────────────────────────────────────────── - -/// Base relation / metric stream source. -#[derive(Debug, Clone)] -pub struct SourceSpec { - /// Table name (SQL) or metric name (PromQL). - pub name: String, -} - -/// How the stream is partitioned. -#[derive(Debug, Clone)] -pub enum PartitionKeys { - /// `by (k1, k2, ...)` — explicit key list. - By(Vec), - /// `without (k1, k2, ...)` — complement; resolved against schema at plan time. - Without(Vec), -} - -impl PartitionKeys { - pub fn keys(&self) -> &[String] { - match self { - PartitionKeys::By(k) | PartitionKeys::Without(k) => k, - } - } - - pub fn is_empty(&self) -> bool { - self.keys().is_empty() - } - - pub fn into_by_keys(self) -> Vec { - match self { - PartitionKeys::By(k) => k, - // For Without, return empty — caller resolves complement. - PartitionKeys::Without(k) => k, - } - } -} - -/// Which column / field the sketch aggregation targets. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum ColumnRef { - /// Explicit column name (SQL: `AVG(price)` → `Named("price")`). - Named(String), - /// The implicit metric sample value (PromQL — always the series value). - SampleValue, - /// All rows / COUNT(*). - Wildcard, -} - -// ── AggIntent helpers ──────────────────────────────────────────────────────── -// -// Step γ7 (PR 13.5): the `AggIntent` helper free fns relocated to their -// canonical home, `intent_algebra::agg_intent`. `relational` re-exports -// the live ones so existing `relational::*` call sites keep compiling -// during the retirement; PR 13 sweeps the consumer imports to the -// canonical path and drops these re-exports with the file. The dead -// helpers (`agg_to_legacy_agg_type`, `agg_quantiles`, -// `accuracy_target_from_legacy` — zero non-test consumers) were deleted -// outright rather than relocated. pub use crate::intent_algebra::agg_intent::{ agg_accuracy, agg_is_exact, agg_is_mergeable, default_cardinality, default_frequency, default_quantile, }; -// ── Accuracy helpers ───────────────────────────────────────────────────────── -// -// The 2026-05 layered-cleanup refactor moved these helpers to -// `sketch_algebra::capability` (their structural home — sketch-family -// error bounds). The thin re-exports below keep `relational::hll_accuracy` / -// `relational::countmin_accuracy` available so the in-file -// `default_cardinality` call site (and any external `algebra::expr::hll_accuracy` -// reference resolved via the back-compat `algebra` alias in `lib.rs`) keep -// compiling. - pub use crate::sketch_algebra::capability::{countmin_accuracy, hll_accuracy}; -// The legacy `WindowSpec` / `WindowKind` types were removed alongside the -// `WindowedAgg` variant they fed. Layer-2 windows are carried by -// `QueryExpr::Window { duration, slide }` directly; window *kind* -// (Tumbling / Sliding / Session) is a canonical L3 concern — -// `query_expr::WindowKind`. +/// Per-partition wrapper that historically lived on the legacy `AggIntent` +/// enum as a `PerPartition { inner, keys }` variant. Canonical L3 +/// represents this shape via `QueryExpr::Aggregate { by: keys, aggs: +/// [inner] }`; this wrapper survives only as the input type for the +/// `physical::sketch_catalog` per-partition sizing helpers. No +/// `asap_l2` equivalent — control_plane-only. +#[derive(Debug, Clone, PartialEq)] +pub struct PerPartitionWrap { + pub inner: AggIntent, + pub keys: Vec, +} -/// A single filter predicate pushed down to the collector. +/// A single filter predicate pushed down to the collector — used by +/// `query_parser::promql::apply_qe_filters` to build an `L2Expr` tree +/// from a flat label-matcher list. Control_plane-only: no `asap_l2` +/// equivalent (its front end builds `L2Expr` directly, never through an +/// intermediate flat predicate list). #[derive(Debug, Clone)] pub struct Predicate { pub col: String, @@ -197,749 +122,3 @@ pub enum FilterVal { Int(i64), Null, } - -// ── Relational algebra ──────────────────────────────────────────────────────── - -/// The **Layer-2 relational** query IR. -/// -/// Every variant is a *node* in the per-language logical query plan tree -/// the `query_parser` front ends (`promql.rs` / `sql.rs`) emit. Leaves -/// are [`QueryExpr::Source`] or [`QueryExpr::Ref`]; interior nodes combine -/// their `input` child(ren) through the operator they implement. -/// -/// # Relationship to the canonical L3 IR -/// -/// Most variants have canonical structural twins in -/// [`crate::intent_algebra::query_expr::QueryExpr`]. The canonical -/// spelling uses `child:` where these L2 variants use `input:`; the -/// typed [`crate::intent_algebra::Predicate`] replaces [`ScalarExpr`] in -/// `Filter` / `Join` / `Aggregate::having` (translation via -/// [`crate::intent_algebra::from_legacy_scalar`]). -/// -/// This is purely a Layer-2 *relational* IR — the sketch-fused -/// `SketchAgg` / `WindowedAgg` variants were removed once the -/// `lower` converter learned to fold the single-statistic -/// sketchable `Aggregate` straight into canonical shapes. The remaining -/// reason the tree is not yet *deleted* outright is that [`ScalarExpr`] -/// still carries four variants (`FunctionCall`, `ScalarSubquery`, -/// `InList`, `Between`) the canonical `Predicate` doesn't cover, and the -/// parsers build `ScalarExpr` directly. -#[derive(Debug, Clone)] -pub enum QueryExpr { - // ── Base relations ──────────────────────────────────────────────────── - /// A named metric stream or table. The outermost leaf. - Source(SourceSpec), - - /// Reference to a CTE / let-binding by name. Resolved at plan time. - Ref(String), - - // ── Filtering & projection ──────────────────────────────────────────── - /// σ — row-level filter (WHERE / PromQL label matchers). - Filter { - pred: ScalarExpr, - input: Box, - }, - - /// π — column projection (SELECT list). - Project { - cols: Vec, - input: Box, - }, - - // ── Aggregation ─────────────────────────────────────────────────────── - /// γ + α — GROUP BY followed by aggregate functions. - /// - /// `keys` is the GROUP BY column list (empty → global aggregate). - /// `aggs` is the list of aggregate expressions to compute. - /// `having` is an optional post-aggregation predicate. - Aggregate { - keys: Vec, - aggs: Vec, - having: Option, - input: Box, - }, - - // ── Time / streaming operators ──────────────────────────────────────── - /// ψ — time window (PromQL `[5m]`; SQL tumbling/sliding window). - Window { - duration: Duration, - slide: Option, - input: Box, - }, - - // The sketch-fused `SketchAgg` / `WindowedAgg` variants were removed: - // they were never parser output — only an intermediate of an earlier - // sketch-lowering pass — and `lower` now folds the - // single-statistic sketchable `Aggregate` straight into canonical - // shapes (`Aggregate { by: [] }` / `Window { Aggregate }`). - - // ── Distributed / multi-stage operators ────────────────────────────── - /// Partition the stream by key-tuple (GROUP BY / `by (dims)`). - Partition { - keys: PartitionKeys, - input: Box, - }, - - /// δ — deduplicate on `cols` (a tuple of columns) before sketch ingestion. - /// SQL `SELECT DISTINCT` lowers to this; the column set may be empty - /// (full-row distinct) or multi-column. PromQL has no direct analog. - Distinct { - cols: Vec, - input: Box, - }, - - /// τ — retain only the top-K entries (heavy hitters). - TopK { - k: u64, - by: Vec, - input: Box, - }, - - /// ⊕ — merge sketches from independent branches (distributed union). - Merge { inputs: Vec }, - - // ── Join operators ──────────────────────────────────────────────────── - /// Relational join. - Join { - kind: JoinKind, - pred: Option, - left: Box, - right: Box, - }, - - // ── Set operators ───────────────────────────────────────────────────── - /// UNION / INTERSECT / EXCEPT (with or without ALL). - SetOp { - kind: SetOpKind, - all: bool, - left: Box, - right: Box, - }, - - // ── Ordering & limiting ─────────────────────────────────────────────── - /// ORDER BY. - Sort { - keys: Vec, - input: Box, - }, - - /// LIMIT [OFFSET]. - Limit { - n: u64, - offset: u64, - input: Box, - }, - - // ── Subquery / CTE ──────────────────────────────────────────────────── - /// SQL `WITH name AS (expr) IN body` or PromQL recording rule binding. - LetBinding { - name: String, - expr: Box, - body: Box, - }, - - // ── PromQL-specific operators ───────────────────────────────────────── - - // Note: `histogram_quantile(φ, )` is no longer a `QueryExpr` - // variant. The PromQL parser substitutes the call with a plain - // `Aggregate { Quantile(φ) }` so - // downstream code (lowerer, optimizer, physical planner) sees a single - // canonical Quantile intent. - /// PromQL sub-query syntax: `[range:resolution]`. - PromQLSubquery { - range: Duration, - resolution: Option, - input: Box, - }, - - /// Binary operation between two instant-vector expressions (PromQL `+`, `/`, …). - /// Also used for SQL arithmetic between sub-relations. - BinaryOp { - op: BinaryOpKind, - lhs: Box, - rhs: Box, - vector_match: Option, - }, -} - -// ── Scalar algebra ──────────────────────────────────────────────────────────── - -/// Scalar expression — computes a single value from a row. -/// -/// Used in [`QueryExpr::Filter`] predicates, [`ProjectItem`] expressions, -/// [`QueryExpr::Aggregate`] HAVING clauses, and JOIN conditions. -#[derive(Debug, Clone)] -pub enum ScalarExpr { - /// Column reference: `t.col` or just `col`. - Column(String), - - /// Literal value. - Literal(LiteralValue), - - /// Arithmetic / comparison / logical / regex binary operator. - BinaryOp { - op: BinaryOpKind, - lhs: Box, - rhs: Box, - }, - - /// Named function call (e.g. `ABS(x)`, `DATE_TRUNC('hour', ts)`). - FunctionCall { name: String, args: Vec }, - - /// Scalar sub-query (`SELECT MAX(price) FROM orders`). - ScalarSubquery(Box), - - /// `expr IN (v1, v2, …)` or `NOT IN (…)`. - InList { - expr: Box, - list: Vec, - negated: bool, - }, - - /// `expr BETWEEN low AND high` or `NOT BETWEEN …`. - Between { - expr: Box, - low: Box, - high: Box, - negated: bool, - }, - - /// `expr IS NULL` / `IS NOT NULL`. - IsNull { - expr: Box, - negated: bool, - }, -} - -// ── Supporting enumerations ─────────────────────────────────────────────────── - -/// A single item in a SELECT projection list. -#[derive(Debug, Clone)] -pub struct ProjectItem { - /// Output column name (SQL `AS alias`; None → use expression name). - pub alias: Option, - pub expr: ScalarExpr, -} - -/// One aggregate function in a GROUP BY / AGGREGATE node. -#[derive(Debug, Clone)] -pub struct AggItem { - /// Output column name. - pub alias: String, - /// The aggregate function. - pub func: AggFunc, - /// Column(s) the function operates on. - pub col: ColumnRef, - /// Whether DISTINCT is applied before aggregation. - pub distinct: bool, -} - -/// All aggregate functions that the algebra supports. -/// -/// "Sketchable" variants (Quantile, CountDistinct, HeavyHitters) can be -/// approximated by a sketch in early pipeline stages; the rest require -/// exact computation. -#[derive(Debug, Clone, PartialEq)] -pub enum AggFunc { - Count, - Sum, - Avg, - Min, - Max, - /// Sample / population standard deviation. - StdDev { - population: bool, - }, - /// Sample / population variance. - Variance { - population: bool, - }, - /// Approximate quantile at φ ∈ (0, 1]. Maps to DDSketch. - Quantile(f64), - /// COUNT DISTINCT — maps to HLL. - CountDistinct, - /// Per-series frequency estimation — maps to CMS / CountSketch. - /// PromQL surface: `count_over_time(metric[range])` (counts - /// samples per series in the window). Distinct from `Count` - /// because `count_over_time` is structurally per-series and - /// always sketchable, where `Count` carries the SQL `COUNT(*)` - /// exact-row-count case that the un-grouped lowering branch - /// pins to `AggIntent::Count{Exact}`. - Frequency, - /// Top-K heavy hitters — maps to CountSketch. - HeavyHitters { - k: u64, - }, - /// PromQL `rate()` — per-second increase over a window. - Rate, - /// PromQL `increase()` — total increase over a window. - Increase, - /// PromQL `delta()` — change over a window (may be negative). - Delta, - /// Arbitrary named aggregate (UDA or extension). - Custom(String), -} - -impl AggFunc { - /// Returns true when this function can be computed from merged partial - /// results: `f(A ∪ B) = combine(f(A), f(B))`. - pub fn is_mergeable(&self) -> bool { - match self { - AggFunc::Avg | AggFunc::StdDev { .. } | AggFunc::Variance { .. } => false, - _ => true, - } - } - - /// Returns true when this function requires sketch approximation to be - /// bandwidth-efficient (i.e. the raw data would be too large to ship). - pub fn is_sketchable(&self) -> bool { - matches!( - self, - AggFunc::Quantile(_) - | AggFunc::CountDistinct - | AggFunc::Frequency - | AggFunc::HeavyHitters { .. } - ) - } - - /// Suggest the appropriate [`AggIntent`] for this function, if any. - /// - /// Canonical Quantile is single-φ; this helper returns one canonical - /// intent. Callers that need multi-φ behaviour build the merge fan-out - /// themselves (cf. `lower::agg_func_to_intents`). - pub fn to_sketch_op(&self) -> Option { - match self { - AggFunc::Quantile(phi) => Some(default_quantile(*phi)), - AggFunc::CountDistinct => Some(default_cardinality()), - AggFunc::Frequency => Some(default_frequency()), - AggFunc::HeavyHitters { .. } => Some(default_frequency()), - AggFunc::Count => Some(AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }), - AggFunc::Sum => Some(AggIntent::Sum { col: None }), - AggFunc::Avg => Some(AggIntent::Avg { col: None }), - AggFunc::Min => Some(AggIntent::Min { col: None }), - AggFunc::Max => Some(AggIntent::Max { col: None }), - _ => None, - } - } -} - -// Leaf algebra types — `BinaryOpKind`, `JoinKind`, `SetOpKind`, -// `VectorMatch` / `VectorMatchKind` / `VectorGrouping` / `GroupSide`, -// `SortKey` — are owned by the canonical `query_expr` module. The L2 -// definitions were byte-identical (modulo extra `Hash` / `serde` derives -// on the canonical side), so `relational` now re-exports them: every -// `relational::BinaryOpKind` reference resolves to the single canonical -// type. `impl Display for BinaryOpKind` moved alongside the type. -pub use crate::intent_algebra::query_expr::{ - BinaryOpKind, GroupSide, JoinKind, SetOpKind, SortKey, VectorGrouping, VectorMatch, - VectorMatchKind, -}; - -/// Scalar literal. -#[derive(Debug, Clone, PartialEq)] -pub enum LiteralValue { - Null, - Bool(bool), - Int(i64), - Float(f64), - Str(String), - Duration(Duration), -} - -impl QueryExpr { - /// Walk the expression tree depth-first and call `f` on every node. - pub fn walk(&self, f: &mut F) { - f(self); - match self { - QueryExpr::Source(_) | QueryExpr::Ref(_) => {} - QueryExpr::Filter { input, .. } - | QueryExpr::Project { input, .. } - | QueryExpr::Window { input, .. } - | QueryExpr::Partition { input, .. } - | QueryExpr::Distinct { input, .. } - | QueryExpr::TopK { input, .. } - | QueryExpr::Sort { input, .. } - | QueryExpr::Limit { input, .. } - | QueryExpr::PromQLSubquery { input, .. } => input.walk(f), - - QueryExpr::Aggregate { input, .. } => input.walk(f), - - QueryExpr::Merge { inputs } => { - for i in inputs { - i.walk(f); - } - } - QueryExpr::Join { left, right, .. } - | QueryExpr::SetOp { left, right, .. } - | QueryExpr::BinaryOp { - lhs: left, - rhs: right, - .. - } => { - left.walk(f); - right.walk(f); - } - QueryExpr::LetBinding { expr, body, .. } => { - expr.walk(f); - body.walk(f); - } - } - } - - /// Returns `true` when the sub-tree contains at least one - /// [`QueryExpr::TopK`] node — the only Layer-2 variant that names a - /// heavy-hitter sketch directly. (Single-statistic sketchable - /// `Aggregate`s are recognised as sketch work only once the converter - /// folds them; at Layer 2 they are indistinguishable from exact - /// aggregates.) - pub fn has_sketch_work(&self) -> bool { - let mut found = false; - self.walk(&mut |n| { - if matches!(n, QueryExpr::TopK { .. }) { - found = true; - } - }); - found - } - - /// Returns the outermost metric/table name from the first `Source` leaf. - pub fn source_name(&self) -> Option<&str> { - match self { - QueryExpr::Source(s) => Some(&s.name), - QueryExpr::Filter { input, .. } - | QueryExpr::Project { input, .. } - | QueryExpr::Window { input, .. } - | QueryExpr::Partition { input, .. } - | QueryExpr::Distinct { input, .. } - | QueryExpr::TopK { input, .. } - | QueryExpr::Sort { input, .. } - | QueryExpr::Limit { input, .. } - | QueryExpr::Aggregate { input, .. } - | QueryExpr::PromQLSubquery { input, .. } => input.source_name(), - QueryExpr::Merge { inputs } => inputs.first()?.source_name(), - QueryExpr::Join { left, .. } - | QueryExpr::SetOp { left, .. } - | QueryExpr::BinaryOp { lhs: left, .. } => left.source_name(), - QueryExpr::LetBinding { body, .. } => body.source_name(), - QueryExpr::Ref(_) => None, - } - } -} - -// ── Predicate → ScalarExpr conversion ──────────────────────────────────────── - -/// Convert a slice of [`Predicate`]s (AND-list) into a single -/// [`ScalarExpr`] tree. An empty slice becomes `Literal(true)`. -fn scalar_from_predicates(preds: &[Predicate]) -> ScalarExpr { - if preds.is_empty() { - return ScalarExpr::Literal(LiteralValue::Bool(true)); - } - let mut iter = preds.iter().map(scalar_from_predicate); - let first = iter.next().unwrap(); - iter.fold(first, |acc, p| ScalarExpr::BinaryOp { - op: BinaryOpKind::And, - lhs: Box::new(acc), - rhs: Box::new(p), - }) -} - -fn scalar_from_predicate(p: &Predicate) -> ScalarExpr { - let col = ScalarExpr::Column(p.col.clone()); - let val = match &p.val { - FilterVal::Str(s) => ScalarExpr::Literal(LiteralValue::Str(s.clone())), - FilterVal::Num(n) => ScalarExpr::Literal(LiteralValue::Float(*n)), - FilterVal::Int(i) => ScalarExpr::Literal(LiteralValue::Int(*i)), - FilterVal::Null => ScalarExpr::Literal(LiteralValue::Null), - }; - match &p.op { - FilterOp::Eq => bin(BinaryOpKind::Compare(CompareOp::Eq), col, val), - FilterOp::Ne => bin(BinaryOpKind::Compare(CompareOp::Ne), col, val), - FilterOp::Lt => bin(BinaryOpKind::Compare(CompareOp::Lt), col, val), - FilterOp::Le => bin(BinaryOpKind::Compare(CompareOp::Le), col, val), - FilterOp::Gt => bin(BinaryOpKind::Compare(CompareOp::Gt), col, val), - FilterOp::Ge => bin(BinaryOpKind::Compare(CompareOp::Ge), col, val), - FilterOp::Like => bin(BinaryOpKind::Compare(CompareOp::Like), col, val), - FilterOp::NotLike => bin(BinaryOpKind::Compare(CompareOp::NotLike), col, val), - FilterOp::IsNull => ScalarExpr::IsNull { - expr: Box::new(col), - negated: false, - }, - FilterOp::IsNotNull => ScalarExpr::IsNull { - expr: Box::new(col), - negated: true, - }, - FilterOp::Regex(r) => bin( - BinaryOpKind::Compare(CompareOp::Regex), - col, - ScalarExpr::Literal(LiteralValue::Str(r.clone())), - ), - FilterOp::NotRegex(r) => bin( - BinaryOpKind::Compare(CompareOp::NotRegex), - col, - ScalarExpr::Literal(LiteralValue::Str(r.clone())), - ), - } -} - -fn bin(op: BinaryOpKind, lhs: ScalarExpr, rhs: ScalarExpr) -> ScalarExpr { - ScalarExpr::BinaryOp { - op, - lhs: Box::new(lhs), - rhs: Box::new(rhs), - } -} - -// ── Display helpers ─────────────────────────────────────────────────────────── - -impl std::fmt::Display for AggFunc { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - AggFunc::Count => write!(f, "COUNT"), - AggFunc::Sum => write!(f, "SUM"), - AggFunc::Avg => write!(f, "AVG"), - AggFunc::Min => write!(f, "MIN"), - AggFunc::Max => write!(f, "MAX"), - AggFunc::StdDev { .. } => write!(f, "STDDEV"), - AggFunc::Variance { .. } => write!(f, "VARIANCE"), - AggFunc::Quantile(p) => write!(f, "QUANTILE({p})"), - AggFunc::CountDistinct => write!(f, "COUNT_DISTINCT"), - AggFunc::Frequency => write!(f, "FREQUENCY"), - AggFunc::HeavyHitters { k } => write!(f, "HEAVY_HITTERS({k})"), - AggFunc::Rate => write!(f, "rate"), - AggFunc::Increase => write!(f, "increase"), - AggFunc::Delta => write!(f, "delta"), - AggFunc::Custom(s) => write!(f, "{s}"), - } - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use std::time::Duration; - - fn src(name: &str) -> QueryExpr { - QueryExpr::Source(SourceSpec { name: name.into() }) - } - - // ── has_sketch_work ─────────────────────────────────────────────────────── - - #[test] - fn has_sketch_work_true_when_topk_present() { - let qe = QueryExpr::TopK { - k: 10, - by: vec![], - input: Box::new(src("m")), - }; - assert!(qe.has_sketch_work()); - } - - #[test] - fn has_sketch_work_false_for_plain_source() { - let qe = QueryExpr::Source(SourceSpec { name: "x".into() }); - assert!(!qe.has_sketch_work()); - } - - // ── source_name ─────────────────────────────────────────────────────────── - - #[test] - fn source_name_extracted_through_chain() { - let qe = QueryExpr::Window { - duration: Duration::from_secs(60), - slide: None, - input: Box::new(QueryExpr::Filter { - pred: ScalarExpr::Literal(LiteralValue::Bool(true)), - input: Box::new(src("my_metric")), - }), - }; - assert_eq!(qe.source_name(), Some("my_metric")); - } - - // ── AggFunc helpers ─────────────────────────────────────────────────────── - - #[test] - fn agg_func_mergeability() { - assert!(AggFunc::Sum.is_mergeable()); - assert!(AggFunc::Count.is_mergeable()); - assert!(AggFunc::Min.is_mergeable()); - assert!(AggFunc::Max.is_mergeable()); - assert!(!AggFunc::Avg.is_mergeable()); - assert!(!AggFunc::StdDev { population: false }.is_mergeable()); - assert!(!AggFunc::Variance { population: true }.is_mergeable()); - } - - #[test] - fn agg_func_sketchability() { - assert!(AggFunc::Quantile(0.99).is_sketchable()); - assert!(AggFunc::CountDistinct.is_sketchable()); - assert!(AggFunc::HeavyHitters { k: 10 }.is_sketchable()); - assert!(!AggFunc::Avg.is_sketchable()); - assert!(!AggFunc::Sum.is_sketchable()); - } - - #[test] - fn agg_func_to_sketch_op_quantile() { - let op = AggFunc::Quantile(0.99).to_sketch_op(); - assert!(matches!(op, Some(AggIntent::Quantile { .. }))); - } - - #[test] - fn agg_func_to_sketch_op_count_distinct() { - let op = AggFunc::CountDistinct.to_sketch_op(); - assert!(matches!(op, Some(AggIntent::Cardinality { .. }))); - } - - #[test] - fn agg_func_to_sketch_op_heavy_hitters() { - let op = AggFunc::HeavyHitters { k: 50 }.to_sketch_op(); - assert!(op.is_some_and(|i| crate::intent_algebra::as_frequency(&i).is_some())); - } - - // ── ScalarExpr predicate list conversion ────────────────────────────────── - - #[test] - fn empty_pred_list_becomes_literal_true() { - let s = scalar_from_predicates(&[]); - assert!(matches!(s, ScalarExpr::Literal(LiteralValue::Bool(true)))); - } - - #[test] - fn two_preds_become_and_tree() { - let preds = vec![ - Predicate { - col: "a".into(), - op: FilterOp::Eq, - val: FilterVal::Int(1), - }, - Predicate { - col: "b".into(), - op: FilterOp::Gt, - val: FilterVal::Num(2.0), - }, - ]; - let s = scalar_from_predicates(&preds); - assert!(matches!( - s, - ScalarExpr::BinaryOp { - op: BinaryOpKind::And, - .. - } - )); - } - - // ── BinaryOpKind display ────────────────────────────────────────────────── - - #[test] - fn binary_op_kind_display() { - assert_eq!(BinaryOpKind::Arith(ArithOp::Add).to_string(), "+"); - assert_eq!(BinaryOpKind::And.to_string(), "AND"); - assert_eq!(BinaryOpKind::Compare(CompareOp::Regex).to_string(), "=~"); - assert_eq!(BinaryOpKind::Compare(CompareOp::NotRegex).to_string(), "!~"); - assert_eq!(BinaryOpKind::Unless.to_string(), "unless"); - } - - // ── Complex nested tree ─────────────────────────────────────────────────── - - #[test] - fn complex_nested_tree() { - // TopK(10, Partition(symbol, Window(5m, Aggregate(Count, Source(price))))) - let qe = QueryExpr::TopK { - k: 10, - by: vec![], - input: Box::new(QueryExpr::Partition { - keys: PartitionKeys::By(vec!["symbol".into()]), - input: Box::new(QueryExpr::Window { - duration: Duration::from_secs(300), - slide: None, - input: Box::new(QueryExpr::Aggregate { - keys: vec![], - aggs: vec![AggItem { - alias: "c".into(), - func: AggFunc::Count, - col: ColumnRef::Wildcard, - distinct: false, - }], - having: None, - input: Box::new(src("price")), - }), - }), - }), - }; - assert!(qe.has_sketch_work()); - assert_eq!(qe.source_name(), Some("price")); - } - - // ── LetBinding and Subquery ─────────────────────────────────────────────── - - #[test] - fn let_binding_construction() { - let expr = QueryExpr::LetBinding { - name: "base".into(), - expr: Box::new(QueryExpr::Source(SourceSpec { name: "cpu".into() })), - body: Box::new(QueryExpr::Ref("base".into())), - }; - match expr { - QueryExpr::LetBinding { name, .. } => assert_eq!(name, "base"), - _ => panic!(), - } - } - - #[test] - fn promql_subquery_node() { - let expr = QueryExpr::PromQLSubquery { - range: Duration::from_secs(3600), - resolution: Some(Duration::from_secs(60)), - input: Box::new(QueryExpr::Source(SourceSpec { name: "m".into() })), - }; - match expr { - QueryExpr::PromQLSubquery { - range, resolution, .. - } => { - assert_eq!(range, Duration::from_secs(3600)); - assert_eq!(resolution, Some(Duration::from_secs(60))); - } - _ => panic!(), - } - } - - // ── AggIntent and related types ────────────────────────────────────────── - - #[test] - fn agg_intent_cardinality_is_mergeable() { - assert!(agg_is_mergeable(&default_cardinality())); - } - - #[test] - fn agg_intent_avg_not_mergeable() { - assert!(!agg_is_mergeable(&AggIntent::Avg { col: None })); - } - - #[test] - fn per_partition_wrap_carries_inner_and_keys() { - let wrap = PerPartitionWrap { - inner: default_cardinality(), - keys: vec!["region".into()], - }; - assert_eq!(wrap.keys, vec!["region".to_string()]); - assert!(agg_is_mergeable(&wrap.inner)); - } - - #[test] - fn partition_keys_without_variant() { - let keys = PartitionKeys::Without(vec!["instance".into()]); - assert_eq!(keys.keys(), &["instance".to_string()]); - assert!(!keys.is_empty()); - } - - #[test] - fn agg_intent_is_exact() { - assert!(agg_is_exact(&AggIntent::Sum { col: None })); - assert!(agg_is_exact(&AggIntent::Min { col: None })); - assert!(agg_is_exact(&AggIntent::Max { col: None })); - assert!(!agg_is_exact(&default_cardinality())); - } -} diff --git a/control_plane/src/physical/window_fusion.rs b/control_plane/src/physical/window_fusion.rs index e8bc024e..bf42aab6 100644 --- a/control_plane/src/physical/window_fusion.rs +++ b/control_plane/src/physical/window_fusion.rs @@ -191,10 +191,7 @@ mod tests { /// A canonical `Scan` leaf — built through `convert_root` so it carries /// the same Binder-built schema a real converted tree would. fn canonical_scan(metric: &str) -> QueryExpr { - convert_root(&LQueryExpr::Source(SourceSpec { - name: metric.into(), - })) - .expect("convert source") + convert_root(&LQueryExpr::Source(SourceSpec::new(metric))).expect("convert source") } /// The canonical `Window { Aggregate { by: [], aggs: [agg] } }` shape — @@ -350,7 +347,7 @@ mod tests { let legacy = LQueryExpr::Window { duration: Duration::from_secs(60), slide: None, - input: Box::new(LQueryExpr::Source(SourceSpec { name: "m".into() })), + input: Box::new(LQueryExpr::Source(SourceSpec::new("m"))), }; let canonical = convert_root(&legacy).expect("convert"); assert!(recognize_windowed_sketch(&canonical).is_none()); @@ -363,17 +360,17 @@ mod tests { // shape the recognizer matches. let legacy = LQueryExpr::Aggregate { keys: vec![], + without: false, aggs: vec![AggItem { - alias: "q".into(), + alias: Some("q".into()), func: AggFunc::Quantile(0.99), col: LColumnRef::SampleValue, - distinct: false, }], having: None, input: Box::new(LQueryExpr::Window { duration: Duration::from_secs(300), slide: None, - input: Box::new(LQueryExpr::Source(SourceSpec { name: "m".into() })), + input: Box::new(LQueryExpr::Source(SourceSpec::new("m"))), }), }; let canonical = convert_root(&legacy).expect("convert"); diff --git a/control_plane/src/query_parser/promql.rs b/control_plane/src/query_parser/promql.rs index 44813f08..8244408d 100644 --- a/control_plane/src/query_parser/promql.rs +++ b/control_plane/src/query_parser/promql.rs @@ -35,15 +35,39 @@ use std::time::Duration; use anyhow::anyhow; use promql_parser::parser::{self, AggregateExpr, Call, Expr, LabelModifier, VectorSelector}; -use crate::intent_algebra::relational::{FilterOp, FilterVal, PartitionKeys, Predicate}; +use crate::intent_algebra::relational::{FilterOp, FilterVal, Predicate}; // ── Walk context ────────────────────────────────────────────────────────────── +/// PromQL `by(labels)` / `without(labels)` aggregation modifier, accumulated +/// as we descend the AST. Control_plane-only walking state — `asap_l2`'s +/// `relational::QueryExpr::Aggregate` has no separate `Partition` node to +/// mirror this against (its `keys`/`without` fields live directly on +/// `Aggregate`, see `intent_algebra::relational`'s module doc), so this +/// folds straight into the nearest `Aggregate`/`Window` via +/// [`fold_group_mod`] instead of wrapping a dedicated node. +#[derive(Clone)] +enum GroupMod { + By(Vec), + Without(Vec), +} + +impl GroupMod { + fn keys(&self) -> &[String] { + match self { + GroupMod::By(k) | GroupMod::Without(k) => k, + } + } + fn is_empty(&self) -> bool { + self.keys().is_empty() + } +} + /// Context accumulated as we descend the AST. #[derive(Default, Clone)] struct WalkCtx { /// GROUP BY / `without` clause from an outer Aggregate node. - partition: Option, + partition: Option, /// Top-K k from an outer `topk` / `bottomk` operator. topk: Option, /// Whether the outer context is a `count()` aggregate (→ CountDistinct). @@ -159,12 +183,12 @@ fn extract_number_param(param: &Option>) -> anyhow::Result { } } -// ── Helpers: PartitionKeys from LabelModifier ───────────────────────────────── +// ── Helpers: GroupMod from LabelModifier ────────────────────────────────────── -fn modifier_to_partition(modifier: &LabelModifier) -> PartitionKeys { +fn modifier_to_partition(modifier: &LabelModifier) -> GroupMod { match modifier { - LabelModifier::Include(labels) => PartitionKeys::By(labels.labels.clone()), - LabelModifier::Exclude(labels) => PartitionKeys::Without(labels.labels.clone()), + LabelModifier::Include(labels) => GroupMod::By(labels.labels.clone()), + LabelModifier::Exclude(labels) => GroupMod::Without(labels.labels.clone()), } } @@ -180,9 +204,8 @@ fn modifier_to_partition(modifier: &LabelModifier) -> PartitionKeys { // | `a op b` binary | BinaryOp { VectorMatch } | use crate::intent_algebra::relational::{ - AggFunc, AggItem, BinaryOpKind, ColumnRef as QeColumnRef, GroupSide, - PartitionKeys as QePartitionKeys, QueryExpr, SourceSpec as QeSourceSpec, VectorGrouping, - VectorMatch, VectorMatchKind, + AggFunc, AggItem, BinaryOpKind, ColumnRef as QeColumnRef, GroupSide, QueryExpr, + SourceSpec as QeSourceSpec, VectorGrouping, VectorMatch, VectorMatchKind, }; use crate::intent_algebra::{ArithOp, CompareOp}; use promql_parser::parser::{token::TokenType, BinaryExpr, VectorMatchCardinality}; @@ -233,18 +256,18 @@ fn walk_qe(expr: &Expr, ctx: WalkCtx) -> anyhow::Result { // HLL-only or CMS-with-heap-only deploy). Expr::VectorSelector(vs) => { let (name, filters) = extract_vs_info(vs); - let source = QueryExpr::Source(QeSourceSpec { name }); + let source = QueryExpr::Source(QeSourceSpec::new(name)); let filtered = apply_qe_filters(source, filters); if ctx.outer_count || ctx.topk.is_some() { Ok(filtered) } else { Ok(QueryExpr::Aggregate { keys: vec![], + without: false, aggs: vec![AggItem { - alias: "value".into(), + alias: Some("value".into()), func: AggFunc::Sum, col: QeColumnRef::SampleValue, - distinct: false, }], having: None, input: Box::new(filtered), @@ -274,13 +297,14 @@ fn walk_aggregate_qe(agg: &AggregateExpr, ctx: WalkCtx) -> anyhow::Result anyhow::Result { let inner_ctx = WalkCtx { @@ -313,7 +337,7 @@ fn walk_aggregate_qe(agg: &AggregateExpr, ctx: WalkCtx) -> anyhow::Result { let inner_ctx = WalkCtx { @@ -324,16 +348,16 @@ fn walk_aggregate_qe(agg: &AggregateExpr, ctx: WalkCtx) -> anyhow::Result { let inner_ctx = WalkCtx { @@ -344,16 +368,16 @@ fn walk_aggregate_qe(agg: &AggregateExpr, ctx: WalkCtx) -> anyhow::Result { let phi = extract_number_param(&agg.param)?; @@ -365,16 +389,16 @@ fn walk_aggregate_qe(agg: &AggregateExpr, ctx: WalkCtx) -> anyhow::Result Err(anyhow!("unsupported PromQL aggregate operator: {other}")), } @@ -419,7 +443,6 @@ fn walk_call_qe(call: &Call, ctx: WalkCtx) -> anyhow::Result { Ok(build_qe_aggregate(source, filters, window, func, ctx)) } _ => { - let func = walk_call_to_op(call, &ctx)?; let (source, filters, window) = if call.func.name == "rate" || call.func.name == "irate" || call.func.name == "increase" @@ -434,6 +457,7 @@ fn walk_call_qe(call: &Call, ctx: WalkCtx) -> anyhow::Result { } else { extract_matrix_arg(call, 0)? }; + let func = walk_call_to_op(call, &ctx, window)?; Ok(build_qe_aggregate(source, filters, window, func, ctx)) } } @@ -503,7 +527,14 @@ fn promql_token_to_binop(tok: TokenType) -> BinaryOpKind { } /// Map a PromQL function call to an [`AggFunc`] (Layer 2 relational operator). -fn walk_call_to_op(call: &Call, ctx: &WalkCtx) -> anyhow::Result { +/// `window` is the range-vector's duration — only `Rate`/`Increase` carry it +/// on the `AggFunc` itself (`asap_l2`'s design: "no separate Window node"). +/// Every other function still relies on the caller wrapping its `Aggregate` +/// in an `L2::Window`, matching this repo's pre-`asap_l2`-merge behavior +/// (see `lower.rs`'s module doc on why `Rate`/`Increase` map to +/// `AggIntent::Sum` here rather than adopting the dedicated intents that +/// window field would otherwise feed). +fn walk_call_to_op(call: &Call, ctx: &WalkCtx, window: Duration) -> anyhow::Result { let name = call.func.name; match name { "quantile_over_time" => { @@ -521,19 +552,19 @@ fn walk_call_to_op(call: &Call, ctx: &WalkCtx) -> anyhow::Result { // `CountDistinct` (HLL distinct counting; the outer // count of inner counts is cardinality). // * Inside `topk(N, count_over_time(...))` → `Count` - // (the topk wrapper expects a count-shaped inner). - // * Otherwise → `Frequency` (per-series sample-count - // estimation; routes to CMS / CountSketch via the - // `AggFunc::Frequency → default_frequency()` lowering). - // This was previously `AggFunc::Count` which the - // un-grouped lowering branch pinned to - // `AggIntent::Count{Exact}` → unsupported by ASAP. + // (the topk wrapper expects a count-shaped inner; the + // grouped-or-windowed `Count` → `Frequency` substitution + // happens in `lower.rs::agg_func_to_intents`). + // * Otherwise → plain `Count`, still windowed here — the + // same `lower.rs` substitution recognizes the windowed + // shape and routes to CMS / CountSketch (per-series + // sample-count estimation), matching the pre-`asap_l2` + // behavior this used to reach via a dedicated + // `AggFunc::Frequency` variant that no longer exists. Ok(if ctx.outer_count { AggFunc::CountDistinct - } else if ctx.topk.is_some() { - AggFunc::Count } else { - AggFunc::Frequency + AggFunc::Count }) } "sum_over_time" | "last_over_time" | "present_over_time" | "absent_over_time" => { @@ -541,12 +572,35 @@ fn walk_call_to_op(call: &Call, ctx: &WalkCtx) -> anyhow::Result { } "delta" | "idelta" | "deriv" | "predict_linear" => Ok(AggFunc::Delta), "changes" | "resets" => Ok(AggFunc::Count), - "rate" | "irate" => Ok(AggFunc::Rate), - "increase" => Ok(AggFunc::Increase), + "rate" | "irate" => Ok(AggFunc::Rate { window }), + "increase" => Ok(AggFunc::Increase { window }), other => Err(anyhow!("unsupported PromQL function: {other}")), } } +/// Short lowercase label for an `AggFunc`, used as the `AggItem` alias. +/// `AggFunc` is foreign (from `asap_l2`) — Rust's orphan rules forbid +/// implementing `Display` for it here, unlike this repo's pre-merge own +/// `AggFunc`, which had one. +fn agg_func_label(f: &AggFunc) -> String { + match f { + AggFunc::Count => "count".into(), + AggFunc::Sum => "sum".into(), + AggFunc::Avg => "avg".into(), + AggFunc::Min => "min".into(), + AggFunc::Max => "max".into(), + AggFunc::StdDev { .. } => "stddev".into(), + AggFunc::Variance { .. } => "variance".into(), + AggFunc::Quantile(_) => "quantile".into(), + AggFunc::CountDistinct => "count_distinct".into(), + AggFunc::HeavyHitters { .. } => "heavy_hitters".into(), + AggFunc::Rate { .. } => "rate".into(), + AggFunc::Increase { .. } => "increase".into(), + AggFunc::Delta => "delta".into(), + other => format!("{other:?}").to_lowercase(), + } +} + /// Build a Layer 2 `QueryExpr`: `Aggregate { AggFunc, input: Window { ... } }`. fn build_qe_aggregate( metric: String, @@ -555,7 +609,7 @@ fn build_qe_aggregate( func: AggFunc, ctx: WalkCtx, ) -> QueryExpr { - let source = QueryExpr::Source(QeSourceSpec { name: metric }); + let source = QueryExpr::Source(QeSourceSpec::new(metric)); let filtered = apply_qe_filters(source, filters); let windowed = QueryExpr::Window { duration: window, @@ -568,136 +622,155 @@ fn build_qe_aggregate( } else { func }; - // Propagate partition keys into the Aggregate's GROUP BY so the lowering - // pass sees Count-with-GROUP-BY → Frequency (not bare Count → no sketch). - let group_keys: Vec = ctx - .partition - .as_ref() - .map(|p| p.keys().to_vec()) - .unwrap_or_default(); - let alias = format!("{}", actual_func).to_lowercase(); - let agg = QueryExpr::Aggregate { - keys: group_keys, + // Propagate partition keys into the Aggregate's GROUP BY so + // `lower.rs`'s `agg_func_to_intents` sees a grouped `Count` → the + // `Frequency` sketch trigger (not a bare, exact `Count`). + let (group_keys, without): (Vec, bool) = match &ctx.partition { + Some(GroupMod::By(k)) => (k.clone(), false), + Some(GroupMod::Without(k)) => (k.clone(), true), + None => (Vec::new(), false), + }; + let alias = agg_func_label(&actual_func); + QueryExpr::Aggregate { + keys: group_keys.into_iter().map(QeColumnRef::Named).collect(), + without, aggs: vec![AggItem { - alias, + alias: Some(alias), func: actual_func, col: QeColumnRef::SampleValue, - distinct: false, }], having: None, input: Box::new(windowed), - }; - // Don't wrap with Partition separately — keys are already in the - // Aggregate. The lowering pass folds them straight into the - // canonical `Aggregate.by: GroupKeys` (no `Partition` node exists in - // the canonical IR). - agg + } } fn apply_qe_filters(input: QueryExpr, filters: Vec) -> QueryExpr { if filters.is_empty() { - input - } else { - use crate::intent_algebra::relational::{BinaryOpKind, LiteralValue, ScalarExpr}; - use crate::intent_algebra::CompareOp; - let pred = filters - .iter() - .fold(ScalarExpr::Literal(LiteralValue::Bool(true)), |acc, p| { - let col = ScalarExpr::Column(p.col.clone()); - let val = match &p.val { - FilterVal::Str(s) => ScalarExpr::Literal(LiteralValue::Str(s.clone())), - FilterVal::Num(n) => ScalarExpr::Literal(LiteralValue::Float(*n)), - FilterVal::Int(i) => ScalarExpr::Literal(LiteralValue::Int(*i)), - FilterVal::Null => ScalarExpr::Literal(LiteralValue::Null), - }; - let this = match &p.op { - FilterOp::Eq => ScalarExpr::BinaryOp { - op: BinaryOpKind::Compare(CompareOp::Eq), - lhs: Box::new(col), - rhs: Box::new(val), - }, - FilterOp::Ne => ScalarExpr::BinaryOp { - op: BinaryOpKind::Compare(CompareOp::Ne), - lhs: Box::new(col), - rhs: Box::new(val), - }, - FilterOp::Lt => ScalarExpr::BinaryOp { - op: BinaryOpKind::Compare(CompareOp::Lt), - lhs: Box::new(col), - rhs: Box::new(val), - }, - FilterOp::Le => ScalarExpr::BinaryOp { - op: BinaryOpKind::Compare(CompareOp::Le), - lhs: Box::new(col), - rhs: Box::new(val), - }, - FilterOp::Gt => ScalarExpr::BinaryOp { - op: BinaryOpKind::Compare(CompareOp::Gt), - lhs: Box::new(col), - rhs: Box::new(val), - }, - FilterOp::Ge => ScalarExpr::BinaryOp { - op: BinaryOpKind::Compare(CompareOp::Ge), - lhs: Box::new(col), - rhs: Box::new(val), - }, - FilterOp::Regex(r) => ScalarExpr::BinaryOp { - op: BinaryOpKind::Compare(CompareOp::Regex), - lhs: Box::new(col), - rhs: Box::new(ScalarExpr::Literal(LiteralValue::Str(r.clone()))), - }, - FilterOp::NotRegex(r) => ScalarExpr::BinaryOp { - op: BinaryOpKind::Compare(CompareOp::NotRegex), - lhs: Box::new(col), - rhs: Box::new(ScalarExpr::Literal(LiteralValue::Str(r.clone()))), - }, - FilterOp::Like => ScalarExpr::BinaryOp { - op: BinaryOpKind::Compare(CompareOp::Like), - lhs: Box::new(col), - rhs: Box::new(val), - }, - FilterOp::NotLike => ScalarExpr::BinaryOp { - op: BinaryOpKind::Compare(CompareOp::NotLike), - lhs: Box::new(col), - rhs: Box::new(val), - }, - FilterOp::IsNull => ScalarExpr::IsNull { - expr: Box::new(col), - negated: false, - }, - FilterOp::IsNotNull => ScalarExpr::IsNull { - expr: Box::new(col), - negated: true, - }, - }; - ScalarExpr::BinaryOp { - op: BinaryOpKind::And, - lhs: Box::new(acc), - rhs: Box::new(this), - } - }); - QueryExpr::Filter { - pred, - input: Box::new(input), - } + return input; } -} + use crate::intent_algebra::{L2Expr, L3Scalar}; -fn apply_qe_partition(input: QueryExpr, partition: Option) -> QueryExpr { - match partition { - None => input, - Some(p) if p.is_empty() => input, - Some(keys) => { - // Convert PromQL by/without → PartitionKeys. - let qe_keys = match keys { - PartitionKeys::By(k) => QePartitionKeys::By(k), - PartitionKeys::Without(k) => QePartitionKeys::Without(k), + let conjuncts: Vec = filters + .iter() + .map(|p| { + let col = L2Expr::Column(QeColumnRef::Named(p.col.clone())); + let val = |v: &FilterVal| match v { + FilterVal::Str(s) => L2Expr::Literal(L3Scalar::Utf8(s.clone())), + FilterVal::Num(n) => L2Expr::Literal(L3Scalar::Float64(*n)), + FilterVal::Int(i) => L2Expr::Literal(L3Scalar::Int64(*i)), + FilterVal::Null => L2Expr::Literal(L3Scalar::Null), }; - QueryExpr::Partition { - keys: qe_keys, - input: Box::new(input), + match &p.op { + FilterOp::Eq => L2Expr::Compare { + left: Box::new(col), + op: CompareOp::Eq, + right: Box::new(val(&p.val)), + }, + FilterOp::Ne => L2Expr::Compare { + left: Box::new(col), + op: CompareOp::Ne, + right: Box::new(val(&p.val)), + }, + FilterOp::Lt => L2Expr::Compare { + left: Box::new(col), + op: CompareOp::Lt, + right: Box::new(val(&p.val)), + }, + FilterOp::Le => L2Expr::Compare { + left: Box::new(col), + op: CompareOp::Le, + right: Box::new(val(&p.val)), + }, + FilterOp::Gt => L2Expr::Compare { + left: Box::new(col), + op: CompareOp::Gt, + right: Box::new(val(&p.val)), + }, + FilterOp::Ge => L2Expr::Compare { + left: Box::new(col), + op: CompareOp::Ge, + right: Box::new(val(&p.val)), + }, + FilterOp::Regex(r) => L2Expr::Compare { + left: Box::new(col), + op: CompareOp::Regex, + right: Box::new(L2Expr::Literal(L3Scalar::Utf8(r.clone()))), + }, + FilterOp::NotRegex(r) => L2Expr::Compare { + left: Box::new(col), + op: CompareOp::NotRegex, + right: Box::new(L2Expr::Literal(L3Scalar::Utf8(r.clone()))), + }, + FilterOp::Like => L2Expr::Compare { + left: Box::new(col), + op: CompareOp::Like, + right: Box::new(val(&p.val)), + }, + FilterOp::NotLike => L2Expr::Compare { + left: Box::new(col), + op: CompareOp::NotLike, + right: Box::new(val(&p.val)), + }, + FilterOp::IsNull => L2Expr::IsNull(Box::new(col)), + FilterOp::IsNotNull => L2Expr::IsNotNull(Box::new(col)), } - } + }) + .collect(); + let pred = if conjuncts.len() == 1 { + conjuncts.into_iter().next().unwrap() + } else { + L2Expr::BoolAnd(conjuncts) + }; + QueryExpr::Filter { + pred, + input: Box::new(input), + } +} + +/// Fold `group` into the nearest `Aggregate` inside `qe` — `asap_l2`'s +/// `Aggregate` carries `keys`/`without` directly (no separate `Partition` +/// node to wrap in; see `intent_algebra::relational`'s module doc). +/// Mirrors `lower.rs`'s `fold_partition_keys`, one layer up (L2, not L3): +/// handles the shapes the walker actually produces (a bare `Aggregate` or +/// a `Window` wrapping one); anything else (`BinaryOp`, a bare `Source`) +/// has no `Aggregate` to fold into and passes through unchanged — e.g. +/// `sum by (host) (a or b)`, where the group modifier belongs to a +/// `BinaryOp` composition, not a reducing aggregate. +fn fold_group_mod(qe: QueryExpr, group: Option<&GroupMod>) -> QueryExpr { + let Some(group) = group else { + return qe; + }; + if group.is_empty() { + return qe; + } + match qe { + QueryExpr::Aggregate { + aggs, + having, + input, + .. + } => QueryExpr::Aggregate { + keys: group + .keys() + .iter() + .cloned() + .map(QeColumnRef::Named) + .collect(), + without: matches!(group, GroupMod::Without(_)), + aggs, + having, + input, + }, + QueryExpr::Window { + duration, + slide, + input, + } => QueryExpr::Window { + duration, + slide, + input: Box::new(fold_group_mod(*input, Some(group))), + }, + other => other, } } From 2ce91517798c57920a615dd12b06da6e65e6343c Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 19 Jul 2026 10:27:09 -0600 Subject: [PATCH 05/11] feat(control_plane): Phase 2 step 5 -- relocate cse.rs to optimizer, adopt asap-plan's algorithm Relocates the workload-level CSE pass from intent_algebra::cse to optimizer::cse, per the Phase 0 decision: ASAPController places this pass in crates/plan ("the cost-aware optimizer layer over the L3 intent algebra"), not alongside the L3 IR type definitions. optimizer is the matching layer here (R1-R12 in engine.rs, and this pass's real consumer, optimizer::cost::workload_cost, which moves with it). Adopts asap_plan::cse's algorithm directly: structural-equality candidate scan (QueryExpr: PartialEq on a Vec) instead of the pre-merge Debug- string-keyed HashMap ("{:?} is not a guaranteed-injective, stable identity contract" per ASAPController's own comment -- a real shortcut this repo's version was taking). CseWorkloadPlan and WorkloadCostPlan now carry asap_ir's own BindingName/QueryId directly instead of a types_v2 wrapper, removing the boundary conversion every QueryExpr::Ref construction site needed. Regression check: optimizer::engine's R8 CommonSubexprElim and this pass are not redundant despite both doing "CSE" -- R8 dedupes Scan leaves across the branches of one Merge node inside a single query tree; this pass dedupes Aggregate-child subtrees across the root queries of a multi-query workload. Intra-tree vs. inter-tree, disjoint inputs, so relocating this pass changes neither what R8 fires on nor when. Full suite re-run confirms: 763 passed (2 new regression-guard tests ported alongside the algorithm), same 1 pre-existing unrelated failure. Co-Authored-By: Claude Sonnet 5 --- control_plane/src/intent_algebra/mod.rs | 15 +- control_plane/src/optimizer/cost/mod.rs | 15 +- .../src/{intent_algebra => optimizer}/cse.rs | 182 +++++++++++++----- control_plane/src/optimizer/mod.rs | 2 + 4 files changed, 148 insertions(+), 66 deletions(-) rename control_plane/src/{intent_algebra => optimizer}/cse.rs (59%) diff --git a/control_plane/src/intent_algebra/mod.rs b/control_plane/src/intent_algebra/mod.rs index 45bff7cf..55267aff 100644 --- a/control_plane/src/intent_algebra/mod.rs +++ b/control_plane/src/intent_algebra/mod.rs @@ -41,10 +41,13 @@ //! may share a `LetBinding` only when the producer's output schema //! has at least one `unique_keys` set. This is the proof point that //! `unique_keys` is load-bearing. -//! - [`dedupe_subtrees`] — basic workload-level CSE pass that hoists +//! - `dedupe_subtrees` — the basic workload-level CSE pass that hoists //! structurally-identical sub-trees into shared `LetBinding`s -//! (`design.md` §6 batched-queries example, ~line 1256). The full -//! alpha-equivalence + nested-CSE algorithm is downstream. +//! (`design.md` §6 batched-queries example, ~line 1256) — lives in +//! `optimizer::cse` as of Phase 2 step 5 (ASAPController places this +//! pass in its cost-aware planning crate, not alongside the L3 IR type +//! definitions). The full alpha-equivalence + nested-CSE algorithm is +//! downstream. //! //! Scope reduction. The PR ships the variants the DC + PromQL deployment //! actually needs (`Scan`, `Window`, `Aggregate`, `LetBinding`, `Ref`). @@ -72,11 +75,6 @@ #![allow(dead_code, unused_imports)] pub mod agg_intent; -pub mod cse; -// `L2Expr` isn't used yet -- `relational.rs` (L2) still builds its own -// `ScalarExpr`, not `L2Expr`; that's the next Phase 2 step. `L3Expr` is -// used now: `query_expr.rs`'s `Predicate` is `L3Expr`-based as of this -// merge. pub mod expr_ir; pub mod query_expr; pub mod schema; @@ -111,7 +109,6 @@ pub use agg_intent::{ default_frequency, default_quantile, frequency, is_frequency_heavy_hitter, output_column, ranking_measure, AggIntent, MathFunc, RankingMeasure, TimeFunc, }; -pub use cse::{dedupe_subtrees, CseWorkloadPlan}; pub use expr_ir::{ArithOp, ColumnRef, CompareOp, Expr, L2Expr, L3Expr, L3Scalar}; pub use query_expr::{ aggregate_output_schema, between, conjoin, label_filter_to_predicate, AtModifier, BinaryOpKind, diff --git a/control_plane/src/optimizer/cost/mod.rs b/control_plane/src/optimizer/cost/mod.rs index 20f070f3..6c10eb11 100644 --- a/control_plane/src/optimizer/cost/mod.rs +++ b/control_plane/src/optimizer/cost/mod.rs @@ -352,9 +352,10 @@ fn apply_delta_decision_with( // the lint baseline clean — mirrors the module-wide allowance on // `intent_algebra/mod.rs` while Phase B sat consumer-less. #[allow(unused_imports)] -use crate::intent_algebra::{AggIntent, BindingScope, QueryExpr, QueryExprError, Schema}; +use asap_ir::intent_algebra::{BindingName, QueryId}; + #[allow(unused_imports)] -use crate::types_v2::{BindingName, QueryId}; +use crate::intent_algebra::{AggIntent, BindingScope, QueryExpr, QueryExprError, Schema}; /// Bundled cost of a multi-query workload, with per-root contributions /// and the savings unlocked by shared-producer credit. Returned by @@ -435,12 +436,7 @@ pub fn workload_cost(plan: &WorkloadCostPlan<'_>) -> Result Column { diff --git a/control_plane/src/intent_algebra/cse.rs b/control_plane/src/optimizer/cse.rs similarity index 59% rename from control_plane/src/intent_algebra/cse.rs rename to control_plane/src/optimizer/cse.rs index 545223f9..1014fce9 100644 --- a/control_plane/src/intent_algebra/cse.rs +++ b/control_plane/src/optimizer/cse.rs @@ -1,30 +1,59 @@ //! Workload-level Common Sub-Expression Elimination. //! -//! Per `control_plane/docs/design.md` §6 batched-queries example (line ~1256 -//! through ~1320). Multi-root planning hoists shared sub-DAGs into -//! `LetBinding`s so the cost model can credit the producer once. +//! ## Phase 2 step 5 (docs/migration-plan-backend-plan.md) +//! +//! Relocated from `intent_algebra::cse` to `optimizer::cse` per the +//! Phase 0 decision: ASAPController places this pass in `crates/plan` +//! ("the cost-aware optimizer layer (L4 decisions) over the L3 intent +//! algebra"), not alongside the L3 IR type definitions — this repo's +//! `optimizer` module is that layer (R1-R12 in `engine.rs`, and this +//! pass's real consumer, `optimizer::cost::workload_cost`). The +//! algorithm is otherwise identical to `asap_plan::cse` (ASAPController's +//! `crates/plan/src/cse.rs`) — adopted directly per the tie-break rule, +//! including its structural-equality candidate scan (`QueryExpr: +//! PartialEq` on a `Vec`, not a `Debug`-string-keyed `HashMap` — `{:?}` +//! is not a guaranteed-injective, stable identity contract, this repo's +//! pre-merge implementation's own shortcut). +//! +//! [`CseWorkloadPlan`] now carries `asap_ir`'s own `BindingName`/`QueryId` +//! directly (`asap_ir::intent_algebra::{BindingName, QueryId}`), not +//! `types_v2`'s separate wrapper types — matching `asap_plan::cse` and +//! removing the boundary conversion the pre-merge version needed at +//! every `QueryExpr::Ref` construction site (`asap_ir`'s `BindingName` +//! was always the *only* type that could actually name a `Ref`/ +//! `LetBinding`; the `types_v2` copy was this pass's own bookkeeping +//! type, not a real second identity). `optimizer::cost::WorkloadCostPlan` +//! — the pass's real consumer — moves with it for the same reason. +//! `types_v2::BindingName` remains the right type everywhere else it's +//! used today (`sketch_algebra::PhysicalExpr`'s own, unrelated L4 +//! binding-name field; `pipeline.rs`'s `QueryId`) — this change is scoped +//! to the CSE↔cost-model boundary only. +//! +//! ## Regression note (R8 `CommonSubexprElim`) +//! +//! `optimizer::engine`'s R8 rule and this pass are *not* redundant, +//! despite both doing "common subexpression elimination": R8 dedupes +//! `Scan` leaves across the **branches of one `Merge` node inside a +//! single query tree**; this pass dedupes `Aggregate`-child subtrees +//! **across the root queries of a multi-query workload**. Intra-tree vs. +//! inter-tree — disjoint inputs, so relocating this pass doesn't change +//! what R8 fires on or when, and both stay. //! -//! Phase F lands the **gate + a basic implementation** that handles the -//! literal "≥2 root queries with identical sub-expressions" case from the -//! design — sufficient to make the workload-cost path observable end-to- -//! end. The fully-general CSE algorithm (alpha-equivalence across -//! `LetBinding` rebinding, schema-merge across compatible-but-not-identical -//! shapes, cross-binding nested CSE) is deferred per design.md §6 line -//! ~562 — it is a downstream optimisation pass, not part of the IR -//! contract Phase F is delivering. +//! Per `control_plane/docs/design.md` §6 batched-queries example (line +//! ~1256 through ~1320). Multi-root planning hoists shared sub-DAGs into +//! `LetBinding`s so the cost model can credit the producer once. //! -//! Legality is gated by [`cse_reuse_is_legal`](super::schema::cse_reuse_is_legal): +//! Legality is gated by [`cse_reuse_is_legal`](crate::intent_algebra::cse_reuse_is_legal): //! a candidate sub-DAG only becomes a `LetBinding` when its output schema //! has at least one `unique_keys` set (§6 line ~1356 — the field is //! load-bearing for this pass). #![allow(dead_code)] -use std::collections::HashMap; +use asap_ir::intent_algebra::{BindingName, QueryId}; +use crate::intent_algebra::cse_reuse_is_legal; use crate::intent_algebra::query_expr::QueryExpr; -use crate::intent_algebra::schema::cse_reuse_is_legal; -use crate::types_v2::{BindingName, QueryId}; /// Multi-root container produced by the CSE pass — mirrors the shape of /// `types_v2::WorkloadPlan` (§6 batched-queries example) but uses the @@ -47,12 +76,12 @@ pub struct CseWorkloadPlan { /// where the duplicate sub-tree used to live. Per design.md §6 line /// ~1272 ("a workload-level CSE pass `core::lower::workload::dedupe_subtrees`"). /// -/// **Phase F scope.** Implements the basic case: identifies sub-trees -/// that appear verbatim (structural equality via `PartialEq`) in ≥2 root -/// inputs and hoists them. Schema-equivalent-but-not-identical sub-trees, -/// alpha-equivalence over inner `LetBinding`s, and recursive nested CSE -/// are deferred — they are the optimisation half of the pass and live -/// downstream of this PR. +/// **Scope.** Implements the basic case: identifies sub-trees that +/// appear verbatim (structural equality via `PartialEq`) in ≥2 root +/// inputs and hoists them. Schema-equivalent-but-not-identical +/// sub-trees, alpha-equivalence over inner `LetBinding`s, and recursive +/// nested CSE are deferred — they are the optimisation half of the pass +/// and live downstream of this PR. /// /// **Legality.** A candidate sub-tree is hoisted only when /// `cse_reuse_is_legal(&candidate.output_schema(), consumer_count)` @@ -69,11 +98,15 @@ pub fn dedupe_subtrees(roots: Vec<(QueryId, QueryExpr)>) -> CseWorkloadPlan { }; } - // Phase F: identify candidate sub-trees that appear as the immediate - // child of an `Aggregate` in ≥2 roots. The batched-queries example - // shape — multiple `Aggregate`s sharing one `Window`-child producer - // — is the case Phase F lights up; richer detection is downstream. - let mut candidate_counts: HashMap = HashMap::new(); + // Identify candidate sub-trees that appear as the immediate child of + // an `Aggregate` in ≥2 roots. The batched-queries example shape — + // multiple `Aggregate`s sharing one `Window`-child producer — is the + // case this lights up; richer detection is downstream. Grouped by + // structural equality (`QueryExpr: PartialEq`), not `Debug` output — + // `{:?}` is not a guaranteed-injective, stable identity contract. The + // candidate set is one entry per distinct root child, so this linear + // scan is bounded by the number of distinct queries. + let mut candidate_counts: Vec<(QueryExpr, usize)> = Vec::new(); for (_, root) in &roots { if let QueryExpr::Aggregate { child, .. } = root { // Skip already-aliased children (a `Ref` is not a candidate @@ -81,23 +114,22 @@ pub fn dedupe_subtrees(roots: Vec<(QueryId, QueryExpr)>) -> CseWorkloadPlan { if matches!(**child, QueryExpr::Ref { .. }) { continue; } - // Use the Debug representation as a structural-key proxy. - // Cheap to compute and matches `PartialEq` for - // `QueryExpr` → adequate for the Phase F basic case. - let key = format!("{child:?}"); - let entry = candidate_counts - .entry(key) - .or_insert_with(|| ((**child).clone(), 0)); - entry.1 += 1; + match candidate_counts + .iter_mut() + .find(|(e, _)| e == child.as_ref()) + { + Some(entry) => entry.1 += 1, + None => candidate_counts.push(((**child).clone(), 1)), + } } } - // Pick the most-shared legal candidate. Phase F hoists at most one - // binding per call; the "hoist all eligible candidates" generalisation - // is a follow-up. Choosing the most-shared first matches the design's - // priority — biggest reuse first. + // Pick the most-shared legal candidate. This pass hoists at most one + // binding per call; the "hoist all eligible candidates" + // generalisation is a follow-up. Choosing the most-shared first + // matches the design's priority — biggest reuse first. let mut chosen: Option<(QueryExpr, usize)> = None; - for (_key, (expr, count)) in candidate_counts.into_iter() { + for (expr, count) in candidate_counts.into_iter() { if count < 2 { continue; } @@ -109,7 +141,7 @@ pub fn dedupe_subtrees(roots: Vec<(QueryId, QueryExpr)>) -> CseWorkloadPlan { if cse_reuse_is_legal(&out_schema, count).is_err() { continue; } - // Bigger fan-in wins; ties broken arbitrarily (HashMap order). + // Bigger fan-in wins; ties broken by input order. match &chosen { Some((_, best_count)) if *best_count >= count => {} _ => chosen = Some((expr, count)), @@ -142,11 +174,7 @@ pub fn dedupe_subtrees(roots: Vec<(QueryId, QueryExpr)>) -> CseWorkloadPlan { output_names, having, child: Box::new(QueryExpr::Ref { - // `QueryExpr::Ref.name` is `asap_ir`'s `BindingName` - // (positional-workload-agnostic) — distinct from this - // module's own `types_v2::BindingName` (CSE-plan-level - // binding identity). Convert at the boundary. - name: asap_ir::intent_algebra::BindingName::new(binding_name.0.clone()), + name: binding_name.clone(), }), }, other => other, @@ -242,6 +270,36 @@ mod tests { assert_eq!(out.roots[0].1, q); } + /// Issue #115 (ASAPController): CSE dedupes on `AggIntent` equality. + /// Before `Quantile` carried its input column, `median(a)` and + /// `median(b)` compared equal, so two aggregates over *different* + /// columns collapsed into one — a wrong answer, not just a missed + /// optimisation. The merged `AggIntent::Quantile { col: Option, .. }` + /// already carries the column, so this is a regression guard, not new + /// behavior this repo needed to add. + #[test] + fn quantiles_over_different_columns_do_not_dedupe() { + let mk = |col: usize| QueryExpr::Aggregate { + by: vec![1].into(), + aggs: vec![AggIntent::Quantile { + col: Some(col), + q: 0.5, + accuracy: AccuracyTarget::Epsilon(0.01), + }], + output_names: Vec::new(), + having: None, + child: Box::new(windowed_scan()), + }; + let (a, b) = (mk(2), mk(3)); + assert_ne!(a, b, "distinct-column quantiles must not compare equal"); + + let out = dedupe_subtrees(vec![(QueryId::new("q1"), a), (QueryId::new("q2"), b)]); + assert_ne!( + out.roots[0].1, out.roots[1].1, + "aggregates over different columns must not collapse" + ); + } + /// design.md §6 batched-queries example basic case: two queries with /// identical `Window` sub-trees — the deduper hoists the shared /// producer into a binding and rewrites each root to reference it. @@ -282,7 +340,7 @@ mod tests { QueryExpr::Aggregate { child, .. } => assert_eq!( **child, QueryExpr::Ref { - name: asap_ir::intent_algebra::BindingName::new("shared_0"), + name: BindingName::new("shared_0"), }, "Aggregate child should be a Ref to the hoisted binding" ), @@ -302,8 +360,8 @@ mod tests { having: None, child: Box::new(windowed_scan()), }; - // q2 uses a different scan (different metric) — Debug repr - // differs → no hoisting. + // q2 uses a different scan (different metric) → structurally + // distinct → no hoisting. let other_scan = QueryExpr::Scan { source: Source::TimeSeries { metric: "different_metric".into(), @@ -340,4 +398,32 @@ mod tests { assert_eq!(out.roots[0].1, q1); assert_eq!(out.roots[1].1, q2); } + + /// Schema without `unique_keys` → CSE refuses to share even if + /// structurally identical (ASAPController's own regression case for + /// the legality gate, ported alongside the algorithm). + #[test] + fn dedupe_subtrees_no_shared_subexpr_when_unique_keys_absent() { + let scan_no_uk = QueryExpr::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("value", DataType::Float64), + ], + 0, + vec![], + ), + }; + let mk = || QueryExpr::Aggregate { + by: vec![].into(), + aggs: vec![AggIntent::Sum { col: None }], + output_names: Vec::new(), + having: None, + child: Box::new(scan_no_uk.clone()), + }; + let out = dedupe_subtrees(vec![(QueryId::new("q1"), mk()), (QueryId::new("q2"), mk())]); + assert!(out.bindings.is_empty(), "no unique_keys → no hoisting"); + } } diff --git a/control_plane/src/optimizer/mod.rs b/control_plane/src/optimizer/mod.rs index a6f358b7..ea248fc6 100644 --- a/control_plane/src/optimizer/mod.rs +++ b/control_plane/src/optimizer/mod.rs @@ -26,12 +26,14 @@ pub mod baseline; pub mod cost; +pub mod cse; pub mod engine; pub mod rules; pub mod trait_def; // Re-exports — preserve the surface that `crate::algebra::QueryOptimizer` // and `crate::planner::*` consumers historically relied on. +pub use cse::{dedupe_subtrees, CseWorkloadPlan}; pub use engine::{DeploymentConstraints, QueryOptimizer}; pub use trait_def::{OptimizerRule, RuleCategory}; From 988deb036c59c84763ab8a3ae6d37338208a8935 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 19 Jul 2026 10:42:34 -0600 Subject: [PATCH 06/11] feat(control_plane): Phase 2 step 6 -- PromQL topk/rate semantic retarget to match ASAPController Adopts ASAPController's RankingMeasure/is_frequency_heavy_hitter gate for topk/bottomk instead of unconditionally forcing AggFunc::Count inside any topk(...): only descending topk ranking by count_over_time(...) now takes the heavy-hitter TopK path; everything else (bottomk, topk over a non-count measure) lowers to a generic Sort+Limit. Fixes a real correctness bug where e.g. topk(k, avg_over_time(...)) was silently treated as a Count/Frequency heavy-hitter aggregate. Rate/Increase/Changes/Resets/Delta/IDelta/Deriv/PredictLinear/ DoubleExpSmoothing now map to their own dedicated AggIntents instead of collapsing onto Sum/Count. This activates capability_for()'s existing Rate/Increase -> ExactAgg(Increase) mapping and correctly routes the archive-only counter-derivative family (Delta/Changes/Resets/...) to None instead of falsely claiming an exact-agg capability. asap_tier_analysis.rs's outer_fn is computed independently of required_capability (straight off the raw PromQL AST function name), so this only needed test-expectation updates, not a dispatch-logic redesign. Adds structural regression tests asserting the raw QueryExpr shape (TopK vs Sort+Limit) for topk/bottomk, not just the flattened ParsedQuery view. Co-Authored-By: Claude Sonnet 5 --- control_plane/src/asap_tier_analysis.rs | 89 ++++++--- control_plane/src/intent_algebra/lower.rs | 185 +++++++++++------ control_plane/src/query_parser/promql.rs | 230 +++++++++++++++++++--- 3 files changed, 389 insertions(+), 115 deletions(-) diff --git a/control_plane/src/asap_tier_analysis.rs b/control_plane/src/asap_tier_analysis.rs index 21ab4449..9bec25b2 100644 --- a/control_plane/src/asap_tier_analysis.rs +++ b/control_plane/src/asap_tier_analysis.rs @@ -1064,19 +1064,19 @@ mod tests { assert!(a.unsupported.is_none(), "{a:?}"); assert_eq!( a.candidates[0].required_capability, - Capability::ExactAgg(AggregationType::Sum) + Capability::ExactAgg(AggregationType::Increase) ); } #[test] fn irate_binds_to_exact_agg() { // `irate` shares `AggFunc::Rate` with `rate` in - // `query_parser::promql`; both lower to `AggIntent::Sum`. + // `query_parser::promql`; both lower to `AggIntent::Rate`. let a = analyze_promql_for_asap_tier("irate(http_requests_total[5m])"); assert!(a.unsupported.is_none(), "{a:?}"); assert_eq!( a.candidates[0].required_capability, - Capability::ExactAgg(AggregationType::Sum) + Capability::ExactAgg(AggregationType::Increase) ); } @@ -1086,7 +1086,7 @@ mod tests { assert!(a.unsupported.is_none(), "{a:?}"); assert_eq!( a.candidates[0].required_capability, - Capability::ExactAgg(AggregationType::Sum) + Capability::ExactAgg(AggregationType::Increase) ); } @@ -1104,14 +1104,17 @@ mod tests { // ── outer_fn — rate vs plain disambiguation ────────────────────────── // // Regression coverage for the PR that retired the engine's - // `query_contains_rate_call` raw-PromQL re-parser. The analyzer's - // lowerer collapses `rate(metric[r])`, `sum_over_time(metric[r])`, - // `sum(metric)`, and the bare selector all onto `AggIntent::Sum` / - // `Capability::ExactAgg(Sum)` — so the engine can't tell from the - // capability alone which the user wrote. The `outer_fn` field on - // `ASAPTierCandidate` carries the rate-vs-plain distinction so the - // engine's reducer dispatch is a typed branch instead of a raw-PromQL - // re-parse. + // `query_contains_rate_call` raw-PromQL re-parser. `outer_fn` is + // computed independently of `required_capability` — straight off the + // raw PromQL function name (`set_counter_fn`), not off the lowered + // `AggIntent` — so the engine's reducer dispatch has the rate-vs- + // increase-vs-plain distinction as a typed branch instead of a + // raw-PromQL re-parse, regardless of whether the capability + // computation happens to agree or differ across cases. `rate`/ + // `irate`/`increase` now bind to distinct-from-`sum`/`sum_over_time` + // capabilities (`ExactAgg(Increase)` vs `ExactAgg(Sum)` — see + // `rate_and_increase_share_capability_but_differ_on_outer_fn` below + // for the pairing that still needs `outer_fn` to disambiguate). #[test] fn rate_candidate_carries_outer_fn_rate() { @@ -1146,15 +1149,15 @@ mod tests { #[test] fn increase_candidate_carries_outer_fn_increase() { - // `increase(metric[r])` shares `Capability::ExactAgg(Sum)` with - // `rate`/`sum_over_time`; the `outer_fn` field carries the + // `increase(metric[r])` shares `Capability::ExactAgg(Increase)` + // with `rate`/`irate`; the `outer_fn` field carries the // distinction so the engine sums deltas in `[t-r,t]` WITHOUT the // rate divisor (issue #301). let a = analyze_promql_for_asap_tier("increase(http_requests_total[5m])"); assert!(a.unsupported.is_none(), "{a:?}"); assert_eq!( a.candidates[0].required_capability, - Capability::ExactAgg(AggregationType::Sum), + Capability::ExactAgg(AggregationType::Increase), "{a:?}" ); assert_eq!(a.candidates[0].outer_fn, OuterFn::Increase, "{a:?}"); @@ -1198,7 +1201,7 @@ mod tests { assert!(a.unsupported.is_none(), "{a:?}"); assert_eq!( a.candidates[0].required_capability, - Capability::ExactAgg(AggregationType::Sum), + Capability::ExactAgg(AggregationType::Increase), "{a:?}" ); assert_eq!(a.candidates[0].outer_fn, OuterFn::Rate, "{a:?}"); @@ -1207,26 +1210,58 @@ mod tests { } #[test] - fn rate_and_sum_over_time_share_capability_but_differ_on_outer_fn() { - // Both collapse to `Capability::ExactAgg(Sum)`; the engine MUST - // disambiguate via the typed `outer_fn` field, not by string- - // parsing the raw PromQL. This test pins the asymmetry the - // engine's dispatch reads off. + fn rate_and_sum_over_time_differ_on_capability_and_outer_fn() { + // Pre-PromQL-frontend-semantic-retarget, `rate`/`irate`/ + // `increase`/`sum_over_time` all collapsed onto `AggIntent::Sum` + // (`Capability::ExactAgg(Sum)`), so `outer_fn` was the *only* + // thing that told the engine `rate(...)` needed a rate-divisor + // step `sum_over_time(...)` didn't. `rate`/`increase` now bind to + // their own dedicated `AggIntent`s (`Capability::ExactAgg( + // Increase)`, distinct from `sum_over_time`'s `ExactAgg(Sum)`) — + // a strictly more precise classification, not a regression: the + // capability itself now carries part of the distinction + // `outer_fn` used to carry alone. let rate = analyze_promql_for_asap_tier("rate(http_requests_total[5m])"); let sot = analyze_promql_for_asap_tier("sum_over_time(http_requests_total[5m])"); - assert_eq!( + assert_ne!( rate.candidates[0].required_capability, sot.candidates[0].required_capability, - "rate and sum_over_time should produce the same Capability" + "rate and sum_over_time should now bind to distinct capabilities" ); - assert_ne!( - rate.candidates[0].outer_fn, sot.candidates[0].outer_fn, - "rate and sum_over_time MUST differ on outer_fn so the engine \ - can dispatch correctly without re-parsing the raw PromQL" + assert_eq!( + rate.candidates[0].required_capability, + Capability::ExactAgg(AggregationType::Increase) + ); + assert_eq!( + sot.candidates[0].required_capability, + Capability::ExactAgg(AggregationType::Sum) ); assert_eq!(rate.candidates[0].outer_fn, OuterFn::Rate); assert_eq!(sot.candidates[0].outer_fn, OuterFn::SumOverTime); } + #[test] + fn rate_and_increase_share_capability_but_differ_on_outer_fn() { + // The pairing that now needs `outer_fn` to disambiguate: `rate` + // and `increase` both bind to `Capability::ExactAgg(Increase)` + // (rate = increase / range, an L4/reducer-level division, not a + // capability difference) — the engine still can't tell from the + // capability alone whether to apply the rate divisor, so + // `outer_fn` carries that distinction. + let rate = analyze_promql_for_asap_tier("rate(http_requests_total[5m])"); + let inc = analyze_promql_for_asap_tier("increase(http_requests_total[5m])"); + assert_eq!( + rate.candidates[0].required_capability, inc.candidates[0].required_capability, + "rate and increase should share the same Capability" + ); + assert_ne!( + rate.candidates[0].outer_fn, inc.candidates[0].outer_fn, + "rate and increase MUST differ on outer_fn so the engine can \ + dispatch correctly without re-parsing the raw PromQL" + ); + assert_eq!(rate.candidates[0].outer_fn, OuterFn::Rate); + assert_eq!(inc.candidates[0].outer_fn, OuterFn::Increase); + } + // ── outer_agg — outer aggregation operator on function results ────── // // Regression coverage for issue #296: the asap engine was rejecting diff --git a/control_plane/src/intent_algebra/lower.rs b/control_plane/src/intent_algebra/lower.rs index c729d1b6..37a8aec4 100644 --- a/control_plane/src/intent_algebra/lower.rs +++ b/control_plane/src/intent_algebra/lower.rs @@ -9,21 +9,43 @@ //! //! Now that `relational.rs` (L2) itself merged onto `asap_l2` (see that //! file's module doc), this converter is control_plane's *own* — not a -//! re-export of `asap_l2::lower::convert_root` — for one deliberate -//! reason: `asap_l2`'s `AggFunc`→`AggIntent` mapping -//! (`agg_func_to_intent` in its `lower.rs`) produces literal -//! `AggIntent::Avg` / `Rate` / `Increase` for those `AggFunc`s, whereas -//! this repo needs the pre-merge behavior a real, tested consumer -//! (`asap_tier_analysis`'s `outer_fn` dispatch) still depends on: -//! `avg_over_time` → a p50 quantile-sketch approximation, and -//! `Rate`/`Increase`/`Delta` → `AggIntent::Sum` (disambiguated via the -//! separate `outer_fn` field, not by intent shape). Reconciling that -//! dispatch to consume the dedicated intents directly is real, -//! deliberately-deferred follow-up work (tracked for the PromQL-frontend -//! semantic-retarget step), not a byproduct of this type merge. Every -//! *structural* piece below (scalar resolution, schema threading, the -//! `GroupKeys` shape) is unchanged from `asap_l2`'s own converter — -//! only the `Aggregate`/`AggFunc` handling is control_plane-specific. +//! re-export of `asap_l2::lower::convert_root` — for one remaining +//! deliberate reason: `avg_over_time` stays a p50 quantile-sketch +//! approximation rather than `asap_l2`'s literal (exact, +//! non-mergeable) `AggIntent::Avg`, because +//! `query_parser::QeCollector::collect_op` (which has no `Avg` arm of +//! its own) relies on that substitution to classify `avg_over_time` as +//! `AggType::Quantile`. Every *structural* piece below (scalar +//! resolution, schema threading, the `GroupKeys` shape) is unchanged +//! from `asap_l2`'s own converter — only this one `Aggregate`/`AggFunc` +//! mapping choice is control_plane-specific. +//! +//! **PromQL-frontend semantic-retarget step (topk/rate precision fix).** +//! `Rate`/`Increase`/`Changes`/`Delta`/`IDelta`/`Deriv`/`PredictLinear`/ +//! `DoubleExpSmoothing`/`Resets` used to all collapse onto +//! `AggIntent::Sum` or `Count` here — a crude placeholder bucketing +//! predating `asap_l2`'s own per-function `AggIntent` vocabulary. They +//! now map onto their own dedicated intents, matching `asap_l2`'s +//! mapping exactly (this repo no longer diverges from it for these). +//! `Rate`/`Increase` activate a real, previously-dormant +//! `capability_for` arm (`Rate | Increase => ExactAgg(Increase)`) for +//! the first time via the PromQL path; the rest are archive-only +//! either way, so the fix is `capability_for` now correctly returning +//! `None` instead of a wrong `Some(...)` for functions the ASAP tier +//! was never actually able to answer that way. `asap_tier_analysis`'s +//! `outer_fn` field is unaffected by any of this — it's computed +//! independently off the raw PromQL function name, not the lowered +//! `AggIntent`. +//! +//! `query_parser::promql`'s `topk`/`bottomk` handling was the other half +//! of this step: it used to force every `topk(...)` argument into a +//! `Count`-shaped inner regardless of what was actually being ranked +//! (silently wrong for `topk(k, avg_over_time(...))`). It now only takes +//! the heavy-hitter `TopK` path when ranking descending by +//! `count_over_time(...)` specifically (`RankingMeasure::Frequency`, +//! the one realised heavy-hitter measure) — everything else, including +//! every `bottomk`, becomes a generic `Sort + Limit`, matching +//! ASAPController's `frontend-promql` design. //! //! Two consequences of adopting `asap_l2::relational::QueryExpr`: //! @@ -502,43 +524,57 @@ fn agg_func_to_intents(func: &AggFunc, frequency_trigger: bool) -> Vec vec![q(*phi)], AggFunc::CountDistinct => vec![crate::intent_algebra::default_cardinality()], AggFunc::HeavyHitters { .. } => vec![crate::intent_algebra::default_frequency()], - // `Rate` / `Increase` / `Delta` map onto `AggIntent::Sum`, not the - // dedicated `AggIntent::Rate` / `Increase` / `Delta` variants — - // the `asap_tier_analysis` engine dispatch deliberately collapses - // all of `rate` / `irate` / `increase` / `sum_over_time` onto one - // `Capability::ExactAgg(Sum)` and disambiguates via the separate - // typed `outer_fn` field instead (see - // `asap_tier_analysis::tests::rate_and_sum_over_time_share_ - // capability_but_differ_on_outer_fn` and the surrounding - // "outer_fn — rate vs plain disambiguation" test block). Using - // the dedicated intents here would fragment that dispatch. - // `Changes`/`Resets` also collapse onto `Count` for the same - // reason (this repo's pre-`asap_l2`-merge `AggFunc` never - // distinguished them from `Count` either) -- and everything else - // in the "counter-derivative range functions" family collapses - // onto `Delta`, matching this repo's pre-merge behavior exactly. - // Properly distinguishing all of these is deferred to the - // PromQL-frontend semantic-retarget step, alongside the - // `outer_fn` reconciliation above. - AggFunc::Rate { .. } | AggFunc::Increase { .. } => vec![AggIntent::Sum { col: None }], - AggFunc::Changes | AggFunc::Resets => { - agg_func_to_intents(&AggFunc::Count, frequency_trigger) + // `Rate` / `Increase` now map onto their own dedicated + // `AggIntent`s (PromQL-frontend semantic-retarget step) -- + // `capability_for` already has a real, tested + // `Rate | Increase => ExactAgg(Increase)` arm (`sketch_algebra:: + // capability`); this activates it via the PromQL path for the + // first time. `asap_tier_analysis`'s `outer_fn` field is + // unaffected -- it's computed independently, straight off the + // raw PromQL function name (`trace_from_promql`'s + // `set_counter_fn`), not off the lowered `AggIntent`, so it + // still tells the engine's reducer *how* to interpret the + // accumulated value (rate needs a divide-by-range step, increase + // doesn't) regardless of what capability got matched. + AggFunc::Rate { .. } => vec![AggIntent::Rate], + AggFunc::Increase { .. } => vec![AggIntent::Increase], + // `Changes` / `Delta` / `IDelta` / `Deriv` / `PredictLinear` / + // `DoubleExpSmoothing` / `Resets` likewise now map onto their own + // dedicated `AggIntent`s instead of collapsing onto `Count`/`Sum` + // -- all are archive-only (`agg_intent::archive_only`; no + // `Bind*` rule exists for any of them, same as before this fix), + // so the real effect is `capability_for` now correctly returning + // `None` (route to archive) instead of the wrong + // `Some(ExactAgg(Sum))` / `Some(CardinalityApprox)` the old + // Sum/Count collapse produced -- these functions were never + // actually answerable from the ASAP tier that way. + AggFunc::Changes => vec![AggIntent::Changes], + AggFunc::Resets => vec![AggIntent::Resets], + AggFunc::Delta => vec![AggIntent::Delta], + AggFunc::IDelta => vec![AggIntent::IDelta], + AggFunc::Deriv => vec![AggIntent::Deriv], + AggFunc::PredictLinear { seconds } => vec![AggIntent::PredictLinear { seconds: *seconds }], + AggFunc::DoubleExpSmoothing { smoothing, trend } => { + vec![AggIntent::DoubleExpSmoothing { + smoothing: *smoothing, + trend: *trend, + }] } - AggFunc::Delta - | AggFunc::IDelta - | AggFunc::Deriv - | AggFunc::PredictLinear { .. } - | AggFunc::DoubleExpSmoothing { .. } => vec![AggIntent::Sum { col: None }], // Every remaining `AggFunc` (native-histogram accessors, - // math/trig, presence, time/calendar, `Group`/`CountValues`, - // the extended range-vector reducers) has no pre-`asap_l2`-merge - // equivalent in this repo's PromQL surface at all -- promql.rs - // doesn't construct any of them today (`walk_call_to_op`'s - // exhaustive function-name table has no arm reaching them), so - // there's no existing behavior to preserve. Map each directly - // onto its like-named `AggIntent` (all archive-only per + // math/trig, time/calendar, `Group`/`CountValues`, the extended + // range-vector reducers) has no pre-`asap_l2`-merge equivalent + // in this repo's PromQL surface at all -- promql.rs doesn't + // construct any of them today (`walk_call_to_op`'s exhaustive + // function-name table has no arm reaching them), so there's no + // existing behavior to preserve. Map each directly onto its + // like-named `AggIntent` (all archive-only per // `agg_intent::archive_only`, so this is inert until a real - // caller constructs one). + // caller constructs one). `Absent` / `AbsentOverTime` / + // `PresentOverTime` / `LastOverTime` *are* constructed by + // promql.rs today (`absent_over_time` / `present_over_time` / + // `last_over_time`) -- listed here rather than above only + // because they were already correctly mapped before this fix + // (never went through the Sum collapse). AggFunc::HistogramCount => vec![AggIntent::HistogramCount], AggFunc::HistogramSum => vec![AggIntent::HistogramSum], AggFunc::HistogramAvg => vec![AggIntent::HistogramAvg], @@ -804,19 +840,50 @@ mod tests { } #[test] - fn rate_and_increase_map_to_sum() { - for func in [ - AggFunc::Rate { - window: Duration::from_secs(300), - }, - AggFunc::Increase { - window: Duration::from_secs(300), - }, - ] { + fn rate_and_increase_map_to_dedicated_intents() { + let cases = [ + ( + AggFunc::Rate { + window: Duration::from_secs(300), + }, + AggIntent::Rate, + ), + ( + AggFunc::Increase { + window: Duration::from_secs(300), + }, + AggIntent::Increase, + ), + ]; + for (func, expected) in cases { let legacy = agg(vec![], false, vec![agg_item("r", func)], src("m")); match convert_root(&legacy).unwrap() { CQueryExpr::Aggregate { aggs, .. } => { - assert!(matches!(aggs.as_slice(), [AggIntent::Sum { col: None }])) + assert_eq!(aggs.as_slice(), [expected]); + } + other => panic!("expected Aggregate, got {other:?}"), + } + } + } + + #[test] + fn changes_resets_delta_family_map_to_dedicated_intents() { + let cases = [ + (AggFunc::Changes, AggIntent::Changes), + (AggFunc::Resets, AggIntent::Resets), + (AggFunc::Delta, AggIntent::Delta), + (AggFunc::IDelta, AggIntent::IDelta), + (AggFunc::Deriv, AggIntent::Deriv), + ( + AggFunc::PredictLinear { seconds: 60.0 }, + AggIntent::PredictLinear { seconds: 60.0 }, + ), + ]; + for (func, expected) in cases { + let legacy = agg(vec![], false, vec![agg_item("x", func)], src("m")); + match convert_root(&legacy).unwrap() { + CQueryExpr::Aggregate { aggs, .. } => { + assert_eq!(aggs.as_slice(), [expected]); } other => panic!("expected Aggregate, got {other:?}"), } diff --git a/control_plane/src/query_parser/promql.rs b/control_plane/src/query_parser/promql.rs index 8244408d..7744ffc9 100644 --- a/control_plane/src/query_parser/promql.rs +++ b/control_plane/src/query_parser/promql.rs @@ -204,10 +204,12 @@ fn modifier_to_partition(modifier: &LabelModifier) -> GroupMod { // | `a op b` binary | BinaryOp { VectorMatch } | use crate::intent_algebra::relational::{ - AggFunc, AggItem, BinaryOpKind, ColumnRef as QeColumnRef, GroupSide, QueryExpr, + AggFunc, AggItem, BinaryOpKind, ColumnRef as QeColumnRef, GroupSide, L2SortKey, QueryExpr, SourceSpec as QeSourceSpec, VectorGrouping, VectorMatch, VectorMatchKind, }; -use crate::intent_algebra::{ArithOp, CompareOp}; +use crate::intent_algebra::{ + is_frequency_heavy_hitter, ArithOp, CompareOp, L2Expr, RankingMeasure, +}; use promql_parser::parser::{token::TokenType, BinaryExpr, VectorMatchCardinality}; /// Parse a PromQL expression string directly into an optimised [`QueryExpr`]. @@ -284,6 +286,21 @@ fn walk_qe(expr: &Expr, ctx: WalkCtx) -> anyhow::Result { } } +/// Whether `expr` (a `topk`/`bottomk` argument) is `count_over_time(...)`, +/// possibly parenthesized — the one PromQL shape ranked by +/// `RankingMeasure::Frequency`, the only measure with a realised +/// heavy-hitter sketch today (`agg_intent::is_frequency_heavy_hitter`). +/// A bare `count(...)` doesn't qualify: it's a *cross-series* reduction +/// (one value, not per-series), so ranking by it isn't a per-series +/// heavy-hitter shape in the first place. +fn is_count_over_time(expr: &Expr) -> bool { + match expr { + Expr::Paren(p) => is_count_over_time(p.expr.as_ref()), + Expr::Call(c) => c.func.name == "count_over_time", + _ => false, + } +} + fn walk_aggregate_qe(agg: &AggregateExpr, ctx: WalkCtx) -> anyhow::Result { let partition = agg.modifier.as_ref().map(modifier_to_partition); let op_name = format!("{}", agg.op); @@ -291,23 +308,72 @@ fn walk_aggregate_qe(agg: &AggregateExpr, ctx: WalkCtx) -> anyhow::Result { let k = extract_number_param(&agg.param)? as u64; + let descending = op_name == "topk"; + // A ranking is the heavy-hitter `TopK` intent only when it + // takes the *top* k (`descending` — `bottomk` never + // qualifies) *and* ranks by a measure with a realised + // heavy-hitter sketch — today, unweighted frequency + // (`count_over_time(...)`) only. Every other measure + // (avg/quantile/rate/a bare selector/...) is + // `RankingMeasure::NonAdditive` and falls through to a + // generic `Sort + Limit` order-by-value below — matching + // ASAPController's `frontend-promql` design (issue #38: "the + // descending-plus-measure rule is shared with the L3 + // canonicalize promotion so the two cannot drift"). Before + // this, `topk(k, )` unconditionally forced a + // `Count`-shaped inner regardless of what was actually being + // ranked — silently wrong for `topk(k, avg_over_time(...))` + // and friends. + let measure = if is_count_over_time(agg.expr.as_ref()) { + RankingMeasure::Frequency + } else { + RankingMeasure::NonAdditive + }; + if is_frequency_heavy_hitter(descending, measure) { + let inner_ctx = WalkCtx { + partition: partition.clone(), + topk: Some(k), + outer_count: false, + }; + let inner = walk_qe(agg.expr.as_ref(), inner_ctx)?; + // Don't fold the group keys into a separate wrapper here — + // the inner Aggregate already has them (or will, once its + // own construction site folds `partition` in). + return Ok(QueryExpr::TopK { + k, + by: partition + .as_ref() + .map(|p| p.keys().iter().cloned().map(QeColumnRef::Named).collect()) + .unwrap_or_default(), + input: Box::new(inner), + }); + } + // Generic order-by-value + limit: `bottomk`, and any `topk` + // ranking by a non-count measure. `ctx.topk` stays `None` so + // `build_qe_aggregate` doesn't force a `Count`-shaped inner. let inner_ctx = WalkCtx { partition: partition.clone(), - topk: Some(k), + topk: None, outer_count: false, }; let inner = walk_qe(agg.expr.as_ref(), inner_ctx)?; - // Don't fold the group keys into a separate wrapper here — the - // inner Aggregate already has them (or will, once its own - // construction site folds `partition` in). - let result = QueryExpr::TopK { - k, - by: partition + let sorted = QueryExpr::Sort { + keys: vec![L2SortKey { + expr: L2Expr::Column(QeColumnRef::SampleValue), + ascending: !descending, + nulls_first: false, + }], + partition_by: partition .as_ref() .map(|p| p.keys().iter().cloned().map(QeColumnRef::Named).collect()) .unwrap_or_default(), input: Box::new(inner), }; + let result = QueryExpr::Limit { + n: k, + offset: 0, + input: Box::new(sorted), + }; Ok(result) } "count" => { @@ -442,6 +508,16 @@ fn walk_call_qe(call: &Call, ctx: WalkCtx) -> anyhow::Result { let func = AggFunc::Quantile(phi); Ok(build_qe_aggregate(source, filters, window, func, ctx)) } + // `predict_linear(v[w], t)` — `t` (seconds into the future) is a + // scalar 2nd argument, so it doesn't fit `walk_call_to_op`'s + // `(call, ctx, window)` shape; special-cased here like + // `quantile_over_time`'s φ argument above. + "predict_linear" => { + let seconds = extract_call_num_arg(call, 1)?; + let (source, filters, window) = extract_matrix_arg(call, 0)?; + let func = AggFunc::PredictLinear { seconds }; + Ok(build_qe_aggregate(source, filters, window, func, ctx)) + } _ => { let (source, filters, window) = if call.func.name == "rate" || call.func.name == "irate" @@ -530,10 +606,12 @@ fn promql_token_to_binop(tok: TokenType) -> BinaryOpKind { /// `window` is the range-vector's duration — only `Rate`/`Increase` carry it /// on the `AggFunc` itself (`asap_l2`'s design: "no separate Window node"). /// Every other function still relies on the caller wrapping its `Aggregate` -/// in an `L2::Window`, matching this repo's pre-`asap_l2`-merge behavior -/// (see `lower.rs`'s module doc on why `Rate`/`Increase` map to -/// `AggIntent::Sum` here rather than adopting the dedicated intents that -/// window field would otherwise feed). +/// in an `L2::Window`. +/// +/// `predict_linear` is handled separately in `walk_call_qe` (its 2nd, +/// scalar argument doesn't fit this function's `(call, ctx, window)` +/// shape, matching how `quantile_over_time`/`histogram_quantile`'s φ +/// argument is already special-cased there). fn walk_call_to_op(call: &Call, ctx: &WalkCtx, window: Duration) -> anyhow::Result { let name = call.func.name; match name { @@ -547,31 +625,52 @@ fn walk_call_to_op(call: &Call, ctx: &WalkCtx, window: Duration) -> anyhow::Resu "stddev_over_time" => Ok(AggFunc::StdDev { population: false }), "stdvar_over_time" => Ok(AggFunc::Variance { population: false }), "count_over_time" => { - // Three cases, in priority order: + // Two cases, in priority order: // * Inside `count by (...) (count_over_time(...))` → // `CountDistinct` (HLL distinct counting; the outer // count of inner counts is cardinality). - // * Inside `topk(N, count_over_time(...))` → `Count` - // (the topk wrapper expects a count-shaped inner; the - // grouped-or-windowed `Count` → `Frequency` substitution - // happens in `lower.rs::agg_func_to_intents`). // * Otherwise → plain `Count`, still windowed here — the - // same `lower.rs` substitution recognizes the windowed - // shape and routes to CMS / CountSketch (per-series - // sample-count estimation), matching the pre-`asap_l2` - // behavior this used to reach via a dedicated - // `AggFunc::Frequency` variant that no longer exists. + // `lower.rs::agg_func_to_intents` grouped-or-windowed + // substitution recognizes the windowed shape and routes + // to CMS / CountSketch (per-series sample-count + // estimation), matching the pre-`asap_l2` behavior this + // used to reach via a dedicated `AggFunc::Frequency` + // variant that no longer exists. `topk(N, count_over_time + // (...))`'s heavy-hitter fusion is handled entirely in + // `walk_aggregate_qe`'s `topk`/`bottomk` arm now (it no + // longer forces `Count` here via `ctx.topk`). Ok(if ctx.outer_count { AggFunc::CountDistinct } else { AggFunc::Count }) } - "sum_over_time" | "last_over_time" | "present_over_time" | "absent_over_time" => { - Ok(AggFunc::Sum) - } - "delta" | "idelta" | "deriv" | "predict_linear" => Ok(AggFunc::Delta), - "changes" | "resets" => Ok(AggFunc::Count), + "sum_over_time" => Ok(AggFunc::Sum), + "last_over_time" => Ok(AggFunc::LastOverTime), + "present_over_time" => Ok(AggFunc::PresentOverTime), + "absent_over_time" => Ok(AggFunc::AbsentOverTime), + // Each of these used to collapse onto `Delta` (delta/idelta/deriv) + // or `Count` (changes/resets) — a crude placeholder bucketing from + // before `asap_l2` gave every one of them its own dedicated + // `AggFunc`/`AggIntent`. All are archive-only (no ASAP-tier + // Bind* rule exists for any of them, same as before this fix) — + // the correctness gain is `capability_for` now correctly + // returning `None` (route to archive) instead of the wrong + // `Some(ExactAgg(Sum))` / `Some(CardinalityApprox))` these used + // to produce, which claimed the ASAP tier could answer a + // `deriv()`/`changes()` query it structurally cannot. + "delta" => Ok(AggFunc::Delta), + "idelta" => Ok(AggFunc::IDelta), + "deriv" => Ok(AggFunc::Deriv), + "changes" => Ok(AggFunc::Changes), + "resets" => Ok(AggFunc::Resets), + // `Rate`/`Increase` now map onto their own dedicated `AggIntent`s + // in `lower.rs` (`capability_for` already has a real, tested + // `Rate | Increase => ExactAgg(Increase)` arm — this activates + // it for the first time via the PromQL path; previously + // collapsed onto `AggIntent::Sum`, which happened to route to + // the same *kind* of exact-precompute capability but under the + // wrong classification). "rate" | "irate" => Ok(AggFunc::Rate { window }), "increase" => Ok(AggFunc::Increase { window }), other => Err(anyhow!("unsupported PromQL function: {other}")), @@ -896,8 +995,81 @@ mod tests { #[test] fn topk_avg_over_time() { + // `topk` ranking by a non-count measure (`avg_over_time`, here) + // is not a heavy-hitter shape — `RankingMeasure::NonAdditive`, + // per `agg_intent::is_frequency_heavy_hitter` — so this becomes + // a generic `Sort + Limit` over `avg_over_time`'s own + // `Aggregate{Quantile(0.5)}` p50 approximation, not a forced + // `Count`/`Frequency` aggregate. Before the topk/rate precision + // fix this incorrectly asserted `[Frequency]`. let pq = pq("topk by (host) (5, avg_over_time(cpu[5m]))"); - assert_eq!(pq.aggregations, vec![AggType::Frequency]); + assert_eq!(pq.aggregations, vec![AggType::Quantile]); + assert_eq!(pq.quantiles, vec![0.5]); + } + + #[test] + fn topk_avg_over_time_is_sort_limit_not_topk_node() { + // Structural check backing `topk_avg_over_time` above: the tree + // must be `Limit { Sort { Aggregate{Quantile} } }`, not + // `QueryExpr::TopK` — confirms the non-heavy-hitter path is + // really taken, not just that the flattened `ParsedQuery` + // happens to read the same. + use crate::intent_algebra::relational::QueryExpr; + + let qe = super::parse_promql_expr("topk by (host) (5, avg_over_time(cpu[5m]))") + .expect("parse should succeed"); + match &qe { + QueryExpr::Limit { n, input, .. } => { + assert_eq!(*n, 5); + assert!( + matches!(input.as_ref(), QueryExpr::Sort { .. }), + "expected Sort under Limit, got {input:?}" + ); + } + other => panic!("expected Limit{{Sort{{...}}}}, got {other:?}"), + } + } + + #[test] + fn bottomk_count_over_time_is_never_heavy_hitter() { + // `bottomk` never qualifies as the heavy-hitter `TopK` intent + // even when ranking by `count_over_time` — `descending` must + // also hold (`is_frequency_heavy_hitter`), and `bottomk` is + // ascending by definition. Must still lower to `Limit{Sort{...}}`. + use crate::intent_algebra::relational::QueryExpr; + + let qe = super::parse_promql_expr("bottomk(5, count_over_time(http_requests_total[5m]))") + .expect("parse should succeed"); + assert!( + !matches!(qe, QueryExpr::TopK { .. }), + "bottomk must never produce the heavy-hitter TopK node, got {qe:?}" + ); + match &qe { + QueryExpr::Limit { n, input, .. } => { + assert_eq!(*n, 5); + match input.as_ref() { + QueryExpr::Sort { keys, .. } => { + assert!(keys[0].ascending, "bottomk must sort ascending"); + } + other => panic!("expected Sort under Limit, got {other:?}"), + } + } + other => panic!("expected Limit{{Sort{{...}}}}, got {other:?}"), + } + } + + #[test] + fn topk_count_over_time_is_topk_node() { + // The one real heavy-hitter shape: `topk` (descending) ranking + // by `count_over_time` (`RankingMeasure::Frequency`). + use crate::intent_algebra::relational::QueryExpr; + + let qe = super::parse_promql_expr("topk(5, count_over_time(http_requests_total[5m]))") + .expect("parse should succeed"); + assert!( + matches!(qe, QueryExpr::TopK { k: 5, .. }), + "expected QueryExpr::TopK{{k: 5, ..}}, got {qe:?}" + ); } // ── count cardinality ───────────────────────────────────────────────────── From fb4ff41bb503dc9d9e2209463d7106bf460c1d49 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 19 Jul 2026 12:47:56 -0600 Subject: [PATCH 07/11] fix(control_plane): route avg_over_time through the exact path, drop p50 approximation AggFunc::Avg now maps onto the literal AggIntent::Avg (matching asap_l2::lower's own mapping) instead of a p50 quantile-sketch approximation. capability_for(&AggIntent::Avg) already correctly returns None -- avg has no ASAP-tier sketch substitute, it needs a cross-policy Sum+Count join -- and ASAPController's own asap-plan treats AggIntent::Avg the same way (pass_through_intents_stay_logical keeps it a whole unsketched logical subtree). avg now routes through the same exact/archive path as Sum/Count/every archive-only intent; QeCollector::collect_op's existing catch-all exact_required = true arm already handles this with no dedicated Avg arm needed. Co-Authored-By: Claude Sonnet 5 --- control_plane/src/intent_algebra/lower.rs | 53 +++++++++++++++-------- control_plane/src/pipeline.rs | 7 ++- control_plane/src/query_parser/promql.rs | 26 +++++++---- 3 files changed, 58 insertions(+), 28 deletions(-) diff --git a/control_plane/src/intent_algebra/lower.rs b/control_plane/src/intent_algebra/lower.rs index 37a8aec4..6f48d8b0 100644 --- a/control_plane/src/intent_algebra/lower.rs +++ b/control_plane/src/intent_algebra/lower.rs @@ -9,16 +9,29 @@ //! //! Now that `relational.rs` (L2) itself merged onto `asap_l2` (see that //! file's module doc), this converter is control_plane's *own* — not a -//! re-export of `asap_l2::lower::convert_root` — for one remaining -//! deliberate reason: `avg_over_time` stays a p50 quantile-sketch -//! approximation rather than `asap_l2`'s literal (exact, -//! non-mergeable) `AggIntent::Avg`, because -//! `query_parser::QeCollector::collect_op` (which has no `Avg` arm of -//! its own) relies on that substitution to classify `avg_over_time` as -//! `AggType::Quantile`. Every *structural* piece below (scalar -//! resolution, schema threading, the `GroupKeys` shape) is unchanged -//! from `asap_l2`'s own converter — only this one `Aggregate`/`AggFunc` -//! mapping choice is control_plane-specific. +//! re-export of `asap_l2::lower::convert_root` — because the +//! `Aggregate` arm's multi-agg fusion and `frequency_trigger` heuristic +//! (see the `Frequency` preservation section below) are control_plane's +//! own dispatch design, with no `asap_l2` equivalent (`asap_l2` always +//! threads a single workload-level `AccuracyTarget` through +//! `agg_func_to_intent` with no grouped/windowed heuristic of its own). +//! Every other *structural* piece below (scalar resolution, schema +//! threading, the `GroupKeys` shape, and now also the full `AggFunc`→ +//! `AggIntent` mapping table itself) is unchanged from `asap_l2`'s own +//! converter — `avg_over_time` used to be the one deliberate mapping +//! divergence (a p50 quantile-sketch approximation in place of +//! `asap_l2`'s literal, exact `AggIntent::Avg`) but that's gone too: +//! `AggFunc::Avg` now maps onto the literal `AggIntent::Avg` exactly +//! like `asap_l2::lower` does, since `capability_for(&AggIntent::Avg)` +//! already correctly returns `None` (no ASAP-tier sketch substitute — +//! `avg` needs a cross-policy Sum+Count join, tracked as a follow-up) +//! and ASAPController's own `crates/plan/src/bind.rs` treats +//! `AggIntent::Avg` the same way (`pass_through_intents_stay_logical` +//! keeps it a whole logical, unsketched subtree). `avg` now routes +//! through the same exact/archive path as `Sum`/`Count`/every +//! archive-only intent — `QeCollector::collect_op`'s existing catch-all +//! `exact_required = true` arm already handles it with no dedicated +//! `Avg` arm needed. //! //! **PromQL-frontend semantic-retarget step (topk/rate precision fix).** //! `Rate`/`Increase`/`Changes`/`Delta`/`IDelta`/`Deriv`/`PredictLinear`/ @@ -511,13 +524,19 @@ fn agg_func_to_intents(func: &AggFunc, frequency_trigger: bool) -> Vec vec![AggIntent::Sum { col: None }], - // `Avg` approximates as the p50 (median) quantile sketch rather - // than a literal (exact, non-mergeable) `AggIntent::Avg` — - // matches `Min`/`Max`'s boundary-quantile treatment and is what - // `query_parser::QeCollector::collect_op` (which has no `Avg` - // arm of its own) relies on to classify `avg_over_time` as - // `AggType::Quantile` with `quantiles: [0.5]`. - AggFunc::Avg => vec![q(0.5)], + // `Avg` maps onto the literal, exact, non-mergeable + // `AggIntent::Avg` — matching `asap_l2::lower`'s own mapping + // exactly. `capability_for(&AggIntent::Avg)` already returns + // `None` (no ASAP-tier sketch substitute; needs a cross-policy + // Sum+Count join, tracked as a follow-up), matching + // ASAPController's own stance: `crates/plan/src/bind.rs`'s + // `pass_through_intents_stay_logical` test keeps `AggIntent::Avg` + // as a whole logical subtree with no sketch binding. So `avg` + // routes through the exact/archive path, same as `Sum`/`Count`/ + // `TopK`/every archive-only intent — `QeCollector::collect_op` + // already handles this correctly via its catch-all + // `exact_required = true` arm, no dedicated `Avg` arm needed. + AggFunc::Avg => vec![AggIntent::Avg { col: None }], AggFunc::Min => vec![AggIntent::Min { col: None }], AggFunc::Max => vec![AggIntent::Max { col: None }], AggFunc::StdDev { .. } | AggFunc::Variance { .. } => vec![q(0.25), q(0.75)], diff --git a/control_plane/src/pipeline.rs b/control_plane/src/pipeline.rs index 20b102c5..abeb2248 100644 --- a/control_plane/src/pipeline.rs +++ b/control_plane/src/pipeline.rs @@ -562,8 +562,11 @@ mod tests { spec.metric_name = "my_custom_metric".into(); let w = Analyzer::new().analyze(spec).unwrap(); assert_eq!(w.metric_name, "my_custom_metric"); - // aggregations still come from parse (avg → DDSketch → Quantile) - assert_eq!(w.aggregations, vec![AggType::Quantile]); + // aggregations still come from parse — `avg_over_time` is exact + // (no ASAP-tier sketch substitute for `AggIntent::Avg`), so + // `aggregations` stays empty and `exact_required` flips instead. + assert_eq!(w.aggregations, Vec::::new()); + assert!(w.exact_required); } /// Explicit time_window overrides the window derived from query_string. diff --git a/control_plane/src/query_parser/promql.rs b/control_plane/src/query_parser/promql.rs index 7744ffc9..445af6cb 100644 --- a/control_plane/src/query_parser/promql.rs +++ b/control_plane/src/query_parser/promql.rs @@ -962,10 +962,17 @@ mod tests { // ── avg_over_time ───────────────────────────────────────────────────────── #[test] - fn avg_over_time_maps_to_p50() { + fn avg_over_time_is_exact() { + // `AggIntent::Avg` has no ASAP-tier sketch substitute + // (`capability_for` returns `None` — needs a cross-policy + // Sum+Count join) and ASAPController's own `asap-plan` treats it + // the same way (`pass_through_intents_stay_logical`), so `avg` + // routes through the exact/archive path like `Sum`/`Count`, + // not the p50-quantile-sketch approximation this used to be. let pq = pq("avg by (symbol) (avg_over_time(financial_last_trade_price[5m]))"); - assert_eq!(pq.aggregations, vec![AggType::Quantile]); - assert_eq!(pq.quantiles, vec![0.5]); + assert_eq!(pq.aggregations, Vec::::new()); + assert!(pq.quantiles.is_empty()); + assert!(pq.exact_required); } // ── min/max_over_time ───────────────────────────────────────────────────── @@ -998,13 +1005,14 @@ mod tests { // `topk` ranking by a non-count measure (`avg_over_time`, here) // is not a heavy-hitter shape — `RankingMeasure::NonAdditive`, // per `agg_intent::is_frequency_heavy_hitter` — so this becomes - // a generic `Sort + Limit` over `avg_over_time`'s own - // `Aggregate{Quantile(0.5)}` p50 approximation, not a forced - // `Count`/`Frequency` aggregate. Before the topk/rate precision - // fix this incorrectly asserted `[Frequency]`. + // a generic `Sort + Limit` over `avg_over_time`'s own exact + // `Aggregate{Avg}`, not a forced `Count`/`Frequency` aggregate. + // Before the topk/rate precision fix this incorrectly asserted + // `[Frequency]`; `avg_over_time` itself is exact (see + // `avg_over_time_is_exact`), not a p50 quantile-sketch anymore. let pq = pq("topk by (host) (5, avg_over_time(cpu[5m]))"); - assert_eq!(pq.aggregations, vec![AggType::Quantile]); - assert_eq!(pq.quantiles, vec![0.5]); + assert_eq!(pq.aggregations, Vec::::new()); + assert!(pq.exact_required); } #[test] From 41bdbcf9513ff3bd654e9cee3d8af6278b71dbc2 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 19 Jul 2026 13:15:55 -0600 Subject: [PATCH 08/11] fix(control_plane): route stddev_over_time/stdvar_over_time through the exact path Same fix class as the avg_over_time correction: AggFunc::StdDev/Variance now map onto their literal AggIntent::StdDev/Variance (matching asap_l2::lower exactly) instead of the two-quantile [q(0.25), q(0.75)] "IQR proxy" approximation. capability_for already declared both archive-only (Avg | StdDev | Variance => None), so the proxy was silently claiming a QuantileApprox ASAP-tier capability neither this repo's own capability table nor ASAPController's asap-plan (pass_through_intents_stay_logical) actually backs with a real bind rule. The Merge-of-siblings fan-out this used to trigger collapses back to a plain single Aggregate. Co-Authored-By: Claude Sonnet 5 --- control_plane/src/intent_algebra/lower.rs | 52 ++++++++++++++++------- control_plane/src/query_parser/promql.rs | 11 +++-- 2 files changed, 45 insertions(+), 18 deletions(-) diff --git a/control_plane/src/intent_algebra/lower.rs b/control_plane/src/intent_algebra/lower.rs index 6f48d8b0..ef5f86c5 100644 --- a/control_plane/src/intent_algebra/lower.rs +++ b/control_plane/src/intent_algebra/lower.rs @@ -509,9 +509,10 @@ pub fn convert(legacy: &LQueryExpr, schema: &Schema) -> Result Vec { let q = |q: f64| AggIntent::Quantile { col: None, @@ -539,7 +540,24 @@ fn agg_func_to_intents(func: &AggFunc, frequency_trigger: bool) -> Vec vec![AggIntent::Avg { col: None }], AggFunc::Min => vec![AggIntent::Min { col: None }], AggFunc::Max => vec![AggIntent::Max { col: None }], - AggFunc::StdDev { .. } | AggFunc::Variance { .. } => vec![q(0.25), q(0.75)], + // `StdDev`/`Variance` map onto their literal `AggIntent`s, + // matching `asap_l2::lower` exactly — same fix as `Avg` above, + // same reasoning: `capability_for` already declares both + // archive-only (`Avg { .. } | StdDev { .. } | Variance { .. } + // => None`), so the former `vec![q(0.25), q(0.75)]` IQR proxy + // (interquartile range as a stand-in for stddev) was silently + // claiming ASAP-tier `QuantileApprox` support neither this + // repo's own capability table nor ASAPController's `asap-plan` + // (`pass_through_intents_stay_logical`) actually backs with a + // real bind rule. + AggFunc::StdDev { population } => vec![AggIntent::StdDev { + col: None, + population: *population, + }], + AggFunc::Variance { population } => vec![AggIntent::Variance { + col: None, + population: *population, + }], AggFunc::Quantile(phi) => vec![q(*phi)], AggFunc::CountDistinct => vec![crate::intent_algebra::default_cardinality()], AggFunc::HeavyHitters { .. } => vec![crate::intent_algebra::default_frequency()], @@ -836,7 +854,12 @@ mod tests { } #[test] - fn stddev_fans_out_into_merge_of_quantile_siblings() { + fn stddev_maps_to_literal_exact_intent() { + // Matches `asap_l2::lower`'s own mapping: no more two-quantile + // IQR-proxy `Merge` fan-out (that claimed a `QuantileApprox` + // ASAP-tier capability `capability_for` never actually backed + // for `StdDev`/`Variance` — same bug class as the old + // `avg → p50` approximation). let legacy = agg( vec![], false, @@ -844,17 +867,16 @@ mod tests { src("m"), ); match convert_root(&legacy).unwrap() { - CQueryExpr::Merge { children } => { - assert_eq!(children.len(), 2); - for c in &children { - assert!(matches!( - c, - CQueryExpr::Aggregate { aggs, .. } - if matches!(aggs.as_slice(), [AggIntent::Quantile { .. }]) - )); - } + CQueryExpr::Aggregate { aggs, .. } => { + assert!(matches!( + aggs.as_slice(), + [AggIntent::StdDev { + population: false, + .. + }] + )); } - other => panic!("expected Merge, got {other:?}"), + other => panic!("expected a plain Aggregate, got {other:?}"), } } diff --git a/control_plane/src/query_parser/promql.rs b/control_plane/src/query_parser/promql.rs index 445af6cb..1273f411 100644 --- a/control_plane/src/query_parser/promql.rs +++ b/control_plane/src/query_parser/promql.rs @@ -1091,10 +1091,15 @@ mod tests { // ── stddev_over_time ────────────────────────────────────────────────────── #[test] - fn stddev_over_time_iqr_proxy() { + fn stddev_over_time_is_exact() { + // `AggIntent::StdDev` has no ASAP-tier sketch substitute + // (`capability_for` returns `None`, same as `Avg`), so + // `stddev_over_time` routes exact -- no more IQR-proxy + // `[q(0.25), q(0.75)]` approximation. let pq = pq("avg by (host) (stddev_over_time(cpu[5m]))"); - assert_eq!(pq.aggregations, vec![AggType::Quantile]); - assert!(pq.quantiles.contains(&0.25) && pq.quantiles.contains(&0.75)); + assert_eq!(pq.aggregations, Vec::::new()); + assert!(pq.quantiles.is_empty()); + assert!(pq.exact_required); } // ── sum_over_time → exact ───────────────────────────────────────────────── From 876b27800f21dabe9513795a9f7bc8497abc6383 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 19 Jul 2026 20:10:57 -0600 Subject: [PATCH 09/11] fix(control_plane): resolve capability_for/rules::dispatch semantic divergences on Count and Min/Max Fixes the two real behavioral divergences found while auditing capability_for against sketch_algebra::rules::dispatch (both claim to answer the same AggIntent -> ASAP-tier-capability question but disagreed): - Count{non-Exact}: capability_for claimed CardinalityApprox (HLL, "distinct count" semantics); rules::dispatch's BindCmsOnCount treats it as a bare frequency point-query (CMS). ASAPController's own crates/plan/src/bind.rs::readout maps Count to SketchQuery::PointCount, confirming CMS is correct -- the CardinalityApprox premise never had a real caller anyway (distinct_over_time/COUNT(DISTINCT) always lower to AggIntent::Cardinality, never Count). AggIntent::Count{non-Exact} is unreachable via this repo's own PromQL frontend today regardless (lower.rs only constructs Count{Exact} or the Extension-based Frequency intent), so this is a consistency fix, not a live routing change. - Min/Max: capability_for claimed QuantileApprox (min = quantile(0), max = quantile(1)), but no rule in sketch_algebra::rules actually implements that -- bind_kll_quantile/bind_ddsketch_quantile only ever match AggIntent::Quantile, never Min/Max, so the promised coverage didn't exist and Min/Max silently fell through to archive regardless. ASAPController's own crates/plan/src/boundary.rs treats Min/Max as an exact mergeable accumulator (SummaryKind::MinMax), same tier as Sum/Rate/Increase -- correct, since comparing two partial extrema needs no approximation at all. bind_exact_agg.rs now binds Min/Max -> AggregationType::MinMax (keyed -> MultipleMinMax), using the data plane's already-fully-wired MinMaxAccumulator; capability_for now returns ExactAgg(MinMax) to match. Also fixes a stale doc-table line (Count{Exact} claimed to return Some(ExactAgg(Sum)); the actual code has returned None since the PR #200/#201 revert). Co-Authored-By: Claude Sonnet 5 --- .../src/sketch_algebra/capability.rs | 85 +++++++++++++------ .../sketch_algebra/rules/bind_exact_agg.rs | 63 ++++++++++++-- 2 files changed, 117 insertions(+), 31 deletions(-) diff --git a/control_plane/src/sketch_algebra/capability.rs b/control_plane/src/sketch_algebra/capability.rs index f3ed639a..c118ef55 100644 --- a/control_plane/src/sketch_algebra/capability.rs +++ b/control_plane/src/sketch_algebra/capability.rs @@ -438,11 +438,11 @@ fn multi_pop_satisfies_single(required: AggregationType, available: AggregationT /// |---|---| /// | `Quantile { q, accuracy }` (accuracy not `Exact`) | `Some(QuantileApprox(Any))` | /// | `Quantile { q, accuracy: Exact }` | `None` (exact must use HashAgg/SortAgg) | -/// | `Min` / `Max` | `Some(QuantileApprox(Any))` — quantile sketches answer min = q(0), max = q(1) | +/// | `Min` / `Max` | `Some(ExactAgg(MinMax))` — exact mergeable accumulator, no approximation needed | /// | `Cardinality { accuracy }` (accuracy not `Exact`) | `Some(CardinalityApprox)` | /// | `Cardinality { accuracy: Exact }` | `None` | -/// | `Count { accuracy: Exact }` | `Some(ExactAgg(Sum))` — count_over_time = sum-of-1s (PR-6 follow-up) | -/// | `Count { accuracy }` (accuracy not `Exact`) | `Some(CardinalityApprox)` | +/// | `Count { accuracy: Exact }` | `None` — no count accumulator exists yet; routes to archive | +/// | `Count { accuracy }` (accuracy not `Exact`) | `Some(FrequencyEstimate(Any))` — bare per-item frequency point-query (CMS) | /// | `TopK { k, accuracy }` (accuracy not `Exact`) | `Some(FrequencyTopk(CmsWithHeap))` | /// | `Frequency { accuracy }` (accuracy not `Exact`) | `Some(FrequencyEstimate(Any))` | /// | `Frequency { accuracy: Exact }` | `None` (exact aggregation; route to archive) | @@ -484,11 +484,7 @@ pub fn capability_for(intent: &AggIntent) -> Option { } AggIntent::Count { accuracy } => { // Count is the legacy bridge — `count_over_time` lowers to - // `Count{accuracy:Exact}` (exact counter, no sketch). When - // the lowerer or callers ask for an approximate count - // (`distinct_over_time` / SQL `COUNT(DISTINCT)`), the - // accuracy is non-Exact and we hand it to the cardinality - // sketch path. + // `Count{accuracy:Exact}` (exact counter, no sketch). // // Exact count routes to archive (`None`). The PR #200/#201 // follow-up flipped this to `ExactAgg(Sum)` on the theory @@ -501,10 +497,28 @@ pub fn capability_for(intent: &AggIntent) -> Option { // until a real `SumCountAccumulator` lands (the // temporal/spatial-split work) — archive counts correctly // in the meantime. + // + // Non-exact `Count` is a relaxed-accuracy `COUNT(*) per + // group` — a bare per-item frequency point-query, matching + // ASAPController's own `crates/plan/src/bind.rs::readout` + // (`AggIntent::Count => SketchQuery::PointCount`) and this + // repo's own `BindCmsOnCount` rule (CMS, no top-k heap). + // Previously mapped to `CardinalityApprox`/HLL under the + // theory that non-exact `Count` meant "distinct count" — + // but `distinct_over_time`/`COUNT(DISTINCT)` always lower to + // `AggIntent::Cardinality`, never to `Count`, so that + // premise never had a real caller; `AggIntent::Count` + // itself is unreachable via this repo's own PromQL frontend + // today regardless of accuracy (`lower.rs` only ever + // constructs `Count{Exact}` or the `Extension`-based + // `Frequency` intent), so this only affects future callers + // (e.g. a SQL frontend) — fixed here for consistency with + // `rules::dispatch` rather than because it changes any + // query routing today. if is_exact(accuracy) { None } else { - Some(Capability::CardinalityApprox) + Some(Capability::FrequencyEstimate(SketchKindHandle::Any)) } } AggIntent::TopK { accuracy, .. } => { @@ -524,13 +538,6 @@ pub fn capability_for(intent: &AggIntent) -> Option { Some(Capability::FrequencyTopk(SketchKindHandle::Any)) } } - // ── Min / Max via quantile sketches ────────────────────────── - // DDSketch / KLL answer min = quantile(0) and max = quantile(1) - // out of the box. No dedicated extrema sketch is needed; route - // these through the quantile-family handler. - AggIntent::Min { .. } | AggIntent::Max { .. } => { - Some(Capability::QuantileApprox(SketchKindHandle::Any)) - } // ── ExactAgg (PR-6 follow-up) ──────────────────────────────── // These intents previously returned `None` and routed to the // archive engine. Now that the data plane carries @@ -543,6 +550,26 @@ pub fn capability_for(intent: &AggIntent) -> Option { AggIntent::Rate | AggIntent::Increase => { Some(Capability::ExactAgg(AggregationType::Increase)) } + // Min / Max are exact, mergeable accumulators — comparing two + // partial min/maxes is exact by construction, no approximation + // needed at all. Previously routed through `QuantileApprox` + // (DDSketch/KLL answer min = quantile(0), max = quantile(1) — + // a strictly worse, lossy answer when an exact accumulator is + // just as cheap) on the theory that a dedicated `MinMax` bind + // rule wasn't worth the cost-model churn versus the + // already-existing quantile-sketch path. That premise didn't + // hold: `bind_kll_quantile`/`bind_ddsketch_quantile` only ever + // matched `AggIntent::Quantile`, never `Min`/`Max`, so no rule + // actually implemented the promised quantile-sketch coverage — + // and matches ASAPController's own `crates/plan/src/boundary.rs` + // (`Min`/`Max` are exact mergeable accumulators, same tier as + // `Sum`/`Rate`/`Increase`). The data plane already has a fully + // wired `MinMaxAccumulator`/`AggregationType::MinMax`, so this + // isn't new infrastructure — see `bind_exact_agg.rs`'s matching + // `AggIntent::Min | AggIntent::Max` arm. + AggIntent::Min { .. } | AggIntent::Max { .. } => { + Some(Capability::ExactAgg(AggregationType::MinMax)) + } // ── Avg / StdDev / Variance: still no ASAP-tier substitute ──── // Avg = Sum / Count, which needs two separate ExactAgg policies // (one for Sum, one for Count) joined at query time. The L4 @@ -687,11 +714,20 @@ mod tests { } #[test] - fn capability_for_count_approximate_returns_cardinality_approx() { + fn capability_for_count_approximate_returns_frequency_estimate() { + // Non-exact `Count` is a relaxed-accuracy `COUNT(*)` point query + // (CMS), matching ASAPController's own `bind.rs::readout` + // (`Count => SketchQuery::PointCount`) and this repo's + // `BindCmsOnCount` rule — not `CardinalityApprox`/HLL, which is + // `AggIntent::Cardinality`'s job (`distinct_over_time`/ + // `COUNT(DISTINCT)` never lower to `Count`). let intent = AggIntent::Count { accuracy: AccuracyTarget::Epsilon(0.01), }; - assert_eq!(capability_for(&intent), Some(Capability::CardinalityApprox)); + assert_eq!( + capability_for(&intent), + Some(Capability::FrequencyEstimate(SketchKindHandle::Any)) + ); } #[test] @@ -729,20 +765,21 @@ mod tests { } #[test] - fn capability_for_min_returns_quantile_approx() { - // Min = quantile(0); DDSketch / KLL answer it directly. + fn capability_for_min_returns_exact_agg_minmax() { + // Min/Max are exact, mergeable accumulators -- no approximation + // needed at all -- matching ASAPController's own + // `crates/plan/src/boundary.rs` treatment. assert_eq!( capability_for(&AggIntent::Min { col: None }), - Some(Capability::QuantileApprox(SketchKindHandle::Any)) + Some(Capability::ExactAgg(AggregationType::MinMax)) ); } #[test] - fn capability_for_max_returns_quantile_approx() { - // Max = quantile(1); DDSketch / KLL answer it directly. + fn capability_for_max_returns_exact_agg_minmax() { assert_eq!( capability_for(&AggIntent::Max { col: None }), - Some(Capability::QuantileApprox(SketchKindHandle::Any)) + Some(Capability::ExactAgg(AggregationType::MinMax)) ); } diff --git a/control_plane/src/sketch_algebra/rules/bind_exact_agg.rs b/control_plane/src/sketch_algebra/rules/bind_exact_agg.rs index 2af99235..6c86aa1c 100644 --- a/control_plane/src/sketch_algebra/rules/bind_exact_agg.rs +++ b/control_plane/src/sketch_algebra/rules/bind_exact_agg.rs @@ -8,6 +8,7 @@ //! | `Sum` | `agg_type: AggregationType::Sum` | //! | `Rate { .. }` | `agg_type: AggregationType::Increase` | //! | `Increase { .. }` | `agg_type: AggregationType::Increase` | +//! | `Min` / `Max` | `agg_type: AggregationType::MinMax` | //! //! `Rate` and `Increase` share the `Increase` accumulator because rate is //! computed as `increase / window_seconds` — a scalar division on the @@ -15,6 +16,14 @@ //! division (when needed) is the L5 emitter's responsibility, not the //! L4 binder's. //! +//! `Min` and `Max` share the `MinMax` accumulator (min/max are exact and +//! mergeable by construction — comparing two partial extrema needs no +//! approximation at all) — matches ASAPController's own +//! `crates/plan/src/boundary.rs`, which realizes `Min`/`Max` as an exact +//! mergeable accumulator, same tier as `Sum`/`Rate`/`Increase`. See the +//! "What this rule does NOT bind" list below for why this superseded the +//! earlier quantile-sketch-only routing. +//! //! ## What this rule does NOT bind //! //! - `AggIntent::Count { accuracy: Exact }` — `count_over_time`. The @@ -34,12 +43,19 @@ //! `HashAgg` / `SortAgg` / `SortMerge` from the deployment's //! exact-physical-operator family (none of which run at the ASAP tier //! today). -//! - `AggIntent::Min` / `AggIntent::Max` — `MinMax` is already covered -//! by the quantile-sketch path (`quantile(0)` / `quantile(1)` via -//! DDSketch / KLL). Adding a separate `MinMax` ExactAgg binding -//! would create a competing rule; not worth the cost-model churn -//! until profile data shows MinMax-precompute is meaningfully -//! cheaper than the quantile-sketch path for some workloads. +//! - `AggIntent::Min` / `AggIntent::Max` — previously left unbound here +//! on the theory that `quantile(0)`/`quantile(1)` (DDSketch/KLL) +//! already covered them, so a dedicated rule would just compete with +//! the sketch-family rules for no accuracy benefit. That theory +//! didn't hold: `bind_kll_quantile`/`bind_ddsketch_quantile` only +//! ever pattern-match `AggIntent::Quantile`, never `Min`/`Max` — no +//! rule actually implemented the promised coverage, so `Min`/`Max` +//! silently fell through to archive regardless of what +//! `capability_for` claimed. Now bound here instead: min/max are +//! exact and mergeable by construction (no approximation needed), +//! matching ASAPController's own treatment +//! (`crates/plan/src/boundary.rs`) and the data plane's existing +//! `MinMaxAccumulator`/`AggregationType::MinMax`. //! //! ## Priority //! @@ -61,7 +77,7 @@ use crate::sketch_algebra::physical_expr::PhysicalExpr; use crate::sketch_algebra::rules::Rule; use crate::types_v2::AccuracyTarget; -/// Bind exact-aggregation intents (Sum / Rate / Increase / Count{Exact}) +/// Bind exact-aggregation intents (Sum / Rate / Increase / Min / Max) /// to `PhysicalExpr::ExactAgg`. See module doc for the mapping table. pub struct BindExactAgg; @@ -119,6 +135,13 @@ impl Rule for BindExactAgg { AggregationType::Increase } } + AggIntent::Min { .. } | AggIntent::Max { .. } => { + if keyed { + AggregationType::MultipleMinMax + } else { + AggregationType::MinMax + } + } // `AggIntent::Count{Exact}` (count_over_time) is intentionally // NOT bound here — see the module doc. It needs a real // count accumulator, which doesn't exist yet; binding it to @@ -226,6 +249,16 @@ mod tests { ); } + #[test] + fn binds_min_to_exact_agg_minmax() { + check_binds(AggIntent::Min { col: None }, AggregationType::MinMax); + } + + #[test] + fn binds_max_to_exact_agg_minmax() { + check_binds(AggIntent::Max { col: None }, AggregationType::MinMax); + } + #[test] fn does_not_bind_count_exact() { // `count_over_time` (Count{Exact}) is NOT bound — the data @@ -346,6 +379,22 @@ mod tests { ); } + #[test] + fn keyed_min_binds_to_multiple_minmax() { + check_keyed_binds( + AggIntent::Min { col: None }, + AggregationType::MultipleMinMax, + ); + } + + #[test] + fn keyed_max_binds_to_multiple_minmax() { + check_keyed_binds( + AggIntent::Max { col: None }, + AggregationType::MultipleMinMax, + ); + } + #[test] fn keyed_count_exact_does_not_bind() { // `count by (...) (count_over_time(...))` — Count{Exact} is From 8e8d450d4e5456918503602e003b9a0d0762bd19 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 20 Jul 2026 06:52:28 -0600 Subject: [PATCH 10/11] refactor(control_plane): capability_for delegates to asap_plan::boundary::implementation_for Bumps the ASAPController pin to 12b3054 (ProjectASAP/ASAPController#138-140: CostModel interface, realize/Realization -> implementation_for/Implementation Cascades-terminology rename, CountSketch/CountSketchWithHeap SummaryKind variants, Implementation::is_satisfied_by) and adds asap-plan/asap-sketch as new git dependencies (same repo/rev as asap-ir/asap-l2; asap-plan depends only on asap-ir, so this pulls in no datafusion/front-end weight). capability_for no longer maintains its own hand-written AggIntent -> ASAP-tier capability match -- that was a second, parallel judgment kept in sync with asap-plan's own implementation_for by hand (and had already drifted twice: the Count/Min-Max divergences fixed in a prior commit). It now keeps only the Extension/Frequency special case (deployment-specific -- asap-plan has no opinion on an intent shape it can't see into, by design) and delegates everything else to asap_plan::boundary::implementation_for, translating the returned Implementation into this repo's coarser Capability vocabulary via the new implementation_to_capability helper. One deliberate override survives the delegation: Count{Exact} is forced to None (archive) rather than trusting implementation_for's ExactAccumulator claim, because the data plane has no working count accumulator (SumAccumulator conflates Sum and Count). All 772 existing tests pass unchanged -- confirms the delegation is behaviorally identical to the hand-written match it replaces, including the Min/Max and Count fixes from the prior commit. Co-Authored-By: Claude Sonnet 5 --- control_plane/Cargo.toml | 9 + .../src/sketch_algebra/capability.rs | 239 ++++++------------ 2 files changed, 93 insertions(+), 155 deletions(-) diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index 639cfaeb..9df525d3 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -52,6 +52,15 @@ asap_types.workspace = true # Also satisfies everything Phase 2 (#395) needs -- asap-l2 (Step 4), # asap-plan/asap-sketch + Implementation::is_satisfied_by (Steps 8-9) -- # 7fcaf91 is a strict descendant of every rev that PR pinned along the way. +# +# asap-plan owns the single authoritative AggIntent -> sketch-vs-exact +# implementation decision (`boundary::implementation_for`, Cascades +# "implementation rule" terminology -- see that crate's doc); capability_for +# delegates to it instead of maintaining its own parallel judgment. asap-plan +# depends only on asap-ir (per its own crate doc's layering invariant), so +# this doesn't pull in datafusion or any front-end weight. asap-sketch is +# asap-plan's own dependency (SummaryKind/SummaryParams), needed here only +# to translate Implementation into this repo's own Capability vocabulary. asap-ir = { git = "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/ProjectASAP/ASAPController", rev = "7fcaf914d87e71407c3a6d7ccac613b867f9c11b" } asap-l2 = { git = "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/ProjectASAP/ASAPController", rev = "7fcaf914d87e71407c3a6d7ccac613b867f9c11b" } asap-sketch = { git = "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/ProjectASAP/ASAPController", rev = "7fcaf914d87e71407c3a6d7ccac613b867f9c11b" } diff --git a/control_plane/src/sketch_algebra/capability.rs b/control_plane/src/sketch_algebra/capability.rs index c118ef55..fc81b89d 100644 --- a/control_plane/src/sketch_algebra/capability.rs +++ b/control_plane/src/sketch_algebra/capability.rs @@ -432,24 +432,19 @@ fn multi_pop_satisfies_single(required: AggregationType, available: AggregationT /// (`intent_algebra::lower::lower_parsed_query`), which is the single /// owner of "what does this PromQL function mean". /// -/// ## Mapping table +/// ## Delegation to `asap-plan` /// -/// | `AggIntent` variant | Returns | -/// |---|---| -/// | `Quantile { q, accuracy }` (accuracy not `Exact`) | `Some(QuantileApprox(Any))` | -/// | `Quantile { q, accuracy: Exact }` | `None` (exact must use HashAgg/SortAgg) | -/// | `Min` / `Max` | `Some(ExactAgg(MinMax))` — exact mergeable accumulator, no approximation needed | -/// | `Cardinality { accuracy }` (accuracy not `Exact`) | `Some(CardinalityApprox)` | -/// | `Cardinality { accuracy: Exact }` | `None` | -/// | `Count { accuracy: Exact }` | `None` — no count accumulator exists yet; routes to archive | -/// | `Count { accuracy }` (accuracy not `Exact`) | `Some(FrequencyEstimate(Any))` — bare per-item frequency point-query (CMS) | -/// | `TopK { k, accuracy }` (accuracy not `Exact`) | `Some(FrequencyTopk(CmsWithHeap))` | -/// | `Frequency { accuracy }` (accuracy not `Exact`) | `Some(FrequencyEstimate(Any))` | -/// | `Frequency { accuracy: Exact }` | `None` (exact aggregation; route to archive) | -/// | `Sum` | `Some(ExactAgg(Sum))` — ASAP-tier exact precompute (PR-6 follow-up) | -/// | `Rate` / `Increase` | `Some(ExactAgg(Increase))` — counter-reset-aware precompute (PR-6 follow-up) | -/// | `Avg` | `None` — needs cross-policy join (Sum + Count); follow-up | -/// | Every archive-only intent | `None` | +/// Beyond the `Extension`/`Frequency` special case (deployment-specific, +/// see below — `asap-plan` deliberately has no opinion on a shape it +/// can't see into), every other `AggIntent` variant's capability is +/// derived from [`asap_plan::boundary::implementation_for`] — the single +/// upstream authority for "how would this intent be realized" — rather +/// than a second, hand-maintained, parallel judgment kept in sync by +/// hand. See [`implementation_to_capability`] for the +/// `asap_sketch::SummaryKind` → `Capability` family translation this +/// still requires (the two crates' capability vocabularies aren't the +/// same *shape*, even once they agree on substance), and its doc comment +/// for the one deliberate override (`Count{Exact}`). pub fn capability_for(intent: &AggIntent) -> Option { if let Some(accuracy) = crate::intent_algebra::as_frequency(intent) { return if is_exact(&accuracy) { @@ -467,152 +462,86 @@ pub fn capability_for(intent: &AggIntent) -> Option { Some(Capability::FrequencyEstimate(SketchKindHandle::Any)) }; } - match intent { - AggIntent::Quantile { accuracy, .. } => { - if is_exact(accuracy) { - None - } else { + implementation_to_capability(asap_plan::boundary::implementation_for(intent)) +} + +/// Translate `asap-plan`'s per-intent implementation decision into this +/// repo's own [`Capability`] vocabulary. +/// +/// `Implementation::Sketch`/`ExactAccumulator` both carry an +/// `asap_sketch::SummaryKind` — this repo's `Capability` groups those +/// into coarser families (`QuantileApprox`/`CardinalityApprox`/ +/// `FrequencyEstimate`/`FrequencyTopk` for sketches; `ExactAgg(AggregationType)` +/// for accumulators) because that's the granularity the sketch index +/// (`is_satisfied_by`) and the wire-shared `sketch_index::Capability` +/// actually match on — the required side never pins a *specific* +/// concrete implementation (`Any`), only the family. This function is +/// exhaustive over `SummaryKind` (no wildcard fallthrough), so a new +/// variant there fails to compile here until given an explicit mapping. +fn implementation_to_capability(implementation: asap_plan::Implementation) -> Option { + use asap_plan::Implementation; + use asap_sketch::SummaryKind; + + match implementation { + Implementation::PassThrough => None, + Implementation::Sketch { kind, .. } => match kind { + SummaryKind::Kll | SummaryKind::DDSketch => { Some(Capability::QuantileApprox(SketchKindHandle::Any)) } - } - AggIntent::Cardinality { accuracy, .. } => { - if is_exact(accuracy) { - None - } else { + SummaryKind::Hll | SummaryKind::Theta | SummaryKind::Kmv => { Some(Capability::CardinalityApprox) } - } - AggIntent::Count { accuracy } => { - // Count is the legacy bridge — `count_over_time` lowers to - // `Count{accuracy:Exact}` (exact counter, no sketch). - // - // Exact count routes to archive (`None`). The PR #200/#201 - // follow-up flipped this to `ExactAgg(Sum)` on the theory - // "count = sum-of-1s" — but the data plane has no count - // accumulator. `SumAccumulator` only tracks `sum: f64` and - // its `query` returns `self.sum` for BOTH `Statistic::Sum` - // and `Statistic::Count`, so a `count_over_time` query - // matched against a `Sum` policy returns the sum of the - // sample VALUES, not the count of samples. Reverted here - // until a real `SumCountAccumulator` lands (the - // temporal/spatial-split work) — archive counts correctly - // in the meantime. - // - // Non-exact `Count` is a relaxed-accuracy `COUNT(*) per - // group` — a bare per-item frequency point-query, matching - // ASAPController's own `crates/plan/src/bind.rs::readout` - // (`AggIntent::Count => SketchQuery::PointCount`) and this - // repo's own `BindCmsOnCount` rule (CMS, no top-k heap). - // Previously mapped to `CardinalityApprox`/HLL under the - // theory that non-exact `Count` meant "distinct count" — - // but `distinct_over_time`/`COUNT(DISTINCT)` always lower to - // `AggIntent::Cardinality`, never to `Count`, so that - // premise never had a real caller; `AggIntent::Count` - // itself is unreachable via this repo's own PromQL frontend - // today regardless of accuracy (`lower.rs` only ever - // constructs `Count{Exact}` or the `Extension`-based - // `Frequency` intent), so this only affects future callers - // (e.g. a SQL frontend) — fixed here for consistency with - // `rules::dispatch` rather than because it changes any - // query routing today. - if is_exact(accuracy) { - None - } else { + SummaryKind::Cms | SummaryKind::CountSketch => { Some(Capability::FrequencyEstimate(SketchKindHandle::Any)) } - } - AggIntent::TopK { accuracy, .. } => { - if is_exact(accuracy) { - // Exact top-k must use HashAgg+Heap; no ASAP-tier sketch. - None - } else { - // Top-k is intrinsically heavy-hitter — only heap-bearing - // handles can enumerate the items. The analyzer doesn't - // care which heap-bearing variant answers (CmsWithHeap or - // CountSketchWithHeap both work — the reducer dispatches - // both through `decode_cms_with_heap_from_msgpack` and - // produces top-k items either way). Return `Any` so - // `is_satisfied_by`'s `handles_compatible_for_topk` - // wildcard accepts whichever variant the ingest tier - // chose to register. + SummaryKind::CmsWithHeap | SummaryKind::CountSketchWithHeap => { Some(Capability::FrequencyTopk(SketchKindHandle::Any)) } - } - // ── ExactAgg (PR-6 follow-up) ──────────────────────────────── - // These intents previously returned `None` and routed to the - // archive engine. Now that the data plane carries - // `Capability::ExactAgg(agg_type)` on ExactAgg-backed sids, - // the analyzer can match them to ASAP-tier exact-precompute - // state instead. `is_satisfied_by` checks `agg_type` equality - // structurally — a sid registered as `ExactAgg(Sum)` only - // satisfies a required `ExactAgg(Sum)`. - AggIntent::Sum { .. } => Some(Capability::ExactAgg(AggregationType::Sum)), - AggIntent::Rate | AggIntent::Increase => { - Some(Capability::ExactAgg(AggregationType::Increase)) - } - // Min / Max are exact, mergeable accumulators — comparing two - // partial min/maxes is exact by construction, no approximation - // needed at all. Previously routed through `QuantileApprox` - // (DDSketch/KLL answer min = quantile(0), max = quantile(1) — - // a strictly worse, lossy answer when an exact accumulator is - // just as cheap) on the theory that a dedicated `MinMax` bind - // rule wasn't worth the cost-model churn versus the - // already-existing quantile-sketch path. That premise didn't - // hold: `bind_kll_quantile`/`bind_ddsketch_quantile` only ever - // matched `AggIntent::Quantile`, never `Min`/`Max`, so no rule - // actually implemented the promised quantile-sketch coverage — - // and matches ASAPController's own `crates/plan/src/boundary.rs` - // (`Min`/`Max` are exact mergeable accumulators, same tier as - // `Sum`/`Rate`/`Increase`). The data plane already has a fully - // wired `MinMaxAccumulator`/`AggregationType::MinMax`, so this - // isn't new infrastructure — see `bind_exact_agg.rs`'s matching - // `AggIntent::Min | AggIntent::Max` arm. - AggIntent::Min { .. } | AggIntent::Max { .. } => { - Some(Capability::ExactAgg(AggregationType::MinMax)) - } - // ── Avg / StdDev / Variance: still no ASAP-tier substitute ──── - // Avg = Sum / Count, which needs two separate ExactAgg policies - // (one for Sum, one for Count) joined at query time. The L4 - // binder doesn't yet emit that pattern, so capability_for keeps - // these on the archive path for now. Follow-up. - AggIntent::Avg { .. } | AggIntent::StdDev { .. } | AggIntent::Variance { .. } => None, - // Archive-only intents — never bind to a ASAP-tier capability; - // routed to the cold tier (Gorilla / Thanos). Includes every - // intent added by the Phase 1 IR merge (none has a `Bind*` rule - // yet) plus the pre-existing archive-only set. `Irate` is - // intentionally absent — folded into `Rate` above (see - // `agg_intent.rs` module docs). - AggIntent::Absent - | AggIntent::AbsentOverTime - | AggIntent::PresentOverTime - | AggIntent::Delta - | AggIntent::Deriv - | AggIntent::PredictLinear { .. } - | AggIntent::DoubleExpSmoothing { .. } - | AggIntent::IDelta - | AggIntent::Resets - | AggIntent::Changes - | AggIntent::HistogramCount - | AggIntent::HistogramSum - | AggIntent::HistogramAvg - | AggIntent::HistogramStdDev - | AggIntent::HistogramStdVar - | AggIntent::HistogramFraction { .. } - | AggIntent::HistogramQuantile { .. } - | AggIntent::Math(_) - | AggIntent::TimeFn(_) - | AggIntent::Group - | AggIntent::CountValues { .. } - | AggIntent::LastOverTime - | AggIntent::FirstOverTime - | AggIntent::MadOverTime - | AggIntent::TsOfMinOverTime - | AggIntent::TsOfMaxOverTime - | AggIntent::TsOfFirstOverTime - | AggIntent::TsOfLastOverTime => None, - // Unrecognized Extension (not the Frequency one, guarded above) -- no - // binding exists for a shape core cannot even see into. - AggIntent::Extension { .. } => None, + SummaryKind::Sum + | SummaryKind::Count + | SummaryKind::MinMax + | SummaryKind::Increase + | SummaryKind::Rate => { + unreachable!( + "{kind:?} is an exact-accumulator SummaryKind, never returned inside \ + Implementation::Sketch by asap_plan::boundary::implementation_for" + ) + } + }, + Implementation::ExactAccumulator { kind, .. } => match kind { + SummaryKind::Sum => Some(Capability::ExactAgg(AggregationType::Sum)), + SummaryKind::MinMax => Some(Capability::ExactAgg(AggregationType::MinMax)), + SummaryKind::Increase | SummaryKind::Rate => { + Some(Capability::ExactAgg(AggregationType::Increase)) + } + // `AggregationType` (this repo's own exact-accumulator-family + // enum) has no `Count` variant — the data plane has no + // working count accumulator (`SumAccumulator` returns `sum` + // for both `Statistic::Sum` and `Statistic::Count`, so a + // `count_over_time` query matched against a `Sum` policy + // would silently return sum-of-values, not sample-count). + // `asap_plan::boundary::implementation_for` still reports + // `Count{Exact}` as an `ExactAccumulator` (it assumes a real + // count accumulator exists, which is true in ASAPController's + // own reference implementation) — deliberately overridden + // here to `None` (archive) until a real + // `SumCountAccumulator` lands. + SummaryKind::Count => None, + SummaryKind::Kll + | SummaryKind::DDSketch + | SummaryKind::Hll + | SummaryKind::Theta + | SummaryKind::Kmv + | SummaryKind::Cms + | SummaryKind::CmsWithHeap + | SummaryKind::CountSketch + | SummaryKind::CountSketchWithHeap => { + unreachable!( + "{kind:?} is a sketch-family SummaryKind, never returned inside \ + Implementation::ExactAccumulator by asap_plan::boundary::implementation_for" + ) + } + }, } } From 490f3fcd949f32066599866111780369f5b1edd6 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 21 Jul 2026 14:04:51 -0600 Subject: [PATCH 11/11] fix(sketch_algebra): let Sum-family sids answer required Increase/Rate Root cause of the 4 data_plane test failures surfaced while re-verifying this rebase (execute_rate_dispatches_to_exact_agg_rate_reducer and friends) -- bisected to Step 6 ("PromQL topk/rate semantic retarget"), not the later capability_for delegation commit. Before Step 6, rate()/increase() lowered to AggIntent::Sum (outer_fn tracked the distinction separately), so a Sum-registered sid matched directly. Step 6 correctly gave Rate/Increase their own AggIntent (matching asap_plan::boundary::implementation_for's SummaryKind::Rate/Increase), which now requires Capability::ExactAgg(Increase) -- but nothing updated Capability::is_satisfied_by to let a Sum-registered sid answer it, and the PR's test plan only ran `cargo test -p control_plane`, so data_plane's own test suite (which is what actually exercises this end-to-end) never caught the regression. The data plane has no storage kind distinct from Sum for "Increase" in the first place: evaluate_exact_agg_rate (sketch_reducer.rs) already reduces Sum/MultipleSum/Increase/MultipleIncrease identically -- all read as raw per-window deltas, divided by the coverage-aware elapsed range. Confirmed via evaluate_exact_agg_rate's own unit tests (already passing, unaffected by this bug) that the reducer layer was never the problem -- only the capability-matching gate upstream of it was too strict. Adds Capability::is_satisfied_by's sum_satisfies_increase, mirroring multi_pop_satisfies_single's single/multi-population direction (a single-pop Sum can only serve a single- or multi-pop Increase requirement if available is multi; MultipleIncrease required still needs a multi-pop available). Does NOT relax the reverse (Increase answering a required Sum / sum_over_time) -- that's already refused explicitly elsewhere (issue #301: reconstructing cumulative-counter sums from per-window deltas is unsound). Also fixes two stale test/doc claims left over from Step 6 that asserted or documented the pre-Step-6 "everything collapses onto ExactAgg(Sum)" behavior as if it still held. Co-Authored-By: Claude Sonnet 5 --- Cargo.lock | 17 ++- .../src/sketch_algebra/capability.rs | 139 ++++++++++++++---- .../query_engines/asap_query_engine/engine.rs | 37 +++-- 3 files changed, 153 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 82a26411..fffaac5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -350,6 +350,15 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "asap-l2" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPController?rev=7fcaf914d87e71407c3a6d7ccac613b867f9c11b#7fcaf914d87e71407c3a6d7ccac613b867f9c11b" +dependencies = [ + "asap-ir", + "thiserror 2.0.18", +] + [[package]] name = "asap-plan" version = "0.1.0" @@ -780,6 +789,7 @@ version = "0.1.0" dependencies = [ "anyhow", "asap-ir", + "asap-l2", "asap-plan", "asap-sketch", "asap_types", @@ -790,7 +800,7 @@ dependencies = [ "http-body-util", "parking_lot", "prometheus", - "promql-parser 0.8.0", + "promql-parser 0.9.0", "prost", "prost-build", "reqwest 0.12.28", @@ -2467,9 +2477,8 @@ dependencies = [ [[package]] name = "promql-parser" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2791a28f8ea7e48f2838999c06d089184d44adb860feab682d45dd190ef718" +version = "0.9.0" +source = "git+https://github.com/ProjectASAP/promql-parser?rev=c51beafb361af4cc95ed62ae377862c660ceb757#c51beafb361af4cc95ed62ae377862c660ceb757" dependencies = [ "cfgrammar", "chrono", diff --git a/control_plane/src/sketch_algebra/capability.rs b/control_plane/src/sketch_algebra/capability.rs index fc81b89d..ef0f50df 100644 --- a/control_plane/src/sketch_algebra/capability.rs +++ b/control_plane/src/sketch_algebra/capability.rs @@ -93,14 +93,27 @@ pub enum Capability { /// `sum_over_time(metric[r])` / `sum(metric)` / bare selector WITHOUT /// re-parsing the raw PromQL string. /// -/// Background: the lowerer collapses every `AggFunc` in +/// Background: until the Phase 2 semantic retarget (ASAPController +/// alignment), the lowerer collapsed every `AggFunc` in /// `{Sum, Rate, Increase, Delta}` onto a single `AggIntent::Sum`, which -/// `capability_for` then maps to `Capability::ExactAgg(Sum)`. That -/// collapse erases the rate-vs-plain distinction the engine needs to +/// `capability_for` then mapped to `Capability::ExactAgg(Sum)` for all +/// of them — erasing the rate-vs-plain distinction the engine needs to /// decide between the plain ExactAgg reducer and the rate-divisor -/// reducer (`evaluate_exact_agg_rate`). Before this enum landed the -/// engine re-walked the raw PromQL via a `query_contains_rate_call` -/// helper to recover the distinction; that was a lossy-lowering smell. +/// reducer (`evaluate_exact_agg_rate`). `OuterFn` was introduced to +/// carry that distinction back (replacing an earlier raw-string +/// `query_contains_rate_call` re-parse). `rate`/`increase` now bind +/// their own `AggIntent::Rate`/`AggIntent::Increase` (matching +/// `asap_plan::boundary::implementation_for`'s `SummaryKind::Rate`/ +/// `Increase` — ASAPController models Rate as a distinct summary +/// family), both mapping to `Capability::ExactAgg(AggregationType::Increase)`; +/// only `sum`/`sum by (...)`/bare selectors and `sum_over_time` still +/// bind `AggIntent::Sum` → `ExactAgg(Sum)`. `OuterFn` still carries the +/// PromQL-function-shape distinction `Capability` doesn't encode (e.g. +/// which divisor/accumulation strategy `evaluate_exact_agg_rate` uses), +/// but the sid-matching predicate is no longer purely `ExactAgg(Sum)` +/// -- see `Capability::is_satisfied_by`'s `sum_satisfies_increase` for +/// how a Sum-registered sid still answers an `ExactAgg(Increase)` +/// required capability. /// /// The walker that populates this lives in `asap_tier_analysis.rs` /// (`trace_from_promql`) — it picks the most-specific counter-function @@ -111,24 +124,18 @@ pub enum Capability { /// /// Post-#299 the agent streams per-window DELTAS for counters. The four /// PromQL counter idioms have genuinely different semantics over those -/// deltas, but they ALL lower to a single `Capability::ExactAgg(Sum)` -/// (the `AggIntent::Sum` collapse erases the function name). Before -/// #301 the engine only distinguished `Rate` from everything else, so -/// `sum`, `sum_over_time`, `increase`, and instant-sum all hit the same -/// reducer path and returned the same (wrong) number. This enum carries -/// the function distinction the engine needs to dispatch correctly: +/// deltas. Before #301 the engine only distinguished `Rate` from +/// everything else, so `sum`, `sum_over_time`, `increase`, and +/// instant-sum all hit the same reducer path and returned the same +/// (wrong) number. This enum carries the function distinction the +/// engine needs to dispatch correctly: /// -/// | Variant | PromQL | Engine dispatch | -/// |---------------|------------------------------|---------------------------------------------------| -/// | `Plain` | `sum(c)` / `sum by (..) (c)` | accumulate ALL windows → cumulative-since-storage | -/// | `Rate` | `rate(c[r])` / `irate(c[r])` | Σ deltas in `[t-r,t]` ÷ min(r, coverage) | -/// | `Increase` | `increase(c[r])` | Σ deltas in `[t-r,t]` (one cumulative number) | -/// | `SumOverTime` | `sum_over_time(c[r])` | capability-miss → archive (can't reconstruct) | -/// -/// The taxonomy lives on `OuterFn` (not the `Capability` algebra) so the -/// sid-matching half stays a pure `ExactAgg(Sum)` predicate — the -/// function distinction is a query-evaluation concern, not a stored-state -/// one. +/// | Variant | PromQL | `required_capability` | Engine dispatch | +/// |---------------|------------------------------|-------------------------|----------------------------------------------------| +/// | `Plain` | `sum(c)` / `sum by (..) (c)` | `ExactAgg(Sum)` | accumulate ALL windows → cumulative-since-storage | +/// | `Rate` | `rate(c[r])` / `irate(c[r])` | `ExactAgg(Increase)` | Σ deltas in `[t-r,t]` ÷ min(r, coverage) | +/// | `Increase` | `increase(c[r])` | `ExactAgg(Increase)` | Σ deltas in `[t-r,t]` (one cumulative number) | +/// | `SumOverTime` | `sum_over_time(c[r])` | `ExactAgg(Sum)` | capability-miss → archive (can't reconstruct) | #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] pub enum OuterFn { /// No range-style counter function in the expression — bare selector, @@ -358,10 +365,14 @@ impl Capability { // serving multi-pop) is NOT allowed — the single-pop // policy has lost the key dimension and can't recover it. // - // Cross-family ExactAgg combos (Sum vs MinMax, etc.) - // remain non-satisfiable: they're different operations. + // Cross-family ExactAgg combos are mostly non-satisfiable + // (Sum vs MinMax, etc. are different operations) — except + // Sum-family serving a required Increase/Rate capability, + // which IS sound; see `sum_satisfies_increase`. (Capability::ExactAgg(req), Capability::ExactAgg(have)) => { - req == have || multi_pop_satisfies_single(*req, *have) + req == have + || multi_pop_satisfies_single(*req, *have) + || sum_satisfies_increase(*req, *have) } _ => false, } @@ -418,6 +429,47 @@ fn multi_pop_satisfies_single(required: AggregationType, available: AggregationT ) } +/// True when an available Sum-family accumulator can answer a required +/// Increase/Rate capability. +/// +/// `rate()`/`increase()` PromQL both lower to a required +/// `Capability::ExactAgg(AggregationType::Increase)` (`capability_for`'s +/// `AggIntent::Rate | AggIntent::Increase` case, matching +/// `asap_plan::boundary::implementation_for`'s `SummaryKind::Increase | +/// SummaryKind::Rate` — ASAPController models Rate as its own summary +/// family). But this workspace's data plane has no storage kind distinct +/// from Sum for it: `evaluate_exact_agg_rate` (`sketch_reducer.rs`) +/// already reduces `Sum`/`MultipleSum`/`Increase`/`MultipleIncrease` +/// identically — all read as `Statistic::Sum` per window, then divided +/// by the coverage-aware elapsed range — so a Sum-registered sid's raw +/// per-window deltas answer a rate query exactly as well as an +/// Increase-registered one's. This is what lets counter metrics ingested +/// as plain `Sum` (not every ingest path distinguishes Increase from +/// Sum at registration time) still answer `rate(...)`/`increase(...)`. +/// +/// Respects the same single/multi-population direction as +/// [`multi_pop_satisfies_single`]: a multi-pop available (`MultipleSum`) +/// can serve a single- or multi-pop required capability; a single-pop +/// available (`Sum`) can only serve a single-pop required one — it has +/// already lost the per-key breakdown a multi-pop required capability +/// would need. +/// +/// Deliberately does NOT apply to a required `Capability::ExactAgg(Sum)` +/// (plain `sum_over_time`) — reconstructing Σ-of-cumulative-counter- +/// samples from per-window deltas is unsound (issue #301); that shape is +/// refused explicitly at the query-dispatch layer, not routed here. +fn sum_satisfies_increase(required: AggregationType, available: AggregationType) -> bool { + matches!( + (required, available), + (AggregationType::Increase, AggregationType::Sum) + | (AggregationType::Increase, AggregationType::MultipleSum) + | ( + AggregationType::MultipleIncrease, + AggregationType::MultipleSum + ) + ) +} + // ── AggIntent → Capability bridge ──────────────────────────────────────────── /// Map a semantic [`AggIntent`] to the ASAP-tier [`Capability`] that can @@ -971,6 +1023,41 @@ mod tests { } } + #[test] + fn is_satisfied_by_sum_family_answers_required_increase() { + // A Sum-registered sid's raw per-window deltas answer a + // rate()/increase() query exactly as well as an Increase- + // registered one's -- evaluate_exact_agg_rate (sketch_reducer.rs) + // reduces both identically. See sum_satisfies_increase's doc. + let required = Capability::ExactAgg(AggregationType::Increase); + assert!(required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); + assert!(required.is_satisfied_by(&Capability::ExactAgg(AggregationType::MultipleSum))); + + let required_multi = Capability::ExactAgg(AggregationType::MultipleIncrease); + assert!(required_multi.is_satisfied_by(&Capability::ExactAgg(AggregationType::MultipleSum))); + } + + #[test] + fn is_satisfied_by_sum_family_does_not_answer_required_multi_increase_from_single_sum() { + // Same single/multi-population direction as multi_pop_satisfies_single: + // a single-pop available (Sum) can't serve a multi-pop required + // capability (MultipleIncrease) -- it already lost the per-key + // breakdown a multi-pop caller needs. + let required = Capability::ExactAgg(AggregationType::MultipleIncrease); + assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); + } + + #[test] + fn is_satisfied_by_increase_does_not_answer_required_sum() { + // The reverse direction is NOT sound: required ExactAgg(Sum) is + // sum_over_time semantics (Σ of raw cumulative-counter samples), + // which an Increase-registered sid's per-window deltas cannot + // reconstruct (issue #301) -- refused explicitly at the + // query-dispatch layer, not routed through here. + let required = Capability::ExactAgg(AggregationType::Sum); + assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Increase))); + } + // ── capability_for: ExactAgg dormancy ──────────────────────────────── #[test] diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index c983274c..268118af 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -3632,6 +3632,8 @@ mod asap_tier_classify_tests { /// the first place. #[test] fn analyzer_candidate_outer_fn_distinguishes_rate_from_sum_over_time() { + use crate::storage_engines::sketch_db::data::AggregationType; + use crate::storage_engines::sketch_db::index::Capability; use control_plane::asap_tier_analysis::{analyze_promql_for_asap_tier, OuterFn}; let rate = analyze_promql_for_asap_tier("rate(http_requests_total[5m])"); let sot = analyze_promql_for_asap_tier("sum_over_time(http_requests_total[5m])"); @@ -3644,18 +3646,29 @@ mod asap_tier_classify_tests { assert!(sum_by_rate.unsupported.is_none() && !sum_by_rate.candidates.is_empty()); assert!(bare.unsupported.is_none() && !bare.candidates.is_empty()); - // Same capability for ALL — the field that disambiguates is - // `outer_fn`, not `required_capability`. + // Whichever query has a `rate(...)` call ANYWHERE in its tree + // (bare `rate(...)` or composed `sum by (...) (rate(...))`) binds + // to `AggIntent::Rate` and now maps to `ExactAgg(Increase)` -- + // matching `asap_plan::boundary::implementation_for`'s + // `SummaryKind::Rate` (ASAPController models Rate as its own + // summary family; see `capability_for`'s module doc and + // `Capability::is_satisfied_by`'s `sum_satisfies_increase` for + // why a Sum-registered sid still answers it). This is a real, + // intentional behavior change from the Phase 2 semantic retarget + // (Rate/Increase used to collapse onto AggIntent::Sum) -- not a + // stale assertion left over from before it. `sot`/`bare` have no + // `rate(...)` anywhere and stay `ExactAgg(Sum)`. assert_eq!( rate.candidates[0].required_capability, - sot.candidates[0].required_capability, + Capability::ExactAgg(AggregationType::Increase), ); assert_eq!( - rate.candidates[0].required_capability, - sum_by_rate.candidates[0].required_capability, + rate.candidates[0].required_capability, sum_by_rate.candidates[0].required_capability, + "composed `sum by (...) (rate(...))` binds the same Rate \ + AggIntent as bare `rate(...)`", ); assert_eq!( - rate.candidates[0].required_capability, + sot.candidates[0].required_capability, bare.candidates[0].required_capability, ); @@ -3672,11 +3685,15 @@ mod asap_tier_classify_tests { } /// `sum by (zone) (rate(http_requests_total[5m]))` end-to-end. - /// The analyzer gives `Capability::ExactAgg(Sum)` with - /// `function="sum"` (outer), `range_seconds=300` (lifted from the - /// inner rate's matrix selector), AND `outer_fn=OuterFn::Rate` + /// The analyzer gives `Capability::ExactAgg(Increase)` (the inner + /// `rate(...)` binds the `AggIntent::Rate` `capability_for` reads; + /// see `analyzer_candidate_outer_fn_distinguishes_rate_from_sum_over_time`) + /// with `function="sum"` (outer), `range_seconds=300` (lifted from + /// the inner rate's matrix selector), AND `outer_fn=OuterFn::Rate` /// (the analyzer's PromQL trace walker flags the inner rate call). - /// The engine dispatches to `evaluate_exact_agg_rate` off the + /// The registered sid here is `ExactAgg(Sum)` -- satisfied via + /// `Capability::is_satisfied_by`'s `sum_satisfies_increase`. The + /// engine dispatches to `evaluate_exact_agg_rate` off the /// typed `outer_fn` field, which folds the per-zone per-window /// sums and divides by 300. #[tokio::test]