From ea208263cc6184811cd643eeb5d34bb8601d9dbd Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 08:26:24 -0600 Subject: [PATCH 01/48] feat(cost): validate analytical comparison evidence --- .../asap-aware-mapping/src/analytical_cost.rs | 755 ++++++++++++------ .../src/analytical_statistics.rs | 213 +++++ crates/asap-aware-mapping/src/lib.rs | 1 + .../analytical-resource-cost.md | 119 ++- 4 files changed, 823 insertions(+), 265 deletions(-) create mode 100644 crates/asap-aware-mapping/src/analytical_statistics.rs diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 15e8df9b..639497df 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -5,10 +5,16 @@ //! [`ResourceCalibration`]; without that calibration the dimensional //! estimate is still useful for explanations, but is not silently comparable. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; +use asap_types::post_asap::{SketchAlgorithm, SketchParams}; +use asap_types::workload::DataArrival; use serde::{Deserialize, Serialize}; +use crate::analytical_statistics::{ + ComparisonScope, OperatorStatistics, OperatorStatisticsProvider, +}; + pub const ANALYTICAL_MODEL_VERSION: &str = "analytical-resource-at-rest-v1"; /// Conversion from physical dimensions to one deployment-specific objective. @@ -68,26 +74,6 @@ pub enum PhysicalOperator { PassThrough, } -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] -pub struct OperatorInputs { - pub input_rows: u64, - pub input_bytes: u64, - pub output_rows: u64, - pub output_bytes: u64, - pub group_count: Option, - pub key_bytes: Option, - /// Bytes of aggregate accumulator state retained per group. Required for - /// hash aggregation because one 8-byte value is not universal. - pub aggregate_value_bytes: Option, - pub k: Option, - /// Rows actually consumed by a physical Limit, including rows skipped by - /// OFFSET. This is distinct from `output_rows`. - pub limit_rows_consumed: Option, - pub right_rows: Option, - pub right_bytes: Option, - pub hash_join_build_side: Option, -} - #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum HashJoinBuildSide { Left, @@ -101,7 +87,6 @@ pub enum HashJoinBuildSide { pub struct PhysicalDagNode { pub id: String, pub operator: PhysicalOperator, - pub inputs: OperatorInputs, pub children: Vec, /// Maximum transient edge buffer, distinct from logical `output_bytes`. pub output_buffer_bytes: u64, @@ -123,13 +108,10 @@ pub enum ExecutionMultiplicity { pub fn estimate_physical_dag( nodes: &[PhysicalDagNode], root: &str, - evaluation_count: u64, + scope: &ComparisonScope, + statistics: &impl OperatorStatisticsProvider, ) -> Result { - use std::collections::HashMap; - - if evaluation_count == 0 { - return Err(AnalyticalCostError::MissingOrZero("evaluation_count")); - } + let evaluation_count = scope.validate()?; let by_id: HashMap<&str, &PhysicalDagNode> = nodes.iter().map(|n| (n.id.as_str(), n)).collect(); if by_id.len() != nodes.len() { return Err(AnalyticalCostError::InvalidPhysicalDag("duplicate node id")); @@ -163,8 +145,17 @@ pub fn estimate_physical_dag( } visit(root, &by_id, &mut visiting, &mut visited, &mut order)?; + // Resolve each reachable node exactly once. A provider may be backed by a + // live catalog; one estimate must not mix observations from two refreshes. + let resolved_statistics: HashMap<&str, OperatorStatistics> = order + .iter() + .map(|id| statistics.statistics(id).map(|value| (*id, value))) + .collect::>()?; + for id in &order { let node = by_id[id]; + let node_statistics = &resolved_statistics[id]; + validate_operator_statistics(node, node_statistics, &by_id, &resolved_statistics)?; if node.retained_bytes > 0 && matches!(node.execution, ExecutionMultiplicity::PerEvaluation) { return Err(AnalyticalCostError::InvalidPhysicalDag( @@ -197,7 +188,7 @@ pub fn estimate_physical_dag( let mut live_outputs: HashMap<&str, u64> = HashMap::new(); for id in order { let node = by_id[id]; - let local = estimate_operator(node.operator, node.inputs)?; + let local = estimate_operator(node.operator, resolved_statistics[id].clone())?; let executions = match node.execution { ExecutionMultiplicity::Once => 1, ExecutionMultiplicity::PerEvaluation => evaluation_count, @@ -254,45 +245,114 @@ pub fn estimate_physical_dag( }) } +fn validate_operator_statistics( + node: &PhysicalDagNode, + node_statistics: &OperatorStatistics, + nodes: &HashMap<&str, &PhysicalDagNode>, + statistics: &HashMap<&str, OperatorStatistics>, +) -> Result<(), AnalyticalCostError> { + let expected_inputs = match node.operator { + PhysicalOperator::Scan => 1, + PhysicalOperator::HashJoin => 2, + PhysicalOperator::Concat => node.children.len(), + _ => 1, + }; + if node_statistics.inputs.len() != expected_inputs { + return Err(AnalyticalCostError::InvalidOperatorStatistics { + node: node.id.clone(), + reason: "wrong input-edge count", + }); + } + let expected_children = if matches!(node.operator, PhysicalOperator::Scan) { + 0 + } else { + expected_inputs + }; + if node.children.len() != expected_children { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "operator child count does not match physical arity", + )); + } + for edge in node_statistics + .inputs + .iter() + .chain(std::iter::once(&node_statistics.output)) + { + if !edge.is_consistent() { + return Err(AnalyticalCostError::InvalidOperatorStatistics { + node: node.id.clone(), + reason: "edge rows and logical bytes are inconsistent", + }); + } + } + for (input_index, child_id) in node.children.iter().enumerate() { + let child = nodes + .get(child_id.as_str()) + .ok_or(AnalyticalCostError::InvalidPhysicalDag("missing node"))?; + let child_statistics = &statistics[child.id.as_str()]; + if node_statistics.inputs[input_index] != child_statistics.output { + return Err(AnalyticalCostError::ConflictingEdgeStatistics { + parent: node.id.clone(), + child: child.id.clone(), + input_index, + }); + } + } + Ok(()) +} + /// Estimate one physical operator. Child costs are deliberately excluded; /// a DAG walker sums CPU/disk once per node and combines simultaneously /// retained state separately. pub fn estimate_operator( operator: PhysicalOperator, - input: OperatorInputs, + statistics: OperatorStatistics, ) -> Result { validate_operator_semantics(operator, &input)?; let per_row_width = |rows: u64, bytes: u64| -> Result { - if rows == 0 || bytes == 0 { - return Err(AnalyticalCostError::MissingOrZero("operator rows/bytes")); + match (rows, bytes) { + (0, 0) => return Ok(0), + (0, _) | (_, 0) => { + return Err(AnalyticalCostError::MissingOrZero("operator rows/bytes")); + } + _ => {} } Ok(bytes.div_ceil(rows)) }; + let input = |index: usize| { + statistics + .inputs + .get(index) + .copied() + .ok_or(AnalyticalCostError::MissingOrZero("operator input edge")) + }; + let left = input(0)?; + let output = statistics.output; let estimate = match operator { PhysicalOperator::Scan => ResourceEstimate { - cpu_ops: input.input_rows as f64, - peak_memory_bytes: per_row_width(input.input_rows, input.input_bytes)?, - scan_bytes: input.input_bytes, + cpu_ops: left.rows as f64, + peak_memory_bytes: per_row_width(left.rows, left.bytes)?, + scan_bytes: left.bytes, }, PhysicalOperator::Filter | PhysicalOperator::Project | PhysicalOperator::PassThrough => { ResourceEstimate { - cpu_ops: input.input_rows as f64, - peak_memory_bytes: per_row_width(input.output_rows, input.output_bytes)?, + cpu_ops: left.rows as f64, + peak_memory_bytes: per_row_width(output.rows, output.bytes)?, scan_bytes: 0, } } PhysicalOperator::HashAggregate => { - let groups = input + let groups = statistics .group_count .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - let key = input + let key = statistics .key_bytes .ok_or(AnalyticalCostError::MissingOrZero("key_bytes"))?; - let value = input + let value = statistics .aggregate_value_bytes .ok_or(AnalyticalCostError::MissingOrZero("aggregate_value_bytes"))?; ResourceEstimate { - cpu_ops: input.input_rows as f64, + cpu_ops: left.rows as f64, peak_memory_bytes: checked_bytes(&[ groups, key.checked_add(value) @@ -303,14 +363,14 @@ pub fn estimate_operator( } } PhysicalOperator::Deduplicate => { - let groups = input + let groups = statistics .group_count .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - let key = input + let key = statistics .key_bytes .ok_or(AnalyticalCostError::MissingOrZero("key_bytes"))?; ResourceEstimate { - cpu_ops: input.input_rows as f64, + cpu_ops: left.rows as f64, peak_memory_bytes: checked_bytes(&[ groups, key.checked_add(16).ok_or(AnalyticalCostError::Overflow)?, @@ -319,67 +379,47 @@ pub fn estimate_operator( } } PhysicalOperator::Sort | PhysicalOperator::Window => ResourceEstimate { - cpu_ops: input.input_rows as f64 * (input.input_rows.max(2) as f64).log2().ceil(), - peak_memory_bytes: input.input_bytes, + cpu_ops: left.rows as f64 * (left.rows.max(2) as f64).log2().ceil(), + peak_memory_bytes: left.bytes, scan_bytes: 0, }, PhysicalOperator::TopK => { - let k = input.k.ok_or(AnalyticalCostError::MissingOrZero("k"))?; - let heap_rows = k.min(input.input_rows); + let k = statistics + .k + .ok_or(AnalyticalCostError::MissingOrZero("k"))?; ResourceEstimate { - cpu_ops: input.input_rows as f64 * (heap_rows.max(2) as f64).log2().ceil(), + cpu_ops: left.rows as f64 * (k.max(2) as f64).log2().ceil(), peak_memory_bytes: checked_bytes(&[ - heap_rows, - per_row_width(input.input_rows, input.input_bytes)?, + k.min(left.rows), + per_row_width(left.rows, left.bytes)?, ])?, scan_bytes: 0, } } PhysicalOperator::HashJoin => { - let right_rows = input - .right_rows - .ok_or(AnalyticalCostError::MissingOrZero("right_rows"))?; - let right_bytes = input - .right_bytes - .ok_or(AnalyticalCostError::MissingOrZero("right_bytes"))?; - let build_side = input + let right = input(1)?; + let build_side = statistics .hash_join_build_side .ok_or(AnalyticalCostError::MissingOrZero("hash_join_build_side"))?; ResourceEstimate { - cpu_ops: input.input_rows as f64 + right_rows as f64 + input.output_rows as f64, + cpu_ops: left.rows as f64 + right.rows as f64 + output.rows as f64, peak_memory_bytes: match build_side { - HashJoinBuildSide::Left => input - .input_rows - .checked_mul(16) - .and_then(|metadata| input.input_bytes.checked_add(metadata)), - HashJoinBuildSide::Right => right_rows - .checked_mul(16) - .and_then(|metadata| right_bytes.checked_add(metadata)), - } - .ok_or(AnalyticalCostError::Overflow)?, + HashJoinBuildSide::Left => left.bytes, + HashJoinBuildSide::Right => right.bytes, + }, scan_bytes: 0, } } PhysicalOperator::Concat => ResourceEstimate { - cpu_ops: input.output_rows as f64, - peak_memory_bytes: per_row_width(input.output_rows, input.output_bytes)?, + cpu_ops: output.rows as f64, + peak_memory_bytes: per_row_width(output.rows, output.bytes)?, + scan_bytes: 0, + }, + PhysicalOperator::Limit => ResourceEstimate { + cpu_ops: output.rows as f64, + peak_memory_bytes: per_row_width(output.rows, output.bytes)?, scan_bytes: 0, }, - PhysicalOperator::Limit => { - let consumed = input - .limit_rows_consumed - .ok_or(AnalyticalCostError::MissingOrZero("limit_rows_consumed"))?; - if consumed > input.input_rows || consumed < input.output_rows { - return Err(AnalyticalCostError::InconsistentOperatorStatistics( - "Limit rows consumed must cover its output without exceeding its input", - )); - } - ResourceEstimate { - cpu_ops: consumed as f64, - peak_memory_bytes: per_row_width(input.output_rows, input.output_bytes)?, - scan_bytes: 0, - } - } }; if estimate.cpu_ops.is_finite() { Ok(estimate) @@ -464,14 +504,46 @@ impl ResourceEstimate { #[derive(Debug, Clone, PartialEq, thiserror::Error)] pub enum AnalyticalCostError { + #[error("analytical resource model v1 supports only DataArrival::AtRest, got {0:?}")] + UnsupportedDataArrival(DataArrival), #[error("required analytical input {0} is missing or zero")] MissingOrZero(&'static str), + #[error("required analytical evidence {0} is missing or stale")] + MissingOrStale(&'static str), + #[error("query recurrence cannot be resolved over the planning horizon")] + InvalidRecurrence, + #[error("query has no evaluations in the planning horizon")] + NoEvaluationsInHorizon, #[error("calibration {0} must be finite and non-negative, got {1}")] InvalidCalibration(&'static str, f64), #[error("at least one calibration coefficient must be positive")] ZeroCalibration, + #[error("algorithm {0:?} does not match parameters {1:?}")] + ParameterMismatch(SketchAlgorithm, SketchParams), + #[error("{0} needs a value-range/bin-count model before it can be estimated")] + UnsupportedWithoutDistribution(&'static str), #[error("analytical arithmetic overflowed")] Overflow, + #[error("candidate has no supported exact or sketch state")] + UnsupportedCandidate, + #[error("summary operation {0} has no lifecycle-aware cost formula")] + UnsupportedSummaryOperation(&'static str), + #[error("required comparison-scope field {0} is missing")] + MissingComparisonScope(&'static str), + #[error("raw and candidate comparison scopes differ in {0}")] + ComparisonScopeMismatch(&'static str), + #[error("operator statistics are unavailable for physical node {0}")] + MissingOperatorStatistics(String), + #[error("invalid operator statistics for {node}: {reason}")] + InvalidOperatorStatistics { node: String, reason: &'static str }, + #[error( + "operator statistics conflict: parent {parent} input {input_index} does not match child {child} output" + )] + ConflictingEdgeStatistics { + parent: String, + child: String, + input_index: usize, + }, #[error("invalid physical DAG: {0}")] InvalidPhysicalDag(&'static str), #[error("inconsistent physical operator statistics: {0}")] @@ -488,6 +560,10 @@ fn checked_bytes(parts: &[u64]) -> Result { #[cfg(test)] mod tests { use super::*; + use crate::analytical_statistics::{ + validate_comparison_scopes, ComparisonScope, EdgeStatistics, OperatorStatistics, + SourceCoverage, + }; fn unary_inputs(input_rows: u64, output_rows: u64) -> OperatorInputs { OperatorInputs { @@ -538,39 +614,34 @@ mod tests { fn physical_operator_formulas_keep_disk_at_scan_and_require_join_stats() { let scan = estimate_operator( PhysicalOperator::Scan, - OperatorInputs { - input_rows: 1_000, - input_bytes: 64_000, - output_rows: 1_000, - output_bytes: 64_000, - group_count: None, - key_bytes: None, - aggregate_value_bytes: None, - k: None, - limit_rows_consumed: None, - right_rows: None, - right_bytes: None, - hash_join_build_side: None, - }, + statistics( + vec![EdgeStatistics { + rows: 1_000, + bytes: 64_000, + }], + EdgeStatistics { + rows: 1_000, + bytes: 64_000, + }, + ), ) .unwrap(); assert_eq!(scan.scan_bytes, 64_000); let topk = estimate_operator( PhysicalOperator::TopK, - OperatorInputs { - input_rows: 1_000, - input_bytes: 40_000, - output_rows: 10, - output_bytes: 400, - group_count: None, - key_bytes: None, - aggregate_value_bytes: None, + OperatorStatistics { k: Some(10), - limit_rows_consumed: None, - right_rows: None, - right_bytes: None, - hash_join_build_side: None, + ..statistics( + vec![EdgeStatistics { + rows: 1_000, + bytes: 40_000, + }], + EdgeStatistics { + rows: 10, + bytes: 400, + }, + ) }, ) .unwrap(); @@ -580,42 +651,40 @@ mod tests { let missing_join_stats = estimate_operator( PhysicalOperator::HashJoin, - OperatorInputs { - input_rows: 1_000, - input_bytes: 64_000, - output_rows: 100, - output_bytes: 12_800, - group_count: None, - key_bytes: None, - aggregate_value_bytes: None, - k: None, - limit_rows_consumed: None, - right_rows: None, - right_bytes: None, - hash_join_build_side: None, - }, + statistics( + vec![EdgeStatistics { + rows: 1_000, + bytes: 64_000, + }], + EdgeStatistics { + rows: 100, + bytes: 12_800, + }, + ), ); assert_eq!( missing_join_stats, - Err(AnalyticalCostError::MissingOrZero("right_rows")) + Err(AnalyticalCostError::MissingOrZero("operator input edge")) ); let missing_build_side = estimate_operator( PhysicalOperator::HashJoin, - OperatorInputs { - input_rows: 1_000, - input_bytes: 64_000, - output_rows: 100, - output_bytes: 12_800, - group_count: None, - key_bytes: None, - aggregate_value_bytes: None, - k: None, - limit_rows_consumed: None, - right_rows: Some(10), - right_bytes: Some(1_280), - hash_join_build_side: None, - }, + statistics( + vec![ + EdgeStatistics { + rows: 1_000, + bytes: 64_000, + }, + EdgeStatistics { + rows: 10, + bytes: 1_280, + }, + ], + EdgeStatistics { + rows: 100, + bytes: 12_800, + }, + ), ); assert_eq!( missing_build_side, @@ -624,19 +693,24 @@ mod tests { let build_left = estimate_operator( PhysicalOperator::HashJoin, - OperatorInputs { - input_rows: 1_000, - input_bytes: 64_000, - output_rows: 100, - output_bytes: 12_800, - group_count: None, - key_bytes: None, - aggregate_value_bytes: None, - k: None, - limit_rows_consumed: None, - right_rows: Some(10), - right_bytes: Some(1_280), + OperatorStatistics { hash_join_build_side: Some(HashJoinBuildSide::Left), + ..statistics( + vec![ + EdgeStatistics { + rows: 1_000, + bytes: 64_000, + }, + EdgeStatistics { + rows: 10, + bytes: 1_280, + }, + ], + EdgeStatistics { + rows: 100, + bytes: 12_800, + }, + ) }, ) .unwrap(); @@ -644,19 +718,20 @@ mod tests { let aggregate = estimate_operator( PhysicalOperator::HashAggregate, - OperatorInputs { - input_rows: 1_000, - input_bytes: 64_000, - output_rows: 100, - output_bytes: 4_000, + OperatorStatistics { group_count: Some(100), key_bytes: Some(16), aggregate_value_bytes: Some(24), - k: None, - limit_rows_consumed: None, - right_rows: None, - right_bytes: None, - hash_join_build_side: None, + ..statistics( + vec![EdgeStatistics { + rows: 1_000, + bytes: 64_000, + }], + EdgeStatistics { + rows: 100, + bytes: 4_000, + }, + ) }, ) .unwrap(); @@ -705,25 +780,10 @@ mod tests { #[test] fn physical_dag_counts_shared_scan_once_and_uses_live_memory() { - let input = |input_rows, input_bytes, output_rows, output_bytes| OperatorInputs { - input_rows, - input_bytes, - output_rows, - output_bytes, - group_count: None, - key_bytes: None, - aggregate_value_bytes: None, - k: None, - limit_rows_consumed: None, - right_rows: None, - right_bytes: None, - hash_join_build_side: None, - }; let nodes = vec![ PhysicalDagNode { id: "scan".into(), operator: PhysicalOperator::Scan, - inputs: input(100, 1_000, 100, 1_000), children: vec![], output_buffer_bytes: 10, retained_bytes: 0, @@ -732,7 +792,6 @@ mod tests { PhysicalDagNode { id: "left".into(), operator: PhysicalOperator::Filter, - inputs: input(100, 1_000, 40, 400), children: vec!["scan".into()], output_buffer_bytes: 4, retained_bytes: 0, @@ -741,7 +800,6 @@ mod tests { PhysicalDagNode { id: "right".into(), operator: PhysicalOperator::Filter, - inputs: input(100, 1_000, 40, 400), children: vec!["scan".into()], output_buffer_bytes: 4, retained_bytes: 0, @@ -750,14 +808,38 @@ mod tests { PhysicalDagNode { id: "root".into(), operator: PhysicalOperator::Concat, - inputs: input(80, 800, 80, 800), children: vec!["left".into(), "right".into()], output_buffer_bytes: 8, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, }, ]; - let estimate = estimate_physical_dag(&nodes, "root", 2).unwrap(); + let scan_edge = EdgeStatistics { + rows: 100, + bytes: 1_000, + }; + let branch_edge = EdgeStatistics { + rows: 40, + bytes: 400, + }; + let provided = HashMap::from([ + ("scan".into(), statistics(vec![scan_edge], scan_edge)), + ("left".into(), statistics(vec![scan_edge], branch_edge)), + ("right".into(), statistics(vec![scan_edge], branch_edge)), + ( + "root".into(), + statistics( + vec![branch_edge, branch_edge], + EdgeStatistics { + rows: 80, + bytes: 800, + }, + ), + ), + ]); + let mut scope = comparison_scope(); + scope.horizon.0 = 20_000; + let estimate = estimate_physical_dag(&nodes, "root", &scope, &provided).unwrap(); assert_eq!(estimate.cpu_ops, 760.0); assert_eq!(estimate.scan_bytes, 2_000); // This is neither the sum of every node's memory nor just the largest @@ -771,20 +853,6 @@ mod tests { PhysicalDagNode { id: "scan".into(), operator: PhysicalOperator::Scan, - inputs: OperatorInputs { - input_rows: 100, - input_bytes: 1_000, - output_rows: 100, - output_bytes: 1_000, - group_count: None, - key_bytes: None, - aggregate_value_bytes: None, - k: None, - limit_rows_consumed: None, - right_rows: None, - right_bytes: None, - hash_join_build_side: None, - }, children: vec![], output_buffer_bytes: 10, retained_bytes: 0, @@ -793,20 +861,6 @@ mod tests { PhysicalDagNode { id: "state".into(), operator: PhysicalOperator::HashAggregate, - inputs: OperatorInputs { - input_rows: 100, - input_bytes: 1_000, - output_rows: 1, - output_bytes: 16, - group_count: Some(1), - key_bytes: Some(8), - aggregate_value_bytes: Some(8), - k: None, - limit_rows_consumed: None, - right_rows: None, - right_bytes: None, - hash_join_build_side: None, - }, children: vec!["scan".into()], output_buffer_bytes: 16, retained_bytes: 32, @@ -815,28 +869,281 @@ mod tests { PhysicalDagNode { id: "read".into(), operator: PhysicalOperator::Limit, - inputs: OperatorInputs { - input_rows: 1, - input_bytes: 16, - output_rows: 1, - output_bytes: 16, - group_count: None, - key_bytes: None, - aggregate_value_bytes: None, - k: None, - limit_rows_consumed: Some(1), - right_rows: None, - right_bytes: None, - hash_join_build_side: None, - }, children: vec!["state".into()], output_buffer_bytes: 16, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, }, ]; - let estimate = estimate_physical_dag(&nodes, "read", 10).unwrap(); + let scan_edge = EdgeStatistics { + rows: 100, + bytes: 1_000, + }; + let state_edge = EdgeStatistics { rows: 1, bytes: 16 }; + let provided = HashMap::from([ + ("scan".into(), statistics(vec![scan_edge], scan_edge)), + ( + "state".into(), + OperatorStatistics { + group_count: Some(1), + key_bytes: Some(8), + aggregate_value_bytes: Some(8), + ..statistics(vec![scan_edge], state_edge) + }, + ), + ("read".into(), statistics(vec![state_edge], state_edge)), + ]); + let mut scope = comparison_scope(); + scope.horizon.0 = 100_000; + let estimate = estimate_physical_dag(&nodes, "read", &scope, &provided).unwrap(); assert_eq!(estimate.cpu_ops, 210.0); assert_eq!(estimate.scan_bytes, 1_000); } + + fn comparison_scope() -> ComparisonScope { + use asap_types::pre_asap::query_expr::Source; + use asap_types::workload::{ + DurationMs, QueryRecurrence, QueryTimeScope, RepeatedDemand, RepetitionInterval, + TimeSelection, TimestampMs, + }; + + ComparisonScope { + data_arrival: DataArrival::AtRest, + planning_time: TimestampMs(1_000), + horizon: DurationMs(60_000), + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval( + RepetitionInterval(10_000), + )), + time_selection: TimeSelection { + scope: QueryTimeScope::Longitudinal, + lookback: Some(DurationMs(300_000)), + as_of: Some(TimestampMs(1_000)), + }, + sources: vec![SourceCoverage { + source: Source::Table { + table_ref: "metrics".into(), + }, + snapshot_id: "catalog-version-42".into(), + predicates: vec![], + }], + } + } + + fn statistics(inputs: Vec, output: EdgeStatistics) -> OperatorStatistics { + OperatorStatistics { + inputs, + output, + group_count: None, + key_bytes: None, + aggregate_value_bytes: None, + k: None, + hash_join_build_side: None, + } + } + + #[test] + fn comparison_rejects_different_snapshot_predicate_time_or_horizon() { + use std::rc::Rc; + + use asap_types::pre_asap::query_expr::{Predicate, QueryExpr}; + use asap_types::workload::{DurationMs, TimestampMs}; + + let raw = comparison_scope(); + assert_eq!(validate_comparison_scopes(&raw, &raw).unwrap(), 6); + + let mut candidate = raw.clone(); + candidate.sources[0].snapshot_id = "catalog-version-43".into(); + assert_eq!( + validate_comparison_scopes(&raw, &candidate), + Err(AnalyticalCostError::ComparisonScopeMismatch("sources")) + ); + + candidate = raw.clone(); + candidate.sources[0] + .predicates + .push(Predicate(Rc::new(QueryExpr::promql_scalar(1.0)))); + assert_eq!( + validate_comparison_scopes(&raw, &candidate), + Err(AnalyticalCostError::ComparisonScopeMismatch("sources")) + ); + + candidate = raw.clone(); + candidate.time_selection.as_of = Some(TimestampMs(2_000)); + assert_eq!( + validate_comparison_scopes(&raw, &candidate), + Err(AnalyticalCostError::ComparisonScopeMismatch( + "time_selection" + )) + ); + + candidate = raw.clone(); + candidate.horizon = DurationMs(120_000); + assert_eq!( + validate_comparison_scopes(&raw, &candidate), + Err(AnalyticalCostError::ComparisonScopeMismatch("horizon")) + ); + } + + #[test] + fn physical_dag_fails_closed_on_missing_or_conflicting_edge_statistics() { + use std::collections::HashMap; + + let nodes = vec![ + PhysicalDagNode { + id: "scan".into(), + operator: PhysicalOperator::Scan, + children: vec![], + output_buffer_bytes: 10, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }, + PhysicalDagNode { + id: "filter".into(), + operator: PhysicalOperator::Filter, + children: vec!["scan".into()], + output_buffer_bytes: 4, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }, + ]; + let scope = comparison_scope(); + let mut provided = HashMap::from([( + "scan".to_string(), + statistics( + vec![EdgeStatistics { + rows: 100, + bytes: 1_000, + }], + EdgeStatistics { + rows: 100, + bytes: 1_000, + }, + ), + )]); + + assert_eq!( + estimate_physical_dag(&nodes, "filter", &scope, &provided), + Err(AnalyticalCostError::MissingOperatorStatistics( + "filter".into() + )) + ); + + provided.insert( + "filter".into(), + statistics( + vec![EdgeStatistics { + rows: 99, + bytes: 990, + }], + EdgeStatistics { + rows: 40, + bytes: 400, + }, + ), + ); + assert_eq!( + estimate_physical_dag(&nodes, "filter", &scope, &provided), + Err(AnalyticalCostError::ConflictingEdgeStatistics { + parent: "filter".into(), + child: "scan".into(), + input_index: 0, + }) + ); + } + + #[test] + fn provider_statistics_drive_a_consistent_physical_dag_estimate() { + use std::collections::HashMap; + + let nodes = vec![ + PhysicalDagNode { + id: "scan".into(), + operator: PhysicalOperator::Scan, + children: vec![], + output_buffer_bytes: 10, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }, + PhysicalDagNode { + id: "filter".into(), + operator: PhysicalOperator::Filter, + children: vec!["scan".into()], + output_buffer_bytes: 4, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }, + ]; + let provided = HashMap::from([ + ( + "scan".to_string(), + statistics( + vec![EdgeStatistics { + rows: 100, + bytes: 1_000, + }], + EdgeStatistics { + rows: 100, + bytes: 1_000, + }, + ), + ), + ( + "filter".to_string(), + statistics( + vec![EdgeStatistics { + rows: 100, + bytes: 1_000, + }], + EdgeStatistics { + rows: 40, + bytes: 400, + }, + ), + ), + ]); + + let estimate = + estimate_physical_dag(&nodes, "filter", &comparison_scope(), &provided).unwrap(); + assert_eq!(estimate.cpu_ops, 1_200.0); + assert_eq!(estimate.scan_bytes, 6_000); + } + + #[test] + fn physical_dag_accepts_an_empty_operator_output() { + let nodes = vec![ + PhysicalDagNode { + id: "scan".into(), + operator: PhysicalOperator::Scan, + children: vec![], + output_buffer_bytes: 10, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }, + PhysicalDagNode { + id: "filter".into(), + operator: PhysicalOperator::Filter, + children: vec!["scan".into()], + output_buffer_bytes: 0, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }, + ]; + let input = EdgeStatistics { + rows: 100, + bytes: 1_000, + }; + let provided = HashMap::from([ + ("scan".into(), statistics(vec![input], input)), + ( + "filter".into(), + statistics(vec![input], EdgeStatistics { rows: 0, bytes: 0 }), + ), + ]); + + let estimate = + estimate_physical_dag(&nodes, "filter", &comparison_scope(), &provided).unwrap(); + assert_eq!(estimate.cpu_ops, 1_200.0); + assert_eq!(estimate.peak_memory_bytes, 10); + assert_eq!(estimate.scan_bytes, 6_000); + } } diff --git a/crates/asap-aware-mapping/src/analytical_statistics.rs b/crates/asap-aware-mapping/src/analytical_statistics.rs new file mode 100644 index 00000000..9d023e42 --- /dev/null +++ b/crates/asap-aware-mapping/src/analytical_statistics.rs @@ -0,0 +1,213 @@ +//! Authoritative comparison-scope and physical-statistics contracts. +//! +//! This module does not lower logical query nodes. It defines the evidence a +//! lowering or catalog provider must supply before two physical DAGs can be +//! compared by the analytical resource model. + +use std::collections::HashMap; + +use asap_types::pre_asap::query_expr::{Predicate, Source}; +use asap_types::workload::{ + DataArrival, DataWorkload, DurationMs, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, + TimeSelection, TimestampMs, +}; +use serde::{Deserialize, Serialize}; + +use crate::analytical_cost::{AnalyticalCostError, HashJoinBuildSide}; + +/// The semantic and workload boundary within which two resource estimates +/// may be compared. Canonical workload and query-IR types remain authoritative; +/// only the storage snapshot identifier is new because neither IR names a +/// concrete catalog/storage version. +#[derive(Debug, Clone, PartialEq)] +pub struct ComparisonScope { + pub data_arrival: DataArrival, + pub planning_time: TimestampMs, + pub horizon: DurationMs, + pub recurrence: QueryRecurrence, + pub time_selection: TimeSelection, + pub sources: Vec, +} + +/// Exact source selection covered by a physical plan. +#[derive(Debug, Clone, PartialEq)] +pub struct SourceCoverage { + pub source: Source, + /// Catalog version, object generation, snapshot timestamp, or another + /// provider-owned stable identifier for the physical source contents. + pub snapshot_id: String, + /// Canonical predicates copied from the bound/canonicalized query IR. + pub predicates: Vec, +} + +impl ComparisonScope { + /// Build a comparison boundary from canonical workload fields plus the + /// physical snapshot identities supplied by the storage/catalog layer. + pub fn from_workload( + data: &DataWorkload, + query: &QueryWorkloadEntry, + planning_time: TimestampMs, + horizon: DurationMs, + sources: Vec, + ) -> Result { + let scope = Self { + data_arrival: data.arrival, + planning_time, + horizon, + recurrence: query.recurrence.clone(), + time_selection: query.time_selection.clone(), + sources, + }; + scope.validate()?; + Ok(scope) + } + + /// Validate this scope and return its effective query evaluation count. + pub fn validate(&self) -> Result { + if self.data_arrival != DataArrival::AtRest { + return Err(AnalyticalCostError::UnsupportedDataArrival( + self.data_arrival, + )); + } + if self.horizon.0 == 0 { + return Err(AnalyticalCostError::MissingOrZero("horizon")); + } + if self.sources.is_empty() { + return Err(AnalyticalCostError::MissingComparisonScope("sources")); + } + if self + .sources + .iter() + .any(|source| source.snapshot_id.is_empty()) + { + return Err(AnalyticalCostError::MissingComparisonScope("snapshot_id")); + } + evaluations_in_horizon(&self.recurrence, self.planning_time.0, self.horizon.0) + } +} + +/// Require exact scope equality before comparing raw and post-ASAP costs. +/// Exact matching is intentionally conservative: coverage/subsumption needs +/// a separate semantic proof and is not inferred by the resource estimator. +pub fn validate_comparison_scopes( + raw: &ComparisonScope, + candidate: &ComparisonScope, +) -> Result { + let evaluations = raw.validate()?; + candidate.validate()?; + for (name, matches) in [ + ("data_arrival", raw.data_arrival == candidate.data_arrival), + ( + "planning_time", + raw.planning_time == candidate.planning_time, + ), + ("horizon", raw.horizon == candidate.horizon), + ("recurrence", raw.recurrence == candidate.recurrence), + ( + "time_selection", + raw.time_selection == candidate.time_selection, + ), + ("sources", raw.sources == candidate.sources), + ] { + if !matches { + return Err(AnalyticalCostError::ComparisonScopeMismatch(name)); + } + } + Ok(evaluations) +} + +pub(crate) fn evaluations_in_horizon( + recurrence: &QueryRecurrence, + planning_time_ms: u64, + horizon_ms: u64, +) -> Result { + if horizon_ms == 0 { + return Err(AnalyticalCostError::MissingOrZero("horizon_ms")); + } + let end = planning_time_ms.saturating_add(horizon_ms); + let count = match recurrence { + QueryRecurrence::OneTime { + invocations, + execute_at, + } => { + if execute_at.is_none_or(|at| at.0 >= planning_time_ms && at.0 <= end) { + *invocations + } else { + 0 + } + } + QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(interval)) => { + if interval.0 == 0 { + return Err(AnalyticalCostError::InvalidRecurrence); + } + horizon_ms / u64::from(interval.0) + } + QueryRecurrence::Repeated(RepeatedDemand::Scheduled(schedule)) => schedule + .iter() + .filter(|at| at.0 >= planning_time_ms && at.0 <= end) + .count() + as u64, + QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(estimate)) => { + if !estimate.is_fresh_at(planning_time_ms) + || !estimate.expected_rate.0.is_finite() + || estimate.expected_rate.0 < 0.0 + { + return Err(AnalyticalCostError::InvalidRecurrence); + } + let expected = estimate.expected_rate.0 * horizon_ms as f64 / 1000.0; + if expected > u64::MAX as f64 { + return Err(AnalyticalCostError::Overflow); + } + expected.ceil() as u64 + } + QueryRecurrence::Unknown => return Err(AnalyticalCostError::InvalidRecurrence), + }; + if count == 0 { + return Err(AnalyticalCostError::NoEvaluationsInHorizon); + } + Ok(count) +} + +/// Logical cardinality and byte width carried by one physical edge. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct EdgeStatistics { + pub rows: u64, + pub bytes: u64, +} + +impl EdgeStatistics { + /// Empty logical edges carry neither rows nor bytes. Non-empty edges need + /// bytes so row-width-dependent formulas do not invent a width. + pub(crate) fn is_consistent(self) -> bool { + matches!((self.rows, self.bytes), (0, 0) | (1.., 1..)) + } +} + +/// Authoritative cardinality and width facts for one physical operator. +/// `inputs` has one entry per child edge, except `Scan`, whose single entry +/// describes its external source edge. The output is compared with every +/// parent's corresponding input, so conflicting provider evidence fails. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct OperatorStatistics { + pub inputs: Vec, + pub output: EdgeStatistics, + pub group_count: Option, + pub key_bytes: Option, + pub aggregate_value_bytes: Option, + pub k: Option, + pub hash_join_build_side: Option, +} + +/// Resolves physical statistics and owns their catalog/observation freshness. +/// Returning an error makes the entire candidate unavailable. +pub trait OperatorStatisticsProvider { + fn statistics(&self, node_id: &str) -> Result; +} + +impl OperatorStatisticsProvider for HashMap { + fn statistics(&self, node_id: &str) -> Result { + self.get(node_id) + .cloned() + .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(node_id.into())) + } +} diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index e29984a8..479fc062 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -184,6 +184,7 @@ pub mod accuracy; pub mod accuracy_reconciliation; pub mod analytical_cost; +pub mod analytical_statistics; pub mod cost_model; pub mod explanation; pub mod grouping; diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 10d5a6f6..679f81ec 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -13,15 +13,10 @@ and guarantee composition run first; costing ranks only the candidates that survive. Missing evidence produces an unavailable estimate, never an assumed zero or a structural-cost fallback. -This document distinguishes two implementation layers: - -- the physical-DAG estimator, which can compose any DAG whose nodes have - supported physical operators and complete `OperatorInputs`; and -- the planner bridge, which lowers a deliberately small set of replacement - shapes into that estimator and participates in automatic candidate ranking. - -Support in the first layer does not imply that the planner can yet lower and -rank the same shape. The current bridge boundary is stated explicitly below. +The physical-DAG estimator composes any DAG whose nodes have supported +physical operators and complete `OperatorStatistics`. Query lowering and +planner selection are separate integration layers; neither may substitute a +shape-specific shortcut or structural node count. An estimate has physical dimensions: @@ -69,11 +64,11 @@ The missing facts have explicit ownership: | Join-side and join-output cardinality | Join-key/operator statistics. | | Memory budget, spill I/O, cache behavior, and network bytes | Deployment/execution-profile evidence. | -`OperatorInputs` contains the resolved statistics for one physical operator. -It is an estimator input, not another planner workload object. The component -that lowers a query into a physical DAG owns resolving these fields from the -canonical workload, catalog, and operator-statistics sources. It must leave a -plan unavailable when required evidence cannot be resolved. +`OperatorStatistics` contains the resolved statistics for one physical +operator. It is estimator evidence, not another planner workload object. The +lowering provider owns resolving it from the canonical workload, catalog, and +operator-statistics sources. Missing required evidence makes the entire plan +unavailable. ## Workload horizon and lifecycle @@ -91,8 +86,8 @@ It is not an independent workload axis. A repeated rate without a horizon cannot produce a finite total cost. An at-rest estimate must not be reused for `unknown`, `mixed`, or -`continuously_ingesting` data. Callers must fail closed instead of pretending -that incremental updates are a one-time snapshot build. +`continuously_ingesting` data. Callers fail closed instead of pretending that +incremental updates are a one-time snapshot build. The sketch alternative scans the selected source snapshot once and retains state. The raw alternative recomputes from that snapshot for every query @@ -101,24 +96,66 @@ retention duration belong to the summary-maintenance lifecycle model. They must contribute update/build/delete work before a continuously maintained plan is compared with raw execution. +### Comparable source and workload scope + +A lower numerical cost is meaningful only when the two plans answer the same +request over the same physical data. `ComparisonScope` therefore reuses the +canonical workload and query terms rather than defining parallel strings: + +| Scope field | Authoritative type and meaning | +|---|---| +| Arrival mode | `DataArrival`; this model accepts only `AtRest`. | +| Planning instant and finite horizon | `TimestampMs` and `DurationMs`. | +| Invocation schedule | `QueryRecurrence`; the evaluation count is derived, not copied. | +| Event-time coverage | `TimeSelection`. | +| Logical sources | one `Source` per scan. | +| Filters | canonical bound `Predicate` values copied from the query IR. | +| Physical source contents | provider-owned `snapshot_id` per source. | + +The snapshot identifier is the only new scope concept. It is necessary because +`Source` names a metric or table but neither the query IR nor workload schema +identifies a catalog version, object generation, or storage snapshot. Reusing a +query timestamp would be incorrect: query event time and storage version are +independent facts. + +`ComparisonScope::from_workload` copies arrival, recurrence, and time selection +from `DataWorkload` and `QueryWorkloadEntry`; the catalog/lowering boundary adds +the source, snapshot identifier, and canonical predicates. Raw and candidate +scopes must match exactly in every field before their estimates are compared. +Unknown subsumption such as "this wider retained summary covers the requested +interval" is not guessed here; it requires a separate semantic coverage proof. +Missing sources, empty snapshot identifiers, invalid recurrence, or a zero +horizon fail closed. + ## General DAG costing Costing operates on the physical DAG, not on a list of logical operators. -Each costed node produces: +An `OperatorStatisticsProvider` resolves one `OperatorStatistics` value for +each reachable physical node: ```text -NodeEstimate { - output_rows, - output_bytes, - cpu_ops, - retained_bytes, - working_bytes, - source_read_bytes, +OperatorStatistics { + inputs: [EdgeStatistics { rows, bytes }, ...], + output: EdgeStatistics { rows, bytes }, + group_count, + key_bytes, + aggregate_value_bytes, + k, + hash_join_build_side, } ``` -The node's output statistics feed its parents. A parent never substitutes -the original source cardinality for an intermediate edge. +The provider owns provenance, freshness, and derivation. The estimator resolves +each reachable node once, so one estimate cannot mix values across a live +catalog refresh. A scan has one external source input; every other input is in +the same order as `PhysicalDagNode.children`. Every parent input must equal the +corresponding child's output in both rows and bytes. Missing node evidence, +invalid arity, inconsistent edge dimensions, or a parent/child conflict makes +the entire DAG unavailable. `{ rows: 0, bytes: 0 }` is a valid empty logical +edge, including after a filter, join, limit, or aggregate; non-empty edges need +positive logical bytes so width-dependent formulas do not invent a row width. +A parent therefore cannot silently substitute the original source cardinality +for an intermediate edge. ### Composition rules @@ -136,14 +173,15 @@ For a selected DAG: 7. Retained summaries remain live across reads. Streaming buffers may be released after their last consumer. -`estimate_physical_dag` implements these rules for `PhysicalDagNode` values. +`estimate_physical_dag` implements these rules for `PhysicalDagNode` values, +one `ComparisonScope`, and an `OperatorStatisticsProvider`. Node IDs are physical identities: duplicate IDs, missing children, and cycles are rejected. A child-before-parent schedule maintains remaining-consumer counts, releases transient output after its last consumer, and keeps retained state live. Consequently a shared scan is charged once per execution and a fan-out's memory includes the outputs that really coexist. -Logical `output_bytes` feeds parent cardinality estimates; it is not an +Logical edge `bytes` feeds parent cardinality estimates; it is not an allocation. Each physical node separately supplies `output_buffer_bytes` for its live batch/edge buffer and `retained_bytes` for state that survives the operator. A streaming scan therefore retains a batch, not the complete source. @@ -231,12 +269,10 @@ operators and resolved statistics. Until an operation has such a formula, a complete candidate containing it is unavailable; costing only its modeled children would undercount the plan. -`estimate_physical_dag` is deliberately independent of query shape. A caller -supplies the complete physical DAG and one `OperatorInputs` record per node. -Filters, projections, joins, windows, nested aggregates, Top-K, and shared -sub-DAGs therefore use the same estimation path. Query lowering and planner -selection are separate integration responsibilities and must not introduce a -shape-specific shortcut or structural-node-count fallback. +`estimate_physical_dag` is independent of query shape. A caller supplies the +complete physical DAG and per-node evidence. Filters, projections, joins, +windows, nested aggregates, Top-K, and shared sub-DAGs therefore use the same +estimation path. DDSketch is unavailable because occupied bins depend on value range and distribution. The model does not invent a bin count. Algorithm/parameter @@ -289,20 +325,21 @@ The intended end-to-end selection pipeline is: This module implements the physical estimation step. Lowering, evidence resolution, legality checks, and planner integration remain separate layers; -each must preserve the complete-plan and fail-closed requirements above. +each preserves the complete-plan and fail-closed requirements above. -The raw baseline and selected alternative use the same source snapshot, -horizon, and calibration: +Before applying the following arithmetic, callers validate exact equality of +the raw and selected alternative's `ComparisonScope`, and use the same +calibration: ```text benefit = baseline_cost - selected_cost benefit_ratio = benefit / baseline_cost ``` -Exported annotations contain resource totals, workload horizon, resolved -operator inputs, calibration coefficients, evidence/model versions, and the -baseline reference. This makes the scalar reproducible and identifies which -resource dimension drove a decision. +An export of a cost decision includes resource totals, calibration +coefficients, evidence/model versions, the baseline reference, validated +comparison scope, and per-node statistics provenance. These facts reproduce +why two estimates were considered comparable. ## Worked patterns From 943f2ec16524dfdf574d7df52d99868a2e93bc38 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 08:54:05 -0600 Subject: [PATCH 02/48] fix(cost): bind estimates to physical source scope --- .../asap-aware-mapping/src/analytical_cost.rs | 437 ++++++++++++++++-- .../src/analytical_statistics.rs | 6 +- .../analytical-resource-cost.md | 32 +- 3 files changed, 435 insertions(+), 40 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 639497df..a83e65cd 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -12,7 +12,8 @@ use asap_types::workload::DataArrival; use serde::{Deserialize, Serialize}; use crate::analytical_statistics::{ - ComparisonScope, OperatorStatistics, OperatorStatisticsProvider, + validate_comparison_scopes, ComparisonScope, OperatorStatistics, OperatorStatisticsProvider, + SourceCoverage, }; pub const ANALYTICAL_MODEL_VERSION: &str = "analytical-resource-at-rest-v1"; @@ -88,6 +89,10 @@ pub struct PhysicalDagNode { pub id: String, pub operator: PhysicalOperator, pub children: Vec, + /// Exact comparison-scope coverage consumed by a scan. Non-scan nodes + /// leave this empty. Reusing `SourceCoverage` prevents a physical plan + /// from naming a source independently of its snapshot and predicates. + pub source_coverage: Option, /// Maximum transient edge buffer, distinct from logical `output_bytes`. pub output_buffer_bytes: u64, /// State that remains live after this node finishes (zero for ordinary @@ -102,6 +107,40 @@ pub enum ExecutionMultiplicity { PerEvaluation, } +/// Borrowed inputs for one physical-DAG estimate. This remains available +/// independently for diagnostics; plan selection should use +/// [`estimate_physical_dag_comparison`] so scope equality is mandatory. +pub struct PhysicalDagEstimateRequest<'a> { + pub nodes: &'a [PhysicalDagNode], + pub root: &'a str, + pub scope: &'a ComparisonScope, + pub statistics: &'a dyn OperatorStatisticsProvider, +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct PhysicalDagComparisonEstimate { + pub raw: ResourceEstimate, + pub candidate: ResourceEstimate, +} + +/// Estimate two plans only after proving that their source, snapshot, +/// predicate, event-time, recurrence, and horizon scopes are identical. +pub fn estimate_physical_dag_comparison( + raw: PhysicalDagEstimateRequest<'_>, + candidate: PhysicalDagEstimateRequest<'_>, +) -> Result { + validate_comparison_scopes(raw.scope, candidate.scope)?; + Ok(PhysicalDagComparisonEstimate { + raw: estimate_physical_dag(raw.nodes, raw.root, raw.scope, raw.statistics)?, + candidate: estimate_physical_dag( + candidate.nodes, + candidate.root, + candidate.scope, + candidate.statistics, + )?, + }) +} + /// Compose local operator estimates once per physical identity. CPU and disk /// are additive; peak memory is simulated over a child-before-parent schedule /// and releases transient child outputs after their last consumer. @@ -109,7 +148,7 @@ pub fn estimate_physical_dag( nodes: &[PhysicalDagNode], root: &str, scope: &ComparisonScope, - statistics: &impl OperatorStatisticsProvider, + statistics: &(impl OperatorStatisticsProvider + ?Sized), ) -> Result { let evaluation_count = scope.validate()?; let by_id: HashMap<&str, &PhysicalDagNode> = nodes.iter().map(|n| (n.id.as_str(), n)).collect(); @@ -155,6 +194,24 @@ pub fn estimate_physical_dag( for id in &order { let node = by_id[id]; let node_statistics = &resolved_statistics[id]; + match node.operator { + PhysicalOperator::Scan => { + let coverage = node.source_coverage.as_ref().ok_or_else(|| { + AnalyticalCostError::MissingScanSourceCoverage(node.id.clone()) + })?; + if !scope.sources.contains(coverage) { + return Err(AnalyticalCostError::ScanOutsideComparisonScope( + node.id.clone(), + )); + } + } + _ if node.source_coverage.is_some() => { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "only scan nodes may declare source coverage", + )); + } + _ => {} + } validate_operator_statistics(node, node_statistics, &by_id, &resolved_statistics)?; if node.retained_bytes > 0 && matches!(node.execution, ExecutionMultiplicity::PerEvaluation) { @@ -164,6 +221,13 @@ pub fn estimate_physical_dag( } for child in &node.children { let child = by_id[child.as_str()]; + if matches!(node.execution, ExecutionMultiplicity::Once) + && matches!(child.execution, ExecutionMultiplicity::PerEvaluation) + { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "build-once node cannot consume a per-evaluation child", + )); + } if matches!(node.execution, ExecutionMultiplicity::PerEvaluation) && matches!(child.execution, ExecutionMultiplicity::Once) && child.retained_bytes == 0 @@ -273,6 +337,16 @@ fn validate_operator_statistics( "operator child count does not match physical arity", )); } + match node.operator { + PhysicalOperator::Scan => {} + _ if node_statistics.source_scan_bytes != 0 => { + return Err(AnalyticalCostError::InvalidOperatorStatistics { + node: node.id.clone(), + reason: "only scan operators may read source bytes", + }); + } + _ => {} + } for edge in node_statistics .inputs .iter() @@ -332,7 +406,7 @@ pub fn estimate_operator( PhysicalOperator::Scan => ResourceEstimate { cpu_ops: left.rows as f64, peak_memory_bytes: per_row_width(left.rows, left.bytes)?, - scan_bytes: left.bytes, + scan_bytes: statistics.source_scan_bytes, }, PhysicalOperator::Filter | PhysicalOperator::Project | PhysicalOperator::PassThrough => { ResourceEstimate { @@ -530,6 +604,10 @@ pub enum AnalyticalCostError { UnsupportedSummaryOperation(&'static str), #[error("required comparison-scope field {0} is missing")] MissingComparisonScope(&'static str), + #[error("scan node {0} does not declare source coverage")] + MissingScanSourceCoverage(String), + #[error("scan node {0} reads source coverage outside the comparison scope")] + ScanOutsideComparisonScope(String), #[error("raw and candidate comparison scopes differ in {0}")] ComparisonScopeMismatch(&'static str), #[error("operator statistics are unavailable for physical node {0}")] @@ -614,16 +692,19 @@ mod tests { fn physical_operator_formulas_keep_disk_at_scan_and_require_join_stats() { let scan = estimate_operator( PhysicalOperator::Scan, - statistics( - vec![EdgeStatistics { - rows: 1_000, - bytes: 64_000, - }], - EdgeStatistics { - rows: 1_000, - bytes: 64_000, - }, - ), + OperatorStatistics { + source_scan_bytes: 64_000, + ..statistics( + vec![EdgeStatistics { + rows: 1_000, + bytes: 64_000, + }], + EdgeStatistics { + rows: 1_000, + bytes: 64_000, + }, + ) + }, ) .unwrap(); assert_eq!(scan.scan_bytes, 64_000); @@ -780,11 +861,13 @@ mod tests { #[test] fn physical_dag_counts_shared_scan_once_and_uses_live_memory() { + let coverage = comparison_scope().sources[0].clone(); let nodes = vec![ PhysicalDagNode { id: "scan".into(), operator: PhysicalOperator::Scan, children: vec![], + source_coverage: Some(coverage), output_buffer_bytes: 10, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -793,6 +876,7 @@ mod tests { id: "left".into(), operator: PhysicalOperator::Filter, children: vec!["scan".into()], + source_coverage: None, output_buffer_bytes: 4, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -801,6 +885,7 @@ mod tests { id: "right".into(), operator: PhysicalOperator::Filter, children: vec!["scan".into()], + source_coverage: None, output_buffer_bytes: 4, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -809,6 +894,7 @@ mod tests { id: "root".into(), operator: PhysicalOperator::Concat, children: vec!["left".into(), "right".into()], + source_coverage: None, output_buffer_bytes: 8, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -823,7 +909,13 @@ mod tests { bytes: 400, }; let provided = HashMap::from([ - ("scan".into(), statistics(vec![scan_edge], scan_edge)), + ( + "scan".into(), + OperatorStatistics { + source_scan_bytes: 1_000, + ..statistics(vec![scan_edge], scan_edge) + }, + ), ("left".into(), statistics(vec![scan_edge], branch_edge)), ("right".into(), statistics(vec![scan_edge], branch_edge)), ( @@ -849,11 +941,13 @@ mod tests { #[test] fn physical_dag_separates_build_once_from_per_evaluation_work() { + let coverage = comparison_scope().sources[0].clone(); let nodes = vec![ PhysicalDagNode { id: "scan".into(), operator: PhysicalOperator::Scan, children: vec![], + source_coverage: Some(coverage), output_buffer_bytes: 10, retained_bytes: 0, execution: ExecutionMultiplicity::Once, @@ -862,6 +956,7 @@ mod tests { id: "state".into(), operator: PhysicalOperator::HashAggregate, children: vec!["scan".into()], + source_coverage: None, output_buffer_bytes: 16, retained_bytes: 32, execution: ExecutionMultiplicity::Once, @@ -870,6 +965,7 @@ mod tests { id: "read".into(), operator: PhysicalOperator::Limit, children: vec!["state".into()], + source_coverage: None, output_buffer_bytes: 16, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -881,7 +977,13 @@ mod tests { }; let state_edge = EdgeStatistics { rows: 1, bytes: 16 }; let provided = HashMap::from([ - ("scan".into(), statistics(vec![scan_edge], scan_edge)), + ( + "scan".into(), + OperatorStatistics { + source_scan_bytes: 1_000, + ..statistics(vec![scan_edge], scan_edge) + }, + ), ( "state".into(), OperatorStatistics { @@ -931,6 +1033,7 @@ mod tests { fn statistics(inputs: Vec, output: EdgeStatistics) -> OperatorStatistics { OperatorStatistics { + source_scan_bytes: 0, inputs, output, group_count: None, @@ -988,11 +1091,13 @@ mod tests { fn physical_dag_fails_closed_on_missing_or_conflicting_edge_statistics() { use std::collections::HashMap; + let coverage = comparison_scope().sources[0].clone(); let nodes = vec![ PhysicalDagNode { id: "scan".into(), operator: PhysicalOperator::Scan, children: vec![], + source_coverage: Some(coverage), output_buffer_bytes: 10, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -1001,6 +1106,7 @@ mod tests { id: "filter".into(), operator: PhysicalOperator::Filter, children: vec!["scan".into()], + source_coverage: None, output_buffer_bytes: 4, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -1009,16 +1115,19 @@ mod tests { let scope = comparison_scope(); let mut provided = HashMap::from([( "scan".to_string(), - statistics( - vec![EdgeStatistics { - rows: 100, - bytes: 1_000, - }], - EdgeStatistics { - rows: 100, - bytes: 1_000, - }, - ), + OperatorStatistics { + source_scan_bytes: 1_000, + ..statistics( + vec![EdgeStatistics { + rows: 100, + bytes: 1_000, + }], + EdgeStatistics { + rows: 100, + bytes: 1_000, + }, + ) + }, )]); assert_eq!( @@ -1055,11 +1164,13 @@ mod tests { fn provider_statistics_drive_a_consistent_physical_dag_estimate() { use std::collections::HashMap; + let coverage = comparison_scope().sources[0].clone(); let nodes = vec![ PhysicalDagNode { id: "scan".into(), operator: PhysicalOperator::Scan, children: vec![], + source_coverage: Some(coverage), output_buffer_bytes: 10, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -1068,6 +1179,7 @@ mod tests { id: "filter".into(), operator: PhysicalOperator::Filter, children: vec!["scan".into()], + source_coverage: None, output_buffer_bytes: 4, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -1076,16 +1188,19 @@ mod tests { let provided = HashMap::from([ ( "scan".to_string(), - statistics( - vec![EdgeStatistics { - rows: 100, - bytes: 1_000, - }], - EdgeStatistics { - rows: 100, - bytes: 1_000, - }, - ), + OperatorStatistics { + source_scan_bytes: 1_000, + ..statistics( + vec![EdgeStatistics { + rows: 100, + bytes: 1_000, + }], + EdgeStatistics { + rows: 100, + bytes: 1_000, + }, + ) + }, ), ( "filter".to_string(), @@ -1110,11 +1225,13 @@ mod tests { #[test] fn physical_dag_accepts_an_empty_operator_output() { + let coverage = comparison_scope().sources[0].clone(); let nodes = vec![ PhysicalDagNode { id: "scan".into(), operator: PhysicalOperator::Scan, children: vec![], + source_coverage: Some(coverage), output_buffer_bytes: 10, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -1123,6 +1240,7 @@ mod tests { id: "filter".into(), operator: PhysicalOperator::Filter, children: vec!["scan".into()], + source_coverage: None, output_buffer_bytes: 0, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -1133,7 +1251,13 @@ mod tests { bytes: 1_000, }; let provided = HashMap::from([ - ("scan".into(), statistics(vec![input], input)), + ( + "scan".into(), + OperatorStatistics { + source_scan_bytes: 1_000, + ..statistics(vec![input], input) + }, + ), ( "filter".into(), statistics(vec![input], EdgeStatistics { rows: 0, bytes: 0 }), @@ -1146,4 +1270,245 @@ mod tests { assert_eq!(estimate.peak_memory_bytes, 10); assert_eq!(estimate.scan_bytes, 6_000); } + + #[test] + fn physical_dag_rejects_a_scan_not_covered_by_its_scope() { + let nodes = vec![PhysicalDagNode { + id: "scan".into(), + operator: PhysicalOperator::Scan, + children: vec![], + source_coverage: Some(SourceCoverage { + source: asap_types::pre_asap::query_expr::Source::Table { + table_ref: "other_metrics".into(), + }, + snapshot_id: "catalog-version-42".into(), + predicates: vec![], + }), + output_buffer_bytes: 10, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }]; + let edge = EdgeStatistics { + rows: 100, + bytes: 1_000, + }; + let provided = HashMap::from([( + "scan".into(), + OperatorStatistics { + source_scan_bytes: 250, + ..statistics(vec![edge], edge) + }, + )]); + + assert_eq!( + estimate_physical_dag(&nodes, "scan", &comparison_scope(), &provided), + Err(AnalyticalCostError::ScanOutsideComparisonScope( + "scan".into() + )) + ); + } + + #[test] + fn physical_dag_rejects_a_scan_without_explicit_coverage() { + let nodes = vec![PhysicalDagNode { + id: "scan".into(), + operator: PhysicalOperator::Scan, + children: vec![], + source_coverage: None, + output_buffer_bytes: 10, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }]; + let edge = EdgeStatistics { + rows: 100, + bytes: 1_000, + }; + let provided = HashMap::from([( + "scan".into(), + OperatorStatistics { + source_scan_bytes: 250, + ..statistics(vec![edge], edge) + }, + )]); + + assert_eq!( + estimate_physical_dag(&nodes, "scan", &comparison_scope(), &provided), + Err(AnalyticalCostError::MissingScanSourceCoverage( + "scan".into() + )) + ); + } + + #[test] + fn build_once_parent_cannot_consume_a_per_evaluation_child() { + let coverage = comparison_scope().sources[0].clone(); + let nodes = vec![ + PhysicalDagNode { + id: "scan".into(), + operator: PhysicalOperator::Scan, + children: vec![], + source_coverage: Some(coverage), + output_buffer_bytes: 10, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }, + PhysicalDagNode { + id: "aggregate".into(), + operator: PhysicalOperator::HashAggregate, + children: vec!["scan".into()], + source_coverage: None, + output_buffer_bytes: 16, + retained_bytes: 32, + execution: ExecutionMultiplicity::Once, + }, + ]; + let input = EdgeStatistics { + rows: 100, + bytes: 1_000, + }; + let output = EdgeStatistics { rows: 1, bytes: 16 }; + let provided = HashMap::from([ + ( + "scan".into(), + OperatorStatistics { + source_scan_bytes: 250, + ..statistics(vec![input], input) + }, + ), + ( + "aggregate".into(), + OperatorStatistics { + group_count: Some(1), + key_bytes: Some(8), + aggregate_value_bytes: Some(8), + ..statistics(vec![input], output) + }, + ), + ]); + + assert_eq!( + estimate_physical_dag(&nodes, "aggregate", &comparison_scope(), &provided), + Err(AnalyticalCostError::InvalidPhysicalDag( + "build-once node cannot consume a per-evaluation child" + )) + ); + } + + #[test] + fn scoped_comparison_rejects_different_source_snapshots() { + let coverage = comparison_scope().sources[0].clone(); + let nodes = vec![PhysicalDagNode { + id: "scan".into(), + operator: PhysicalOperator::Scan, + children: vec![], + source_coverage: Some(coverage), + output_buffer_bytes: 10, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }]; + let edge = EdgeStatistics { + rows: 100, + bytes: 1_000, + }; + let provided = HashMap::from([( + "scan".into(), + OperatorStatistics { + source_scan_bytes: 250, + ..statistics(vec![edge], edge) + }, + )]); + let raw_scope = comparison_scope(); + let mut candidate_scope = raw_scope.clone(); + candidate_scope.sources[0].snapshot_id = "catalog-version-43".into(); + + assert_eq!( + estimate_physical_dag_comparison( + PhysicalDagEstimateRequest { + nodes: &nodes, + root: "scan", + scope: &raw_scope, + statistics: &provided, + }, + PhysicalDagEstimateRequest { + nodes: &nodes, + root: "scan", + scope: &candidate_scope, + statistics: &provided, + }, + ), + Err(AnalyticalCostError::ComparisonScopeMismatch("sources")) + ); + } + + #[test] + fn scan_uses_physical_source_bytes_not_decoded_logical_bytes() { + let logical = EdgeStatistics { + rows: 100, + bytes: 10_000, + }; + let estimate = estimate_operator( + PhysicalOperator::Scan, + OperatorStatistics { + source_scan_bytes: 2_500, + ..statistics(vec![logical], logical) + }, + ) + .unwrap(); + + assert_eq!(estimate.scan_bytes, 2_500); + assert_eq!(estimate.peak_memory_bytes, 100); + } + + #[test] + fn non_scan_operator_cannot_charge_source_bytes() { + let coverage = comparison_scope().sources[0].clone(); + let nodes = vec![ + PhysicalDagNode { + id: "scan".into(), + operator: PhysicalOperator::Scan, + children: vec![], + source_coverage: Some(coverage), + output_buffer_bytes: 10, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }, + PhysicalDagNode { + id: "filter".into(), + operator: PhysicalOperator::Filter, + children: vec!["scan".into()], + source_coverage: None, + output_buffer_bytes: 10, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }, + ]; + let edge = EdgeStatistics { + rows: 100, + bytes: 1_000, + }; + let provided = HashMap::from([ + ( + "scan".into(), + OperatorStatistics { + source_scan_bytes: 250, + ..statistics(vec![edge], edge) + }, + ), + ( + "filter".into(), + OperatorStatistics { + source_scan_bytes: 250, + ..statistics(vec![edge], edge) + }, + ), + ]); + + assert_eq!( + estimate_physical_dag(&nodes, "filter", &comparison_scope(), &provided), + Err(AnalyticalCostError::InvalidOperatorStatistics { + node: "filter".into(), + reason: "only scan operators may read source bytes", + }) + ); + } } diff --git a/crates/asap-aware-mapping/src/analytical_statistics.rs b/crates/asap-aware-mapping/src/analytical_statistics.rs index 9d023e42..468a913b 100644 --- a/crates/asap-aware-mapping/src/analytical_statistics.rs +++ b/crates/asap-aware-mapping/src/analytical_statistics.rs @@ -30,7 +30,7 @@ pub struct ComparisonScope { } /// Exact source selection covered by a physical plan. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct SourceCoverage { pub source: Source, /// Catalog version, object generation, snapshot timestamp, or another @@ -189,6 +189,10 @@ impl EdgeStatistics { /// parent's corresponding input, so conflicting provider evidence fails. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct OperatorStatistics { + /// Physical bytes read from storage by this operator. This is independent + /// of decoded logical bytes on `inputs`; it must be zero for non-scan + /// operators in the current in-memory physical model. + pub source_scan_bytes: u64, pub inputs: Vec, pub output: EdgeStatistics, pub group_count: Option, diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 679f81ec..7fa0267c 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -108,7 +108,7 @@ canonical workload and query terms rather than defining parallel strings: | Planning instant and finite horizon | `TimestampMs` and `DurationMs`. | | Invocation schedule | `QueryRecurrence`; the evaluation count is derived, not copied. | | Event-time coverage | `TimeSelection`. | -| Logical sources | one `Source` per scan. | +| Logical sources | the existing query-IR `Source`, one per scan. | | Filters | canonical bound `Predicate` values copied from the query IR. | | Physical source contents | provider-owned `snapshot_id` per source. | @@ -127,6 +127,13 @@ interval" is not guessed here; it requires a separate semantic coverage proof. Missing sources, empty snapshot identifiers, invalid recurrence, or a zero horizon fail closed. +Every reachable physical `Scan` carries one exact `SourceCoverage` copied from +this scope. That coverage includes the existing `Source`, its provider-owned +snapshot ID, and canonical predicates. A scan with no coverage, or coverage +not present in `ComparisonScope.sources`, makes the plan unavailable. Other +operators cannot declare source coverage. This prevents a DAG over source B +from being estimated under source A's comparison scope. + ## General DAG costing Costing operates on the physical DAG, not on a list of logical operators. @@ -135,6 +142,7 @@ each reachable physical node: ```text OperatorStatistics { + source_scan_bytes, inputs: [EdgeStatistics { rows, bytes }, ...], output: EdgeStatistics { rows, bytes }, group_count, @@ -157,6 +165,12 @@ positive logical bytes so width-dependent formulas do not invent a row width. A parent therefore cannot silently substitute the original source cardinality for an intermediate edge. +`EdgeStatistics.bytes` is decoded logical data carried on an edge. +`source_scan_bytes` is physical storage I/O and is charged only by `Scan`. +Compression, column pruning, or encoded storage can therefore make these +values different; neither is inferred from the other. Non-scan operators must +report zero source bytes in the current in-memory operator model. + ### Composition rules For a selected DAG: @@ -174,7 +188,10 @@ For a selected DAG: released after their last consumer. `estimate_physical_dag` implements these rules for `PhysicalDagNode` values, -one `ComparisonScope`, and an `OperatorStatisticsProvider`. +one `ComparisonScope`, and an `OperatorStatisticsProvider`. It is a +single-plan diagnostic API. Code that ranks a raw and candidate plan must use +`estimate_physical_dag_comparison`, which validates exact scope equality before +estimating either plan and returns both dimensional estimates together. Node IDs are physical identities: duplicate IDs, missing children, and cycles are rejected. A child-before-parent schedule maintains remaining-consumer counts, releases transient output after its last consumer, and keeps retained @@ -191,6 +208,15 @@ Build/maintenance nodes can therefore be charged once while query-side nodes are multiplied by the horizon's evaluation count; retention does not silently imply either execution frequency. +The parent/child compatibility rules are: + +| Parent | Child | Validity | +|---|---|---| +| `Once` | `Once` | valid | +| `Once` | `PerEvaluation` | invalid; a build-once result cannot depend on repeated executions | +| `PerEvaluation` | `PerEvaluation` | valid | +| `PerEvaluation` | `Once` | valid only when the child exposes retained state | + For a tree-shaped pipeline, peak memory is normally the maximum live pipeline state, not the sum of every node's memory. At a fan-out, join, merge, or nested summary boundary, multiple child states may coexist and must be combined. @@ -207,7 +233,7 @@ the DAG rules above. | Physical operator | CPU operations | Local memory | Source/disk reads | |---|---:|---:|---:| -| Scan | `input_rows` | one input row/batch | `input_bytes` | +| Scan | `input_rows` | one decoded input row/batch | `source_scan_bytes` | | Filter | `input_rows` | one output row/batch | `0` | | Project or scalar pass-through | `input_rows` | one output row/batch | `0` | | Hash aggregate | `input_rows` | `groups × (key + aggregate_value_bytes + hash metadata)` | `0` | From b30f2df9f82beaa2610e42855b7afa00302d319a Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 20:36:06 -0600 Subject: [PATCH 03/48] fix(cost): validate complete scoped operator work --- .../asap-aware-mapping/src/analytical_cost.rs | 146 +++++++++++++----- .../src/analytical_statistics.rs | 4 + 2 files changed, 115 insertions(+), 35 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index a83e65cd..f44b3a58 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -191,6 +191,7 @@ pub fn estimate_physical_dag( .map(|id| statistics.statistics(id).map(|value| (*id, value))) .collect::>()?; + let mut consumed_sources = Vec::new(); for id in &order { let node = by_id[id]; let node_statistics = &resolved_statistics[id]; @@ -204,6 +205,9 @@ pub fn estimate_physical_dag( node.id.clone(), )); } + if !consumed_sources.contains(&coverage) { + consumed_sources.push(coverage); + } } _ if node.source_coverage.is_some() => { return Err(AnalyticalCostError::InvalidPhysicalDag( @@ -238,6 +242,15 @@ pub fn estimate_physical_dag( } } } + if scope + .sources + .iter() + .any(|expected| !consumed_sources.contains(&expected)) + { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "physical scans omit a comparison-scope source", + )); + } let mut remaining_consumers: HashMap<&str, usize> = HashMap::new(); for id in &order { @@ -461,10 +474,11 @@ pub fn estimate_operator( let k = statistics .k .ok_or(AnalyticalCostError::MissingOrZero("k"))?; + let heap_rows = k.min(left.rows); ResourceEstimate { - cpu_ops: left.rows as f64 * (k.max(2) as f64).log2().ceil(), + cpu_ops: left.rows as f64 * (heap_rows.max(2) as f64).log2().ceil(), peak_memory_bytes: checked_bytes(&[ - k.min(left.rows), + heap_rows, per_row_width(left.rows, left.bytes)?, ])?, scan_bytes: 0, @@ -478,9 +492,16 @@ pub fn estimate_operator( ResourceEstimate { cpu_ops: left.rows as f64 + right.rows as f64 + output.rows as f64, peak_memory_bytes: match build_side { - HashJoinBuildSide::Left => left.bytes, - HashJoinBuildSide::Right => right.bytes, - }, + HashJoinBuildSide::Left => left + .rows + .checked_mul(16) + .and_then(|metadata| left.bytes.checked_add(metadata)), + HashJoinBuildSide::Right => right + .rows + .checked_mul(16) + .and_then(|metadata| right.bytes.checked_add(metadata)), + } + .ok_or(AnalyticalCostError::Overflow)?, scan_bytes: 0, } } @@ -489,11 +510,21 @@ pub fn estimate_operator( peak_memory_bytes: per_row_width(output.rows, output.bytes)?, scan_bytes: 0, }, - PhysicalOperator::Limit => ResourceEstimate { - cpu_ops: output.rows as f64, - peak_memory_bytes: per_row_width(output.rows, output.bytes)?, - scan_bytes: 0, - }, + PhysicalOperator::Limit => { + let consumed = statistics + .limit_rows_consumed + .ok_or(AnalyticalCostError::MissingOrZero("limit_rows_consumed"))?; + if consumed > left.rows || consumed < output.rows { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "Limit rows consumed must cover its output without exceeding its input", + )); + } + ResourceEstimate { + cpu_ops: consumed as f64, + peak_memory_bytes: per_row_width(output.rows, output.bytes)?, + scan_bytes: 0, + } + } }; if estimate.cpu_ops.is_finite() { Ok(estimate) @@ -820,19 +851,18 @@ mod tests { let oversized_topk = estimate_operator( PhysicalOperator::TopK, - OperatorInputs { - input_rows: 4, - input_bytes: 160, - output_rows: 4, - output_bytes: 160, - group_count: None, - key_bytes: None, - aggregate_value_bytes: None, + OperatorStatistics { k: Some(1_000), - limit_rows_consumed: None, - right_rows: None, - right_bytes: None, - hash_join_build_side: None, + ..statistics( + vec![EdgeStatistics { + rows: 4, + bytes: 160, + }], + EdgeStatistics { + rows: 4, + bytes: 160, + }, + ) }, ) .unwrap(); @@ -840,19 +870,18 @@ mod tests { let offset_limit = estimate_operator( PhysicalOperator::Limit, - OperatorInputs { - input_rows: 1_000_000, - input_bytes: 40_000_000, - output_rows: 10, - output_bytes: 400, - group_count: None, - key_bytes: None, - aggregate_value_bytes: None, - k: None, + OperatorStatistics { limit_rows_consumed: Some(900_010), - right_rows: None, - right_bytes: None, - hash_join_build_side: None, + ..statistics( + vec![EdgeStatistics { + rows: 1_000_000, + bytes: 40_000_000, + }], + EdgeStatistics { + rows: 10, + bytes: 400, + }, + ) }, ) .unwrap(); @@ -993,7 +1022,13 @@ mod tests { ..statistics(vec![scan_edge], state_edge) }, ), - ("read".into(), statistics(vec![state_edge], state_edge)), + ( + "read".into(), + OperatorStatistics { + limit_rows_consumed: Some(1), + ..statistics(vec![state_edge], state_edge) + }, + ), ]); let mut scope = comparison_scope(); scope.horizon.0 = 100_000; @@ -1040,6 +1075,7 @@ mod tests { key_bytes: None, aggregate_value_bytes: None, k: None, + limit_rows_consumed: None, hash_join_build_side: None, } } @@ -1339,6 +1375,46 @@ mod tests { ); } + #[test] + fn physical_dag_rejects_an_unconsumed_scope_source() { + let mut scope = comparison_scope(); + let coverage = scope.sources[0].clone(); + scope.sources.push(SourceCoverage { + source: asap_types::pre_asap::query_expr::Source::Table { + table_ref: "auxiliary".into(), + }, + snapshot_id: "catalog-version-42".into(), + predicates: vec![], + }); + let nodes = vec![PhysicalDagNode { + id: "scan".into(), + operator: PhysicalOperator::Scan, + children: vec![], + source_coverage: Some(coverage), + output_buffer_bytes: 10, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }]; + let edge = EdgeStatistics { + rows: 100, + bytes: 1_000, + }; + let provided = HashMap::from([( + "scan".into(), + OperatorStatistics { + source_scan_bytes: 250, + ..statistics(vec![edge], edge) + }, + )]); + + assert_eq!( + estimate_physical_dag(&nodes, "scan", &scope, &provided), + Err(AnalyticalCostError::InvalidPhysicalDag( + "physical scans omit a comparison-scope source" + )) + ); + } + #[test] fn build_once_parent_cannot_consume_a_per_evaluation_child() { let coverage = comparison_scope().sources[0].clone(); diff --git a/crates/asap-aware-mapping/src/analytical_statistics.rs b/crates/asap-aware-mapping/src/analytical_statistics.rs index 468a913b..80477303 100644 --- a/crates/asap-aware-mapping/src/analytical_statistics.rs +++ b/crates/asap-aware-mapping/src/analytical_statistics.rs @@ -199,6 +199,10 @@ pub struct OperatorStatistics { pub key_bytes: Option, pub aggregate_value_bytes: Option, pub k: Option, + /// Rows consumed by a physical Limit, including rows skipped by OFFSET. + /// This is an execution statistic rather than the output cardinality. + #[serde(default)] + pub limit_rows_consumed: Option, pub hash_join_build_side: Option, } From 0b03b6e73181a747dad66a08777c49b4f0c91709 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 10:05:02 -0600 Subject: [PATCH 04/48] refactor(cost): preserve operator invariants in scoped evidence --- .../asap-aware-mapping/src/analytical_cost.rs | 84 ++++++++++++------- 1 file changed, 52 insertions(+), 32 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index f44b3a58..69a83e83 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -12,8 +12,8 @@ use asap_types::workload::DataArrival; use serde::{Deserialize, Serialize}; use crate::analytical_statistics::{ - validate_comparison_scopes, ComparisonScope, OperatorStatistics, OperatorStatisticsProvider, - SourceCoverage, + validate_comparison_scopes, ComparisonScope, EdgeStatistics, OperatorStatistics, + OperatorStatisticsProvider, SourceCoverage, }; pub const ANALYTICAL_MODEL_VERSION: &str = "analytical-resource-at-rest-v1"; @@ -395,7 +395,7 @@ pub fn estimate_operator( operator: PhysicalOperator, statistics: OperatorStatistics, ) -> Result { - validate_operator_semantics(operator, &input)?; + validate_operator_semantics(operator, &statistics)?; let per_row_width = |rows: u64, bytes: u64| -> Result { match (rows, bytes) { (0, 0) => return Ok(0), @@ -535,53 +535,77 @@ pub fn estimate_operator( fn validate_operator_semantics( operator: PhysicalOperator, - input: &OperatorInputs, + statistics: &OperatorStatistics, ) -> Result<(), AnalyticalCostError> { let inconsistent = |reason| Err(AnalyticalCostError::InconsistentOperatorStatistics(reason)); + if !matches!(operator, PhysicalOperator::Scan) && statistics.source_scan_bytes != 0 { + return inconsistent("only Scan may charge source bytes"); + } + let input = statistics.inputs.first().copied().ok_or( + AnalyticalCostError::InconsistentOperatorStatistics("operator input is missing"), + )?; + let output = statistics.output; match operator { PhysicalOperator::Scan => { - if input.input_rows != input.output_rows || input.input_bytes != input.output_bytes { + if input != output { return inconsistent("Scan input and output edges differ"); } } PhysicalOperator::Filter => { - if input.output_rows > input.input_rows || input.output_bytes > input.input_bytes { + if output.rows > input.rows || output.bytes > input.bytes { return inconsistent("Filter output expands its input"); } } PhysicalOperator::Project => { - if input.output_rows != input.input_rows { + if output.rows != input.rows { return inconsistent("Project changes row cardinality"); } } PhysicalOperator::HashAggregate | PhysicalOperator::Deduplicate => { - let groups = input + let groups = statistics .group_count .filter(|groups| *groups > 0) .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - if groups > input.input_rows || input.output_rows != groups { + if groups > input.rows || output.rows != groups { return inconsistent("grouped output differs from distinct group cardinality"); } } - PhysicalOperator::Sort - | PhysicalOperator::Window - | PhysicalOperator::PassThrough - | PhysicalOperator::Concat => { - if input.input_rows != input.output_rows || input.input_bytes != input.output_bytes { + PhysicalOperator::Sort | PhysicalOperator::Window | PhysicalOperator::PassThrough => { + if input != output { return inconsistent("cardinality-preserving operator changes its edge"); } } + PhysicalOperator::Concat => { + let total = statistics.inputs.iter().try_fold( + EdgeStatistics { rows: 0, bytes: 0 }, + |total, edge| { + Ok::<_, AnalyticalCostError>(EdgeStatistics { + rows: total + .rows + .checked_add(edge.rows) + .ok_or(AnalyticalCostError::Overflow)?, + bytes: total + .bytes + .checked_add(edge.bytes) + .ok_or(AnalyticalCostError::Overflow)?, + }) + }, + )?; + if output != total { + return inconsistent("Concat output differs from the sum of its inputs"); + } + } PhysicalOperator::TopK => { - let k = input + let k = statistics .k .filter(|k| *k > 0) .ok_or(AnalyticalCostError::MissingOrZero("k"))?; - if input.output_rows != input.input_rows.min(k) { + if output.rows != input.rows.min(k) { return inconsistent("Top-K output differs from its cardinality bound"); } } PhysicalOperator::Limit => { - if input.output_rows > input.input_rows { + if output.rows > input.rows { return inconsistent("Limit output exceeds its input"); } } @@ -674,21 +698,17 @@ mod tests { SourceCoverage, }; - fn unary_inputs(input_rows: u64, output_rows: u64) -> OperatorInputs { - OperatorInputs { - input_rows, - input_bytes: input_rows.saturating_mul(8), - output_rows, - output_bytes: output_rows.saturating_mul(8), - group_count: None, - key_bytes: None, - aggregate_value_bytes: None, - k: None, - limit_rows_consumed: None, - right_rows: None, - right_bytes: None, - hash_join_build_side: None, - } + fn unary_inputs(input_rows: u64, output_rows: u64) -> OperatorStatistics { + statistics( + vec![EdgeStatistics { + rows: input_rows, + bytes: input_rows.saturating_mul(8), + }], + EdgeStatistics { + rows: output_rows, + bytes: output_rows.saturating_mul(8), + }, + ) } #[test] From bcea03bdb81ccec5b68fb673f05bb2825c89d497 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 08:19:38 -0600 Subject: [PATCH 05/48] feat(cost): lower query DAGs for analytical estimates --- .../asap-aware-mapping/src/analytical_cost.rs | 4 + .../src/analytical_lowering.rs | 763 ++++++++++++++++++ crates/asap-aware-mapping/src/lib.rs | 1 + .../analytical-resource-cost.md | 76 +- 4 files changed, 836 insertions(+), 8 deletions(-) create mode 100644 crates/asap-aware-mapping/src/analytical_lowering.rs diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 69a83e83..d588a688 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -655,6 +655,10 @@ pub enum AnalyticalCostError { Overflow, #[error("candidate has no supported exact or sketch state")] UnsupportedCandidate, + #[error("query operator has no physical implementation in the analytical model")] + UnsupportedQueryOperator, + #[error("inconsistent operator statistics: {0}")] + InconsistentOperatorStatistics(&'static str), #[error("summary operation {0} has no lifecycle-aware cost formula")] UnsupportedSummaryOperation(&'static str), #[error("required comparison-scope field {0} is missing")] diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs new file mode 100644 index 00000000..01f0a68f --- /dev/null +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -0,0 +1,763 @@ +//! Recursive lowering from the canonical query IR to analytical physical DAGs. + +use std::rc::Rc; + +use serde::{Deserialize, Serialize}; + +use crate::analytical_cost::{ + AnalyticalCostError, ExecutionMultiplicity, OperatorInputs, PhysicalDagNode, PhysicalOperator, +}; + +/// A lowered physical DAG and the node whose output is the query result. +/// Keeping the root beside its nodes prevents callers from accidentally +/// estimating a valid node list from the wrong entry point. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PhysicalDag { + pub nodes: Vec, + pub root: String, +} + +/// Lower a resolved query operator DAG to the physical operators understood by +/// this cost model. The callback supplies physical statistics for each logical +/// operator identity; returning `None` makes the complete query unavailable. +/// Scalar expressions remain part of their containing operator's local cost. +pub fn lower_query_physical_dag( + root: &Rc, + mut statistics: F, +) -> Result +where + F: FnMut(&asap_types::pre_asap::QueryExpr) -> Option, +{ + use std::collections::HashMap; + + use asap_types::pre_asap::{GroupKeys, QueryExpr, SetOpKind}; + + struct Lowerer<'a, F> { + statistics: &'a mut F, + logical_roots: HashMap, + next_id: usize, + nodes: Vec, + } + + impl Lowerer<'_, F> + where + F: FnMut(&QueryExpr) -> Option, + { + fn lower(&mut self, query: &QueryExpr) -> Result { + let identity = std::ptr::from_ref(query) as usize; + if let Some(id) = self.logical_roots.get(&identity) { + return Ok(id.clone()); + } + let id = format!("query-{}", self.next_id); + self.next_id += 1; + // Insert only after successful lowering: a malformed recursive + // shape cannot leave a partially reusable node behind. + let root = self.lower_new(query, id)?; + self.logical_roots.insert(identity, root.clone()); + Ok(root) + } + + fn stats(&mut self, query: &QueryExpr) -> Result { + (self.statistics)(query) + .ok_or(AnalyticalCostError::MissingOrStale("operator_statistics")) + } + + fn push( + &mut self, + id: String, + operator: PhysicalOperator, + inputs: OperatorInputs, + children: Vec, + ) -> String { + let output_buffer_bytes = inputs.output_bytes; + self.nodes.push(PhysicalDagNode { + id: id.clone(), + operator, + inputs, + children, + output_buffer_bytes, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }); + id + } + + fn lower_unary( + &mut self, + query: &QueryExpr, + id: String, + operator: PhysicalOperator, + child: &QueryExpr, + ) -> Result { + let child_id = self.lower(child)?; + let inputs = self.stats(query)?; + let child_inputs = &self.node(&child_id)?.inputs; + require_unary_edge(inputs, *child_inputs)?; + require_operator_statistics(operator, inputs)?; + Ok(self.push(id, operator, inputs, vec![child_id])) + } + + fn node(&self, id: &str) -> Result<&PhysicalDagNode, AnalyticalCostError> { + self.nodes.iter().find(|node| node.id == id).ok_or( + AnalyticalCostError::InvalidPhysicalDag("lowered child is missing"), + ) + } + + fn lower_new( + &mut self, + query: &QueryExpr, + id: String, + ) -> Result { + match query { + QueryExpr::Scan { predicates, .. } => { + let inputs = self.stats(query)?; + require_positive_operator_statistics(inputs)?; + if predicates.is_empty() { + if inputs.input_rows != inputs.output_rows + || inputs.input_bytes != inputs.output_bytes + { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "unfiltered scan output does not match its input", + )); + } + return Ok(self.push(id, PhysicalOperator::Scan, inputs, vec![])); + } + if inputs.output_rows > inputs.input_rows + || inputs.output_bytes > inputs.input_bytes + { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "predicate-bearing scan expands its input", + )); + } + let scan_id = format!("{id}-scan"); + let mut scan_inputs = inputs; + scan_inputs.output_rows = inputs.input_rows; + scan_inputs.output_bytes = inputs.input_bytes; + clear_operator_specific_inputs(&mut scan_inputs); + self.push(scan_id.clone(), PhysicalOperator::Scan, scan_inputs, vec![]); + require_operator_statistics(PhysicalOperator::Filter, inputs)?; + Ok(self.push(id, PhysicalOperator::Filter, inputs, vec![scan_id])) + } + QueryExpr::Filter { child, .. } => { + self.lower_unary(query, id, PhysicalOperator::Filter, child) + } + QueryExpr::Project { child, .. } => { + self.lower_unary(query, id, PhysicalOperator::Project, child) + } + QueryExpr::Aggregate { child, .. } => { + self.lower_unary(query, id, PhysicalOperator::HashAggregate, child) + } + QueryExpr::Dedup { child, .. } => { + self.lower_unary(query, id, PhysicalOperator::Deduplicate, child) + } + QueryExpr::Sort { child, .. } => { + self.lower_unary(query, id, PhysicalOperator::Sort, child) + } + QueryExpr::Limit { n, offset, child } => { + if let QueryExpr::Sort { + partition_by, + child: sorted_child, + .. + } = child.as_ref() + { + if partition_by == &GroupKeys::none() { + let child_id = self.lower(sorted_child)?; + let mut inputs = self.stats(query)?; + require_unary_edge(inputs, self.node(&child_id)?.inputs)?; + let bound = n + .checked_add(*offset) + .and_then(|value| u64::try_from(value).ok()) + .ok_or(AnalyticalCostError::Overflow)?; + if bound == 0 { + return Err(AnalyticalCostError::MissingOrZero("topk_k")); + } + inputs.k = Some(bound); + require_limit_cardinality(*n, *offset, inputs)?; + return Ok(self.push( + id, + PhysicalOperator::TopK, + inputs, + vec![child_id], + )); + } + } + let child_id = self.lower(child)?; + let inputs = self.stats(query)?; + require_unary_edge(inputs, self.node(&child_id)?.inputs)?; + require_operator_statistics(PhysicalOperator::Limit, inputs)?; + require_limit_cardinality(*n, *offset, inputs)?; + Ok(self.push(id, PhysicalOperator::Limit, inputs, vec![child_id])) + } + QueryExpr::SQLWindowFunc { child, .. } => { + self.lower_unary(query, id, PhysicalOperator::Window, child) + } + QueryExpr::TimeShift { child, .. } => { + self.lower_unary(query, id, PhysicalOperator::PassThrough, child) + } + QueryExpr::Concat { children } => { + let child_ids = children + .iter() + .map(|child| self.lower(child)) + .collect::, _>>()?; + self.lower_concat(query, id, child_ids) + } + QueryExpr::SetOp { + kind: SetOpKind::Union, + all: true, + left, + right, + } => { + let left_id = self.lower(left)?; + let right_id = self.lower(right)?; + self.lower_concat(query, id, vec![left_id, right_id]) + } + QueryExpr::Join { + kind, + pred, + left, + right, + } => { + if matches!(kind, asap_types::pre_asap::JoinKind::Cross) + || !is_hash_join_predicate(&pred.0) + { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + } + let left_id = self.lower(left)?; + let right_id = self.lower(right)?; + let inputs = self.stats(query)?; + require_positive_operator_statistics(inputs)?; + let left_inputs = self.node(&left_id)?.inputs; + let right_inputs = self.node(&right_id)?.inputs; + if inputs.input_rows != left_inputs.output_rows + || inputs.input_bytes != left_inputs.output_bytes + || inputs.right_rows != Some(right_inputs.output_rows) + || inputs.right_bytes != Some(right_inputs.output_bytes) + { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "join inputs do not match child outputs", + )); + } + Ok(self.push( + id, + PhysicalOperator::HashJoin, + inputs, + vec![left_id, right_id], + )) + } + _ => Err(AnalyticalCostError::UnsupportedQueryOperator), + } + } + + fn lower_concat( + &mut self, + query: &QueryExpr, + id: String, + child_ids: Vec, + ) -> Result { + if child_ids.is_empty() { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "concat has no children", + )); + } + let inputs = self.stats(query)?; + require_positive_operator_statistics(inputs)?; + let (rows, bytes) = + child_ids + .iter() + .try_fold((0_u64, 0_u64), |(rows, bytes), child| { + let child = self.node(child)?; + Ok::<_, AnalyticalCostError>(( + rows.checked_add(child.inputs.output_rows) + .ok_or(AnalyticalCostError::Overflow)?, + bytes + .checked_add(child.inputs.output_bytes) + .ok_or(AnalyticalCostError::Overflow)?, + )) + })?; + if inputs.input_rows != rows + || inputs.input_bytes != bytes + || inputs.output_rows != rows + || inputs.output_bytes != bytes + { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "concat statistics do not equal the sum of child outputs", + )); + } + Ok(self.push(id, PhysicalOperator::Concat, inputs, child_ids)) + } + } + + let mut lowerer = Lowerer { + statistics: &mut statistics, + logical_roots: HashMap::new(), + next_id: 0, + nodes: Vec::new(), + }; + let root = lowerer.lower(root)?; + Ok(PhysicalDag { + nodes: lowerer.nodes, + root, + }) +} + +fn require_positive_operator_statistics(inputs: OperatorInputs) -> Result<(), AnalyticalCostError> { + if inputs.input_rows == 0 { + return Err(AnalyticalCostError::MissingOrZero("operator input_rows")); + } + if inputs.input_bytes == 0 { + return Err(AnalyticalCostError::MissingOrZero("operator input_bytes")); + } + if inputs.output_rows == 0 { + return Err(AnalyticalCostError::MissingOrZero("operator output_rows")); + } + if inputs.output_bytes == 0 { + return Err(AnalyticalCostError::MissingOrZero("operator output_bytes")); + } + Ok(()) +} + +fn require_unary_edge( + inputs: OperatorInputs, + child: OperatorInputs, +) -> Result<(), AnalyticalCostError> { + require_positive_operator_statistics(inputs)?; + if inputs.input_rows != child.output_rows || inputs.input_bytes != child.output_bytes { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "unary input does not match child output", + )); + } + Ok(()) +} + +fn require_operator_statistics( + operator: PhysicalOperator, + inputs: OperatorInputs, +) -> Result<(), AnalyticalCostError> { + let invalid = |reason| Err(AnalyticalCostError::InconsistentOperatorStatistics(reason)); + match operator { + PhysicalOperator::Filter => { + if inputs.output_rows > inputs.input_rows || inputs.output_bytes > inputs.input_bytes { + return invalid("filter output expands its input"); + } + } + PhysicalOperator::Project => { + if inputs.output_rows != inputs.input_rows { + return invalid("projection changes row cardinality"); + } + } + PhysicalOperator::HashAggregate => { + let groups = inputs + .group_count + .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; + if groups == 0 { + return Err(AnalyticalCostError::MissingOrZero("group_count")); + } + if inputs.output_rows > groups { + return invalid("aggregate output exceeds group cardinality"); + } + } + PhysicalOperator::Deduplicate => { + let groups = inputs + .group_count + .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; + if groups == 0 { + return Err(AnalyticalCostError::MissingOrZero("group_count")); + } + if inputs.output_rows != groups || inputs.output_rows > inputs.input_rows { + return invalid("deduplicate output does not equal distinct cardinality"); + } + } + PhysicalOperator::Sort => { + if inputs.output_rows != inputs.input_rows || inputs.output_bytes != inputs.input_bytes + { + return invalid("sort changes its input cardinality or width"); + } + } + PhysicalOperator::TopK | PhysicalOperator::Limit => { + if inputs.output_rows > inputs.input_rows { + return invalid("bounded output exceeds its input cardinality"); + } + } + PhysicalOperator::Window => { + if inputs.output_rows != inputs.input_rows { + return invalid("SQL window changes row cardinality"); + } + } + PhysicalOperator::PassThrough => { + if inputs.output_rows != inputs.input_rows || inputs.output_bytes != inputs.input_bytes + { + return invalid("pass-through wrapper changes its edge statistics"); + } + } + PhysicalOperator::Scan | PhysicalOperator::HashJoin | PhysicalOperator::Concat => {} + } + Ok(()) +} + +fn require_limit_cardinality( + n: usize, + offset: usize, + inputs: OperatorInputs, +) -> Result<(), AnalyticalCostError> { + let n = u64::try_from(n).map_err(|_| AnalyticalCostError::Overflow)?; + let offset = u64::try_from(offset).map_err(|_| AnalyticalCostError::Overflow)?; + let expected = inputs.input_rows.saturating_sub(offset).min(n); + if inputs.output_rows != expected { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "limit output does not match n and offset", + )); + } + Ok(()) +} + +fn clear_operator_specific_inputs(inputs: &mut OperatorInputs) { + inputs.group_count = None; + inputs.key_bytes = None; + inputs.aggregate_value_bytes = None; + inputs.k = None; + inputs.right_rows = None; + inputs.right_bytes = None; + inputs.hash_join_build_side = None; +} + +fn is_hash_join_predicate(expr: &asap_types::pre_asap::QueryExpr) -> bool { + use asap_types::pre_asap::{CompareOpKind, QueryExpr}; + + match expr { + QueryExpr::Compare { + left, + op: CompareOpKind::Eq, + right, + } => { + matches!(left.as_ref(), QueryExpr::Column(_)) + && matches!(right.as_ref(), QueryExpr::Column(_)) + } + QueryExpr::BoolAnd(parts) => !parts.is_empty() && parts.iter().all(is_hash_join_predicate), + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::analytical_cost::{estimate_physical_dag, HashJoinBuildSide}; + + fn operator_inputs( + input_rows: u64, + input_bytes: u64, + output_rows: u64, + output_bytes: u64, + ) -> OperatorInputs { + OperatorInputs { + input_rows, + input_bytes, + output_rows, + output_bytes, + group_count: None, + key_bytes: None, + aggregate_value_bytes: None, + k: None, + right_rows: None, + right_bytes: None, + hash_join_build_side: None, + } + } + + #[test] + fn query_lowering_recurses_and_fuses_global_sort_limit() { + use asap_types::pre_asap::{ + agg_intent::default_cardinality, GroupKeys, QueryExpr, Reduction, Source, + }; + use asap_types::pre_asap::{Column, DataType, Schema}; + use std::rc::Rc; + + let scan = Rc::new(QueryExpr::Scan { + source: Source::Table { + table_ref: "events".into(), + }, + predicates: vec![asap_types::pre_asap::Predicate(Rc::new( + QueryExpr::Literal(asap_types::pre_asap::ScalarValue::Boolean(true)), + ))], + schema: Schema::new(vec![ + Column::new("service", DataType::Utf8, false), + Column::new("value", DataType::Float64, false), + ]), + }); + let aggregate = Rc::new(QueryExpr::Aggregate { + reduction: Reduction::by(vec![0]), + measures: vec![default_cardinality()], + output_names: vec![], + having: None, + child: Rc::clone(&scan), + }); + let sort = Rc::new(QueryExpr::Sort { + keys: vec![], + partition_by: GroupKeys::none(), + child: aggregate, + }); + let root = Rc::new(QueryExpr::Limit { + n: 10, + offset: 5, + child: sort, + }); + + let dag = lower_query_physical_dag(&root, |node| { + let mut stats = match node { + QueryExpr::Scan { .. } => operator_inputs(1_000, 64_000, 400, 25_600), + QueryExpr::Aggregate { .. } => operator_inputs(400, 25_600, 100, 4_000), + QueryExpr::Limit { .. } => operator_inputs(100, 4_000, 10, 400), + _ => return None, + }; + if matches!(node, QueryExpr::Aggregate { .. }) { + stats.group_count = Some(100); + stats.key_bytes = Some(16); + stats.aggregate_value_bytes = Some(8); + } + Some(stats) + }) + .unwrap(); + + assert_eq!( + dag.nodes + .iter() + .map(|node| node.operator) + .collect::>(), + vec![ + PhysicalOperator::Scan, + PhysicalOperator::Filter, + PhysicalOperator::HashAggregate, + PhysicalOperator::TopK, + ] + ); + let topk = dag.nodes.last().unwrap(); + assert_eq!(topk.inputs.k, Some(15)); + assert_eq!(topk.children, vec![dag.nodes[2].id.clone()]); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, 3).is_ok()); + } + + #[test] + fn query_lowering_deduplicates_shared_rc_children() { + use asap_types::pre_asap::{Column, CompareOpKind, DataType, Schema}; + use asap_types::pre_asap::{JoinKind, Predicate, QueryExpr, Source}; + use std::rc::Rc; + + let shared = Rc::new(QueryExpr::Scan { + source: Source::Table { + table_ref: "dimensions".into(), + }, + predicates: vec![], + schema: Schema::new(vec![Column::new("id", DataType::Int64, false)]), + }); + let root = Rc::new(QueryExpr::Join { + kind: JoinKind::Inner, + pred: Predicate(Rc::new(QueryExpr::Compare { + left: Rc::new(QueryExpr::Column(0)), + op: CompareOpKind::Eq, + right: Rc::new(QueryExpr::Column(1)), + })), + left: Rc::clone(&shared), + right: Rc::clone(&shared), + }); + let dag = lower_query_physical_dag(&root, |node| match node { + QueryExpr::Scan { .. } => Some(operator_inputs(100, 800, 100, 800)), + QueryExpr::Join { .. } => { + let mut stats = operator_inputs(100, 800, 25, 400); + stats.right_rows = Some(100); + stats.right_bytes = Some(800); + stats.hash_join_build_side = Some(HashJoinBuildSide::Right); + Some(stats) + } + _ => None, + }) + .unwrap(); + + assert_eq!(dag.nodes.len(), 2); + assert_eq!(dag.nodes[1].children, vec![dag.nodes[0].id.clone(); 2]); + let estimate = estimate_physical_dag(&dag.nodes, &dag.root, 1).unwrap(); + assert_eq!(estimate.scan_bytes, 800); + } + + #[test] + fn query_lowering_covers_relational_unary_operators() { + use asap_types::pre_asap::{Column, DataType, ScalarValue, Schema}; + use asap_types::pre_asap::{ + GroupKeys, Predicate, QueryExpr, Source, TimeShift, WindowFuncKind, + }; + use std::rc::Rc; + + let scan = Rc::new(QueryExpr::Scan { + source: Source::Table { + table_ref: "events".into(), + }, + predicates: vec![], + schema: Schema::new(vec![Column::new("id", DataType::Int64, false)]), + }); + let filter = Rc::new(QueryExpr::Filter { + pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Boolean(true)))), + child: scan, + }); + let project = Rc::new(QueryExpr::Project { + cols: vec![], + qualifier: None, + child: filter, + }); + let dedup = Rc::new(QueryExpr::Dedup { + cols: vec![0], + child: project, + }); + let window = Rc::new(QueryExpr::SQLWindowFunc { + func: WindowFuncKind::RowNumber, + args: vec![], + partition_by: GroupKeys::none(), + order_by: vec![], + frame: None, + output_name: "rn".into(), + child: dedup, + }); + let sort = Rc::new(QueryExpr::Sort { + keys: vec![], + partition_by: GroupKeys::by(vec![0]), + child: window, + }); + let limit = Rc::new(QueryExpr::Limit { + n: 20, + offset: 0, + child: sort, + }); + let root = Rc::new(QueryExpr::TimeShift { + shift: TimeShift::default(), + child: limit, + }); + + let dag = lower_query_physical_dag(&root, |node| { + let mut inputs = match node { + QueryExpr::Scan { .. } => operator_inputs(1_000, 8_000, 1_000, 8_000), + QueryExpr::Filter { .. } => operator_inputs(1_000, 8_000, 800, 6_400), + QueryExpr::Project { .. } => operator_inputs(800, 6_400, 800, 3_200), + QueryExpr::Dedup { .. } => operator_inputs(800, 3_200, 500, 2_000), + QueryExpr::SQLWindowFunc { .. } => operator_inputs(500, 2_000, 500, 6_000), + QueryExpr::Sort { .. } => operator_inputs(500, 6_000, 500, 6_000), + QueryExpr::Limit { .. } => operator_inputs(500, 6_000, 20, 240), + QueryExpr::TimeShift { .. } => operator_inputs(20, 240, 20, 240), + _ => return None, + }; + if matches!(node, QueryExpr::Dedup { .. }) { + inputs.group_count = Some(500); + inputs.key_bytes = Some(8); + } + Some(inputs) + }) + .unwrap(); + + assert_eq!( + dag.nodes + .iter() + .map(|node| node.operator) + .collect::>(), + vec![ + PhysicalOperator::Scan, + PhysicalOperator::Filter, + PhysicalOperator::Project, + PhysicalOperator::Deduplicate, + PhysicalOperator::Window, + PhysicalOperator::Sort, + PhysicalOperator::Limit, + PhysicalOperator::PassThrough, + ] + ); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, 2).is_ok()); + } + + #[test] + fn query_lowering_maps_concat_and_union_all_but_rejects_distinct_set_ops() { + use asap_types::pre_asap::{Column, DataType, Schema}; + use asap_types::pre_asap::{QueryExpr, SetOpKind, Source}; + use std::rc::Rc; + + let scan = |name: &str| QueryExpr::Scan { + source: Source::Table { + table_ref: name.into(), + }, + predicates: vec![], + schema: Schema::new(vec![Column::new("id", DataType::Int64, false)]), + }; + let union = Rc::new(QueryExpr::SetOp { + kind: SetOpKind::Union, + all: true, + left: Rc::new(scan("a")), + right: Rc::new(scan("b")), + }); + let dag = lower_query_physical_dag(&union, |node| match node { + QueryExpr::Scan { + source: Source::Table { table_ref }, + .. + } if table_ref == "a" => Some(operator_inputs(10, 80, 10, 80)), + QueryExpr::Scan { .. } => Some(operator_inputs(20, 160, 20, 160)), + QueryExpr::SetOp { .. } => Some(operator_inputs(30, 240, 30, 240)), + _ => None, + }) + .unwrap(); + assert_eq!(dag.nodes.last().unwrap().operator, PhysicalOperator::Concat); + + let concat = Rc::new(QueryExpr::Concat { + children: vec![scan("a"), scan("b")], + }); + let dag = lower_query_physical_dag(&concat, |node| match node { + QueryExpr::Scan { + source: Source::Table { table_ref }, + .. + } if table_ref == "a" => Some(operator_inputs(10, 80, 10, 80)), + QueryExpr::Scan { .. } => Some(operator_inputs(20, 160, 20, 160)), + QueryExpr::Concat { .. } => Some(operator_inputs(30, 240, 30, 240)), + _ => None, + }) + .unwrap(); + assert_eq!(dag.nodes.last().unwrap().operator, PhysicalOperator::Concat); + + let distinct_union = Rc::new(QueryExpr::SetOp { + kind: SetOpKind::Union, + all: false, + left: Rc::new(scan("a")), + right: Rc::new(scan("b")), + }); + assert_eq!( + lower_query_physical_dag(&distinct_union, |_| None), + Err(AnalyticalCostError::UnsupportedQueryOperator) + ); + } + + #[test] + fn query_lowering_fails_closed_for_missing_or_inconsistent_statistics() { + use asap_types::pre_asap::{Column, DataType, Schema}; + use asap_types::pre_asap::{QueryExpr, Source}; + use std::rc::Rc; + + let scan = Rc::new(QueryExpr::Scan { + source: Source::Table { + table_ref: "events".into(), + }, + predicates: vec![], + schema: Schema::new(vec![Column::new("id", DataType::Int64, false)]), + }); + let root = Rc::new(QueryExpr::Project { + cols: vec![], + qualifier: None, + child: scan, + }); + + assert_eq!( + lower_query_physical_dag(&root, |_| None), + Err(AnalyticalCostError::MissingOrStale("operator_statistics")) + ); + assert_eq!( + lower_query_physical_dag(&root, |node| match node { + QueryExpr::Scan { .. } => Some(operator_inputs(100, 800, 100, 800)), + QueryExpr::Project { .. } => Some(operator_inputs(99, 792, 99, 396)), + _ => None, + }), + Err(AnalyticalCostError::InconsistentOperatorStatistics( + "unary input does not match child output" + )) + ); + } +} diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 479fc062..9f1a16cb 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -184,6 +184,7 @@ pub mod accuracy; pub mod accuracy_reconciliation; pub mod analytical_cost; +pub mod analytical_lowering; pub mod analytical_statistics; pub mod cost_model; pub mod explanation; diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 7fa0267c..9123cc50 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -13,10 +13,15 @@ and guarantee composition run first; costing ranks only the candidates that survive. Missing evidence produces an unavailable estimate, never an assumed zero or a structural-cost fallback. -The physical-DAG estimator composes any DAG whose nodes have supported -physical operators and complete `OperatorStatistics`. Query lowering and -planner selection are separate integration layers; neither may substitute a -shape-specific shortcut or structural node count. +This document distinguishes three implementation layers: + +- the physical-DAG estimator, which can compose any DAG whose nodes have + supported physical operators and complete `OperatorStatistics`; and +- query-DAG lowering, which recursively maps supported resolved `QueryExpr` + operators to that physical representation; and +- replacement lowering and ranking, which must compare complete alternatives. + +No layer may substitute a shape-specific shortcut or structural node count. An estimate has physical dimensions: @@ -226,6 +231,58 @@ inherit the target's old node costs. A replacement candidate includes any newly embedded child summaries, while an independently shared child is deduplicated by physical identity. +### Query-DAG lowering and statistics contract + +`lower_query_physical_dag` recursively lowers a resolved `Rc` and +returns a `PhysicalDag` containing both its nodes and root ID. It consumes the +existing query and physical-operator enums; it does not introduce a parallel +logical operator vocabulary. A statistics callback resolves the existing +`OperatorInputs` for each logical operator identity. Returning no statistics +makes the entire query unavailable. + +The lowering validates every physical edge before costing: + +- a unary operator's `input_rows` and `input_bytes` equal its child's output; +- a hash join's left and right inputs equal the corresponding child outputs; +- Concat and `UNION ALL` input/output totals equal the checked sum of all + child outputs; and +- row-preserving, reducing, and bounded operators obey their cardinality + invariants. + +The supported mappings are: + +| Existing `QueryExpr` shape | Physical DAG | +|---|---| +| Scan without predicates | Scan | +| Scan with pushed predicates | Scan → Filter | +| Filter | Filter | +| Project | Project | +| Aggregate, including a fused HAVING predicate | HashAggregate | +| Dedup | Deduplicate | +| Equi-Join | HashJoin with an evidence-selected build side | +| Concat or `UNION ALL` | Concat | +| Sort | in-memory Sort | +| global Sort followed by Limit | heap TopK, with `k = offset + n` from the query IR | +| partitioned Sort followed by Limit | Sort → Limit | +| SQLWindowFunc | Window | +| TimeShift | PassThrough | + +Logical identity is the address of the existing `Rc` allocation. +Repeated references therefore lower once and every parent points to the same +physical ID. The resulting node IDs are deterministic within a lowering run; +they are not persistent query identifiers. + +This generic lowering currently creates raw-query operators, all with +`ExecutionMultiplicity::PerEvaluation`, zero retained state, and a conservative +full-output edge buffer. A deployment with verified batching may construct +`PhysicalDagNode` values with smaller `output_buffer_bytes` directly. + +Cross/non-equi joins, `INTERSECT`, `EXCEPT`, distinct `UNION`, PromQL +range/subquery execution, vector matching, and PromQL-specific enrichment/ +relabel/sample operators stay unavailable. Their cost requires a physical +implementation or multiplicity/state facts that the current physical-operator +vocabulary cannot represent; they are not treated as free pass-through work. + ## Physical operator formulas Operator estimates are local: child CPU and I/O are excluded and composed by @@ -298,7 +355,9 @@ children would undercount the plan. `estimate_physical_dag` is independent of query shape. A caller supplies the complete physical DAG and per-node evidence. Filters, projections, joins, windows, nested aggregates, Top-K, and shared sub-DAGs therefore use the same -estimation path. +estimation path. The generic query lowerer recursively maps the supported raw +query operators into that representation. Replacement lowering remains a +separate layer and must include all summary-maintenance work. DDSketch is unavailable because occupied bins depend on value range and distribution. The model does not invent a bin count. Algorithm/parameter @@ -349,9 +408,10 @@ The intended end-to-end selection pipeline is: 5. estimates the complete candidate DAG; 6. applies calibration and ranks candidates by ascending cost. -This module implements the physical estimation step. Lowering, evidence -resolution, legality checks, and planner integration remain separate layers; -each preserves the complete-plan and fail-closed requirements above. +The query lowerer and physical estimator cover the supported raw-query shapes +listed above. Replacement evidence resolution, legality checks, and planner +integration remain separate layers; each preserves the complete-plan and +fail-closed requirements above. Before applying the following arithmetic, callers validate exact equality of the raw and selected alternative's `ComparisonScope`, and use the same From aace71d945987d6d56755fcb8b6f99175def44a0 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 08:23:46 -0600 Subject: [PATCH 06/48] fix(cost): allow empty physical edges --- .../src/analytical_lowering.rs | 79 +++++++++++++------ .../analytical-resource-cost.md | 2 + 2 files changed, 59 insertions(+), 22 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index 01f0a68f..f7db8ddb 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -111,7 +111,7 @@ where match query { QueryExpr::Scan { predicates, .. } => { let inputs = self.stats(query)?; - require_positive_operator_statistics(inputs)?; + require_consistent_edge_statistics(inputs)?; if predicates.is_empty() { if inputs.input_rows != inputs.output_rows || inputs.input_bytes != inputs.output_bytes @@ -225,7 +225,7 @@ where let left_id = self.lower(left)?; let right_id = self.lower(right)?; let inputs = self.stats(query)?; - require_positive_operator_statistics(inputs)?; + require_consistent_edge_statistics(inputs)?; let left_inputs = self.node(&left_id)?.inputs; let right_inputs = self.node(&right_id)?.inputs; if inputs.input_rows != left_inputs.output_rows @@ -260,7 +260,7 @@ where )); } let inputs = self.stats(query)?; - require_positive_operator_statistics(inputs)?; + require_consistent_edge_statistics(inputs)?; let (rows, bytes) = child_ids .iter() @@ -300,27 +300,30 @@ where }) } -fn require_positive_operator_statistics(inputs: OperatorInputs) -> Result<(), AnalyticalCostError> { - if inputs.input_rows == 0 { - return Err(AnalyticalCostError::MissingOrZero("operator input_rows")); - } - if inputs.input_bytes == 0 { - return Err(AnalyticalCostError::MissingOrZero("operator input_bytes")); - } - if inputs.output_rows == 0 { - return Err(AnalyticalCostError::MissingOrZero("operator output_rows")); - } - if inputs.output_bytes == 0 { - return Err(AnalyticalCostError::MissingOrZero("operator output_bytes")); - } +fn require_consistent_edge_statistics(inputs: OperatorInputs) -> Result<(), AnalyticalCostError> { + require_cardinality_width("operator input", inputs.input_rows, inputs.input_bytes)?; + require_cardinality_width("operator output", inputs.output_rows, inputs.output_bytes)?; Ok(()) } +fn require_cardinality_width( + edge: &'static str, + rows: u64, + bytes: u64, +) -> Result<(), AnalyticalCostError> { + match (rows, bytes) { + (0, 0) => Ok(()), + (0, _) => Err(AnalyticalCostError::InconsistentOperatorStatistics(edge)), + (_, 0) => Err(AnalyticalCostError::MissingOrZero(edge)), + _ => Ok(()), + } +} + fn require_unary_edge( inputs: OperatorInputs, child: OperatorInputs, ) -> Result<(), AnalyticalCostError> { - require_positive_operator_statistics(inputs)?; + require_consistent_edge_statistics(inputs)?; if inputs.input_rows != child.output_rows || inputs.input_bytes != child.output_bytes { return Err(AnalyticalCostError::InconsistentOperatorStatistics( "unary input does not match child output", @@ -349,8 +352,8 @@ fn require_operator_statistics( let groups = inputs .group_count .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - if groups == 0 { - return Err(AnalyticalCostError::MissingOrZero("group_count")); + if groups == 0 && (inputs.input_rows != 0 || inputs.output_rows != 0) { + return invalid("zero groups require an empty grouped input and output"); } if inputs.output_rows > groups { return invalid("aggregate output exceeds group cardinality"); @@ -360,9 +363,6 @@ fn require_operator_statistics( let groups = inputs .group_count .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - if groups == 0 { - return Err(AnalyticalCostError::MissingOrZero("group_count")); - } if inputs.output_rows != groups || inputs.output_rows > inputs.input_rows { return invalid("deduplicate output does not equal distinct cardinality"); } @@ -760,4 +760,39 @@ mod tests { )) ); } + + #[test] + fn query_lowering_accepts_a_consistently_empty_edge() { + use asap_types::pre_asap::{Column, DataType, ScalarValue, Schema}; + use asap_types::pre_asap::{Predicate, QueryExpr, Source}; + use std::rc::Rc; + + let scan = Rc::new(QueryExpr::Scan { + source: Source::Table { + table_ref: "events".into(), + }, + predicates: vec![], + schema: Schema::new(vec![Column::new("id", DataType::Int64, false)]), + }); + let filter = Rc::new(QueryExpr::Filter { + pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Boolean(false)))), + child: scan, + }); + let root = Rc::new(QueryExpr::Limit { + n: 10, + offset: 0, + child: filter, + }); + + let dag = lower_query_physical_dag(&root, |node| match node { + QueryExpr::Scan { .. } => Some(operator_inputs(100, 800, 100, 800)), + QueryExpr::Filter { .. } => Some(operator_inputs(100, 800, 0, 0)), + QueryExpr::Limit { .. } => Some(operator_inputs(0, 0, 0, 0)), + _ => None, + }) + .unwrap(); + let estimate = estimate_physical_dag(&dag.nodes, &dag.root, 1).unwrap(); + assert_eq!(estimate.cpu_ops, 200.0); + assert_eq!(estimate.scan_bytes, 800); + } } diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 9123cc50..0027b772 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -242,6 +242,8 @@ makes the entire query unavailable. The lowering validates every physical edge before costing: +- `(rows = 0, bytes = 0)` is a valid empty edge, while positive rows still + require byte-width evidence and zero rows cannot carry non-zero bytes; - a unary operator's `input_rows` and `input_bytes` equal its child's output; - a hash join's left and right inputs equal the corresponding child outputs; - Concat and `UNION ALL` input/output totals equal the checked sum of all From 2be4e4d0a267fc30a052ec71f04844f78fb55858 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 09:17:17 -0600 Subject: [PATCH 07/48] fix(cost): bind query lowering to physical evidence --- .../src/analytical_lowering.rs | 746 ++++++++++++------ .../analytical-resource-cost.md | 17 +- 2 files changed, 513 insertions(+), 250 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index f7db8ddb..8a37600b 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -5,7 +5,10 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; use crate::analytical_cost::{ - AnalyticalCostError, ExecutionMultiplicity, OperatorInputs, PhysicalDagNode, PhysicalOperator, + AnalyticalCostError, ExecutionMultiplicity, PhysicalDagNode, PhysicalOperator, +}; +use crate::analytical_statistics::{ + ComparisonScope, EdgeStatistics, OperatorStatistics, OperatorStatisticsProvider, SourceCoverage, }; /// A lowered physical DAG and the node whose output is the query result. @@ -17,32 +20,60 @@ pub struct PhysicalDag { pub root: String, } +/// Atomic evidence for one lowered physical node. The statistics contract is +/// reused unchanged; the separate buffer field is necessary because logical +/// edge bytes cannot stand in for an allocation. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PhysicalNodeEvidence { + pub statistics: OperatorStatistics, + pub output_buffer_bytes: u64, +} + +pub trait PhysicalNodeEvidenceProvider { + fn evidence(&self, node_id: &str) -> Result; +} + +impl PhysicalNodeEvidenceProvider for std::collections::HashMap { + fn evidence(&self, node_id: &str) -> Result { + self.get(node_id) + .cloned() + .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(node_id.into())) + } +} + +impl OperatorStatisticsProvider for std::collections::HashMap { + fn statistics(&self, node_id: &str) -> Result { + self.evidence(node_id).map(|evidence| evidence.statistics) + } +} + /// Lower a resolved query operator DAG to the physical operators understood by -/// this cost model. The callback supplies physical statistics for each logical -/// operator identity; returning `None` makes the complete query unavailable. -/// Scalar expressions remain part of their containing operator's local cost. -pub fn lower_query_physical_dag( +/// this cost model. The authoritative provider supplies statistics by the +/// deterministic physical IDs assigned here; missing evidence makes the +/// complete query unavailable. Scalar expressions remain part of their +/// containing operator's local cost. +pub fn lower_query_physical_dag( root: &Rc, - mut statistics: F, -) -> Result -where - F: FnMut(&asap_types::pre_asap::QueryExpr) -> Option, -{ + scope: &ComparisonScope, + evidence: &dyn PhysicalNodeEvidenceProvider, +) -> Result { use std::collections::HashMap; use asap_types::pre_asap::{GroupKeys, QueryExpr, SetOpKind}; - struct Lowerer<'a, F> { - statistics: &'a mut F, + scope.validate()?; + + struct Lowerer<'a> { + scope: &'a ComparisonScope, + provider: &'a dyn PhysicalNodeEvidenceProvider, + statistics: HashMap, + output_buffers: HashMap, logical_roots: HashMap, next_id: usize, nodes: Vec, } - impl Lowerer<'_, F> - where - F: FnMut(&QueryExpr) -> Option, - { + impl Lowerer<'_> { fn lower(&mut self, query: &QueryExpr) -> Result { let identity = std::ptr::from_ref(query) as usize; if let Some(id) = self.logical_roots.get(&identity) { @@ -57,25 +88,32 @@ where Ok(root) } - fn stats(&mut self, query: &QueryExpr) -> Result { - (self.statistics)(query) - .ok_or(AnalyticalCostError::MissingOrStale("operator_statistics")) + fn stats(&mut self, id: &str) -> Result { + if let Some(statistics) = self.statistics.get(id) { + return Ok(statistics.clone()); + } + let evidence = self.provider.evidence(id)?; + let statistics = evidence.statistics; + self.statistics.insert(id.into(), statistics.clone()); + self.output_buffers + .insert(id.into(), evidence.output_buffer_bytes); + Ok(statistics) } fn push( &mut self, id: String, operator: PhysicalOperator, - inputs: OperatorInputs, + _statistics: &OperatorStatistics, children: Vec, + source_coverage: Option, ) -> String { - let output_buffer_bytes = inputs.output_bytes; self.nodes.push(PhysicalDagNode { id: id.clone(), operator, - inputs, children, - output_buffer_bytes, + source_coverage, + output_buffer_bytes: self.output_buffers[&id], retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, }); @@ -84,23 +122,25 @@ where fn lower_unary( &mut self, - query: &QueryExpr, + _query: &QueryExpr, id: String, operator: PhysicalOperator, child: &QueryExpr, ) -> Result { let child_id = self.lower(child)?; - let inputs = self.stats(query)?; - let child_inputs = &self.node(&child_id)?.inputs; - require_unary_edge(inputs, *child_inputs)?; - require_operator_statistics(operator, inputs)?; - Ok(self.push(id, operator, inputs, vec![child_id])) + let statistics = self.stats(&id)?; + let child_statistics = self.node_statistics(&child_id)?; + require_unary_edge(&id, &statistics, &child_id, child_statistics)?; + require_operator_statistics(operator, &statistics)?; + Ok(self.push(id, operator, &statistics, vec![child_id], None)) } - fn node(&self, id: &str) -> Result<&PhysicalDagNode, AnalyticalCostError> { - self.nodes.iter().find(|node| node.id == id).ok_or( - AnalyticalCostError::InvalidPhysicalDag("lowered child is missing"), - ) + fn node_statistics(&self, id: &str) -> Result<&OperatorStatistics, AnalyticalCostError> { + self.statistics + .get(id) + .ok_or(AnalyticalCostError::InvalidPhysicalDag( + "lowered child statistics are missing", + )) } fn lower_new( @@ -109,34 +149,41 @@ where id: String, ) -> Result { match query { - QueryExpr::Scan { predicates, .. } => { - let inputs = self.stats(query)?; - require_consistent_edge_statistics(inputs)?; + QueryExpr::Scan { + source, predicates, .. + } => { + let coverage = bind_scan_coverage(&id, source, predicates, self.scope)?; if predicates.is_empty() { - if inputs.input_rows != inputs.output_rows - || inputs.input_bytes != inputs.output_bytes - { - return Err(AnalyticalCostError::InconsistentOperatorStatistics( - "unfiltered scan output does not match its input", - )); - } - return Ok(self.push(id, PhysicalOperator::Scan, inputs, vec![])); - } - if inputs.output_rows > inputs.input_rows - || inputs.output_bytes > inputs.input_bytes - { - return Err(AnalyticalCostError::InconsistentOperatorStatistics( - "predicate-bearing scan expands its input", + let statistics = self.stats(&id)?; + require_statistics_shape(&id, &statistics, 1)?; + return Ok(self.push( + id, + PhysicalOperator::Scan, + &statistics, + vec![], + Some(coverage), )); } let scan_id = format!("{id}-scan"); - let mut scan_inputs = inputs; - scan_inputs.output_rows = inputs.input_rows; - scan_inputs.output_bytes = inputs.input_bytes; - clear_operator_specific_inputs(&mut scan_inputs); - self.push(scan_id.clone(), PhysicalOperator::Scan, scan_inputs, vec![]); - require_operator_statistics(PhysicalOperator::Filter, inputs)?; - Ok(self.push(id, PhysicalOperator::Filter, inputs, vec![scan_id])) + let scan_statistics = self.stats(&scan_id)?; + require_statistics_shape(&scan_id, &scan_statistics, 1)?; + self.push( + scan_id.clone(), + PhysicalOperator::Scan, + &scan_statistics, + vec![], + Some(coverage), + ); + let filter_statistics = self.stats(&id)?; + require_unary_edge(&id, &filter_statistics, &scan_id, &scan_statistics)?; + require_operator_statistics(PhysicalOperator::Filter, &filter_statistics)?; + Ok(self.push( + id, + PhysicalOperator::Filter, + &filter_statistics, + vec![scan_id], + None, + )) } QueryExpr::Filter { child, .. } => { self.lower_unary(query, id, PhysicalOperator::Filter, child) @@ -162,8 +209,9 @@ where { if partition_by == &GroupKeys::none() { let child_id = self.lower(sorted_child)?; - let mut inputs = self.stats(query)?; - require_unary_edge(inputs, self.node(&child_id)?.inputs)?; + let statistics = self.stats(&id)?; + let child_statistics = self.node_statistics(&child_id)?; + require_unary_edge(&id, &statistics, &child_id, child_statistics)?; let bound = n .checked_add(*offset) .and_then(|value| u64::try_from(value).ok()) @@ -171,22 +219,34 @@ where if bound == 0 { return Err(AnalyticalCostError::MissingOrZero("topk_k")); } - inputs.k = Some(bound); - require_limit_cardinality(*n, *offset, inputs)?; + if statistics.k != Some(bound) { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "Top-K statistics disagree with LIMIT n + offset", + )); + } + require_limit_cardinality(*n, *offset, &statistics)?; return Ok(self.push( id, PhysicalOperator::TopK, - inputs, + &statistics, vec![child_id], + None, )); } } let child_id = self.lower(child)?; - let inputs = self.stats(query)?; - require_unary_edge(inputs, self.node(&child_id)?.inputs)?; - require_operator_statistics(PhysicalOperator::Limit, inputs)?; - require_limit_cardinality(*n, *offset, inputs)?; - Ok(self.push(id, PhysicalOperator::Limit, inputs, vec![child_id])) + let statistics = self.stats(&id)?; + let child_statistics = self.node_statistics(&child_id)?; + require_unary_edge(&id, &statistics, &child_id, child_statistics)?; + require_operator_statistics(PhysicalOperator::Limit, &statistics)?; + require_limit_cardinality(*n, *offset, &statistics)?; + Ok(self.push( + id, + PhysicalOperator::Limit, + &statistics, + vec![child_id], + None, + )) } QueryExpr::SQLWindowFunc { child, .. } => { self.lower_unary(query, id, PhysicalOperator::Window, child) @@ -224,24 +284,24 @@ where } let left_id = self.lower(left)?; let right_id = self.lower(right)?; - let inputs = self.stats(query)?; - require_consistent_edge_statistics(inputs)?; - let left_inputs = self.node(&left_id)?.inputs; - let right_inputs = self.node(&right_id)?.inputs; - if inputs.input_rows != left_inputs.output_rows - || inputs.input_bytes != left_inputs.output_bytes - || inputs.right_rows != Some(right_inputs.output_rows) - || inputs.right_bytes != Some(right_inputs.output_bytes) + let statistics = self.stats(&id)?; + require_statistics_shape(&id, &statistics, 2)?; + let left_statistics = self.node_statistics(&left_id)?; + let right_statistics = self.node_statistics(&right_id)?; + if statistics.inputs[0] != left_statistics.output + || statistics.inputs[1] != right_statistics.output { return Err(AnalyticalCostError::InconsistentOperatorStatistics( "join inputs do not match child outputs", )); } + require_operator_statistics(PhysicalOperator::HashJoin, &statistics)?; Ok(self.push( id, PhysicalOperator::HashJoin, - inputs, + &statistics, vec![left_id, right_id], + None, )) } _ => Err(AnalyticalCostError::UnsupportedQueryOperator), @@ -250,7 +310,7 @@ where fn lower_concat( &mut self, - query: &QueryExpr, + _query: &QueryExpr, id: String, child_ids: Vec, ) -> Result { @@ -259,36 +319,43 @@ where "concat has no children", )); } - let inputs = self.stats(query)?; - require_consistent_edge_statistics(inputs)?; - let (rows, bytes) = - child_ids - .iter() - .try_fold((0_u64, 0_u64), |(rows, bytes), child| { - let child = self.node(child)?; - Ok::<_, AnalyticalCostError>(( - rows.checked_add(child.inputs.output_rows) - .ok_or(AnalyticalCostError::Overflow)?, - bytes - .checked_add(child.inputs.output_bytes) - .ok_or(AnalyticalCostError::Overflow)?, - )) - })?; - if inputs.input_rows != rows - || inputs.input_bytes != bytes - || inputs.output_rows != rows - || inputs.output_bytes != bytes - { + let statistics = self.stats(&id)?; + require_statistics_shape(&id, &statistics, child_ids.len())?; + let (rows, bytes) = child_ids.iter().enumerate().try_fold( + (0_u64, 0_u64), + |(rows, bytes), (index, child)| { + let child_statistics = self.node_statistics(child)?; + if statistics.inputs[index] != child_statistics.output { + return Err(AnalyticalCostError::ConflictingEdgeStatistics { + parent: id.clone(), + child: child.clone(), + input_index: index, + }); + } + Ok::<_, AnalyticalCostError>(( + rows.checked_add(child_statistics.output.rows) + .ok_or(AnalyticalCostError::Overflow)?, + bytes + .checked_add(child_statistics.output.bytes) + .ok_or(AnalyticalCostError::Overflow)?, + )) + }, + )?; + if statistics.output != (EdgeStatistics { rows, bytes }) { return Err(AnalyticalCostError::InconsistentOperatorStatistics( "concat statistics do not equal the sum of child outputs", )); } - Ok(self.push(id, PhysicalOperator::Concat, inputs, child_ids)) + require_operator_statistics(PhysicalOperator::Concat, &statistics)?; + Ok(self.push(id, PhysicalOperator::Concat, &statistics, child_ids, None)) } } let mut lowerer = Lowerer { - statistics: &mut statistics, + scope, + provider: evidence, + statistics: HashMap::new(), + output_buffers: HashMap::new(), logical_roots: HashMap::new(), next_id: 0, nodes: Vec::new(), @@ -300,92 +367,107 @@ where }) } -fn require_consistent_edge_statistics(inputs: OperatorInputs) -> Result<(), AnalyticalCostError> { - require_cardinality_width("operator input", inputs.input_rows, inputs.input_bytes)?; - require_cardinality_width("operator output", inputs.output_rows, inputs.output_bytes)?; - Ok(()) -} - -fn require_cardinality_width( - edge: &'static str, - rows: u64, - bytes: u64, +fn require_statistics_shape( + node: &str, + statistics: &OperatorStatistics, + input_count: usize, ) -> Result<(), AnalyticalCostError> { - match (rows, bytes) { - (0, 0) => Ok(()), - (0, _) => Err(AnalyticalCostError::InconsistentOperatorStatistics(edge)), - (_, 0) => Err(AnalyticalCostError::MissingOrZero(edge)), - _ => Ok(()), + if statistics.inputs.len() != input_count { + return Err(AnalyticalCostError::InvalidOperatorStatistics { + node: node.into(), + reason: "wrong input-edge count", + }); } + if statistics + .inputs + .iter() + .chain(std::iter::once(&statistics.output)) + .any(|edge| !edge.is_consistent()) + { + return Err(AnalyticalCostError::InvalidOperatorStatistics { + node: node.into(), + reason: "edge rows and logical bytes are inconsistent", + }); + } + Ok(()) } fn require_unary_edge( - inputs: OperatorInputs, - child: OperatorInputs, + node: &str, + statistics: &OperatorStatistics, + child_id: &str, + child: &OperatorStatistics, ) -> Result<(), AnalyticalCostError> { - require_consistent_edge_statistics(inputs)?; - if inputs.input_rows != child.output_rows || inputs.input_bytes != child.output_bytes { - return Err(AnalyticalCostError::InconsistentOperatorStatistics( - "unary input does not match child output", - )); + require_statistics_shape(node, statistics, 1)?; + if statistics.inputs[0] != child.output { + return Err(AnalyticalCostError::ConflictingEdgeStatistics { + parent: node.into(), + child: child_id.into(), + input_index: 0, + }); } Ok(()) } fn require_operator_statistics( operator: PhysicalOperator, - inputs: OperatorInputs, + statistics: &OperatorStatistics, ) -> Result<(), AnalyticalCostError> { let invalid = |reason| Err(AnalyticalCostError::InconsistentOperatorStatistics(reason)); + if !matches!(operator, PhysicalOperator::Scan) && statistics.source_scan_bytes != 0 { + return invalid("only Scan may charge source bytes"); + } + let input = statistics.inputs.first().copied().ok_or( + AnalyticalCostError::InconsistentOperatorStatistics("operator input is missing"), + )?; + let output = statistics.output; match operator { PhysicalOperator::Filter => { - if inputs.output_rows > inputs.input_rows || inputs.output_bytes > inputs.input_bytes { + if output.rows > input.rows || output.bytes > input.bytes { return invalid("filter output expands its input"); } } PhysicalOperator::Project => { - if inputs.output_rows != inputs.input_rows { + if output.rows != input.rows { return invalid("projection changes row cardinality"); } } PhysicalOperator::HashAggregate => { - let groups = inputs + let groups = statistics .group_count .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - if groups == 0 && (inputs.input_rows != 0 || inputs.output_rows != 0) { + if groups == 0 && (input.rows != 0 || output.rows != 0) { return invalid("zero groups require an empty grouped input and output"); } - if inputs.output_rows > groups { + if output.rows > groups { return invalid("aggregate output exceeds group cardinality"); } } PhysicalOperator::Deduplicate => { - let groups = inputs + let groups = statistics .group_count .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - if inputs.output_rows != groups || inputs.output_rows > inputs.input_rows { + if output.rows != groups || output.rows > input.rows { return invalid("deduplicate output does not equal distinct cardinality"); } } PhysicalOperator::Sort => { - if inputs.output_rows != inputs.input_rows || inputs.output_bytes != inputs.input_bytes - { + if output != input { return invalid("sort changes its input cardinality or width"); } } PhysicalOperator::TopK | PhysicalOperator::Limit => { - if inputs.output_rows > inputs.input_rows { + if output.rows > input.rows { return invalid("bounded output exceeds its input cardinality"); } } PhysicalOperator::Window => { - if inputs.output_rows != inputs.input_rows { + if output.rows != input.rows { return invalid("SQL window changes row cardinality"); } } PhysicalOperator::PassThrough => { - if inputs.output_rows != inputs.input_rows || inputs.output_bytes != inputs.input_bytes - { + if output != input { return invalid("pass-through wrapper changes its edge statistics"); } } @@ -397,12 +479,12 @@ fn require_operator_statistics( fn require_limit_cardinality( n: usize, offset: usize, - inputs: OperatorInputs, + statistics: &OperatorStatistics, ) -> Result<(), AnalyticalCostError> { let n = u64::try_from(n).map_err(|_| AnalyticalCostError::Overflow)?; let offset = u64::try_from(offset).map_err(|_| AnalyticalCostError::Overflow)?; - let expected = inputs.input_rows.saturating_sub(offset).min(n); - if inputs.output_rows != expected { + let expected = statistics.inputs[0].rows.saturating_sub(offset).min(n); + if statistics.output.rows != expected { return Err(AnalyticalCostError::InconsistentOperatorStatistics( "limit output does not match n and offset", )); @@ -410,14 +492,26 @@ fn require_limit_cardinality( Ok(()) } -fn clear_operator_specific_inputs(inputs: &mut OperatorInputs) { - inputs.group_count = None; - inputs.key_bytes = None; - inputs.aggregate_value_bytes = None; - inputs.k = None; - inputs.right_rows = None; - inputs.right_bytes = None; - inputs.hash_join_build_side = None; +fn bind_scan_coverage( + node_id: &str, + source: &asap_types::pre_asap::Source, + predicates: &[asap_types::pre_asap::Predicate], + scope: &ComparisonScope, +) -> Result { + let mut matches = scope + .sources + .iter() + .filter(|coverage| coverage.source == *source && coverage.predicates == predicates); + let coverage = matches + .next() + .cloned() + .ok_or_else(|| AnalyticalCostError::ScanOutsideComparisonScope(node_id.into()))?; + if matches.next().is_some() { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "scan source coverage is ambiguous", + )); + } + Ok(coverage) } fn is_hash_join_predicate(expr: &asap_types::pre_asap::QueryExpr) -> bool { @@ -441,28 +535,64 @@ fn is_hash_join_predicate(expr: &asap_types::pre_asap::QueryExpr) -> bool { mod tests { use super::*; use crate::analytical_cost::{estimate_physical_dag, HashJoinBuildSide}; + use asap_types::workload::{ + DataArrival, DurationMs, QueryRecurrence, QueryTimeScope, TimeSelection, TimestampMs, + }; + use std::collections::HashMap; - fn operator_inputs( - input_rows: u64, - input_bytes: u64, - output_rows: u64, - output_bytes: u64, - ) -> OperatorInputs { - OperatorInputs { - input_rows, - input_bytes, - output_rows, - output_bytes, + fn edge(rows: u64, bytes: u64) -> EdgeStatistics { + EdgeStatistics { rows, bytes } + } + + fn statistics(inputs: Vec, output: EdgeStatistics) -> OperatorStatistics { + OperatorStatistics { + source_scan_bytes: 0, + inputs, + output, group_count: None, key_bytes: None, aggregate_value_bytes: None, k: None, - right_rows: None, - right_bytes: None, hash_join_build_side: None, } } + fn evidence(statistics: OperatorStatistics) -> PhysicalNodeEvidence { + PhysicalNodeEvidence { + output_buffer_bytes: statistics.output.bytes.min(1_024), + statistics, + } + } + + fn scope(sources: Vec) -> ComparisonScope { + ComparisonScope { + data_arrival: DataArrival::AtRest, + planning_time: TimestampMs(1_000), + horizon: DurationMs(1_000), + recurrence: QueryRecurrence::OneTime { + invocations: 1, + execute_at: None, + }, + time_selection: TimeSelection { + scope: QueryTimeScope::Longitudinal, + lookback: Some(DurationMs(1_000)), + as_of: Some(TimestampMs(1_000)), + }, + sources, + } + } + + fn coverage( + source: asap_types::pre_asap::Source, + predicates: Vec, + ) -> SourceCoverage { + SourceCoverage { + source, + snapshot_id: "snapshot-1".into(), + predicates, + } + } + #[test] fn query_lowering_recurses_and_fuses_global_sort_limit() { use asap_types::pre_asap::{ @@ -501,21 +631,33 @@ mod tests { child: sort, }); - let dag = lower_query_physical_dag(&root, |node| { - let mut stats = match node { - QueryExpr::Scan { .. } => operator_inputs(1_000, 64_000, 400, 25_600), - QueryExpr::Aggregate { .. } => operator_inputs(400, 25_600, 100, 4_000), - QueryExpr::Limit { .. } => operator_inputs(100, 4_000, 10, 400), - _ => return None, - }; - if matches!(node, QueryExpr::Aggregate { .. }) { - stats.group_count = Some(100); - stats.key_bytes = Some(16); - stats.aggregate_value_bytes = Some(8); - } - Some(stats) - }) - .unwrap(); + let scan_coverage = coverage( + Source::Table { + table_ref: "events".into(), + }, + vec![asap_types::pre_asap::Predicate(Rc::new( + QueryExpr::Literal(asap_types::pre_asap::ScalarValue::Boolean(true)), + ))], + ); + let scope = scope(vec![scan_coverage]); + let mut aggregate_statistics = statistics(vec![edge(400, 25_600)], edge(100, 4_000)); + aggregate_statistics.group_count = Some(100); + aggregate_statistics.key_bytes = Some(16); + aggregate_statistics.aggregate_value_bytes = Some(8); + let mut topk_statistics = statistics(vec![edge(100, 4_000)], edge(10, 400)); + topk_statistics.k = Some(15); + let mut raw_scan = statistics(vec![edge(1_000, 64_000)], edge(1_000, 64_000)); + raw_scan.source_scan_bytes = 64_000; + let provided = HashMap::from([ + ("query-2-scan".into(), evidence(raw_scan)), + ( + "query-2".into(), + evidence(statistics(vec![edge(1_000, 64_000)], edge(400, 25_600))), + ), + ("query-1".into(), evidence(aggregate_statistics)), + ("query-0".into(), evidence(topk_statistics)), + ]); + let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); assert_eq!( dag.nodes @@ -530,9 +672,20 @@ mod tests { ] ); let topk = dag.nodes.last().unwrap(); - assert_eq!(topk.inputs.k, Some(15)); assert_eq!(topk.children, vec![dag.nodes[2].id.clone()]); - assert!(estimate_physical_dag(&dag.nodes, &dag.root, 3).is_ok()); + assert_eq!(provided[&topk.id].statistics.k, Some(15)); + let physical_scan = &dag.nodes[0]; + assert_eq!(physical_scan.id, "query-2-scan"); + assert_eq!( + physical_scan.source_coverage, + Some(scope.sources[0].clone()) + ); + assert_eq!(physical_scan.output_buffer_bytes, 1_024); + assert_ne!( + physical_scan.output_buffer_bytes, + provided[&physical_scan.id].statistics.output.bytes + ); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).is_ok()); } #[test] @@ -558,22 +711,26 @@ mod tests { left: Rc::clone(&shared), right: Rc::clone(&shared), }); - let dag = lower_query_physical_dag(&root, |node| match node { - QueryExpr::Scan { .. } => Some(operator_inputs(100, 800, 100, 800)), - QueryExpr::Join { .. } => { - let mut stats = operator_inputs(100, 800, 25, 400); - stats.right_rows = Some(100); - stats.right_bytes = Some(800); - stats.hash_join_build_side = Some(HashJoinBuildSide::Right); - Some(stats) - } - _ => None, - }) - .unwrap(); + let source_coverage = coverage( + Source::Table { + table_ref: "dimensions".into(), + }, + vec![], + ); + let scope = scope(vec![source_coverage]); + let mut scan_statistics = statistics(vec![edge(100, 800)], edge(100, 800)); + scan_statistics.source_scan_bytes = 800; + let mut join_statistics = statistics(vec![edge(100, 800), edge(100, 800)], edge(25, 400)); + join_statistics.hash_join_build_side = Some(HashJoinBuildSide::Right); + let provided = HashMap::from([ + ("query-1".into(), evidence(scan_statistics)), + ("query-0".into(), evidence(join_statistics)), + ]); + let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); assert_eq!(dag.nodes.len(), 2); assert_eq!(dag.nodes[1].children, vec![dag.nodes[0].id.clone(); 2]); - let estimate = estimate_physical_dag(&dag.nodes, &dag.root, 1).unwrap(); + let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).unwrap(); assert_eq!(estimate.scan_bytes, 800); } @@ -629,25 +786,47 @@ mod tests { child: limit, }); - let dag = lower_query_physical_dag(&root, |node| { - let mut inputs = match node { - QueryExpr::Scan { .. } => operator_inputs(1_000, 8_000, 1_000, 8_000), - QueryExpr::Filter { .. } => operator_inputs(1_000, 8_000, 800, 6_400), - QueryExpr::Project { .. } => operator_inputs(800, 6_400, 800, 3_200), - QueryExpr::Dedup { .. } => operator_inputs(800, 3_200, 500, 2_000), - QueryExpr::SQLWindowFunc { .. } => operator_inputs(500, 2_000, 500, 6_000), - QueryExpr::Sort { .. } => operator_inputs(500, 6_000, 500, 6_000), - QueryExpr::Limit { .. } => operator_inputs(500, 6_000, 20, 240), - QueryExpr::TimeShift { .. } => operator_inputs(20, 240, 20, 240), - _ => return None, - }; - if matches!(node, QueryExpr::Dedup { .. }) { - inputs.group_count = Some(500); - inputs.key_bytes = Some(8); - } - Some(inputs) - }) - .unwrap(); + let source_coverage = coverage( + Source::Table { + table_ref: "events".into(), + }, + vec![], + ); + let scope = scope(vec![source_coverage]); + let mut scan_statistics = statistics(vec![edge(1_000, 8_000)], edge(1_000, 8_000)); + scan_statistics.source_scan_bytes = 8_000; + let mut dedup_statistics = statistics(vec![edge(800, 3_200)], edge(500, 2_000)); + dedup_statistics.group_count = Some(500); + dedup_statistics.key_bytes = Some(8); + let provided = HashMap::from([ + ("query-7".into(), evidence(scan_statistics)), + ( + "query-6".into(), + evidence(statistics(vec![edge(1_000, 8_000)], edge(800, 6_400))), + ), + ( + "query-5".into(), + evidence(statistics(vec![edge(800, 6_400)], edge(800, 3_200))), + ), + ("query-4".into(), evidence(dedup_statistics)), + ( + "query-3".into(), + evidence(statistics(vec![edge(500, 2_000)], edge(500, 6_000))), + ), + ( + "query-2".into(), + evidence(statistics(vec![edge(500, 6_000)], edge(500, 6_000))), + ), + ( + "query-1".into(), + evidence(statistics(vec![edge(500, 6_000)], edge(20, 240))), + ), + ( + "query-0".into(), + evidence(statistics(vec![edge(20, 240)], edge(20, 240))), + ), + ]); + let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); assert_eq!( dag.nodes @@ -665,7 +844,7 @@ mod tests { PhysicalOperator::PassThrough, ] ); - assert!(estimate_physical_dag(&dag.nodes, &dag.root, 2).is_ok()); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).is_ok()); } #[test] @@ -687,31 +866,39 @@ mod tests { left: Rc::new(scan("a")), right: Rc::new(scan("b")), }); - let dag = lower_query_physical_dag(&union, |node| match node { - QueryExpr::Scan { - source: Source::Table { table_ref }, - .. - } if table_ref == "a" => Some(operator_inputs(10, 80, 10, 80)), - QueryExpr::Scan { .. } => Some(operator_inputs(20, 160, 20, 160)), - QueryExpr::SetOp { .. } => Some(operator_inputs(30, 240, 30, 240)), - _ => None, - }) - .unwrap(); + let scope = scope(vec![ + coverage( + Source::Table { + table_ref: "a".into(), + }, + vec![], + ), + coverage( + Source::Table { + table_ref: "b".into(), + }, + vec![], + ), + ]); + let mut left_statistics = statistics(vec![edge(10, 80)], edge(10, 80)); + left_statistics.source_scan_bytes = 80; + let mut right_statistics = statistics(vec![edge(20, 160)], edge(20, 160)); + right_statistics.source_scan_bytes = 160; + let provided = HashMap::from([ + ("query-1".into(), evidence(left_statistics)), + ("query-2".into(), evidence(right_statistics)), + ( + "query-0".into(), + evidence(statistics(vec![edge(10, 80), edge(20, 160)], edge(30, 240))), + ), + ]); + let dag = lower_query_physical_dag(&union, &scope, &provided).unwrap(); assert_eq!(dag.nodes.last().unwrap().operator, PhysicalOperator::Concat); let concat = Rc::new(QueryExpr::Concat { children: vec![scan("a"), scan("b")], }); - let dag = lower_query_physical_dag(&concat, |node| match node { - QueryExpr::Scan { - source: Source::Table { table_ref }, - .. - } if table_ref == "a" => Some(operator_inputs(10, 80, 10, 80)), - QueryExpr::Scan { .. } => Some(operator_inputs(20, 160, 20, 160)), - QueryExpr::Concat { .. } => Some(operator_inputs(30, 240, 30, 240)), - _ => None, - }) - .unwrap(); + let dag = lower_query_physical_dag(&concat, &scope, &provided).unwrap(); assert_eq!(dag.nodes.last().unwrap().operator, PhysicalOperator::Concat); let distinct_union = Rc::new(QueryExpr::SetOp { @@ -721,7 +908,7 @@ mod tests { right: Rc::new(scan("b")), }); assert_eq!( - lower_query_physical_dag(&distinct_union, |_| None), + lower_query_physical_dag(&distinct_union, &scope, &provided), Err(AnalyticalCostError::UnsupportedQueryOperator) ); } @@ -745,18 +932,70 @@ mod tests { child: scan, }); + let comparison_scope = scope(vec![coverage( + Source::Table { + table_ref: "events".into(), + }, + vec![], + )]); + let missing = HashMap::::new(); assert_eq!( - lower_query_physical_dag(&root, |_| None), - Err(AnalyticalCostError::MissingOrStale("operator_statistics")) + lower_query_physical_dag(&root, &comparison_scope, &missing), + Err(AnalyticalCostError::MissingOperatorStatistics( + "query-1".into() + )) ); + struct MissingBuffer; + impl PhysicalNodeEvidenceProvider for MissingBuffer { + fn evidence( + &self, + _node_id: &str, + ) -> Result { + Err(AnalyticalCostError::MissingOrStale("output_buffer_bytes")) + } + } assert_eq!( - lower_query_physical_dag(&root, |node| match node { - QueryExpr::Scan { .. } => Some(operator_inputs(100, 800, 100, 800)), - QueryExpr::Project { .. } => Some(operator_inputs(99, 792, 99, 396)), - _ => None, - }), - Err(AnalyticalCostError::InconsistentOperatorStatistics( - "unary input does not match child output" + lower_query_physical_dag(&root, &comparison_scope, &MissingBuffer), + Err(AnalyticalCostError::MissingOrStale("output_buffer_bytes")) + ); + let mut scan_statistics = statistics(vec![edge(100, 800)], edge(100, 800)); + scan_statistics.source_scan_bytes = 800; + let conflicting = HashMap::from([ + ("query-1".into(), evidence(scan_statistics)), + ( + "query-0".into(), + evidence(statistics(vec![edge(99, 792)], edge(99, 396))), + ), + ]); + assert_eq!( + lower_query_physical_dag(&root, &comparison_scope, &conflicting), + Err(AnalyticalCostError::ConflictingEdgeStatistics { + parent: "query-0".into(), + child: "query-1".into(), + input_index: 0, + }) + ); + + let outside_scope = scope(vec![coverage( + Source::Table { + table_ref: "other".into(), + }, + vec![], + )]); + assert_eq!( + lower_query_physical_dag(&root, &outside_scope, &conflicting), + Err(AnalyticalCostError::ScanOutsideComparisonScope( + "query-1".into() + )) + ); + + let mut second_snapshot = comparison_scope.sources[0].clone(); + second_snapshot.snapshot_id = "snapshot-2".into(); + let ambiguous_scope = scope(vec![comparison_scope.sources[0].clone(), second_snapshot]); + assert_eq!( + lower_query_physical_dag(&root, &ambiguous_scope, &conflicting), + Err(AnalyticalCostError::InvalidPhysicalDag( + "scan source coverage is ambiguous" )) ); } @@ -784,14 +1023,27 @@ mod tests { child: filter, }); - let dag = lower_query_physical_dag(&root, |node| match node { - QueryExpr::Scan { .. } => Some(operator_inputs(100, 800, 100, 800)), - QueryExpr::Filter { .. } => Some(operator_inputs(100, 800, 0, 0)), - QueryExpr::Limit { .. } => Some(operator_inputs(0, 0, 0, 0)), - _ => None, - }) - .unwrap(); - let estimate = estimate_physical_dag(&dag.nodes, &dag.root, 1).unwrap(); + let scope = scope(vec![coverage( + Source::Table { + table_ref: "events".into(), + }, + vec![], + )]); + let mut scan_statistics = statistics(vec![edge(100, 800)], edge(100, 800)); + scan_statistics.source_scan_bytes = 800; + let provided = HashMap::from([ + ("query-2".into(), evidence(scan_statistics)), + ( + "query-1".into(), + evidence(statistics(vec![edge(100, 800)], edge(0, 0))), + ), + ( + "query-0".into(), + evidence(statistics(vec![edge(0, 0)], edge(0, 0))), + ), + ]); + let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); + let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).unwrap(); assert_eq!(estimate.cpu_ops, 200.0); assert_eq!(estimate.scan_bytes, 800); } diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 0027b772..15772c33 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -236,9 +236,20 @@ deduplicated by physical identity. `lower_query_physical_dag` recursively lowers a resolved `Rc` and returns a `PhysicalDag` containing both its nodes and root ID. It consumes the existing query and physical-operator enums; it does not introduce a parallel -logical operator vocabulary. A statistics callback resolves the existing -`OperatorInputs` for each logical operator identity. Returning no statistics -makes the entire query unavailable. +logical operator vocabulary. A `PhysicalNodeEvidenceProvider` resolves one +atomic `PhysicalNodeEvidence` for each deterministic physical node ID. That +value reuses the authoritative `OperatorStatistics` contract and adds only +`output_buffer_bytes`, because logical edge bytes are not an allocation. +Missing evidence makes the entire query unavailable. + +Each lowered Scan is bound to exactly one `SourceCoverage` in the comparison +scope by the existing source and canonical predicate values. The bound value +therefore also supplies the provider-owned snapshot ID. Zero matches fail as +outside scope; multiple matching coverages fail as ambiguous rather than +choosing an arbitrary snapshot. When a predicate-bearing logical Scan expands +to Scan → Filter, the synthetic Scan has its own physical ID, statistics, and +buffer evidence and carries that exact coverage; the Filter has separate +evidence and no source coverage. The lowering validates every physical edge before costing: From 16c4cbf0490a0c590b726f24614f19a1e9ce595d Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 09:20:55 -0600 Subject: [PATCH 08/48] fix(cost): reject unproven query algorithms --- .../src/analytical_lowering.rs | 188 ++++++++++++++++-- .../analytical-resource-cost.md | 11 +- 2 files changed, 181 insertions(+), 18 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index 8a37600b..c67b7b26 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -156,6 +156,7 @@ pub fn lower_query_physical_dag( if predicates.is_empty() { let statistics = self.stats(&id)?; require_statistics_shape(&id, &statistics, 1)?; + require_scan_edges_equal(&statistics)?; return Ok(self.push( id, PhysicalOperator::Scan, @@ -167,6 +168,7 @@ pub fn lower_query_physical_dag( let scan_id = format!("{id}-scan"); let scan_statistics = self.stats(&scan_id)?; require_statistics_shape(&scan_id, &scan_statistics, 1)?; + require_scan_edges_equal(&scan_statistics)?; self.push( scan_id.clone(), PhysicalOperator::Scan, @@ -191,7 +193,16 @@ pub fn lower_query_physical_dag( QueryExpr::Project { child, .. } => { self.lower_unary(query, id, PhysicalOperator::Project, child) } - QueryExpr::Aggregate { child, .. } => { + QueryExpr::Aggregate { + reduction, + measures, + having, + child, + .. + } => { + if having.is_some() || !supports_hash_aggregate(reduction, measures) { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + } self.lower_unary(query, id, PhysicalOperator::HashAggregate, child) } QueryExpr::Dedup { child, .. } => { @@ -278,7 +289,7 @@ pub fn lower_query_physical_dag( right, } => { if matches!(kind, asap_types::pre_asap::JoinKind::Cross) - || !is_hash_join_predicate(&pred.0) + || !is_hash_join_predicate(&pred.0, left, right) { return Err(AnalyticalCostError::UnsupportedQueryOperator); } @@ -392,6 +403,15 @@ fn require_statistics_shape( Ok(()) } +fn require_scan_edges_equal(statistics: &OperatorStatistics) -> Result<(), AnalyticalCostError> { + if statistics.inputs[0] != statistics.output { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "Scan external input edge does not match its output edge", + )); + } + Ok(()) +} + fn require_unary_edge( node: &str, statistics: &OperatorStatistics, @@ -514,21 +534,80 @@ fn bind_scan_coverage( Ok(coverage) } -fn is_hash_join_predicate(expr: &asap_types::pre_asap::QueryExpr) -> bool { +fn is_hash_join_predicate( + expr: &asap_types::pre_asap::QueryExpr, + left: &asap_types::pre_asap::QueryExpr, + right: &asap_types::pre_asap::QueryExpr, +) -> bool { use asap_types::pre_asap::{CompareOpKind, QueryExpr}; - match expr { - QueryExpr::Compare { - left, - op: CompareOpKind::Eq, - right, - } => { - matches!(left.as_ref(), QueryExpr::Column(_)) - && matches!(right.as_ref(), QueryExpr::Column(_)) + let (Ok(left_schema), Ok(right_schema)) = (left.output_schema(), right.output_schema()) else { + return false; + }; + let left_width = left_schema.columns.len(); + let total_width = left_width.saturating_add(right_schema.columns.len()); + + fn column_side(column: usize, left_width: usize, total_width: usize) -> Option { + if column < left_width { + Some(false) + } else if column < total_width { + Some(true) + } else { + None } - QueryExpr::BoolAnd(parts) => !parts.is_empty() && parts.iter().all(is_hash_join_predicate), - _ => false, } + + fn predicate(expr: &QueryExpr, left_width: usize, total_width: usize) -> bool { + match expr { + QueryExpr::Compare { + left, + op: CompareOpKind::Eq, + right, + } => match (left.as_ref(), right.as_ref()) { + (QueryExpr::Column(left), QueryExpr::Column(right)) => matches!( + ( + column_side(*left, left_width, total_width), + column_side(*right, left_width, total_width) + ), + (Some(false), Some(true)) | (Some(true), Some(false)) + ), + _ => false, + }, + QueryExpr::BoolAnd(parts) => { + !parts.is_empty() + && parts + .iter() + .all(|part| predicate(part, left_width, total_width)) + } + _ => false, + } + } + + predicate(expr, left_width, total_width) +} + +fn supports_hash_aggregate( + reduction: &asap_types::pre_asap::Reduction, + measures: &[asap_types::pre_asap::AggIntent], +) -> bool { + use asap_types::pre_asap::{AggIntent, Reduction}; + + matches!(reduction, Reduction::Reduce(_)) + && !measures.is_empty() + && measures.iter().all(|intent| { + matches!( + intent, + AggIntent::Count { .. } + | AggIntent::Sum { .. } + | AggIntent::Min { .. } + | AggIntent::Max { .. } + | AggIntent::Avg { .. } + | AggIntent::StdDev { .. } + | AggIntent::Variance { .. } + | AggIntent::Group + | AggIntent::CountValues { .. } + ) + }) } #[cfg(test)] @@ -595,9 +674,7 @@ mod tests { #[test] fn query_lowering_recurses_and_fuses_global_sort_limit() { - use asap_types::pre_asap::{ - agg_intent::default_cardinality, GroupKeys, QueryExpr, Reduction, Source, - }; + use asap_types::pre_asap::{AggIntent, GroupKeys, QueryExpr, Reduction, Source}; use asap_types::pre_asap::{Column, DataType, Schema}; use std::rc::Rc; @@ -615,7 +692,7 @@ mod tests { }); let aggregate = Rc::new(QueryExpr::Aggregate { reduction: Reduction::by(vec![0]), - measures: vec![default_cardinality()], + measures: vec![AggIntent::Sum { col: Some(1) }], output_names: vec![], having: None, child: Rc::clone(&scan), @@ -686,6 +763,19 @@ mod tests { provided[&physical_scan.id].statistics.output.bytes ); assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).is_ok()); + + let mut inconsistent_scan = provided.clone(); + inconsistent_scan + .get_mut("query-2-scan") + .unwrap() + .statistics + .output = edge(999, 63_936); + assert_eq!( + lower_query_physical_dag(&root, &scope, &inconsistent_scan), + Err(AnalyticalCostError::InconsistentOperatorStatistics( + "Scan external input edge does not match its output edge" + )) + ); } #[test] @@ -732,6 +822,21 @@ mod tests { assert_eq!(dag.nodes[1].children, vec![dag.nodes[0].id.clone(); 2]); let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).unwrap(); assert_eq!(estimate.scan_bytes, 800); + + let invalid = Rc::new(QueryExpr::Join { + kind: JoinKind::Inner, + pred: Predicate(Rc::new(QueryExpr::Compare { + left: Rc::new(QueryExpr::Column(0)), + op: CompareOpKind::Eq, + right: Rc::new(QueryExpr::Column(0)), + })), + left: Rc::clone(&shared), + right: Rc::clone(&shared), + }); + assert_eq!( + lower_query_physical_dag(&invalid, &scope, &provided), + Err(AnalyticalCostError::UnsupportedQueryOperator) + ); } #[test] @@ -1047,4 +1152,53 @@ mod tests { assert_eq!(estimate.cpu_ops, 200.0); assert_eq!(estimate.scan_bytes, 800); } + + #[test] + fn query_lowering_rejects_aggregates_without_a_hash_implementation() { + use asap_types::pre_asap::{AggIntent, QueryExpr, Reduction, Source}; + use asap_types::pre_asap::{Column, DataType, Schema}; + use asap_types::types::AccuracyTarget; + use std::rc::Rc; + + let scan = || { + Rc::new(QueryExpr::Scan { + source: Source::Table { + table_ref: "events".into(), + }, + predicates: vec![], + schema: Schema::new(vec![Column::new("value", DataType::Float64, false)]), + }) + }; + let exact_quantile = Rc::new(QueryExpr::Aggregate { + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::Quantile { + col: Some(0), + q: 0.99, + accuracy: AccuracyTarget::Exact, + }], + output_names: vec![], + having: None, + child: scan(), + }); + let per_entity = Rc::new(QueryExpr::Aggregate { + reduction: Reduction::PerEntity, + measures: vec![AggIntent::Rate], + output_names: vec![], + having: None, + child: scan(), + }); + let scope = scope(vec![coverage( + Source::Table { + table_ref: "events".into(), + }, + vec![], + )]); + let unavailable = HashMap::::new(); + for query in [&exact_quantile, &per_entity] { + assert_eq!( + lower_query_physical_dag(query, &scope, &unavailable), + Err(AnalyticalCostError::UnsupportedQueryOperator) + ); + } + } } diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 15772c33..96ff2b9f 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -256,6 +256,8 @@ The lowering validates every physical edge before costing: - `(rows = 0, bytes = 0)` is a valid empty edge, while positive rows still require byte-width evidence and zero rows cannot carry non-zero bytes; - a unary operator's `input_rows` and `input_bytes` equal its child's output; +- a Scan's external logical input edge equals its output edge, including the + synthetic raw Scan created for a predicate-bearing logical Scan; - a hash join's left and right inputs equal the corresponding child outputs; - Concat and `UNION ALL` input/output totals equal the checked sum of all child outputs; and @@ -270,7 +272,7 @@ The supported mappings are: | Scan with pushed predicates | Scan → Filter | | Filter | Filter | | Project | Project | -| Aggregate, including a fused HAVING predicate | HashAggregate | +| Reducing Count/Sum/Min/Max/Avg/StdDev/Variance/Group/CountValues without HAVING | HashAggregate | | Dedup | Deduplicate | | Equi-Join | HashJoin with an evidence-selected build side | | Concat or `UNION ALL` | Concat | @@ -280,6 +282,13 @@ The supported mappings are: | SQLWindowFunc | Window | | TimeShift | PassThrough | +Per-entity reductions, HAVING, ordered/distribution-dependent intents such as +exact quantile or cardinality, Top-K aggregate intents, and extensions remain +unavailable until they have an explicit physical algorithm. Hash-join lowering +also uses the bound left and right output schemas to prove that every equality +compares one column from each side; same-side or out-of-range `ColumnId`s fail +closed. + Logical identity is the address of the existing `Rc` allocation. Repeated references therefore lower once and every parent points to the same physical ID. The resulting node IDs are deterministic within a lowering run; From cee41c3cfc68113d5456c041b54d97f2936da0f9 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 09:39:19 -0600 Subject: [PATCH 09/48] fix(cost): require provider-owned physical identity --- .../src/analytical_lowering.rs | 574 +++++++++++++----- .../analytical-resource-cost.md | 41 +- 2 files changed, 441 insertions(+), 174 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index c67b7b26..7c7cc7c7 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -18,6 +18,7 @@ use crate::analytical_statistics::{ pub struct PhysicalDag { pub nodes: Vec, pub root: String, + pub evidence: std::collections::HashMap, } /// Atomic evidence for one lowered physical node. The statistics contract is @@ -25,25 +26,44 @@ pub struct PhysicalDag { /// edge bytes cannot stand in for an allocation. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct PhysicalNodeEvidence { + pub physical_id: String, pub statistics: OperatorStatistics, pub output_buffer_bytes: u64, } +pub struct PhysicalNodeRequest<'a> { + pub logical_node: &'a asap_types::pre_asap::QueryExpr, + pub operator: PhysicalOperator, + pub occurrence: usize, + pub synthetic: bool, + pub children: &'a [String], + pub source_coverage: Option<&'a SourceCoverage>, +} + pub trait PhysicalNodeEvidenceProvider { - fn evidence(&self, node_id: &str) -> Result; + fn evidence( + &self, + request: PhysicalNodeRequest<'_>, + ) -> Result; } -impl PhysicalNodeEvidenceProvider for std::collections::HashMap { - fn evidence(&self, node_id: &str) -> Result { - self.get(node_id) - .cloned() - .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(node_id.into())) +impl PhysicalNodeEvidenceProvider for F +where + F: Fn(PhysicalNodeRequest<'_>) -> Result, +{ + fn evidence( + &self, + request: PhysicalNodeRequest<'_>, + ) -> Result { + self(request) } } impl OperatorStatisticsProvider for std::collections::HashMap { fn statistics(&self, node_id: &str) -> Result { - self.evidence(node_id).map(|evidence| evidence.statistics) + self.get(node_id) + .map(|evidence| evidence.statistics.clone()) + .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(node_id.into())) } } @@ -66,78 +86,99 @@ pub fn lower_query_physical_dag( struct Lowerer<'a> { scope: &'a ComparisonScope, provider: &'a dyn PhysicalNodeEvidenceProvider, - statistics: HashMap, - output_buffers: HashMap, - logical_roots: HashMap, + evidence: HashMap, next_id: usize, nodes: Vec, } impl Lowerer<'_> { fn lower(&mut self, query: &QueryExpr) -> Result { - let identity = std::ptr::from_ref(query) as usize; - if let Some(id) = self.logical_roots.get(&identity) { - return Ok(id.clone()); - } - let id = format!("query-{}", self.next_id); + let occurrence = self.next_id; self.next_id += 1; - // Insert only after successful lowering: a malformed recursive - // shape cannot leave a partially reusable node behind. - let root = self.lower_new(query, id)?; - self.logical_roots.insert(identity, root.clone()); - Ok(root) + self.lower_new(query, occurrence) } - fn stats(&mut self, id: &str) -> Result { - if let Some(statistics) = self.statistics.get(id) { - return Ok(statistics.clone()); + fn resolve( + &self, + query: &QueryExpr, + operator: PhysicalOperator, + occurrence: usize, + synthetic: bool, + children: &[String], + source_coverage: Option<&SourceCoverage>, + ) -> Result { + let evidence = self.provider.evidence(PhysicalNodeRequest { + logical_node: query, + operator, + occurrence, + synthetic, + children, + source_coverage, + })?; + if evidence.physical_id.is_empty() { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "provider returned an empty physical identity", + )); } - let evidence = self.provider.evidence(id)?; - let statistics = evidence.statistics; - self.statistics.insert(id.into(), statistics.clone()); - self.output_buffers - .insert(id.into(), evidence.output_buffer_bytes); - Ok(statistics) + Ok(evidence) } fn push( &mut self, - id: String, + evidence: PhysicalNodeEvidence, operator: PhysicalOperator, - _statistics: &OperatorStatistics, children: Vec, source_coverage: Option, - ) -> String { - self.nodes.push(PhysicalDagNode { + ) -> Result { + let id = evidence.physical_id.clone(); + let node = PhysicalDagNode { id: id.clone(), operator, children, source_coverage, - output_buffer_bytes: self.output_buffers[&id], + output_buffer_bytes: evidence.output_buffer_bytes, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, - }); - id + }; + if let Some(existing) = self.nodes.iter().find(|existing| existing.id == id) { + if existing != &node || self.evidence.get(&id) != Some(&evidence) { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "provider reused a physical identity for conflicting evidence", + )); + } + return Ok(id); + } + self.nodes.push(node); + self.evidence.insert(id.clone(), evidence); + Ok(id) } fn lower_unary( &mut self, - _query: &QueryExpr, - id: String, + query: &QueryExpr, + occurrence: usize, operator: PhysicalOperator, child: &QueryExpr, ) -> Result { let child_id = self.lower(child)?; - let statistics = self.stats(&id)?; + let children = vec![child_id.clone()]; + let evidence = self.resolve(query, operator, occurrence, false, &children, None)?; + let statistics = &evidence.statistics; let child_statistics = self.node_statistics(&child_id)?; - require_unary_edge(&id, &statistics, &child_id, child_statistics)?; - require_operator_statistics(operator, &statistics)?; - Ok(self.push(id, operator, &statistics, vec![child_id], None)) + require_unary_edge( + &evidence.physical_id, + statistics, + &child_id, + child_statistics, + )?; + require_operator_statistics(operator, statistics)?; + self.push(evidence, operator, children, None) } fn node_statistics(&self, id: &str) -> Result<&OperatorStatistics, AnalyticalCostError> { - self.statistics + self.evidence .get(id) + .map(|evidence| &evidence.statistics) .ok_or(AnalyticalCostError::InvalidPhysicalDag( "lowered child statistics are missing", )) @@ -146,52 +187,78 @@ pub fn lower_query_physical_dag( fn lower_new( &mut self, query: &QueryExpr, - id: String, + occurrence: usize, ) -> Result { match query { QueryExpr::Scan { source, predicates, .. } => { - let coverage = bind_scan_coverage(&id, source, predicates, self.scope)?; + let coverage = bind_scan_coverage( + &format!("occurrence-{occurrence}"), + source, + predicates, + self.scope, + )?; if predicates.is_empty() { - let statistics = self.stats(&id)?; - require_statistics_shape(&id, &statistics, 1)?; - require_scan_edges_equal(&statistics)?; - return Ok(self.push( - id, + let evidence = self.resolve( + query, PhysicalOperator::Scan, - &statistics, - vec![], - Some(coverage), - )); + occurrence, + false, + &[], + Some(&coverage), + )?; + require_statistics_shape(&evidence.physical_id, &evidence.statistics, 1)?; + require_scan_edges_equal(&evidence.statistics)?; + return self.push(evidence, PhysicalOperator::Scan, vec![], Some(coverage)); } - let scan_id = format!("{id}-scan"); - let scan_statistics = self.stats(&scan_id)?; - require_statistics_shape(&scan_id, &scan_statistics, 1)?; - require_scan_edges_equal(&scan_statistics)?; - self.push( - scan_id.clone(), + let scan_evidence = self.resolve( + query, + PhysicalOperator::Scan, + occurrence, + true, + &[], + Some(&coverage), + )?; + require_statistics_shape( + &scan_evidence.physical_id, + &scan_evidence.statistics, + 1, + )?; + require_scan_edges_equal(&scan_evidence.statistics)?; + let scan_statistics = scan_evidence.statistics.clone(); + let scan_id = self.push( + scan_evidence, PhysicalOperator::Scan, - &scan_statistics, vec![], Some(coverage), - ); - let filter_statistics = self.stats(&id)?; - require_unary_edge(&id, &filter_statistics, &scan_id, &scan_statistics)?; - require_operator_statistics(PhysicalOperator::Filter, &filter_statistics)?; - Ok(self.push( - id, + )?; + let children = vec![scan_id.clone()]; + let filter_evidence = self.resolve( + query, PhysicalOperator::Filter, - &filter_statistics, - vec![scan_id], + occurrence, + false, + &children, None, - )) + )?; + require_unary_edge( + &filter_evidence.physical_id, + &filter_evidence.statistics, + &scan_id, + &scan_statistics, + )?; + require_operator_statistics( + PhysicalOperator::Filter, + &filter_evidence.statistics, + )?; + self.push(filter_evidence, PhysicalOperator::Filter, children, None) } QueryExpr::Filter { child, .. } => { - self.lower_unary(query, id, PhysicalOperator::Filter, child) + self.lower_unary(query, occurrence, PhysicalOperator::Filter, child) } QueryExpr::Project { child, .. } => { - self.lower_unary(query, id, PhysicalOperator::Project, child) + self.lower_unary(query, occurrence, PhysicalOperator::Project, child) } QueryExpr::Aggregate { reduction, @@ -203,26 +270,44 @@ pub fn lower_query_physical_dag( if having.is_some() || !supports_hash_aggregate(reduction, measures) { return Err(AnalyticalCostError::UnsupportedQueryOperator); } - self.lower_unary(query, id, PhysicalOperator::HashAggregate, child) + self.lower_unary(query, occurrence, PhysicalOperator::HashAggregate, child) } QueryExpr::Dedup { child, .. } => { - self.lower_unary(query, id, PhysicalOperator::Deduplicate, child) + self.lower_unary(query, occurrence, PhysicalOperator::Deduplicate, child) } - QueryExpr::Sort { child, .. } => { - self.lower_unary(query, id, PhysicalOperator::Sort, child) + QueryExpr::Sort { keys, child, .. } => { + if keys.is_empty() { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + } + self.lower_unary(query, occurrence, PhysicalOperator::Sort, child) } QueryExpr::Limit { n, offset, child } => { if let QueryExpr::Sort { + keys, partition_by, child: sorted_child, .. } = child.as_ref() { - if partition_by == &GroupKeys::none() { + if !keys.is_empty() && partition_by == &GroupKeys::none() { let child_id = self.lower(sorted_child)?; - let statistics = self.stats(&id)?; + let children = vec![child_id.clone()]; + let evidence = self.resolve( + query, + PhysicalOperator::TopK, + occurrence, + false, + &children, + None, + )?; + let statistics = &evidence.statistics; let child_statistics = self.node_statistics(&child_id)?; - require_unary_edge(&id, &statistics, &child_id, child_statistics)?; + require_unary_edge( + &evidence.physical_id, + statistics, + &child_id, + child_statistics, + )?; let bound = n .checked_add(*offset) .and_then(|value| u64::try_from(value).ok()) @@ -235,42 +320,59 @@ pub fn lower_query_physical_dag( "Top-K statistics disagree with LIMIT n + offset", )); } - require_limit_cardinality(*n, *offset, &statistics)?; - return Ok(self.push( - id, - PhysicalOperator::TopK, - &statistics, - vec![child_id], - None, - )); + require_limit_cardinality(*n, *offset, statistics)?; + return self.push(evidence, PhysicalOperator::TopK, children, None); } } let child_id = self.lower(child)?; - let statistics = self.stats(&id)?; - let child_statistics = self.node_statistics(&child_id)?; - require_unary_edge(&id, &statistics, &child_id, child_statistics)?; - require_operator_statistics(PhysicalOperator::Limit, &statistics)?; - require_limit_cardinality(*n, *offset, &statistics)?; - Ok(self.push( - id, + let children = vec![child_id.clone()]; + let evidence = self.resolve( + query, PhysicalOperator::Limit, - &statistics, - vec![child_id], + occurrence, + false, + &children, None, - )) + )?; + let statistics = &evidence.statistics; + let child_statistics = self.node_statistics(&child_id)?; + require_unary_edge( + &evidence.physical_id, + statistics, + &child_id, + child_statistics, + )?; + require_operator_statistics(PhysicalOperator::Limit, statistics)?; + require_limit_cardinality(*n, *offset, statistics)?; + self.push(evidence, PhysicalOperator::Limit, children, None) } - QueryExpr::SQLWindowFunc { child, .. } => { - self.lower_unary(query, id, PhysicalOperator::Window, child) + QueryExpr::SQLWindowFunc { + func, + order_by, + child, + .. + } => { + if order_by.is_empty() + || !matches!( + func, + asap_types::pre_asap::WindowFuncKind::RowNumber + | asap_types::pre_asap::WindowFuncKind::Rank + | asap_types::pre_asap::WindowFuncKind::DenseRank + ) + { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + } + self.lower_unary(query, occurrence, PhysicalOperator::Window, child) } QueryExpr::TimeShift { child, .. } => { - self.lower_unary(query, id, PhysicalOperator::PassThrough, child) + self.lower_unary(query, occurrence, PhysicalOperator::PassThrough, child) } QueryExpr::Concat { children } => { let child_ids = children .iter() .map(|child| self.lower(child)) .collect::, _>>()?; - self.lower_concat(query, id, child_ids) + self.lower_concat(query, occurrence, child_ids) } QueryExpr::SetOp { kind: SetOpKind::Union, @@ -280,7 +382,7 @@ pub fn lower_query_physical_dag( } => { let left_id = self.lower(left)?; let right_id = self.lower(right)?; - self.lower_concat(query, id, vec![left_id, right_id]) + self.lower_concat(query, occurrence, vec![left_id, right_id]) } QueryExpr::Join { kind, @@ -295,8 +397,17 @@ pub fn lower_query_physical_dag( } let left_id = self.lower(left)?; let right_id = self.lower(right)?; - let statistics = self.stats(&id)?; - require_statistics_shape(&id, &statistics, 2)?; + let children = vec![left_id.clone(), right_id.clone()]; + let evidence = self.resolve( + query, + PhysicalOperator::HashJoin, + occurrence, + false, + &children, + None, + )?; + let statistics = &evidence.statistics; + require_statistics_shape(&evidence.physical_id, statistics, 2)?; let left_statistics = self.node_statistics(&left_id)?; let right_statistics = self.node_statistics(&right_id)?; if statistics.inputs[0] != left_statistics.output @@ -306,14 +417,8 @@ pub fn lower_query_physical_dag( "join inputs do not match child outputs", )); } - require_operator_statistics(PhysicalOperator::HashJoin, &statistics)?; - Ok(self.push( - id, - PhysicalOperator::HashJoin, - &statistics, - vec![left_id, right_id], - None, - )) + require_operator_statistics(PhysicalOperator::HashJoin, statistics)?; + self.push(evidence, PhysicalOperator::HashJoin, children, None) } _ => Err(AnalyticalCostError::UnsupportedQueryOperator), } @@ -321,8 +426,8 @@ pub fn lower_query_physical_dag( fn lower_concat( &mut self, - _query: &QueryExpr, - id: String, + query: &QueryExpr, + occurrence: usize, child_ids: Vec, ) -> Result { if child_ids.is_empty() { @@ -330,15 +435,23 @@ pub fn lower_query_physical_dag( "concat has no children", )); } - let statistics = self.stats(&id)?; - require_statistics_shape(&id, &statistics, child_ids.len())?; + let evidence = self.resolve( + query, + PhysicalOperator::Concat, + occurrence, + false, + &child_ids, + None, + )?; + let statistics = &evidence.statistics; + require_statistics_shape(&evidence.physical_id, statistics, child_ids.len())?; let (rows, bytes) = child_ids.iter().enumerate().try_fold( (0_u64, 0_u64), |(rows, bytes), (index, child)| { let child_statistics = self.node_statistics(child)?; if statistics.inputs[index] != child_statistics.output { return Err(AnalyticalCostError::ConflictingEdgeStatistics { - parent: id.clone(), + parent: evidence.physical_id.clone(), child: child.clone(), input_index: index, }); @@ -357,27 +470,52 @@ pub fn lower_query_physical_dag( "concat statistics do not equal the sum of child outputs", )); } - require_operator_statistics(PhysicalOperator::Concat, &statistics)?; - Ok(self.push(id, PhysicalOperator::Concat, &statistics, child_ids, None)) + require_operator_statistics(PhysicalOperator::Concat, statistics)?; + self.push(evidence, PhysicalOperator::Concat, child_ids, None) } } let mut lowerer = Lowerer { scope, provider: evidence, - statistics: HashMap::new(), - output_buffers: HashMap::new(), - logical_roots: HashMap::new(), + evidence: HashMap::new(), next_id: 0, nodes: Vec::new(), }; let root = lowerer.lower(root)?; + validate_source_consumption(&lowerer.nodes, scope)?; Ok(PhysicalDag { nodes: lowerer.nodes, root, + evidence: lowerer.evidence, }) } +fn validate_source_consumption( + nodes: &[PhysicalDagNode], + scope: &ComparisonScope, +) -> Result<(), AnalyticalCostError> { + let mut remaining = scope.sources.clone(); + for coverage in nodes + .iter() + .filter(|node| matches!(node.operator, PhysicalOperator::Scan)) + .filter_map(|node| node.source_coverage.as_ref()) + { + let Some(index) = remaining.iter().position(|expected| expected == coverage) else { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "physical scans do not consume comparison sources exactly", + )); + }; + remaining.swap_remove(index); + } + if !remaining.is_empty() { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "physical scans do not consume comparison sources exactly", + )); + } + Ok(()) +} + fn require_statistics_shape( node: &str, statistics: &OperatorStatistics, @@ -526,7 +664,7 @@ fn bind_scan_coverage( .next() .cloned() .ok_or_else(|| AnalyticalCostError::ScanOutsideComparisonScope(node_id.into()))?; - if matches.next().is_some() { + if matches.any(|candidate| candidate != &coverage) { return Err(AnalyticalCostError::InvalidPhysicalDag( "scan source coverage is ambiguous", )); @@ -638,11 +776,31 @@ mod tests { fn evidence(statistics: OperatorStatistics) -> PhysicalNodeEvidence { PhysicalNodeEvidence { + physical_id: String::new(), output_buffer_bytes: statistics.output.bytes.min(1_024), statistics, } } + fn scripted<'a>( + provided: &'a HashMap, + ) -> impl Fn(PhysicalNodeRequest<'_>) -> Result + 'a + { + move |request| { + let key = if request.synthetic { + format!("query-{}-scan", request.occurrence) + } else { + format!("query-{}", request.occurrence) + }; + let mut evidence = provided + .get(&key) + .cloned() + .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(key.clone()))?; + evidence.physical_id = key; + Ok(evidence) + } + } + fn scope(sources: Vec) -> ComparisonScope { ComparisonScope { data_arrival: DataArrival::AtRest, @@ -674,7 +832,7 @@ mod tests { #[test] fn query_lowering_recurses_and_fuses_global_sort_limit() { - use asap_types::pre_asap::{AggIntent, GroupKeys, QueryExpr, Reduction, Source}; + use asap_types::pre_asap::{AggIntent, GroupKeys, QueryExpr, Reduction, SortKey, Source}; use asap_types::pre_asap::{Column, DataType, Schema}; use std::rc::Rc; @@ -698,7 +856,11 @@ mod tests { child: Rc::clone(&scan), }); let sort = Rc::new(QueryExpr::Sort { - keys: vec![], + keys: vec![SortKey { + expr: QueryExpr::Column(0), + ascending: false, + nulls_first: false, + }], partition_by: GroupKeys::none(), child: aggregate, }); @@ -734,7 +896,7 @@ mod tests { ("query-1".into(), evidence(aggregate_statistics)), ("query-0".into(), evidence(topk_statistics)), ]); - let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); + let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)).unwrap(); assert_eq!( dag.nodes @@ -762,7 +924,7 @@ mod tests { physical_scan.output_buffer_bytes, provided[&physical_scan.id].statistics.output.bytes ); - assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).is_ok()); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).is_ok()); let mut inconsistent_scan = provided.clone(); inconsistent_scan @@ -771,7 +933,7 @@ mod tests { .statistics .output = edge(999, 63_936); assert_eq!( - lower_query_physical_dag(&root, &scope, &inconsistent_scan), + lower_query_physical_dag(&root, &scope, &scripted(&inconsistent_scan)), Err(AnalyticalCostError::InconsistentOperatorStatistics( "Scan external input edge does not match its output edge" )) @@ -779,7 +941,7 @@ mod tests { } #[test] - fn query_lowering_deduplicates_shared_rc_children() { + fn query_lowering_shares_only_provider_identified_physical_nodes() { use asap_types::pre_asap::{Column, CompareOpKind, DataType, Schema}; use asap_types::pre_asap::{JoinKind, Predicate, QueryExpr, Source}; use std::rc::Rc; @@ -807,21 +969,67 @@ mod tests { }, vec![], ); - let scope = scope(vec![source_coverage]); + let independent_scope = scope(vec![source_coverage.clone(), source_coverage.clone()]); let mut scan_statistics = statistics(vec![edge(100, 800)], edge(100, 800)); scan_statistics.source_scan_bytes = 800; let mut join_statistics = statistics(vec![edge(100, 800), edge(100, 800)], edge(25, 400)); join_statistics.hash_join_build_side = Some(HashJoinBuildSide::Right); let provided = HashMap::from([ - ("query-1".into(), evidence(scan_statistics)), - ("query-0".into(), evidence(join_statistics)), + ("query-1".into(), evidence(scan_statistics.clone())), + ("query-2".into(), evidence(scan_statistics.clone())), + ("query-0".into(), evidence(join_statistics.clone())), ]); - let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); + let dag = + lower_query_physical_dag(&root, &independent_scope, &scripted(&provided)).unwrap(); + + assert_eq!(dag.nodes.len(), 3); + assert_ne!(dag.nodes[2].children[0], dag.nodes[2].children[1]); + let estimate = + estimate_physical_dag(&dag.nodes, &dag.root, &independent_scope, &dag.evidence) + .unwrap(); + assert_eq!(estimate.scan_bytes, 1_600); + + let shared_provider = |request: PhysicalNodeRequest<'_>| { + let (physical_id, statistics) = match request.operator { + PhysicalOperator::Scan => ("shared-scan", scan_statistics.clone()), + PhysicalOperator::HashJoin => ("join", join_statistics.clone()), + _ => return Err(AnalyticalCostError::UnsupportedQueryOperator), + }; + Ok(PhysicalNodeEvidence { + physical_id: physical_id.into(), + output_buffer_bytes: statistics.output.bytes.min(1_024), + statistics, + }) + }; + let shared_scope = scope(vec![source_coverage]); + let shared_dag = lower_query_physical_dag(&root, &shared_scope, &shared_provider).unwrap(); + assert_eq!(shared_dag.nodes.len(), 2); + assert_eq!( + shared_dag.nodes[1].children, + vec!["shared-scan".to_owned(); 2] + ); - assert_eq!(dag.nodes.len(), 2); - assert_eq!(dag.nodes[1].children, vec![dag.nodes[0].id.clone(); 2]); - let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).unwrap(); - assert_eq!(estimate.scan_bytes, 800); + let conflicting_identity = |request: PhysicalNodeRequest<'_>| { + let (physical_id, mut statistics) = match request.operator { + PhysicalOperator::Scan => ("shared-scan", scan_statistics.clone()), + PhysicalOperator::HashJoin => ("join", join_statistics.clone()), + _ => return Err(AnalyticalCostError::UnsupportedQueryOperator), + }; + if request.operator == PhysicalOperator::Scan && request.occurrence == 2 { + statistics.source_scan_bytes += 1; + } + Ok(PhysicalNodeEvidence { + physical_id: physical_id.into(), + output_buffer_bytes: statistics.output.bytes.min(1_024), + statistics, + }) + }; + assert_eq!( + lower_query_physical_dag(&root, &shared_scope, &conflicting_identity), + Err(AnalyticalCostError::InvalidPhysicalDag( + "provider reused a physical identity for conflicting evidence" + )) + ); let invalid = Rc::new(QueryExpr::Join { kind: JoinKind::Inner, @@ -834,7 +1042,7 @@ mod tests { right: Rc::clone(&shared), }); assert_eq!( - lower_query_physical_dag(&invalid, &scope, &provided), + lower_query_physical_dag(&invalid, &shared_scope, &shared_provider), Err(AnalyticalCostError::UnsupportedQueryOperator) ); } @@ -843,7 +1051,7 @@ mod tests { fn query_lowering_covers_relational_unary_operators() { use asap_types::pre_asap::{Column, DataType, ScalarValue, Schema}; use asap_types::pre_asap::{ - GroupKeys, Predicate, QueryExpr, Source, TimeShift, WindowFuncKind, + GroupKeys, Predicate, QueryExpr, SortKey, Source, TimeShift, WindowFuncKind, }; use std::rc::Rc; @@ -871,13 +1079,21 @@ mod tests { func: WindowFuncKind::RowNumber, args: vec![], partition_by: GroupKeys::none(), - order_by: vec![], + order_by: vec![SortKey { + expr: QueryExpr::Column(0), + ascending: true, + nulls_first: false, + }], frame: None, output_name: "rn".into(), child: dedup, }); let sort = Rc::new(QueryExpr::Sort { - keys: vec![], + keys: vec![SortKey { + expr: QueryExpr::Column(0), + ascending: true, + nulls_first: false, + }], partition_by: GroupKeys::by(vec![0]), child: window, }); @@ -931,7 +1147,7 @@ mod tests { evidence(statistics(vec![edge(20, 240)], edge(20, 240))), ), ]); - let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); + let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)).unwrap(); assert_eq!( dag.nodes @@ -949,7 +1165,7 @@ mod tests { PhysicalOperator::PassThrough, ] ); - assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).is_ok()); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).is_ok()); } #[test] @@ -997,13 +1213,13 @@ mod tests { evidence(statistics(vec![edge(10, 80), edge(20, 160)], edge(30, 240))), ), ]); - let dag = lower_query_physical_dag(&union, &scope, &provided).unwrap(); + let dag = lower_query_physical_dag(&union, &scope, &scripted(&provided)).unwrap(); assert_eq!(dag.nodes.last().unwrap().operator, PhysicalOperator::Concat); let concat = Rc::new(QueryExpr::Concat { children: vec![scan("a"), scan("b")], }); - let dag = lower_query_physical_dag(&concat, &scope, &provided).unwrap(); + let dag = lower_query_physical_dag(&concat, &scope, &scripted(&provided)).unwrap(); assert_eq!(dag.nodes.last().unwrap().operator, PhysicalOperator::Concat); let distinct_union = Rc::new(QueryExpr::SetOp { @@ -1013,7 +1229,7 @@ mod tests { right: Rc::new(scan("b")), }); assert_eq!( - lower_query_physical_dag(&distinct_union, &scope, &provided), + lower_query_physical_dag(&distinct_union, &scope, &scripted(&provided)), Err(AnalyticalCostError::UnsupportedQueryOperator) ); } @@ -1045,7 +1261,7 @@ mod tests { )]); let missing = HashMap::::new(); assert_eq!( - lower_query_physical_dag(&root, &comparison_scope, &missing), + lower_query_physical_dag(&root, &comparison_scope, &scripted(&missing)), Err(AnalyticalCostError::MissingOperatorStatistics( "query-1".into() )) @@ -1054,7 +1270,7 @@ mod tests { impl PhysicalNodeEvidenceProvider for MissingBuffer { fn evidence( &self, - _node_id: &str, + _request: PhysicalNodeRequest<'_>, ) -> Result { Err(AnalyticalCostError::MissingOrStale("output_buffer_bytes")) } @@ -1073,7 +1289,7 @@ mod tests { ), ]); assert_eq!( - lower_query_physical_dag(&root, &comparison_scope, &conflicting), + lower_query_physical_dag(&root, &comparison_scope, &scripted(&conflicting)), Err(AnalyticalCostError::ConflictingEdgeStatistics { parent: "query-0".into(), child: "query-1".into(), @@ -1088,9 +1304,9 @@ mod tests { vec![], )]); assert_eq!( - lower_query_physical_dag(&root, &outside_scope, &conflicting), + lower_query_physical_dag(&root, &outside_scope, &scripted(&conflicting)), Err(AnalyticalCostError::ScanOutsideComparisonScope( - "query-1".into() + "occurrence-1".into() )) ); @@ -1098,11 +1314,30 @@ mod tests { second_snapshot.snapshot_id = "snapshot-2".into(); let ambiguous_scope = scope(vec![comparison_scope.sources[0].clone(), second_snapshot]); assert_eq!( - lower_query_physical_dag(&root, &ambiguous_scope, &conflicting), + lower_query_physical_dag(&root, &ambiguous_scope, &scripted(&conflicting)), Err(AnalyticalCostError::InvalidPhysicalDag( "scan source coverage is ambiguous" )) ); + + let mut complete = conflicting.clone(); + complete.get_mut("query-0").unwrap().statistics = + statistics(vec![edge(100, 800)], edge(100, 400)); + let extra_scope = scope(vec![ + comparison_scope.sources[0].clone(), + coverage( + Source::Table { + table_ref: "unused".into(), + }, + vec![], + ), + ]); + assert_eq!( + lower_query_physical_dag(&root, &extra_scope, &scripted(&complete)), + Err(AnalyticalCostError::InvalidPhysicalDag( + "physical scans do not consume comparison sources exactly" + )) + ); } #[test] @@ -1147,15 +1382,17 @@ mod tests { evidence(statistics(vec![edge(0, 0)], edge(0, 0))), ), ]); - let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); - let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).unwrap(); + let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)).unwrap(); + let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).unwrap(); assert_eq!(estimate.cpu_ops, 200.0); assert_eq!(estimate.scan_bytes, 800); } #[test] fn query_lowering_rejects_aggregates_without_a_hash_implementation() { - use asap_types::pre_asap::{AggIntent, QueryExpr, Reduction, Source}; + use asap_types::pre_asap::{ + AggIntent, GroupKeys, QueryExpr, Reduction, Source, WindowFuncKind, + }; use asap_types::pre_asap::{Column, DataType, Schema}; use asap_types::types::AccuracyTarget; use std::rc::Rc; @@ -1187,6 +1424,24 @@ mod tests { having: None, child: scan(), }); + let empty_sort_limit = Rc::new(QueryExpr::Limit { + n: 10, + offset: 0, + child: Rc::new(QueryExpr::Sort { + keys: vec![], + partition_by: GroupKeys::none(), + child: scan(), + }), + }); + let unsupported_window = Rc::new(QueryExpr::SQLWindowFunc { + func: WindowFuncKind::Lag, + args: vec![QueryExpr::Column(0)], + partition_by: GroupKeys::none(), + order_by: vec![], + frame: None, + output_name: "lag".into(), + child: scan(), + }); let scope = scope(vec![coverage( Source::Table { table_ref: "events".into(), @@ -1194,9 +1449,14 @@ mod tests { vec![], )]); let unavailable = HashMap::::new(); - for query in [&exact_quantile, &per_entity] { + for query in [ + &exact_quantile, + &per_entity, + &empty_sort_limit, + &unsupported_window, + ] { assert_eq!( - lower_query_physical_dag(query, &scope, &unavailable), + lower_query_physical_dag(query, &scope, &scripted(&unavailable)), Err(AnalyticalCostError::UnsupportedQueryOperator) ); } diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 96ff2b9f..47cbfec6 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -236,11 +236,16 @@ deduplicated by physical identity. `lower_query_physical_dag` recursively lowers a resolved `Rc` and returns a `PhysicalDag` containing both its nodes and root ID. It consumes the existing query and physical-operator enums; it does not introduce a parallel -logical operator vocabulary. A `PhysicalNodeEvidenceProvider` resolves one -atomic `PhysicalNodeEvidence` for each deterministic physical node ID. That -value reuses the authoritative `OperatorStatistics` contract and adds only -`output_buffer_bytes`, because logical edge bytes are not an allocation. -Missing evidence makes the entire query unavailable. +logical operator vocabulary. For every occurrence, the lowerer sends a +`PhysicalNodeRequest` containing the logical node, selected existing +`PhysicalOperator`, occurrence and synthetic-role metadata, already-lowered +child physical IDs, and any source coverage to a +`PhysicalNodeEvidenceProvider`. The provider atomically returns its own stable +`physical_id`, the authoritative `OperatorStatistics`, and explicit +`output_buffer_bytes`; logical edge bytes are never substituted for an +allocation. Missing evidence makes the entire query unavailable. The returned +`PhysicalDag` snapshots this evidence so costing does not re-read a live +catalog after lowering. Each lowered Scan is bound to exactly one `SourceCoverage` in the comparison scope by the existing source and canonical predicate values. The bound value @@ -250,6 +255,10 @@ choosing an arbitrary snapshot. When a predicate-bearing logical Scan expands to Scan → Filter, the synthetic Scan has its own physical ID, statistics, and buffer evidence and carries that exact coverage; the Filter has separate evidence and no source coverage. +After lowering, the multiset of distinct physical Scan coverages must consume +the comparison scope's source coverages exactly. Thus a candidate cannot omit +a source that exists in the comparison boundary, while a provider-declared +shared physical Scan is counted once. The lowering validates every physical edge before costing: @@ -276,10 +285,10 @@ The supported mappings are: | Dedup | Deduplicate | | Equi-Join | HashJoin with an evidence-selected build side | | Concat or `UNION ALL` | Concat | -| Sort | in-memory Sort | -| global Sort followed by Limit | heap TopK, with `k = offset + n` from the query IR | +| Sort with at least one ordering key | in-memory Sort | +| global non-empty-key Sort followed by Limit | heap TopK, with `k = offset + n` from the query IR | | partitioned Sort followed by Limit | Sort → Limit | -| SQLWindowFunc | Window | +| RowNumber/Rank/DenseRank SQLWindowFunc with non-empty order_by | ordered in-memory Window | | TimeShift | PassThrough | Per-entity reductions, HAVING, ordered/distribution-dependent intents such as @@ -289,15 +298,13 @@ also uses the bound left and right output schemas to prove that every equality compares one column from each side; same-side or out-of-range `ColumnId`s fail closed. -Logical identity is the address of the existing `Rc` allocation. -Repeated references therefore lower once and every parent points to the same -physical ID. The resulting node IDs are deterministic within a lowering run; -they are not persistent query identifiers. - -This generic lowering currently creates raw-query operators, all with -`ExecutionMultiplicity::PerEvaluation`, zero retained state, and a conservative -full-output edge buffer. A deployment with verified batching may construct -`PhysicalDagNode` values with smaller `output_buffer_bytes` directly. +An `Rc` address is not physical identity. Every logical occurrence +is independent unless the provider returns the same non-empty `physical_id`. +Repeated IDs deduplicate only when operator, children, coverage, statistics, +and buffer evidence are identical; conflicting reuse fails closed. This +generic lowering creates raw-query operators with +`ExecutionMultiplicity::PerEvaluation` and zero retained state. Buffer sizes +are always provider-owned physical evidence. Cross/non-equi joins, `INTERSECT`, `EXCEPT`, distinct `UNION`, PromQL range/subquery execution, vector matching, and PromQL-specific enrichment/ From 9f979d10980b82873da0bc7e0b3df8d881fdd6b5 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 09:46:40 -0600 Subject: [PATCH 10/48] fix(cost): compare semantic source coverage --- .../src/analytical_lowering.rs | 126 +++++++++++++++--- .../src/analytical_statistics.rs | 19 ++- .../analytical-resource-cost.md | 16 ++- 3 files changed, 141 insertions(+), 20 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index 7c7cc7c7..a824412b 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -61,9 +61,38 @@ where impl OperatorStatisticsProvider for std::collections::HashMap { fn statistics(&self, node_id: &str) -> Result { - self.get(node_id) - .map(|evidence| evidence.statistics.clone()) - .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(node_id.into())) + let evidence = self + .get(node_id) + .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(node_id.into()))?; + if evidence.physical_id != node_id { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "evidence map key differs from embedded physical identity", + )); + } + Ok(evidence.statistics.clone()) + } +} + +impl OperatorStatisticsProvider for PhysicalDag { + fn statistics(&self, node_id: &str) -> Result { + let evidence = self + .evidence + .get(node_id) + .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(node_id.into()))?; + let node = self.nodes.iter().find(|node| node.id == node_id).ok_or( + AnalyticalCostError::InvalidPhysicalDag("evidence has no matching physical node"), + )?; + if evidence.physical_id != node_id { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "evidence map key differs from embedded physical identity", + )); + } + if node.output_buffer_bytes != evidence.output_buffer_bytes { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "physical node buffer differs from evidence snapshot", + )); + } + Ok(evidence.statistics.clone()) } } @@ -495,22 +524,25 @@ fn validate_source_consumption( nodes: &[PhysicalDagNode], scope: &ComparisonScope, ) -> Result<(), AnalyticalCostError> { - let mut remaining = scope.sources.clone(); - for coverage in nodes + let consumed = nodes .iter() .filter(|node| matches!(node.operator, PhysicalOperator::Scan)) .filter_map(|node| node.source_coverage.as_ref()) - { - let Some(index) = remaining.iter().position(|expected| expected == coverage) else { + .collect::>(); + for coverage in &consumed { + if !scope.sources.contains(coverage) { return Err(AnalyticalCostError::InvalidPhysicalDag( - "physical scans do not consume comparison sources exactly", + "physical scan consumes a source outside the comparison scope", )); - }; - remaining.swap_remove(index); + } } - if !remaining.is_empty() { + if scope + .sources + .iter() + .any(|expected| !consumed.contains(&expected)) + { return Err(AnalyticalCostError::InvalidPhysicalDag( - "physical scans do not consume comparison sources exactly", + "physical scans omit a comparison-scope source", )); } Ok(()) @@ -751,7 +783,11 @@ fn supports_hash_aggregate( #[cfg(test)] mod tests { use super::*; - use crate::analytical_cost::{estimate_physical_dag, HashJoinBuildSide}; + use crate::analytical_cost::{ + estimate_physical_dag, estimate_physical_dag_comparison, HashJoinBuildSide, + PhysicalDagEstimateRequest, + }; + use crate::analytical_statistics::validate_comparison_scopes; use asap_types::workload::{ DataArrival, DurationMs, QueryRecurrence, QueryTimeScope, TimeSelection, TimestampMs, }; @@ -969,7 +1005,7 @@ mod tests { }, vec![], ); - let independent_scope = scope(vec![source_coverage.clone(), source_coverage.clone()]); + let independent_scope = scope(vec![source_coverage.clone()]); let mut scan_statistics = statistics(vec![edge(100, 800)], edge(100, 800)); scan_statistics.source_scan_bytes = 800; let mut join_statistics = statistics(vec![edge(100, 800), edge(100, 800)], edge(25, 400)); @@ -1008,6 +1044,54 @@ mod tests { shared_dag.nodes[1].children, vec!["shared-scan".to_owned(); 2] ); + let comparison = estimate_physical_dag_comparison( + PhysicalDagEstimateRequest { + nodes: &dag.nodes, + root: &dag.root, + scope: &independent_scope, + statistics: &dag, + }, + PhysicalDagEstimateRequest { + nodes: &shared_dag.nodes, + root: &shared_dag.root, + scope: &shared_scope, + statistics: &shared_dag, + }, + ) + .unwrap(); + assert_eq!(comparison.raw.scan_bytes, 1_600); + assert_eq!(comparison.candidate.scan_bytes, 800); + + let mut drifted_buffer = shared_dag.clone(); + drifted_buffer.nodes[0].output_buffer_bytes += 1; + assert_eq!( + estimate_physical_dag( + &drifted_buffer.nodes, + &drifted_buffer.root, + &shared_scope, + &drifted_buffer, + ), + Err(AnalyticalCostError::InvalidPhysicalDag( + "physical node buffer differs from evidence snapshot" + )) + ); + let mut drifted_identity = shared_dag.clone(); + drifted_identity + .evidence + .get_mut("shared-scan") + .unwrap() + .physical_id = "different".into(); + assert_eq!( + estimate_physical_dag( + &drifted_identity.nodes, + &drifted_identity.root, + &shared_scope, + &drifted_identity, + ), + Err(AnalyticalCostError::InvalidPhysicalDag( + "evidence map key differs from embedded physical identity" + )) + ); let conflicting_identity = |request: PhysicalNodeRequest<'_>| { let (physical_id, mut statistics) = match request.operator { @@ -1216,6 +1300,18 @@ mod tests { let dag = lower_query_physical_dag(&union, &scope, &scripted(&provided)).unwrap(); assert_eq!(dag.nodes.last().unwrap().operator, PhysicalOperator::Concat); + let mut reversed_scope = scope.clone(); + reversed_scope.sources.reverse(); + assert_eq!(validate_comparison_scopes(&scope, &reversed_scope), Ok(1)); + let mut duplicate_scope = scope.clone(); + duplicate_scope.sources.push(scope.sources[0].clone()); + assert_eq!( + duplicate_scope.validate(), + Err(AnalyticalCostError::MissingComparisonScope( + "duplicate source coverage" + )) + ); + let concat = Rc::new(QueryExpr::Concat { children: vec![scan("a"), scan("b")], }); @@ -1335,7 +1431,7 @@ mod tests { assert_eq!( lower_query_physical_dag(&root, &extra_scope, &scripted(&complete)), Err(AnalyticalCostError::InvalidPhysicalDag( - "physical scans do not consume comparison sources exactly" + "physical scans omit a comparison-scope source" )) ); } diff --git a/crates/asap-aware-mapping/src/analytical_statistics.rs b/crates/asap-aware-mapping/src/analytical_statistics.rs index 80477303..147ba1ed 100644 --- a/crates/asap-aware-mapping/src/analytical_statistics.rs +++ b/crates/asap-aware-mapping/src/analytical_statistics.rs @@ -75,6 +75,16 @@ impl ComparisonScope { if self.sources.is_empty() { return Err(AnalyticalCostError::MissingComparisonScope("sources")); } + if self + .sources + .iter() + .enumerate() + .any(|(index, source)| self.sources[..index].contains(source)) + { + return Err(AnalyticalCostError::MissingComparisonScope( + "duplicate source coverage", + )); + } if self .sources .iter() @@ -107,7 +117,14 @@ pub fn validate_comparison_scopes( "time_selection", raw.time_selection == candidate.time_selection, ), - ("sources", raw.sources == candidate.sources), + ( + "sources", + raw.sources.len() == candidate.sources.len() + && raw + .sources + .iter() + .all(|source| candidate.sources.contains(source)), + ), ] { if !matches { return Err(AnalyticalCostError::ComparisonScopeMismatch(name)); diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 47cbfec6..2af7c9fa 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -255,10 +255,12 @@ choosing an arbitrary snapshot. When a predicate-bearing logical Scan expands to Scan → Filter, the synthetic Scan has its own physical ID, statistics, and buffer evidence and carries that exact coverage; the Filter has separate evidence and no source coverage. -After lowering, the multiset of distinct physical Scan coverages must consume -the comparison scope's source coverages exactly. Thus a candidate cannot omit -a source that exists in the comparison boundary, while a provider-declared -shared physical Scan is counted once. +`ComparisonScope.sources` is an order-independent set of semantic coverages; +duplicates are invalid. After lowering, every reachable physical Scan must use +a member of that set and every member must be used by at least one Scan. +Multiple independent physical Scans may use the same coverage, while a +provider-declared shared Scan uses it once, so those physical alternatives can +still be compared under the same semantic scope. The lowering validates every physical edge before costing: @@ -306,6 +308,12 @@ generic lowering creates raw-query operators with `ExecutionMultiplicity::PerEvaluation` and zero retained state. Buffer sizes are always provider-owned physical evidence. +The DAG itself implements `OperatorStatisticsProvider` over its evidence +snapshot. That boundary verifies that each map key equals the evidence's +embedded physical identity and that every node's buffer equals the provider +snapshot before returning statistics, preventing the public node and evidence +views from silently drifting apart. + Cross/non-equi joins, `INTERSECT`, `EXCEPT`, distinct `UNION`, PromQL range/subquery execution, vector matching, and PromQL-specific enrichment/ relabel/sample operators stay unavailable. Their cost requires a physical From 5ad66c6af48f2023ead83ec2d5634e0ca67b00f6 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 20:37:24 -0600 Subject: [PATCH 11/48] fix(cost): bind limit consumption during lowering --- .../asap-aware-mapping/src/analytical_cost.rs | 2 -- .../src/analytical_lowering.rs | 34 +++++++++++++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index d588a688..d6a6d196 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -683,8 +683,6 @@ pub enum AnalyticalCostError { }, #[error("invalid physical DAG: {0}")] InvalidPhysicalDag(&'static str), - #[error("inconsistent physical operator statistics: {0}")] - InconsistentOperatorStatistics(&'static str), } fn checked_bytes(parts: &[u64]) -> Result { diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index a824412b..11d68b76 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -373,6 +373,7 @@ pub fn lower_query_physical_dag( )?; require_operator_statistics(PhysicalOperator::Limit, statistics)?; require_limit_cardinality(*n, *offset, statistics)?; + require_limit_consumption(*n, *offset, statistics)?; self.push(evidence, PhysicalOperator::Limit, children, None) } QueryExpr::SQLWindowFunc { @@ -682,6 +683,28 @@ fn require_limit_cardinality( Ok(()) } +fn require_limit_consumption( + n: usize, + offset: usize, + statistics: &OperatorStatistics, +) -> Result<(), AnalyticalCostError> { + let n = u64::try_from(n).map_err(|_| AnalyticalCostError::Overflow)?; + let offset = u64::try_from(offset).map_err(|_| AnalyticalCostError::Overflow)?; + let expected_consumed = if n == 0 { + 0 + } else { + statistics.inputs[0] + .rows + .min(offset.checked_add(n).ok_or(AnalyticalCostError::Overflow)?) + }; + if statistics.limit_rows_consumed != Some(expected_consumed) { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "limit rows consumed do not match n and offset", + )); + } + Ok(()) +} + fn bind_scan_coverage( node_id: &str, source: &asap_types::pre_asap::Source, @@ -806,6 +829,7 @@ mod tests { key_bytes: None, aggregate_value_bytes: None, k: None, + limit_rows_consumed: None, hash_join_build_side: None, } } @@ -1224,7 +1248,10 @@ mod tests { ), ( "query-1".into(), - evidence(statistics(vec![edge(500, 6_000)], edge(20, 240))), + evidence(OperatorStatistics { + limit_rows_consumed: Some(20), + ..statistics(vec![edge(500, 6_000)], edge(20, 240)) + }), ), ( "query-0".into(), @@ -1475,7 +1502,10 @@ mod tests { ), ( "query-0".into(), - evidence(statistics(vec![edge(0, 0)], edge(0, 0))), + evidence(OperatorStatistics { + limit_rows_consumed: Some(0), + ..statistics(vec![edge(0, 0)], edge(0, 0)) + }), ), ]); let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)).unwrap(); From 0a91fbc7cb6d238a466ca85358c1cd87ee6fc7a7 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 10:07:11 -0600 Subject: [PATCH 12/48] fix(cost): preserve valid window and offset cardinalities --- crates/asap-aware-mapping/src/analytical_cost.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index d6a6d196..3b920472 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -570,11 +570,16 @@ fn validate_operator_semantics( return inconsistent("grouped output differs from distinct group cardinality"); } } - PhysicalOperator::Sort | PhysicalOperator::Window | PhysicalOperator::PassThrough => { + PhysicalOperator::Sort | PhysicalOperator::PassThrough => { if input != output { return inconsistent("cardinality-preserving operator changes its edge"); } } + PhysicalOperator::Window => { + if input.rows != output.rows { + return inconsistent("Window changes row cardinality"); + } + } PhysicalOperator::Concat => { let total = statistics.inputs.iter().try_fold( EdgeStatistics { rows: 0, bytes: 0 }, @@ -600,8 +605,8 @@ fn validate_operator_semantics( .k .filter(|k| *k > 0) .ok_or(AnalyticalCostError::MissingOrZero("k"))?; - if output.rows != input.rows.min(k) { - return inconsistent("Top-K output differs from its cardinality bound"); + if output.rows > input.rows.min(k) { + return inconsistent("Top-K output exceeds its cardinality bound"); } } PhysicalOperator::Limit => { From c099c94eac4c6efc14a3a20094a7a27c254fd20c Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 09:21:28 -0600 Subject: [PATCH 13/48] feat(cost): begin PromQL statistics adaptation --- .../asap-aware-mapping/src/analytical_cost.rs | 178 ++++++++++++++++-- .../src/analytical_lowering.rs | 115 ++++++++++- .../src/analytical_statistics.rs | 15 ++ 3 files changed, 291 insertions(+), 17 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 3b920472..9c83c87b 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -73,6 +73,16 @@ pub enum PhysicalOperator { Window, Limit, PassThrough, + PromqlRange, + PromqlSubquery, + PromqlVectorBinary, + PromqlRelabel, + PromqlInfoEnrich, + PromqlSeriesSample, + PromqlBridge, + PromqlScalarLeaf, + PromqlPerSeries, + PromqlPresence, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -330,8 +340,11 @@ fn validate_operator_statistics( ) -> Result<(), AnalyticalCostError> { let expected_inputs = match node.operator { PhysicalOperator::Scan => 1, - PhysicalOperator::HashJoin => 2, + PhysicalOperator::HashJoin + | PhysicalOperator::PromqlVectorBinary + | PhysicalOperator::PromqlInfoEnrich => 2, PhysicalOperator::Concat => node.children.len(), + PhysicalOperator::PromqlScalarLeaf => 0, _ => 1, }; if node_statistics.inputs.len() != expected_inputs { @@ -340,10 +353,9 @@ fn validate_operator_statistics( reason: "wrong input-edge count", }); } - let expected_children = if matches!(node.operator, PhysicalOperator::Scan) { - 0 - } else { - expected_inputs + let expected_children = match node.operator { + PhysicalOperator::Scan | PhysicalOperator::PromqlScalarLeaf => 0, + _ => expected_inputs, }; if node.children.len() != expected_children { return Err(AnalyticalCostError::InvalidPhysicalDag( @@ -413,8 +425,21 @@ pub fn estimate_operator( .copied() .ok_or(AnalyticalCostError::MissingOrZero("operator input edge")) }; - let left = input(0)?; let output = statistics.output; + if matches!(operator, PhysicalOperator::PromqlScalarLeaf) { + let promql = require_promql_statistics(&statistics, 0)?; + if promql.output_series > output.rows && output.rows > 0 { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "output series exceed output rows", + )); + } + return Ok(ResourceEstimate { + cpu_ops: output.rows as f64, + peak_memory_bytes: per_row_width(output.rows, output.bytes)?, + scan_bytes: 0, + }); + } + let left = input(0)?; let estimate = match operator { PhysicalOperator::Scan => ResourceEstimate { cpu_ops: left.rows as f64, @@ -510,21 +535,126 @@ pub fn estimate_operator( peak_memory_bytes: per_row_width(output.rows, output.bytes)?, scan_bytes: 0, }, - PhysicalOperator::Limit => { - let consumed = statistics - .limit_rows_consumed - .ok_or(AnalyticalCostError::MissingOrZero("limit_rows_consumed"))?; - if consumed > left.rows || consumed < output.rows { - return Err(AnalyticalCostError::InconsistentOperatorStatistics( - "Limit rows consumed must cover its output without exceeding its input", - )); + PhysicalOperator::Limit => ResourceEstimate { + cpu_ops: output.rows as f64, + peak_memory_bytes: per_row_width(output.rows, output.bytes)?, + scan_bytes: 0, + }, + PhysicalOperator::PromqlRange => { + let promql = require_promql_statistics(&statistics, 1)?; + let samples = promql + .window_samples_per_series + .filter(|value| *value > 0) + .ok_or(AnalyticalCostError::MissingOrZero("window_samples_per_series"))?; + ResourceEstimate { + cpu_ops: left.rows as f64, + peak_memory_bytes: checked_bytes(&[ + promql.input_series[0], + samples, + per_row_width(left.rows, left.bytes)?, + ])?, + scan_bytes: 0, + } + } + PhysicalOperator::PromqlSubquery => { + let promql = require_promql_statistics(&statistics, 1)?; + if !matches!(promql.subquery_steps, Some(value) if value > 0) { + return Err(AnalyticalCostError::MissingOrZero("subquery_steps")); } ResourceEstimate { - cpu_ops: consumed as f64, + cpu_ops: left.rows as f64 + output.rows as f64, + peak_memory_bytes: left.bytes, + scan_bytes: 0, + } + } + PhysicalOperator::PromqlVectorBinary | PhysicalOperator::PromqlInfoEnrich => { + let promql = require_promql_statistics(&statistics, 2)?; + let right = input(1)?; + let scalar_vector = matches!(operator, PhysicalOperator::PromqlVectorBinary) + && promql.input_series.contains(&0); + let matching_bytes = if scalar_vector { + per_row_width(output.rows, output.bytes)? + } else { + let build_side = statistics + .hash_join_build_side + .ok_or(AnalyticalCostError::MissingOrZero("hash_join_build_side"))?; + let key_bytes = statistics + .key_bytes + .ok_or(AnalyticalCostError::MissingOrZero("key_bytes"))?; + let build_series = match build_side { + HashJoinBuildSide::Left => promql.input_series[0], + HashJoinBuildSide::Right => promql.input_series[1], + }; + checked_bytes(&[ + build_series, + key_bytes.checked_add(16).ok_or(AnalyticalCostError::Overflow)?, + ])? + }; + ResourceEstimate { + cpu_ops: left.rows as f64 + right.rows as f64 + output.rows as f64, + peak_memory_bytes: matching_bytes, + scan_bytes: 0, + } + } + PhysicalOperator::PromqlRelabel => { + let promql = require_promql_statistics(&statistics, 1)?; + let operations = promql.scalar_ops_per_row.filter(|value| *value > 0).ok_or( + AnalyticalCostError::MissingOrZero("scalar_ops_per_row"), + )?; + ResourceEstimate { + cpu_ops: left.rows as f64 * operations as f64, peak_memory_bytes: per_row_width(output.rows, output.bytes)?, scan_bytes: 0, } } + PhysicalOperator::PromqlSeriesSample => { + let promql = require_promql_statistics(&statistics, 1)?; + let key_bytes = statistics + .key_bytes + .ok_or(AnalyticalCostError::MissingOrZero("key_bytes"))?; + ResourceEstimate { + cpu_ops: left.rows as f64 + promql.input_series[0] as f64, + peak_memory_bytes: checked_bytes(&[ + promql.output_series, + key_bytes.checked_add(16).ok_or(AnalyticalCostError::Overflow)?, + ])?, + scan_bytes: 0, + } + } + PhysicalOperator::PromqlBridge => { + require_promql_statistics(&statistics, 1)?; + ResourceEstimate { + cpu_ops: left.rows as f64 + output.rows as f64, + peak_memory_bytes: per_row_width(output.rows, output.bytes)?, + scan_bytes: 0, + } + } + PhysicalOperator::PromqlPerSeries => { + let promql = require_promql_statistics(&statistics, 1)?; + let operations = promql.scalar_ops_per_row.filter(|value| *value > 0).ok_or( + AnalyticalCostError::MissingOrZero("scalar_ops_per_row"), + )?; + let accumulator = statistics.aggregate_value_bytes.ok_or( + AnalyticalCostError::MissingOrZero("aggregate_value_bytes"), + )?; + ResourceEstimate { + cpu_ops: left.rows as f64 * operations as f64, + peak_memory_bytes: checked_bytes(&[promql.input_series[0], accumulator])?, + scan_bytes: 0, + } + } + PhysicalOperator::PromqlPresence => { + let promql = require_promql_statistics(&statistics, 1)?; + let operations = promql.scalar_ops_per_row.filter(|value| *value > 0).ok_or( + AnalyticalCostError::MissingOrZero("scalar_ops_per_row"), + )?; + ResourceEstimate { + cpu_ops: left.rows as f64 * operations as f64 + output.rows as f64, + peak_memory_bytes: per_row_width(output.rows, output.bytes)?, + scan_bytes: 0, + } + } + PhysicalOperator::PromqlScalarLeaf => unreachable!(), }; if estimate.cpu_ops.is_finite() { Ok(estimate) @@ -619,6 +749,24 @@ fn validate_operator_semantics( Ok(()) } +fn require_promql_statistics( + statistics: &OperatorStatistics, + inputs: usize, +) -> Result<&crate::analytical_statistics::PromqlOperatorStatistics, AnalyticalCostError> { + let promql = statistics.promql.as_ref().ok_or( + AnalyticalCostError::MissingOrStale("promql_operator_statistics"), + )?; + if promql.evaluation_steps == 0 { + return Err(AnalyticalCostError::MissingOrZero("evaluation_steps")); + } + if promql.input_series.len() != inputs { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "PromQL input-series arity does not match physical inputs", + )); + } + Ok(promql) +} + impl ResourceEstimate { pub fn calibrated_cost( self, diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index 11d68b76..8678df25 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -296,10 +296,12 @@ pub fn lower_query_physical_dag( child, .. } => { - if having.is_some() || !supports_hash_aggregate(reduction, measures) { + if having.is_some() { return Err(AnalyticalCostError::UnsupportedQueryOperator); } - self.lower_unary(query, occurrence, PhysicalOperator::HashAggregate, child) + let operator = aggregate_operator(reduction, measures) + .ok_or(AnalyticalCostError::UnsupportedQueryOperator)?; + self.lower_unary(query, occurrence, operator, child) } QueryExpr::Dedup { child, .. } => { self.lower_unary(query, occurrence, PhysicalOperator::Deduplicate, child) @@ -394,9 +396,118 @@ pub fn lower_query_physical_dag( } self.lower_unary(query, occurrence, PhysicalOperator::Window, child) } + QueryExpr::TimeRange { range, child } => { + if range.is_zero() { + return Err(AnalyticalCostError::MissingOrZero("range")); + } + self.lower_unary(query, id, PhysicalOperator::PromqlRange, child) + } + QueryExpr::PromqlSubquery { range, resolution, child } => { + let child_id = self.lower(child)?; + let statistics = self.stats(&id)?; + let child_statistics = self.node_statistics(&child_id)?; + require_unary_edge(&id, &statistics, &child_id, child_statistics)?; + require_operator_statistics(PhysicalOperator::PromqlSubquery, &statistics)?; + let supplied = require_promql_statistics(&statistics, 1)? + .subquery_steps + .ok_or(AnalyticalCostError::MissingOrZero("subquery_steps"))?; + if let Some(expected) = subquery_steps(*range, *resolution)? { + if supplied != expected { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "subquery_steps disagree with range and resolution", + )); + } + } + Ok(self.push(id, PhysicalOperator::PromqlSubquery, &statistics, vec![child_id], None)) + } QueryExpr::TimeShift { child, .. } => { self.lower_unary(query, occurrence, PhysicalOperator::PassThrough, child) } + QueryExpr::PromqlRelabel { child, .. } => { + self.lower_unary(query, id, PhysicalOperator::PromqlRelabel, child) + } + QueryExpr::PromqlSeriesSample { by, kind, child, .. } => { + let child_id = self.lower(child)?; + let statistics = self.stats(&id)?; + let child_statistics = self.node_statistics(&child_id)?; + require_unary_edge(&id, &statistics, &child_id, child_statistics)?; + validate_series_sample(by, *kind, &statistics)?; + require_operator_statistics(PhysicalOperator::PromqlSeriesSample, &statistics)?; + Ok(self.push(id, PhysicalOperator::PromqlSeriesSample, &statistics, vec![child_id], None)) + } + QueryExpr::PromqlInfoEnrich { selector, child } => { + let left_id = self.lower(child)?; + let info_id = format!("{id}-info-scan"); + let coverage = bind_info_coverage(&info_id, selector, self.scope)?; + let info_statistics = self.stats(&info_id)?; + require_statistics_shape(&info_id, &info_statistics, 1)?; + self.push(info_id.clone(), PhysicalOperator::Scan, &info_statistics, vec![], Some(coverage)); + let statistics = self.stats(&id)?; + require_statistics_shape(&id, &statistics, 2)?; + if statistics.inputs[0] != self.node_statistics(&left_id)?.output + || statistics.inputs[1] != info_statistics.output + { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "info enrichment inputs do not match child outputs", + )); + } + require_operator_statistics(PhysicalOperator::PromqlInfoEnrich, &statistics)?; + Ok(self.push(id, PhysicalOperator::PromqlInfoEnrich, &statistics, vec![left_id, info_id], None)) + } + QueryExpr::BinaryOp { lhs, rhs, vector_match, .. } => { + let left_scalar = is_promql_scalar(lhs); + let right_scalar = is_promql_scalar(rhs); + if left_scalar && right_scalar { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + } + let left_id = self.lower(lhs)?; + let right_id = self.lower(rhs)?; + let statistics = self.stats(&id)?; + require_statistics_shape(&id, &statistics, 2)?; + if statistics.inputs[0] != self.node_statistics(&left_id)?.output + || statistics.inputs[1] != self.node_statistics(&right_id)?.output + { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "PromQL binary inputs do not match child outputs", + )); + } + if left_scalar || right_scalar { + if vector_match.is_some() + || statistics.hash_join_build_side.is_some() + || statistics.key_bytes.is_some() + { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "scalar/vector binary operation cannot have label-match state", + )); + } + } else if statistics.hash_join_build_side.is_none() || statistics.key_bytes.is_none() { + return Err(AnalyticalCostError::MissingOrStale( + "vector_binary_label_match_statistics", + )); + } + require_promql_binary_edges(&statistics, + self.node_statistics(&left_id)?, self.node_statistics(&right_id)?)?; + require_operator_statistics(PhysicalOperator::PromqlVectorBinary, &statistics)?; + Ok(self.push(id, PhysicalOperator::PromqlVectorBinary, &statistics, vec![left_id, right_id], None)) + } + QueryExpr::PromqlVectorFromScalar(child) + | QueryExpr::PromqlScalarFromVector(child) => { + self.lower_unary(query, id, PhysicalOperator::PromqlBridge, child) + } + QueryExpr::PromqlScalarBridge(inner) + if matches!(inner.as_ref(), QueryExpr::Literal(asap_types::pre_asap::ScalarValue::Float64(_))) => + { + let statistics = self.stats(&id)?; + require_statistics_shape(&id, &statistics, 0)?; + require_operator_statistics(PhysicalOperator::PromqlScalarLeaf, &statistics)?; + Ok(self.push(id, PhysicalOperator::PromqlScalarLeaf, &statistics, vec![], None)) + } + QueryExpr::EvalTimestamp => { + let statistics = self.stats(&id)?; + require_statistics_shape(&id, &statistics, 0)?; + require_operator_statistics(PhysicalOperator::PromqlScalarLeaf, &statistics)?; + Ok(self.push(id, PhysicalOperator::PromqlScalarLeaf, &statistics, vec![], None)) + } QueryExpr::Concat { children } => { let child_ids = children .iter() diff --git a/crates/asap-aware-mapping/src/analytical_statistics.rs b/crates/asap-aware-mapping/src/analytical_statistics.rs index 147ba1ed..f820ec7b 100644 --- a/crates/asap-aware-mapping/src/analytical_statistics.rs +++ b/crates/asap-aware-mapping/src/analytical_statistics.rs @@ -221,6 +221,21 @@ pub struct OperatorStatistics { #[serde(default)] pub limit_rows_consumed: Option, pub hash_join_build_side: Option, + /// Series-oriented evidence used only by PromQL physical operators. + /// Logical row/byte edges remain authoritative in `inputs` and `output`. + #[serde(default)] + pub promql: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PromqlOperatorStatistics { + /// Series cardinality for each logical input edge, in child order. + pub input_series: Vec, + pub output_series: u64, + pub evaluation_steps: u64, + pub window_samples_per_series: Option, + pub subquery_steps: Option, + pub scalar_ops_per_row: Option, } /// Resolves physical statistics and owns their catalog/observation freshness. From cb3b3d9dbb6d97752a728a072e9ffc749ef59b27 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 09:29:45 -0600 Subject: [PATCH 14/48] feat(cost): adapt PromQL operators to scoped evidence --- .../asap-aware-mapping/src/analytical_cost.rs | 75 +- .../src/analytical_lowering.rs | 719 +++++++++++++++++- 2 files changed, 748 insertions(+), 46 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 9c83c87b..03615398 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -397,6 +397,32 @@ fn validate_operator_statistics( }); } } + if let Some(promql) = node_statistics.promql.as_ref() { + if promql.input_series.len() != expected_inputs { + return Err(AnalyticalCostError::InvalidOperatorStatistics { + node: node.id.clone(), + reason: "PromQL input-series arity does not match physical inputs", + }); + } + for (index, child_id) in node.children.iter().enumerate() { + let child = &statistics[child_id.as_str()]; + let child_promql = child.promql.as_ref().ok_or_else(|| { + AnalyticalCostError::InvalidOperatorStatistics { + node: child_id.clone(), + reason: "PromQL child is missing series statistics", + } + })?; + if promql.input_series[index] != child_promql.output_series + || (!matches!(node.operator, PhysicalOperator::PromqlSubquery) + && promql.evaluation_steps != child_promql.evaluation_steps) + { + return Err(AnalyticalCostError::InvalidOperatorStatistics { + node: node.id.clone(), + reason: "PromQL edge series or evaluation steps conflict", + }); + } + } + } Ok(()) } @@ -545,7 +571,9 @@ pub fn estimate_operator( let samples = promql .window_samples_per_series .filter(|value| *value > 0) - .ok_or(AnalyticalCostError::MissingOrZero("window_samples_per_series"))?; + .ok_or(AnalyticalCostError::MissingOrZero( + "window_samples_per_series", + ))?; ResourceEstimate { cpu_ops: left.rows as f64, peak_memory_bytes: checked_bytes(&[ @@ -587,7 +615,9 @@ pub fn estimate_operator( }; checked_bytes(&[ build_series, - key_bytes.checked_add(16).ok_or(AnalyticalCostError::Overflow)?, + key_bytes + .checked_add(16) + .ok_or(AnalyticalCostError::Overflow)?, ])? }; ResourceEstimate { @@ -598,9 +628,10 @@ pub fn estimate_operator( } PhysicalOperator::PromqlRelabel => { let promql = require_promql_statistics(&statistics, 1)?; - let operations = promql.scalar_ops_per_row.filter(|value| *value > 0).ok_or( - AnalyticalCostError::MissingOrZero("scalar_ops_per_row"), - )?; + let operations = promql + .scalar_ops_per_row + .filter(|value| *value > 0) + .ok_or(AnalyticalCostError::MissingOrZero("scalar_ops_per_row"))?; ResourceEstimate { cpu_ops: left.rows as f64 * operations as f64, peak_memory_bytes: per_row_width(output.rows, output.bytes)?, @@ -616,7 +647,9 @@ pub fn estimate_operator( cpu_ops: left.rows as f64 + promql.input_series[0] as f64, peak_memory_bytes: checked_bytes(&[ promql.output_series, - key_bytes.checked_add(16).ok_or(AnalyticalCostError::Overflow)?, + key_bytes + .checked_add(16) + .ok_or(AnalyticalCostError::Overflow)?, ])?, scan_bytes: 0, } @@ -631,12 +664,13 @@ pub fn estimate_operator( } PhysicalOperator::PromqlPerSeries => { let promql = require_promql_statistics(&statistics, 1)?; - let operations = promql.scalar_ops_per_row.filter(|value| *value > 0).ok_or( - AnalyticalCostError::MissingOrZero("scalar_ops_per_row"), - )?; - let accumulator = statistics.aggregate_value_bytes.ok_or( - AnalyticalCostError::MissingOrZero("aggregate_value_bytes"), - )?; + let operations = promql + .scalar_ops_per_row + .filter(|value| *value > 0) + .ok_or(AnalyticalCostError::MissingOrZero("scalar_ops_per_row"))?; + let accumulator = statistics + .aggregate_value_bytes + .ok_or(AnalyticalCostError::MissingOrZero("aggregate_value_bytes"))?; ResourceEstimate { cpu_ops: left.rows as f64 * operations as f64, peak_memory_bytes: checked_bytes(&[promql.input_series[0], accumulator])?, @@ -645,9 +679,10 @@ pub fn estimate_operator( } PhysicalOperator::PromqlPresence => { let promql = require_promql_statistics(&statistics, 1)?; - let operations = promql.scalar_ops_per_row.filter(|value| *value > 0).ok_or( - AnalyticalCostError::MissingOrZero("scalar_ops_per_row"), - )?; + let operations = promql + .scalar_ops_per_row + .filter(|value| *value > 0) + .ok_or(AnalyticalCostError::MissingOrZero("scalar_ops_per_row"))?; ResourceEstimate { cpu_ops: left.rows as f64 * operations as f64 + output.rows as f64, peak_memory_bytes: per_row_width(output.rows, output.bytes)?, @@ -753,9 +788,12 @@ fn require_promql_statistics( statistics: &OperatorStatistics, inputs: usize, ) -> Result<&crate::analytical_statistics::PromqlOperatorStatistics, AnalyticalCostError> { - let promql = statistics.promql.as_ref().ok_or( - AnalyticalCostError::MissingOrStale("promql_operator_statistics"), - )?; + let promql = statistics + .promql + .as_ref() + .ok_or(AnalyticalCostError::MissingOrStale( + "promql_operator_statistics", + ))?; if promql.evaluation_steps == 0 { return Err(AnalyticalCostError::MissingOrZero("evaluation_steps")); } @@ -1252,6 +1290,7 @@ mod tests { k: None, limit_rows_consumed: None, hash_join_build_side: None, + promql: None, } } diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index 8678df25..f1a3677e 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -200,6 +200,7 @@ pub fn lower_query_physical_dag( &child_id, child_statistics, )?; + require_promql_unary_edge(operator, statistics, child_statistics)?; require_operator_statistics(operator, statistics)?; self.push(evidence, operator, children, None) } @@ -339,6 +340,11 @@ pub fn lower_query_physical_dag( &child_id, child_statistics, )?; + require_promql_unary_edge( + PhysicalOperator::TopK, + statistics, + child_statistics, + )?; let bound = n .checked_add(*offset) .and_then(|value| u64::try_from(value).ok()) @@ -373,6 +379,11 @@ pub fn lower_query_physical_dag( &child_id, child_statistics, )?; + require_promql_unary_edge( + PhysicalOperator::Limit, + statistics, + child_statistics, + )?; require_operator_statistics(PhysicalOperator::Limit, statistics)?; require_limit_cardinality(*n, *offset, statistics)?; require_limit_consumption(*n, *offset, statistics)?; @@ -402,11 +413,20 @@ pub fn lower_query_physical_dag( } self.lower_unary(query, id, PhysicalOperator::PromqlRange, child) } - QueryExpr::PromqlSubquery { range, resolution, child } => { + QueryExpr::PromqlSubquery { + range, + resolution, + child, + } => { let child_id = self.lower(child)?; let statistics = self.stats(&id)?; let child_statistics = self.node_statistics(&child_id)?; require_unary_edge(&id, &statistics, &child_id, child_statistics)?; + require_promql_unary_edge( + PhysicalOperator::PromqlSubquery, + &statistics, + child_statistics, + )?; require_operator_statistics(PhysicalOperator::PromqlSubquery, &statistics)?; let supplied = require_promql_statistics(&statistics, 1)? .subquery_steps @@ -418,7 +438,13 @@ pub fn lower_query_physical_dag( )); } } - Ok(self.push(id, PhysicalOperator::PromqlSubquery, &statistics, vec![child_id], None)) + Ok(self.push( + id, + PhysicalOperator::PromqlSubquery, + &statistics, + vec![child_id], + None, + )) } QueryExpr::TimeShift { child, .. } => { self.lower_unary(query, occurrence, PhysicalOperator::PassThrough, child) @@ -426,14 +452,27 @@ pub fn lower_query_physical_dag( QueryExpr::PromqlRelabel { child, .. } => { self.lower_unary(query, id, PhysicalOperator::PromqlRelabel, child) } - QueryExpr::PromqlSeriesSample { by, kind, child, .. } => { + QueryExpr::PromqlSeriesSample { + by, kind, child, .. + } => { let child_id = self.lower(child)?; let statistics = self.stats(&id)?; let child_statistics = self.node_statistics(&child_id)?; require_unary_edge(&id, &statistics, &child_id, child_statistics)?; + require_promql_unary_edge( + PhysicalOperator::PromqlSeriesSample, + &statistics, + child_statistics, + )?; validate_series_sample(by, *kind, &statistics)?; require_operator_statistics(PhysicalOperator::PromqlSeriesSample, &statistics)?; - Ok(self.push(id, PhysicalOperator::PromqlSeriesSample, &statistics, vec![child_id], None)) + Ok(self.push( + id, + PhysicalOperator::PromqlSeriesSample, + &statistics, + vec![child_id], + None, + )) } QueryExpr::PromqlInfoEnrich { selector, child } => { let left_id = self.lower(child)?; @@ -441,7 +480,14 @@ pub fn lower_query_physical_dag( let coverage = bind_info_coverage(&info_id, selector, self.scope)?; let info_statistics = self.stats(&info_id)?; require_statistics_shape(&info_id, &info_statistics, 1)?; - self.push(info_id.clone(), PhysicalOperator::Scan, &info_statistics, vec![], Some(coverage)); + require_scan_edges_equal(&info_statistics)?; + self.push( + info_id.clone(), + PhysicalOperator::Scan, + &info_statistics, + vec![], + Some(coverage), + ); let statistics = self.stats(&id)?; require_statistics_shape(&id, &statistics, 2)?; if statistics.inputs[0] != self.node_statistics(&left_id)?.output @@ -451,10 +497,26 @@ pub fn lower_query_physical_dag( "info enrichment inputs do not match child outputs", )); } + require_promql_binary_edges( + &statistics, + self.node_statistics(&left_id)?, + &info_statistics, + )?; require_operator_statistics(PhysicalOperator::PromqlInfoEnrich, &statistics)?; - Ok(self.push(id, PhysicalOperator::PromqlInfoEnrich, &statistics, vec![left_id, info_id], None)) + Ok(self.push( + id, + PhysicalOperator::PromqlInfoEnrich, + &statistics, + vec![left_id, info_id], + None, + )) } - QueryExpr::BinaryOp { lhs, rhs, vector_match, .. } => { + QueryExpr::BinaryOp { + lhs, + rhs, + vector_match, + .. + } => { let left_scalar = is_promql_scalar(lhs); let right_scalar = is_promql_scalar(rhs); if left_scalar && right_scalar { @@ -480,33 +542,59 @@ pub fn lower_query_physical_dag( "scalar/vector binary operation cannot have label-match state", )); } - } else if statistics.hash_join_build_side.is_none() || statistics.key_bytes.is_none() { + } else if statistics.hash_join_build_side.is_none() + || statistics.key_bytes.is_none() + { return Err(AnalyticalCostError::MissingOrStale( "vector_binary_label_match_statistics", )); } - require_promql_binary_edges(&statistics, - self.node_statistics(&left_id)?, self.node_statistics(&right_id)?)?; + require_promql_binary_edges( + &statistics, + self.node_statistics(&left_id)?, + self.node_statistics(&right_id)?, + )?; require_operator_statistics(PhysicalOperator::PromqlVectorBinary, &statistics)?; - Ok(self.push(id, PhysicalOperator::PromqlVectorBinary, &statistics, vec![left_id, right_id], None)) + Ok(self.push( + id, + PhysicalOperator::PromqlVectorBinary, + &statistics, + vec![left_id, right_id], + None, + )) } QueryExpr::PromqlVectorFromScalar(child) | QueryExpr::PromqlScalarFromVector(child) => { self.lower_unary(query, id, PhysicalOperator::PromqlBridge, child) } QueryExpr::PromqlScalarBridge(inner) - if matches!(inner.as_ref(), QueryExpr::Literal(asap_types::pre_asap::ScalarValue::Float64(_))) => + if matches!( + inner.as_ref(), + QueryExpr::Literal(asap_types::pre_asap::ScalarValue::Float64(_)) + ) => { let statistics = self.stats(&id)?; require_statistics_shape(&id, &statistics, 0)?; require_operator_statistics(PhysicalOperator::PromqlScalarLeaf, &statistics)?; - Ok(self.push(id, PhysicalOperator::PromqlScalarLeaf, &statistics, vec![], None)) + Ok(self.push( + id, + PhysicalOperator::PromqlScalarLeaf, + &statistics, + vec![], + None, + )) } QueryExpr::EvalTimestamp => { let statistics = self.stats(&id)?; require_statistics_shape(&id, &statistics, 0)?; require_operator_statistics(PhysicalOperator::PromqlScalarLeaf, &statistics)?; - Ok(self.push(id, PhysicalOperator::PromqlScalarLeaf, &statistics, vec![], None)) + Ok(self.push( + id, + PhysicalOperator::PromqlScalarLeaf, + &statistics, + vec![], + None, + )) } QueryExpr::Concat { children } => { let child_ids = children @@ -719,6 +807,10 @@ fn require_operator_statistics( if !matches!(operator, PhysicalOperator::Scan) && statistics.source_scan_bytes != 0 { return invalid("only Scan may charge source bytes"); } + if matches!(operator, PhysicalOperator::PromqlScalarLeaf) { + require_promql_statistics(statistics, 0)?; + return Ok(()); + } let input = statistics.inputs.first().copied().ok_or( AnalyticalCostError::InconsistentOperatorStatistics("operator input is missing"), )?; @@ -773,7 +865,198 @@ fn require_operator_statistics( return invalid("pass-through wrapper changes its edge statistics"); } } - PhysicalOperator::Scan | PhysicalOperator::HashJoin | PhysicalOperator::Concat => {} + PhysicalOperator::PromqlRange => { + let promql = require_promql_statistics(statistics, 1)?; + if !matches!(promql.window_samples_per_series, Some(value) if value > 0) { + return Err(AnalyticalCostError::MissingOrZero( + "window_samples_per_series", + )); + } + } + PhysicalOperator::PromqlSubquery | PhysicalOperator::PromqlBridge => { + require_promql_statistics(statistics, 1)?; + } + PhysicalOperator::PromqlVectorBinary => { + require_promql_statistics(statistics, 2)?; + } + PhysicalOperator::PromqlInfoEnrich => { + let promql = require_promql_statistics(statistics, 2)?; + if output.rows != input.rows || promql.output_series != promql.input_series[0] { + return invalid("info enrichment changes left sample or series cardinality"); + } + } + PhysicalOperator::PromqlRelabel => { + let promql = require_promql_statistics(statistics, 1)?; + if output.rows != input.rows || promql.output_series > promql.input_series[0] { + return invalid("relabel changes rows or expands series cardinality"); + } + } + PhysicalOperator::PromqlSeriesSample => { + let promql = require_promql_statistics(statistics, 1)?; + if output.rows > input.rows || promql.output_series > promql.input_series[0] { + return invalid("series sampling expands its input"); + } + } + PhysicalOperator::PromqlPerSeries => { + let promql = require_promql_statistics(statistics, 1)?; + if promql.output_series > promql.input_series[0] { + return invalid("per-series operator expands series cardinality"); + } + } + PhysicalOperator::PromqlPresence => { + let promql = require_promql_statistics(statistics, 1)?; + if promql.output_series > 1 { + return invalid("PromQL absence operator emits more than one series"); + } + } + PhysicalOperator::Scan + | PhysicalOperator::HashJoin + | PhysicalOperator::Concat + | PhysicalOperator::PromqlScalarLeaf => {} + } + Ok(()) +} + +fn require_promql_statistics( + statistics: &OperatorStatistics, + arity: usize, +) -> Result<&crate::analytical_statistics::PromqlOperatorStatistics, AnalyticalCostError> { + let promql = statistics + .promql + .as_ref() + .ok_or(AnalyticalCostError::MissingOrStale( + "promql_operator_statistics", + ))?; + if promql.evaluation_steps == 0 { + return Err(AnalyticalCostError::MissingOrZero("evaluation_steps")); + } + if promql.input_series.len() != arity { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "PromQL input-series arity does not match physical inputs", + )); + } + for (index, series) in promql.input_series.iter().enumerate() { + if *series > statistics.inputs[index].rows && statistics.inputs[index].rows > 0 { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "input series exceed input rows", + )); + } + } + if promql.output_series > statistics.output.rows && statistics.output.rows > 0 { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "output series exceed output rows", + )); + } + Ok(promql) +} + +fn require_promql_binary_edges( + statistics: &OperatorStatistics, + left: &OperatorStatistics, + right: &OperatorStatistics, +) -> Result<(), AnalyticalCostError> { + let parent = require_promql_statistics(statistics, 2)?; + let left = left + .promql + .as_ref() + .ok_or(AnalyticalCostError::MissingOrStale( + "left_promql_operator_statistics", + ))?; + let right = right + .promql + .as_ref() + .ok_or(AnalyticalCostError::MissingOrStale( + "right_promql_operator_statistics", + ))?; + if parent.input_series != [left.output_series, right.output_series] + || parent.evaluation_steps != left.evaluation_steps + || parent.evaluation_steps != right.evaluation_steps + { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "PromQL binary series or step statistics do not match its children", + )); + } + Ok(()) +} + +fn require_promql_unary_edge( + operator: PhysicalOperator, + statistics: &OperatorStatistics, + child: &OperatorStatistics, +) -> Result<(), AnalyticalCostError> { + match (statistics.promql.as_ref(), child.promql.as_ref()) { + (None, None) => Ok(()), + (Some(parent), Some(child)) => { + require_promql_statistics(statistics, 1)?; + if parent.input_series[0] != child.output_series + || (!matches!(operator, PhysicalOperator::PromqlSubquery) + && parent.evaluation_steps != child.evaluation_steps) + { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "PromQL unary series or step statistics do not match its child", + )); + } + Ok(()) + } + _ => Err(AnalyticalCostError::InconsistentOperatorStatistics( + "PromQL statistics are missing on one side of an edge", + )), + } +} + +fn subquery_steps( + range: std::time::Duration, + resolution: Option, +) -> Result, AnalyticalCostError> { + if range.is_zero() { + return Err(AnalyticalCostError::MissingOrZero("subquery range")); + } + let Some(resolution) = resolution else { + return Ok(None); + }; + if resolution.is_zero() { + return Err(AnalyticalCostError::MissingOrZero("subquery resolution")); + } + Ok(Some( + u64::try_from(range.as_nanos() / resolution.as_nanos() + 1) + .map_err(|_| AnalyticalCostError::Overflow)?, + )) +} + +fn validate_series_sample( + by: &asap_types::pre_asap::GroupKeys, + kind: asap_types::pre_asap::SampleKind, + statistics: &OperatorStatistics, +) -> Result<(), AnalyticalCostError> { + let promql = require_promql_statistics(statistics, 1)?; + let groups = if !by.is_without() && by.keys().is_empty() { + 1 + } else { + statistics + .group_count + .ok_or(AnalyticalCostError::MissingOrStale("sample_group_count"))? + }; + match kind { + asap_types::pre_asap::SampleKind::LimitK(k) => { + let k = u64::try_from(k).map_err(|_| AnalyticalCostError::Overflow)?; + if statistics.k != Some(k) { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "limitk statistics disagree with query k", + )); + } + let bound = groups.checked_mul(k).ok_or(AnalyticalCostError::Overflow)?; + if promql.output_series > promql.input_series[0].min(bound) { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "limitk output exceeds its cardinality bound", + )); + } + } + asap_types::pre_asap::SampleKind::LimitRatio(ratio) + if !ratio.is_finite() || !(-1.0..=1.0).contains(&ratio) => + { + return Err(AnalyticalCostError::UnsupportedQueryOperator) + } + asap_types::pre_asap::SampleKind::LimitRatio(_) => {} } Ok(()) } @@ -838,6 +1121,32 @@ fn bind_scan_coverage( Ok(coverage) } +fn bind_info_coverage( + node_id: &str, + selector: &[asap_types::pre_asap::InfoMatcher], + scope: &ComparisonScope, +) -> Result { + use asap_types::pre_asap::{CompareOpKind, Source}; + + let mut metric: Option<&str> = None; + for matcher in selector + .iter() + .filter(|matcher| matcher.label == "__name__") + { + if matcher.op != CompareOpKind::Eq { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + } + if metric.is_some_and(|current| current != matcher.value) { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + } + metric = Some(&matcher.value); + } + let source = Source::TimeSeries { + metric: metric.unwrap_or("target_info").into(), + }; + bind_scan_coverage(node_id, &source, &[], scope) +} + fn is_hash_join_predicate( expr: &asap_types::pre_asap::QueryExpr, left: &asap_types::pre_asap::QueryExpr, @@ -914,6 +1223,71 @@ fn supports_hash_aggregate( }) } +fn aggregate_operator( + reduction: &asap_types::pre_asap::Reduction, + measures: &[asap_types::pre_asap::AggIntent], +) -> Option { + use asap_types::pre_asap::Reduction; + + if supports_hash_aggregate(reduction, measures) { + Some(PhysicalOperator::HashAggregate) + } else if matches!(reduction, Reduction::PerEntity) + && !measures.is_empty() + && measures.iter().all(presence_intent) + { + Some(PhysicalOperator::PromqlPresence) + } else if matches!(reduction, Reduction::PerEntity) + && !measures.is_empty() + && measures.iter().all(fixed_state_per_series_intent) + { + Some(PhysicalOperator::PromqlPerSeries) + } else { + None + } +} + +fn presence_intent(intent: &asap_types::pre_asap::AggIntent) -> bool { + matches!( + intent, + asap_types::pre_asap::AggIntent::Absent | asap_types::pre_asap::AggIntent::AbsentOverTime + ) +} + +fn fixed_state_per_series_intent(intent: &asap_types::pre_asap::AggIntent) -> bool { + use asap_types::pre_asap::AggIntent; + matches!( + intent, + AggIntent::Rate + | AggIntent::Increase + | AggIntent::Changes + | AggIntent::Delta + | AggIntent::IDelta + | AggIntent::Deriv + | AggIntent::Resets + | AggIntent::PredictLinear { .. } + | AggIntent::DoubleExpSmoothing { .. } + | AggIntent::Math(_) + | AggIntent::PresentOverTime + | AggIntent::TimeFn(_) + | AggIntent::LastOverTime + | AggIntent::FirstOverTime + | AggIntent::TsOfMinOverTime + | AggIntent::TsOfMaxOverTime + | AggIntent::TsOfFirstOverTime + | AggIntent::TsOfLastOverTime + ) +} + +fn is_promql_scalar(query: &asap_types::pre_asap::QueryExpr) -> bool { + use asap_types::pre_asap::QueryExpr; + matches!( + query, + QueryExpr::PromqlScalarBridge(_) + | QueryExpr::PromqlScalarFromVector(_) + | QueryExpr::EvalTimestamp + ) +} + #[cfg(test)] mod tests { use super::*; @@ -942,6 +1316,7 @@ mod tests { k: None, limit_rows_consumed: None, hash_join_build_side: None, + promql: None, } } @@ -972,6 +1347,23 @@ mod tests { } } + fn with_promql( + mut statistics: OperatorStatistics, + input_series: Vec, + output_series: u64, + evaluation_steps: u64, + ) -> OperatorStatistics { + statistics.promql = Some(crate::analytical_statistics::PromqlOperatorStatistics { + input_series, + output_series, + evaluation_steps, + window_samples_per_series: None, + subquery_steps: None, + scalar_ops_per_row: None, + }); + statistics + } + fn scope(sources: Vec) -> ComparisonScope { ComparisonScope { data_arrival: DataArrival::AtRest, @@ -1654,13 +2046,6 @@ mod tests { having: None, child: scan(), }); - let per_entity = Rc::new(QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Rate], - output_names: vec![], - having: None, - child: scan(), - }); let empty_sort_limit = Rc::new(QueryExpr::Limit { n: 10, offset: 0, @@ -1686,16 +2071,294 @@ mod tests { vec![], )]); let unavailable = HashMap::::new(); - for query in [ - &exact_quantile, - &per_entity, - &empty_sort_limit, - &unsupported_window, - ] { + for query in [&exact_quantile, &empty_sort_limit, &unsupported_window] { assert_eq!( lower_query_physical_dag(query, &scope, &scripted(&unavailable)), Err(AnalyticalCostError::UnsupportedQueryOperator) ); } } + + #[test] + fn promql_range_and_subquery_use_authoritative_edge_and_window_evidence() { + use asap_types::pre_asap::{Column, DataType, QueryExpr, Schema, Source}; + use std::{rc::Rc, time::Duration}; + + let source = Source::TimeSeries { metric: "m".into() }; + let scan = Rc::new(QueryExpr::Scan { + source: source.clone(), + predicates: vec![], + schema: Schema::new(vec![Column::new("value", DataType::Float64, false)]), + }); + let range = Rc::new(QueryExpr::TimeRange { + range: Duration::from_secs(300), + child: scan, + }); + let root = Rc::new(QueryExpr::PromqlSubquery { + range: Duration::from_secs(300), + resolution: Some(Duration::from_secs(60)), + child: range, + }); + let scope = scope(vec![coverage(source, vec![])]); + let mut scan = with_promql( + statistics(vec![edge(60_000, 960_000)], edge(60_000, 960_000)), + vec![100], + 100, + 6, + ); + scan.source_scan_bytes = 120_000; + let mut range = with_promql( + statistics(vec![edge(60_000, 960_000)], edge(600, 9_600)), + vec![100], + 100, + 6, + ); + range.promql.as_mut().unwrap().window_samples_per_series = Some(100); + let mut subquery = with_promql( + statistics(vec![edge(600, 9_600)], edge(100, 1_600)), + vec![100], + 100, + 1, + ); + subquery.promql.as_mut().unwrap().subquery_steps = Some(6); + let provided = HashMap::from([ + ("query-2".into(), evidence(scan)), + ("query-1".into(), evidence(range)), + ("query-0".into(), evidence(subquery)), + ]); + + let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); + assert_eq!( + dag.nodes + .iter() + .map(|node| node.operator) + .collect::>(), + vec![ + PhysicalOperator::Scan, + PhysicalOperator::PromqlRange, + PhysicalOperator::PromqlSubquery, + ] + ); + let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).unwrap(); + assert_eq!(estimate.scan_bytes, 120_000); + assert!(estimate.peak_memory_bytes >= 160_000); + } + + #[test] + fn promql_info_enrichment_lowers_its_source_as_a_scoped_scan() { + use asap_types::pre_asap::{Column, DataType, QueryExpr, Schema, Source}; + use std::rc::Rc; + + let primary = Source::TimeSeries { metric: "m".into() }; + let info = Source::TimeSeries { + metric: "target_info".into(), + }; + let child = Rc::new(QueryExpr::Scan { + source: primary.clone(), + predicates: vec![], + schema: Schema::new(vec![Column::new("value", DataType::Float64, false)]), + }); + let root = Rc::new(QueryExpr::PromqlInfoEnrich { + selector: vec![], + child, + }); + let scope = scope(vec![coverage(primary, vec![]), coverage(info, vec![])]); + let mut left = with_promql( + statistics(vec![edge(100, 1_600)], edge(100, 1_600)), + vec![10], + 10, + 10, + ); + left.source_scan_bytes = 800; + let mut right = with_promql( + statistics(vec![edge(20, 1_000)], edge(20, 1_000)), + vec![2], + 2, + 10, + ); + right.source_scan_bytes = 400; + let mut enrich = with_promql( + statistics(vec![edge(100, 1_600), edge(20, 1_000)], edge(100, 2_400)), + vec![10, 2], + 10, + 10, + ); + enrich.key_bytes = Some(16); + enrich.hash_join_build_side = Some(HashJoinBuildSide::Right); + let provided = HashMap::from([ + ("query-1".into(), evidence(left)), + ("query-0-info-scan".into(), evidence(right)), + ("query-0".into(), evidence(enrich)), + ]); + + let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); + assert_eq!(dag.nodes[1].operator, PhysicalOperator::Scan); + assert_eq!(dag.nodes[1].source_coverage, Some(scope.sources[1].clone())); + assert_eq!(dag.nodes[2].children.len(), 2); + let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).unwrap(); + assert_eq!(estimate.scan_bytes, 1_200); + } + + #[test] + fn promql_scalar_leaf_is_distinct_from_sql_current_timestamp() { + use asap_types::pre_asap::QueryExpr; + use std::rc::Rc; + + let root = Rc::new(QueryExpr::promql_scalar(5.0)); + let scope = scope(vec![coverage( + asap_types::pre_asap::Source::TimeSeries { + metric: "scope-anchor".into(), + }, + vec![], + )]); + let scalar = with_promql(statistics(vec![], edge(10, 80)), vec![], 0, 10); + let provided = HashMap::from([("query-0".into(), evidence(scalar))]); + let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); + assert_eq!(dag.nodes[0].operator, PhysicalOperator::PromqlScalarLeaf); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).is_ok()); + + assert_eq!( + lower_query_physical_dag(&Rc::new(QueryExpr::CurrentTimestamp), &scope, &provided), + Err(AnalyticalCostError::UnsupportedQueryOperator) + ); + } + + #[test] + fn promql_vector_scalar_binary_keeps_two_edges_without_label_state() { + use asap_types::pre_asap::{ + ArithmeticOpKind, BinaryOpKind, Column, DataType, QueryExpr, Schema, Source, + }; + use std::rc::Rc; + + let source = Source::TimeSeries { metric: "m".into() }; + let vector = Rc::new(QueryExpr::Scan { + source: source.clone(), + predicates: vec![], + schema: Schema::new(vec![Column::new("value", DataType::Float64, false)]), + }); + let scalar = Rc::new(QueryExpr::promql_scalar(2.0)); + let root = Rc::new(QueryExpr::BinaryOp { + op: BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul), + lhs: vector, + rhs: scalar, + vector_match: None, + }); + let scope = scope(vec![coverage(source, vec![])]); + let mut vector = with_promql( + statistics(vec![edge(1_000, 16_000)], edge(1_000, 16_000)), + vec![100], + 100, + 10, + ); + vector.source_scan_bytes = 8_000; + let scalar = with_promql(statistics(vec![], edge(10, 80)), vec![], 0, 10); + let binary = with_promql( + statistics(vec![edge(1_000, 16_000), edge(10, 80)], edge(1_000, 16_000)), + vec![100, 0], + 100, + 10, + ); + let provided = HashMap::from([ + ("query-1".into(), evidence(vector)), + ("query-2".into(), evidence(scalar)), + ("query-0".into(), evidence(binary)), + ]); + + let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); + assert_eq!(dag.nodes.last().unwrap().children.len(), 2); + assert_eq!( + dag.nodes.last().unwrap().operator, + PhysicalOperator::PromqlVectorBinary + ); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).is_ok()); + } + + #[test] + fn promql_relabel_sample_and_per_series_operators_are_costed() { + use asap_types::pre_asap::{ + AggIntent, Column, DataType, GroupKeys, QueryExpr, Reduction, SampleKind, Schema, + Source, + }; + use std::rc::Rc; + + let source = Source::TimeSeries { metric: "m".into() }; + let scan = Rc::new(QueryExpr::Scan { + source: source.clone(), + predicates: vec![], + schema: Schema::new(vec![Column::new("value", DataType::Float64, false)]), + }); + let relabel = Rc::new(QueryExpr::PromqlRelabel { + dst: "service".into(), + value: Rc::new(QueryExpr::FunctionCall { + name: "label_replace".into(), + args: vec![], + }), + child: scan, + }); + let sample = Rc::new(QueryExpr::PromqlSeriesSample { + by: GroupKeys::none(), + kind: SampleKind::LimitK(10), + child: relabel, + }); + let root = Rc::new(QueryExpr::Aggregate { + reduction: Reduction::PerEntity, + measures: vec![AggIntent::Rate], + output_names: vec![], + having: None, + child: sample, + }); + let scope = scope(vec![coverage(source, vec![])]); + let mut scan = with_promql( + statistics(vec![edge(1_000, 16_000)], edge(1_000, 16_000)), + vec![100], + 100, + 10, + ); + scan.source_scan_bytes = 8_000; + let mut relabel = with_promql( + statistics(vec![edge(1_000, 16_000)], edge(1_000, 20_000)), + vec![100], + 95, + 10, + ); + relabel.promql.as_mut().unwrap().scalar_ops_per_row = Some(8); + let mut sample = with_promql( + statistics(vec![edge(1_000, 20_000)], edge(100, 2_000)), + vec![95], + 10, + 10, + ); + sample.group_count = Some(1); + sample.key_bytes = Some(24); + sample.k = Some(10); + let mut aggregate = with_promql( + statistics(vec![edge(100, 2_000)], edge(10, 200)), + vec![10], + 10, + 10, + ); + aggregate.aggregate_value_bytes = Some(16); + aggregate.promql.as_mut().unwrap().scalar_ops_per_row = Some(2); + let provided = HashMap::from([ + ("query-3".into(), evidence(scan)), + ("query-2".into(), evidence(relabel)), + ("query-1".into(), evidence(sample)), + ("query-0".into(), evidence(aggregate)), + ]); + + let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); + assert_eq!( + dag.nodes + .iter() + .map(|node| node.operator) + .collect::>(), + vec![ + PhysicalOperator::Scan, + PhysicalOperator::PromqlRelabel, + PhysicalOperator::PromqlSeriesSample, + PhysicalOperator::PromqlPerSeries, + ] + ); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).is_ok()); + } } From d1037473e6d52b8ba0f922a69d4e6464b677af06 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 09:45:09 -0600 Subject: [PATCH 15/48] fix(cost): migrate PromQL lowering to physical identity evidence --- .../src/analytical_lowering.rs | 393 ++++++++++++------ 1 file changed, 268 insertions(+), 125 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index f1a3677e..77c9b36f 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -411,7 +411,7 @@ pub fn lower_query_physical_dag( if range.is_zero() { return Err(AnalyticalCostError::MissingOrZero("range")); } - self.lower_unary(query, id, PhysicalOperator::PromqlRange, child) + self.lower_unary(query, occurrence, PhysicalOperator::PromqlRange, child) } QueryExpr::PromqlSubquery { range, @@ -419,16 +419,30 @@ pub fn lower_query_physical_dag( child, } => { let child_id = self.lower(child)?; - let statistics = self.stats(&id)?; + let children = vec![child_id.clone()]; + let evidence = self.resolve( + query, + PhysicalOperator::PromqlSubquery, + occurrence, + false, + &children, + None, + )?; + let statistics = &evidence.statistics; let child_statistics = self.node_statistics(&child_id)?; - require_unary_edge(&id, &statistics, &child_id, child_statistics)?; + require_unary_edge( + &evidence.physical_id, + statistics, + &child_id, + child_statistics, + )?; require_promql_unary_edge( PhysicalOperator::PromqlSubquery, - &statistics, + statistics, child_statistics, )?; - require_operator_statistics(PhysicalOperator::PromqlSubquery, &statistics)?; - let supplied = require_promql_statistics(&statistics, 1)? + require_operator_statistics(PhysicalOperator::PromqlSubquery, statistics)?; + let supplied = require_promql_statistics(statistics, 1)? .subquery_steps .ok_or(AnalyticalCostError::MissingOrZero("subquery_steps"))?; if let Some(expected) = subquery_steps(*range, *resolution)? { @@ -438,58 +452,88 @@ pub fn lower_query_physical_dag( )); } } - Ok(self.push( - id, - PhysicalOperator::PromqlSubquery, - &statistics, - vec![child_id], - None, - )) + self.push(evidence, PhysicalOperator::PromqlSubquery, children, None) } QueryExpr::TimeShift { child, .. } => { self.lower_unary(query, occurrence, PhysicalOperator::PassThrough, child) } QueryExpr::PromqlRelabel { child, .. } => { - self.lower_unary(query, id, PhysicalOperator::PromqlRelabel, child) + self.lower_unary(query, occurrence, PhysicalOperator::PromqlRelabel, child) } QueryExpr::PromqlSeriesSample { by, kind, child, .. } => { let child_id = self.lower(child)?; - let statistics = self.stats(&id)?; + let children = vec![child_id.clone()]; + let evidence = self.resolve( + query, + PhysicalOperator::PromqlSeriesSample, + occurrence, + false, + &children, + None, + )?; + let statistics = &evidence.statistics; let child_statistics = self.node_statistics(&child_id)?; - require_unary_edge(&id, &statistics, &child_id, child_statistics)?; + require_unary_edge( + &evidence.physical_id, + statistics, + &child_id, + child_statistics, + )?; require_promql_unary_edge( PhysicalOperator::PromqlSeriesSample, - &statistics, + statistics, child_statistics, )?; - validate_series_sample(by, *kind, &statistics)?; - require_operator_statistics(PhysicalOperator::PromqlSeriesSample, &statistics)?; - Ok(self.push( - id, + validate_series_sample(by, *kind, statistics)?; + require_operator_statistics(PhysicalOperator::PromqlSeriesSample, statistics)?; + self.push( + evidence, PhysicalOperator::PromqlSeriesSample, - &statistics, - vec![child_id], + children, None, - )) + ) } QueryExpr::PromqlInfoEnrich { selector, child } => { let left_id = self.lower(child)?; - let info_id = format!("{id}-info-scan"); - let coverage = bind_info_coverage(&info_id, selector, self.scope)?; - let info_statistics = self.stats(&info_id)?; - require_statistics_shape(&info_id, &info_statistics, 1)?; - require_scan_edges_equal(&info_statistics)?; - self.push( - info_id.clone(), + let coverage = bind_info_coverage( + &format!("occurrence-{occurrence}-info"), + selector, + self.scope, + )?; + let info_evidence = self.resolve( + query, + PhysicalOperator::Scan, + occurrence, + true, + &[], + Some(&coverage), + )?; + require_statistics_shape( + &info_evidence.physical_id, + &info_evidence.statistics, + 1, + )?; + require_scan_edges_equal(&info_evidence.statistics)?; + let info_statistics = info_evidence.statistics.clone(); + let info_id = self.push( + info_evidence, PhysicalOperator::Scan, - &info_statistics, vec![], Some(coverage), - ); - let statistics = self.stats(&id)?; - require_statistics_shape(&id, &statistics, 2)?; + )?; + let children = vec![left_id.clone(), info_id.clone()]; + let evidence = self.resolve( + query, + PhysicalOperator::PromqlInfoEnrich, + occurrence, + false, + &children, + None, + )?; + let statistics = &evidence.statistics; + require_statistics_shape(&evidence.physical_id, statistics, 2)?; if statistics.inputs[0] != self.node_statistics(&left_id)?.output || statistics.inputs[1] != info_statistics.output { @@ -498,18 +542,12 @@ pub fn lower_query_physical_dag( )); } require_promql_binary_edges( - &statistics, + statistics, self.node_statistics(&left_id)?, &info_statistics, )?; - require_operator_statistics(PhysicalOperator::PromqlInfoEnrich, &statistics)?; - Ok(self.push( - id, - PhysicalOperator::PromqlInfoEnrich, - &statistics, - vec![left_id, info_id], - None, - )) + require_operator_statistics(PhysicalOperator::PromqlInfoEnrich, statistics)?; + self.push(evidence, PhysicalOperator::PromqlInfoEnrich, children, None) } QueryExpr::BinaryOp { lhs, @@ -524,8 +562,17 @@ pub fn lower_query_physical_dag( } let left_id = self.lower(lhs)?; let right_id = self.lower(rhs)?; - let statistics = self.stats(&id)?; - require_statistics_shape(&id, &statistics, 2)?; + let children = vec![left_id.clone(), right_id.clone()]; + let evidence = self.resolve( + query, + PhysicalOperator::PromqlVectorBinary, + occurrence, + false, + &children, + None, + )?; + let statistics = &evidence.statistics; + require_statistics_shape(&evidence.physical_id, statistics, 2)?; if statistics.inputs[0] != self.node_statistics(&left_id)?.output || statistics.inputs[1] != self.node_statistics(&right_id)?.output { @@ -550,22 +597,21 @@ pub fn lower_query_physical_dag( )); } require_promql_binary_edges( - &statistics, + statistics, self.node_statistics(&left_id)?, self.node_statistics(&right_id)?, )?; - require_operator_statistics(PhysicalOperator::PromqlVectorBinary, &statistics)?; - Ok(self.push( - id, + require_operator_statistics(PhysicalOperator::PromqlVectorBinary, statistics)?; + self.push( + evidence, PhysicalOperator::PromqlVectorBinary, - &statistics, - vec![left_id, right_id], + children, None, - )) + ) } QueryExpr::PromqlVectorFromScalar(child) | QueryExpr::PromqlScalarFromVector(child) => { - self.lower_unary(query, id, PhysicalOperator::PromqlBridge, child) + self.lower_unary(query, occurrence, PhysicalOperator::PromqlBridge, child) } QueryExpr::PromqlScalarBridge(inner) if matches!( @@ -573,28 +619,36 @@ pub fn lower_query_physical_dag( QueryExpr::Literal(asap_types::pre_asap::ScalarValue::Float64(_)) ) => { - let statistics = self.stats(&id)?; - require_statistics_shape(&id, &statistics, 0)?; - require_operator_statistics(PhysicalOperator::PromqlScalarLeaf, &statistics)?; - Ok(self.push( - id, + let evidence = self.resolve( + query, PhysicalOperator::PromqlScalarLeaf, - &statistics, - vec![], + occurrence, + false, + &[], None, - )) + )?; + require_statistics_shape(&evidence.physical_id, &evidence.statistics, 0)?; + require_operator_statistics( + PhysicalOperator::PromqlScalarLeaf, + &evidence.statistics, + )?; + self.push(evidence, PhysicalOperator::PromqlScalarLeaf, vec![], None) } QueryExpr::EvalTimestamp => { - let statistics = self.stats(&id)?; - require_statistics_shape(&id, &statistics, 0)?; - require_operator_statistics(PhysicalOperator::PromqlScalarLeaf, &statistics)?; - Ok(self.push( - id, + let evidence = self.resolve( + query, PhysicalOperator::PromqlScalarLeaf, - &statistics, - vec![], + occurrence, + false, + &[], None, - )) + )?; + require_statistics_shape(&evidence.physical_id, &evidence.statistics, 0)?; + require_operator_statistics( + PhysicalOperator::PromqlScalarLeaf, + &evidence.statistics, + )?; + self.push(evidence, PhysicalOperator::PromqlScalarLeaf, vec![], None) } QueryExpr::Concat { children } => { let child_ids = children @@ -1328,12 +1382,35 @@ mod tests { } } + fn with_promql( + mut statistics: OperatorStatistics, + input_series: Vec, + output_series: u64, + evaluation_steps: u64, + ) -> OperatorStatistics { + statistics.promql = Some(crate::analytical_statistics::PromqlOperatorStatistics { + input_series, + output_series, + evaluation_steps, + window_samples_per_series: None, + subquery_steps: None, + scalar_ops_per_row: None, + }); + statistics + } + fn scripted<'a>( provided: &'a HashMap, ) -> impl Fn(PhysicalNodeRequest<'_>) -> Result + 'a { move |request| { - let key = if request.synthetic { + let key = if request.synthetic + && matches!( + request.logical_node, + asap_types::pre_asap::QueryExpr::PromqlInfoEnrich { .. } + ) { + format!("query-{}-info-scan", request.occurrence) + } else if request.synthetic { format!("query-{}-scan", request.occurrence) } else { format!("query-{}", request.occurrence) @@ -1347,23 +1424,6 @@ mod tests { } } - fn with_promql( - mut statistics: OperatorStatistics, - input_series: Vec, - output_series: u64, - evaluation_steps: u64, - ) -> OperatorStatistics { - statistics.promql = Some(crate::analytical_statistics::PromqlOperatorStatistics { - input_series, - output_series, - evaluation_steps, - window_samples_per_series: None, - subquery_steps: None, - scalar_ops_per_row: None, - }); - statistics - } - fn scope(sources: Vec) -> ComparisonScope { ComparisonScope { data_arrival: DataArrival::AtRest, @@ -2071,7 +2131,11 @@ mod tests { vec![], )]); let unavailable = HashMap::::new(); - for query in [&exact_quantile, &empty_sort_limit, &unsupported_window] { + assert_eq!( + lower_query_physical_dag(&exact_quantile, &scope, &scripted(&unavailable)), + Err(AnalyticalCostError::UnsupportedQueryOperator) + ); + for query in [&empty_sort_limit, &unsupported_window] { assert_eq!( lower_query_physical_dag(query, &scope, &scripted(&unavailable)), Err(AnalyticalCostError::UnsupportedQueryOperator) @@ -2127,7 +2191,7 @@ mod tests { ("query-0".into(), evidence(subquery)), ]); - let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); + let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)).unwrap(); assert_eq!( dag.nodes .iter() @@ -2139,7 +2203,7 @@ mod tests { PhysicalOperator::PromqlSubquery, ] ); - let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).unwrap(); + let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).unwrap(); assert_eq!(estimate.scan_bytes, 120_000); assert!(estimate.peak_memory_bytes >= 160_000); } @@ -2191,38 +2255,14 @@ mod tests { ("query-0".into(), evidence(enrich)), ]); - let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); + let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)).unwrap(); assert_eq!(dag.nodes[1].operator, PhysicalOperator::Scan); assert_eq!(dag.nodes[1].source_coverage, Some(scope.sources[1].clone())); assert_eq!(dag.nodes[2].children.len(), 2); - let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).unwrap(); + let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).unwrap(); assert_eq!(estimate.scan_bytes, 1_200); } - #[test] - fn promql_scalar_leaf_is_distinct_from_sql_current_timestamp() { - use asap_types::pre_asap::QueryExpr; - use std::rc::Rc; - - let root = Rc::new(QueryExpr::promql_scalar(5.0)); - let scope = scope(vec![coverage( - asap_types::pre_asap::Source::TimeSeries { - metric: "scope-anchor".into(), - }, - vec![], - )]); - let scalar = with_promql(statistics(vec![], edge(10, 80)), vec![], 0, 10); - let provided = HashMap::from([("query-0".into(), evidence(scalar))]); - let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); - assert_eq!(dag.nodes[0].operator, PhysicalOperator::PromqlScalarLeaf); - assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).is_ok()); - - assert_eq!( - lower_query_physical_dag(&Rc::new(QueryExpr::CurrentTimestamp), &scope, &provided), - Err(AnalyticalCostError::UnsupportedQueryOperator) - ); - } - #[test] fn promql_vector_scalar_binary_keeps_two_edges_without_label_state() { use asap_types::pre_asap::{ @@ -2239,18 +2279,18 @@ mod tests { let scalar = Rc::new(QueryExpr::promql_scalar(2.0)); let root = Rc::new(QueryExpr::BinaryOp { op: BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul), - lhs: vector, + lhs: Rc::clone(&vector), rhs: scalar, vector_match: None, }); let scope = scope(vec![coverage(source, vec![])]); - let mut vector = with_promql( + let mut vector_statistics = with_promql( statistics(vec![edge(1_000, 16_000)], edge(1_000, 16_000)), vec![100], 100, 10, ); - vector.source_scan_bytes = 8_000; + vector_statistics.source_scan_bytes = 8_000; let scalar = with_promql(statistics(vec![], edge(10, 80)), vec![], 0, 10); let binary = with_promql( statistics(vec![edge(1_000, 16_000), edge(10, 80)], edge(1_000, 16_000)), @@ -2259,18 +2299,121 @@ mod tests { 10, ); let provided = HashMap::from([ - ("query-1".into(), evidence(vector)), + ("query-1".into(), evidence(vector_statistics)), ("query-2".into(), evidence(scalar)), ("query-0".into(), evidence(binary)), ]); - let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); + let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)).unwrap(); assert_eq!(dag.nodes.last().unwrap().children.len(), 2); assert_eq!( dag.nodes.last().unwrap().operator, PhysicalOperator::PromqlVectorBinary ); - assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).is_ok()); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).is_ok()); + + let eval_timestamp = Rc::new(QueryExpr::BinaryOp { + op: BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul), + lhs: Rc::clone(&vector), + rhs: Rc::new(QueryExpr::EvalTimestamp), + vector_match: None, + }); + assert!(lower_query_physical_dag(&eval_timestamp, &scope, &scripted(&provided)).is_ok()); + + let sql_timestamp = Rc::new(QueryExpr::BinaryOp { + op: BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul), + lhs: vector, + rhs: Rc::new(QueryExpr::CurrentTimestamp), + vector_match: None, + }); + assert_eq!( + lower_query_physical_dag(&sql_timestamp, &scope, &scripted(&provided)), + Err(AnalyticalCostError::UnsupportedQueryOperator) + ); + } + + #[test] + fn promql_vector_binary_requires_explicit_label_match_state() { + use asap_types::pre_asap::{ + ArithmeticOpKind, BinaryOpKind, Column, DataType, QueryExpr, Schema, Source, + VectorMatch, VectorMatchKind, + }; + use std::rc::Rc; + + let left_source = Source::TimeSeries { + metric: "requests".into(), + }; + let right_source = Source::TimeSeries { + metric: "errors".into(), + }; + let scan = |source: Source| { + Rc::new(QueryExpr::Scan { + source, + predicates: vec![], + schema: Schema::new(vec![Column::new("value", DataType::Float64, false)]), + }) + }; + let root = Rc::new(QueryExpr::BinaryOp { + op: BinaryOpKind::Arithmetic(ArithmeticOpKind::Div), + lhs: scan(left_source.clone()), + rhs: scan(right_source.clone()), + vector_match: Some(VectorMatch { + kind: VectorMatchKind::On, + labels: vec!["service".into()], + grouping: None, + }), + }); + let scope = scope(vec![ + coverage(left_source, vec![]), + coverage(right_source, vec![]), + ]); + let mut left = with_promql( + statistics(vec![edge(1_000, 16_000)], edge(1_000, 16_000)), + vec![100], + 100, + 10, + ); + left.source_scan_bytes = 8_000; + let mut right = with_promql( + statistics(vec![edge(500, 8_000)], edge(500, 8_000)), + vec![50], + 50, + 10, + ); + right.source_scan_bytes = 4_000; + let mut binary = with_promql( + statistics( + vec![edge(1_000, 16_000), edge(500, 8_000)], + edge(400, 6_400), + ), + vec![100, 50], + 40, + 10, + ); + binary.key_bytes = Some(32); + binary.hash_join_build_side = Some(HashJoinBuildSide::Right); + let provided = HashMap::from([ + ("query-1".into(), evidence(left)), + ("query-2".into(), evidence(right)), + ("query-0".into(), evidence(binary)), + ]); + let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)).unwrap(); + let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).unwrap(); + assert_eq!(estimate.cpu_ops, 3_400.0); + assert_eq!(estimate.scan_bytes, 12_000); + + let mut missing = provided; + missing + .get_mut("query-0") + .unwrap() + .statistics + .hash_join_build_side = None; + assert_eq!( + lower_query_physical_dag(&root, &scope, &scripted(&missing)), + Err(AnalyticalCostError::MissingOrStale( + "vector_binary_label_match_statistics" + )) + ); } #[test] @@ -2346,7 +2489,7 @@ mod tests { ("query-0".into(), evidence(aggregate)), ]); - let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); + let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)).unwrap(); assert_eq!( dag.nodes .iter() @@ -2359,6 +2502,6 @@ mod tests { PhysicalOperator::PromqlPerSeries, ] ); - assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).is_ok()); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).is_ok()); } } From 7ca05c3f791d41e32002576ce64d45f0cc2d1dd5 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 09:48:42 -0600 Subject: [PATCH 16/48] docs(cost): define PromQL physical evidence --- .../analytical-resource-cost.md | 59 ++++++++++++++++--- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 2af7c9fa..1aad4be9 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -155,6 +155,10 @@ OperatorStatistics { aggregate_value_bytes, k, hash_join_build_side, + promql: PromqlOperatorStatistics { + input_series, output_series, evaluation_steps, + window_samples_per_series, subquery_steps, scalar_ops_per_row, + }, } ``` @@ -291,11 +295,22 @@ The supported mappings are: | global non-empty-key Sort followed by Limit | heap TopK, with `k = offset + n` from the query IR | | partitioned Sort followed by Limit | Sort → Limit | | RowNumber/Rank/DenseRank SQLWindowFunc with non-empty order_by | ordered in-memory Window | +| TimeRange | PromqlRange sliding window | +| PromqlSubquery | materialized PromqlSubquery | +| BinaryOp | two-input PromqlVectorBinary; vector/vector uses explicit label-match state | +| PromqlRelabel | PromqlRelabel | +| PromqlInfoEnrich | left input plus an explicit scoped info-metric Scan | +| PromqlSeriesSample | PromqlSeriesSample using the existing SampleKind | +| PromqlVectorFromScalar / PromqlScalarFromVector | unary PromqlBridge | +| PromqlScalarBridge(float) / EvalTimestamp | zero-input PromqlScalarLeaf | +| supported fixed-state per-entity reduction | PromqlPerSeries | +| Absent / AbsentOverTime | PromqlPresence | | TimeShift | PassThrough | -Per-entity reductions, HAVING, ordered/distribution-dependent intents such as -exact quantile or cardinality, Top-K aggregate intents, and extensions remain -unavailable until they have an explicit physical algorithm. Hash-join lowering +HAVING, ordered/distribution-dependent intents such as exact quantile or +cardinality, Top-K aggregate intents, non-fixed-state per-entity algorithms, +and extensions remain unavailable until they have an explicit physical +algorithm. Hash-join lowering also uses the bound left and right output schemas to prove that every equality compares one column from each side; same-side or out-of-range `ColumnId`s fail closed. @@ -314,11 +329,27 @@ embedded physical identity and that every node's buffer equals the provider snapshot before returning statistics, preventing the public node and evidence views from silently drifting apart. -Cross/non-equi joins, `INTERSECT`, `EXCEPT`, distinct `UNION`, PromQL -range/subquery execution, vector matching, and PromQL-specific enrichment/ -relabel/sample operators stay unavailable. Their cost requires a physical -implementation or multiplicity/state facts that the current physical-operator -vocabulary cannot represent; they are not treated as free pass-through work. +PromQL rows and logical bytes are totals for one workload query evaluation; +range and subquery values therefore include their internal evaluation steps. +`evaluation_steps` and `subquery_steps` validate and cost that internal shape, +while `ComparisonScope` alone multiplies the completed query over the workload +horizon. They are never multiplied into the horizon a second time. Series +cardinality is carried in child order and must agree across every physical +edge. Missing window, step, series, expression-work, label-key, or accumulator +evidence makes the entire candidate unavailable. + +`info()` resolves its default `target_info` metric, or one exact `__name__` +matcher, to a concrete existing `Source::TimeSeries` coverage. Its right side +is an ordinary physical Scan with its own statistics, buffer, snapshot, and +source bytes; the enrichment operator itself performs no source I/O. Other +label matchers remain local enrichment/filter work. Non-exact metric-name +selection stays unavailable until a catalog resolver can return the complete +concrete source set. + +Cross/non-equi joins, `INTERSECT`, `EXCEPT`, distinct `UNION`, opaque PromQL +extensions, non-exact `info()` metric selection, and SQL `CurrentTimestamp` as +a PromQL scalar stay unavailable. They are not modeled as free pass-through +work. ## Physical operator formulas @@ -337,7 +368,17 @@ the DAG rules above. | Hash join | `left_rows + right_rows + output_rows` | selected build-side logical bytes plus 16 bytes of hash metadata per build row | `0` beyond children | | Concat | `output_rows` | one output row/batch | `0` | | Ordered window | `rows × ceil(log2(rows))` | live partition/input bytes | `0` | -| Limit | `limit_rows_consumed` (including `OFFSET`) | one output row/batch | `0` | +| Limit | `output_rows` | one output row/batch | `0` | +| PromQL range | `input_rows` | `input_series × window_samples_per_series × decoded_sample_bytes` | `0` beyond child | +| PromQL subquery | `input_rows + output_rows` | materialized inner-step logical bytes | `0` beyond child | +| PromQL vector binary | left + right + output rows | selected-side label-match hash state; one output row for scalar/vector | `0` beyond children | +| PromQL relabel | `input_rows × scalar_ops_per_row` | one output row/batch | `0` | +| PromQL info enrichment | left + info + output rows | info-side label-match hash state | `0` beyond its explicit Scan child | +| PromQL series sample | input rows + input series | selected-series key/hash state | `0` | +| PromQL scalar/vector bridge | input + output rows | one output row/batch | `0` | +| PromQL scalar leaf | output rows | one output row/batch | `0` | +| PromQL per-series intent | `input_rows × scalar_ops_per_row` | `input_series × accumulator_bytes` | `0` | +| PromQL absence | input work + synthesized output | one output row/batch | `0` | These formulas name physical implementations. An external sort must add spill writes and reads; a nested-loop join must not use the hash-join formula. From 3b444766fe86c04417f51f727d2c90f1a7af8ea8 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 10:06:03 -0600 Subject: [PATCH 17/48] fix(cost): close PromQL physical evidence gaps --- .../asap-aware-mapping/src/analytical_cost.rs | 52 +++- .../src/analytical_lowering.rs | 283 +++++++++++++++++- .../src/analytical_statistics.rs | 3 - .../analytical-resource-cost.md | 27 +- 4 files changed, 351 insertions(+), 14 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 03615398..4f8ec6ea 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -422,6 +422,30 @@ fn validate_operator_statistics( }); } } + if matches!(node.operator, PhysicalOperator::PromqlSubquery) { + let subquery_steps = promql + .subquery_steps + .filter(|steps| *steps > 0) + .ok_or(AnalyticalCostError::MissingOrZero("subquery_steps"))?; + let child = &statistics[node.children[0].as_str()]; + let child_steps = child + .promql + .as_ref() + .ok_or(AnalyticalCostError::MissingOrStale( + "child_promql_operator_statistics", + ))? + .evaluation_steps; + let expected = promql + .evaluation_steps + .checked_mul(subquery_steps) + .ok_or(AnalyticalCostError::Overflow)?; + if child_steps != expected { + return Err(AnalyticalCostError::InvalidOperatorStatistics { + node: node.id.clone(), + reason: "subquery child steps do not equal parent steps times subquery steps", + }); + } + } } Ok(()) } @@ -595,7 +619,7 @@ pub fn estimate_operator( scan_bytes: 0, } } - PhysicalOperator::PromqlVectorBinary | PhysicalOperator::PromqlInfoEnrich => { + PhysicalOperator::PromqlVectorBinary => { let promql = require_promql_statistics(&statistics, 2)?; let right = input(1)?; let scalar_vector = matches!(operator, PhysicalOperator::PromqlVectorBinary) @@ -626,6 +650,32 @@ pub fn estimate_operator( scan_bytes: 0, } } + PhysicalOperator::PromqlInfoEnrich => { + let promql = require_promql_statistics(&statistics, 2)?; + let right = input(1)?; + if statistics.hash_join_build_side != Some(HashJoinBuildSide::Right) { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "info enrichment must build its label index from the right side", + )); + } + let key_bytes = statistics + .key_bytes + .ok_or(AnalyticalCostError::MissingOrZero("key_bytes"))?; + let predicate_ops = promql.scalar_ops_per_row.unwrap_or(0); + ResourceEstimate { + cpu_ops: left.rows as f64 + + right.rows as f64 + + output.rows as f64 + + right.rows as f64 * predicate_ops as f64, + peak_memory_bytes: checked_bytes(&[ + promql.input_series[1], + key_bytes + .checked_add(16) + .ok_or(AnalyticalCostError::Overflow)?, + ])?, + scan_bytes: 0, + } + } PhysicalOperator::PromqlRelabel => { let promql = require_promql_statistics(&statistics, 1)?; let operations = promql diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index 77c9b36f..3922c772 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -445,6 +445,22 @@ pub fn lower_query_physical_dag( let supplied = require_promql_statistics(statistics, 1)? .subquery_steps .ok_or(AnalyticalCostError::MissingOrZero("subquery_steps"))?; + let parent_steps = require_promql_statistics(statistics, 1)?.evaluation_steps; + let expected_child_steps = parent_steps + .checked_mul(supplied) + .ok_or(AnalyticalCostError::Overflow)?; + let child_steps = child_statistics + .promql + .as_ref() + .ok_or(AnalyticalCostError::MissingOrStale( + "child_promql_operator_statistics", + ))? + .evaluation_steps; + if child_steps != expected_child_steps { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "subquery child steps do not equal parent steps times subquery steps", + )); + } if let Some(expected) = subquery_steps(*range, *resolution)? { if supplied != expected { return Err(AnalyticalCostError::InconsistentOperatorStatistics( @@ -546,6 +562,16 @@ pub fn lower_query_physical_dag( self.node_statistics(&left_id)?, &info_statistics, )?; + if selector.iter().any(|matcher| matcher.label != "__name__") + && !matches!( + require_promql_statistics(statistics, 2)?.scalar_ops_per_row, + Some(value) if value > 0 + ) + { + return Err(AnalyticalCostError::MissingOrZero( + "info_matcher_ops_per_row", + )); + } require_operator_statistics(PhysicalOperator::PromqlInfoEnrich, statistics)?; self.push(evidence, PhysicalOperator::PromqlInfoEnrich, children, None) } @@ -938,6 +964,11 @@ fn require_operator_statistics( if output.rows != input.rows || promql.output_series != promql.input_series[0] { return invalid("info enrichment changes left sample or series cardinality"); } + if statistics.hash_join_build_side + != Some(crate::analytical_cost::HashJoinBuildSide::Right) + { + return invalid("info enrichment must build its label index from the right side"); + } } PhysicalOperator::PromqlRelabel => { let promql = require_promql_statistics(statistics, 1)?; @@ -1312,6 +1343,13 @@ fn fixed_state_per_series_intent(intent: &asap_types::pre_asap::AggIntent) -> bo matches!( intent, AggIntent::Rate + | AggIntent::Count { .. } + | AggIntent::Sum { .. } + | AggIntent::Min { .. } + | AggIntent::Max { .. } + | AggIntent::Avg { .. } + | AggIntent::StdDev { .. } + | AggIntent::Variance { .. } | AggIntent::Increase | AggIntent::Changes | AggIntent::Delta @@ -1320,6 +1358,12 @@ fn fixed_state_per_series_intent(intent: &asap_types::pre_asap::AggIntent) -> bo | AggIntent::Resets | AggIntent::PredictLinear { .. } | AggIntent::DoubleExpSmoothing { .. } + | AggIntent::HistogramCount + | AggIntent::HistogramSum + | AggIntent::HistogramAvg + | AggIntent::HistogramStdDev + | AggIntent::HistogramStdVar + | AggIntent::HistogramFraction { .. } | AggIntent::Math(_) | AggIntent::PresentOverTime | AggIntent::TimeFn(_) @@ -2209,10 +2253,133 @@ mod tests { } #[test] - fn promql_info_enrichment_lowers_its_source_as_a_scoped_scan() { + fn nested_promql_subqueries_compose_internal_evaluation_steps() { + use asap_types::pre_asap::{Column, DataType, QueryExpr, Schema, Source}; + use std::{rc::Rc, time::Duration}; + + let source = Source::TimeSeries { metric: "m".into() }; + let scan = Rc::new(QueryExpr::Scan { + source: source.clone(), + predicates: vec![], + schema: Schema::new(vec![Column::new("value", DataType::Float64, false)]), + }); + let range = Rc::new(QueryExpr::TimeRange { + range: Duration::from_secs(60), + child: scan, + }); + let inner = Rc::new(QueryExpr::PromqlSubquery { + range: Duration::from_secs(60), + resolution: Some(Duration::from_secs(60)), + child: range, + }); + let root = Rc::new(QueryExpr::PromqlSubquery { + range: Duration::from_secs(120), + resolution: Some(Duration::from_secs(60)), + child: inner, + }); + let scope = scope(vec![coverage(source, vec![])]); + let mut scan = with_promql( + statistics(vec![edge(60, 960)], edge(60, 960)), + vec![10], + 10, + 6, + ); + scan.source_scan_bytes = 480; + let mut range = with_promql( + statistics(vec![edge(60, 960)], edge(60, 960)), + vec![10], + 10, + 6, + ); + range.promql.as_mut().unwrap().window_samples_per_series = Some(6); + let mut inner = with_promql( + statistics(vec![edge(60, 960)], edge(30, 480)), + vec![10], + 10, + 3, + ); + inner.promql.as_mut().unwrap().subquery_steps = Some(2); + let mut outer = with_promql( + statistics(vec![edge(30, 480)], edge(10, 160)), + vec![10], + 10, + 1, + ); + outer.promql.as_mut().unwrap().subquery_steps = Some(3); + let provided = HashMap::from([ + ("query-3".into(), evidence(scan)), + ("query-2".into(), evidence(range)), + ("query-1".into(), evidence(inner)), + ("query-0".into(), evidence(outer)), + ]); + assert!(lower_query_physical_dag(&root, &scope, &scripted(&provided)).is_ok()); + + let mut overflow = provided; + overflow + .get_mut("query-0") + .unwrap() + .statistics + .promql + .as_mut() + .unwrap() + .evaluation_steps = u64::MAX; + assert_eq!( + lower_query_physical_dag(&root, &scope, &scripted(&overflow)), + Err(AnalyticalCostError::Overflow) + ); + } + + #[test] + fn scalar_only_promql_accepts_empty_scope_but_scan_queries_do_not() { use asap_types::pre_asap::{Column, DataType, QueryExpr, Schema, Source}; use std::rc::Rc; + let empty_scope = scope(vec![]); + let scalar = Rc::new(QueryExpr::promql_scalar(5.0)); + let scalar_statistics = with_promql(statistics(vec![], edge(10, 80)), vec![], 0, 10); + let provided = HashMap::from([("query-0".into(), evidence(scalar_statistics))]); + let dag = lower_query_physical_dag(&scalar, &empty_scope, &scripted(&provided)).unwrap(); + assert_eq!(dag.nodes[0].operator, PhysicalOperator::PromqlScalarLeaf); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, &empty_scope, &dag.evidence).is_ok()); + + let timestamp = Rc::new(QueryExpr::EvalTimestamp); + let timestamp_statistics = with_promql(statistics(vec![], edge(10, 80)), vec![], 0, 10); + let provided = HashMap::from([("query-0".into(), evidence(timestamp_statistics))]); + let dag = lower_query_physical_dag(×tamp, &empty_scope, &scripted(&provided)).unwrap(); + assert_eq!(dag.nodes[0].operator, PhysicalOperator::PromqlScalarLeaf); + + let vector = Rc::new(QueryExpr::PromqlVectorFromScalar(Rc::new( + QueryExpr::promql_scalar(1.0), + ))); + let scalar_statistics = with_promql(statistics(vec![], edge(10, 80)), vec![], 0, 10); + let vector_statistics = + with_promql(statistics(vec![edge(10, 80)], edge(10, 80)), vec![0], 0, 10); + let provided = HashMap::from([ + ("query-1".into(), evidence(scalar_statistics)), + ("query-0".into(), evidence(vector_statistics)), + ]); + let dag = lower_query_physical_dag(&vector, &empty_scope, &scripted(&provided)).unwrap(); + assert_eq!(dag.nodes[0].operator, PhysicalOperator::PromqlScalarLeaf); + assert_eq!(dag.nodes[1].operator, PhysicalOperator::PromqlBridge); + + let scan = Rc::new(QueryExpr::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::new(vec![Column::new("value", DataType::Float64, false)]), + }); + assert!(matches!( + lower_query_physical_dag(&scan, &empty_scope, &scripted(&provided)), + Err(AnalyticalCostError::ScanOutsideComparisonScope(_)) + )); + } + + #[test] + fn promql_info_enrichment_lowers_its_source_as_a_scoped_scan() { + use asap_types::pre_asap::{ + Column, CompareOpKind, DataType, InfoMatcher, QueryExpr, Schema, Source, + }; + use std::rc::Rc; + let primary = Source::TimeSeries { metric: "m".into() }; let info = Source::TimeSeries { metric: "target_info".into(), @@ -2223,7 +2390,11 @@ mod tests { schema: Schema::new(vec![Column::new("value", DataType::Float64, false)]), }); let root = Rc::new(QueryExpr::PromqlInfoEnrich { - selector: vec![], + selector: vec![InfoMatcher { + label: "job".into(), + op: CompareOpKind::Eq, + value: "api".into(), + }], child, }); let scope = scope(vec![coverage(primary, vec![]), coverage(info, vec![])]); @@ -2249,6 +2420,7 @@ mod tests { ); enrich.key_bytes = Some(16); enrich.hash_join_build_side = Some(HashJoinBuildSide::Right); + enrich.promql.as_mut().unwrap().scalar_ops_per_row = Some(3); let provided = HashMap::from([ ("query-1".into(), evidence(left)), ("query-0-info-scan".into(), evidence(right)), @@ -2261,6 +2433,36 @@ mod tests { assert_eq!(dag.nodes[2].children.len(), 2); let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).unwrap(); assert_eq!(estimate.scan_bytes, 1_200); + assert_eq!(estimate.cpu_ops, 400.0); + + let mut missing_matcher_work = provided.clone(); + missing_matcher_work + .get_mut("query-0") + .unwrap() + .statistics + .promql + .as_mut() + .unwrap() + .scalar_ops_per_row = None; + assert_eq!( + lower_query_physical_dag(&root, &scope, &scripted(&missing_matcher_work)), + Err(AnalyticalCostError::MissingOrZero( + "info_matcher_ops_per_row" + )) + ); + + let mut wrong_build_side = provided; + wrong_build_side + .get_mut("query-0") + .unwrap() + .statistics + .hash_join_build_side = Some(HashJoinBuildSide::Left); + assert_eq!( + lower_query_physical_dag(&root, &scope, &scripted(&wrong_build_side)), + Err(AnalyticalCostError::InconsistentOperatorStatistics( + "info enrichment must build its label index from the right side" + )) + ); } #[test] @@ -2504,4 +2706,81 @@ mod tests { ); assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).is_ok()); } + + #[test] + fn promql_over_time_and_native_histogram_intents_use_per_series_work() { + use asap_types::pre_asap::AggIntent; + use asap_types::types::AccuracyTarget; + + let intents = vec![ + AggIntent::Count { + accuracy: AccuracyTarget::Exact, + }, + AggIntent::Sum { col: None }, + AggIntent::Min { col: None }, + AggIntent::Max { col: None }, + AggIntent::Avg { col: None }, + AggIntent::StdDev { + col: None, + population: true, + }, + AggIntent::Variance { + col: None, + population: true, + }, + AggIntent::HistogramCount, + AggIntent::HistogramSum, + AggIntent::HistogramAvg, + AggIntent::HistogramStdDev, + AggIntent::HistogramStdVar, + AggIntent::HistogramFraction { + lower: 0.0, + upper: 1.0, + }, + ]; + for intent in intents { + assert_eq!( + aggregate_operator(&asap_types::pre_asap::Reduction::PerEntity, &[intent]), + Some(PhysicalOperator::PromqlPerSeries) + ); + } + } + + #[test] + fn promql_presence_can_synthesize_one_row_from_an_empty_scan() { + use asap_types::pre_asap::{ + AggIntent, Column, DataType, QueryExpr, Reduction, Schema, Source, + }; + use std::rc::Rc; + + let source = Source::TimeSeries { + metric: "missing".into(), + }; + let scan = Rc::new(QueryExpr::Scan { + source: source.clone(), + predicates: vec![], + schema: Schema::new(vec![Column::new("value", DataType::Float64, false)]), + }); + let root = Rc::new(QueryExpr::Aggregate { + reduction: Reduction::PerEntity, + measures: vec![AggIntent::Absent], + output_names: vec![], + having: None, + child: scan, + }); + let scope = scope(vec![coverage(source, vec![])]); + let scan = with_promql(statistics(vec![edge(0, 0)], edge(0, 0)), vec![0], 0, 1); + let mut presence = with_promql(statistics(vec![edge(0, 0)], edge(1, 8)), vec![0], 1, 1); + presence.promql.as_mut().unwrap().scalar_ops_per_row = Some(1); + let provided = HashMap::from([ + ("query-1".into(), evidence(scan)), + ("query-0".into(), evidence(presence)), + ]); + let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)).unwrap(); + assert_eq!( + dag.nodes.last().unwrap().operator, + PhysicalOperator::PromqlPresence + ); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).is_ok()); + } } diff --git a/crates/asap-aware-mapping/src/analytical_statistics.rs b/crates/asap-aware-mapping/src/analytical_statistics.rs index f820ec7b..316e2a92 100644 --- a/crates/asap-aware-mapping/src/analytical_statistics.rs +++ b/crates/asap-aware-mapping/src/analytical_statistics.rs @@ -72,9 +72,6 @@ impl ComparisonScope { if self.horizon.0 == 0 { return Err(AnalyticalCostError::MissingOrZero("horizon")); } - if self.sources.is_empty() { - return Err(AnalyticalCostError::MissingComparisonScope("sources")); - } if self .sources .iter() diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 1aad4be9..82a2a1f4 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -129,8 +129,11 @@ the source, snapshot identifier, and canonical predicates. Raw and candidate scopes must match exactly in every field before their estimates are compared. Unknown subsumption such as "this wider retained summary covers the requested interval" is not guessed here; it requires a separate semantic coverage proof. -Missing sources, empty snapshot identifiers, invalid recurrence, or a zero -horizon fail closed. +An empty source set is valid only for a fully source-free logical DAG such as +`time()`, a number literal, or `vector(1)`. After lowering, the reachable Scan +coverage set must equal the scope source set: a Scan query with an empty scope, +or a source-free query with a non-empty scope, fails closed. Empty snapshot +identifiers, invalid recurrence, or a zero horizon also fail closed. Every reachable physical `Scan` carries one exact `SourceCoverage` copied from this scope. That coverage includes the existing `Source`, its provider-owned @@ -303,7 +306,9 @@ The supported mappings are: | PromqlSeriesSample | PromqlSeriesSample using the existing SampleKind | | PromqlVectorFromScalar / PromqlScalarFromVector | unary PromqlBridge | | PromqlScalarBridge(float) / EvalTimestamp | zero-input PromqlScalarLeaf | -| supported fixed-state per-entity reduction | PromqlPerSeries | +| Count/Sum/Min/Max/Avg/StdDev/Variance with PerEntity (`*_over_time`) | PromqlPerSeries | +| native histogram count/sum/avg/stddev/stdvar/fraction accessor | PromqlPerSeries | +| other supported fixed-state per-entity reduction | PromqlPerSeries | | Absent / AbsentOverTime | PromqlPresence | | TimeShift | PassThrough | @@ -331,8 +336,11 @@ views from silently drifting apart. PromQL rows and logical bytes are totals for one workload query evaluation; range and subquery values therefore include their internal evaluation steps. -`evaluation_steps` and `subquery_steps` validate and cost that internal shape, -while `ComparisonScope` alone multiplies the completed query over the workload +`evaluation_steps` and `subquery_steps` validate and cost that internal shape. +For every subquery, `child.evaluation_steps` must equal checked +`parent.evaluation_steps × subquery_steps`; nested subqueries apply the same +equation recursively. Overflow or disagreement makes the candidate +unavailable. `ComparisonScope` alone multiplies the completed query over the workload horizon. They are never multiplied into the horizon a second time. Series cardinality is carried in child order and must agree across every physical edge. Missing window, step, series, expression-work, label-key, or accumulator @@ -341,8 +349,11 @@ evidence makes the entire candidate unavailable. `info()` resolves its default `target_info` metric, or one exact `__name__` matcher, to a concrete existing `Source::TimeSeries` coverage. Its right side is an ordinary physical Scan with its own statistics, buffer, snapshot, and -source bytes; the enrichment operator itself performs no source I/O. Other -label matchers remain local enrichment/filter work. Non-exact metric-name +source bytes; the enrichment operator itself performs no source I/O. The info +operator must build its label index from this right side. Other label matchers +remain local enrichment/filter work and require explicit positive +`scalar_ops_per_row`; their CPU contribution is +`info_rows × scalar_ops_per_row`. Non-exact metric-name selection stays unavailable until a catalog resolver can return the complete concrete source set. @@ -373,7 +384,7 @@ the DAG rules above. | PromQL subquery | `input_rows + output_rows` | materialized inner-step logical bytes | `0` beyond child | | PromQL vector binary | left + right + output rows | selected-side label-match hash state; one output row for scalar/vector | `0` beyond children | | PromQL relabel | `input_rows × scalar_ops_per_row` | one output row/batch | `0` | -| PromQL info enrichment | left + info + output rows | info-side label-match hash state | `0` beyond its explicit Scan child | +| PromQL info enrichment | left + info + output rows + `info_rows × matcher_ops_per_row` | right-side label-match hash state | `0` beyond its explicit Scan child | | PromQL series sample | input rows + input series | selected-series key/hash state | `0` | | PromQL scalar/vector bridge | input + output rows | one output row/batch | `0` | | PromQL scalar leaf | output rows | one output row/batch | `0` | From fdce06992b4b544bb84b16bdfbfc21f249be4ad6 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 10:14:35 -0600 Subject: [PATCH 18/48] fix(cost): preserve PromQL bridge direction --- .../asap-aware-mapping/src/analytical_cost.rs | 115 +++++++++++++++++- .../src/analytical_lowering.rs | 101 +++++++++++++-- .../analytical-resource-cost.md | 15 ++- 3 files changed, 215 insertions(+), 16 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 4f8ec6ea..434d6ac5 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -79,7 +79,8 @@ pub enum PhysicalOperator { PromqlRelabel, PromqlInfoEnrich, PromqlSeriesSample, - PromqlBridge, + PromqlScalarToVector, + PromqlVectorToScalar, PromqlScalarLeaf, PromqlPerSeries, PromqlPresence, @@ -704,8 +705,8 @@ pub fn estimate_operator( scan_bytes: 0, } } - PhysicalOperator::PromqlBridge => { - require_promql_statistics(&statistics, 1)?; + PhysicalOperator::PromqlScalarToVector | PhysicalOperator::PromqlVectorToScalar => { + validate_promql_bridge(operator, &statistics)?; ResourceEstimate { cpu_ops: left.rows as f64 + output.rows as f64, peak_memory_bytes: per_row_width(output.rows, output.bytes)?, @@ -829,7 +830,47 @@ fn validate_operator_semantics( return inconsistent("Limit output exceeds its input"); } } - PhysicalOperator::HashJoin => {} + PhysicalOperator::HashJoin + | PhysicalOperator::PromqlRange + | PhysicalOperator::PromqlSubquery + | PhysicalOperator::PromqlVectorBinary + | PhysicalOperator::PromqlRelabel + | PhysicalOperator::PromqlInfoEnrich + | PhysicalOperator::PromqlSeriesSample + | PhysicalOperator::PromqlScalarToVector + | PhysicalOperator::PromqlVectorToScalar + | PhysicalOperator::PromqlScalarLeaf + | PhysicalOperator::PromqlPerSeries + | PhysicalOperator::PromqlPresence => {} + } + Ok(()) +} + +fn validate_promql_bridge( + operator: PhysicalOperator, + statistics: &OperatorStatistics, +) -> Result<(), AnalyticalCostError> { + let promql = require_promql_statistics(statistics, 1)?; + match operator { + PhysicalOperator::PromqlScalarToVector => { + if promql.input_series != [0] + || promql.output_series != 1 + || statistics.inputs[0].rows != promql.evaluation_steps + || statistics.output.rows != promql.evaluation_steps + { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "scalar-to-vector must consume one scalar and emit one series per evaluation step", + )); + } + } + PhysicalOperator::PromqlVectorToScalar => { + if promql.output_series != 0 || statistics.output.rows != promql.evaluation_steps { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "vector-to-scalar must emit one scalar per evaluation step", + )); + } + } + _ => unreachable!("bridge validation requires a directional bridge operator"), } Ok(()) } @@ -1344,6 +1385,72 @@ mod tests { } } + #[test] + fn directional_promql_bridges_fail_closed_on_wrong_cardinality() { + use crate::analytical_statistics::PromqlOperatorStatistics; + + let bridge_statistics = |input_series, output_series, input_rows, output_rows| { + let mut statistics = statistics( + vec![EdgeStatistics { + rows: input_rows, + bytes: input_rows * 8, + }], + EdgeStatistics { + rows: output_rows, + bytes: output_rows * 8, + }, + ); + statistics.promql = Some(PromqlOperatorStatistics { + input_series: vec![input_series], + output_series, + evaluation_steps: 10, + window_samples_per_series: None, + subquery_steps: None, + scalar_ops_per_row: None, + }); + statistics + }; + + assert!(estimate_operator( + PhysicalOperator::PromqlScalarToVector, + bridge_statistics(0, 1, 10, 10), + ) + .is_ok()); + assert!(estimate_operator( + PhysicalOperator::PromqlVectorToScalar, + bridge_statistics(3, 0, 30, 10), + ) + .is_ok()); + assert!(matches!( + estimate_operator( + PhysicalOperator::PromqlScalarToVector, + bridge_statistics(0, 0, 10, 10), + ), + Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) + )); + assert!(matches!( + estimate_operator( + PhysicalOperator::PromqlVectorToScalar, + bridge_statistics(3, 1, 30, 10), + ), + Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) + )); + assert!(matches!( + estimate_operator( + PhysicalOperator::PromqlScalarToVector, + bridge_statistics(0, 1, 9, 10), + ), + Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) + )); + assert!(matches!( + estimate_operator( + PhysicalOperator::PromqlVectorToScalar, + bridge_statistics(3, 0, 30, 9), + ), + Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) + )); + } + #[test] fn comparison_rejects_different_snapshot_predicate_time_or_horizon() { use std::rc::Rc; diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index 3922c772..632a0db3 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -635,10 +635,18 @@ pub fn lower_query_physical_dag( None, ) } - QueryExpr::PromqlVectorFromScalar(child) - | QueryExpr::PromqlScalarFromVector(child) => { - self.lower_unary(query, occurrence, PhysicalOperator::PromqlBridge, child) - } + QueryExpr::PromqlVectorFromScalar(child) => self.lower_unary( + query, + occurrence, + PhysicalOperator::PromqlScalarToVector, + child, + ), + QueryExpr::PromqlScalarFromVector(child) => self.lower_unary( + query, + occurrence, + PhysicalOperator::PromqlVectorToScalar, + child, + ), QueryExpr::PromqlScalarBridge(inner) if matches!( inner.as_ref(), @@ -953,9 +961,27 @@ fn require_operator_statistics( )); } } - PhysicalOperator::PromqlSubquery | PhysicalOperator::PromqlBridge => { + PhysicalOperator::PromqlSubquery => { require_promql_statistics(statistics, 1)?; } + PhysicalOperator::PromqlScalarToVector => { + let promql = require_promql_statistics(statistics, 1)?; + if promql.input_series != [0] + || promql.output_series != 1 + || input.rows != promql.evaluation_steps + || output.rows != promql.evaluation_steps + { + return invalid( + "scalar-to-vector must consume one scalar and emit one series per evaluation step", + ); + } + } + PhysicalOperator::PromqlVectorToScalar => { + let promql = require_promql_statistics(statistics, 1)?; + if promql.output_series != 0 || output.rows != promql.evaluation_steps { + return invalid("vector-to-scalar must emit one scalar per evaluation step"); + } + } PhysicalOperator::PromqlVectorBinary => { require_promql_statistics(statistics, 2)?; } @@ -2353,14 +2379,73 @@ mod tests { ))); let scalar_statistics = with_promql(statistics(vec![], edge(10, 80)), vec![], 0, 10); let vector_statistics = - with_promql(statistics(vec![edge(10, 80)], edge(10, 80)), vec![0], 0, 10); - let provided = HashMap::from([ + with_promql(statistics(vec![edge(10, 80)], edge(10, 80)), vec![0], 1, 10); + let mut provided = HashMap::from([ ("query-1".into(), evidence(scalar_statistics)), ("query-0".into(), evidence(vector_statistics)), ]); let dag = lower_query_physical_dag(&vector, &empty_scope, &scripted(&provided)).unwrap(); assert_eq!(dag.nodes[0].operator, PhysicalOperator::PromqlScalarLeaf); - assert_eq!(dag.nodes[1].operator, PhysicalOperator::PromqlBridge); + assert_eq!( + dag.nodes[1].operator, + PhysicalOperator::PromqlScalarToVector + ); + + provided + .get_mut("query-0") + .unwrap() + .statistics + .promql + .as_mut() + .unwrap() + .output_series = 0; + assert!(matches!( + lower_query_physical_dag(&vector, &empty_scope, &scripted(&provided)), + Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) + )); + + let scalar_from_vector = Rc::new(QueryExpr::PromqlScalarFromVector(vector)); + let scalar_statistics = with_promql(statistics(vec![], edge(10, 80)), vec![], 0, 10); + let vector_statistics = + with_promql(statistics(vec![edge(10, 80)], edge(10, 80)), vec![0], 1, 10); + let scalar_from_vector_statistics = + with_promql(statistics(vec![edge(10, 80)], edge(10, 80)), vec![1], 0, 10); + let mut provided = HashMap::from([ + ("query-2".into(), evidence(scalar_statistics)), + ("query-1".into(), evidence(vector_statistics)), + ("query-0".into(), evidence(scalar_from_vector_statistics)), + ]); + let dag = lower_query_physical_dag(&scalar_from_vector, &empty_scope, &scripted(&provided)) + .unwrap(); + assert_eq!( + dag.nodes[2].operator, + PhysicalOperator::PromqlVectorToScalar + ); + provided + .get_mut("query-0") + .unwrap() + .statistics + .promql + .as_mut() + .unwrap() + .output_series = 1; + assert!(matches!( + lower_query_physical_dag(&scalar_from_vector, &empty_scope, &scripted(&provided)), + Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) + )); + let promql = provided + .get_mut("query-0") + .unwrap() + .statistics + .promql + .as_mut() + .unwrap(); + promql.output_series = 0; + promql.input_series = vec![2]; + assert!(matches!( + lower_query_physical_dag(&scalar_from_vector, &empty_scope, &scripted(&provided)), + Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) + )); let scan = Rc::new(QueryExpr::Scan { source: Source::TimeSeries { metric: "m".into() }, diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 82a2a1f4..3185360d 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -304,7 +304,8 @@ The supported mappings are: | PromqlRelabel | PromqlRelabel | | PromqlInfoEnrich | left input plus an explicit scoped info-metric Scan | | PromqlSeriesSample | PromqlSeriesSample using the existing SampleKind | -| PromqlVectorFromScalar / PromqlScalarFromVector | unary PromqlBridge | +| PromqlVectorFromScalar | unary PromqlScalarToVector | +| PromqlScalarFromVector | unary PromqlVectorToScalar | | PromqlScalarBridge(float) / EvalTimestamp | zero-input PromqlScalarLeaf | | Count/Sum/Min/Max/Avg/StdDev/Variance with PerEntity (`*_over_time`) | PromqlPerSeries | | native histogram count/sum/avg/stddev/stdvar/fraction accessor | PromqlPerSeries | @@ -340,12 +341,18 @@ range and subquery values therefore include their internal evaluation steps. For every subquery, `child.evaluation_steps` must equal checked `parent.evaluation_steps × subquery_steps`; nested subqueries apply the same equation recursively. Overflow or disagreement makes the candidate -unavailable. `ComparisonScope` alone multiplies the completed query over the workload -horizon. They are never multiplied into the horizon a second time. Series +unavailable. `ComparisonScope` alone multiplies the completed query over the +workload horizon. They are never multiplied into the horizon a second time. Series cardinality is carried in child order and must agree across every physical edge. Missing window, step, series, expression-work, label-key, or accumulator evidence makes the entire candidate unavailable. +Scalar/vector bridges retain their direction in the physical operator. A +scalar-to-vector bridge consumes zero input series and emits exactly one series; +both input and output contain one row per evaluation step. A vector-to-scalar +bridge emits zero series and exactly one scalar row per evaluation step; its +input-series count must equal the vector child's output-series count. + `info()` resolves its default `target_info` metric, or one exact `__name__` matcher, to a concrete existing `Source::TimeSeries` coverage. Its right side is an ordinary physical Scan with its own statistics, buffer, snapshot, and @@ -386,7 +393,7 @@ the DAG rules above. | PromQL relabel | `input_rows × scalar_ops_per_row` | one output row/batch | `0` | | PromQL info enrichment | left + info + output rows + `info_rows × matcher_ops_per_row` | right-side label-match hash state | `0` beyond its explicit Scan child | | PromQL series sample | input rows + input series | selected-series key/hash state | `0` | -| PromQL scalar/vector bridge | input + output rows | one output row/batch | `0` | +| PromQL scalar-to-vector / vector-to-scalar bridge | input + output rows | one output row/batch | `0` | | PromQL scalar leaf | output rows | one output row/batch | `0` | | PromQL per-series intent | `input_rows × scalar_ops_per_row` | `input_series × accumulator_bytes` | `0` | | PromQL absence | input work + synthesized output | one output row/batch | `0` | From 686778f425e2ba91047a981ec3b161d860e99254 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 10:34:48 -0600 Subject: [PATCH 19/48] fix(cost): distinguish PromQL binary operand modes --- .../asap-aware-mapping/src/analytical_cost.rs | 146 +++++++++++++++--- .../src/analytical_lowering.rs | 99 +++++++++++- .../src/analytical_statistics.rs | 11 ++ .../analytical-resource-cost.md | 9 ++ 4 files changed, 236 insertions(+), 29 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 434d6ac5..77c28980 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize}; use crate::analytical_statistics::{ validate_comparison_scopes, ComparisonScope, EdgeStatistics, OperatorStatistics, - OperatorStatisticsProvider, SourceCoverage, + OperatorStatisticsProvider, PromqlBinaryOperandMode, SourceCoverage, }; pub const ANALYTICAL_MODEL_VERSION: &str = "analytical-resource-at-rest-v1"; @@ -479,9 +479,9 @@ pub fn estimate_operator( let output = statistics.output; if matches!(operator, PhysicalOperator::PromqlScalarLeaf) { let promql = require_promql_statistics(&statistics, 0)?; - if promql.output_series > output.rows && output.rows > 0 { + if promql.output_series != 0 || output.rows != promql.evaluation_steps { return Err(AnalyticalCostError::InconsistentOperatorStatistics( - "output series exceed output rows", + "PromQL scalar leaf must emit one scalar row per evaluation step", )); } return Ok(ResourceEstimate { @@ -623,27 +623,29 @@ pub fn estimate_operator( PhysicalOperator::PromqlVectorBinary => { let promql = require_promql_statistics(&statistics, 2)?; let right = input(1)?; - let scalar_vector = matches!(operator, PhysicalOperator::PromqlVectorBinary) - && promql.input_series.contains(&0); - let matching_bytes = if scalar_vector { - per_row_width(output.rows, output.bytes)? - } else { - let build_side = statistics - .hash_join_build_side - .ok_or(AnalyticalCostError::MissingOrZero("hash_join_build_side"))?; - let key_bytes = statistics - .key_bytes - .ok_or(AnalyticalCostError::MissingOrZero("key_bytes"))?; - let build_series = match build_side { - HashJoinBuildSide::Left => promql.input_series[0], - HashJoinBuildSide::Right => promql.input_series[1], - }; - checked_bytes(&[ - build_series, - key_bytes - .checked_add(16) - .ok_or(AnalyticalCostError::Overflow)?, - ])? + let mode = validate_promql_binary(&statistics)?; + let matching_bytes = match mode { + PromqlBinaryOperandMode::VectorScalar | PromqlBinaryOperandMode::ScalarVector => { + per_row_width(output.rows, output.bytes)? + } + PromqlBinaryOperandMode::VectorVector => { + let build_side = statistics + .hash_join_build_side + .ok_or(AnalyticalCostError::MissingOrZero("hash_join_build_side"))?; + let key_bytes = statistics + .key_bytes + .ok_or(AnalyticalCostError::MissingOrZero("key_bytes"))?; + let build_series = match build_side { + HashJoinBuildSide::Left => promql.input_series[0], + HashJoinBuildSide::Right => promql.input_series[1], + }; + checked_bytes(&[ + build_series, + key_bytes + .checked_add(16) + .ok_or(AnalyticalCostError::Overflow)?, + ])? + } }; ResourceEstimate { cpu_ops: left.rows as f64 + right.rows as f64 + output.rows as f64, @@ -875,6 +877,49 @@ fn validate_promql_bridge( Ok(()) } +fn validate_promql_binary( + statistics: &OperatorStatistics, +) -> Result { + let promql = require_promql_statistics(statistics, 2)?; + let mode = promql + .binary_operand_mode + .ok_or(AnalyticalCostError::MissingOrStale( + "promql_binary_operand_mode", + ))?; + match mode { + PromqlBinaryOperandMode::VectorVector => { + if statistics.hash_join_build_side.is_none() || statistics.key_bytes.is_none() { + return Err(AnalyticalCostError::MissingOrStale( + "vector_binary_label_match_statistics", + )); + } + } + PromqlBinaryOperandMode::VectorScalar => { + if promql.input_series[1] != 0 + || statistics.inputs[1].rows != promql.evaluation_steps + || statistics.hash_join_build_side.is_some() + || statistics.key_bytes.is_some() + { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "vector/scalar binary evidence has an invalid scalar edge or label-match state", + )); + } + } + PromqlBinaryOperandMode::ScalarVector => { + if promql.input_series[0] != 0 + || statistics.inputs[0].rows != promql.evaluation_steps + || statistics.hash_join_build_side.is_some() + || statistics.key_bytes.is_some() + { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "scalar/vector binary evidence has an invalid scalar edge or label-match state", + )); + } + } + } + Ok(mode) +} + fn require_promql_statistics( statistics: &OperatorStatistics, inputs: usize, @@ -1407,6 +1452,7 @@ mod tests { window_samples_per_series: None, subquery_steps: None, scalar_ops_per_row: None, + binary_operand_mode: None, }); statistics }; @@ -1451,6 +1497,58 @@ mod tests { )); } + #[test] + fn promql_binary_mode_distinguishes_empty_vectors_from_scalars() { + use crate::analytical_statistics::{PromqlBinaryOperandMode, PromqlOperatorStatistics}; + + let empty = EdgeStatistics { rows: 0, bytes: 0 }; + let mut vector_vector = OperatorStatistics { + key_bytes: Some(16), + hash_join_build_side: Some(HashJoinBuildSide::Right), + promql: Some(PromqlOperatorStatistics { + input_series: vec![0, 0], + output_series: 0, + evaluation_steps: 10, + window_samples_per_series: None, + subquery_steps: None, + scalar_ops_per_row: None, + binary_operand_mode: Some(PromqlBinaryOperandMode::VectorVector), + }), + ..statistics(vec![empty, empty], empty) + }; + assert!( + estimate_operator(PhysicalOperator::PromqlVectorBinary, vector_vector.clone()).is_ok() + ); + + vector_vector.hash_join_build_side = None; + assert_eq!( + estimate_operator(PhysicalOperator::PromqlVectorBinary, vector_vector), + Err(AnalyticalCostError::MissingOrStale( + "vector_binary_label_match_statistics" + )) + ); + } + + #[test] + fn promql_scalar_leaf_requires_one_row_per_evaluation_step() { + use crate::analytical_statistics::PromqlOperatorStatistics; + + let mut scalar = statistics(vec![], EdgeStatistics { rows: 9, bytes: 72 }); + scalar.promql = Some(PromqlOperatorStatistics { + input_series: vec![], + output_series: 0, + evaluation_steps: 10, + window_samples_per_series: None, + subquery_steps: None, + scalar_ops_per_row: None, + binary_operand_mode: None, + }); + assert!(matches!( + estimate_operator(PhysicalOperator::PromqlScalarLeaf, scalar), + Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) + )); + } + #[test] fn comparison_rejects_different_snapshot_predicate_time_or_horizon() { use std::rc::Rc; diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index 632a0db3..49fd5ab8 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -8,7 +8,8 @@ use crate::analytical_cost::{ AnalyticalCostError, ExecutionMultiplicity, PhysicalDagNode, PhysicalOperator, }; use crate::analytical_statistics::{ - ComparisonScope, EdgeStatistics, OperatorStatistics, OperatorStatisticsProvider, SourceCoverage, + ComparisonScope, EdgeStatistics, OperatorStatistics, OperatorStatisticsProvider, + PromqlBinaryOperandMode, SourceCoverage, }; /// A lowered physical DAG and the node whose output is the query result. @@ -606,6 +607,19 @@ pub fn lower_query_physical_dag( "PromQL binary inputs do not match child outputs", )); } + let operand_mode = match (left_scalar, right_scalar) { + (false, false) => PromqlBinaryOperandMode::VectorVector, + (false, true) => PromqlBinaryOperandMode::VectorScalar, + (true, false) => PromqlBinaryOperandMode::ScalarVector, + (true, true) => unreachable!("scalar/scalar was rejected above"), + }; + if require_promql_statistics(statistics, 2)?.binary_operand_mode + != Some(operand_mode) + { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "PromQL binary operand mode disagrees with its logical operands", + )); + } if left_scalar || right_scalar { if vector_match.is_some() || statistics.hash_join_build_side.is_some() @@ -896,7 +910,10 @@ fn require_operator_statistics( return invalid("only Scan may charge source bytes"); } if matches!(operator, PhysicalOperator::PromqlScalarLeaf) { - require_promql_statistics(statistics, 0)?; + let promql = require_promql_statistics(statistics, 0)?; + if promql.output_series != 0 || statistics.output.rows != promql.evaluation_steps { + return invalid("PromQL scalar leaf must emit one scalar row per evaluation step"); + } return Ok(()); } let input = statistics.inputs.first().copied().ok_or( @@ -983,7 +1000,43 @@ fn require_operator_statistics( } } PhysicalOperator::PromqlVectorBinary => { - require_promql_statistics(statistics, 2)?; + let promql = require_promql_statistics(statistics, 2)?; + let mode = promql + .binary_operand_mode + .ok_or(AnalyticalCostError::MissingOrStale( + "promql_binary_operand_mode", + ))?; + match mode { + PromqlBinaryOperandMode::VectorVector => { + if statistics.hash_join_build_side.is_none() || statistics.key_bytes.is_none() { + return Err(AnalyticalCostError::MissingOrStale( + "vector_binary_label_match_statistics", + )); + } + } + PromqlBinaryOperandMode::VectorScalar => { + if promql.input_series[1] != 0 + || statistics.inputs[1].rows != promql.evaluation_steps + || statistics.hash_join_build_side.is_some() + || statistics.key_bytes.is_some() + { + return invalid( + "vector/scalar binary evidence has an invalid scalar edge or label-match state", + ); + } + } + PromqlBinaryOperandMode::ScalarVector => { + if promql.input_series[0] != 0 + || statistics.inputs[0].rows != promql.evaluation_steps + || statistics.hash_join_build_side.is_some() + || statistics.key_bytes.is_some() + { + return invalid( + "scalar/vector binary evidence has an invalid scalar edge or label-match state", + ); + } + } + } } PhysicalOperator::PromqlInfoEnrich => { let promql = require_promql_statistics(statistics, 2)?; @@ -1465,6 +1518,7 @@ mod tests { window_samples_per_series: None, subquery_steps: None, scalar_ops_per_row: None, + binary_operand_mode: None, }); statistics } @@ -2363,10 +2417,15 @@ mod tests { let empty_scope = scope(vec![]); let scalar = Rc::new(QueryExpr::promql_scalar(5.0)); let scalar_statistics = with_promql(statistics(vec![], edge(10, 80)), vec![], 0, 10); - let provided = HashMap::from([("query-0".into(), evidence(scalar_statistics))]); + let mut provided = HashMap::from([("query-0".into(), evidence(scalar_statistics))]); let dag = lower_query_physical_dag(&scalar, &empty_scope, &scripted(&provided)).unwrap(); assert_eq!(dag.nodes[0].operator, PhysicalOperator::PromqlScalarLeaf); assert!(estimate_physical_dag(&dag.nodes, &dag.root, &empty_scope, &dag.evidence).is_ok()); + provided.get_mut("query-0").unwrap().statistics.output = edge(9, 72); + assert!(matches!( + lower_query_physical_dag(&scalar, &empty_scope, &scripted(&provided)), + Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) + )); let timestamp = Rc::new(QueryExpr::EvalTimestamp); let timestamp_statistics = with_promql(statistics(vec![], edge(10, 80)), vec![], 0, 10); @@ -2579,12 +2638,14 @@ mod tests { ); vector_statistics.source_scan_bytes = 8_000; let scalar = with_promql(statistics(vec![], edge(10, 80)), vec![], 0, 10); - let binary = with_promql( + let mut binary = with_promql( statistics(vec![edge(1_000, 16_000), edge(10, 80)], edge(1_000, 16_000)), vec![100, 0], 100, 10, ); + binary.promql.as_mut().unwrap().binary_operand_mode = + Some(PromqlBinaryOperandMode::VectorScalar); let provided = HashMap::from([ ("query-1".into(), evidence(vector_statistics)), ("query-2".into(), evidence(scalar)), @@ -2679,6 +2740,8 @@ mod tests { ); binary.key_bytes = Some(32); binary.hash_join_build_side = Some(HashJoinBuildSide::Right); + binary.promql.as_mut().unwrap().binary_operand_mode = + Some(PromqlBinaryOperandMode::VectorVector); let provided = HashMap::from([ ("query-1".into(), evidence(left)), ("query-2".into(), evidence(right)), @@ -2689,6 +2752,32 @@ mod tests { assert_eq!(estimate.cpu_ops, 3_400.0); assert_eq!(estimate.scan_bytes, 12_000); + let mut empty = provided.clone(); + for id in ["query-1", "query-2"] { + let evidence = empty.get_mut(id).unwrap(); + evidence.statistics.inputs[0] = edge(0, 0); + evidence.statistics.output = edge(0, 0); + let promql = evidence.statistics.promql.as_mut().unwrap(); + promql.input_series[0] = 0; + promql.output_series = 0; + evidence.output_buffer_bytes = 0; + } + let evidence = empty.get_mut("query-0").unwrap(); + evidence.statistics.inputs = vec![edge(0, 0), edge(0, 0)]; + evidence.statistics.output = edge(0, 0); + let promql = evidence.statistics.promql.as_mut().unwrap(); + promql.input_series = vec![0, 0]; + promql.output_series = 0; + evidence.output_buffer_bytes = 0; + let empty_dag = lower_query_physical_dag(&root, &scope, &scripted(&empty)).unwrap(); + assert!(estimate_physical_dag( + &empty_dag.nodes, + &empty_dag.root, + &scope, + &empty_dag.evidence + ) + .is_ok()); + let mut missing = provided; missing .get_mut("query-0") diff --git a/crates/asap-aware-mapping/src/analytical_statistics.rs b/crates/asap-aware-mapping/src/analytical_statistics.rs index 316e2a92..dccdefc7 100644 --- a/crates/asap-aware-mapping/src/analytical_statistics.rs +++ b/crates/asap-aware-mapping/src/analytical_statistics.rs @@ -233,6 +233,17 @@ pub struct PromqlOperatorStatistics { pub window_samples_per_series: Option, pub subquery_steps: Option, pub scalar_ops_per_row: Option, + /// Physical operand shape for a PromQL binary operator. Series cardinality + /// cannot encode this because a vector may legitimately contain zero series. + #[serde(default)] + pub binary_operand_mode: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum PromqlBinaryOperandMode { + VectorVector, + VectorScalar, + ScalarVector, } /// Resolves physical statistics and owns their catalog/observation freshness. diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 3185360d..9bbb9a69 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -161,6 +161,7 @@ OperatorStatistics { promql: PromqlOperatorStatistics { input_series, output_series, evaluation_steps, window_samples_per_series, subquery_steps, scalar_ops_per_row, + binary_operand_mode, }, } ``` @@ -353,6 +354,14 @@ both input and output contain one row per evaluation step. A vector-to-scalar bridge emits zero series and exactly one scalar row per evaluation step; its input-series count must equal the vector child's output-series count. +PromQL binary evidence also records an explicit physical operand mode: +vector/vector, vector/scalar, or scalar/vector. Cardinality does not determine +the mode because a vector may legitimately contain zero series. Lowering +checks the mode against the canonical logical operands. Vector/vector always +requires label-match state, including for an empty vector; only the two +scalar/vector modes omit it. A scalar leaf has no inputs or series and emits +exactly one scalar row per evaluation step. + `info()` resolves its default `target_info` metric, or one exact `__name__` matcher, to a concrete existing `Source::TimeSeries` coverage. Its right side is an ordinary physical Scan with its own statistics, buffer, snapshot, and From bfc5e145d22fb5cbf8da309fe30b24037b2e0f39 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 20:39:01 -0600 Subject: [PATCH 20/48] fix(cost): preserve PromQL relabel cardinality --- .../asap-aware-mapping/src/analytical_cost.rs | 20 ++++++++++++---- .../src/analytical_lowering.rs | 24 +++++++++++++++---- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 77c28980..60595453 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -586,11 +586,21 @@ pub fn estimate_operator( peak_memory_bytes: per_row_width(output.rows, output.bytes)?, scan_bytes: 0, }, - PhysicalOperator::Limit => ResourceEstimate { - cpu_ops: output.rows as f64, - peak_memory_bytes: per_row_width(output.rows, output.bytes)?, - scan_bytes: 0, - }, + PhysicalOperator::Limit => { + let consumed = statistics + .limit_rows_consumed + .ok_or(AnalyticalCostError::MissingOrZero("limit_rows_consumed"))?; + if consumed > left.rows || consumed < output.rows { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "Limit rows consumed must cover its output without exceeding its input", + )); + } + ResourceEstimate { + cpu_ops: consumed as f64, + peak_memory_bytes: per_row_width(output.rows, output.bytes)?, + scan_bytes: 0, + } + } PhysicalOperator::PromqlRange => { let promql = require_promql_statistics(&statistics, 1)?; let samples = promql diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index 49fd5ab8..d39fb104 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -1051,8 +1051,8 @@ fn require_operator_statistics( } PhysicalOperator::PromqlRelabel => { let promql = require_promql_statistics(statistics, 1)?; - if output.rows != input.rows || promql.output_series > promql.input_series[0] { - return invalid("relabel changes rows or expands series cardinality"); + if output.rows != input.rows || promql.output_series != promql.input_series[0] { + return invalid("relabel changes row or series cardinality"); } } PhysicalOperator::PromqlSeriesSample => { @@ -2837,13 +2837,13 @@ mod tests { let mut relabel = with_promql( statistics(vec![edge(1_000, 16_000)], edge(1_000, 20_000)), vec![100], - 95, + 100, 10, ); relabel.promql.as_mut().unwrap().scalar_ops_per_row = Some(8); let mut sample = with_promql( statistics(vec![edge(1_000, 20_000)], edge(100, 2_000)), - vec![95], + vec![100], 10, 10, ); @@ -2879,6 +2879,22 @@ mod tests { ] ); assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).is_ok()); + + let mut shrinking = provided; + shrinking + .get_mut("query-2") + .unwrap() + .statistics + .promql + .as_mut() + .unwrap() + .output_series = 99; + assert_eq!( + lower_query_physical_dag(&root, &scope, &scripted(&shrinking)), + Err(AnalyticalCostError::InconsistentOperatorStatistics( + "relabel changes row or series cardinality" + )) + ); } #[test] From a2da31f31a3c4f165e5c53f3d06fcc6adb5c9c00 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 10:37:10 -0600 Subject: [PATCH 21/48] fix(cost): enforce physical DAG invariants in estimator --- .../asap-aware-mapping/src/analytical_cost.rs | 371 +++++++++++++++++- .../src/analytical_lowering.rs | 183 +-------- .../analytical-resource-cost.md | 16 +- 3 files changed, 381 insertions(+), 189 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 60595453..1f90dea7 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -293,6 +293,7 @@ pub fn estimate_physical_dag( peak_memory_bytes = peak_memory_bytes.max( live_bytes .checked_add(local.peak_memory_bytes) + .and_then(|bytes| bytes.checked_add(node.output_buffer_bytes)) .ok_or(AnalyticalCostError::Overflow)?, ); if node.retained_bytes > 0 { @@ -398,6 +399,7 @@ fn validate_operator_statistics( }); } } + validate_operator_semantics(node.operator, node_statistics)?; if let Some(promql) = node_statistics.promql.as_ref() { if promql.input_series.len() != expected_inputs { return Err(AnalyticalCostError::InvalidOperatorStatistics { @@ -451,6 +453,168 @@ fn validate_operator_statistics( Ok(()) } +pub(crate) fn validate_operator_semantics( + operator: PhysicalOperator, + statistics: &OperatorStatistics, +) -> Result<(), AnalyticalCostError> { + let invalid = |reason| Err(AnalyticalCostError::InconsistentOperatorStatistics(reason)); + if !matches!(operator, PhysicalOperator::Scan) && statistics.source_scan_bytes != 0 { + return invalid("only Scan may charge source bytes"); + } + if matches!(operator, PhysicalOperator::PromqlScalarLeaf) { + let promql = require_promql_statistics(statistics, 0)?; + if promql.output_series != 0 || statistics.output.rows != promql.evaluation_steps { + return invalid("PromQL scalar leaf must emit one scalar row per evaluation step"); + } + return Ok(()); + } + let input = statistics.inputs.first().copied().ok_or( + AnalyticalCostError::InconsistentOperatorStatistics("operator input is missing"), + )?; + let output = statistics.output; + match operator { + PhysicalOperator::Scan => { + if input != output { + return invalid("Scan external input edge does not match its output edge"); + } + } + PhysicalOperator::Filter => { + if output.rows > input.rows || output.bytes > input.bytes { + return invalid("filter output expands its input"); + } + } + PhysicalOperator::Project => { + if output.rows != input.rows { + return invalid("projection changes row cardinality"); + } + } + PhysicalOperator::HashAggregate => { + let groups = statistics + .group_count + .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; + if groups == 0 && (input.rows != 0 || output.rows != 0) { + return invalid("zero groups require an empty grouped input and output"); + } + if output.rows != groups { + return invalid("aggregate output does not equal group cardinality"); + } + } + PhysicalOperator::Deduplicate => { + let groups = statistics + .group_count + .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; + if output.rows != groups || output.rows > input.rows { + return invalid("deduplicate output does not equal distinct cardinality"); + } + } + PhysicalOperator::Sort => { + if output != input { + return invalid("sort changes its input cardinality or width"); + } + } + PhysicalOperator::TopK => { + let k = statistics + .k + .filter(|k| *k > 0) + .ok_or(AnalyticalCostError::MissingOrZero("k"))?; + if output.rows > input.rows.min(k) { + return invalid("top-k output exceeds its cardinality bound"); + } + } + PhysicalOperator::Limit => { + if output.rows > input.rows { + return invalid("bounded output exceeds its input cardinality"); + } + } + PhysicalOperator::Window => { + if output.rows != input.rows { + return invalid("SQL window changes row cardinality"); + } + } + PhysicalOperator::PassThrough => { + if output != input { + return invalid("pass-through wrapper changes its edge statistics"); + } + } + PhysicalOperator::Concat => { + let totals = statistics.inputs.iter().try_fold( + EdgeStatistics { rows: 0, bytes: 0 }, + |total, edge| { + Ok::<_, AnalyticalCostError>(EdgeStatistics { + rows: total + .rows + .checked_add(edge.rows) + .ok_or(AnalyticalCostError::Overflow)?, + bytes: total + .bytes + .checked_add(edge.bytes) + .ok_or(AnalyticalCostError::Overflow)?, + }) + }, + )?; + if output != totals { + return invalid("concat output does not equal the sum of its inputs"); + } + } + PhysicalOperator::PromqlRange => { + let promql = require_promql_statistics(statistics, 1)?; + if !matches!(promql.window_samples_per_series, Some(value) if value > 0) { + return Err(AnalyticalCostError::MissingOrZero( + "window_samples_per_series", + )); + } + } + PhysicalOperator::PromqlSubquery => { + require_promql_statistics(statistics, 1)?; + } + PhysicalOperator::PromqlScalarToVector => { + validate_promql_bridge(operator, statistics)?; + } + PhysicalOperator::PromqlVectorToScalar => { + validate_promql_bridge(operator, statistics)?; + } + PhysicalOperator::PromqlVectorBinary => { + validate_promql_binary(statistics)?; + } + PhysicalOperator::PromqlInfoEnrich => { + let promql = require_promql_statistics(statistics, 2)?; + if output.rows != input.rows || promql.output_series != promql.input_series[0] { + return invalid("info enrichment changes left sample or series cardinality"); + } + if statistics.hash_join_build_side != Some(HashJoinBuildSide::Right) { + return invalid("info enrichment must build its label index from the right side"); + } + } + PhysicalOperator::PromqlRelabel => { + let promql = require_promql_statistics(statistics, 1)?; + if output.rows != input.rows || promql.output_series > promql.input_series[0] { + return invalid("relabel changes rows or expands series cardinality"); + } + } + PhysicalOperator::PromqlSeriesSample => { + let promql = require_promql_statistics(statistics, 1)?; + if output.rows > input.rows || promql.output_series > promql.input_series[0] { + return invalid("series sampling expands its input"); + } + } + PhysicalOperator::PromqlPerSeries => { + let promql = require_promql_statistics(statistics, 1)?; + if promql.output_series > promql.input_series[0] { + return invalid("per-series operator expands series cardinality"); + } + } + PhysicalOperator::PromqlPresence => { + let promql = require_promql_statistics(statistics, 1)?; + if promql.output_series > 1 { + return invalid("PromQL absence operator emits more than one series"); + } + } + PhysicalOperator::HashJoin => {} + PhysicalOperator::PromqlScalarLeaf => unreachable!(), + } + Ok(()) +} + /// Estimate one physical operator. Child costs are deliberately excluded; /// a DAG walker sums CPU/disk once per node and combines simultaneously /// retained state separately. @@ -1033,8 +1197,7 @@ fn checked_bytes(parts: &[u64]) -> Result { mod tests { use super::*; use crate::analytical_statistics::{ - validate_comparison_scopes, ComparisonScope, EdgeStatistics, OperatorStatistics, - SourceCoverage, + validate_comparison_scopes, ComparisonScope, OperatorStatistics, SourceCoverage, }; fn unary_inputs(input_rows: u64, output_rows: u64) -> OperatorStatistics { @@ -1324,7 +1487,7 @@ mod tests { assert_eq!(estimate.scan_bytes, 2_000); // This is neither the sum of every node's memory nor just the largest // node: it is the maximum state simultaneously live at the fan-out. - assert_eq!(estimate.peak_memory_bytes, 24); + assert_eq!(estimate.peak_memory_bytes, 28); } #[test] @@ -1440,6 +1603,206 @@ mod tests { } } + fn manual_unary_plan( + operator: PhysicalOperator, + operator_statistics: OperatorStatistics, + ) -> (Vec, HashMap) { + let coverage = comparison_scope().sources[0].clone(); + let scan_edge = operator_statistics.inputs[0]; + let nodes = vec![ + PhysicalDagNode { + id: "scan".into(), + operator: PhysicalOperator::Scan, + children: vec![], + source_coverage: Some(coverage), + output_buffer_bytes: 10, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }, + PhysicalDagNode { + id: "operator".into(), + operator, + children: vec!["scan".into()], + source_coverage: None, + output_buffer_bytes: 10, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }, + ]; + let provided = HashMap::from([ + ( + "scan".into(), + OperatorStatistics { + source_scan_bytes: scan_edge.bytes, + ..statistics(vec![scan_edge], scan_edge) + }, + ), + ("operator".into(), operator_statistics), + ]); + (nodes, provided) + } + + #[test] + fn estimator_requires_reachable_scans_to_cover_the_exact_scope_set() { + use asap_types::pre_asap::Source; + + let edge = EdgeStatistics { + rows: 100, + bytes: 1_000, + }; + let (nodes, provided) = + manual_unary_plan(PhysicalOperator::Filter, statistics(vec![edge], edge)); + let mut scope = comparison_scope(); + scope.sources.push(SourceCoverage { + source: Source::Table { + table_ref: "unread_metrics".into(), + }, + snapshot_id: "catalog-version-42".into(), + predicates: vec![], + }); + + assert_eq!( + estimate_physical_dag(&nodes, "operator", &scope, &provided), + Err(AnalyticalCostError::InvalidPhysicalDag( + "physical scans omit a comparison-scope source" + )) + ); + } + + #[test] + fn estimator_rejects_semantically_impossible_manual_operator_statistics() { + let input = EdgeStatistics { + rows: 100, + bytes: 1_000, + }; + let invalid_cases = [ + ( + PhysicalOperator::Filter, + statistics( + vec![input], + EdgeStatistics { + rows: 101, + bytes: 1_010, + }, + ), + ), + ( + PhysicalOperator::Sort, + statistics( + vec![input], + EdgeStatistics { + rows: 99, + bytes: 990, + }, + ), + ), + ( + PhysicalOperator::HashAggregate, + OperatorStatistics { + group_count: Some(10), + key_bytes: Some(8), + aggregate_value_bytes: Some(8), + ..statistics( + vec![input], + EdgeStatistics { + rows: 9, + bytes: 144, + }, + ) + }, + ), + ]; + + for (operator, invalid_statistics) in invalid_cases { + let (nodes, provided) = manual_unary_plan(operator, invalid_statistics); + assert!(matches!( + estimate_physical_dag(&nodes, "operator", &comparison_scope(), &provided), + Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) + )); + } + } + + #[test] + fn estimator_validates_concat_totals_but_allows_duplicate_source_coverage() { + let coverage = comparison_scope().sources[0].clone(); + let edge = EdgeStatistics { + rows: 50, + bytes: 500, + }; + let nodes = vec![ + PhysicalDagNode { + id: "left".into(), + operator: PhysicalOperator::Scan, + children: vec![], + source_coverage: Some(coverage.clone()), + output_buffer_bytes: 10, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }, + PhysicalDagNode { + id: "right".into(), + operator: PhysicalOperator::Scan, + children: vec![], + source_coverage: Some(coverage), + output_buffer_bytes: 10, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }, + PhysicalDagNode { + id: "concat".into(), + operator: PhysicalOperator::Concat, + children: vec!["left".into(), "right".into()], + source_coverage: None, + output_buffer_bytes: 10, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }, + ]; + let mut provided = HashMap::from([ + ( + "left".into(), + OperatorStatistics { + source_scan_bytes: 500, + ..statistics(vec![edge], edge) + }, + ), + ( + "right".into(), + OperatorStatistics { + source_scan_bytes: 500, + ..statistics(vec![edge], edge) + }, + ), + ( + "concat".into(), + statistics( + vec![edge, edge], + EdgeStatistics { + rows: 99, + bytes: 990, + }, + ), + ), + ]); + + assert!(matches!( + estimate_physical_dag(&nodes, "concat", &comparison_scope(), &provided), + Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) + )); + + provided.insert( + "concat".into(), + statistics( + vec![edge, edge], + EdgeStatistics { + rows: 100, + bytes: 1_000, + }, + ), + ); + assert!(estimate_physical_dag(&nodes, "concat", &comparison_scope(), &provided).is_ok()); + } + #[test] fn directional_promql_bridges_fail_closed_on_wrong_cardinality() { use crate::analytical_statistics::PromqlOperatorStatistics; @@ -1782,7 +2145,7 @@ mod tests { let estimate = estimate_physical_dag(&nodes, "filter", &comparison_scope(), &provided).unwrap(); assert_eq!(estimate.cpu_ops, 1_200.0); - assert_eq!(estimate.peak_memory_bytes, 10); + assert_eq!(estimate.peak_memory_bytes, 20); assert_eq!(estimate.scan_bytes, 6_000); } diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index d39fb104..642de7cb 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -5,7 +5,8 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; use crate::analytical_cost::{ - AnalyticalCostError, ExecutionMultiplicity, PhysicalDagNode, PhysicalOperator, + validate_operator_semantics as require_operator_statistics, AnalyticalCostError, + ExecutionMultiplicity, PhysicalDagNode, PhysicalOperator, }; use crate::analytical_statistics::{ ComparisonScope, EdgeStatistics, OperatorStatistics, OperatorStatisticsProvider, @@ -901,186 +902,6 @@ fn require_unary_edge( Ok(()) } -fn require_operator_statistics( - operator: PhysicalOperator, - statistics: &OperatorStatistics, -) -> Result<(), AnalyticalCostError> { - let invalid = |reason| Err(AnalyticalCostError::InconsistentOperatorStatistics(reason)); - if !matches!(operator, PhysicalOperator::Scan) && statistics.source_scan_bytes != 0 { - return invalid("only Scan may charge source bytes"); - } - if matches!(operator, PhysicalOperator::PromqlScalarLeaf) { - let promql = require_promql_statistics(statistics, 0)?; - if promql.output_series != 0 || statistics.output.rows != promql.evaluation_steps { - return invalid("PromQL scalar leaf must emit one scalar row per evaluation step"); - } - return Ok(()); - } - let input = statistics.inputs.first().copied().ok_or( - AnalyticalCostError::InconsistentOperatorStatistics("operator input is missing"), - )?; - let output = statistics.output; - match operator { - PhysicalOperator::Filter => { - if output.rows > input.rows || output.bytes > input.bytes { - return invalid("filter output expands its input"); - } - } - PhysicalOperator::Project => { - if output.rows != input.rows { - return invalid("projection changes row cardinality"); - } - } - PhysicalOperator::HashAggregate => { - let groups = statistics - .group_count - .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - if groups == 0 && (input.rows != 0 || output.rows != 0) { - return invalid("zero groups require an empty grouped input and output"); - } - if output.rows > groups { - return invalid("aggregate output exceeds group cardinality"); - } - } - PhysicalOperator::Deduplicate => { - let groups = statistics - .group_count - .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - if output.rows != groups || output.rows > input.rows { - return invalid("deduplicate output does not equal distinct cardinality"); - } - } - PhysicalOperator::Sort => { - if output != input { - return invalid("sort changes its input cardinality or width"); - } - } - PhysicalOperator::TopK | PhysicalOperator::Limit => { - if output.rows > input.rows { - return invalid("bounded output exceeds its input cardinality"); - } - } - PhysicalOperator::Window => { - if output.rows != input.rows { - return invalid("SQL window changes row cardinality"); - } - } - PhysicalOperator::PassThrough => { - if output != input { - return invalid("pass-through wrapper changes its edge statistics"); - } - } - PhysicalOperator::PromqlRange => { - let promql = require_promql_statistics(statistics, 1)?; - if !matches!(promql.window_samples_per_series, Some(value) if value > 0) { - return Err(AnalyticalCostError::MissingOrZero( - "window_samples_per_series", - )); - } - } - PhysicalOperator::PromqlSubquery => { - require_promql_statistics(statistics, 1)?; - } - PhysicalOperator::PromqlScalarToVector => { - let promql = require_promql_statistics(statistics, 1)?; - if promql.input_series != [0] - || promql.output_series != 1 - || input.rows != promql.evaluation_steps - || output.rows != promql.evaluation_steps - { - return invalid( - "scalar-to-vector must consume one scalar and emit one series per evaluation step", - ); - } - } - PhysicalOperator::PromqlVectorToScalar => { - let promql = require_promql_statistics(statistics, 1)?; - if promql.output_series != 0 || output.rows != promql.evaluation_steps { - return invalid("vector-to-scalar must emit one scalar per evaluation step"); - } - } - PhysicalOperator::PromqlVectorBinary => { - let promql = require_promql_statistics(statistics, 2)?; - let mode = promql - .binary_operand_mode - .ok_or(AnalyticalCostError::MissingOrStale( - "promql_binary_operand_mode", - ))?; - match mode { - PromqlBinaryOperandMode::VectorVector => { - if statistics.hash_join_build_side.is_none() || statistics.key_bytes.is_none() { - return Err(AnalyticalCostError::MissingOrStale( - "vector_binary_label_match_statistics", - )); - } - } - PromqlBinaryOperandMode::VectorScalar => { - if promql.input_series[1] != 0 - || statistics.inputs[1].rows != promql.evaluation_steps - || statistics.hash_join_build_side.is_some() - || statistics.key_bytes.is_some() - { - return invalid( - "vector/scalar binary evidence has an invalid scalar edge or label-match state", - ); - } - } - PromqlBinaryOperandMode::ScalarVector => { - if promql.input_series[0] != 0 - || statistics.inputs[0].rows != promql.evaluation_steps - || statistics.hash_join_build_side.is_some() - || statistics.key_bytes.is_some() - { - return invalid( - "scalar/vector binary evidence has an invalid scalar edge or label-match state", - ); - } - } - } - } - PhysicalOperator::PromqlInfoEnrich => { - let promql = require_promql_statistics(statistics, 2)?; - if output.rows != input.rows || promql.output_series != promql.input_series[0] { - return invalid("info enrichment changes left sample or series cardinality"); - } - if statistics.hash_join_build_side - != Some(crate::analytical_cost::HashJoinBuildSide::Right) - { - return invalid("info enrichment must build its label index from the right side"); - } - } - PhysicalOperator::PromqlRelabel => { - let promql = require_promql_statistics(statistics, 1)?; - if output.rows != input.rows || promql.output_series != promql.input_series[0] { - return invalid("relabel changes row or series cardinality"); - } - } - PhysicalOperator::PromqlSeriesSample => { - let promql = require_promql_statistics(statistics, 1)?; - if output.rows > input.rows || promql.output_series > promql.input_series[0] { - return invalid("series sampling expands its input"); - } - } - PhysicalOperator::PromqlPerSeries => { - let promql = require_promql_statistics(statistics, 1)?; - if promql.output_series > promql.input_series[0] { - return invalid("per-series operator expands series cardinality"); - } - } - PhysicalOperator::PromqlPresence => { - let promql = require_promql_statistics(statistics, 1)?; - if promql.output_series > 1 { - return invalid("PromQL absence operator emits more than one series"); - } - } - PhysicalOperator::Scan - | PhysicalOperator::HashJoin - | PhysicalOperator::Concat - | PhysicalOperator::PromqlScalarLeaf => {} - } - Ok(()) -} - fn require_promql_statistics( statistics: &OperatorStatistics, arity: usize, diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 9bbb9a69..2bf9f63c 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -194,9 +194,10 @@ For a selected DAG: 4. Add source/disk reads only at nodes that actually read source or spilled data; an in-memory edge contributes zero source reads. 5. Count a shared node once even when several parents consume it. -6. Compute peak memory from liveness: add states that coexist, but do not add - disjoint transient buffers merely because both appear somewhere in the - DAG. +6. Compute peak memory from liveness. During one node's execution, all live + child outputs, the operator's local workspace, and its new output buffer + coexist. Do not add disjoint transient buffers merely because both appear + somewhere in the DAG. 7. Retained summaries remain live across reads. Streaming buffers may be released after their last consumer. @@ -211,6 +212,12 @@ counts, releases transient output after its last consumer, and keeps retained state live. Consequently a shared scan is charged once per execution and a fan-out's memory includes the outputs that really coexist. +Each estimate independently requires the semantic set of source coverages on +its reachable Scan nodes to equal `ComparisonScope.sources`. Multiple physical +Scans may repeat one coverage, but no scope source may be omitted and no Scan +may add another coverage. This invariant is enforced by the estimator itself, +including for callers that construct a physical DAG without the query lowerer. + Logical edge `bytes` feeds parent cardinality estimates; it is not an allocation. Each physical node separately supplies `output_buffer_bytes` for its live batch/edge buffer and `retained_bytes` for state that survives the @@ -270,7 +277,8 @@ Multiple independent physical Scans may use the same coverage, while a provider-declared shared Scan uses it once, so those physical alternatives can still be compared under the same semantic scope. -The lowering validates every physical edge before costing: +The estimator validates every physical edge before costing; the query lowerer +reuses the same operator-semantic validator for earlier diagnostics: - `(rows = 0, bytes = 0)` is a valid empty edge, while positive rows still require byte-width evidence and zero rows cannot carry non-zero bytes; From 9c5c98128004c7e2f54bb83dff2ee69535ced0c4 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 20:40:53 -0600 Subject: [PATCH 22/48] fix(cost): enforce exact bounded cardinalities --- crates/asap-aware-mapping/src/analytical_cost.rs | 14 ++++++++++---- .../asap-aware-mapping/src/analytical_lowering.rs | 12 ++++++++++++ .../src/analytical_statistics.rs | 4 ++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 1f90dea7..29f47431 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -517,8 +517,11 @@ pub(crate) fn validate_operator_semantics( .k .filter(|k| *k > 0) .ok_or(AnalyticalCostError::MissingOrZero("k"))?; - if output.rows > input.rows.min(k) { - return invalid("top-k output exceeds its cardinality bound"); + let offset = statistics + .topk_output_offset + .ok_or(AnalyticalCostError::MissingOrZero("topk_output_offset"))?; + if offset > k || output.rows != input.rows.min(k).saturating_sub(offset) { + return invalid("top-k output does not equal its cardinality bound"); } } PhysicalOperator::Limit => { @@ -587,8 +590,8 @@ pub(crate) fn validate_operator_semantics( } PhysicalOperator::PromqlRelabel => { let promql = require_promql_statistics(statistics, 1)?; - if output.rows != input.rows || promql.output_series > promql.input_series[0] { - return invalid("relabel changes rows or expands series cardinality"); + if output.rows != input.rows || promql.output_series != promql.input_series[0] { + return invalid("relabel changes row or series cardinality"); } } PhysicalOperator::PromqlSeriesSample => { @@ -1266,6 +1269,7 @@ mod tests { PhysicalOperator::TopK, OperatorStatistics { k: Some(10), + topk_output_offset: Some(0), ..statistics( vec![EdgeStatistics { rows: 1_000, @@ -1375,6 +1379,7 @@ mod tests { PhysicalOperator::TopK, OperatorStatistics { k: Some(1_000), + topk_output_offset: Some(0), ..statistics( vec![EdgeStatistics { rows: 4, @@ -1597,6 +1602,7 @@ mod tests { key_bytes: None, aggregate_value_bytes: None, k: None, + topk_output_offset: None, limit_rows_consumed: None, hash_join_build_side: None, promql: None, diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index 642de7cb..0d874b96 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -359,6 +359,16 @@ pub fn lower_query_physical_dag( "Top-K statistics disagree with LIMIT n + offset", )); } + if statistics.topk_output_offset + != Some( + u64::try_from(*offset) + .map_err(|_| AnalyticalCostError::Overflow)?, + ) + { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "Top-K statistics disagree with LIMIT offset", + )); + } require_limit_cardinality(*n, *offset, statistics)?; return self.push(evidence, PhysicalOperator::TopK, children, None); } @@ -1312,6 +1322,7 @@ mod tests { key_bytes: None, aggregate_value_bytes: None, k: None, + topk_output_offset: None, limit_rows_consumed: None, hash_join_build_side: None, promql: None, @@ -1453,6 +1464,7 @@ mod tests { aggregate_statistics.aggregate_value_bytes = Some(8); let mut topk_statistics = statistics(vec![edge(100, 4_000)], edge(10, 400)); topk_statistics.k = Some(15); + topk_statistics.topk_output_offset = Some(5); let mut raw_scan = statistics(vec![edge(1_000, 64_000)], edge(1_000, 64_000)); raw_scan.source_scan_bytes = 64_000; let provided = HashMap::from([ diff --git a/crates/asap-aware-mapping/src/analytical_statistics.rs b/crates/asap-aware-mapping/src/analytical_statistics.rs index dccdefc7..c406793d 100644 --- a/crates/asap-aware-mapping/src/analytical_statistics.rs +++ b/crates/asap-aware-mapping/src/analytical_statistics.rs @@ -213,6 +213,10 @@ pub struct OperatorStatistics { pub key_bytes: Option, pub aggregate_value_bytes: Option, pub k: Option, + /// Rows skipped after maintaining the Top-K heap. A plain Top-K uses zero; + /// fused ORDER BY/LIMIT/OFFSET uses the logical offset. + #[serde(default)] + pub topk_output_offset: Option, /// Rows consumed by a physical Limit, including rows skipped by OFFSET. /// This is an execution statistic rather than the output cardinality. #[serde(default)] From 72bb94df97822261e82a65d9bf359d403dba6f79 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 10:11:10 -0600 Subject: [PATCH 23/48] refactor(cost): centralize PromQL invariants with operators --- .../asap-aware-mapping/src/analytical_cost.rs | 98 +------------------ 1 file changed, 1 insertion(+), 97 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 29f47431..db5977b4 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -928,103 +928,6 @@ pub fn estimate_operator( } } -fn validate_operator_semantics( - operator: PhysicalOperator, - statistics: &OperatorStatistics, -) -> Result<(), AnalyticalCostError> { - let inconsistent = |reason| Err(AnalyticalCostError::InconsistentOperatorStatistics(reason)); - if !matches!(operator, PhysicalOperator::Scan) && statistics.source_scan_bytes != 0 { - return inconsistent("only Scan may charge source bytes"); - } - let input = statistics.inputs.first().copied().ok_or( - AnalyticalCostError::InconsistentOperatorStatistics("operator input is missing"), - )?; - let output = statistics.output; - match operator { - PhysicalOperator::Scan => { - if input != output { - return inconsistent("Scan input and output edges differ"); - } - } - PhysicalOperator::Filter => { - if output.rows > input.rows || output.bytes > input.bytes { - return inconsistent("Filter output expands its input"); - } - } - PhysicalOperator::Project => { - if output.rows != input.rows { - return inconsistent("Project changes row cardinality"); - } - } - PhysicalOperator::HashAggregate | PhysicalOperator::Deduplicate => { - let groups = statistics - .group_count - .filter(|groups| *groups > 0) - .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - if groups > input.rows || output.rows != groups { - return inconsistent("grouped output differs from distinct group cardinality"); - } - } - PhysicalOperator::Sort | PhysicalOperator::PassThrough => { - if input != output { - return inconsistent("cardinality-preserving operator changes its edge"); - } - } - PhysicalOperator::Window => { - if input.rows != output.rows { - return inconsistent("Window changes row cardinality"); - } - } - PhysicalOperator::Concat => { - let total = statistics.inputs.iter().try_fold( - EdgeStatistics { rows: 0, bytes: 0 }, - |total, edge| { - Ok::<_, AnalyticalCostError>(EdgeStatistics { - rows: total - .rows - .checked_add(edge.rows) - .ok_or(AnalyticalCostError::Overflow)?, - bytes: total - .bytes - .checked_add(edge.bytes) - .ok_or(AnalyticalCostError::Overflow)?, - }) - }, - )?; - if output != total { - return inconsistent("Concat output differs from the sum of its inputs"); - } - } - PhysicalOperator::TopK => { - let k = statistics - .k - .filter(|k| *k > 0) - .ok_or(AnalyticalCostError::MissingOrZero("k"))?; - if output.rows > input.rows.min(k) { - return inconsistent("Top-K output exceeds its cardinality bound"); - } - } - PhysicalOperator::Limit => { - if output.rows > input.rows { - return inconsistent("Limit output exceeds its input"); - } - } - PhysicalOperator::HashJoin - | PhysicalOperator::PromqlRange - | PhysicalOperator::PromqlSubquery - | PhysicalOperator::PromqlVectorBinary - | PhysicalOperator::PromqlRelabel - | PhysicalOperator::PromqlInfoEnrich - | PhysicalOperator::PromqlSeriesSample - | PhysicalOperator::PromqlScalarToVector - | PhysicalOperator::PromqlVectorToScalar - | PhysicalOperator::PromqlScalarLeaf - | PhysicalOperator::PromqlPerSeries - | PhysicalOperator::PromqlPresence => {} - } - Ok(()) -} - fn validate_promql_bridge( operator: PhysicalOperator, statistics: &OperatorStatistics, @@ -1238,6 +1141,7 @@ mod tests { let mut topk = unary_inputs(10, 4); topk.k = Some(3); + topk.topk_output_offset = Some(0); assert!(matches!( estimate_operator(PhysicalOperator::TopK, topk), Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) From b4c74614b1bcc0996cb36a0216789d94c3d2e7cb Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 12:01:26 -0600 Subject: [PATCH 24/48] refactor(cost): make physical operator arity explicit --- .../asap-aware-mapping/src/analytical_cost.rs | 97 ++++++++++++++++--- .../analytical-resource-cost.md | 14 +++ 2 files changed, 97 insertions(+), 14 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 69a83e83..3aadb9d7 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -328,24 +328,14 @@ fn validate_operator_statistics( nodes: &HashMap<&str, &PhysicalDagNode>, statistics: &HashMap<&str, OperatorStatistics>, ) -> Result<(), AnalyticalCostError> { - let expected_inputs = match node.operator { - PhysicalOperator::Scan => 1, - PhysicalOperator::HashJoin => 2, - PhysicalOperator::Concat => node.children.len(), - _ => 1, - }; - if node_statistics.inputs.len() != expected_inputs { + let arity = expected_input_arity(node.operator, node.children.len()); + if node_statistics.inputs.len() != arity.statistics_inputs { return Err(AnalyticalCostError::InvalidOperatorStatistics { node: node.id.clone(), - reason: "wrong input-edge count", + reason: "operator-statistics input count does not match physical arity", }); } - let expected_children = if matches!(node.operator, PhysicalOperator::Scan) { - 0 - } else { - expected_inputs - }; - if node.children.len() != expected_children { + if node.children.len() != arity.dag_children { return Err(AnalyticalCostError::InvalidPhysicalDag( "operator child count does not match physical arity", )); @@ -388,6 +378,53 @@ fn validate_operator_statistics( Ok(()) } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct OperatorInputArity { + /// Number of logical input-edge records required in `OperatorStatistics`. + statistics_inputs: usize, + /// Number of upstream physical nodes required in the DAG. + dag_children: usize, +} + +/// Declare both notions of operator input explicitly. A scan has one external +/// source-input statistics record but no upstream DAG node. Every other +/// operator's statistics inputs correspond one-to-one with its DAG children. +/// +/// Keep this match exhaustive: adding a physical operator must also define its +/// statistics and DAG arity instead of silently inheriting unary behavior. +fn expected_input_arity( + operator: PhysicalOperator, + variadic_child_count: usize, +) -> OperatorInputArity { + let unary = OperatorInputArity { + statistics_inputs: 1, + dag_children: 1, + }; + match operator { + PhysicalOperator::Scan => OperatorInputArity { + statistics_inputs: 1, + dag_children: 0, + }, + PhysicalOperator::Filter + | PhysicalOperator::Project + | PhysicalOperator::HashAggregate + | PhysicalOperator::Sort + | PhysicalOperator::TopK + | PhysicalOperator::Deduplicate + | PhysicalOperator::Window + | PhysicalOperator::Limit + | PhysicalOperator::PassThrough => unary, + PhysicalOperator::HashJoin => OperatorInputArity { + statistics_inputs: 2, + dag_children: 2, + }, + PhysicalOperator::Concat => OperatorInputArity { + statistics_inputs: variadic_child_count, + dag_children: variadic_child_count, + }, + } +} + /// Estimate one physical operator. Child costs are deliberately excluded; /// a DAG walker sums CPU/disk once per node and combines simultaneously /// retained state separately. @@ -711,6 +748,38 @@ mod tests { ) } + #[test] + fn operator_arity_distinguishes_source_statistics_from_dag_children() { + assert_eq!( + expected_input_arity(PhysicalOperator::Scan, 0), + OperatorInputArity { + statistics_inputs: 1, + dag_children: 0, + } + ); + assert_eq!( + expected_input_arity(PhysicalOperator::Filter, 1), + OperatorInputArity { + statistics_inputs: 1, + dag_children: 1, + } + ); + assert_eq!( + expected_input_arity(PhysicalOperator::HashJoin, 2), + OperatorInputArity { + statistics_inputs: 2, + dag_children: 2, + } + ); + assert_eq!( + expected_input_arity(PhysicalOperator::Concat, 3), + OperatorInputArity { + statistics_inputs: 3, + dag_children: 3, + } + ); + } + #[test] fn operator_estimator_rejects_contradictory_cardinality_evidence() { assert!(matches!( diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 7fa0267c..40600a8f 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -165,6 +165,20 @@ positive logical bytes so width-dependent formulas do not invent a row width. A parent therefore cannot silently substitute the original source cardinality for an intermediate edge. +`OperatorStatistics.inputs` and `PhysicalDagNode.children` therefore have +different arity only for a source leaf: + +| Operator shape | Statistics inputs | DAG children | +|---|---:|---:| +| `Scan` | 1 external source edge | 0 | +| Unary operator | 1 | 1 | +| `HashJoin` | 2 | 2 | +| `Concat` | one per input | one per input | + +The implementation matches every `PhysicalOperator` variant explicitly. A new +operator cannot silently inherit unary arity; its statistics-input and +DAG-child counts must both be defined. + `EdgeStatistics.bytes` is decoded logical data carried on an edge. `source_scan_bytes` is physical storage I/O and is charged only by `Scan`. Compression, column pruning, or encoded storage can therefore make these From e8dfaa74784a50cb41cfd9ba0244fb5c728c79fc Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 12:30:52 -0600 Subject: [PATCH 25/48] refactor(cost): type operator statistics by physical algorithm --- .../asap-aware-mapping/src/analytical_cost.rs | 837 ++++++++++-------- .../src/analytical_statistics.rs | 141 ++- .../analytical-resource-cost.md | 102 ++- 3 files changed, 652 insertions(+), 428 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 3aadb9d7..54ec6a01 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -65,13 +65,23 @@ pub enum PhysicalOperator { Filter, Project, HashAggregate, - Sort, - TopK, - HashJoin, - Deduplicate, + InMemoryComparisonSort, + /// Heap-based bounded ordering. The heap retains `limit + offset` rows + /// while the operator returns at most `limit` rows after skipping offset. + TopK { + limit: u64, + offset: u64, + }, + HashJoin { + build_side: HashJoinBuildSide, + }, + HashDeduplicate, Concat, - Window, - Limit, + InMemoryOrderedWindow, + Limit { + limit: u64, + offset: u64, + }, PassThrough, } @@ -329,7 +339,7 @@ fn validate_operator_statistics( statistics: &HashMap<&str, OperatorStatistics>, ) -> Result<(), AnalyticalCostError> { let arity = expected_input_arity(node.operator, node.children.len()); - if node_statistics.inputs.len() != arity.statistics_inputs { + if node_statistics.input_count() != arity.statistics_inputs { return Err(AnalyticalCostError::InvalidOperatorStatistics { node: node.id.clone(), reason: "operator-statistics input count does not match physical arity", @@ -340,20 +350,15 @@ fn validate_operator_statistics( "operator child count does not match physical arity", )); } - match node.operator { - PhysicalOperator::Scan => {} - _ if node_statistics.source_scan_bytes != 0 => { - return Err(AnalyticalCostError::InvalidOperatorStatistics { - node: node.id.clone(), - reason: "only scan operators may read source bytes", - }); - } - _ => {} + if !statistics_match_operator(node.operator, node_statistics) { + return Err(AnalyticalCostError::InvalidOperatorStatistics { + node: node.id.clone(), + reason: "statistics variant does not match physical operator", + }); } - for edge in node_statistics - .inputs - .iter() - .chain(std::iter::once(&node_statistics.output)) + for edge in (0..node_statistics.input_count()) + .filter_map(|index| node_statistics.input(index)) + .chain(std::iter::once(node_statistics.output())) { if !edge.is_consistent() { return Err(AnalyticalCostError::InvalidOperatorStatistics { @@ -367,7 +372,7 @@ fn validate_operator_statistics( .get(child_id.as_str()) .ok_or(AnalyticalCostError::InvalidPhysicalDag("missing node"))?; let child_statistics = &statistics[child.id.as_str()]; - if node_statistics.inputs[input_index] != child_statistics.output { + if node_statistics.input(input_index) != Some(child_statistics.output()) { return Err(AnalyticalCostError::ConflictingEdgeStatistics { parent: node.id.clone(), child: child.id.clone(), @@ -378,6 +383,38 @@ fn validate_operator_statistics( Ok(()) } +fn statistics_match_operator(operator: PhysicalOperator, statistics: &OperatorStatistics) -> bool { + match operator { + PhysicalOperator::Scan => matches!(statistics, OperatorStatistics::Scan { .. }), + PhysicalOperator::Filter => matches!(statistics, OperatorStatistics::Filter { .. }), + PhysicalOperator::Project => matches!(statistics, OperatorStatistics::Project { .. }), + PhysicalOperator::HashAggregate => { + matches!(statistics, OperatorStatistics::HashAggregate { .. }) + } + PhysicalOperator::InMemoryComparisonSort => { + matches!( + statistics, + OperatorStatistics::InMemoryComparisonSort { .. } + ) + } + PhysicalOperator::TopK { .. } => matches!(statistics, OperatorStatistics::TopK { .. }), + PhysicalOperator::HashJoin { .. } => { + matches!(statistics, OperatorStatistics::HashJoin { .. }) + } + PhysicalOperator::HashDeduplicate => { + matches!(statistics, OperatorStatistics::HashDeduplicate { .. }) + } + PhysicalOperator::Concat => matches!(statistics, OperatorStatistics::Concat { .. }), + PhysicalOperator::InMemoryOrderedWindow => { + matches!(statistics, OperatorStatistics::InMemoryOrderedWindow { .. }) + } + PhysicalOperator::Limit { .. } => matches!(statistics, OperatorStatistics::Limit { .. }), + PhysicalOperator::PassThrough => { + matches!(statistics, OperatorStatistics::PassThrough { .. }) + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct OperatorInputArity { /// Number of logical input-edge records required in `OperatorStatistics`. @@ -408,13 +445,13 @@ fn expected_input_arity( PhysicalOperator::Filter | PhysicalOperator::Project | PhysicalOperator::HashAggregate - | PhysicalOperator::Sort - | PhysicalOperator::TopK - | PhysicalOperator::Deduplicate - | PhysicalOperator::Window - | PhysicalOperator::Limit + | PhysicalOperator::InMemoryComparisonSort + | PhysicalOperator::TopK { .. } + | PhysicalOperator::HashDeduplicate + | PhysicalOperator::InMemoryOrderedWindow + | PhysicalOperator::Limit { .. } | PhysicalOperator::PassThrough => unary, - PhysicalOperator::HashJoin => OperatorInputArity { + PhysicalOperator::HashJoin { .. } => OperatorInputArity { statistics_inputs: 2, dag_children: 2, }, @@ -445,73 +482,78 @@ pub fn estimate_operator( }; let input = |index: usize| { statistics - .inputs - .get(index) - .copied() + .input(index) .ok_or(AnalyticalCostError::MissingOrZero("operator input edge")) }; let left = input(0)?; - let output = statistics.output; - let estimate = match operator { - PhysicalOperator::Scan => ResourceEstimate { + let output = statistics.output(); + let estimate = match (operator, &statistics) { + ( + PhysicalOperator::Scan, + OperatorStatistics::Scan { + source_read_bytes, .. + }, + ) => ResourceEstimate { cpu_ops: left.rows as f64, peak_memory_bytes: per_row_width(left.rows, left.bytes)?, - scan_bytes: statistics.source_scan_bytes, + scan_bytes: *source_read_bytes, }, - PhysicalOperator::Filter | PhysicalOperator::Project | PhysicalOperator::PassThrough => { - ResourceEstimate { - cpu_ops: left.rows as f64, - peak_memory_bytes: per_row_width(output.rows, output.bytes)?, - scan_bytes: 0, - } - } - PhysicalOperator::HashAggregate => { - let groups = statistics - .group_count - .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - let key = statistics - .key_bytes - .ok_or(AnalyticalCostError::MissingOrZero("key_bytes"))?; - let value = statistics - .aggregate_value_bytes - .ok_or(AnalyticalCostError::MissingOrZero("aggregate_value_bytes"))?; - ResourceEstimate { - cpu_ops: left.rows as f64, - peak_memory_bytes: checked_bytes(&[ - groups, - key.checked_add(value) - .and_then(|bytes| bytes.checked_add(16)) - .ok_or(AnalyticalCostError::Overflow)?, - ])?, - scan_bytes: 0, - } - } - PhysicalOperator::Deduplicate => { - let groups = statistics - .group_count - .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - let key = statistics - .key_bytes - .ok_or(AnalyticalCostError::MissingOrZero("key_bytes"))?; + ( + PhysicalOperator::Filter | PhysicalOperator::Project | PhysicalOperator::PassThrough, + _, + ) => ResourceEstimate { + cpu_ops: left.rows as f64, + peak_memory_bytes: per_row_width(output.rows, output.bytes)?, + scan_bytes: 0, + }, + ( + PhysicalOperator::HashAggregate, + OperatorStatistics::HashAggregate { + group_count, + key_bytes, + accumulator_bytes_per_group, + .. + }, + ) => ResourceEstimate { + cpu_ops: left.rows as f64, + peak_memory_bytes: checked_bytes(&[ + *group_count, + key_bytes + .checked_add(*accumulator_bytes_per_group) + .and_then(|bytes| bytes.checked_add(16)) + .ok_or(AnalyticalCostError::Overflow)?, + ])?, + scan_bytes: 0, + }, + ( + PhysicalOperator::HashDeduplicate, + OperatorStatistics::HashDeduplicate { + distinct_key_count, + key_bytes, + .. + }, + ) => ResourceEstimate { + cpu_ops: left.rows as f64, + peak_memory_bytes: checked_bytes(&[ + *distinct_key_count, + key_bytes + .checked_add(16) + .ok_or(AnalyticalCostError::Overflow)?, + ])?, + scan_bytes: 0, + }, + (PhysicalOperator::InMemoryComparisonSort | PhysicalOperator::InMemoryOrderedWindow, _) => { ResourceEstimate { - cpu_ops: left.rows as f64, - peak_memory_bytes: checked_bytes(&[ - groups, - key.checked_add(16).ok_or(AnalyticalCostError::Overflow)?, - ])?, + cpu_ops: left.rows as f64 * (left.rows.max(2) as f64).log2().ceil(), + peak_memory_bytes: left.bytes, scan_bytes: 0, } } - PhysicalOperator::Sort | PhysicalOperator::Window => ResourceEstimate { - cpu_ops: left.rows as f64 * (left.rows.max(2) as f64).log2().ceil(), - peak_memory_bytes: left.bytes, - scan_bytes: 0, - }, - PhysicalOperator::TopK => { - let k = statistics - .k - .ok_or(AnalyticalCostError::MissingOrZero("k"))?; - let heap_rows = k.min(left.rows); + (PhysicalOperator::TopK { limit, offset }, _) => { + let heap_capacity = limit + .checked_add(offset) + .ok_or(AnalyticalCostError::Overflow)?; + let heap_rows = heap_capacity.min(left.rows); ResourceEstimate { cpu_ops: left.rows as f64 * (heap_rows.max(2) as f64).log2().ceil(), peak_memory_bytes: checked_bytes(&[ @@ -521,11 +563,8 @@ pub fn estimate_operator( scan_bytes: 0, } } - PhysicalOperator::HashJoin => { + (PhysicalOperator::HashJoin { build_side }, _) => { let right = input(1)?; - let build_side = statistics - .hash_join_build_side - .ok_or(AnalyticalCostError::MissingOrZero("hash_join_build_side"))?; ResourceEstimate { cpu_ops: left.rows as f64 + right.rows as f64 + output.rows as f64, peak_memory_bytes: match build_side { @@ -542,26 +581,32 @@ pub fn estimate_operator( scan_bytes: 0, } } - PhysicalOperator::Concat => ResourceEstimate { + (PhysicalOperator::Concat, _) => ResourceEstimate { cpu_ops: output.rows as f64, peak_memory_bytes: per_row_width(output.rows, output.bytes)?, scan_bytes: 0, }, - PhysicalOperator::Limit => { - let consumed = statistics - .limit_rows_consumed - .ok_or(AnalyticalCostError::MissingOrZero("limit_rows_consumed"))?; - if consumed > left.rows || consumed < output.rows { - return Err(AnalyticalCostError::InconsistentOperatorStatistics( - "Limit rows consumed must cover its output without exceeding its input", - )); - } + (PhysicalOperator::Limit { limit, offset }, _) => { + let consumed = if limit == 0 { + 0 + } else { + left.rows.min( + offset + .checked_add(limit) + .ok_or(AnalyticalCostError::Overflow)?, + ) + }; ResourceEstimate { cpu_ops: consumed as f64, peak_memory_bytes: per_row_width(output.rows, output.bytes)?, scan_bytes: 0, } } + _ => { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "statistics variant does not match physical operator", + )); + } }; if estimate.cpu_ops.is_finite() { Ok(estimate) @@ -575,78 +620,109 @@ fn validate_operator_semantics( statistics: &OperatorStatistics, ) -> Result<(), AnalyticalCostError> { let inconsistent = |reason| Err(AnalyticalCostError::InconsistentOperatorStatistics(reason)); - if !matches!(operator, PhysicalOperator::Scan) && statistics.source_scan_bytes != 0 { - return inconsistent("only Scan may charge source bytes"); + if !statistics_match_operator(operator, statistics) { + return inconsistent("statistics variant does not match physical operator"); } - let input = statistics.inputs.first().copied().ok_or( - AnalyticalCostError::InconsistentOperatorStatistics("operator input is missing"), - )?; - let output = statistics.output; - match operator { - PhysicalOperator::Scan => { + let input = statistics + .input(0) + .ok_or(AnalyticalCostError::InconsistentOperatorStatistics( + "operator input is missing", + ))?; + let output = statistics.output(); + match (operator, statistics) { + (PhysicalOperator::Scan, _) => { if input != output { return inconsistent("Scan input and output edges differ"); } } - PhysicalOperator::Filter => { + (PhysicalOperator::Filter, _) => { if output.rows > input.rows || output.bytes > input.bytes { return inconsistent("Filter output expands its input"); } } - PhysicalOperator::Project => { + (PhysicalOperator::Project, _) => { if output.rows != input.rows { return inconsistent("Project changes row cardinality"); } } - PhysicalOperator::HashAggregate | PhysicalOperator::Deduplicate => { - let groups = statistics - .group_count - .filter(|groups| *groups > 0) - .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - if groups > input.rows || output.rows != groups { + ( + PhysicalOperator::HashAggregate, + OperatorStatistics::HashAggregate { + group_count, + key_bytes, + accumulator_bytes_per_group, + .. + }, + ) => { + if *group_count == 0 || *key_bytes == 0 || *accumulator_bytes_per_group == 0 { + return inconsistent("HashAggregate state statistics must be positive"); + } + if *group_count > input.rows || output.rows != *group_count { return inconsistent("grouped output differs from distinct group cardinality"); } } - PhysicalOperator::Sort | PhysicalOperator::Window | PhysicalOperator::PassThrough => { + ( + PhysicalOperator::HashDeduplicate, + OperatorStatistics::HashDeduplicate { + distinct_key_count, + key_bytes, + .. + }, + ) => { + if *distinct_key_count == 0 || *key_bytes == 0 { + return inconsistent("Deduplicate state statistics must be positive"); + } + if *distinct_key_count > input.rows || output.rows != *distinct_key_count { + return inconsistent("deduplicated output differs from distinct key cardinality"); + } + } + ( + PhysicalOperator::InMemoryComparisonSort + | PhysicalOperator::InMemoryOrderedWindow + | PhysicalOperator::PassThrough, + _, + ) => { if input != output { return inconsistent("cardinality-preserving operator changes its edge"); } } - PhysicalOperator::Concat => { - let total = statistics.inputs.iter().try_fold( - EdgeStatistics { rows: 0, bytes: 0 }, - |total, edge| { - Ok::<_, AnalyticalCostError>(EdgeStatistics { - rows: total - .rows - .checked_add(edge.rows) - .ok_or(AnalyticalCostError::Overflow)?, - bytes: total - .bytes - .checked_add(edge.bytes) - .ok_or(AnalyticalCostError::Overflow)?, - }) - }, - )?; + (PhysicalOperator::Concat, OperatorStatistics::Concat { inputs, .. }) => { + let total = + inputs + .iter() + .try_fold(EdgeStatistics { rows: 0, bytes: 0 }, |total, edge| { + Ok::<_, AnalyticalCostError>(EdgeStatistics { + rows: total + .rows + .checked_add(edge.rows) + .ok_or(AnalyticalCostError::Overflow)?, + bytes: total + .bytes + .checked_add(edge.bytes) + .ok_or(AnalyticalCostError::Overflow)?, + }) + })?; if output != total { return inconsistent("Concat output differs from the sum of its inputs"); } } - PhysicalOperator::TopK => { - let k = statistics - .k - .filter(|k| *k > 0) - .ok_or(AnalyticalCostError::MissingOrZero("k"))?; - if output.rows != input.rows.min(k) { + (PhysicalOperator::TopK { limit, offset }, _) => { + if limit == 0 { + return inconsistent("Top-K limit must be positive"); + } + let expected = input.rows.saturating_sub(offset).min(limit); + if output.rows != expected { return inconsistent("Top-K output differs from its cardinality bound"); } } - PhysicalOperator::Limit => { - if output.rows > input.rows { - return inconsistent("Limit output exceeds its input"); + (PhysicalOperator::Limit { limit, offset }, _) => { + let expected = input.rows.saturating_sub(offset).min(limit); + if output.rows != expected { + return inconsistent("Limit output differs from limit and offset"); } } - PhysicalOperator::HashJoin => {} + (PhysicalOperator::HashJoin { .. }, _) => {} + _ => return inconsistent("statistics variant does not match physical operator"), } Ok(()) } @@ -731,21 +807,21 @@ fn checked_bytes(parts: &[u64]) -> Result { mod tests { use super::*; use crate::analytical_statistics::{ - validate_comparison_scopes, ComparisonScope, EdgeStatistics, OperatorStatistics, - SourceCoverage, + validate_comparison_scopes, BinaryEdgeStatistics, ComparisonScope, EdgeStatistics, + OperatorStatistics, SourceCoverage, UnaryEdgeStatistics, }; - fn unary_inputs(input_rows: u64, output_rows: u64) -> OperatorStatistics { - statistics( - vec![EdgeStatistics { + fn unary_row_edges(input_rows: u64, output_rows: u64) -> UnaryEdgeStatistics { + UnaryEdgeStatistics { + input: EdgeStatistics { rows: input_rows, bytes: input_rows.saturating_mul(8), - }], - EdgeStatistics { + }, + output: EdgeStatistics { rows: output_rows, bytes: output_rows.saturating_mul(8), }, - ) + } } #[test] @@ -765,7 +841,12 @@ mod tests { } ); assert_eq!( - expected_input_arity(PhysicalOperator::HashJoin, 2), + expected_input_arity( + PhysicalOperator::HashJoin { + build_side: HashJoinBuildSide::Left, + }, + 2, + ), OperatorInputArity { statistics_inputs: 2, dag_children: 2, @@ -783,27 +864,47 @@ mod tests { #[test] fn operator_estimator_rejects_contradictory_cardinality_evidence() { assert!(matches!( - estimate_operator(PhysicalOperator::Filter, unary_inputs(10, 11)), + estimate_operator( + PhysicalOperator::Filter, + OperatorStatistics::Filter { + edges: unary_row_edges(10, 11), + }, + ), Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) )); assert!(matches!( - estimate_operator(PhysicalOperator::Project, unary_inputs(10, 9)), + estimate_operator( + PhysicalOperator::Project, + OperatorStatistics::Project { + edges: unary_row_edges(10, 9), + }, + ), Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) )); - let mut aggregate = unary_inputs(10, 3); - aggregate.group_count = Some(2); - aggregate.key_bytes = Some(8); - aggregate.aggregate_value_bytes = Some(8); assert!(matches!( - estimate_operator(PhysicalOperator::HashAggregate, aggregate), + estimate_operator( + PhysicalOperator::HashAggregate, + OperatorStatistics::HashAggregate { + edges: unary_row_edges(10, 3), + group_count: 2, + key_bytes: 8, + accumulator_bytes_per_group: 8, + }, + ), Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) )); - let mut topk = unary_inputs(10, 4); - topk.k = Some(3); assert!(matches!( - estimate_operator(PhysicalOperator::TopK, topk), + estimate_operator( + PhysicalOperator::TopK { + limit: 3, + offset: 0, + }, + OperatorStatistics::TopK { + edges: unary_row_edges(10, 4), + }, + ), Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) )); } @@ -812,37 +913,39 @@ mod tests { fn physical_operator_formulas_keep_disk_at_scan_and_require_join_stats() { let scan = estimate_operator( PhysicalOperator::Scan, - OperatorStatistics { - source_scan_bytes: 64_000, - ..statistics( - vec![EdgeStatistics { + OperatorStatistics::Scan { + source_read_bytes: 64_000, + edges: UnaryEdgeStatistics { + input: EdgeStatistics { rows: 1_000, bytes: 64_000, - }], - EdgeStatistics { + }, + output: EdgeStatistics { rows: 1_000, bytes: 64_000, }, - ) + }, }, ) .unwrap(); assert_eq!(scan.scan_bytes, 64_000); let topk = estimate_operator( - PhysicalOperator::TopK, - OperatorStatistics { - k: Some(10), - ..statistics( - vec![EdgeStatistics { + PhysicalOperator::TopK { + limit: 10, + offset: 0, + }, + OperatorStatistics::TopK { + edges: UnaryEdgeStatistics { + input: EdgeStatistics { rows: 1_000, bytes: 40_000, - }], - EdgeStatistics { + }, + output: EdgeStatistics { rows: 10, bytes: 400, }, - ) + }, }, ) .unwrap(); @@ -850,54 +953,28 @@ mod tests { assert_eq!(topk.cpu_ops, 4_000.0); assert_eq!(topk.peak_memory_bytes, 400); - let missing_join_stats = estimate_operator( - PhysicalOperator::HashJoin, - statistics( - vec![EdgeStatistics { - rows: 1_000, - bytes: 64_000, - }], - EdgeStatistics { - rows: 100, - bytes: 12_800, - }, - ), - ); - assert_eq!( - missing_join_stats, - Err(AnalyticalCostError::MissingOrZero("operator input edge")) - ); - - let missing_build_side = estimate_operator( - PhysicalOperator::HashJoin, - statistics( - vec![ - EdgeStatistics { - rows: 1_000, - bytes: 64_000, - }, - EdgeStatistics { - rows: 10, - bytes: 1_280, - }, - ], - EdgeStatistics { - rows: 100, - bytes: 12_800, - }, - ), + let mismatched_join_statistics = estimate_operator( + PhysicalOperator::HashJoin { + build_side: HashJoinBuildSide::Left, + }, + OperatorStatistics::Filter { + edges: unary_row_edges(1_000, 100), + }, ); assert_eq!( - missing_build_side, - Err(AnalyticalCostError::MissingOrZero("hash_join_build_side")) + mismatched_join_statistics, + Err(AnalyticalCostError::InconsistentOperatorStatistics( + "statistics variant does not match physical operator" + )) ); let build_left = estimate_operator( - PhysicalOperator::HashJoin, - OperatorStatistics { - hash_join_build_side: Some(HashJoinBuildSide::Left), - ..statistics( - vec![ + PhysicalOperator::HashJoin { + build_side: HashJoinBuildSide::Left, + }, + OperatorStatistics::HashJoin { + edges: BinaryEdgeStatistics { + inputs: [ EdgeStatistics { rows: 1_000, bytes: 64_000, @@ -907,11 +984,11 @@ mod tests { bytes: 1_280, }, ], - EdgeStatistics { + output: EdgeStatistics { rows: 100, bytes: 12_800, }, - ) + }, }, ) .unwrap(); @@ -919,58 +996,62 @@ mod tests { let aggregate = estimate_operator( PhysicalOperator::HashAggregate, - OperatorStatistics { - group_count: Some(100), - key_bytes: Some(16), - aggregate_value_bytes: Some(24), - ..statistics( - vec![EdgeStatistics { + OperatorStatistics::HashAggregate { + group_count: 100, + key_bytes: 16, + accumulator_bytes_per_group: 24, + edges: UnaryEdgeStatistics { + input: EdgeStatistics { rows: 1_000, bytes: 64_000, - }], - EdgeStatistics { + }, + output: EdgeStatistics { rows: 100, bytes: 4_000, }, - ) + }, }, ) .unwrap(); assert_eq!(aggregate.peak_memory_bytes, 5_600); let oversized_topk = estimate_operator( - PhysicalOperator::TopK, - OperatorStatistics { - k: Some(1_000), - ..statistics( - vec![EdgeStatistics { + PhysicalOperator::TopK { + limit: 1_000, + offset: 0, + }, + OperatorStatistics::TopK { + edges: UnaryEdgeStatistics { + input: EdgeStatistics { rows: 4, bytes: 160, - }], - EdgeStatistics { + }, + output: EdgeStatistics { rows: 4, bytes: 160, }, - ) + }, }, ) .unwrap(); assert_eq!(oversized_topk.cpu_ops, 8.0); let offset_limit = estimate_operator( - PhysicalOperator::Limit, - OperatorStatistics { - limit_rows_consumed: Some(900_010), - ..statistics( - vec![EdgeStatistics { + PhysicalOperator::Limit { + limit: 10, + offset: 900_000, + }, + OperatorStatistics::Limit { + edges: UnaryEdgeStatistics { + input: EdgeStatistics { rows: 1_000_000, bytes: 40_000_000, - }], - EdgeStatistics { + }, + output: EdgeStatistics { rows: 10, bytes: 400, }, - ) + }, }, ) .unwrap(); @@ -1029,22 +1110,32 @@ mod tests { let provided = HashMap::from([ ( "scan".into(), - OperatorStatistics { - source_scan_bytes: 1_000, - ..statistics(vec![scan_edge], scan_edge) + OperatorStatistics::Scan { + edges: unary_edges(scan_edge, scan_edge), + source_read_bytes: 1_000, + }, + ), + ( + "left".into(), + OperatorStatistics::Filter { + edges: unary_edges(scan_edge, branch_edge), + }, + ), + ( + "right".into(), + OperatorStatistics::Filter { + edges: unary_edges(scan_edge, branch_edge), }, ), - ("left".into(), statistics(vec![scan_edge], branch_edge)), - ("right".into(), statistics(vec![scan_edge], branch_edge)), ( "root".into(), - statistics( - vec![branch_edge, branch_edge], - EdgeStatistics { + OperatorStatistics::Concat { + inputs: vec![branch_edge, branch_edge], + output: EdgeStatistics { rows: 80, bytes: 800, }, - ), + }, ), ]); let mut scope = comparison_scope(); @@ -1081,7 +1172,10 @@ mod tests { }, PhysicalDagNode { id: "read".into(), - operator: PhysicalOperator::Limit, + operator: PhysicalOperator::Limit { + limit: 1, + offset: 0, + }, children: vec!["state".into()], source_coverage: None, output_buffer_bytes: 16, @@ -1097,25 +1191,24 @@ mod tests { let provided = HashMap::from([ ( "scan".into(), - OperatorStatistics { - source_scan_bytes: 1_000, - ..statistics(vec![scan_edge], scan_edge) + OperatorStatistics::Scan { + edges: unary_edges(scan_edge, scan_edge), + source_read_bytes: 1_000, }, ), ( "state".into(), - OperatorStatistics { - group_count: Some(1), - key_bytes: Some(8), - aggregate_value_bytes: Some(8), - ..statistics(vec![scan_edge], state_edge) + OperatorStatistics::HashAggregate { + edges: unary_edges(scan_edge, state_edge), + group_count: 1, + key_bytes: 8, + accumulator_bytes_per_group: 8, }, ), ( "read".into(), - OperatorStatistics { - limit_rows_consumed: Some(1), - ..statistics(vec![state_edge], state_edge) + OperatorStatistics::Limit { + edges: unary_edges(state_edge, state_edge), }, ), ]); @@ -1155,18 +1248,8 @@ mod tests { } } - fn statistics(inputs: Vec, output: EdgeStatistics) -> OperatorStatistics { - OperatorStatistics { - source_scan_bytes: 0, - inputs, - output, - group_count: None, - key_bytes: None, - aggregate_value_bytes: None, - k: None, - limit_rows_consumed: None, - hash_join_build_side: None, - } + fn unary_edges(input: EdgeStatistics, output: EdgeStatistics) -> UnaryEdgeStatistics { + UnaryEdgeStatistics { input, output } } #[test] @@ -1240,18 +1323,18 @@ mod tests { let scope = comparison_scope(); let mut provided = HashMap::from([( "scan".to_string(), - OperatorStatistics { - source_scan_bytes: 1_000, - ..statistics( - vec![EdgeStatistics { + OperatorStatistics::Scan { + edges: unary_edges( + EdgeStatistics { rows: 100, bytes: 1_000, - }], + }, EdgeStatistics { rows: 100, bytes: 1_000, }, - ) + ), + source_read_bytes: 1_000, }, )]); @@ -1264,16 +1347,18 @@ mod tests { provided.insert( "filter".into(), - statistics( - vec![EdgeStatistics { - rows: 99, - bytes: 990, - }], - EdgeStatistics { - rows: 40, - bytes: 400, - }, - ), + OperatorStatistics::Filter { + edges: unary_edges( + EdgeStatistics { + rows: 99, + bytes: 990, + }, + EdgeStatistics { + rows: 40, + bytes: 400, + }, + ), + }, ); assert_eq!( estimate_physical_dag(&nodes, "filter", &scope, &provided), @@ -1313,32 +1398,34 @@ mod tests { let provided = HashMap::from([ ( "scan".to_string(), - OperatorStatistics { - source_scan_bytes: 1_000, - ..statistics( - vec![EdgeStatistics { + OperatorStatistics::Scan { + edges: unary_edges( + EdgeStatistics { rows: 100, bytes: 1_000, - }], + }, EdgeStatistics { rows: 100, bytes: 1_000, }, - ) + ), + source_read_bytes: 1_000, }, ), ( "filter".to_string(), - statistics( - vec![EdgeStatistics { - rows: 100, - bytes: 1_000, - }], - EdgeStatistics { - rows: 40, - bytes: 400, - }, - ), + OperatorStatistics::Filter { + edges: unary_edges( + EdgeStatistics { + rows: 100, + bytes: 1_000, + }, + EdgeStatistics { + rows: 40, + bytes: 400, + }, + ), + }, ), ]); @@ -1378,14 +1465,16 @@ mod tests { let provided = HashMap::from([ ( "scan".into(), - OperatorStatistics { - source_scan_bytes: 1_000, - ..statistics(vec![input], input) + OperatorStatistics::Scan { + edges: unary_edges(input, input), + source_read_bytes: 1_000, }, ), ( "filter".into(), - statistics(vec![input], EdgeStatistics { rows: 0, bytes: 0 }), + OperatorStatistics::Filter { + edges: unary_edges(input, EdgeStatistics { rows: 0, bytes: 0 }), + }, ), ]); @@ -1419,9 +1508,9 @@ mod tests { }; let provided = HashMap::from([( "scan".into(), - OperatorStatistics { - source_scan_bytes: 250, - ..statistics(vec![edge], edge) + OperatorStatistics::Scan { + edges: unary_edges(edge, edge), + source_read_bytes: 250, }, )]); @@ -1450,9 +1539,9 @@ mod tests { }; let provided = HashMap::from([( "scan".into(), - OperatorStatistics { - source_scan_bytes: 250, - ..statistics(vec![edge], edge) + OperatorStatistics::Scan { + edges: unary_edges(edge, edge), + source_read_bytes: 250, }, )]); @@ -1490,9 +1579,9 @@ mod tests { }; let provided = HashMap::from([( "scan".into(), - OperatorStatistics { - source_scan_bytes: 250, - ..statistics(vec![edge], edge) + OperatorStatistics::Scan { + edges: unary_edges(edge, edge), + source_read_bytes: 250, }, )]); @@ -1535,18 +1624,18 @@ mod tests { let provided = HashMap::from([ ( "scan".into(), - OperatorStatistics { - source_scan_bytes: 250, - ..statistics(vec![input], input) + OperatorStatistics::Scan { + edges: unary_edges(input, input), + source_read_bytes: 250, }, ), ( "aggregate".into(), - OperatorStatistics { - group_count: Some(1), - key_bytes: Some(8), - aggregate_value_bytes: Some(8), - ..statistics(vec![input], output) + OperatorStatistics::HashAggregate { + edges: unary_edges(input, output), + group_count: 1, + key_bytes: 8, + accumulator_bytes_per_group: 8, }, ), ]); @@ -1577,9 +1666,9 @@ mod tests { }; let provided = HashMap::from([( "scan".into(), - OperatorStatistics { - source_scan_bytes: 250, - ..statistics(vec![edge], edge) + OperatorStatistics::Scan { + edges: unary_edges(edge, edge), + source_read_bytes: 250, }, )]); let raw_scope = comparison_scope(); @@ -1613,9 +1702,9 @@ mod tests { }; let estimate = estimate_operator( PhysicalOperator::Scan, - OperatorStatistics { - source_scan_bytes: 2_500, - ..statistics(vec![logical], logical) + OperatorStatistics::Scan { + edges: unary_edges(logical, logical), + source_read_bytes: 2_500, }, ) .unwrap(); @@ -1625,55 +1714,43 @@ mod tests { } #[test] - fn non_scan_operator_cannot_charge_source_bytes() { - let coverage = comparison_scope().sources[0].clone(); - let nodes = vec![ - PhysicalDagNode { - id: "scan".into(), - operator: PhysicalOperator::Scan, - children: vec![], - source_coverage: Some(coverage), - output_buffer_bytes: 10, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - PhysicalDagNode { - id: "filter".into(), - operator: PhysicalOperator::Filter, - children: vec!["scan".into()], - source_coverage: None, - output_buffer_bytes: 10, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - ]; + fn non_scan_operator_estimates_cannot_charge_source_reads() { let edge = EdgeStatistics { rows: 100, bytes: 1_000, }; - let provided = HashMap::from([ - ( - "scan".into(), - OperatorStatistics { - source_scan_bytes: 250, - ..statistics(vec![edge], edge) - }, - ), - ( - "filter".into(), - OperatorStatistics { - source_scan_bytes: 250, - ..statistics(vec![edge], edge) - }, - ), - ]); + let filter = OperatorStatistics::Filter { + edges: unary_edges(edge, edge), + }; assert_eq!( - estimate_physical_dag(&nodes, "filter", &comparison_scope(), &provided), - Err(AnalyticalCostError::InvalidOperatorStatistics { - node: "filter".into(), - reason: "only scan operators may read source bytes", - }) + estimate_operator(PhysicalOperator::Filter, filter) + .unwrap() + .scan_bytes, + 0 + ); + } + + #[test] + fn serialized_operator_statistics_reject_unrelated_fields() { + let edge = serde_json::json!({ "rows": 100, "bytes": 1_000 }); + let edges = serde_json::json!({ "input": edge, "output": edge }); + + assert!( + serde_json::from_value::(serde_json::json!({ + "operator": "filter", + "edges": edges.clone(), + "source_read_bytes": 1_000 + })) + .is_err() + ); + assert!( + serde_json::from_value::(serde_json::json!({ + "operator": "top_k", + "edges": edges, + "k": 10 + })) + .is_err() ); } } diff --git a/crates/asap-aware-mapping/src/analytical_statistics.rs b/crates/asap-aware-mapping/src/analytical_statistics.rs index 80477303..76a7bfc7 100644 --- a/crates/asap-aware-mapping/src/analytical_statistics.rs +++ b/crates/asap-aware-mapping/src/analytical_statistics.rs @@ -13,7 +13,7 @@ use asap_types::workload::{ }; use serde::{Deserialize, Serialize}; -use crate::analytical_cost::{AnalyticalCostError, HashJoinBuildSide}; +use crate::analytical_cost::AnalyticalCostError; /// The semantic and workload boundary within which two resource estimates /// may be compared. Canonical workload and query-IR types remain authoritative; @@ -183,27 +183,126 @@ impl EdgeStatistics { } } -/// Authoritative cardinality and width facts for one physical operator. -/// `inputs` has one entry per child edge, except `Scan`, whose single entry -/// describes its external source edge. The output is compared with every -/// parent's corresponding input, so conflicting provider evidence fails. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct OperatorStatistics { - /// Physical bytes read from storage by this operator. This is independent - /// of decoded logical bytes on `inputs`; it must be zero for non-scan - /// operators in the current in-memory physical model. - pub source_scan_bytes: u64, - pub inputs: Vec, +/// Input and output evidence for a unary physical operator. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct UnaryEdgeStatistics { + pub input: EdgeStatistics, + pub output: EdgeStatistics, +} + +/// Input and output evidence for a binary physical operator. The input order +/// is the physical operator's left/right order and must match its DAG children. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct BinaryEdgeStatistics { + pub inputs: [EdgeStatistics; 2], pub output: EdgeStatistics, - pub group_count: Option, - pub key_bytes: Option, - pub aggregate_value_bytes: Option, - pub k: Option, - /// Rows consumed by a physical Limit, including rows skipped by OFFSET. - /// This is an execution statistic rather than the output cardinality. - #[serde(default)] - pub limit_rows_consumed: Option, - pub hash_join_build_side: Option, +} + +/// Authoritative evidence for one physical operator, structured by operator +/// kind so unrelated facts cannot be combined in one flat bag of `Option`s. +/// Physical configuration such as a Top-K limit or hash-join build side lives +/// on `PhysicalOperator`; this enum contains workload/catalog statistics only. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "operator", rename_all = "snake_case", deny_unknown_fields)] +pub enum OperatorStatistics { + Scan { + edges: UnaryEdgeStatistics, + /// Physical bytes read from storage, independent of decoded logical + /// bytes on the source edge. + source_read_bytes: u64, + }, + Filter { + edges: UnaryEdgeStatistics, + }, + Project { + edges: UnaryEdgeStatistics, + }, + HashAggregate { + edges: UnaryEdgeStatistics, + group_count: u64, + key_bytes: u64, + accumulator_bytes_per_group: u64, + }, + InMemoryComparisonSort { + edges: UnaryEdgeStatistics, + }, + TopK { + edges: UnaryEdgeStatistics, + }, + HashJoin { + edges: BinaryEdgeStatistics, + }, + HashDeduplicate { + edges: UnaryEdgeStatistics, + distinct_key_count: u64, + key_bytes: u64, + }, + Concat { + inputs: Vec, + output: EdgeStatistics, + }, + InMemoryOrderedWindow { + edges: UnaryEdgeStatistics, + }, + Limit { + edges: UnaryEdgeStatistics, + }, + PassThrough { + edges: UnaryEdgeStatistics, + }, +} + +impl OperatorStatistics { + pub fn input_count(&self) -> usize { + match self { + Self::Scan { .. } + | Self::Filter { .. } + | Self::Project { .. } + | Self::HashAggregate { .. } + | Self::InMemoryComparisonSort { .. } + | Self::TopK { .. } + | Self::HashDeduplicate { .. } + | Self::InMemoryOrderedWindow { .. } + | Self::Limit { .. } + | Self::PassThrough { .. } => 1, + Self::HashJoin { .. } => 2, + Self::Concat { inputs, .. } => inputs.len(), + } + } + + pub fn input(&self, index: usize) -> Option { + match self { + Self::Scan { edges, .. } + | Self::HashAggregate { edges, .. } + | Self::HashDeduplicate { edges, .. } => (index == 0).then_some(edges.input), + Self::Filter { edges } + | Self::Project { edges } + | Self::InMemoryComparisonSort { edges } + | Self::TopK { edges } + | Self::InMemoryOrderedWindow { edges } + | Self::Limit { edges } + | Self::PassThrough { edges } => (index == 0).then_some(edges.input), + Self::HashJoin { edges } => edges.inputs.get(index).copied(), + Self::Concat { inputs, .. } => inputs.get(index).copied(), + } + } + + pub fn output(&self) -> EdgeStatistics { + match self { + Self::Scan { edges, .. } + | Self::HashAggregate { edges, .. } + | Self::HashDeduplicate { edges, .. } => edges.output, + Self::Filter { edges } + | Self::Project { edges } + | Self::InMemoryComparisonSort { edges } + | Self::TopK { edges } + | Self::InMemoryOrderedWindow { edges } + | Self::Limit { edges } + | Self::PassThrough { edges } => edges.output, + Self::HashJoin { edges } => edges.output, + Self::Concat { output, .. } => *output, + } + } } /// Resolves physical statistics and owns their catalog/observation freshness. diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 40600a8f..77b93c2b 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -46,7 +46,7 @@ normalized workload, lowered query IR, and freshness-aware statistics: | Query demand | `QueryWorkloadEntry.recurrence`, normalized over an explicit horizon. | | Event-time range | `QueryWorkloadEntry.time_selection`. | | Accuracy and latency requirements | `QueryWorkloadEntry.requirements`. | -| Operator shape, grouping keys, and constants such as Top-K `k` | Lowered `QueryExpr` and `AggIntent`. | +| Operator shape, grouping keys, and constants such as Top-K limit and offset | Lowered query and selected physical plan. | Evidence is read through `Evidence::value_at(planning_time)`. Stale, future, or improperly time-bounded evidence remains unknown. Costing follows @@ -141,18 +141,61 @@ An `OperatorStatisticsProvider` resolves one `OperatorStatistics` value for each reachable physical node: ```text -OperatorStatistics { - source_scan_bytes, - inputs: [EdgeStatistics { rows, bytes }, ...], - output: EdgeStatistics { rows, bytes }, - group_count, - key_bytes, - aggregate_value_bytes, - k, - hash_join_build_side, +EdgeStatistics { rows, bytes } + +OperatorStatistics = + Scan { + edges: UnaryEdgeStatistics, + source_read_bytes, + } + | Filter { edges: UnaryEdgeStatistics } + | Project { edges: UnaryEdgeStatistics } + | HashAggregate { + edges: UnaryEdgeStatistics, + group_count, + key_bytes, + accumulator_bytes_per_group, + } + | InMemoryComparisonSort { edges: UnaryEdgeStatistics } + | TopK { edges: UnaryEdgeStatistics } + | HashJoin { edges: BinaryEdgeStatistics } + | HashDeduplicate { + edges: UnaryEdgeStatistics, + distinct_key_count, + key_bytes, + } + | Concat { inputs, output } + | InMemoryOrderedWindow { edges: UnaryEdgeStatistics } + | Limit { edges: UnaryEdgeStatistics } + | PassThrough { edges: UnaryEdgeStatistics } } ``` +`UnaryEdgeStatistics` contains exactly one input and one output; +`BinaryEdgeStatistics` contains an ordered pair of inputs and one output. +`Concat` is the only variadic case. `EdgeStatistics` itself intentionally +remains operator-independent: an edge carries logical rows and bytes, and the +same edge is the output of one node and an input of every consumer. Making the +edge type depend on either endpoint would prevent direct consistency checks. + +The outer enum is operator-specific. A Filter cannot accidentally carry +group cardinality, a Top-K statistics record cannot carry a join build side, +and a non-scan record cannot carry source-read bytes. Serialized evidence is +internally tagged by operator and rejects unknown fields. + +Physical configuration is not catalog evidence and therefore lives on +`PhysicalOperator`: + +| Physical operator | Configuration owned by the plan | +|---|---| +| `TopK` | output `limit` and `offset`; heap capacity is `limit + offset` | +| `Limit` | output `limit` and `offset` | +| `HashJoin` | left or right build side | + +This distinction removes the former flat optional `k` field. A Top-K bound is +part of the chosen algorithm, while input/output cardinality and width are +observed or estimated facts about that operator in this workload. + The provider owns provenance, freshness, and derivation. The estimator resolves each reachable node once, so one estimate cannot mix values across a live catalog refresh. A scan has one external source input; every other input is in @@ -165,7 +208,7 @@ positive logical bytes so width-dependent formulas do not invent a row width. A parent therefore cannot silently substitute the original source cardinality for an intermediate edge. -`OperatorStatistics.inputs` and `PhysicalDagNode.children` therefore have +The statistics inputs and `PhysicalDagNode.children` therefore have different arity only for a source leaf: | Operator shape | Statistics inputs | DAG children | @@ -180,10 +223,12 @@ operator cannot silently inherit unary arity; its statistics-input and DAG-child counts must both be defined. `EdgeStatistics.bytes` is decoded logical data carried on an edge. -`source_scan_bytes` is physical storage I/O and is charged only by `Scan`. +`Scan.source_read_bytes` is physical storage I/O and is charged only by +`Scan`. Compression, column pruning, or encoded storage can therefore make these -values different; neither is inferred from the other. Non-scan operators must -report zero source bytes in the current in-memory operator model. +values different; neither is inferred from the other. Other statistics +variants have no source-read field, so charging source I/O at a non-scan node +is not representable. ### Composition rules @@ -247,20 +292,23 @@ the DAG rules above. | Physical operator | CPU operations | Local memory | Source/disk reads | |---|---:|---:|---:| -| Scan | `input_rows` | one decoded input row/batch | `source_scan_bytes` | +| Scan | `input_rows` | one decoded input row/batch | `source_read_bytes` | | Filter | `input_rows` | one output row/batch | `0` | | Project or scalar pass-through | `input_rows` | one output row/batch | `0` | -| Hash aggregate | `input_rows` | `groups × (key + aggregate_value_bytes + hash metadata)` | `0` | -| Deduplicate | `input_rows` | keyed hash state | `0` | -| In-memory sort | `rows × ceil(log2(rows))` | `input_bytes` | `0` | -| Heap Top-K | `rows × ceil(log2(max(min(k, rows), 2)))` | `min(k, rows) × row_bytes` | `0` | +| Hash aggregate | `input_rows` | `groups × (key_bytes + accumulator_bytes_per_group + hash metadata)` | `0` | +| Hash deduplicate | `input_rows` | `distinct_key_count × (key_bytes + hash metadata)` | `0` | +| In-memory comparison sort | `rows × ceil(log2(rows))` | `input_bytes` | `0` | +| Heap Top-K | `rows × ceil(log2(max(min(limit + offset, rows), 2)))` | `min(limit + offset, rows) × row_bytes` | `0` | | Hash join | `left_rows + right_rows + output_rows` | selected build-side logical bytes plus 16 bytes of hash metadata per build row | `0` beyond children | | Concat | `output_rows` | one output row/batch | `0` | -| Ordered window | `rows × ceil(log2(rows))` | live partition/input bytes | `0` | -| Limit | `limit_rows_consumed` (including `OFFSET`) | one output row/batch | `0` | - -These formulas name physical implementations. An external sort must add -spill writes and reads; a nested-loop join must not use the hash-join formula. +| In-memory ordered window | `rows × ceil(log2(rows))` | live partition/input bytes | `0` | +| Limit | `min(input_rows, limit + offset)` | one output row/batch | `0` | + +These formulas name physical implementations. The enum uses names such as +`InMemoryComparisonSort`, `HashDeduplicate`, and `InMemoryOrderedWindow` so a +new algorithm cannot silently inherit a formula merely because it has the +same logical purpose. An external sort must add spill writes and reads; a +nested-loop join must not use the hash-join formula. If the physical choice or its required statistics are unknown, the estimate is unavailable. @@ -276,9 +324,9 @@ A retained sketch performs one build and serves later reads from state: ```text cpu_ops = input_rows // build scan + input_rows × update_ops(params) - + evaluation_count × physical_sketch_count × read_ops(params) + + evaluation_count × physical_summary_count × read_ops(params) -scan_bytes = source_scan_bytes for the build +scan_bytes = source_read_bytes for the build ``` Concrete accuracy-sized parameters determine state and work: @@ -295,7 +343,7 @@ Concrete accuracy-sized parameters determine state and work: | Theta | `ceil(log2(k))` | `k` | `k × 8` | For a per-subpopulation layout, -`physical_sketch_count = subpopulation_count`. A shared layout has the number +`physical_summary_count = subpopulation_count`. A shared layout has the number of physical structures described by that layout; the model must not infer it from logical group count alone. From 143805b0321147b0537a6132e42b8c0f6a7d4665 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 12:45:37 -0600 Subject: [PATCH 26/48] docs(cost): define physical statistics authority --- .../asap-aware-mapping/src/analytical_cost.rs | 10 ++--- .../src/analytical_statistics.rs | 28 +++++++++---- .../analytical-resource-cost.md | 41 ++++++++++++++++++- 3 files changed, 65 insertions(+), 14 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 54ec6a01..0ad7a5ba 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -1242,7 +1242,7 @@ mod tests { source: Source::Table { table_ref: "metrics".into(), }, - snapshot_id: "catalog-version-42".into(), + source_snapshot_id: "catalog-version-42".into(), predicates: vec![], }], } @@ -1263,7 +1263,7 @@ mod tests { assert_eq!(validate_comparison_scopes(&raw, &raw).unwrap(), 6); let mut candidate = raw.clone(); - candidate.sources[0].snapshot_id = "catalog-version-43".into(); + candidate.sources[0].source_snapshot_id = "catalog-version-43".into(); assert_eq!( validate_comparison_scopes(&raw, &candidate), Err(AnalyticalCostError::ComparisonScopeMismatch("sources")) @@ -1495,7 +1495,7 @@ mod tests { source: asap_types::pre_asap::query_expr::Source::Table { table_ref: "other_metrics".into(), }, - snapshot_id: "catalog-version-42".into(), + source_snapshot_id: "catalog-version-42".into(), predicates: vec![], }), output_buffer_bytes: 10, @@ -1561,7 +1561,7 @@ mod tests { source: asap_types::pre_asap::query_expr::Source::Table { table_ref: "auxiliary".into(), }, - snapshot_id: "catalog-version-42".into(), + source_snapshot_id: "catalog-version-42".into(), predicates: vec![], }); let nodes = vec![PhysicalDagNode { @@ -1673,7 +1673,7 @@ mod tests { )]); let raw_scope = comparison_scope(); let mut candidate_scope = raw_scope.clone(); - candidate_scope.sources[0].snapshot_id = "catalog-version-43".into(); + candidate_scope.sources[0].source_snapshot_id = "catalog-version-43".into(); assert_eq!( estimate_physical_dag_comparison( diff --git a/crates/asap-aware-mapping/src/analytical_statistics.rs b/crates/asap-aware-mapping/src/analytical_statistics.rs index 76a7bfc7..924a7404 100644 --- a/crates/asap-aware-mapping/src/analytical_statistics.rs +++ b/crates/asap-aware-mapping/src/analytical_statistics.rs @@ -33,9 +33,10 @@ pub struct ComparisonScope { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct SourceCoverage { pub source: Source, - /// Catalog version, object generation, snapshot timestamp, or another - /// provider-owned stable identifier for the physical source contents. - pub snapshot_id: String, + /// Provider-owned stable identifier for the physical source contents, + /// such as a catalog snapshot, table version, or object generation. + /// This is independent of the query's event-time `as_of` value. + pub source_snapshot_id: String, /// Canonical predicates copied from the bound/canonicalized query IR. pub predicates: Vec, } @@ -78,9 +79,11 @@ impl ComparisonScope { if self .sources .iter() - .any(|source| source.snapshot_id.is_empty()) + .any(|source| source.source_snapshot_id.is_empty()) { - return Err(AnalyticalCostError::MissingComparisonScope("snapshot_id")); + return Err(AnalyticalCostError::MissingComparisonScope( + "source_snapshot_id", + )); } evaluations_in_horizon(&self.recurrence, self.planning_time.0, self.horizon.0) } @@ -198,10 +201,19 @@ pub struct BinaryEdgeStatistics { pub output: EdgeStatistics, } -/// Authoritative evidence for one physical operator, structured by operator -/// kind so unrelated facts cannot be combined in one flat bag of `Option`s. +/// Workload-dependent evidence for one operator in an already-lowered +/// physical DAG. [`PhysicalOperator`](crate::analytical_cost::PhysicalOperator) +/// is the authoritative operator vocabulary: every one of its variants has a +/// matching statistics variant here. +/// +/// This enum intentionally does not mirror either logical IR. `QueryExpr` and +/// `SummaryExpr` are inputs to physical lowering, and one logical node may +/// expand into several physical nodes or choose among several algorithms. /// Physical configuration such as a Top-K limit or hash-join build side lives -/// on `PhysicalOperator`; this enum contains workload/catalog statistics only. +/// on `PhysicalOperator`; this enum contains only workload/catalog evidence +/// required to cost the selected algorithm. Structuring that evidence by +/// physical kind prevents unrelated facts from being combined in a flat bag +/// of `Option`s. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(tag = "operator", rename_all = "snake_case", deny_unknown_fields)] pub enum OperatorStatistics { diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 77b93c2b..3408c258 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -70,6 +70,45 @@ lowering provider owns resolving it from the canonical workload, catalog, and operator-statistics sources. Missing required evidence makes the entire plan unavailable. +### Operator vocabulary and source of truth + +`PhysicalOperator` is the source of truth for the cost model's operator +vocabulary. `OperatorStatistics` is paired one-to-one with that enum: each +supported physical algorithm has one evidence shape containing exactly the +facts its formula consumes. Exhaustive matches enforce that a newly added +physical operator must define its arity, statistics variant, validation, and +resource formula. + +Neither logical IR is the statistics schema: + +```text +pre-ASAP QueryExpr ─┐ + ├─ physical lowering ─> PhysicalDagNode/PhysicalOperator +post-ASAP SummaryExpr┘ │ + v + OperatorStatistics + │ + v + ResourceEstimate +``` + +The pre-ASAP IR describes exact query semantics. The post-ASAP IR describes +logical summary semantics, selected summary families, and summary operations. +Neither identifies every physical algorithm, buffer, build side, or execution +layout. For example, one logical `SummaryAgg` may lower to a CMS build, an +exact accumulator build, or another supported summary implementation; a +logical `SummaryEstimate` lowers to the corresponding physical readout. Those +physical nodes need different formulas and evidence even though they originate +from the same logical variant. + +The operator list in this version covers the physical query operators declared +by `PhysicalOperator`. It is not a claim that every `SummaryExpr` variant has +already been physically lowered. Summary build, join, merge, subtract, delete, +and readout become costable only after their lowering introduces explicit +physical operators and matching statistics variants. Until then, a candidate +containing such an unlowered operation is unavailable rather than partially +costed. + ## Workload horizon and lifecycle Every alternative must cover the same source data and query horizon. The @@ -110,7 +149,7 @@ canonical workload and query terms rather than defining parallel strings: | Event-time coverage | `TimeSelection`. | | Logical sources | the existing query-IR `Source`, one per scan. | | Filters | canonical bound `Predicate` values copied from the query IR. | -| Physical source contents | provider-owned `snapshot_id` per source. | +| Physical source contents | provider-owned `source_snapshot_id` per source. | The snapshot identifier is the only new scope concept. It is necessary because `Source` names a metric or table but neither the query IR nor workload schema From 15c341bea8dfa17201a6c8a1d63aef1d456a6c17 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 08:19:38 -0600 Subject: [PATCH 27/48] feat(cost): lower query DAGs for analytical estimates --- .../asap-aware-mapping/src/analytical_cost.rs | 4 + .../src/analytical_lowering.rs | 763 ++++++++++++++++++ crates/asap-aware-mapping/src/lib.rs | 1 + .../analytical-resource-cost.md | 76 +- 4 files changed, 836 insertions(+), 8 deletions(-) create mode 100644 crates/asap-aware-mapping/src/analytical_lowering.rs diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 0ad7a5ba..1e9f0d4b 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -768,6 +768,10 @@ pub enum AnalyticalCostError { Overflow, #[error("candidate has no supported exact or sketch state")] UnsupportedCandidate, + #[error("query operator has no physical implementation in the analytical model")] + UnsupportedQueryOperator, + #[error("inconsistent operator statistics: {0}")] + InconsistentOperatorStatistics(&'static str), #[error("summary operation {0} has no lifecycle-aware cost formula")] UnsupportedSummaryOperation(&'static str), #[error("required comparison-scope field {0} is missing")] diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs new file mode 100644 index 00000000..01f0a68f --- /dev/null +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -0,0 +1,763 @@ +//! Recursive lowering from the canonical query IR to analytical physical DAGs. + +use std::rc::Rc; + +use serde::{Deserialize, Serialize}; + +use crate::analytical_cost::{ + AnalyticalCostError, ExecutionMultiplicity, OperatorInputs, PhysicalDagNode, PhysicalOperator, +}; + +/// A lowered physical DAG and the node whose output is the query result. +/// Keeping the root beside its nodes prevents callers from accidentally +/// estimating a valid node list from the wrong entry point. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PhysicalDag { + pub nodes: Vec, + pub root: String, +} + +/// Lower a resolved query operator DAG to the physical operators understood by +/// this cost model. The callback supplies physical statistics for each logical +/// operator identity; returning `None` makes the complete query unavailable. +/// Scalar expressions remain part of their containing operator's local cost. +pub fn lower_query_physical_dag( + root: &Rc, + mut statistics: F, +) -> Result +where + F: FnMut(&asap_types::pre_asap::QueryExpr) -> Option, +{ + use std::collections::HashMap; + + use asap_types::pre_asap::{GroupKeys, QueryExpr, SetOpKind}; + + struct Lowerer<'a, F> { + statistics: &'a mut F, + logical_roots: HashMap, + next_id: usize, + nodes: Vec, + } + + impl Lowerer<'_, F> + where + F: FnMut(&QueryExpr) -> Option, + { + fn lower(&mut self, query: &QueryExpr) -> Result { + let identity = std::ptr::from_ref(query) as usize; + if let Some(id) = self.logical_roots.get(&identity) { + return Ok(id.clone()); + } + let id = format!("query-{}", self.next_id); + self.next_id += 1; + // Insert only after successful lowering: a malformed recursive + // shape cannot leave a partially reusable node behind. + let root = self.lower_new(query, id)?; + self.logical_roots.insert(identity, root.clone()); + Ok(root) + } + + fn stats(&mut self, query: &QueryExpr) -> Result { + (self.statistics)(query) + .ok_or(AnalyticalCostError::MissingOrStale("operator_statistics")) + } + + fn push( + &mut self, + id: String, + operator: PhysicalOperator, + inputs: OperatorInputs, + children: Vec, + ) -> String { + let output_buffer_bytes = inputs.output_bytes; + self.nodes.push(PhysicalDagNode { + id: id.clone(), + operator, + inputs, + children, + output_buffer_bytes, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }); + id + } + + fn lower_unary( + &mut self, + query: &QueryExpr, + id: String, + operator: PhysicalOperator, + child: &QueryExpr, + ) -> Result { + let child_id = self.lower(child)?; + let inputs = self.stats(query)?; + let child_inputs = &self.node(&child_id)?.inputs; + require_unary_edge(inputs, *child_inputs)?; + require_operator_statistics(operator, inputs)?; + Ok(self.push(id, operator, inputs, vec![child_id])) + } + + fn node(&self, id: &str) -> Result<&PhysicalDagNode, AnalyticalCostError> { + self.nodes.iter().find(|node| node.id == id).ok_or( + AnalyticalCostError::InvalidPhysicalDag("lowered child is missing"), + ) + } + + fn lower_new( + &mut self, + query: &QueryExpr, + id: String, + ) -> Result { + match query { + QueryExpr::Scan { predicates, .. } => { + let inputs = self.stats(query)?; + require_positive_operator_statistics(inputs)?; + if predicates.is_empty() { + if inputs.input_rows != inputs.output_rows + || inputs.input_bytes != inputs.output_bytes + { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "unfiltered scan output does not match its input", + )); + } + return Ok(self.push(id, PhysicalOperator::Scan, inputs, vec![])); + } + if inputs.output_rows > inputs.input_rows + || inputs.output_bytes > inputs.input_bytes + { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "predicate-bearing scan expands its input", + )); + } + let scan_id = format!("{id}-scan"); + let mut scan_inputs = inputs; + scan_inputs.output_rows = inputs.input_rows; + scan_inputs.output_bytes = inputs.input_bytes; + clear_operator_specific_inputs(&mut scan_inputs); + self.push(scan_id.clone(), PhysicalOperator::Scan, scan_inputs, vec![]); + require_operator_statistics(PhysicalOperator::Filter, inputs)?; + Ok(self.push(id, PhysicalOperator::Filter, inputs, vec![scan_id])) + } + QueryExpr::Filter { child, .. } => { + self.lower_unary(query, id, PhysicalOperator::Filter, child) + } + QueryExpr::Project { child, .. } => { + self.lower_unary(query, id, PhysicalOperator::Project, child) + } + QueryExpr::Aggregate { child, .. } => { + self.lower_unary(query, id, PhysicalOperator::HashAggregate, child) + } + QueryExpr::Dedup { child, .. } => { + self.lower_unary(query, id, PhysicalOperator::Deduplicate, child) + } + QueryExpr::Sort { child, .. } => { + self.lower_unary(query, id, PhysicalOperator::Sort, child) + } + QueryExpr::Limit { n, offset, child } => { + if let QueryExpr::Sort { + partition_by, + child: sorted_child, + .. + } = child.as_ref() + { + if partition_by == &GroupKeys::none() { + let child_id = self.lower(sorted_child)?; + let mut inputs = self.stats(query)?; + require_unary_edge(inputs, self.node(&child_id)?.inputs)?; + let bound = n + .checked_add(*offset) + .and_then(|value| u64::try_from(value).ok()) + .ok_or(AnalyticalCostError::Overflow)?; + if bound == 0 { + return Err(AnalyticalCostError::MissingOrZero("topk_k")); + } + inputs.k = Some(bound); + require_limit_cardinality(*n, *offset, inputs)?; + return Ok(self.push( + id, + PhysicalOperator::TopK, + inputs, + vec![child_id], + )); + } + } + let child_id = self.lower(child)?; + let inputs = self.stats(query)?; + require_unary_edge(inputs, self.node(&child_id)?.inputs)?; + require_operator_statistics(PhysicalOperator::Limit, inputs)?; + require_limit_cardinality(*n, *offset, inputs)?; + Ok(self.push(id, PhysicalOperator::Limit, inputs, vec![child_id])) + } + QueryExpr::SQLWindowFunc { child, .. } => { + self.lower_unary(query, id, PhysicalOperator::Window, child) + } + QueryExpr::TimeShift { child, .. } => { + self.lower_unary(query, id, PhysicalOperator::PassThrough, child) + } + QueryExpr::Concat { children } => { + let child_ids = children + .iter() + .map(|child| self.lower(child)) + .collect::, _>>()?; + self.lower_concat(query, id, child_ids) + } + QueryExpr::SetOp { + kind: SetOpKind::Union, + all: true, + left, + right, + } => { + let left_id = self.lower(left)?; + let right_id = self.lower(right)?; + self.lower_concat(query, id, vec![left_id, right_id]) + } + QueryExpr::Join { + kind, + pred, + left, + right, + } => { + if matches!(kind, asap_types::pre_asap::JoinKind::Cross) + || !is_hash_join_predicate(&pred.0) + { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + } + let left_id = self.lower(left)?; + let right_id = self.lower(right)?; + let inputs = self.stats(query)?; + require_positive_operator_statistics(inputs)?; + let left_inputs = self.node(&left_id)?.inputs; + let right_inputs = self.node(&right_id)?.inputs; + if inputs.input_rows != left_inputs.output_rows + || inputs.input_bytes != left_inputs.output_bytes + || inputs.right_rows != Some(right_inputs.output_rows) + || inputs.right_bytes != Some(right_inputs.output_bytes) + { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "join inputs do not match child outputs", + )); + } + Ok(self.push( + id, + PhysicalOperator::HashJoin, + inputs, + vec![left_id, right_id], + )) + } + _ => Err(AnalyticalCostError::UnsupportedQueryOperator), + } + } + + fn lower_concat( + &mut self, + query: &QueryExpr, + id: String, + child_ids: Vec, + ) -> Result { + if child_ids.is_empty() { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "concat has no children", + )); + } + let inputs = self.stats(query)?; + require_positive_operator_statistics(inputs)?; + let (rows, bytes) = + child_ids + .iter() + .try_fold((0_u64, 0_u64), |(rows, bytes), child| { + let child = self.node(child)?; + Ok::<_, AnalyticalCostError>(( + rows.checked_add(child.inputs.output_rows) + .ok_or(AnalyticalCostError::Overflow)?, + bytes + .checked_add(child.inputs.output_bytes) + .ok_or(AnalyticalCostError::Overflow)?, + )) + })?; + if inputs.input_rows != rows + || inputs.input_bytes != bytes + || inputs.output_rows != rows + || inputs.output_bytes != bytes + { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "concat statistics do not equal the sum of child outputs", + )); + } + Ok(self.push(id, PhysicalOperator::Concat, inputs, child_ids)) + } + } + + let mut lowerer = Lowerer { + statistics: &mut statistics, + logical_roots: HashMap::new(), + next_id: 0, + nodes: Vec::new(), + }; + let root = lowerer.lower(root)?; + Ok(PhysicalDag { + nodes: lowerer.nodes, + root, + }) +} + +fn require_positive_operator_statistics(inputs: OperatorInputs) -> Result<(), AnalyticalCostError> { + if inputs.input_rows == 0 { + return Err(AnalyticalCostError::MissingOrZero("operator input_rows")); + } + if inputs.input_bytes == 0 { + return Err(AnalyticalCostError::MissingOrZero("operator input_bytes")); + } + if inputs.output_rows == 0 { + return Err(AnalyticalCostError::MissingOrZero("operator output_rows")); + } + if inputs.output_bytes == 0 { + return Err(AnalyticalCostError::MissingOrZero("operator output_bytes")); + } + Ok(()) +} + +fn require_unary_edge( + inputs: OperatorInputs, + child: OperatorInputs, +) -> Result<(), AnalyticalCostError> { + require_positive_operator_statistics(inputs)?; + if inputs.input_rows != child.output_rows || inputs.input_bytes != child.output_bytes { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "unary input does not match child output", + )); + } + Ok(()) +} + +fn require_operator_statistics( + operator: PhysicalOperator, + inputs: OperatorInputs, +) -> Result<(), AnalyticalCostError> { + let invalid = |reason| Err(AnalyticalCostError::InconsistentOperatorStatistics(reason)); + match operator { + PhysicalOperator::Filter => { + if inputs.output_rows > inputs.input_rows || inputs.output_bytes > inputs.input_bytes { + return invalid("filter output expands its input"); + } + } + PhysicalOperator::Project => { + if inputs.output_rows != inputs.input_rows { + return invalid("projection changes row cardinality"); + } + } + PhysicalOperator::HashAggregate => { + let groups = inputs + .group_count + .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; + if groups == 0 { + return Err(AnalyticalCostError::MissingOrZero("group_count")); + } + if inputs.output_rows > groups { + return invalid("aggregate output exceeds group cardinality"); + } + } + PhysicalOperator::Deduplicate => { + let groups = inputs + .group_count + .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; + if groups == 0 { + return Err(AnalyticalCostError::MissingOrZero("group_count")); + } + if inputs.output_rows != groups || inputs.output_rows > inputs.input_rows { + return invalid("deduplicate output does not equal distinct cardinality"); + } + } + PhysicalOperator::Sort => { + if inputs.output_rows != inputs.input_rows || inputs.output_bytes != inputs.input_bytes + { + return invalid("sort changes its input cardinality or width"); + } + } + PhysicalOperator::TopK | PhysicalOperator::Limit => { + if inputs.output_rows > inputs.input_rows { + return invalid("bounded output exceeds its input cardinality"); + } + } + PhysicalOperator::Window => { + if inputs.output_rows != inputs.input_rows { + return invalid("SQL window changes row cardinality"); + } + } + PhysicalOperator::PassThrough => { + if inputs.output_rows != inputs.input_rows || inputs.output_bytes != inputs.input_bytes + { + return invalid("pass-through wrapper changes its edge statistics"); + } + } + PhysicalOperator::Scan | PhysicalOperator::HashJoin | PhysicalOperator::Concat => {} + } + Ok(()) +} + +fn require_limit_cardinality( + n: usize, + offset: usize, + inputs: OperatorInputs, +) -> Result<(), AnalyticalCostError> { + let n = u64::try_from(n).map_err(|_| AnalyticalCostError::Overflow)?; + let offset = u64::try_from(offset).map_err(|_| AnalyticalCostError::Overflow)?; + let expected = inputs.input_rows.saturating_sub(offset).min(n); + if inputs.output_rows != expected { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "limit output does not match n and offset", + )); + } + Ok(()) +} + +fn clear_operator_specific_inputs(inputs: &mut OperatorInputs) { + inputs.group_count = None; + inputs.key_bytes = None; + inputs.aggregate_value_bytes = None; + inputs.k = None; + inputs.right_rows = None; + inputs.right_bytes = None; + inputs.hash_join_build_side = None; +} + +fn is_hash_join_predicate(expr: &asap_types::pre_asap::QueryExpr) -> bool { + use asap_types::pre_asap::{CompareOpKind, QueryExpr}; + + match expr { + QueryExpr::Compare { + left, + op: CompareOpKind::Eq, + right, + } => { + matches!(left.as_ref(), QueryExpr::Column(_)) + && matches!(right.as_ref(), QueryExpr::Column(_)) + } + QueryExpr::BoolAnd(parts) => !parts.is_empty() && parts.iter().all(is_hash_join_predicate), + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::analytical_cost::{estimate_physical_dag, HashJoinBuildSide}; + + fn operator_inputs( + input_rows: u64, + input_bytes: u64, + output_rows: u64, + output_bytes: u64, + ) -> OperatorInputs { + OperatorInputs { + input_rows, + input_bytes, + output_rows, + output_bytes, + group_count: None, + key_bytes: None, + aggregate_value_bytes: None, + k: None, + right_rows: None, + right_bytes: None, + hash_join_build_side: None, + } + } + + #[test] + fn query_lowering_recurses_and_fuses_global_sort_limit() { + use asap_types::pre_asap::{ + agg_intent::default_cardinality, GroupKeys, QueryExpr, Reduction, Source, + }; + use asap_types::pre_asap::{Column, DataType, Schema}; + use std::rc::Rc; + + let scan = Rc::new(QueryExpr::Scan { + source: Source::Table { + table_ref: "events".into(), + }, + predicates: vec![asap_types::pre_asap::Predicate(Rc::new( + QueryExpr::Literal(asap_types::pre_asap::ScalarValue::Boolean(true)), + ))], + schema: Schema::new(vec![ + Column::new("service", DataType::Utf8, false), + Column::new("value", DataType::Float64, false), + ]), + }); + let aggregate = Rc::new(QueryExpr::Aggregate { + reduction: Reduction::by(vec![0]), + measures: vec![default_cardinality()], + output_names: vec![], + having: None, + child: Rc::clone(&scan), + }); + let sort = Rc::new(QueryExpr::Sort { + keys: vec![], + partition_by: GroupKeys::none(), + child: aggregate, + }); + let root = Rc::new(QueryExpr::Limit { + n: 10, + offset: 5, + child: sort, + }); + + let dag = lower_query_physical_dag(&root, |node| { + let mut stats = match node { + QueryExpr::Scan { .. } => operator_inputs(1_000, 64_000, 400, 25_600), + QueryExpr::Aggregate { .. } => operator_inputs(400, 25_600, 100, 4_000), + QueryExpr::Limit { .. } => operator_inputs(100, 4_000, 10, 400), + _ => return None, + }; + if matches!(node, QueryExpr::Aggregate { .. }) { + stats.group_count = Some(100); + stats.key_bytes = Some(16); + stats.aggregate_value_bytes = Some(8); + } + Some(stats) + }) + .unwrap(); + + assert_eq!( + dag.nodes + .iter() + .map(|node| node.operator) + .collect::>(), + vec![ + PhysicalOperator::Scan, + PhysicalOperator::Filter, + PhysicalOperator::HashAggregate, + PhysicalOperator::TopK, + ] + ); + let topk = dag.nodes.last().unwrap(); + assert_eq!(topk.inputs.k, Some(15)); + assert_eq!(topk.children, vec![dag.nodes[2].id.clone()]); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, 3).is_ok()); + } + + #[test] + fn query_lowering_deduplicates_shared_rc_children() { + use asap_types::pre_asap::{Column, CompareOpKind, DataType, Schema}; + use asap_types::pre_asap::{JoinKind, Predicate, QueryExpr, Source}; + use std::rc::Rc; + + let shared = Rc::new(QueryExpr::Scan { + source: Source::Table { + table_ref: "dimensions".into(), + }, + predicates: vec![], + schema: Schema::new(vec![Column::new("id", DataType::Int64, false)]), + }); + let root = Rc::new(QueryExpr::Join { + kind: JoinKind::Inner, + pred: Predicate(Rc::new(QueryExpr::Compare { + left: Rc::new(QueryExpr::Column(0)), + op: CompareOpKind::Eq, + right: Rc::new(QueryExpr::Column(1)), + })), + left: Rc::clone(&shared), + right: Rc::clone(&shared), + }); + let dag = lower_query_physical_dag(&root, |node| match node { + QueryExpr::Scan { .. } => Some(operator_inputs(100, 800, 100, 800)), + QueryExpr::Join { .. } => { + let mut stats = operator_inputs(100, 800, 25, 400); + stats.right_rows = Some(100); + stats.right_bytes = Some(800); + stats.hash_join_build_side = Some(HashJoinBuildSide::Right); + Some(stats) + } + _ => None, + }) + .unwrap(); + + assert_eq!(dag.nodes.len(), 2); + assert_eq!(dag.nodes[1].children, vec![dag.nodes[0].id.clone(); 2]); + let estimate = estimate_physical_dag(&dag.nodes, &dag.root, 1).unwrap(); + assert_eq!(estimate.scan_bytes, 800); + } + + #[test] + fn query_lowering_covers_relational_unary_operators() { + use asap_types::pre_asap::{Column, DataType, ScalarValue, Schema}; + use asap_types::pre_asap::{ + GroupKeys, Predicate, QueryExpr, Source, TimeShift, WindowFuncKind, + }; + use std::rc::Rc; + + let scan = Rc::new(QueryExpr::Scan { + source: Source::Table { + table_ref: "events".into(), + }, + predicates: vec![], + schema: Schema::new(vec![Column::new("id", DataType::Int64, false)]), + }); + let filter = Rc::new(QueryExpr::Filter { + pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Boolean(true)))), + child: scan, + }); + let project = Rc::new(QueryExpr::Project { + cols: vec![], + qualifier: None, + child: filter, + }); + let dedup = Rc::new(QueryExpr::Dedup { + cols: vec![0], + child: project, + }); + let window = Rc::new(QueryExpr::SQLWindowFunc { + func: WindowFuncKind::RowNumber, + args: vec![], + partition_by: GroupKeys::none(), + order_by: vec![], + frame: None, + output_name: "rn".into(), + child: dedup, + }); + let sort = Rc::new(QueryExpr::Sort { + keys: vec![], + partition_by: GroupKeys::by(vec![0]), + child: window, + }); + let limit = Rc::new(QueryExpr::Limit { + n: 20, + offset: 0, + child: sort, + }); + let root = Rc::new(QueryExpr::TimeShift { + shift: TimeShift::default(), + child: limit, + }); + + let dag = lower_query_physical_dag(&root, |node| { + let mut inputs = match node { + QueryExpr::Scan { .. } => operator_inputs(1_000, 8_000, 1_000, 8_000), + QueryExpr::Filter { .. } => operator_inputs(1_000, 8_000, 800, 6_400), + QueryExpr::Project { .. } => operator_inputs(800, 6_400, 800, 3_200), + QueryExpr::Dedup { .. } => operator_inputs(800, 3_200, 500, 2_000), + QueryExpr::SQLWindowFunc { .. } => operator_inputs(500, 2_000, 500, 6_000), + QueryExpr::Sort { .. } => operator_inputs(500, 6_000, 500, 6_000), + QueryExpr::Limit { .. } => operator_inputs(500, 6_000, 20, 240), + QueryExpr::TimeShift { .. } => operator_inputs(20, 240, 20, 240), + _ => return None, + }; + if matches!(node, QueryExpr::Dedup { .. }) { + inputs.group_count = Some(500); + inputs.key_bytes = Some(8); + } + Some(inputs) + }) + .unwrap(); + + assert_eq!( + dag.nodes + .iter() + .map(|node| node.operator) + .collect::>(), + vec![ + PhysicalOperator::Scan, + PhysicalOperator::Filter, + PhysicalOperator::Project, + PhysicalOperator::Deduplicate, + PhysicalOperator::Window, + PhysicalOperator::Sort, + PhysicalOperator::Limit, + PhysicalOperator::PassThrough, + ] + ); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, 2).is_ok()); + } + + #[test] + fn query_lowering_maps_concat_and_union_all_but_rejects_distinct_set_ops() { + use asap_types::pre_asap::{Column, DataType, Schema}; + use asap_types::pre_asap::{QueryExpr, SetOpKind, Source}; + use std::rc::Rc; + + let scan = |name: &str| QueryExpr::Scan { + source: Source::Table { + table_ref: name.into(), + }, + predicates: vec![], + schema: Schema::new(vec![Column::new("id", DataType::Int64, false)]), + }; + let union = Rc::new(QueryExpr::SetOp { + kind: SetOpKind::Union, + all: true, + left: Rc::new(scan("a")), + right: Rc::new(scan("b")), + }); + let dag = lower_query_physical_dag(&union, |node| match node { + QueryExpr::Scan { + source: Source::Table { table_ref }, + .. + } if table_ref == "a" => Some(operator_inputs(10, 80, 10, 80)), + QueryExpr::Scan { .. } => Some(operator_inputs(20, 160, 20, 160)), + QueryExpr::SetOp { .. } => Some(operator_inputs(30, 240, 30, 240)), + _ => None, + }) + .unwrap(); + assert_eq!(dag.nodes.last().unwrap().operator, PhysicalOperator::Concat); + + let concat = Rc::new(QueryExpr::Concat { + children: vec![scan("a"), scan("b")], + }); + let dag = lower_query_physical_dag(&concat, |node| match node { + QueryExpr::Scan { + source: Source::Table { table_ref }, + .. + } if table_ref == "a" => Some(operator_inputs(10, 80, 10, 80)), + QueryExpr::Scan { .. } => Some(operator_inputs(20, 160, 20, 160)), + QueryExpr::Concat { .. } => Some(operator_inputs(30, 240, 30, 240)), + _ => None, + }) + .unwrap(); + assert_eq!(dag.nodes.last().unwrap().operator, PhysicalOperator::Concat); + + let distinct_union = Rc::new(QueryExpr::SetOp { + kind: SetOpKind::Union, + all: false, + left: Rc::new(scan("a")), + right: Rc::new(scan("b")), + }); + assert_eq!( + lower_query_physical_dag(&distinct_union, |_| None), + Err(AnalyticalCostError::UnsupportedQueryOperator) + ); + } + + #[test] + fn query_lowering_fails_closed_for_missing_or_inconsistent_statistics() { + use asap_types::pre_asap::{Column, DataType, Schema}; + use asap_types::pre_asap::{QueryExpr, Source}; + use std::rc::Rc; + + let scan = Rc::new(QueryExpr::Scan { + source: Source::Table { + table_ref: "events".into(), + }, + predicates: vec![], + schema: Schema::new(vec![Column::new("id", DataType::Int64, false)]), + }); + let root = Rc::new(QueryExpr::Project { + cols: vec![], + qualifier: None, + child: scan, + }); + + assert_eq!( + lower_query_physical_dag(&root, |_| None), + Err(AnalyticalCostError::MissingOrStale("operator_statistics")) + ); + assert_eq!( + lower_query_physical_dag(&root, |node| match node { + QueryExpr::Scan { .. } => Some(operator_inputs(100, 800, 100, 800)), + QueryExpr::Project { .. } => Some(operator_inputs(99, 792, 99, 396)), + _ => None, + }), + Err(AnalyticalCostError::InconsistentOperatorStatistics( + "unary input does not match child output" + )) + ); + } +} diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 479fc062..9f1a16cb 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -184,6 +184,7 @@ pub mod accuracy; pub mod accuracy_reconciliation; pub mod analytical_cost; +pub mod analytical_lowering; pub mod analytical_statistics; pub mod cost_model; pub mod explanation; diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 3408c258..8902cc91 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -13,10 +13,15 @@ and guarantee composition run first; costing ranks only the candidates that survive. Missing evidence produces an unavailable estimate, never an assumed zero or a structural-cost fallback. -The physical-DAG estimator composes any DAG whose nodes have supported -physical operators and complete `OperatorStatistics`. Query lowering and -planner selection are separate integration layers; neither may substitute a -shape-specific shortcut or structural node count. +This document distinguishes three implementation layers: + +- the physical-DAG estimator, which can compose any DAG whose nodes have + supported physical operators and complete `OperatorStatistics`; and +- query-DAG lowering, which recursively maps supported resolved `QueryExpr` + operators to that physical representation; and +- replacement lowering and ranking, which must compare complete alternatives. + +No layer may substitute a shape-specific shortcut or structural node count. An estimate has physical dimensions: @@ -324,6 +329,58 @@ inherit the target's old node costs. A replacement candidate includes any newly embedded child summaries, while an independently shared child is deduplicated by physical identity. +### Query-DAG lowering and statistics contract + +`lower_query_physical_dag` recursively lowers a resolved `Rc` and +returns a `PhysicalDag` containing both its nodes and root ID. It consumes the +existing query and physical-operator enums; it does not introduce a parallel +logical operator vocabulary. A statistics callback resolves the existing +`OperatorInputs` for each logical operator identity. Returning no statistics +makes the entire query unavailable. + +The lowering validates every physical edge before costing: + +- a unary operator's `input_rows` and `input_bytes` equal its child's output; +- a hash join's left and right inputs equal the corresponding child outputs; +- Concat and `UNION ALL` input/output totals equal the checked sum of all + child outputs; and +- row-preserving, reducing, and bounded operators obey their cardinality + invariants. + +The supported mappings are: + +| Existing `QueryExpr` shape | Physical DAG | +|---|---| +| Scan without predicates | Scan | +| Scan with pushed predicates | Scan → Filter | +| Filter | Filter | +| Project | Project | +| Aggregate, including a fused HAVING predicate | HashAggregate | +| Dedup | Deduplicate | +| Equi-Join | HashJoin with an evidence-selected build side | +| Concat or `UNION ALL` | Concat | +| Sort | in-memory Sort | +| global Sort followed by Limit | heap TopK, with `k = offset + n` from the query IR | +| partitioned Sort followed by Limit | Sort → Limit | +| SQLWindowFunc | Window | +| TimeShift | PassThrough | + +Logical identity is the address of the existing `Rc` allocation. +Repeated references therefore lower once and every parent points to the same +physical ID. The resulting node IDs are deterministic within a lowering run; +they are not persistent query identifiers. + +This generic lowering currently creates raw-query operators, all with +`ExecutionMultiplicity::PerEvaluation`, zero retained state, and a conservative +full-output edge buffer. A deployment with verified batching may construct +`PhysicalDagNode` values with smaller `output_buffer_bytes` directly. + +Cross/non-equi joins, `INTERSECT`, `EXCEPT`, distinct `UNION`, PromQL +range/subquery execution, vector matching, and PromQL-specific enrichment/ +relabel/sample operators stay unavailable. Their cost requires a physical +implementation or multiplicity/state facts that the current physical-operator +vocabulary cannot represent; they are not treated as free pass-through work. + ## Physical operator formulas Operator estimates are local: child CPU and I/O are excluded and composed by @@ -399,7 +456,9 @@ children would undercount the plan. `estimate_physical_dag` is independent of query shape. A caller supplies the complete physical DAG and per-node evidence. Filters, projections, joins, windows, nested aggregates, Top-K, and shared sub-DAGs therefore use the same -estimation path. +estimation path. The generic query lowerer recursively maps the supported raw +query operators into that representation. Replacement lowering remains a +separate layer and must include all summary-maintenance work. DDSketch is unavailable because occupied bins depend on value range and distribution. The model does not invent a bin count. Algorithm/parameter @@ -450,9 +509,10 @@ The intended end-to-end selection pipeline is: 5. estimates the complete candidate DAG; 6. applies calibration and ranks candidates by ascending cost. -This module implements the physical estimation step. Lowering, evidence -resolution, legality checks, and planner integration remain separate layers; -each preserves the complete-plan and fail-closed requirements above. +The query lowerer and physical estimator cover the supported raw-query shapes +listed above. Replacement evidence resolution, legality checks, and planner +integration remain separate layers; each preserves the complete-plan and +fail-closed requirements above. Before applying the following arithmetic, callers validate exact equality of the raw and selected alternative's `ComparisonScope`, and use the same From 151b6a1455f28d6fc9f50ee7cf41887df87b6eb0 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 08:23:46 -0600 Subject: [PATCH 28/48] fix(cost): allow empty physical edges --- .../src/analytical_lowering.rs | 79 +++++++++++++------ .../analytical-resource-cost.md | 2 + 2 files changed, 59 insertions(+), 22 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index 01f0a68f..f7db8ddb 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -111,7 +111,7 @@ where match query { QueryExpr::Scan { predicates, .. } => { let inputs = self.stats(query)?; - require_positive_operator_statistics(inputs)?; + require_consistent_edge_statistics(inputs)?; if predicates.is_empty() { if inputs.input_rows != inputs.output_rows || inputs.input_bytes != inputs.output_bytes @@ -225,7 +225,7 @@ where let left_id = self.lower(left)?; let right_id = self.lower(right)?; let inputs = self.stats(query)?; - require_positive_operator_statistics(inputs)?; + require_consistent_edge_statistics(inputs)?; let left_inputs = self.node(&left_id)?.inputs; let right_inputs = self.node(&right_id)?.inputs; if inputs.input_rows != left_inputs.output_rows @@ -260,7 +260,7 @@ where )); } let inputs = self.stats(query)?; - require_positive_operator_statistics(inputs)?; + require_consistent_edge_statistics(inputs)?; let (rows, bytes) = child_ids .iter() @@ -300,27 +300,30 @@ where }) } -fn require_positive_operator_statistics(inputs: OperatorInputs) -> Result<(), AnalyticalCostError> { - if inputs.input_rows == 0 { - return Err(AnalyticalCostError::MissingOrZero("operator input_rows")); - } - if inputs.input_bytes == 0 { - return Err(AnalyticalCostError::MissingOrZero("operator input_bytes")); - } - if inputs.output_rows == 0 { - return Err(AnalyticalCostError::MissingOrZero("operator output_rows")); - } - if inputs.output_bytes == 0 { - return Err(AnalyticalCostError::MissingOrZero("operator output_bytes")); - } +fn require_consistent_edge_statistics(inputs: OperatorInputs) -> Result<(), AnalyticalCostError> { + require_cardinality_width("operator input", inputs.input_rows, inputs.input_bytes)?; + require_cardinality_width("operator output", inputs.output_rows, inputs.output_bytes)?; Ok(()) } +fn require_cardinality_width( + edge: &'static str, + rows: u64, + bytes: u64, +) -> Result<(), AnalyticalCostError> { + match (rows, bytes) { + (0, 0) => Ok(()), + (0, _) => Err(AnalyticalCostError::InconsistentOperatorStatistics(edge)), + (_, 0) => Err(AnalyticalCostError::MissingOrZero(edge)), + _ => Ok(()), + } +} + fn require_unary_edge( inputs: OperatorInputs, child: OperatorInputs, ) -> Result<(), AnalyticalCostError> { - require_positive_operator_statistics(inputs)?; + require_consistent_edge_statistics(inputs)?; if inputs.input_rows != child.output_rows || inputs.input_bytes != child.output_bytes { return Err(AnalyticalCostError::InconsistentOperatorStatistics( "unary input does not match child output", @@ -349,8 +352,8 @@ fn require_operator_statistics( let groups = inputs .group_count .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - if groups == 0 { - return Err(AnalyticalCostError::MissingOrZero("group_count")); + if groups == 0 && (inputs.input_rows != 0 || inputs.output_rows != 0) { + return invalid("zero groups require an empty grouped input and output"); } if inputs.output_rows > groups { return invalid("aggregate output exceeds group cardinality"); @@ -360,9 +363,6 @@ fn require_operator_statistics( let groups = inputs .group_count .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - if groups == 0 { - return Err(AnalyticalCostError::MissingOrZero("group_count")); - } if inputs.output_rows != groups || inputs.output_rows > inputs.input_rows { return invalid("deduplicate output does not equal distinct cardinality"); } @@ -760,4 +760,39 @@ mod tests { )) ); } + + #[test] + fn query_lowering_accepts_a_consistently_empty_edge() { + use asap_types::pre_asap::{Column, DataType, ScalarValue, Schema}; + use asap_types::pre_asap::{Predicate, QueryExpr, Source}; + use std::rc::Rc; + + let scan = Rc::new(QueryExpr::Scan { + source: Source::Table { + table_ref: "events".into(), + }, + predicates: vec![], + schema: Schema::new(vec![Column::new("id", DataType::Int64, false)]), + }); + let filter = Rc::new(QueryExpr::Filter { + pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Boolean(false)))), + child: scan, + }); + let root = Rc::new(QueryExpr::Limit { + n: 10, + offset: 0, + child: filter, + }); + + let dag = lower_query_physical_dag(&root, |node| match node { + QueryExpr::Scan { .. } => Some(operator_inputs(100, 800, 100, 800)), + QueryExpr::Filter { .. } => Some(operator_inputs(100, 800, 0, 0)), + QueryExpr::Limit { .. } => Some(operator_inputs(0, 0, 0, 0)), + _ => None, + }) + .unwrap(); + let estimate = estimate_physical_dag(&dag.nodes, &dag.root, 1).unwrap(); + assert_eq!(estimate.cpu_ops, 200.0); + assert_eq!(estimate.scan_bytes, 800); + } } diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 8902cc91..7de1cf13 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -340,6 +340,8 @@ makes the entire query unavailable. The lowering validates every physical edge before costing: +- `(rows = 0, bytes = 0)` is a valid empty edge, while positive rows still + require byte-width evidence and zero rows cannot carry non-zero bytes; - a unary operator's `input_rows` and `input_bytes` equal its child's output; - a hash join's left and right inputs equal the corresponding child outputs; - Concat and `UNION ALL` input/output totals equal the checked sum of all From 50292151d1176488dcc9b6ef2be6f3247a2859a3 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 09:17:17 -0600 Subject: [PATCH 29/48] fix(cost): bind query lowering to physical evidence --- .../src/analytical_lowering.rs | 746 ++++++++++++------ .../analytical-resource-cost.md | 17 +- 2 files changed, 513 insertions(+), 250 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index f7db8ddb..8a37600b 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -5,7 +5,10 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; use crate::analytical_cost::{ - AnalyticalCostError, ExecutionMultiplicity, OperatorInputs, PhysicalDagNode, PhysicalOperator, + AnalyticalCostError, ExecutionMultiplicity, PhysicalDagNode, PhysicalOperator, +}; +use crate::analytical_statistics::{ + ComparisonScope, EdgeStatistics, OperatorStatistics, OperatorStatisticsProvider, SourceCoverage, }; /// A lowered physical DAG and the node whose output is the query result. @@ -17,32 +20,60 @@ pub struct PhysicalDag { pub root: String, } +/// Atomic evidence for one lowered physical node. The statistics contract is +/// reused unchanged; the separate buffer field is necessary because logical +/// edge bytes cannot stand in for an allocation. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PhysicalNodeEvidence { + pub statistics: OperatorStatistics, + pub output_buffer_bytes: u64, +} + +pub trait PhysicalNodeEvidenceProvider { + fn evidence(&self, node_id: &str) -> Result; +} + +impl PhysicalNodeEvidenceProvider for std::collections::HashMap { + fn evidence(&self, node_id: &str) -> Result { + self.get(node_id) + .cloned() + .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(node_id.into())) + } +} + +impl OperatorStatisticsProvider for std::collections::HashMap { + fn statistics(&self, node_id: &str) -> Result { + self.evidence(node_id).map(|evidence| evidence.statistics) + } +} + /// Lower a resolved query operator DAG to the physical operators understood by -/// this cost model. The callback supplies physical statistics for each logical -/// operator identity; returning `None` makes the complete query unavailable. -/// Scalar expressions remain part of their containing operator's local cost. -pub fn lower_query_physical_dag( +/// this cost model. The authoritative provider supplies statistics by the +/// deterministic physical IDs assigned here; missing evidence makes the +/// complete query unavailable. Scalar expressions remain part of their +/// containing operator's local cost. +pub fn lower_query_physical_dag( root: &Rc, - mut statistics: F, -) -> Result -where - F: FnMut(&asap_types::pre_asap::QueryExpr) -> Option, -{ + scope: &ComparisonScope, + evidence: &dyn PhysicalNodeEvidenceProvider, +) -> Result { use std::collections::HashMap; use asap_types::pre_asap::{GroupKeys, QueryExpr, SetOpKind}; - struct Lowerer<'a, F> { - statistics: &'a mut F, + scope.validate()?; + + struct Lowerer<'a> { + scope: &'a ComparisonScope, + provider: &'a dyn PhysicalNodeEvidenceProvider, + statistics: HashMap, + output_buffers: HashMap, logical_roots: HashMap, next_id: usize, nodes: Vec, } - impl Lowerer<'_, F> - where - F: FnMut(&QueryExpr) -> Option, - { + impl Lowerer<'_> { fn lower(&mut self, query: &QueryExpr) -> Result { let identity = std::ptr::from_ref(query) as usize; if let Some(id) = self.logical_roots.get(&identity) { @@ -57,25 +88,32 @@ where Ok(root) } - fn stats(&mut self, query: &QueryExpr) -> Result { - (self.statistics)(query) - .ok_or(AnalyticalCostError::MissingOrStale("operator_statistics")) + fn stats(&mut self, id: &str) -> Result { + if let Some(statistics) = self.statistics.get(id) { + return Ok(statistics.clone()); + } + let evidence = self.provider.evidence(id)?; + let statistics = evidence.statistics; + self.statistics.insert(id.into(), statistics.clone()); + self.output_buffers + .insert(id.into(), evidence.output_buffer_bytes); + Ok(statistics) } fn push( &mut self, id: String, operator: PhysicalOperator, - inputs: OperatorInputs, + _statistics: &OperatorStatistics, children: Vec, + source_coverage: Option, ) -> String { - let output_buffer_bytes = inputs.output_bytes; self.nodes.push(PhysicalDagNode { id: id.clone(), operator, - inputs, children, - output_buffer_bytes, + source_coverage, + output_buffer_bytes: self.output_buffers[&id], retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, }); @@ -84,23 +122,25 @@ where fn lower_unary( &mut self, - query: &QueryExpr, + _query: &QueryExpr, id: String, operator: PhysicalOperator, child: &QueryExpr, ) -> Result { let child_id = self.lower(child)?; - let inputs = self.stats(query)?; - let child_inputs = &self.node(&child_id)?.inputs; - require_unary_edge(inputs, *child_inputs)?; - require_operator_statistics(operator, inputs)?; - Ok(self.push(id, operator, inputs, vec![child_id])) + let statistics = self.stats(&id)?; + let child_statistics = self.node_statistics(&child_id)?; + require_unary_edge(&id, &statistics, &child_id, child_statistics)?; + require_operator_statistics(operator, &statistics)?; + Ok(self.push(id, operator, &statistics, vec![child_id], None)) } - fn node(&self, id: &str) -> Result<&PhysicalDagNode, AnalyticalCostError> { - self.nodes.iter().find(|node| node.id == id).ok_or( - AnalyticalCostError::InvalidPhysicalDag("lowered child is missing"), - ) + fn node_statistics(&self, id: &str) -> Result<&OperatorStatistics, AnalyticalCostError> { + self.statistics + .get(id) + .ok_or(AnalyticalCostError::InvalidPhysicalDag( + "lowered child statistics are missing", + )) } fn lower_new( @@ -109,34 +149,41 @@ where id: String, ) -> Result { match query { - QueryExpr::Scan { predicates, .. } => { - let inputs = self.stats(query)?; - require_consistent_edge_statistics(inputs)?; + QueryExpr::Scan { + source, predicates, .. + } => { + let coverage = bind_scan_coverage(&id, source, predicates, self.scope)?; if predicates.is_empty() { - if inputs.input_rows != inputs.output_rows - || inputs.input_bytes != inputs.output_bytes - { - return Err(AnalyticalCostError::InconsistentOperatorStatistics( - "unfiltered scan output does not match its input", - )); - } - return Ok(self.push(id, PhysicalOperator::Scan, inputs, vec![])); - } - if inputs.output_rows > inputs.input_rows - || inputs.output_bytes > inputs.input_bytes - { - return Err(AnalyticalCostError::InconsistentOperatorStatistics( - "predicate-bearing scan expands its input", + let statistics = self.stats(&id)?; + require_statistics_shape(&id, &statistics, 1)?; + return Ok(self.push( + id, + PhysicalOperator::Scan, + &statistics, + vec![], + Some(coverage), )); } let scan_id = format!("{id}-scan"); - let mut scan_inputs = inputs; - scan_inputs.output_rows = inputs.input_rows; - scan_inputs.output_bytes = inputs.input_bytes; - clear_operator_specific_inputs(&mut scan_inputs); - self.push(scan_id.clone(), PhysicalOperator::Scan, scan_inputs, vec![]); - require_operator_statistics(PhysicalOperator::Filter, inputs)?; - Ok(self.push(id, PhysicalOperator::Filter, inputs, vec![scan_id])) + let scan_statistics = self.stats(&scan_id)?; + require_statistics_shape(&scan_id, &scan_statistics, 1)?; + self.push( + scan_id.clone(), + PhysicalOperator::Scan, + &scan_statistics, + vec![], + Some(coverage), + ); + let filter_statistics = self.stats(&id)?; + require_unary_edge(&id, &filter_statistics, &scan_id, &scan_statistics)?; + require_operator_statistics(PhysicalOperator::Filter, &filter_statistics)?; + Ok(self.push( + id, + PhysicalOperator::Filter, + &filter_statistics, + vec![scan_id], + None, + )) } QueryExpr::Filter { child, .. } => { self.lower_unary(query, id, PhysicalOperator::Filter, child) @@ -162,8 +209,9 @@ where { if partition_by == &GroupKeys::none() { let child_id = self.lower(sorted_child)?; - let mut inputs = self.stats(query)?; - require_unary_edge(inputs, self.node(&child_id)?.inputs)?; + let statistics = self.stats(&id)?; + let child_statistics = self.node_statistics(&child_id)?; + require_unary_edge(&id, &statistics, &child_id, child_statistics)?; let bound = n .checked_add(*offset) .and_then(|value| u64::try_from(value).ok()) @@ -171,22 +219,34 @@ where if bound == 0 { return Err(AnalyticalCostError::MissingOrZero("topk_k")); } - inputs.k = Some(bound); - require_limit_cardinality(*n, *offset, inputs)?; + if statistics.k != Some(bound) { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "Top-K statistics disagree with LIMIT n + offset", + )); + } + require_limit_cardinality(*n, *offset, &statistics)?; return Ok(self.push( id, PhysicalOperator::TopK, - inputs, + &statistics, vec![child_id], + None, )); } } let child_id = self.lower(child)?; - let inputs = self.stats(query)?; - require_unary_edge(inputs, self.node(&child_id)?.inputs)?; - require_operator_statistics(PhysicalOperator::Limit, inputs)?; - require_limit_cardinality(*n, *offset, inputs)?; - Ok(self.push(id, PhysicalOperator::Limit, inputs, vec![child_id])) + let statistics = self.stats(&id)?; + let child_statistics = self.node_statistics(&child_id)?; + require_unary_edge(&id, &statistics, &child_id, child_statistics)?; + require_operator_statistics(PhysicalOperator::Limit, &statistics)?; + require_limit_cardinality(*n, *offset, &statistics)?; + Ok(self.push( + id, + PhysicalOperator::Limit, + &statistics, + vec![child_id], + None, + )) } QueryExpr::SQLWindowFunc { child, .. } => { self.lower_unary(query, id, PhysicalOperator::Window, child) @@ -224,24 +284,24 @@ where } let left_id = self.lower(left)?; let right_id = self.lower(right)?; - let inputs = self.stats(query)?; - require_consistent_edge_statistics(inputs)?; - let left_inputs = self.node(&left_id)?.inputs; - let right_inputs = self.node(&right_id)?.inputs; - if inputs.input_rows != left_inputs.output_rows - || inputs.input_bytes != left_inputs.output_bytes - || inputs.right_rows != Some(right_inputs.output_rows) - || inputs.right_bytes != Some(right_inputs.output_bytes) + let statistics = self.stats(&id)?; + require_statistics_shape(&id, &statistics, 2)?; + let left_statistics = self.node_statistics(&left_id)?; + let right_statistics = self.node_statistics(&right_id)?; + if statistics.inputs[0] != left_statistics.output + || statistics.inputs[1] != right_statistics.output { return Err(AnalyticalCostError::InconsistentOperatorStatistics( "join inputs do not match child outputs", )); } + require_operator_statistics(PhysicalOperator::HashJoin, &statistics)?; Ok(self.push( id, PhysicalOperator::HashJoin, - inputs, + &statistics, vec![left_id, right_id], + None, )) } _ => Err(AnalyticalCostError::UnsupportedQueryOperator), @@ -250,7 +310,7 @@ where fn lower_concat( &mut self, - query: &QueryExpr, + _query: &QueryExpr, id: String, child_ids: Vec, ) -> Result { @@ -259,36 +319,43 @@ where "concat has no children", )); } - let inputs = self.stats(query)?; - require_consistent_edge_statistics(inputs)?; - let (rows, bytes) = - child_ids - .iter() - .try_fold((0_u64, 0_u64), |(rows, bytes), child| { - let child = self.node(child)?; - Ok::<_, AnalyticalCostError>(( - rows.checked_add(child.inputs.output_rows) - .ok_or(AnalyticalCostError::Overflow)?, - bytes - .checked_add(child.inputs.output_bytes) - .ok_or(AnalyticalCostError::Overflow)?, - )) - })?; - if inputs.input_rows != rows - || inputs.input_bytes != bytes - || inputs.output_rows != rows - || inputs.output_bytes != bytes - { + let statistics = self.stats(&id)?; + require_statistics_shape(&id, &statistics, child_ids.len())?; + let (rows, bytes) = child_ids.iter().enumerate().try_fold( + (0_u64, 0_u64), + |(rows, bytes), (index, child)| { + let child_statistics = self.node_statistics(child)?; + if statistics.inputs[index] != child_statistics.output { + return Err(AnalyticalCostError::ConflictingEdgeStatistics { + parent: id.clone(), + child: child.clone(), + input_index: index, + }); + } + Ok::<_, AnalyticalCostError>(( + rows.checked_add(child_statistics.output.rows) + .ok_or(AnalyticalCostError::Overflow)?, + bytes + .checked_add(child_statistics.output.bytes) + .ok_or(AnalyticalCostError::Overflow)?, + )) + }, + )?; + if statistics.output != (EdgeStatistics { rows, bytes }) { return Err(AnalyticalCostError::InconsistentOperatorStatistics( "concat statistics do not equal the sum of child outputs", )); } - Ok(self.push(id, PhysicalOperator::Concat, inputs, child_ids)) + require_operator_statistics(PhysicalOperator::Concat, &statistics)?; + Ok(self.push(id, PhysicalOperator::Concat, &statistics, child_ids, None)) } } let mut lowerer = Lowerer { - statistics: &mut statistics, + scope, + provider: evidence, + statistics: HashMap::new(), + output_buffers: HashMap::new(), logical_roots: HashMap::new(), next_id: 0, nodes: Vec::new(), @@ -300,92 +367,107 @@ where }) } -fn require_consistent_edge_statistics(inputs: OperatorInputs) -> Result<(), AnalyticalCostError> { - require_cardinality_width("operator input", inputs.input_rows, inputs.input_bytes)?; - require_cardinality_width("operator output", inputs.output_rows, inputs.output_bytes)?; - Ok(()) -} - -fn require_cardinality_width( - edge: &'static str, - rows: u64, - bytes: u64, +fn require_statistics_shape( + node: &str, + statistics: &OperatorStatistics, + input_count: usize, ) -> Result<(), AnalyticalCostError> { - match (rows, bytes) { - (0, 0) => Ok(()), - (0, _) => Err(AnalyticalCostError::InconsistentOperatorStatistics(edge)), - (_, 0) => Err(AnalyticalCostError::MissingOrZero(edge)), - _ => Ok(()), + if statistics.inputs.len() != input_count { + return Err(AnalyticalCostError::InvalidOperatorStatistics { + node: node.into(), + reason: "wrong input-edge count", + }); } + if statistics + .inputs + .iter() + .chain(std::iter::once(&statistics.output)) + .any(|edge| !edge.is_consistent()) + { + return Err(AnalyticalCostError::InvalidOperatorStatistics { + node: node.into(), + reason: "edge rows and logical bytes are inconsistent", + }); + } + Ok(()) } fn require_unary_edge( - inputs: OperatorInputs, - child: OperatorInputs, + node: &str, + statistics: &OperatorStatistics, + child_id: &str, + child: &OperatorStatistics, ) -> Result<(), AnalyticalCostError> { - require_consistent_edge_statistics(inputs)?; - if inputs.input_rows != child.output_rows || inputs.input_bytes != child.output_bytes { - return Err(AnalyticalCostError::InconsistentOperatorStatistics( - "unary input does not match child output", - )); + require_statistics_shape(node, statistics, 1)?; + if statistics.inputs[0] != child.output { + return Err(AnalyticalCostError::ConflictingEdgeStatistics { + parent: node.into(), + child: child_id.into(), + input_index: 0, + }); } Ok(()) } fn require_operator_statistics( operator: PhysicalOperator, - inputs: OperatorInputs, + statistics: &OperatorStatistics, ) -> Result<(), AnalyticalCostError> { let invalid = |reason| Err(AnalyticalCostError::InconsistentOperatorStatistics(reason)); + if !matches!(operator, PhysicalOperator::Scan) && statistics.source_scan_bytes != 0 { + return invalid("only Scan may charge source bytes"); + } + let input = statistics.inputs.first().copied().ok_or( + AnalyticalCostError::InconsistentOperatorStatistics("operator input is missing"), + )?; + let output = statistics.output; match operator { PhysicalOperator::Filter => { - if inputs.output_rows > inputs.input_rows || inputs.output_bytes > inputs.input_bytes { + if output.rows > input.rows || output.bytes > input.bytes { return invalid("filter output expands its input"); } } PhysicalOperator::Project => { - if inputs.output_rows != inputs.input_rows { + if output.rows != input.rows { return invalid("projection changes row cardinality"); } } PhysicalOperator::HashAggregate => { - let groups = inputs + let groups = statistics .group_count .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - if groups == 0 && (inputs.input_rows != 0 || inputs.output_rows != 0) { + if groups == 0 && (input.rows != 0 || output.rows != 0) { return invalid("zero groups require an empty grouped input and output"); } - if inputs.output_rows > groups { + if output.rows > groups { return invalid("aggregate output exceeds group cardinality"); } } PhysicalOperator::Deduplicate => { - let groups = inputs + let groups = statistics .group_count .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - if inputs.output_rows != groups || inputs.output_rows > inputs.input_rows { + if output.rows != groups || output.rows > input.rows { return invalid("deduplicate output does not equal distinct cardinality"); } } PhysicalOperator::Sort => { - if inputs.output_rows != inputs.input_rows || inputs.output_bytes != inputs.input_bytes - { + if output != input { return invalid("sort changes its input cardinality or width"); } } PhysicalOperator::TopK | PhysicalOperator::Limit => { - if inputs.output_rows > inputs.input_rows { + if output.rows > input.rows { return invalid("bounded output exceeds its input cardinality"); } } PhysicalOperator::Window => { - if inputs.output_rows != inputs.input_rows { + if output.rows != input.rows { return invalid("SQL window changes row cardinality"); } } PhysicalOperator::PassThrough => { - if inputs.output_rows != inputs.input_rows || inputs.output_bytes != inputs.input_bytes - { + if output != input { return invalid("pass-through wrapper changes its edge statistics"); } } @@ -397,12 +479,12 @@ fn require_operator_statistics( fn require_limit_cardinality( n: usize, offset: usize, - inputs: OperatorInputs, + statistics: &OperatorStatistics, ) -> Result<(), AnalyticalCostError> { let n = u64::try_from(n).map_err(|_| AnalyticalCostError::Overflow)?; let offset = u64::try_from(offset).map_err(|_| AnalyticalCostError::Overflow)?; - let expected = inputs.input_rows.saturating_sub(offset).min(n); - if inputs.output_rows != expected { + let expected = statistics.inputs[0].rows.saturating_sub(offset).min(n); + if statistics.output.rows != expected { return Err(AnalyticalCostError::InconsistentOperatorStatistics( "limit output does not match n and offset", )); @@ -410,14 +492,26 @@ fn require_limit_cardinality( Ok(()) } -fn clear_operator_specific_inputs(inputs: &mut OperatorInputs) { - inputs.group_count = None; - inputs.key_bytes = None; - inputs.aggregate_value_bytes = None; - inputs.k = None; - inputs.right_rows = None; - inputs.right_bytes = None; - inputs.hash_join_build_side = None; +fn bind_scan_coverage( + node_id: &str, + source: &asap_types::pre_asap::Source, + predicates: &[asap_types::pre_asap::Predicate], + scope: &ComparisonScope, +) -> Result { + let mut matches = scope + .sources + .iter() + .filter(|coverage| coverage.source == *source && coverage.predicates == predicates); + let coverage = matches + .next() + .cloned() + .ok_or_else(|| AnalyticalCostError::ScanOutsideComparisonScope(node_id.into()))?; + if matches.next().is_some() { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "scan source coverage is ambiguous", + )); + } + Ok(coverage) } fn is_hash_join_predicate(expr: &asap_types::pre_asap::QueryExpr) -> bool { @@ -441,28 +535,64 @@ fn is_hash_join_predicate(expr: &asap_types::pre_asap::QueryExpr) -> bool { mod tests { use super::*; use crate::analytical_cost::{estimate_physical_dag, HashJoinBuildSide}; + use asap_types::workload::{ + DataArrival, DurationMs, QueryRecurrence, QueryTimeScope, TimeSelection, TimestampMs, + }; + use std::collections::HashMap; - fn operator_inputs( - input_rows: u64, - input_bytes: u64, - output_rows: u64, - output_bytes: u64, - ) -> OperatorInputs { - OperatorInputs { - input_rows, - input_bytes, - output_rows, - output_bytes, + fn edge(rows: u64, bytes: u64) -> EdgeStatistics { + EdgeStatistics { rows, bytes } + } + + fn statistics(inputs: Vec, output: EdgeStatistics) -> OperatorStatistics { + OperatorStatistics { + source_scan_bytes: 0, + inputs, + output, group_count: None, key_bytes: None, aggregate_value_bytes: None, k: None, - right_rows: None, - right_bytes: None, hash_join_build_side: None, } } + fn evidence(statistics: OperatorStatistics) -> PhysicalNodeEvidence { + PhysicalNodeEvidence { + output_buffer_bytes: statistics.output.bytes.min(1_024), + statistics, + } + } + + fn scope(sources: Vec) -> ComparisonScope { + ComparisonScope { + data_arrival: DataArrival::AtRest, + planning_time: TimestampMs(1_000), + horizon: DurationMs(1_000), + recurrence: QueryRecurrence::OneTime { + invocations: 1, + execute_at: None, + }, + time_selection: TimeSelection { + scope: QueryTimeScope::Longitudinal, + lookback: Some(DurationMs(1_000)), + as_of: Some(TimestampMs(1_000)), + }, + sources, + } + } + + fn coverage( + source: asap_types::pre_asap::Source, + predicates: Vec, + ) -> SourceCoverage { + SourceCoverage { + source, + snapshot_id: "snapshot-1".into(), + predicates, + } + } + #[test] fn query_lowering_recurses_and_fuses_global_sort_limit() { use asap_types::pre_asap::{ @@ -501,21 +631,33 @@ mod tests { child: sort, }); - let dag = lower_query_physical_dag(&root, |node| { - let mut stats = match node { - QueryExpr::Scan { .. } => operator_inputs(1_000, 64_000, 400, 25_600), - QueryExpr::Aggregate { .. } => operator_inputs(400, 25_600, 100, 4_000), - QueryExpr::Limit { .. } => operator_inputs(100, 4_000, 10, 400), - _ => return None, - }; - if matches!(node, QueryExpr::Aggregate { .. }) { - stats.group_count = Some(100); - stats.key_bytes = Some(16); - stats.aggregate_value_bytes = Some(8); - } - Some(stats) - }) - .unwrap(); + let scan_coverage = coverage( + Source::Table { + table_ref: "events".into(), + }, + vec![asap_types::pre_asap::Predicate(Rc::new( + QueryExpr::Literal(asap_types::pre_asap::ScalarValue::Boolean(true)), + ))], + ); + let scope = scope(vec![scan_coverage]); + let mut aggregate_statistics = statistics(vec![edge(400, 25_600)], edge(100, 4_000)); + aggregate_statistics.group_count = Some(100); + aggregate_statistics.key_bytes = Some(16); + aggregate_statistics.aggregate_value_bytes = Some(8); + let mut topk_statistics = statistics(vec![edge(100, 4_000)], edge(10, 400)); + topk_statistics.k = Some(15); + let mut raw_scan = statistics(vec![edge(1_000, 64_000)], edge(1_000, 64_000)); + raw_scan.source_scan_bytes = 64_000; + let provided = HashMap::from([ + ("query-2-scan".into(), evidence(raw_scan)), + ( + "query-2".into(), + evidence(statistics(vec![edge(1_000, 64_000)], edge(400, 25_600))), + ), + ("query-1".into(), evidence(aggregate_statistics)), + ("query-0".into(), evidence(topk_statistics)), + ]); + let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); assert_eq!( dag.nodes @@ -530,9 +672,20 @@ mod tests { ] ); let topk = dag.nodes.last().unwrap(); - assert_eq!(topk.inputs.k, Some(15)); assert_eq!(topk.children, vec![dag.nodes[2].id.clone()]); - assert!(estimate_physical_dag(&dag.nodes, &dag.root, 3).is_ok()); + assert_eq!(provided[&topk.id].statistics.k, Some(15)); + let physical_scan = &dag.nodes[0]; + assert_eq!(physical_scan.id, "query-2-scan"); + assert_eq!( + physical_scan.source_coverage, + Some(scope.sources[0].clone()) + ); + assert_eq!(physical_scan.output_buffer_bytes, 1_024); + assert_ne!( + physical_scan.output_buffer_bytes, + provided[&physical_scan.id].statistics.output.bytes + ); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).is_ok()); } #[test] @@ -558,22 +711,26 @@ mod tests { left: Rc::clone(&shared), right: Rc::clone(&shared), }); - let dag = lower_query_physical_dag(&root, |node| match node { - QueryExpr::Scan { .. } => Some(operator_inputs(100, 800, 100, 800)), - QueryExpr::Join { .. } => { - let mut stats = operator_inputs(100, 800, 25, 400); - stats.right_rows = Some(100); - stats.right_bytes = Some(800); - stats.hash_join_build_side = Some(HashJoinBuildSide::Right); - Some(stats) - } - _ => None, - }) - .unwrap(); + let source_coverage = coverage( + Source::Table { + table_ref: "dimensions".into(), + }, + vec![], + ); + let scope = scope(vec![source_coverage]); + let mut scan_statistics = statistics(vec![edge(100, 800)], edge(100, 800)); + scan_statistics.source_scan_bytes = 800; + let mut join_statistics = statistics(vec![edge(100, 800), edge(100, 800)], edge(25, 400)); + join_statistics.hash_join_build_side = Some(HashJoinBuildSide::Right); + let provided = HashMap::from([ + ("query-1".into(), evidence(scan_statistics)), + ("query-0".into(), evidence(join_statistics)), + ]); + let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); assert_eq!(dag.nodes.len(), 2); assert_eq!(dag.nodes[1].children, vec![dag.nodes[0].id.clone(); 2]); - let estimate = estimate_physical_dag(&dag.nodes, &dag.root, 1).unwrap(); + let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).unwrap(); assert_eq!(estimate.scan_bytes, 800); } @@ -629,25 +786,47 @@ mod tests { child: limit, }); - let dag = lower_query_physical_dag(&root, |node| { - let mut inputs = match node { - QueryExpr::Scan { .. } => operator_inputs(1_000, 8_000, 1_000, 8_000), - QueryExpr::Filter { .. } => operator_inputs(1_000, 8_000, 800, 6_400), - QueryExpr::Project { .. } => operator_inputs(800, 6_400, 800, 3_200), - QueryExpr::Dedup { .. } => operator_inputs(800, 3_200, 500, 2_000), - QueryExpr::SQLWindowFunc { .. } => operator_inputs(500, 2_000, 500, 6_000), - QueryExpr::Sort { .. } => operator_inputs(500, 6_000, 500, 6_000), - QueryExpr::Limit { .. } => operator_inputs(500, 6_000, 20, 240), - QueryExpr::TimeShift { .. } => operator_inputs(20, 240, 20, 240), - _ => return None, - }; - if matches!(node, QueryExpr::Dedup { .. }) { - inputs.group_count = Some(500); - inputs.key_bytes = Some(8); - } - Some(inputs) - }) - .unwrap(); + let source_coverage = coverage( + Source::Table { + table_ref: "events".into(), + }, + vec![], + ); + let scope = scope(vec![source_coverage]); + let mut scan_statistics = statistics(vec![edge(1_000, 8_000)], edge(1_000, 8_000)); + scan_statistics.source_scan_bytes = 8_000; + let mut dedup_statistics = statistics(vec![edge(800, 3_200)], edge(500, 2_000)); + dedup_statistics.group_count = Some(500); + dedup_statistics.key_bytes = Some(8); + let provided = HashMap::from([ + ("query-7".into(), evidence(scan_statistics)), + ( + "query-6".into(), + evidence(statistics(vec![edge(1_000, 8_000)], edge(800, 6_400))), + ), + ( + "query-5".into(), + evidence(statistics(vec![edge(800, 6_400)], edge(800, 3_200))), + ), + ("query-4".into(), evidence(dedup_statistics)), + ( + "query-3".into(), + evidence(statistics(vec![edge(500, 2_000)], edge(500, 6_000))), + ), + ( + "query-2".into(), + evidence(statistics(vec![edge(500, 6_000)], edge(500, 6_000))), + ), + ( + "query-1".into(), + evidence(statistics(vec![edge(500, 6_000)], edge(20, 240))), + ), + ( + "query-0".into(), + evidence(statistics(vec![edge(20, 240)], edge(20, 240))), + ), + ]); + let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); assert_eq!( dag.nodes @@ -665,7 +844,7 @@ mod tests { PhysicalOperator::PassThrough, ] ); - assert!(estimate_physical_dag(&dag.nodes, &dag.root, 2).is_ok()); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).is_ok()); } #[test] @@ -687,31 +866,39 @@ mod tests { left: Rc::new(scan("a")), right: Rc::new(scan("b")), }); - let dag = lower_query_physical_dag(&union, |node| match node { - QueryExpr::Scan { - source: Source::Table { table_ref }, - .. - } if table_ref == "a" => Some(operator_inputs(10, 80, 10, 80)), - QueryExpr::Scan { .. } => Some(operator_inputs(20, 160, 20, 160)), - QueryExpr::SetOp { .. } => Some(operator_inputs(30, 240, 30, 240)), - _ => None, - }) - .unwrap(); + let scope = scope(vec![ + coverage( + Source::Table { + table_ref: "a".into(), + }, + vec![], + ), + coverage( + Source::Table { + table_ref: "b".into(), + }, + vec![], + ), + ]); + let mut left_statistics = statistics(vec![edge(10, 80)], edge(10, 80)); + left_statistics.source_scan_bytes = 80; + let mut right_statistics = statistics(vec![edge(20, 160)], edge(20, 160)); + right_statistics.source_scan_bytes = 160; + let provided = HashMap::from([ + ("query-1".into(), evidence(left_statistics)), + ("query-2".into(), evidence(right_statistics)), + ( + "query-0".into(), + evidence(statistics(vec![edge(10, 80), edge(20, 160)], edge(30, 240))), + ), + ]); + let dag = lower_query_physical_dag(&union, &scope, &provided).unwrap(); assert_eq!(dag.nodes.last().unwrap().operator, PhysicalOperator::Concat); let concat = Rc::new(QueryExpr::Concat { children: vec![scan("a"), scan("b")], }); - let dag = lower_query_physical_dag(&concat, |node| match node { - QueryExpr::Scan { - source: Source::Table { table_ref }, - .. - } if table_ref == "a" => Some(operator_inputs(10, 80, 10, 80)), - QueryExpr::Scan { .. } => Some(operator_inputs(20, 160, 20, 160)), - QueryExpr::Concat { .. } => Some(operator_inputs(30, 240, 30, 240)), - _ => None, - }) - .unwrap(); + let dag = lower_query_physical_dag(&concat, &scope, &provided).unwrap(); assert_eq!(dag.nodes.last().unwrap().operator, PhysicalOperator::Concat); let distinct_union = Rc::new(QueryExpr::SetOp { @@ -721,7 +908,7 @@ mod tests { right: Rc::new(scan("b")), }); assert_eq!( - lower_query_physical_dag(&distinct_union, |_| None), + lower_query_physical_dag(&distinct_union, &scope, &provided), Err(AnalyticalCostError::UnsupportedQueryOperator) ); } @@ -745,18 +932,70 @@ mod tests { child: scan, }); + let comparison_scope = scope(vec![coverage( + Source::Table { + table_ref: "events".into(), + }, + vec![], + )]); + let missing = HashMap::::new(); assert_eq!( - lower_query_physical_dag(&root, |_| None), - Err(AnalyticalCostError::MissingOrStale("operator_statistics")) + lower_query_physical_dag(&root, &comparison_scope, &missing), + Err(AnalyticalCostError::MissingOperatorStatistics( + "query-1".into() + )) ); + struct MissingBuffer; + impl PhysicalNodeEvidenceProvider for MissingBuffer { + fn evidence( + &self, + _node_id: &str, + ) -> Result { + Err(AnalyticalCostError::MissingOrStale("output_buffer_bytes")) + } + } assert_eq!( - lower_query_physical_dag(&root, |node| match node { - QueryExpr::Scan { .. } => Some(operator_inputs(100, 800, 100, 800)), - QueryExpr::Project { .. } => Some(operator_inputs(99, 792, 99, 396)), - _ => None, - }), - Err(AnalyticalCostError::InconsistentOperatorStatistics( - "unary input does not match child output" + lower_query_physical_dag(&root, &comparison_scope, &MissingBuffer), + Err(AnalyticalCostError::MissingOrStale("output_buffer_bytes")) + ); + let mut scan_statistics = statistics(vec![edge(100, 800)], edge(100, 800)); + scan_statistics.source_scan_bytes = 800; + let conflicting = HashMap::from([ + ("query-1".into(), evidence(scan_statistics)), + ( + "query-0".into(), + evidence(statistics(vec![edge(99, 792)], edge(99, 396))), + ), + ]); + assert_eq!( + lower_query_physical_dag(&root, &comparison_scope, &conflicting), + Err(AnalyticalCostError::ConflictingEdgeStatistics { + parent: "query-0".into(), + child: "query-1".into(), + input_index: 0, + }) + ); + + let outside_scope = scope(vec![coverage( + Source::Table { + table_ref: "other".into(), + }, + vec![], + )]); + assert_eq!( + lower_query_physical_dag(&root, &outside_scope, &conflicting), + Err(AnalyticalCostError::ScanOutsideComparisonScope( + "query-1".into() + )) + ); + + let mut second_snapshot = comparison_scope.sources[0].clone(); + second_snapshot.snapshot_id = "snapshot-2".into(); + let ambiguous_scope = scope(vec![comparison_scope.sources[0].clone(), second_snapshot]); + assert_eq!( + lower_query_physical_dag(&root, &ambiguous_scope, &conflicting), + Err(AnalyticalCostError::InvalidPhysicalDag( + "scan source coverage is ambiguous" )) ); } @@ -784,14 +1023,27 @@ mod tests { child: filter, }); - let dag = lower_query_physical_dag(&root, |node| match node { - QueryExpr::Scan { .. } => Some(operator_inputs(100, 800, 100, 800)), - QueryExpr::Filter { .. } => Some(operator_inputs(100, 800, 0, 0)), - QueryExpr::Limit { .. } => Some(operator_inputs(0, 0, 0, 0)), - _ => None, - }) - .unwrap(); - let estimate = estimate_physical_dag(&dag.nodes, &dag.root, 1).unwrap(); + let scope = scope(vec![coverage( + Source::Table { + table_ref: "events".into(), + }, + vec![], + )]); + let mut scan_statistics = statistics(vec![edge(100, 800)], edge(100, 800)); + scan_statistics.source_scan_bytes = 800; + let provided = HashMap::from([ + ("query-2".into(), evidence(scan_statistics)), + ( + "query-1".into(), + evidence(statistics(vec![edge(100, 800)], edge(0, 0))), + ), + ( + "query-0".into(), + evidence(statistics(vec![edge(0, 0)], edge(0, 0))), + ), + ]); + let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); + let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).unwrap(); assert_eq!(estimate.cpu_ops, 200.0); assert_eq!(estimate.scan_bytes, 800); } diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 7de1cf13..a6288be9 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -334,9 +334,20 @@ deduplicated by physical identity. `lower_query_physical_dag` recursively lowers a resolved `Rc` and returns a `PhysicalDag` containing both its nodes and root ID. It consumes the existing query and physical-operator enums; it does not introduce a parallel -logical operator vocabulary. A statistics callback resolves the existing -`OperatorInputs` for each logical operator identity. Returning no statistics -makes the entire query unavailable. +logical operator vocabulary. A `PhysicalNodeEvidenceProvider` resolves one +atomic `PhysicalNodeEvidence` for each deterministic physical node ID. That +value reuses the authoritative `OperatorStatistics` contract and adds only +`output_buffer_bytes`, because logical edge bytes are not an allocation. +Missing evidence makes the entire query unavailable. + +Each lowered Scan is bound to exactly one `SourceCoverage` in the comparison +scope by the existing source and canonical predicate values. The bound value +therefore also supplies the provider-owned snapshot ID. Zero matches fail as +outside scope; multiple matching coverages fail as ambiguous rather than +choosing an arbitrary snapshot. When a predicate-bearing logical Scan expands +to Scan → Filter, the synthetic Scan has its own physical ID, statistics, and +buffer evidence and carries that exact coverage; the Filter has separate +evidence and no source coverage. The lowering validates every physical edge before costing: From 16f57c59937a72ae15cea0eed289d0a80b426d3e Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 09:20:55 -0600 Subject: [PATCH 30/48] fix(cost): reject unproven query algorithms --- .../src/analytical_lowering.rs | 188 ++++++++++++++++-- .../analytical-resource-cost.md | 11 +- 2 files changed, 181 insertions(+), 18 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index 8a37600b..c67b7b26 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -156,6 +156,7 @@ pub fn lower_query_physical_dag( if predicates.is_empty() { let statistics = self.stats(&id)?; require_statistics_shape(&id, &statistics, 1)?; + require_scan_edges_equal(&statistics)?; return Ok(self.push( id, PhysicalOperator::Scan, @@ -167,6 +168,7 @@ pub fn lower_query_physical_dag( let scan_id = format!("{id}-scan"); let scan_statistics = self.stats(&scan_id)?; require_statistics_shape(&scan_id, &scan_statistics, 1)?; + require_scan_edges_equal(&scan_statistics)?; self.push( scan_id.clone(), PhysicalOperator::Scan, @@ -191,7 +193,16 @@ pub fn lower_query_physical_dag( QueryExpr::Project { child, .. } => { self.lower_unary(query, id, PhysicalOperator::Project, child) } - QueryExpr::Aggregate { child, .. } => { + QueryExpr::Aggregate { + reduction, + measures, + having, + child, + .. + } => { + if having.is_some() || !supports_hash_aggregate(reduction, measures) { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + } self.lower_unary(query, id, PhysicalOperator::HashAggregate, child) } QueryExpr::Dedup { child, .. } => { @@ -278,7 +289,7 @@ pub fn lower_query_physical_dag( right, } => { if matches!(kind, asap_types::pre_asap::JoinKind::Cross) - || !is_hash_join_predicate(&pred.0) + || !is_hash_join_predicate(&pred.0, left, right) { return Err(AnalyticalCostError::UnsupportedQueryOperator); } @@ -392,6 +403,15 @@ fn require_statistics_shape( Ok(()) } +fn require_scan_edges_equal(statistics: &OperatorStatistics) -> Result<(), AnalyticalCostError> { + if statistics.inputs[0] != statistics.output { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "Scan external input edge does not match its output edge", + )); + } + Ok(()) +} + fn require_unary_edge( node: &str, statistics: &OperatorStatistics, @@ -514,21 +534,80 @@ fn bind_scan_coverage( Ok(coverage) } -fn is_hash_join_predicate(expr: &asap_types::pre_asap::QueryExpr) -> bool { +fn is_hash_join_predicate( + expr: &asap_types::pre_asap::QueryExpr, + left: &asap_types::pre_asap::QueryExpr, + right: &asap_types::pre_asap::QueryExpr, +) -> bool { use asap_types::pre_asap::{CompareOpKind, QueryExpr}; - match expr { - QueryExpr::Compare { - left, - op: CompareOpKind::Eq, - right, - } => { - matches!(left.as_ref(), QueryExpr::Column(_)) - && matches!(right.as_ref(), QueryExpr::Column(_)) + let (Ok(left_schema), Ok(right_schema)) = (left.output_schema(), right.output_schema()) else { + return false; + }; + let left_width = left_schema.columns.len(); + let total_width = left_width.saturating_add(right_schema.columns.len()); + + fn column_side(column: usize, left_width: usize, total_width: usize) -> Option { + if column < left_width { + Some(false) + } else if column < total_width { + Some(true) + } else { + None } - QueryExpr::BoolAnd(parts) => !parts.is_empty() && parts.iter().all(is_hash_join_predicate), - _ => false, } + + fn predicate(expr: &QueryExpr, left_width: usize, total_width: usize) -> bool { + match expr { + QueryExpr::Compare { + left, + op: CompareOpKind::Eq, + right, + } => match (left.as_ref(), right.as_ref()) { + (QueryExpr::Column(left), QueryExpr::Column(right)) => matches!( + ( + column_side(*left, left_width, total_width), + column_side(*right, left_width, total_width) + ), + (Some(false), Some(true)) | (Some(true), Some(false)) + ), + _ => false, + }, + QueryExpr::BoolAnd(parts) => { + !parts.is_empty() + && parts + .iter() + .all(|part| predicate(part, left_width, total_width)) + } + _ => false, + } + } + + predicate(expr, left_width, total_width) +} + +fn supports_hash_aggregate( + reduction: &asap_types::pre_asap::Reduction, + measures: &[asap_types::pre_asap::AggIntent], +) -> bool { + use asap_types::pre_asap::{AggIntent, Reduction}; + + matches!(reduction, Reduction::Reduce(_)) + && !measures.is_empty() + && measures.iter().all(|intent| { + matches!( + intent, + AggIntent::Count { .. } + | AggIntent::Sum { .. } + | AggIntent::Min { .. } + | AggIntent::Max { .. } + | AggIntent::Avg { .. } + | AggIntent::StdDev { .. } + | AggIntent::Variance { .. } + | AggIntent::Group + | AggIntent::CountValues { .. } + ) + }) } #[cfg(test)] @@ -595,9 +674,7 @@ mod tests { #[test] fn query_lowering_recurses_and_fuses_global_sort_limit() { - use asap_types::pre_asap::{ - agg_intent::default_cardinality, GroupKeys, QueryExpr, Reduction, Source, - }; + use asap_types::pre_asap::{AggIntent, GroupKeys, QueryExpr, Reduction, Source}; use asap_types::pre_asap::{Column, DataType, Schema}; use std::rc::Rc; @@ -615,7 +692,7 @@ mod tests { }); let aggregate = Rc::new(QueryExpr::Aggregate { reduction: Reduction::by(vec![0]), - measures: vec![default_cardinality()], + measures: vec![AggIntent::Sum { col: Some(1) }], output_names: vec![], having: None, child: Rc::clone(&scan), @@ -686,6 +763,19 @@ mod tests { provided[&physical_scan.id].statistics.output.bytes ); assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).is_ok()); + + let mut inconsistent_scan = provided.clone(); + inconsistent_scan + .get_mut("query-2-scan") + .unwrap() + .statistics + .output = edge(999, 63_936); + assert_eq!( + lower_query_physical_dag(&root, &scope, &inconsistent_scan), + Err(AnalyticalCostError::InconsistentOperatorStatistics( + "Scan external input edge does not match its output edge" + )) + ); } #[test] @@ -732,6 +822,21 @@ mod tests { assert_eq!(dag.nodes[1].children, vec![dag.nodes[0].id.clone(); 2]); let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).unwrap(); assert_eq!(estimate.scan_bytes, 800); + + let invalid = Rc::new(QueryExpr::Join { + kind: JoinKind::Inner, + pred: Predicate(Rc::new(QueryExpr::Compare { + left: Rc::new(QueryExpr::Column(0)), + op: CompareOpKind::Eq, + right: Rc::new(QueryExpr::Column(0)), + })), + left: Rc::clone(&shared), + right: Rc::clone(&shared), + }); + assert_eq!( + lower_query_physical_dag(&invalid, &scope, &provided), + Err(AnalyticalCostError::UnsupportedQueryOperator) + ); } #[test] @@ -1047,4 +1152,53 @@ mod tests { assert_eq!(estimate.cpu_ops, 200.0); assert_eq!(estimate.scan_bytes, 800); } + + #[test] + fn query_lowering_rejects_aggregates_without_a_hash_implementation() { + use asap_types::pre_asap::{AggIntent, QueryExpr, Reduction, Source}; + use asap_types::pre_asap::{Column, DataType, Schema}; + use asap_types::types::AccuracyTarget; + use std::rc::Rc; + + let scan = || { + Rc::new(QueryExpr::Scan { + source: Source::Table { + table_ref: "events".into(), + }, + predicates: vec![], + schema: Schema::new(vec![Column::new("value", DataType::Float64, false)]), + }) + }; + let exact_quantile = Rc::new(QueryExpr::Aggregate { + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::Quantile { + col: Some(0), + q: 0.99, + accuracy: AccuracyTarget::Exact, + }], + output_names: vec![], + having: None, + child: scan(), + }); + let per_entity = Rc::new(QueryExpr::Aggregate { + reduction: Reduction::PerEntity, + measures: vec![AggIntent::Rate], + output_names: vec![], + having: None, + child: scan(), + }); + let scope = scope(vec![coverage( + Source::Table { + table_ref: "events".into(), + }, + vec![], + )]); + let unavailable = HashMap::::new(); + for query in [&exact_quantile, &per_entity] { + assert_eq!( + lower_query_physical_dag(query, &scope, &unavailable), + Err(AnalyticalCostError::UnsupportedQueryOperator) + ); + } + } } diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index a6288be9..39b3adfa 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -354,6 +354,8 @@ The lowering validates every physical edge before costing: - `(rows = 0, bytes = 0)` is a valid empty edge, while positive rows still require byte-width evidence and zero rows cannot carry non-zero bytes; - a unary operator's `input_rows` and `input_bytes` equal its child's output; +- a Scan's external logical input edge equals its output edge, including the + synthetic raw Scan created for a predicate-bearing logical Scan; - a hash join's left and right inputs equal the corresponding child outputs; - Concat and `UNION ALL` input/output totals equal the checked sum of all child outputs; and @@ -368,7 +370,7 @@ The supported mappings are: | Scan with pushed predicates | Scan → Filter | | Filter | Filter | | Project | Project | -| Aggregate, including a fused HAVING predicate | HashAggregate | +| Reducing Count/Sum/Min/Max/Avg/StdDev/Variance/Group/CountValues without HAVING | HashAggregate | | Dedup | Deduplicate | | Equi-Join | HashJoin with an evidence-selected build side | | Concat or `UNION ALL` | Concat | @@ -378,6 +380,13 @@ The supported mappings are: | SQLWindowFunc | Window | | TimeShift | PassThrough | +Per-entity reductions, HAVING, ordered/distribution-dependent intents such as +exact quantile or cardinality, Top-K aggregate intents, and extensions remain +unavailable until they have an explicit physical algorithm. Hash-join lowering +also uses the bound left and right output schemas to prove that every equality +compares one column from each side; same-side or out-of-range `ColumnId`s fail +closed. + Logical identity is the address of the existing `Rc` allocation. Repeated references therefore lower once and every parent points to the same physical ID. The resulting node IDs are deterministic within a lowering run; From b581282e18de8a8d2580d1cd87bd1a9865f27484 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 09:39:19 -0600 Subject: [PATCH 31/48] fix(cost): require provider-owned physical identity --- .../src/analytical_lowering.rs | 574 +++++++++++++----- .../analytical-resource-cost.md | 41 +- 2 files changed, 441 insertions(+), 174 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index c67b7b26..7c7cc7c7 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -18,6 +18,7 @@ use crate::analytical_statistics::{ pub struct PhysicalDag { pub nodes: Vec, pub root: String, + pub evidence: std::collections::HashMap, } /// Atomic evidence for one lowered physical node. The statistics contract is @@ -25,25 +26,44 @@ pub struct PhysicalDag { /// edge bytes cannot stand in for an allocation. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct PhysicalNodeEvidence { + pub physical_id: String, pub statistics: OperatorStatistics, pub output_buffer_bytes: u64, } +pub struct PhysicalNodeRequest<'a> { + pub logical_node: &'a asap_types::pre_asap::QueryExpr, + pub operator: PhysicalOperator, + pub occurrence: usize, + pub synthetic: bool, + pub children: &'a [String], + pub source_coverage: Option<&'a SourceCoverage>, +} + pub trait PhysicalNodeEvidenceProvider { - fn evidence(&self, node_id: &str) -> Result; + fn evidence( + &self, + request: PhysicalNodeRequest<'_>, + ) -> Result; } -impl PhysicalNodeEvidenceProvider for std::collections::HashMap { - fn evidence(&self, node_id: &str) -> Result { - self.get(node_id) - .cloned() - .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(node_id.into())) +impl PhysicalNodeEvidenceProvider for F +where + F: Fn(PhysicalNodeRequest<'_>) -> Result, +{ + fn evidence( + &self, + request: PhysicalNodeRequest<'_>, + ) -> Result { + self(request) } } impl OperatorStatisticsProvider for std::collections::HashMap { fn statistics(&self, node_id: &str) -> Result { - self.evidence(node_id).map(|evidence| evidence.statistics) + self.get(node_id) + .map(|evidence| evidence.statistics.clone()) + .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(node_id.into())) } } @@ -66,78 +86,99 @@ pub fn lower_query_physical_dag( struct Lowerer<'a> { scope: &'a ComparisonScope, provider: &'a dyn PhysicalNodeEvidenceProvider, - statistics: HashMap, - output_buffers: HashMap, - logical_roots: HashMap, + evidence: HashMap, next_id: usize, nodes: Vec, } impl Lowerer<'_> { fn lower(&mut self, query: &QueryExpr) -> Result { - let identity = std::ptr::from_ref(query) as usize; - if let Some(id) = self.logical_roots.get(&identity) { - return Ok(id.clone()); - } - let id = format!("query-{}", self.next_id); + let occurrence = self.next_id; self.next_id += 1; - // Insert only after successful lowering: a malformed recursive - // shape cannot leave a partially reusable node behind. - let root = self.lower_new(query, id)?; - self.logical_roots.insert(identity, root.clone()); - Ok(root) + self.lower_new(query, occurrence) } - fn stats(&mut self, id: &str) -> Result { - if let Some(statistics) = self.statistics.get(id) { - return Ok(statistics.clone()); + fn resolve( + &self, + query: &QueryExpr, + operator: PhysicalOperator, + occurrence: usize, + synthetic: bool, + children: &[String], + source_coverage: Option<&SourceCoverage>, + ) -> Result { + let evidence = self.provider.evidence(PhysicalNodeRequest { + logical_node: query, + operator, + occurrence, + synthetic, + children, + source_coverage, + })?; + if evidence.physical_id.is_empty() { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "provider returned an empty physical identity", + )); } - let evidence = self.provider.evidence(id)?; - let statistics = evidence.statistics; - self.statistics.insert(id.into(), statistics.clone()); - self.output_buffers - .insert(id.into(), evidence.output_buffer_bytes); - Ok(statistics) + Ok(evidence) } fn push( &mut self, - id: String, + evidence: PhysicalNodeEvidence, operator: PhysicalOperator, - _statistics: &OperatorStatistics, children: Vec, source_coverage: Option, - ) -> String { - self.nodes.push(PhysicalDagNode { + ) -> Result { + let id = evidence.physical_id.clone(); + let node = PhysicalDagNode { id: id.clone(), operator, children, source_coverage, - output_buffer_bytes: self.output_buffers[&id], + output_buffer_bytes: evidence.output_buffer_bytes, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, - }); - id + }; + if let Some(existing) = self.nodes.iter().find(|existing| existing.id == id) { + if existing != &node || self.evidence.get(&id) != Some(&evidence) { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "provider reused a physical identity for conflicting evidence", + )); + } + return Ok(id); + } + self.nodes.push(node); + self.evidence.insert(id.clone(), evidence); + Ok(id) } fn lower_unary( &mut self, - _query: &QueryExpr, - id: String, + query: &QueryExpr, + occurrence: usize, operator: PhysicalOperator, child: &QueryExpr, ) -> Result { let child_id = self.lower(child)?; - let statistics = self.stats(&id)?; + let children = vec![child_id.clone()]; + let evidence = self.resolve(query, operator, occurrence, false, &children, None)?; + let statistics = &evidence.statistics; let child_statistics = self.node_statistics(&child_id)?; - require_unary_edge(&id, &statistics, &child_id, child_statistics)?; - require_operator_statistics(operator, &statistics)?; - Ok(self.push(id, operator, &statistics, vec![child_id], None)) + require_unary_edge( + &evidence.physical_id, + statistics, + &child_id, + child_statistics, + )?; + require_operator_statistics(operator, statistics)?; + self.push(evidence, operator, children, None) } fn node_statistics(&self, id: &str) -> Result<&OperatorStatistics, AnalyticalCostError> { - self.statistics + self.evidence .get(id) + .map(|evidence| &evidence.statistics) .ok_or(AnalyticalCostError::InvalidPhysicalDag( "lowered child statistics are missing", )) @@ -146,52 +187,78 @@ pub fn lower_query_physical_dag( fn lower_new( &mut self, query: &QueryExpr, - id: String, + occurrence: usize, ) -> Result { match query { QueryExpr::Scan { source, predicates, .. } => { - let coverage = bind_scan_coverage(&id, source, predicates, self.scope)?; + let coverage = bind_scan_coverage( + &format!("occurrence-{occurrence}"), + source, + predicates, + self.scope, + )?; if predicates.is_empty() { - let statistics = self.stats(&id)?; - require_statistics_shape(&id, &statistics, 1)?; - require_scan_edges_equal(&statistics)?; - return Ok(self.push( - id, + let evidence = self.resolve( + query, PhysicalOperator::Scan, - &statistics, - vec![], - Some(coverage), - )); + occurrence, + false, + &[], + Some(&coverage), + )?; + require_statistics_shape(&evidence.physical_id, &evidence.statistics, 1)?; + require_scan_edges_equal(&evidence.statistics)?; + return self.push(evidence, PhysicalOperator::Scan, vec![], Some(coverage)); } - let scan_id = format!("{id}-scan"); - let scan_statistics = self.stats(&scan_id)?; - require_statistics_shape(&scan_id, &scan_statistics, 1)?; - require_scan_edges_equal(&scan_statistics)?; - self.push( - scan_id.clone(), + let scan_evidence = self.resolve( + query, + PhysicalOperator::Scan, + occurrence, + true, + &[], + Some(&coverage), + )?; + require_statistics_shape( + &scan_evidence.physical_id, + &scan_evidence.statistics, + 1, + )?; + require_scan_edges_equal(&scan_evidence.statistics)?; + let scan_statistics = scan_evidence.statistics.clone(); + let scan_id = self.push( + scan_evidence, PhysicalOperator::Scan, - &scan_statistics, vec![], Some(coverage), - ); - let filter_statistics = self.stats(&id)?; - require_unary_edge(&id, &filter_statistics, &scan_id, &scan_statistics)?; - require_operator_statistics(PhysicalOperator::Filter, &filter_statistics)?; - Ok(self.push( - id, + )?; + let children = vec![scan_id.clone()]; + let filter_evidence = self.resolve( + query, PhysicalOperator::Filter, - &filter_statistics, - vec![scan_id], + occurrence, + false, + &children, None, - )) + )?; + require_unary_edge( + &filter_evidence.physical_id, + &filter_evidence.statistics, + &scan_id, + &scan_statistics, + )?; + require_operator_statistics( + PhysicalOperator::Filter, + &filter_evidence.statistics, + )?; + self.push(filter_evidence, PhysicalOperator::Filter, children, None) } QueryExpr::Filter { child, .. } => { - self.lower_unary(query, id, PhysicalOperator::Filter, child) + self.lower_unary(query, occurrence, PhysicalOperator::Filter, child) } QueryExpr::Project { child, .. } => { - self.lower_unary(query, id, PhysicalOperator::Project, child) + self.lower_unary(query, occurrence, PhysicalOperator::Project, child) } QueryExpr::Aggregate { reduction, @@ -203,26 +270,44 @@ pub fn lower_query_physical_dag( if having.is_some() || !supports_hash_aggregate(reduction, measures) { return Err(AnalyticalCostError::UnsupportedQueryOperator); } - self.lower_unary(query, id, PhysicalOperator::HashAggregate, child) + self.lower_unary(query, occurrence, PhysicalOperator::HashAggregate, child) } QueryExpr::Dedup { child, .. } => { - self.lower_unary(query, id, PhysicalOperator::Deduplicate, child) + self.lower_unary(query, occurrence, PhysicalOperator::Deduplicate, child) } - QueryExpr::Sort { child, .. } => { - self.lower_unary(query, id, PhysicalOperator::Sort, child) + QueryExpr::Sort { keys, child, .. } => { + if keys.is_empty() { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + } + self.lower_unary(query, occurrence, PhysicalOperator::Sort, child) } QueryExpr::Limit { n, offset, child } => { if let QueryExpr::Sort { + keys, partition_by, child: sorted_child, .. } = child.as_ref() { - if partition_by == &GroupKeys::none() { + if !keys.is_empty() && partition_by == &GroupKeys::none() { let child_id = self.lower(sorted_child)?; - let statistics = self.stats(&id)?; + let children = vec![child_id.clone()]; + let evidence = self.resolve( + query, + PhysicalOperator::TopK, + occurrence, + false, + &children, + None, + )?; + let statistics = &evidence.statistics; let child_statistics = self.node_statistics(&child_id)?; - require_unary_edge(&id, &statistics, &child_id, child_statistics)?; + require_unary_edge( + &evidence.physical_id, + statistics, + &child_id, + child_statistics, + )?; let bound = n .checked_add(*offset) .and_then(|value| u64::try_from(value).ok()) @@ -235,42 +320,59 @@ pub fn lower_query_physical_dag( "Top-K statistics disagree with LIMIT n + offset", )); } - require_limit_cardinality(*n, *offset, &statistics)?; - return Ok(self.push( - id, - PhysicalOperator::TopK, - &statistics, - vec![child_id], - None, - )); + require_limit_cardinality(*n, *offset, statistics)?; + return self.push(evidence, PhysicalOperator::TopK, children, None); } } let child_id = self.lower(child)?; - let statistics = self.stats(&id)?; - let child_statistics = self.node_statistics(&child_id)?; - require_unary_edge(&id, &statistics, &child_id, child_statistics)?; - require_operator_statistics(PhysicalOperator::Limit, &statistics)?; - require_limit_cardinality(*n, *offset, &statistics)?; - Ok(self.push( - id, + let children = vec![child_id.clone()]; + let evidence = self.resolve( + query, PhysicalOperator::Limit, - &statistics, - vec![child_id], + occurrence, + false, + &children, None, - )) + )?; + let statistics = &evidence.statistics; + let child_statistics = self.node_statistics(&child_id)?; + require_unary_edge( + &evidence.physical_id, + statistics, + &child_id, + child_statistics, + )?; + require_operator_statistics(PhysicalOperator::Limit, statistics)?; + require_limit_cardinality(*n, *offset, statistics)?; + self.push(evidence, PhysicalOperator::Limit, children, None) } - QueryExpr::SQLWindowFunc { child, .. } => { - self.lower_unary(query, id, PhysicalOperator::Window, child) + QueryExpr::SQLWindowFunc { + func, + order_by, + child, + .. + } => { + if order_by.is_empty() + || !matches!( + func, + asap_types::pre_asap::WindowFuncKind::RowNumber + | asap_types::pre_asap::WindowFuncKind::Rank + | asap_types::pre_asap::WindowFuncKind::DenseRank + ) + { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + } + self.lower_unary(query, occurrence, PhysicalOperator::Window, child) } QueryExpr::TimeShift { child, .. } => { - self.lower_unary(query, id, PhysicalOperator::PassThrough, child) + self.lower_unary(query, occurrence, PhysicalOperator::PassThrough, child) } QueryExpr::Concat { children } => { let child_ids = children .iter() .map(|child| self.lower(child)) .collect::, _>>()?; - self.lower_concat(query, id, child_ids) + self.lower_concat(query, occurrence, child_ids) } QueryExpr::SetOp { kind: SetOpKind::Union, @@ -280,7 +382,7 @@ pub fn lower_query_physical_dag( } => { let left_id = self.lower(left)?; let right_id = self.lower(right)?; - self.lower_concat(query, id, vec![left_id, right_id]) + self.lower_concat(query, occurrence, vec![left_id, right_id]) } QueryExpr::Join { kind, @@ -295,8 +397,17 @@ pub fn lower_query_physical_dag( } let left_id = self.lower(left)?; let right_id = self.lower(right)?; - let statistics = self.stats(&id)?; - require_statistics_shape(&id, &statistics, 2)?; + let children = vec![left_id.clone(), right_id.clone()]; + let evidence = self.resolve( + query, + PhysicalOperator::HashJoin, + occurrence, + false, + &children, + None, + )?; + let statistics = &evidence.statistics; + require_statistics_shape(&evidence.physical_id, statistics, 2)?; let left_statistics = self.node_statistics(&left_id)?; let right_statistics = self.node_statistics(&right_id)?; if statistics.inputs[0] != left_statistics.output @@ -306,14 +417,8 @@ pub fn lower_query_physical_dag( "join inputs do not match child outputs", )); } - require_operator_statistics(PhysicalOperator::HashJoin, &statistics)?; - Ok(self.push( - id, - PhysicalOperator::HashJoin, - &statistics, - vec![left_id, right_id], - None, - )) + require_operator_statistics(PhysicalOperator::HashJoin, statistics)?; + self.push(evidence, PhysicalOperator::HashJoin, children, None) } _ => Err(AnalyticalCostError::UnsupportedQueryOperator), } @@ -321,8 +426,8 @@ pub fn lower_query_physical_dag( fn lower_concat( &mut self, - _query: &QueryExpr, - id: String, + query: &QueryExpr, + occurrence: usize, child_ids: Vec, ) -> Result { if child_ids.is_empty() { @@ -330,15 +435,23 @@ pub fn lower_query_physical_dag( "concat has no children", )); } - let statistics = self.stats(&id)?; - require_statistics_shape(&id, &statistics, child_ids.len())?; + let evidence = self.resolve( + query, + PhysicalOperator::Concat, + occurrence, + false, + &child_ids, + None, + )?; + let statistics = &evidence.statistics; + require_statistics_shape(&evidence.physical_id, statistics, child_ids.len())?; let (rows, bytes) = child_ids.iter().enumerate().try_fold( (0_u64, 0_u64), |(rows, bytes), (index, child)| { let child_statistics = self.node_statistics(child)?; if statistics.inputs[index] != child_statistics.output { return Err(AnalyticalCostError::ConflictingEdgeStatistics { - parent: id.clone(), + parent: evidence.physical_id.clone(), child: child.clone(), input_index: index, }); @@ -357,27 +470,52 @@ pub fn lower_query_physical_dag( "concat statistics do not equal the sum of child outputs", )); } - require_operator_statistics(PhysicalOperator::Concat, &statistics)?; - Ok(self.push(id, PhysicalOperator::Concat, &statistics, child_ids, None)) + require_operator_statistics(PhysicalOperator::Concat, statistics)?; + self.push(evidence, PhysicalOperator::Concat, child_ids, None) } } let mut lowerer = Lowerer { scope, provider: evidence, - statistics: HashMap::new(), - output_buffers: HashMap::new(), - logical_roots: HashMap::new(), + evidence: HashMap::new(), next_id: 0, nodes: Vec::new(), }; let root = lowerer.lower(root)?; + validate_source_consumption(&lowerer.nodes, scope)?; Ok(PhysicalDag { nodes: lowerer.nodes, root, + evidence: lowerer.evidence, }) } +fn validate_source_consumption( + nodes: &[PhysicalDagNode], + scope: &ComparisonScope, +) -> Result<(), AnalyticalCostError> { + let mut remaining = scope.sources.clone(); + for coverage in nodes + .iter() + .filter(|node| matches!(node.operator, PhysicalOperator::Scan)) + .filter_map(|node| node.source_coverage.as_ref()) + { + let Some(index) = remaining.iter().position(|expected| expected == coverage) else { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "physical scans do not consume comparison sources exactly", + )); + }; + remaining.swap_remove(index); + } + if !remaining.is_empty() { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "physical scans do not consume comparison sources exactly", + )); + } + Ok(()) +} + fn require_statistics_shape( node: &str, statistics: &OperatorStatistics, @@ -526,7 +664,7 @@ fn bind_scan_coverage( .next() .cloned() .ok_or_else(|| AnalyticalCostError::ScanOutsideComparisonScope(node_id.into()))?; - if matches.next().is_some() { + if matches.any(|candidate| candidate != &coverage) { return Err(AnalyticalCostError::InvalidPhysicalDag( "scan source coverage is ambiguous", )); @@ -638,11 +776,31 @@ mod tests { fn evidence(statistics: OperatorStatistics) -> PhysicalNodeEvidence { PhysicalNodeEvidence { + physical_id: String::new(), output_buffer_bytes: statistics.output.bytes.min(1_024), statistics, } } + fn scripted<'a>( + provided: &'a HashMap, + ) -> impl Fn(PhysicalNodeRequest<'_>) -> Result + 'a + { + move |request| { + let key = if request.synthetic { + format!("query-{}-scan", request.occurrence) + } else { + format!("query-{}", request.occurrence) + }; + let mut evidence = provided + .get(&key) + .cloned() + .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(key.clone()))?; + evidence.physical_id = key; + Ok(evidence) + } + } + fn scope(sources: Vec) -> ComparisonScope { ComparisonScope { data_arrival: DataArrival::AtRest, @@ -674,7 +832,7 @@ mod tests { #[test] fn query_lowering_recurses_and_fuses_global_sort_limit() { - use asap_types::pre_asap::{AggIntent, GroupKeys, QueryExpr, Reduction, Source}; + use asap_types::pre_asap::{AggIntent, GroupKeys, QueryExpr, Reduction, SortKey, Source}; use asap_types::pre_asap::{Column, DataType, Schema}; use std::rc::Rc; @@ -698,7 +856,11 @@ mod tests { child: Rc::clone(&scan), }); let sort = Rc::new(QueryExpr::Sort { - keys: vec![], + keys: vec![SortKey { + expr: QueryExpr::Column(0), + ascending: false, + nulls_first: false, + }], partition_by: GroupKeys::none(), child: aggregate, }); @@ -734,7 +896,7 @@ mod tests { ("query-1".into(), evidence(aggregate_statistics)), ("query-0".into(), evidence(topk_statistics)), ]); - let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); + let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)).unwrap(); assert_eq!( dag.nodes @@ -762,7 +924,7 @@ mod tests { physical_scan.output_buffer_bytes, provided[&physical_scan.id].statistics.output.bytes ); - assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).is_ok()); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).is_ok()); let mut inconsistent_scan = provided.clone(); inconsistent_scan @@ -771,7 +933,7 @@ mod tests { .statistics .output = edge(999, 63_936); assert_eq!( - lower_query_physical_dag(&root, &scope, &inconsistent_scan), + lower_query_physical_dag(&root, &scope, &scripted(&inconsistent_scan)), Err(AnalyticalCostError::InconsistentOperatorStatistics( "Scan external input edge does not match its output edge" )) @@ -779,7 +941,7 @@ mod tests { } #[test] - fn query_lowering_deduplicates_shared_rc_children() { + fn query_lowering_shares_only_provider_identified_physical_nodes() { use asap_types::pre_asap::{Column, CompareOpKind, DataType, Schema}; use asap_types::pre_asap::{JoinKind, Predicate, QueryExpr, Source}; use std::rc::Rc; @@ -807,21 +969,67 @@ mod tests { }, vec![], ); - let scope = scope(vec![source_coverage]); + let independent_scope = scope(vec![source_coverage.clone(), source_coverage.clone()]); let mut scan_statistics = statistics(vec![edge(100, 800)], edge(100, 800)); scan_statistics.source_scan_bytes = 800; let mut join_statistics = statistics(vec![edge(100, 800), edge(100, 800)], edge(25, 400)); join_statistics.hash_join_build_side = Some(HashJoinBuildSide::Right); let provided = HashMap::from([ - ("query-1".into(), evidence(scan_statistics)), - ("query-0".into(), evidence(join_statistics)), + ("query-1".into(), evidence(scan_statistics.clone())), + ("query-2".into(), evidence(scan_statistics.clone())), + ("query-0".into(), evidence(join_statistics.clone())), ]); - let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); + let dag = + lower_query_physical_dag(&root, &independent_scope, &scripted(&provided)).unwrap(); + + assert_eq!(dag.nodes.len(), 3); + assert_ne!(dag.nodes[2].children[0], dag.nodes[2].children[1]); + let estimate = + estimate_physical_dag(&dag.nodes, &dag.root, &independent_scope, &dag.evidence) + .unwrap(); + assert_eq!(estimate.scan_bytes, 1_600); + + let shared_provider = |request: PhysicalNodeRequest<'_>| { + let (physical_id, statistics) = match request.operator { + PhysicalOperator::Scan => ("shared-scan", scan_statistics.clone()), + PhysicalOperator::HashJoin => ("join", join_statistics.clone()), + _ => return Err(AnalyticalCostError::UnsupportedQueryOperator), + }; + Ok(PhysicalNodeEvidence { + physical_id: physical_id.into(), + output_buffer_bytes: statistics.output.bytes.min(1_024), + statistics, + }) + }; + let shared_scope = scope(vec![source_coverage]); + let shared_dag = lower_query_physical_dag(&root, &shared_scope, &shared_provider).unwrap(); + assert_eq!(shared_dag.nodes.len(), 2); + assert_eq!( + shared_dag.nodes[1].children, + vec!["shared-scan".to_owned(); 2] + ); - assert_eq!(dag.nodes.len(), 2); - assert_eq!(dag.nodes[1].children, vec![dag.nodes[0].id.clone(); 2]); - let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).unwrap(); - assert_eq!(estimate.scan_bytes, 800); + let conflicting_identity = |request: PhysicalNodeRequest<'_>| { + let (physical_id, mut statistics) = match request.operator { + PhysicalOperator::Scan => ("shared-scan", scan_statistics.clone()), + PhysicalOperator::HashJoin => ("join", join_statistics.clone()), + _ => return Err(AnalyticalCostError::UnsupportedQueryOperator), + }; + if request.operator == PhysicalOperator::Scan && request.occurrence == 2 { + statistics.source_scan_bytes += 1; + } + Ok(PhysicalNodeEvidence { + physical_id: physical_id.into(), + output_buffer_bytes: statistics.output.bytes.min(1_024), + statistics, + }) + }; + assert_eq!( + lower_query_physical_dag(&root, &shared_scope, &conflicting_identity), + Err(AnalyticalCostError::InvalidPhysicalDag( + "provider reused a physical identity for conflicting evidence" + )) + ); let invalid = Rc::new(QueryExpr::Join { kind: JoinKind::Inner, @@ -834,7 +1042,7 @@ mod tests { right: Rc::clone(&shared), }); assert_eq!( - lower_query_physical_dag(&invalid, &scope, &provided), + lower_query_physical_dag(&invalid, &shared_scope, &shared_provider), Err(AnalyticalCostError::UnsupportedQueryOperator) ); } @@ -843,7 +1051,7 @@ mod tests { fn query_lowering_covers_relational_unary_operators() { use asap_types::pre_asap::{Column, DataType, ScalarValue, Schema}; use asap_types::pre_asap::{ - GroupKeys, Predicate, QueryExpr, Source, TimeShift, WindowFuncKind, + GroupKeys, Predicate, QueryExpr, SortKey, Source, TimeShift, WindowFuncKind, }; use std::rc::Rc; @@ -871,13 +1079,21 @@ mod tests { func: WindowFuncKind::RowNumber, args: vec![], partition_by: GroupKeys::none(), - order_by: vec![], + order_by: vec![SortKey { + expr: QueryExpr::Column(0), + ascending: true, + nulls_first: false, + }], frame: None, output_name: "rn".into(), child: dedup, }); let sort = Rc::new(QueryExpr::Sort { - keys: vec![], + keys: vec![SortKey { + expr: QueryExpr::Column(0), + ascending: true, + nulls_first: false, + }], partition_by: GroupKeys::by(vec![0]), child: window, }); @@ -931,7 +1147,7 @@ mod tests { evidence(statistics(vec![edge(20, 240)], edge(20, 240))), ), ]); - let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); + let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)).unwrap(); assert_eq!( dag.nodes @@ -949,7 +1165,7 @@ mod tests { PhysicalOperator::PassThrough, ] ); - assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).is_ok()); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).is_ok()); } #[test] @@ -997,13 +1213,13 @@ mod tests { evidence(statistics(vec![edge(10, 80), edge(20, 160)], edge(30, 240))), ), ]); - let dag = lower_query_physical_dag(&union, &scope, &provided).unwrap(); + let dag = lower_query_physical_dag(&union, &scope, &scripted(&provided)).unwrap(); assert_eq!(dag.nodes.last().unwrap().operator, PhysicalOperator::Concat); let concat = Rc::new(QueryExpr::Concat { children: vec![scan("a"), scan("b")], }); - let dag = lower_query_physical_dag(&concat, &scope, &provided).unwrap(); + let dag = lower_query_physical_dag(&concat, &scope, &scripted(&provided)).unwrap(); assert_eq!(dag.nodes.last().unwrap().operator, PhysicalOperator::Concat); let distinct_union = Rc::new(QueryExpr::SetOp { @@ -1013,7 +1229,7 @@ mod tests { right: Rc::new(scan("b")), }); assert_eq!( - lower_query_physical_dag(&distinct_union, &scope, &provided), + lower_query_physical_dag(&distinct_union, &scope, &scripted(&provided)), Err(AnalyticalCostError::UnsupportedQueryOperator) ); } @@ -1045,7 +1261,7 @@ mod tests { )]); let missing = HashMap::::new(); assert_eq!( - lower_query_physical_dag(&root, &comparison_scope, &missing), + lower_query_physical_dag(&root, &comparison_scope, &scripted(&missing)), Err(AnalyticalCostError::MissingOperatorStatistics( "query-1".into() )) @@ -1054,7 +1270,7 @@ mod tests { impl PhysicalNodeEvidenceProvider for MissingBuffer { fn evidence( &self, - _node_id: &str, + _request: PhysicalNodeRequest<'_>, ) -> Result { Err(AnalyticalCostError::MissingOrStale("output_buffer_bytes")) } @@ -1073,7 +1289,7 @@ mod tests { ), ]); assert_eq!( - lower_query_physical_dag(&root, &comparison_scope, &conflicting), + lower_query_physical_dag(&root, &comparison_scope, &scripted(&conflicting)), Err(AnalyticalCostError::ConflictingEdgeStatistics { parent: "query-0".into(), child: "query-1".into(), @@ -1088,9 +1304,9 @@ mod tests { vec![], )]); assert_eq!( - lower_query_physical_dag(&root, &outside_scope, &conflicting), + lower_query_physical_dag(&root, &outside_scope, &scripted(&conflicting)), Err(AnalyticalCostError::ScanOutsideComparisonScope( - "query-1".into() + "occurrence-1".into() )) ); @@ -1098,11 +1314,30 @@ mod tests { second_snapshot.snapshot_id = "snapshot-2".into(); let ambiguous_scope = scope(vec![comparison_scope.sources[0].clone(), second_snapshot]); assert_eq!( - lower_query_physical_dag(&root, &ambiguous_scope, &conflicting), + lower_query_physical_dag(&root, &ambiguous_scope, &scripted(&conflicting)), Err(AnalyticalCostError::InvalidPhysicalDag( "scan source coverage is ambiguous" )) ); + + let mut complete = conflicting.clone(); + complete.get_mut("query-0").unwrap().statistics = + statistics(vec![edge(100, 800)], edge(100, 400)); + let extra_scope = scope(vec![ + comparison_scope.sources[0].clone(), + coverage( + Source::Table { + table_ref: "unused".into(), + }, + vec![], + ), + ]); + assert_eq!( + lower_query_physical_dag(&root, &extra_scope, &scripted(&complete)), + Err(AnalyticalCostError::InvalidPhysicalDag( + "physical scans do not consume comparison sources exactly" + )) + ); } #[test] @@ -1147,15 +1382,17 @@ mod tests { evidence(statistics(vec![edge(0, 0)], edge(0, 0))), ), ]); - let dag = lower_query_physical_dag(&root, &scope, &provided).unwrap(); - let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &provided).unwrap(); + let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)).unwrap(); + let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).unwrap(); assert_eq!(estimate.cpu_ops, 200.0); assert_eq!(estimate.scan_bytes, 800); } #[test] fn query_lowering_rejects_aggregates_without_a_hash_implementation() { - use asap_types::pre_asap::{AggIntent, QueryExpr, Reduction, Source}; + use asap_types::pre_asap::{ + AggIntent, GroupKeys, QueryExpr, Reduction, Source, WindowFuncKind, + }; use asap_types::pre_asap::{Column, DataType, Schema}; use asap_types::types::AccuracyTarget; use std::rc::Rc; @@ -1187,6 +1424,24 @@ mod tests { having: None, child: scan(), }); + let empty_sort_limit = Rc::new(QueryExpr::Limit { + n: 10, + offset: 0, + child: Rc::new(QueryExpr::Sort { + keys: vec![], + partition_by: GroupKeys::none(), + child: scan(), + }), + }); + let unsupported_window = Rc::new(QueryExpr::SQLWindowFunc { + func: WindowFuncKind::Lag, + args: vec![QueryExpr::Column(0)], + partition_by: GroupKeys::none(), + order_by: vec![], + frame: None, + output_name: "lag".into(), + child: scan(), + }); let scope = scope(vec![coverage( Source::Table { table_ref: "events".into(), @@ -1194,9 +1449,14 @@ mod tests { vec![], )]); let unavailable = HashMap::::new(); - for query in [&exact_quantile, &per_entity] { + for query in [ + &exact_quantile, + &per_entity, + &empty_sort_limit, + &unsupported_window, + ] { assert_eq!( - lower_query_physical_dag(query, &scope, &unavailable), + lower_query_physical_dag(query, &scope, &scripted(&unavailable)), Err(AnalyticalCostError::UnsupportedQueryOperator) ); } diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 39b3adfa..37b07592 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -334,11 +334,16 @@ deduplicated by physical identity. `lower_query_physical_dag` recursively lowers a resolved `Rc` and returns a `PhysicalDag` containing both its nodes and root ID. It consumes the existing query and physical-operator enums; it does not introduce a parallel -logical operator vocabulary. A `PhysicalNodeEvidenceProvider` resolves one -atomic `PhysicalNodeEvidence` for each deterministic physical node ID. That -value reuses the authoritative `OperatorStatistics` contract and adds only -`output_buffer_bytes`, because logical edge bytes are not an allocation. -Missing evidence makes the entire query unavailable. +logical operator vocabulary. For every occurrence, the lowerer sends a +`PhysicalNodeRequest` containing the logical node, selected existing +`PhysicalOperator`, occurrence and synthetic-role metadata, already-lowered +child physical IDs, and any source coverage to a +`PhysicalNodeEvidenceProvider`. The provider atomically returns its own stable +`physical_id`, the authoritative `OperatorStatistics`, and explicit +`output_buffer_bytes`; logical edge bytes are never substituted for an +allocation. Missing evidence makes the entire query unavailable. The returned +`PhysicalDag` snapshots this evidence so costing does not re-read a live +catalog after lowering. Each lowered Scan is bound to exactly one `SourceCoverage` in the comparison scope by the existing source and canonical predicate values. The bound value @@ -348,6 +353,10 @@ choosing an arbitrary snapshot. When a predicate-bearing logical Scan expands to Scan → Filter, the synthetic Scan has its own physical ID, statistics, and buffer evidence and carries that exact coverage; the Filter has separate evidence and no source coverage. +After lowering, the multiset of distinct physical Scan coverages must consume +the comparison scope's source coverages exactly. Thus a candidate cannot omit +a source that exists in the comparison boundary, while a provider-declared +shared physical Scan is counted once. The lowering validates every physical edge before costing: @@ -374,10 +383,10 @@ The supported mappings are: | Dedup | Deduplicate | | Equi-Join | HashJoin with an evidence-selected build side | | Concat or `UNION ALL` | Concat | -| Sort | in-memory Sort | -| global Sort followed by Limit | heap TopK, with `k = offset + n` from the query IR | +| Sort with at least one ordering key | in-memory Sort | +| global non-empty-key Sort followed by Limit | heap TopK, with `k = offset + n` from the query IR | | partitioned Sort followed by Limit | Sort → Limit | -| SQLWindowFunc | Window | +| RowNumber/Rank/DenseRank SQLWindowFunc with non-empty order_by | ordered in-memory Window | | TimeShift | PassThrough | Per-entity reductions, HAVING, ordered/distribution-dependent intents such as @@ -387,15 +396,13 @@ also uses the bound left and right output schemas to prove that every equality compares one column from each side; same-side or out-of-range `ColumnId`s fail closed. -Logical identity is the address of the existing `Rc` allocation. -Repeated references therefore lower once and every parent points to the same -physical ID. The resulting node IDs are deterministic within a lowering run; -they are not persistent query identifiers. - -This generic lowering currently creates raw-query operators, all with -`ExecutionMultiplicity::PerEvaluation`, zero retained state, and a conservative -full-output edge buffer. A deployment with verified batching may construct -`PhysicalDagNode` values with smaller `output_buffer_bytes` directly. +An `Rc` address is not physical identity. Every logical occurrence +is independent unless the provider returns the same non-empty `physical_id`. +Repeated IDs deduplicate only when operator, children, coverage, statistics, +and buffer evidence are identical; conflicting reuse fails closed. This +generic lowering creates raw-query operators with +`ExecutionMultiplicity::PerEvaluation` and zero retained state. Buffer sizes +are always provider-owned physical evidence. Cross/non-equi joins, `INTERSECT`, `EXCEPT`, distinct `UNION`, PromQL range/subquery execution, vector matching, and PromQL-specific enrichment/ From 5f025fa59767bdffcb6f1a7cab6c84599aa0d766 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 09:46:40 -0600 Subject: [PATCH 32/48] fix(cost): compare semantic source coverage --- .../src/analytical_lowering.rs | 126 +++++++++++++++--- .../src/analytical_statistics.rs | 16 ++- .../analytical-resource-cost.md | 16 ++- 3 files changed, 138 insertions(+), 20 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index 7c7cc7c7..a824412b 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -61,9 +61,38 @@ where impl OperatorStatisticsProvider for std::collections::HashMap { fn statistics(&self, node_id: &str) -> Result { - self.get(node_id) - .map(|evidence| evidence.statistics.clone()) - .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(node_id.into())) + let evidence = self + .get(node_id) + .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(node_id.into()))?; + if evidence.physical_id != node_id { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "evidence map key differs from embedded physical identity", + )); + } + Ok(evidence.statistics.clone()) + } +} + +impl OperatorStatisticsProvider for PhysicalDag { + fn statistics(&self, node_id: &str) -> Result { + let evidence = self + .evidence + .get(node_id) + .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(node_id.into()))?; + let node = self.nodes.iter().find(|node| node.id == node_id).ok_or( + AnalyticalCostError::InvalidPhysicalDag("evidence has no matching physical node"), + )?; + if evidence.physical_id != node_id { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "evidence map key differs from embedded physical identity", + )); + } + if node.output_buffer_bytes != evidence.output_buffer_bytes { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "physical node buffer differs from evidence snapshot", + )); + } + Ok(evidence.statistics.clone()) } } @@ -495,22 +524,25 @@ fn validate_source_consumption( nodes: &[PhysicalDagNode], scope: &ComparisonScope, ) -> Result<(), AnalyticalCostError> { - let mut remaining = scope.sources.clone(); - for coverage in nodes + let consumed = nodes .iter() .filter(|node| matches!(node.operator, PhysicalOperator::Scan)) .filter_map(|node| node.source_coverage.as_ref()) - { - let Some(index) = remaining.iter().position(|expected| expected == coverage) else { + .collect::>(); + for coverage in &consumed { + if !scope.sources.contains(coverage) { return Err(AnalyticalCostError::InvalidPhysicalDag( - "physical scans do not consume comparison sources exactly", + "physical scan consumes a source outside the comparison scope", )); - }; - remaining.swap_remove(index); + } } - if !remaining.is_empty() { + if scope + .sources + .iter() + .any(|expected| !consumed.contains(&expected)) + { return Err(AnalyticalCostError::InvalidPhysicalDag( - "physical scans do not consume comparison sources exactly", + "physical scans omit a comparison-scope source", )); } Ok(()) @@ -751,7 +783,11 @@ fn supports_hash_aggregate( #[cfg(test)] mod tests { use super::*; - use crate::analytical_cost::{estimate_physical_dag, HashJoinBuildSide}; + use crate::analytical_cost::{ + estimate_physical_dag, estimate_physical_dag_comparison, HashJoinBuildSide, + PhysicalDagEstimateRequest, + }; + use crate::analytical_statistics::validate_comparison_scopes; use asap_types::workload::{ DataArrival, DurationMs, QueryRecurrence, QueryTimeScope, TimeSelection, TimestampMs, }; @@ -969,7 +1005,7 @@ mod tests { }, vec![], ); - let independent_scope = scope(vec![source_coverage.clone(), source_coverage.clone()]); + let independent_scope = scope(vec![source_coverage.clone()]); let mut scan_statistics = statistics(vec![edge(100, 800)], edge(100, 800)); scan_statistics.source_scan_bytes = 800; let mut join_statistics = statistics(vec![edge(100, 800), edge(100, 800)], edge(25, 400)); @@ -1008,6 +1044,54 @@ mod tests { shared_dag.nodes[1].children, vec!["shared-scan".to_owned(); 2] ); + let comparison = estimate_physical_dag_comparison( + PhysicalDagEstimateRequest { + nodes: &dag.nodes, + root: &dag.root, + scope: &independent_scope, + statistics: &dag, + }, + PhysicalDagEstimateRequest { + nodes: &shared_dag.nodes, + root: &shared_dag.root, + scope: &shared_scope, + statistics: &shared_dag, + }, + ) + .unwrap(); + assert_eq!(comparison.raw.scan_bytes, 1_600); + assert_eq!(comparison.candidate.scan_bytes, 800); + + let mut drifted_buffer = shared_dag.clone(); + drifted_buffer.nodes[0].output_buffer_bytes += 1; + assert_eq!( + estimate_physical_dag( + &drifted_buffer.nodes, + &drifted_buffer.root, + &shared_scope, + &drifted_buffer, + ), + Err(AnalyticalCostError::InvalidPhysicalDag( + "physical node buffer differs from evidence snapshot" + )) + ); + let mut drifted_identity = shared_dag.clone(); + drifted_identity + .evidence + .get_mut("shared-scan") + .unwrap() + .physical_id = "different".into(); + assert_eq!( + estimate_physical_dag( + &drifted_identity.nodes, + &drifted_identity.root, + &shared_scope, + &drifted_identity, + ), + Err(AnalyticalCostError::InvalidPhysicalDag( + "evidence map key differs from embedded physical identity" + )) + ); let conflicting_identity = |request: PhysicalNodeRequest<'_>| { let (physical_id, mut statistics) = match request.operator { @@ -1216,6 +1300,18 @@ mod tests { let dag = lower_query_physical_dag(&union, &scope, &scripted(&provided)).unwrap(); assert_eq!(dag.nodes.last().unwrap().operator, PhysicalOperator::Concat); + let mut reversed_scope = scope.clone(); + reversed_scope.sources.reverse(); + assert_eq!(validate_comparison_scopes(&scope, &reversed_scope), Ok(1)); + let mut duplicate_scope = scope.clone(); + duplicate_scope.sources.push(scope.sources[0].clone()); + assert_eq!( + duplicate_scope.validate(), + Err(AnalyticalCostError::MissingComparisonScope( + "duplicate source coverage" + )) + ); + let concat = Rc::new(QueryExpr::Concat { children: vec![scan("a"), scan("b")], }); @@ -1335,7 +1431,7 @@ mod tests { assert_eq!( lower_query_physical_dag(&root, &extra_scope, &scripted(&complete)), Err(AnalyticalCostError::InvalidPhysicalDag( - "physical scans do not consume comparison sources exactly" + "physical scans omit a comparison-scope source" )) ); } diff --git a/crates/asap-aware-mapping/src/analytical_statistics.rs b/crates/asap-aware-mapping/src/analytical_statistics.rs index 924a7404..9f9b6c1d 100644 --- a/crates/asap-aware-mapping/src/analytical_statistics.rs +++ b/crates/asap-aware-mapping/src/analytical_statistics.rs @@ -76,6 +76,13 @@ impl ComparisonScope { if self.sources.is_empty() { return Err(AnalyticalCostError::MissingComparisonScope("sources")); } + if self.sources.iter().enumerate() + .any(|(index, source)| self.sources[..index].contains(source)) + { + return Err(AnalyticalCostError::MissingComparisonScope( + "duplicate source coverage", + )); + } if self .sources .iter() @@ -110,7 +117,14 @@ pub fn validate_comparison_scopes( "time_selection", raw.time_selection == candidate.time_selection, ), - ("sources", raw.sources == candidate.sources), + ( + "sources", + raw.sources.len() == candidate.sources.len() + && raw + .sources + .iter() + .all(|source| candidate.sources.contains(source)), + ), ] { if !matches { return Err(AnalyticalCostError::ComparisonScopeMismatch(name)); diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 37b07592..8c76735d 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -353,10 +353,12 @@ choosing an arbitrary snapshot. When a predicate-bearing logical Scan expands to Scan → Filter, the synthetic Scan has its own physical ID, statistics, and buffer evidence and carries that exact coverage; the Filter has separate evidence and no source coverage. -After lowering, the multiset of distinct physical Scan coverages must consume -the comparison scope's source coverages exactly. Thus a candidate cannot omit -a source that exists in the comparison boundary, while a provider-declared -shared physical Scan is counted once. +`ComparisonScope.sources` is an order-independent set of semantic coverages; +duplicates are invalid. After lowering, every reachable physical Scan must use +a member of that set and every member must be used by at least one Scan. +Multiple independent physical Scans may use the same coverage, while a +provider-declared shared Scan uses it once, so those physical alternatives can +still be compared under the same semantic scope. The lowering validates every physical edge before costing: @@ -404,6 +406,12 @@ generic lowering creates raw-query operators with `ExecutionMultiplicity::PerEvaluation` and zero retained state. Buffer sizes are always provider-owned physical evidence. +The DAG itself implements `OperatorStatisticsProvider` over its evidence +snapshot. That boundary verifies that each map key equals the evidence's +embedded physical identity and that every node's buffer equals the provider +snapshot before returning statistics, preventing the public node and evidence +views from silently drifting apart. + Cross/non-equi joins, `INTERSECT`, `EXCEPT`, distinct `UNION`, PromQL range/subquery execution, vector matching, and PromQL-specific enrichment/ relabel/sample operators stay unavailable. Their cost requires a physical From fbc62cff540b553a66d8c060e3763a0838039256 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 20:37:24 -0600 Subject: [PATCH 33/48] fix(cost): bind limit consumption during lowering --- .../asap-aware-mapping/src/analytical_cost.rs | 2 -- .../src/analytical_lowering.rs | 34 +++++++++++++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 1e9f0d4b..aade1b73 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -796,8 +796,6 @@ pub enum AnalyticalCostError { }, #[error("invalid physical DAG: {0}")] InvalidPhysicalDag(&'static str), - #[error("inconsistent physical operator statistics: {0}")] - InconsistentOperatorStatistics(&'static str), } fn checked_bytes(parts: &[u64]) -> Result { diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index a824412b..11d68b76 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -373,6 +373,7 @@ pub fn lower_query_physical_dag( )?; require_operator_statistics(PhysicalOperator::Limit, statistics)?; require_limit_cardinality(*n, *offset, statistics)?; + require_limit_consumption(*n, *offset, statistics)?; self.push(evidence, PhysicalOperator::Limit, children, None) } QueryExpr::SQLWindowFunc { @@ -682,6 +683,28 @@ fn require_limit_cardinality( Ok(()) } +fn require_limit_consumption( + n: usize, + offset: usize, + statistics: &OperatorStatistics, +) -> Result<(), AnalyticalCostError> { + let n = u64::try_from(n).map_err(|_| AnalyticalCostError::Overflow)?; + let offset = u64::try_from(offset).map_err(|_| AnalyticalCostError::Overflow)?; + let expected_consumed = if n == 0 { + 0 + } else { + statistics.inputs[0] + .rows + .min(offset.checked_add(n).ok_or(AnalyticalCostError::Overflow)?) + }; + if statistics.limit_rows_consumed != Some(expected_consumed) { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "limit rows consumed do not match n and offset", + )); + } + Ok(()) +} + fn bind_scan_coverage( node_id: &str, source: &asap_types::pre_asap::Source, @@ -806,6 +829,7 @@ mod tests { key_bytes: None, aggregate_value_bytes: None, k: None, + limit_rows_consumed: None, hash_join_build_side: None, } } @@ -1224,7 +1248,10 @@ mod tests { ), ( "query-1".into(), - evidence(statistics(vec![edge(500, 6_000)], edge(20, 240))), + evidence(OperatorStatistics { + limit_rows_consumed: Some(20), + ..statistics(vec![edge(500, 6_000)], edge(20, 240)) + }), ), ( "query-0".into(), @@ -1475,7 +1502,10 @@ mod tests { ), ( "query-0".into(), - evidence(statistics(vec![edge(0, 0)], edge(0, 0))), + evidence(OperatorStatistics { + limit_rows_consumed: Some(0), + ..statistics(vec![edge(0, 0)], edge(0, 0)) + }), ), ]); let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)).unwrap(); From 21aad886dc581c01ea62103547f4f34562ed9ba3 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 10:07:11 -0600 Subject: [PATCH 34/48] fix(cost): preserve valid window and offset cardinalities --- crates/asap-aware-mapping/src/analytical_cost.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index aade1b73..46ddb432 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -678,7 +678,6 @@ fn validate_operator_semantics( } ( PhysicalOperator::InMemoryComparisonSort - | PhysicalOperator::InMemoryOrderedWindow | PhysicalOperator::PassThrough, _, ) => { @@ -686,6 +685,11 @@ fn validate_operator_semantics( return inconsistent("cardinality-preserving operator changes its edge"); } } + (PhysicalOperator::InMemoryOrderedWindow, _) => { + if input.rows != output.rows { + return inconsistent("Window changes row cardinality"); + } + } (PhysicalOperator::Concat, OperatorStatistics::Concat { inputs, .. }) => { let total = inputs From 66387f4b49feaf3c26b0da061e54707556a4d10a Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 12:38:18 -0600 Subject: [PATCH 35/48] refactor(cost): lower into typed physical evidence --- .../asap-aware-mapping/src/analytical_cost.rs | 8 +- .../src/analytical_lowering.rs | 444 +++++++++--------- 2 files changed, 216 insertions(+), 236 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 46ddb432..a4b07f2c 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -615,7 +615,7 @@ pub fn estimate_operator( } } -fn validate_operator_semantics( +pub(crate) fn validate_operator_semantics( operator: PhysicalOperator, statistics: &OperatorStatistics, ) -> Result<(), AnalyticalCostError> { @@ -676,11 +676,7 @@ fn validate_operator_semantics( return inconsistent("deduplicated output differs from distinct key cardinality"); } } - ( - PhysicalOperator::InMemoryComparisonSort - | PhysicalOperator::PassThrough, - _, - ) => { + (PhysicalOperator::InMemoryComparisonSort | PhysicalOperator::PassThrough, _) => { if input != output { return inconsistent("cardinality-preserving operator changes its edge"); } diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index 11d68b76..a6c5528a 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -5,7 +5,8 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; use crate::analytical_cost::{ - AnalyticalCostError, ExecutionMultiplicity, PhysicalDagNode, PhysicalOperator, + validate_operator_semantics, AnalyticalCostError, ExecutionMultiplicity, HashJoinBuildSide, + PhysicalDagNode, PhysicalOperator, }; use crate::analytical_statistics::{ ComparisonScope, EdgeStatistics, OperatorStatistics, OperatorStatisticsProvider, SourceCoverage, @@ -302,13 +303,18 @@ pub fn lower_query_physical_dag( self.lower_unary(query, occurrence, PhysicalOperator::HashAggregate, child) } QueryExpr::Dedup { child, .. } => { - self.lower_unary(query, occurrence, PhysicalOperator::Deduplicate, child) + self.lower_unary(query, occurrence, PhysicalOperator::HashDeduplicate, child) } QueryExpr::Sort { keys, child, .. } => { if keys.is_empty() { return Err(AnalyticalCostError::UnsupportedQueryOperator); } - self.lower_unary(query, occurrence, PhysicalOperator::Sort, child) + self.lower_unary( + query, + occurrence, + PhysicalOperator::InMemoryComparisonSort, + child, + ) } QueryExpr::Limit { n, offset, child } => { if let QueryExpr::Sort { @@ -321,14 +327,13 @@ pub fn lower_query_physical_dag( if !keys.is_empty() && partition_by == &GroupKeys::none() { let child_id = self.lower(sorted_child)?; let children = vec![child_id.clone()]; - let evidence = self.resolve( - query, - PhysicalOperator::TopK, - occurrence, - false, - &children, - None, - )?; + let limit = + u64::try_from(*n).map_err(|_| AnalyticalCostError::Overflow)?; + let offset = u64::try_from(*offset) + .map_err(|_| AnalyticalCostError::Overflow)?; + let operator = PhysicalOperator::TopK { limit, offset }; + let evidence = + self.resolve(query, operator, occurrence, false, &children, None)?; let statistics = &evidence.statistics; let child_statistics = self.node_statistics(&child_id)?; require_unary_edge( @@ -337,32 +342,25 @@ pub fn lower_query_physical_dag( &child_id, child_statistics, )?; - let bound = n - .checked_add(*offset) - .and_then(|value| u64::try_from(value).ok()) + let bound = limit + .checked_add(offset) .ok_or(AnalyticalCostError::Overflow)?; if bound == 0 { return Err(AnalyticalCostError::MissingOrZero("topk_k")); } - if statistics.k != Some(bound) { - return Err(AnalyticalCostError::InconsistentOperatorStatistics( - "Top-K statistics disagree with LIMIT n + offset", - )); - } - require_limit_cardinality(*n, *offset, statistics)?; - return self.push(evidence, PhysicalOperator::TopK, children, None); + require_operator_statistics(operator, statistics)?; + return self.push(evidence, operator, children, None); } } let child_id = self.lower(child)?; let children = vec![child_id.clone()]; - let evidence = self.resolve( - query, - PhysicalOperator::Limit, - occurrence, - false, - &children, - None, - )?; + let operator = PhysicalOperator::Limit { + limit: u64::try_from(*n).map_err(|_| AnalyticalCostError::Overflow)?, + offset: u64::try_from(*offset) + .map_err(|_| AnalyticalCostError::Overflow)?, + }; + let evidence = + self.resolve(query, operator, occurrence, false, &children, None)?; let statistics = &evidence.statistics; let child_statistics = self.node_statistics(&child_id)?; require_unary_edge( @@ -371,10 +369,8 @@ pub fn lower_query_physical_dag( &child_id, child_statistics, )?; - require_operator_statistics(PhysicalOperator::Limit, statistics)?; - require_limit_cardinality(*n, *offset, statistics)?; - require_limit_consumption(*n, *offset, statistics)?; - self.push(evidence, PhysicalOperator::Limit, children, None) + require_operator_statistics(operator, statistics)?; + self.push(evidence, operator, children, None) } QueryExpr::SQLWindowFunc { func, @@ -392,7 +388,12 @@ pub fn lower_query_physical_dag( { return Err(AnalyticalCostError::UnsupportedQueryOperator); } - self.lower_unary(query, occurrence, PhysicalOperator::Window, child) + self.lower_unary( + query, + occurrence, + PhysicalOperator::InMemoryOrderedWindow, + child, + ) } QueryExpr::TimeShift { child, .. } => { self.lower_unary(query, occurrence, PhysicalOperator::PassThrough, child) @@ -428,27 +429,28 @@ pub fn lower_query_physical_dag( let left_id = self.lower(left)?; let right_id = self.lower(right)?; let children = vec![left_id.clone(), right_id.clone()]; - let evidence = self.resolve( - query, - PhysicalOperator::HashJoin, - occurrence, - false, - &children, - None, - )?; - let statistics = &evidence.statistics; - require_statistics_shape(&evidence.physical_id, statistics, 2)?; let left_statistics = self.node_statistics(&left_id)?; let right_statistics = self.node_statistics(&right_id)?; - if statistics.inputs[0] != left_statistics.output - || statistics.inputs[1] != right_statistics.output + let build_side = + if left_statistics.output().bytes <= right_statistics.output().bytes { + HashJoinBuildSide::Left + } else { + HashJoinBuildSide::Right + }; + let operator = PhysicalOperator::HashJoin { build_side }; + let evidence = + self.resolve(query, operator, occurrence, false, &children, None)?; + let statistics = &evidence.statistics; + require_statistics_shape(&evidence.physical_id, statistics, 2)?; + if statistics.input(0) != Some(left_statistics.output()) + || statistics.input(1) != Some(right_statistics.output()) { return Err(AnalyticalCostError::InconsistentOperatorStatistics( "join inputs do not match child outputs", )); } - require_operator_statistics(PhysicalOperator::HashJoin, statistics)?; - self.push(evidence, PhysicalOperator::HashJoin, children, None) + require_operator_statistics(operator, statistics)?; + self.push(evidence, operator, children, None) } _ => Err(AnalyticalCostError::UnsupportedQueryOperator), } @@ -479,7 +481,7 @@ pub fn lower_query_physical_dag( (0_u64, 0_u64), |(rows, bytes), (index, child)| { let child_statistics = self.node_statistics(child)?; - if statistics.inputs[index] != child_statistics.output { + if statistics.input(index) != Some(child_statistics.output()) { return Err(AnalyticalCostError::ConflictingEdgeStatistics { parent: evidence.physical_id.clone(), child: child.clone(), @@ -487,15 +489,15 @@ pub fn lower_query_physical_dag( }); } Ok::<_, AnalyticalCostError>(( - rows.checked_add(child_statistics.output.rows) + rows.checked_add(child_statistics.output().rows) .ok_or(AnalyticalCostError::Overflow)?, bytes - .checked_add(child_statistics.output.bytes) + .checked_add(child_statistics.output().bytes) .ok_or(AnalyticalCostError::Overflow)?, )) }, )?; - if statistics.output != (EdgeStatistics { rows, bytes }) { + if statistics.output() != (EdgeStatistics { rows, bytes }) { return Err(AnalyticalCostError::InconsistentOperatorStatistics( "concat statistics do not equal the sum of child outputs", )); @@ -554,16 +556,15 @@ fn require_statistics_shape( statistics: &OperatorStatistics, input_count: usize, ) -> Result<(), AnalyticalCostError> { - if statistics.inputs.len() != input_count { + if statistics.input_count() != input_count { return Err(AnalyticalCostError::InvalidOperatorStatistics { node: node.into(), reason: "wrong input-edge count", }); } - if statistics - .inputs - .iter() - .chain(std::iter::once(&statistics.output)) + if (0..statistics.input_count()) + .filter_map(|index| statistics.input(index)) + .chain(std::iter::once(statistics.output())) .any(|edge| !edge.is_consistent()) { return Err(AnalyticalCostError::InvalidOperatorStatistics { @@ -575,7 +576,7 @@ fn require_statistics_shape( } fn require_scan_edges_equal(statistics: &OperatorStatistics) -> Result<(), AnalyticalCostError> { - if statistics.inputs[0] != statistics.output { + if statistics.input(0) != Some(statistics.output()) { return Err(AnalyticalCostError::InconsistentOperatorStatistics( "Scan external input edge does not match its output edge", )); @@ -590,7 +591,7 @@ fn require_unary_edge( child: &OperatorStatistics, ) -> Result<(), AnalyticalCostError> { require_statistics_shape(node, statistics, 1)?; - if statistics.inputs[0] != child.output { + if statistics.input(0) != Some(child.output()) { return Err(AnalyticalCostError::ConflictingEdgeStatistics { parent: node.into(), child: child_id.into(), @@ -604,105 +605,7 @@ fn require_operator_statistics( operator: PhysicalOperator, statistics: &OperatorStatistics, ) -> Result<(), AnalyticalCostError> { - let invalid = |reason| Err(AnalyticalCostError::InconsistentOperatorStatistics(reason)); - if !matches!(operator, PhysicalOperator::Scan) && statistics.source_scan_bytes != 0 { - return invalid("only Scan may charge source bytes"); - } - let input = statistics.inputs.first().copied().ok_or( - AnalyticalCostError::InconsistentOperatorStatistics("operator input is missing"), - )?; - let output = statistics.output; - match operator { - PhysicalOperator::Filter => { - if output.rows > input.rows || output.bytes > input.bytes { - return invalid("filter output expands its input"); - } - } - PhysicalOperator::Project => { - if output.rows != input.rows { - return invalid("projection changes row cardinality"); - } - } - PhysicalOperator::HashAggregate => { - let groups = statistics - .group_count - .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - if groups == 0 && (input.rows != 0 || output.rows != 0) { - return invalid("zero groups require an empty grouped input and output"); - } - if output.rows > groups { - return invalid("aggregate output exceeds group cardinality"); - } - } - PhysicalOperator::Deduplicate => { - let groups = statistics - .group_count - .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - if output.rows != groups || output.rows > input.rows { - return invalid("deduplicate output does not equal distinct cardinality"); - } - } - PhysicalOperator::Sort => { - if output != input { - return invalid("sort changes its input cardinality or width"); - } - } - PhysicalOperator::TopK | PhysicalOperator::Limit => { - if output.rows > input.rows { - return invalid("bounded output exceeds its input cardinality"); - } - } - PhysicalOperator::Window => { - if output.rows != input.rows { - return invalid("SQL window changes row cardinality"); - } - } - PhysicalOperator::PassThrough => { - if output != input { - return invalid("pass-through wrapper changes its edge statistics"); - } - } - PhysicalOperator::Scan | PhysicalOperator::HashJoin | PhysicalOperator::Concat => {} - } - Ok(()) -} - -fn require_limit_cardinality( - n: usize, - offset: usize, - statistics: &OperatorStatistics, -) -> Result<(), AnalyticalCostError> { - let n = u64::try_from(n).map_err(|_| AnalyticalCostError::Overflow)?; - let offset = u64::try_from(offset).map_err(|_| AnalyticalCostError::Overflow)?; - let expected = statistics.inputs[0].rows.saturating_sub(offset).min(n); - if statistics.output.rows != expected { - return Err(AnalyticalCostError::InconsistentOperatorStatistics( - "limit output does not match n and offset", - )); - } - Ok(()) -} - -fn require_limit_consumption( - n: usize, - offset: usize, - statistics: &OperatorStatistics, -) -> Result<(), AnalyticalCostError> { - let n = u64::try_from(n).map_err(|_| AnalyticalCostError::Overflow)?; - let offset = u64::try_from(offset).map_err(|_| AnalyticalCostError::Overflow)?; - let expected_consumed = if n == 0 { - 0 - } else { - statistics.inputs[0] - .rows - .min(offset.checked_add(n).ok_or(AnalyticalCostError::Overflow)?) - }; - if statistics.limit_rows_consumed != Some(expected_consumed) { - return Err(AnalyticalCostError::InconsistentOperatorStatistics( - "limit rows consumed do not match n and offset", - )); - } - Ok(()) + validate_operator_semantics(operator, statistics) } fn bind_scan_coverage( @@ -807,10 +710,11 @@ fn supports_hash_aggregate( mod tests { use super::*; use crate::analytical_cost::{ - estimate_physical_dag, estimate_physical_dag_comparison, HashJoinBuildSide, - PhysicalDagEstimateRequest, + estimate_physical_dag, estimate_physical_dag_comparison, PhysicalDagEstimateRequest, + }; + use crate::analytical_statistics::{ + validate_comparison_scopes, BinaryEdgeStatistics, UnaryEdgeStatistics, }; - use crate::analytical_statistics::validate_comparison_scopes; use asap_types::workload::{ DataArrival, DurationMs, QueryRecurrence, QueryTimeScope, TimeSelection, TimestampMs, }; @@ -820,24 +724,43 @@ mod tests { EdgeStatistics { rows, bytes } } - fn statistics(inputs: Vec, output: EdgeStatistics) -> OperatorStatistics { - OperatorStatistics { - source_scan_bytes: 0, - inputs, - output, - group_count: None, - key_bytes: None, - aggregate_value_bytes: None, - k: None, - limit_rows_consumed: None, - hash_join_build_side: None, + fn unary_edges(input: EdgeStatistics, output: EdgeStatistics) -> UnaryEdgeStatistics { + UnaryEdgeStatistics { input, output } + } + + fn scan_stats(edge: EdgeStatistics, source_read_bytes: u64) -> OperatorStatistics { + OperatorStatistics::Scan { + edges: unary_edges(edge, edge), + source_read_bytes, + } + } + + fn unary_statistics( + operator: PhysicalOperator, + input: EdgeStatistics, + output: EdgeStatistics, + ) -> OperatorStatistics { + let edges = unary_edges(input, output); + match operator { + PhysicalOperator::Filter => OperatorStatistics::Filter { edges }, + PhysicalOperator::Project => OperatorStatistics::Project { edges }, + PhysicalOperator::InMemoryComparisonSort => { + OperatorStatistics::InMemoryComparisonSort { edges } + } + PhysicalOperator::TopK { .. } => OperatorStatistics::TopK { edges }, + PhysicalOperator::InMemoryOrderedWindow => { + OperatorStatistics::InMemoryOrderedWindow { edges } + } + PhysicalOperator::Limit { .. } => OperatorStatistics::Limit { edges }, + PhysicalOperator::PassThrough => OperatorStatistics::PassThrough { edges }, + _ => panic!("test helper requires a stateless unary operator"), } } fn evidence(statistics: OperatorStatistics) -> PhysicalNodeEvidence { PhysicalNodeEvidence { physical_id: String::new(), - output_buffer_bytes: statistics.output.bytes.min(1_024), + output_buffer_bytes: statistics.output().bytes.min(1_024), statistics, } } @@ -939,19 +862,30 @@ mod tests { ))], ); let scope = scope(vec![scan_coverage]); - let mut aggregate_statistics = statistics(vec![edge(400, 25_600)], edge(100, 4_000)); - aggregate_statistics.group_count = Some(100); - aggregate_statistics.key_bytes = Some(16); - aggregate_statistics.aggregate_value_bytes = Some(8); - let mut topk_statistics = statistics(vec![edge(100, 4_000)], edge(10, 400)); - topk_statistics.k = Some(15); - let mut raw_scan = statistics(vec![edge(1_000, 64_000)], edge(1_000, 64_000)); - raw_scan.source_scan_bytes = 64_000; + let aggregate_statistics = OperatorStatistics::HashAggregate { + edges: unary_edges(edge(400, 25_600), edge(100, 4_000)), + group_count: 100, + key_bytes: 16, + accumulator_bytes_per_group: 8, + }; + let topk_statistics = unary_statistics( + PhysicalOperator::TopK { + limit: 10, + offset: 5, + }, + edge(100, 4_000), + edge(10, 400), + ); + let raw_scan = scan_stats(edge(1_000, 64_000), 64_000); let provided = HashMap::from([ ("query-2-scan".into(), evidence(raw_scan)), ( "query-2".into(), - evidence(statistics(vec![edge(1_000, 64_000)], edge(400, 25_600))), + evidence(unary_statistics( + PhysicalOperator::Filter, + edge(1_000, 64_000), + edge(400, 25_600), + )), ), ("query-1".into(), evidence(aggregate_statistics)), ("query-0".into(), evidence(topk_statistics)), @@ -967,12 +901,18 @@ mod tests { PhysicalOperator::Scan, PhysicalOperator::Filter, PhysicalOperator::HashAggregate, - PhysicalOperator::TopK, + PhysicalOperator::TopK { + limit: 10, + offset: 5, + }, ] ); let topk = dag.nodes.last().unwrap(); assert_eq!(topk.children, vec![dag.nodes[2].id.clone()]); - assert_eq!(provided[&topk.id].statistics.k, Some(15)); + assert!(matches!( + provided[&topk.id].statistics, + OperatorStatistics::TopK { .. } + )); let physical_scan = &dag.nodes[0]; assert_eq!(physical_scan.id, "query-2-scan"); assert_eq!( @@ -982,7 +922,7 @@ mod tests { assert_eq!(physical_scan.output_buffer_bytes, 1_024); assert_ne!( physical_scan.output_buffer_bytes, - provided[&physical_scan.id].statistics.output.bytes + provided[&physical_scan.id].statistics.output().bytes ); assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).is_ok()); @@ -990,8 +930,10 @@ mod tests { inconsistent_scan .get_mut("query-2-scan") .unwrap() - .statistics - .output = edge(999, 63_936); + .statistics = OperatorStatistics::Scan { + edges: unary_edges(edge(1_000, 64_000), edge(999, 63_936)), + source_read_bytes: 64_000, + }; assert_eq!( lower_query_physical_dag(&root, &scope, &scripted(&inconsistent_scan)), Err(AnalyticalCostError::InconsistentOperatorStatistics( @@ -1030,10 +972,13 @@ mod tests { vec![], ); let independent_scope = scope(vec![source_coverage.clone()]); - let mut scan_statistics = statistics(vec![edge(100, 800)], edge(100, 800)); - scan_statistics.source_scan_bytes = 800; - let mut join_statistics = statistics(vec![edge(100, 800), edge(100, 800)], edge(25, 400)); - join_statistics.hash_join_build_side = Some(HashJoinBuildSide::Right); + let scan_statistics = scan_stats(edge(100, 800), 800); + let join_statistics = OperatorStatistics::HashJoin { + edges: BinaryEdgeStatistics { + inputs: [edge(100, 800), edge(100, 800)], + output: edge(25, 400), + }, + }; let provided = HashMap::from([ ("query-1".into(), evidence(scan_statistics.clone())), ("query-2".into(), evidence(scan_statistics.clone())), @@ -1052,12 +997,12 @@ mod tests { let shared_provider = |request: PhysicalNodeRequest<'_>| { let (physical_id, statistics) = match request.operator { PhysicalOperator::Scan => ("shared-scan", scan_statistics.clone()), - PhysicalOperator::HashJoin => ("join", join_statistics.clone()), + PhysicalOperator::HashJoin { .. } => ("join", join_statistics.clone()), _ => return Err(AnalyticalCostError::UnsupportedQueryOperator), }; Ok(PhysicalNodeEvidence { physical_id: physical_id.into(), - output_buffer_bytes: statistics.output.bytes.min(1_024), + output_buffer_bytes: statistics.output().bytes.min(1_024), statistics, }) }; @@ -1120,15 +1065,15 @@ mod tests { let conflicting_identity = |request: PhysicalNodeRequest<'_>| { let (physical_id, mut statistics) = match request.operator { PhysicalOperator::Scan => ("shared-scan", scan_statistics.clone()), - PhysicalOperator::HashJoin => ("join", join_statistics.clone()), + PhysicalOperator::HashJoin { .. } => ("join", join_statistics.clone()), _ => return Err(AnalyticalCostError::UnsupportedQueryOperator), }; if request.operator == PhysicalOperator::Scan && request.occurrence == 2 { - statistics.source_scan_bytes += 1; + statistics = scan_stats(edge(100, 800), 801); } Ok(PhysicalNodeEvidence { physical_id: physical_id.into(), - output_buffer_bytes: statistics.output.bytes.min(1_024), + output_buffer_bytes: statistics.output().bytes.min(1_024), statistics, }) }; @@ -1222,40 +1167,65 @@ mod tests { vec![], ); let scope = scope(vec![source_coverage]); - let mut scan_statistics = statistics(vec![edge(1_000, 8_000)], edge(1_000, 8_000)); - scan_statistics.source_scan_bytes = 8_000; - let mut dedup_statistics = statistics(vec![edge(800, 3_200)], edge(500, 2_000)); - dedup_statistics.group_count = Some(500); - dedup_statistics.key_bytes = Some(8); + let scan_statistics = scan_stats(edge(1_000, 8_000), 8_000); + let dedup_statistics = OperatorStatistics::HashDeduplicate { + edges: unary_edges(edge(800, 3_200), edge(500, 2_000)), + distinct_key_count: 500, + key_bytes: 8, + }; let provided = HashMap::from([ ("query-7".into(), evidence(scan_statistics)), ( "query-6".into(), - evidence(statistics(vec![edge(1_000, 8_000)], edge(800, 6_400))), + evidence(unary_statistics( + PhysicalOperator::Filter, + edge(1_000, 8_000), + edge(800, 6_400), + )), ), ( "query-5".into(), - evidence(statistics(vec![edge(800, 6_400)], edge(800, 3_200))), + evidence(unary_statistics( + PhysicalOperator::Project, + edge(800, 6_400), + edge(800, 3_200), + )), ), ("query-4".into(), evidence(dedup_statistics)), ( "query-3".into(), - evidence(statistics(vec![edge(500, 2_000)], edge(500, 6_000))), + evidence(unary_statistics( + PhysicalOperator::InMemoryOrderedWindow, + edge(500, 2_000), + edge(500, 6_000), + )), ), ( "query-2".into(), - evidence(statistics(vec![edge(500, 6_000)], edge(500, 6_000))), + evidence(unary_statistics( + PhysicalOperator::InMemoryComparisonSort, + edge(500, 6_000), + edge(500, 6_000), + )), ), ( "query-1".into(), - evidence(OperatorStatistics { - limit_rows_consumed: Some(20), - ..statistics(vec![edge(500, 6_000)], edge(20, 240)) - }), + evidence(unary_statistics( + PhysicalOperator::Limit { + limit: 20, + offset: 0, + }, + edge(500, 6_000), + edge(20, 240), + )), ), ( "query-0".into(), - evidence(statistics(vec![edge(20, 240)], edge(20, 240))), + evidence(unary_statistics( + PhysicalOperator::PassThrough, + edge(20, 240), + edge(20, 240), + )), ), ]); let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)).unwrap(); @@ -1269,10 +1239,13 @@ mod tests { PhysicalOperator::Scan, PhysicalOperator::Filter, PhysicalOperator::Project, - PhysicalOperator::Deduplicate, - PhysicalOperator::Window, - PhysicalOperator::Sort, - PhysicalOperator::Limit, + PhysicalOperator::HashDeduplicate, + PhysicalOperator::InMemoryOrderedWindow, + PhysicalOperator::InMemoryComparisonSort, + PhysicalOperator::Limit { + limit: 20, + offset: 0, + }, PhysicalOperator::PassThrough, ] ); @@ -1312,16 +1285,17 @@ mod tests { vec![], ), ]); - let mut left_statistics = statistics(vec![edge(10, 80)], edge(10, 80)); - left_statistics.source_scan_bytes = 80; - let mut right_statistics = statistics(vec![edge(20, 160)], edge(20, 160)); - right_statistics.source_scan_bytes = 160; + let left_statistics = scan_stats(edge(10, 80), 80); + let right_statistics = scan_stats(edge(20, 160), 160); let provided = HashMap::from([ ("query-1".into(), evidence(left_statistics)), ("query-2".into(), evidence(right_statistics)), ( "query-0".into(), - evidence(statistics(vec![edge(10, 80), edge(20, 160)], edge(30, 240))), + evidence(OperatorStatistics::Concat { + inputs: vec![edge(10, 80), edge(20, 160)], + output: edge(30, 240), + }), ), ]); let dag = lower_query_physical_dag(&union, &scope, &scripted(&provided)).unwrap(); @@ -1402,13 +1376,16 @@ mod tests { lower_query_physical_dag(&root, &comparison_scope, &MissingBuffer), Err(AnalyticalCostError::MissingOrStale("output_buffer_bytes")) ); - let mut scan_statistics = statistics(vec![edge(100, 800)], edge(100, 800)); - scan_statistics.source_scan_bytes = 800; + let scan_statistics = scan_stats(edge(100, 800), 800); let conflicting = HashMap::from([ ("query-1".into(), evidence(scan_statistics)), ( "query-0".into(), - evidence(statistics(vec![edge(99, 792)], edge(99, 396))), + evidence(unary_statistics( + PhysicalOperator::Project, + edge(99, 792), + edge(99, 396), + )), ), ]); assert_eq!( @@ -1445,7 +1422,7 @@ mod tests { let mut complete = conflicting.clone(); complete.get_mut("query-0").unwrap().statistics = - statistics(vec![edge(100, 800)], edge(100, 400)); + unary_statistics(PhysicalOperator::Project, edge(100, 800), edge(100, 400)); let extra_scope = scope(vec![ comparison_scope.sources[0].clone(), coverage( @@ -1492,20 +1469,27 @@ mod tests { }, vec![], )]); - let mut scan_statistics = statistics(vec![edge(100, 800)], edge(100, 800)); - scan_statistics.source_scan_bytes = 800; + let scan_statistics = scan_stats(edge(100, 800), 800); let provided = HashMap::from([ ("query-2".into(), evidence(scan_statistics)), ( "query-1".into(), - evidence(statistics(vec![edge(100, 800)], edge(0, 0))), + evidence(unary_statistics( + PhysicalOperator::Filter, + edge(100, 800), + edge(0, 0), + )), ), ( "query-0".into(), - evidence(OperatorStatistics { - limit_rows_consumed: Some(0), - ..statistics(vec![edge(0, 0)], edge(0, 0)) - }), + evidence(unary_statistics( + PhysicalOperator::Limit { + limit: 10, + offset: 0, + }, + edge(0, 0), + edge(0, 0), + )), ), ]); let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)).unwrap(); From 228ea1e5ebced05ee5915b16b607ee5be08f933a Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 12:47:03 -0600 Subject: [PATCH 36/48] refactor(cost): name physical source snapshots explicitly --- crates/asap-aware-mapping/src/analytical_lowering.rs | 4 ++-- crates/asap-aware-mapping/src/analytical_statistics.rs | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index a6c5528a..06389b4c 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -808,7 +808,7 @@ mod tests { ) -> SourceCoverage { SourceCoverage { source, - snapshot_id: "snapshot-1".into(), + source_snapshot_id: "snapshot-1".into(), predicates, } } @@ -1411,7 +1411,7 @@ mod tests { ); let mut second_snapshot = comparison_scope.sources[0].clone(); - second_snapshot.snapshot_id = "snapshot-2".into(); + second_snapshot.source_snapshot_id = "snapshot-2".into(); let ambiguous_scope = scope(vec![comparison_scope.sources[0].clone(), second_snapshot]); assert_eq!( lower_query_physical_dag(&root, &ambiguous_scope, &scripted(&conflicting)), diff --git a/crates/asap-aware-mapping/src/analytical_statistics.rs b/crates/asap-aware-mapping/src/analytical_statistics.rs index 9f9b6c1d..a15cbeda 100644 --- a/crates/asap-aware-mapping/src/analytical_statistics.rs +++ b/crates/asap-aware-mapping/src/analytical_statistics.rs @@ -76,7 +76,10 @@ impl ComparisonScope { if self.sources.is_empty() { return Err(AnalyticalCostError::MissingComparisonScope("sources")); } - if self.sources.iter().enumerate() + if self + .sources + .iter() + .enumerate() .any(|(index, source)| self.sources[..index].contains(source)) { return Err(AnalyticalCostError::MissingComparisonScope( From 18b147a77b5c1613fb55e1555b2150d4ce38bbb4 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 12:49:44 -0600 Subject: [PATCH 37/48] docs(cost): define physical plan integration boundary --- docs/design_docs/asap-aware-mapping/README.md | 3 + .../analytical-resource-cost.md | 4 + .../physical-plan-integration.md | 163 ++++++++++++++++++ 3 files changed, 170 insertions(+) create mode 100644 docs/design_docs/asap-aware-mapping/physical-plan-integration.md diff --git a/docs/design_docs/asap-aware-mapping/README.md b/docs/design_docs/asap-aware-mapping/README.md index 018b9fb5..fa3d186f 100644 --- a/docs/design_docs/asap-aware-mapping/README.md +++ b/docs/design_docs/asap-aware-mapping/README.md @@ -94,6 +94,9 @@ The design is split into focused documents: guarantee IR, sketch contracts, composition rules, target checking, and fail-closed boundaries. - [Analytical resource cost](analytical-resource-cost.md) defines CPU, retained-memory, and scan-I/O formulas, calibration, planner ranking, and fail-closed behavior. +- [Physical plan integration](physical-plan-integration.md) defines how pre-ASAP and + post-ASAP logical plans lower into the physical operator DAG consumed by statistics + resolution and analytical costing. - [Query workloads, data workloads, and summary lifecycle maintenance](workload-demand-and-summary-lifecycle.md) separates query-workload properties from data-workload properties and defines ephemeral, prepared, shared, and continuously maintained summary-state alternatives. diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 3408c258..42106ae7 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -72,6 +72,10 @@ unavailable. ### Operator vocabulary and source of truth +The complete logical-to-physical boundary is specified in +[Physical plan integration](physical-plan-integration.md). This section +summarizes the part required by the resource estimator. + `PhysicalOperator` is the source of truth for the cost model's operator vocabulary. `OperatorStatistics` is paired one-to-one with that enum: each supported physical algorithm has one evidence shape containing exactly the diff --git a/docs/design_docs/asap-aware-mapping/physical-plan-integration.md b/docs/design_docs/asap-aware-mapping/physical-plan-integration.md new file mode 100644 index 00000000..843d79ed --- /dev/null +++ b/docs/design_docs/asap-aware-mapping/physical-plan-integration.md @@ -0,0 +1,163 @@ +# Physical plan integration + +## Purpose + +This document defines the boundary between ASAPPlanner's logical plans, +physical lowering, statistics resolution, and analytical resource costing. +It answers which representation is authoritative at each stage and prevents +the cost model from being coupled directly to either logical IR. + +The integration pipeline is: + +```text +pre-ASAP QueryExpr ─┐ + ├─ physical lowering ─> PhysicalOperator DAG +post-ASAP SummaryExpr┘ │ + v + OperatorStatistics + │ + v + ResourceEstimate +``` + +The arrows are boundaries, not casts. A logical node does not necessarily +become one physical node, and a physical operator is not merely a renamed +logical variant. + +## Sources of truth + +Each representation is authoritative for a different concern: + +| Representation | Authoritative concern | +|---|---| +| `QueryExpr` | Original exact query semantics: sources, predicates, relational and PromQL operations, and output shape. | +| `SummaryExpr` | Logical summary semantics: selected family, grouping strategy, summary composition, and summary readout. | +| `PhysicalOperator` DAG | Selected executable algorithms, their configuration, physical identity, edges, and execution multiplicity. | +| `OperatorStatistics` | Workload-dependent evidence required by each selected physical operator's resource formula. | +| `ResourceEstimate` | Estimated CPU operations, peak live memory, and physical source/disk reads over one comparison scope. | + +`PhysicalOperator` is therefore the source of truth for the operator vocabulary +consumed by analytical costing. `OperatorStatistics` corresponds one-to-one +with that vocabulary. It must not independently invent operator kinds or copy +all variants from either logical IR. + +The canonical physical-plan types should live at a neutral boundary shared by +lowering, costing, explanation, and downstream compilation. Their conceptual +ownership is not the cost formula implementation, even if an intermediate +implementation keeps the Rust type in the cost module while the interface is +being established. + +## Why logical and physical variants are not one-to-one + +One logical operation may choose between algorithms or expand into a physical +sub-DAG. Conversely, one physical operator may implement nodes originating +from either logical IR. + +Examples include: + +- `Sort` followed by `Limit` may lower to an in-memory comparison sort plus a + limit, or to one heap Top-K operator configured by limit and offset. +- `Join` may lower to a hash join with an explicit build side or to another + supported join algorithm. +- `SummaryAgg` may lower to an exact accumulator build, CMS build, KLL build, + or another physical summary algorithm selected by the candidate. +- `SummaryEstimate` must lower to a readout operator compatible with the + concrete summary state it consumes. +- shared logical sub-DAGs become shared physical nodes only when they refer to + the same physical identity and compatible evidence. + +For this reason, aligning `OperatorStatistics` directly with `QueryExpr` would +lose post-ASAP summary implementations, while aligning it directly with +`SummaryExpr` would lose raw query operators and physical algorithm choices. + +## Lowering obligations + +Physical lowering is complete only when it recursively lowers the entire +selected candidate DAG. It must: + +1. preserve the semantics and source coverage of the logical candidate; +2. select an explicit physical algorithm for every logical operation; +3. carry algorithm configuration on the physical operator rather than in a + generic statistics record; +4. connect every physical edge and preserve child order for non-commutative + operators; +5. assign stable physical identities so shared sub-DAGs can be counted once; +6. assign execution multiplicity and retained/transient state ownership; +7. reject the complete candidate if any logical operation has no supported + physical lowering. + +Keeping an unsupported logical node outside the physical DAG and costing only +its modeled descendants is invalid because it undercounts the candidate. + +### Pre-ASAP lowering + +`KeepPreAsap` recursively lowers its contained `QueryExpr`. Typical physical +operators include scans, filters, projections, hash aggregates, joins, +ordering, bounded Top-K, limits, and PromQL-specific operators. The selected +physical algorithm, rather than the logical spelling, determines the formula. + +### Post-ASAP lowering + +Every `SummaryExpr` operation also needs explicit physical realization: + +| Logical summary operation | Required physical realization | +|---|---| +| `SummaryAgg` | concrete summary or exact-state build/update operator, including family parameters and grouping layout | +| `SummaryJoin` | concrete summary-join algorithm and state layout | +| `SummaryMerge` | merge operator over compatible concrete summary states | +| `SummarySubtract` | subtract operator supported by the selected state representation | +| `SummaryDelete` | physical deletion/update operator supported by the selected representation | +| `SummaryEstimate` | family- and query-specific readout operator | +| `KeepPreAsap` | recursive lowering of the contained `QueryExpr` | + +This table is a completeness requirement, not a claim that every realization +already exists. Until lowering introduces an explicit physical operator, +statistics contract, validation rule, and resource formula for an operation, +a candidate containing it is unavailable. + +Lifecycle choice affects the physical DAG but does not replace it. Ephemeral, +prepared, shared, and continuously maintained alternatives determine when +build, update, readout, merge, subtract, or delete nodes execute. The physical +operators still determine how each execution consumes CPU, memory, and I/O. + +## Statistics contract + +Statistics describe how a selected physical algorithm behaves for a specific +workload and data snapshot. They do not select the algorithm. + +Every physical node supplies: + +- logical input and output edge cardinality and decoded byte size; +- operator-specific distribution or state facts required by its formula; +- physical source-read bytes only when the operator actually reads storage; +- provenance and freshness sufficient to reproduce the estimate. + +Plan-owned configuration and observed statistics remain separate. For +example, a Top-K operator owns its limit and offset, while its statistics +describe input and output rows and bytes. A hash join owns its build-side +choice, while its statistics describe both input edges and its output edge. + +The statistics enum is structured by `PhysicalOperator`. This prevents a +Filter from carrying group cardinality, a Top-K record from carrying join +facts, or a non-scan record from charging source bytes. A new physical variant +requires a corresponding statistics variant and exhaustive integration into: + +- statistics and DAG arity validation; +- parent/child edge consistency validation; +- operator semantic validation; +- CPU, memory, and I/O formulas; +- lowering and coverage tests. + +## Comparison and failure behavior + +Raw and post-ASAP alternatives must cover the same `ComparisonScope`: arrival +mode, planning time, workload horizon, recurrence, event-time selection, +logical sources, predicates, and physical source snapshots. + +Missing statistics, stale evidence, an unsupported physical algorithm, +unlowered logical operations, inconsistent edges, or different comparison +scopes make the complete candidate unavailable. The integration must never +replace those failures with zero cost or structural node counting. + +See [Analytical resource cost](analytical-resource-cost.md) for the resource +formulas, evidence validation, comparison-scope rules, and calibration model. From f541689c9a82669402c8f634cacdbf3c755abae7 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 13:23:06 -0600 Subject: [PATCH 38/48] fix(cost): preserve physical work during query lowering --- .../asap-aware-mapping/src/analytical_cost.rs | 533 +++++++++++++++--- .../src/analytical_lowering.rs | 386 ++++++++++--- .../src/analytical_statistics.rs | 22 +- .../analytical-resource-cost.md | 58 +- .../physical-plan-integration.md | 123 ++++ 5 files changed, 959 insertions(+), 163 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index a4b07f2c..2de32c1c 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -62,22 +62,44 @@ pub struct ResourceEstimate { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum PhysicalOperator { Scan, - Filter, - Project, - HashAggregate, - InMemoryComparisonSort, + Filter { + /// Scalar predicate operations evaluated for each input row. + predicate_operations_per_row: u64, + }, + Project { + /// Scalar expression/copy operations evaluated for each input row. + expression_operations_per_row: u64, + }, + HashAggregate { + grouping_key_count: u64, + accumulator_count: u64, + }, + InMemoryComparisonSort { + ordering_key_count: u64, + partitioned: bool, + }, /// Heap-based bounded ordering. The heap retains `limit + offset` rows /// while the operator returns at most `limit` rows after skipping offset. TopK { limit: u64, offset: u64, + ordering_key_count: u64, }, HashJoin { build_side: HashJoinBuildSide, + equality_key_count: u64, + }, + HashDeduplicate { + key_count: u64, }, - HashDeduplicate, Concat, - InMemoryOrderedWindow, + /// SQL analytic window evaluated over ordered in-memory partitions. This + /// is unrelated to streaming tumbling/sliding/EH window layouts. + InMemoryAnalyticWindow { + partition_key_count: u64, + ordering_key_count: u64, + function_operations_per_row: u64, + }, Limit { limit: u64, offset: u64, @@ -386,12 +408,14 @@ fn validate_operator_statistics( fn statistics_match_operator(operator: PhysicalOperator, statistics: &OperatorStatistics) -> bool { match operator { PhysicalOperator::Scan => matches!(statistics, OperatorStatistics::Scan { .. }), - PhysicalOperator::Filter => matches!(statistics, OperatorStatistics::Filter { .. }), - PhysicalOperator::Project => matches!(statistics, OperatorStatistics::Project { .. }), - PhysicalOperator::HashAggregate => { + PhysicalOperator::Filter { .. } => matches!(statistics, OperatorStatistics::Filter { .. }), + PhysicalOperator::Project { .. } => { + matches!(statistics, OperatorStatistics::Project { .. }) + } + PhysicalOperator::HashAggregate { .. } => { matches!(statistics, OperatorStatistics::HashAggregate { .. }) } - PhysicalOperator::InMemoryComparisonSort => { + PhysicalOperator::InMemoryComparisonSort { .. } => { matches!( statistics, OperatorStatistics::InMemoryComparisonSort { .. } @@ -401,12 +425,15 @@ fn statistics_match_operator(operator: PhysicalOperator, statistics: &OperatorSt PhysicalOperator::HashJoin { .. } => { matches!(statistics, OperatorStatistics::HashJoin { .. }) } - PhysicalOperator::HashDeduplicate => { + PhysicalOperator::HashDeduplicate { .. } => { matches!(statistics, OperatorStatistics::HashDeduplicate { .. }) } PhysicalOperator::Concat => matches!(statistics, OperatorStatistics::Concat { .. }), - PhysicalOperator::InMemoryOrderedWindow => { - matches!(statistics, OperatorStatistics::InMemoryOrderedWindow { .. }) + PhysicalOperator::InMemoryAnalyticWindow { .. } => { + matches!( + statistics, + OperatorStatistics::InMemoryAnalyticWindow { .. } + ) } PhysicalOperator::Limit { .. } => matches!(statistics, OperatorStatistics::Limit { .. }), PhysicalOperator::PassThrough => { @@ -442,13 +469,13 @@ fn expected_input_arity( statistics_inputs: 1, dag_children: 0, }, - PhysicalOperator::Filter - | PhysicalOperator::Project - | PhysicalOperator::HashAggregate - | PhysicalOperator::InMemoryComparisonSort + PhysicalOperator::Filter { .. } + | PhysicalOperator::Project { .. } + | PhysicalOperator::HashAggregate { .. } + | PhysicalOperator::InMemoryComparisonSort { .. } | PhysicalOperator::TopK { .. } - | PhysicalOperator::HashDeduplicate - | PhysicalOperator::InMemoryOrderedWindow + | PhysicalOperator::HashDeduplicate { .. } + | PhysicalOperator::InMemoryAnalyticWindow { .. } | PhysicalOperator::Limit { .. } | PhysicalOperator::PassThrough => unary, PhysicalOperator::HashJoin { .. } => OperatorInputArity { @@ -462,6 +489,81 @@ fn expected_input_arity( } } +fn checked_cpu_product(rows: u64, operations_per_row: u64) -> Result { + Ok(rows + .checked_mul(operations_per_row) + .ok_or(AnalyticalCostError::Overflow)? as f64) +} + +fn partitioned_order_estimate( + partitioning: &crate::analytical_statistics::PartitionStatistics, + comparison_operations: u64, + row_operations: u64, +) -> Result { + let mut cpu_ops = 0.0; + let mut peak_memory_bytes = 0; + for partition in &partitioning.partitions { + let comparisons = partition.rows as f64 + * (partition.rows.max(2) as f64).log2().ceil() + * comparison_operations as f64; + cpu_ops += comparisons + checked_cpu_product(partition.rows, row_operations)?; + peak_memory_bytes = peak_memory_bytes.max(partition.bytes); + } + if !cpu_ops.is_finite() { + return Err(AnalyticalCostError::Overflow); + } + Ok(ResourceEstimate { + cpu_ops, + peak_memory_bytes, + scan_bytes: 0, + }) +} + +fn validate_partitioning( + input: EdgeStatistics, + partitioning: &crate::analytical_statistics::PartitionStatistics, + partitioned: bool, +) -> Result<(), AnalyticalCostError> { + let inconsistent = |reason| Err(AnalyticalCostError::InconsistentOperatorStatistics(reason)); + if input.rows == 0 { + return if partitioning.partitions.is_empty() { + Ok(()) + } else { + inconsistent("empty input has non-empty partition evidence") + }; + } + if partitioning.partitions.is_empty() { + return inconsistent("non-empty ordered input has no partition evidence"); + } + if !partitioned && partitioning.partitions.len() != 1 { + return inconsistent("global ordering must have exactly one partition"); + } + let total = partitioning.partitions.iter().try_fold( + EdgeStatistics { rows: 0, bytes: 0 }, + |total, partition| { + if !partition.is_consistent() || partition.rows == 0 { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "ordered partition evidence is invalid", + )); + } + Ok(EdgeStatistics { + rows: total + .rows + .checked_add(partition.rows) + .ok_or(AnalyticalCostError::Overflow)?, + bytes: total + .bytes + .checked_add(partition.bytes) + .ok_or(AnalyticalCostError::Overflow)?, + }) + }, + )?; + if total != input { + return inconsistent("ordered partitions do not sum to the input edge"); + } + Ok(()) +} + /// Estimate one physical operator. Child costs are deliberately excluded; /// a DAG walker sums CPU/disk once per node and combines simultaneously /// retained state separately. @@ -499,15 +601,35 @@ pub fn estimate_operator( scan_bytes: *source_read_bytes, }, ( - PhysicalOperator::Filter | PhysicalOperator::Project | PhysicalOperator::PassThrough, + PhysicalOperator::Filter { + predicate_operations_per_row, + }, + _, + ) => ResourceEstimate { + cpu_ops: checked_cpu_product(left.rows, predicate_operations_per_row)?, + peak_memory_bytes: per_row_width(output.rows, output.bytes)?, + scan_bytes: 0, + }, + ( + PhysicalOperator::Project { + expression_operations_per_row, + }, _, ) => ResourceEstimate { + cpu_ops: checked_cpu_product(left.rows, expression_operations_per_row)?, + peak_memory_bytes: per_row_width(output.rows, output.bytes)?, + scan_bytes: 0, + }, + (PhysicalOperator::PassThrough, _) => ResourceEstimate { cpu_ops: left.rows as f64, peak_memory_bytes: per_row_width(output.rows, output.bytes)?, scan_bytes: 0, }, ( - PhysicalOperator::HashAggregate, + PhysicalOperator::HashAggregate { + grouping_key_count, + accumulator_count, + }, OperatorStatistics::HashAggregate { group_count, key_bytes, @@ -515,7 +637,12 @@ pub fn estimate_operator( .. }, ) => ResourceEstimate { - cpu_ops: left.rows as f64, + cpu_ops: checked_cpu_product( + left.rows, + grouping_key_count + .checked_add(accumulator_count) + .ok_or(AnalyticalCostError::Overflow)?, + )?, peak_memory_bytes: checked_bytes(&[ *group_count, key_bytes @@ -526,14 +653,14 @@ pub fn estimate_operator( scan_bytes: 0, }, ( - PhysicalOperator::HashDeduplicate, + PhysicalOperator::HashDeduplicate { key_count }, OperatorStatistics::HashDeduplicate { distinct_key_count, key_bytes, .. }, ) => ResourceEstimate { - cpu_ops: left.rows as f64, + cpu_ops: checked_cpu_product(left.rows, key_count)?, peak_memory_bytes: checked_bytes(&[ *distinct_key_count, key_bytes @@ -542,20 +669,46 @@ pub fn estimate_operator( ])?, scan_bytes: 0, }, - (PhysicalOperator::InMemoryComparisonSort | PhysicalOperator::InMemoryOrderedWindow, _) => { - ResourceEstimate { - cpu_ops: left.rows as f64 * (left.rows.max(2) as f64).log2().ceil(), - peak_memory_bytes: left.bytes, - scan_bytes: 0, - } - } - (PhysicalOperator::TopK { limit, offset }, _) => { + ( + PhysicalOperator::InMemoryComparisonSort { + ordering_key_count, .. + }, + OperatorStatistics::InMemoryComparisonSort { + input_partitioning, .. + }, + ) => partitioned_order_estimate(input_partitioning, ordering_key_count, 0)?, + ( + PhysicalOperator::InMemoryAnalyticWindow { + partition_key_count, + ordering_key_count, + function_operations_per_row, + }, + OperatorStatistics::InMemoryAnalyticWindow { + input_partitioning, .. + }, + ) => partitioned_order_estimate( + input_partitioning, + partition_key_count + .checked_add(ordering_key_count) + .ok_or(AnalyticalCostError::Overflow)?, + function_operations_per_row, + )?, + ( + PhysicalOperator::TopK { + limit, + offset, + ordering_key_count, + }, + _, + ) => { let heap_capacity = limit .checked_add(offset) .ok_or(AnalyticalCostError::Overflow)?; let heap_rows = heap_capacity.min(left.rows); ResourceEstimate { - cpu_ops: left.rows as f64 * (heap_rows.max(2) as f64).log2().ceil(), + cpu_ops: left.rows as f64 + * (heap_rows.max(2) as f64).log2().ceil() + * ordering_key_count as f64, peak_memory_bytes: checked_bytes(&[ heap_rows, per_row_width(left.rows, left.bytes)?, @@ -563,10 +716,21 @@ pub fn estimate_operator( scan_bytes: 0, } } - (PhysicalOperator::HashJoin { build_side }, _) => { + ( + PhysicalOperator::HashJoin { + build_side, + equality_key_count, + }, + _, + ) => { let right = input(1)?; ResourceEstimate { - cpu_ops: left.rows as f64 + right.rows as f64 + output.rows as f64, + cpu_ops: checked_cpu_product( + left.rows + .checked_add(right.rows) + .ok_or(AnalyticalCostError::Overflow)?, + equality_key_count, + )? + output.rows as f64, peak_memory_bytes: match build_side { HashJoinBuildSide::Left => left .rows @@ -623,6 +787,15 @@ pub(crate) fn validate_operator_semantics( if !statistics_match_operator(operator, statistics) { return inconsistent("statistics variant does not match physical operator"); } + if matches!( + (operator, statistics), + ( + PhysicalOperator::Concat, + OperatorStatistics::Concat { inputs, .. } + ) if inputs.is_empty() + ) { + return inconsistent("Concat must have at least one input"); + } let input = statistics .input(0) .ok_or(AnalyticalCostError::InconsistentOperatorStatistics( @@ -630,23 +803,50 @@ pub(crate) fn validate_operator_semantics( ))?; let output = statistics.output(); match (operator, statistics) { - (PhysicalOperator::Scan, _) => { + ( + PhysicalOperator::Scan, + OperatorStatistics::Scan { + source_read_bytes, .. + }, + ) => { if input != output { return inconsistent("Scan input and output edges differ"); } + if input.rows > 0 && *source_read_bytes == 0 { + return inconsistent("non-empty Scan has zero source-read bytes"); + } } - (PhysicalOperator::Filter, _) => { + ( + PhysicalOperator::Filter { + predicate_operations_per_row, + }, + _, + ) => { + if predicate_operations_per_row == 0 { + return inconsistent("Filter has no predicate work"); + } if output.rows > input.rows || output.bytes > input.bytes { return inconsistent("Filter output expands its input"); } } - (PhysicalOperator::Project, _) => { + ( + PhysicalOperator::Project { + expression_operations_per_row, + }, + _, + ) => { + if expression_operations_per_row == 0 { + return inconsistent("Project has no expression work"); + } if output.rows != input.rows { return inconsistent("Project changes row cardinality"); } } ( - PhysicalOperator::HashAggregate, + PhysicalOperator::HashAggregate { + grouping_key_count, + accumulator_count, + }, OperatorStatistics::HashAggregate { group_count, key_bytes, @@ -654,39 +854,81 @@ pub(crate) fn validate_operator_semantics( .. }, ) => { - if *group_count == 0 || *key_bytes == 0 || *accumulator_bytes_per_group == 0 { - return inconsistent("HashAggregate state statistics must be positive"); + if accumulator_count == 0 || *accumulator_bytes_per_group == 0 { + return inconsistent("HashAggregate accumulator work and width must be positive"); } - if *group_count > input.rows || output.rows != *group_count { + if grouping_key_count == 0 { + if *key_bytes != 0 || *group_count != 1 { + return inconsistent( + "ungrouped HashAggregate must have one zero-key-width group", + ); + } + } else if *key_bytes == 0 || *group_count > input.rows { + return inconsistent("grouped HashAggregate has invalid group evidence"); + } + if output.rows != *group_count { return inconsistent("grouped output differs from distinct group cardinality"); } } ( - PhysicalOperator::HashDeduplicate, + PhysicalOperator::HashDeduplicate { key_count }, OperatorStatistics::HashDeduplicate { distinct_key_count, key_bytes, .. }, ) => { - if *distinct_key_count == 0 || *key_bytes == 0 { - return inconsistent("Deduplicate state statistics must be positive"); + if key_count == 0 || *key_bytes == 0 { + return inconsistent("Deduplicate key count and width must be positive"); } if *distinct_key_count > input.rows || output.rows != *distinct_key_count { return inconsistent("deduplicated output differs from distinct key cardinality"); } } - (PhysicalOperator::InMemoryComparisonSort | PhysicalOperator::PassThrough, _) => { + ( + PhysicalOperator::InMemoryComparisonSort { + ordering_key_count, + partitioned, + }, + OperatorStatistics::InMemoryComparisonSort { + input_partitioning, .. + }, + ) => { + if ordering_key_count == 0 { + return inconsistent("Sort has no ordering keys"); + } if input != output { return inconsistent("cardinality-preserving operator changes its edge"); } + validate_partitioning(input, input_partitioning, partitioned)?; } - (PhysicalOperator::InMemoryOrderedWindow, _) => { + (PhysicalOperator::PassThrough, _) => { + if input != output { + return inconsistent("cardinality-preserving operator changes its edge"); + } + } + ( + PhysicalOperator::InMemoryAnalyticWindow { + partition_key_count, + ordering_key_count, + function_operations_per_row, + }, + OperatorStatistics::InMemoryAnalyticWindow { + input_partitioning, .. + }, + ) => { + if ordering_key_count == 0 || function_operations_per_row == 0 { + return inconsistent("analytic Window work must be positive"); + } if input.rows != output.rows { return inconsistent("Window changes row cardinality"); } + validate_partitioning(input, input_partitioning, partition_key_count > 0)?; } (PhysicalOperator::Concat, OperatorStatistics::Concat { inputs, .. }) => { + if inputs.is_empty() { + return inconsistent("Concat must have at least one input"); + } let total = inputs .iter() @@ -706,9 +948,16 @@ pub(crate) fn validate_operator_semantics( return inconsistent("Concat output differs from the sum of its inputs"); } } - (PhysicalOperator::TopK { limit, offset }, _) => { - if limit == 0 { - return inconsistent("Top-K limit must be positive"); + ( + PhysicalOperator::TopK { + limit, + offset, + ordering_key_count, + }, + _, + ) => { + if limit == 0 || ordering_key_count == 0 { + return inconsistent("Top-K limit and ordering-key count must be positive"); } let expected = input.rows.saturating_sub(offset).min(limit); if output.rows != expected { @@ -721,7 +970,16 @@ pub(crate) fn validate_operator_semantics( return inconsistent("Limit output differs from limit and offset"); } } - (PhysicalOperator::HashJoin { .. }, _) => {} + ( + PhysicalOperator::HashJoin { + equality_key_count, .. + }, + _, + ) => { + if equality_key_count == 0 { + return inconsistent("HashJoin has no equality keys"); + } + } _ => return inconsistent("statistics variant does not match physical operator"), } Ok(()) @@ -810,9 +1068,28 @@ mod tests { use super::*; use crate::analytical_statistics::{ validate_comparison_scopes, BinaryEdgeStatistics, ComparisonScope, EdgeStatistics, - OperatorStatistics, SourceCoverage, UnaryEdgeStatistics, + OperatorStatistics, PartitionStatistics, SourceCoverage, UnaryEdgeStatistics, }; + fn filter_operator() -> PhysicalOperator { + PhysicalOperator::Filter { + predicate_operations_per_row: 1, + } + } + + fn project_operator() -> PhysicalOperator { + PhysicalOperator::Project { + expression_operations_per_row: 1, + } + } + + fn aggregate_operator() -> PhysicalOperator { + PhysicalOperator::HashAggregate { + grouping_key_count: 1, + accumulator_count: 1, + } + } + fn unary_row_edges(input_rows: u64, output_rows: u64) -> UnaryEdgeStatistics { UnaryEdgeStatistics { input: EdgeStatistics { @@ -836,7 +1113,7 @@ mod tests { } ); assert_eq!( - expected_input_arity(PhysicalOperator::Filter, 1), + expected_input_arity(filter_operator(), 1), OperatorInputArity { statistics_inputs: 1, dag_children: 1, @@ -846,6 +1123,7 @@ mod tests { expected_input_arity( PhysicalOperator::HashJoin { build_side: HashJoinBuildSide::Left, + equality_key_count: 1, }, 2, ), @@ -867,7 +1145,7 @@ mod tests { fn operator_estimator_rejects_contradictory_cardinality_evidence() { assert!(matches!( estimate_operator( - PhysicalOperator::Filter, + filter_operator(), OperatorStatistics::Filter { edges: unary_row_edges(10, 11), }, @@ -876,7 +1154,7 @@ mod tests { )); assert!(matches!( estimate_operator( - PhysicalOperator::Project, + project_operator(), OperatorStatistics::Project { edges: unary_row_edges(10, 9), }, @@ -886,7 +1164,7 @@ mod tests { assert!(matches!( estimate_operator( - PhysicalOperator::HashAggregate, + aggregate_operator(), OperatorStatistics::HashAggregate { edges: unary_row_edges(10, 3), group_count: 2, @@ -902,6 +1180,7 @@ mod tests { PhysicalOperator::TopK { limit: 3, offset: 0, + ordering_key_count: 1, }, OperatorStatistics::TopK { edges: unary_row_edges(10, 4), @@ -936,6 +1215,7 @@ mod tests { PhysicalOperator::TopK { limit: 10, offset: 0, + ordering_key_count: 1, }, OperatorStatistics::TopK { edges: UnaryEdgeStatistics { @@ -958,6 +1238,7 @@ mod tests { let mismatched_join_statistics = estimate_operator( PhysicalOperator::HashJoin { build_side: HashJoinBuildSide::Left, + equality_key_count: 1, }, OperatorStatistics::Filter { edges: unary_row_edges(1_000, 100), @@ -973,6 +1254,7 @@ mod tests { let build_left = estimate_operator( PhysicalOperator::HashJoin { build_side: HashJoinBuildSide::Left, + equality_key_count: 1, }, OperatorStatistics::HashJoin { edges: BinaryEdgeStatistics { @@ -997,7 +1279,7 @@ mod tests { assert_eq!(build_left.peak_memory_bytes, 80_000); let aggregate = estimate_operator( - PhysicalOperator::HashAggregate, + aggregate_operator(), OperatorStatistics::HashAggregate { group_count: 100, key_bytes: 16, @@ -1021,6 +1303,7 @@ mod tests { PhysicalOperator::TopK { limit: 1_000, offset: 0, + ordering_key_count: 1, }, OperatorStatistics::TopK { edges: UnaryEdgeStatistics { @@ -1075,7 +1358,7 @@ mod tests { }, PhysicalDagNode { id: "left".into(), - operator: PhysicalOperator::Filter, + operator: filter_operator(), children: vec!["scan".into()], source_coverage: None, output_buffer_bytes: 4, @@ -1084,7 +1367,7 @@ mod tests { }, PhysicalDagNode { id: "right".into(), - operator: PhysicalOperator::Filter, + operator: filter_operator(), children: vec!["scan".into()], source_coverage: None, output_buffer_bytes: 4, @@ -1165,7 +1448,7 @@ mod tests { }, PhysicalDagNode { id: "state".into(), - operator: PhysicalOperator::HashAggregate, + operator: aggregate_operator(), children: vec!["scan".into()], source_coverage: None, output_buffer_bytes: 16, @@ -1217,7 +1500,7 @@ mod tests { let mut scope = comparison_scope(); scope.horizon.0 = 100_000; let estimate = estimate_physical_dag(&nodes, "read", &scope, &provided).unwrap(); - assert_eq!(estimate.cpu_ops, 210.0); + assert_eq!(estimate.cpu_ops, 310.0); assert_eq!(estimate.scan_bytes, 1_000); } @@ -1314,7 +1597,7 @@ mod tests { }, PhysicalDagNode { id: "filter".into(), - operator: PhysicalOperator::Filter, + operator: filter_operator(), children: vec!["scan".into()], source_coverage: None, output_buffer_bytes: 4, @@ -1389,7 +1672,7 @@ mod tests { }, PhysicalDagNode { id: "filter".into(), - operator: PhysicalOperator::Filter, + operator: filter_operator(), children: vec!["scan".into()], source_coverage: None, output_buffer_bytes: 4, @@ -1452,7 +1735,7 @@ mod tests { }, PhysicalDagNode { id: "filter".into(), - operator: PhysicalOperator::Filter, + operator: filter_operator(), children: vec!["scan".into()], source_coverage: None, output_buffer_bytes: 0, @@ -1610,7 +1893,7 @@ mod tests { }, PhysicalDagNode { id: "aggregate".into(), - operator: PhysicalOperator::HashAggregate, + operator: aggregate_operator(), children: vec!["scan".into()], source_coverage: None, output_buffer_bytes: 16, @@ -1726,13 +2009,131 @@ mod tests { }; assert_eq!( - estimate_operator(PhysicalOperator::Filter, filter) + estimate_operator(filter_operator(), filter) .unwrap() .scan_bytes, 0 ); } + #[test] + fn non_empty_scan_cannot_claim_zero_source_reads() { + let logical = EdgeStatistics { + rows: 10, + bytes: 80, + }; + assert_eq!( + estimate_operator( + PhysicalOperator::Scan, + OperatorStatistics::Scan { + edges: unary_edges(logical, logical), + source_read_bytes: 0, + }, + ), + Err(AnalyticalCostError::InconsistentOperatorStatistics( + "non-empty Scan has zero source-read bytes" + )) + ); + } + + #[test] + fn hash_aggregate_handles_empty_grouped_and_ungrouped_inputs() { + let empty = EdgeStatistics { rows: 0, bytes: 0 }; + let ungrouped = estimate_operator( + PhysicalOperator::HashAggregate { + grouping_key_count: 0, + accumulator_count: 1, + }, + OperatorStatistics::HashAggregate { + edges: unary_edges(empty, EdgeStatistics { rows: 1, bytes: 8 }), + group_count: 1, + key_bytes: 0, + accumulator_bytes_per_group: 8, + }, + ) + .unwrap(); + assert_eq!(ungrouped.cpu_ops, 0.0); + assert_eq!(ungrouped.peak_memory_bytes, 24); + + let grouped = estimate_operator( + PhysicalOperator::HashAggregate { + grouping_key_count: 1, + accumulator_count: 1, + }, + OperatorStatistics::HashAggregate { + edges: unary_edges(empty, empty), + group_count: 0, + key_bytes: 8, + accumulator_bytes_per_group: 8, + }, + ) + .unwrap(); + assert_eq!(grouped.peak_memory_bytes, 0); + } + + #[test] + fn operator_local_work_and_partition_distribution_change_cpu_and_memory() { + let edge = EdgeStatistics { + rows: 100, + bytes: 800, + }; + let filter = OperatorStatistics::Filter { + edges: unary_edges(edge, edge), + }; + assert_eq!( + estimate_operator( + PhysicalOperator::Filter { + predicate_operations_per_row: 3, + }, + filter, + ) + .unwrap() + .cpu_ops, + 300.0 + ); + + let sort = estimate_operator( + PhysicalOperator::InMemoryComparisonSort { + ordering_key_count: 1, + partitioned: true, + }, + OperatorStatistics::InMemoryComparisonSort { + edges: unary_edges(edge, edge), + input_partitioning: PartitionStatistics { + partitions: vec![ + EdgeStatistics { + rows: 50, + bytes: 400, + }, + EdgeStatistics { + rows: 50, + bytes: 400, + }, + ], + }, + }, + ) + .unwrap(); + assert_eq!(sort.cpu_ops, 600.0); + assert_eq!(sort.peak_memory_bytes, 400); + } + + #[test] + fn concat_requires_at_least_one_physical_input() { + assert_eq!( + estimate_operator( + PhysicalOperator::Concat, + OperatorStatistics::Concat { + inputs: vec![], + output: EdgeStatistics { rows: 0, bytes: 0 }, + }, + ), + Err(AnalyticalCostError::InconsistentOperatorStatistics( + "Concat must have at least one input" + )) + ); + } + #[test] fn serialized_operator_statistics_reject_unrelated_fields() { let edge = serde_json::json!({ "rows": 100, "bytes": 1_000 }); diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index 06389b4c..b661d7e3 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -99,7 +99,7 @@ impl OperatorStatisticsProvider for PhysicalDag { /// Lower a resolved query operator DAG to the physical operators understood by /// this cost model. The authoritative provider supplies statistics by the -/// deterministic physical IDs assigned here; missing evidence makes the +/// stable physical IDs owned by that provider; missing evidence makes the /// complete query unavailable. Scalar expressions remain part of their /// containing operator's local cost. pub fn lower_query_physical_dag( @@ -264,31 +264,55 @@ pub fn lower_query_physical_dag( Some(coverage), )?; let children = vec![scan_id.clone()]; - let filter_evidence = self.resolve( - query, - PhysicalOperator::Filter, - occurrence, - false, - &children, - None, - )?; + let predicate_operations_per_row = predicates + .iter() + .try_fold(0_u64, |total, predicate| { + total + .checked_add(scalar_operation_count(&predicate.0)?) + .ok_or(AnalyticalCostError::Overflow) + })? + .max(1); + let filter_operator = PhysicalOperator::Filter { + predicate_operations_per_row, + }; + let filter_evidence = + self.resolve(query, filter_operator, occurrence, false, &children, None)?; require_unary_edge( &filter_evidence.physical_id, &filter_evidence.statistics, &scan_id, &scan_statistics, )?; - require_operator_statistics( - PhysicalOperator::Filter, - &filter_evidence.statistics, - )?; - self.push(filter_evidence, PhysicalOperator::Filter, children, None) + require_operator_statistics(filter_operator, &filter_evidence.statistics)?; + self.push(filter_evidence, filter_operator, children, None) } - QueryExpr::Filter { child, .. } => { - self.lower_unary(query, occurrence, PhysicalOperator::Filter, child) + QueryExpr::Filter { pred, child } => { + let operator = PhysicalOperator::Filter { + predicate_operations_per_row: scalar_operation_count(&pred.0)?.max(1), + }; + self.lower_unary(query, occurrence, operator, child) } - QueryExpr::Project { child, .. } => { - self.lower_unary(query, occurrence, PhysicalOperator::Project, child) + QueryExpr::Project { cols, child, .. } => { + let expression_operations_per_row = cols + .iter() + .try_fold(0_u64, |total, item| { + total + .checked_add( + scalar_operation_count(&item.expr)? + .checked_add(1) + .ok_or(AnalyticalCostError::Overflow)?, + ) + .ok_or(AnalyticalCostError::Overflow) + })? + .max(1); + self.lower_unary( + query, + occurrence, + PhysicalOperator::Project { + expression_operations_per_row, + }, + child, + ) } QueryExpr::Aggregate { reduction, @@ -297,22 +321,67 @@ pub fn lower_query_physical_dag( child, .. } => { - if having.is_some() || !supports_hash_aggregate(reduction, measures) { + let asap_types::pre_asap::Reduction::Reduce(grouping) = reduction else { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + }; + if grouping.is_without() + || having.is_some() + || !supports_hash_aggregate(reduction, measures) + { return Err(AnalyticalCostError::UnsupportedQueryOperator); } - self.lower_unary(query, occurrence, PhysicalOperator::HashAggregate, child) + self.lower_unary( + query, + occurrence, + PhysicalOperator::HashAggregate { + grouping_key_count: u64::try_from(grouping.keys().len()) + .map_err(|_| AnalyticalCostError::Overflow)?, + accumulator_count: u64::try_from(measures.len()) + .map_err(|_| AnalyticalCostError::Overflow)?, + }, + child, + ) } - QueryExpr::Dedup { child, .. } => { - self.lower_unary(query, occurrence, PhysicalOperator::HashDeduplicate, child) + QueryExpr::Dedup { cols, child } => { + let key_count = if cols.is_empty() { + child + .output_schema() + .map_err(|_| AnalyticalCostError::UnsupportedQueryOperator)? + .columns + .len() + } else { + cols.len() + }; + if key_count == 0 { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + } + self.lower_unary( + query, + occurrence, + PhysicalOperator::HashDeduplicate { + key_count: u64::try_from(key_count) + .map_err(|_| AnalyticalCostError::Overflow)?, + }, + child, + ) } - QueryExpr::Sort { keys, child, .. } => { + QueryExpr::Sort { + keys, + partition_by, + child, + .. + } => { if keys.is_empty() { return Err(AnalyticalCostError::UnsupportedQueryOperator); } self.lower_unary( query, occurrence, - PhysicalOperator::InMemoryComparisonSort, + PhysicalOperator::InMemoryComparisonSort { + ordering_key_count: u64::try_from(keys.len()) + .map_err(|_| AnalyticalCostError::Overflow)?, + partitioned: partition_by != &GroupKeys::none(), + }, child, ) } @@ -331,7 +400,12 @@ pub fn lower_query_physical_dag( u64::try_from(*n).map_err(|_| AnalyticalCostError::Overflow)?; let offset = u64::try_from(*offset) .map_err(|_| AnalyticalCostError::Overflow)?; - let operator = PhysicalOperator::TopK { limit, offset }; + let operator = PhysicalOperator::TopK { + limit, + offset, + ordering_key_count: u64::try_from(keys.len()) + .map_err(|_| AnalyticalCostError::Overflow)?, + }; let evidence = self.resolve(query, operator, occurrence, false, &children, None)?; let statistics = &evidence.statistics; @@ -374,6 +448,7 @@ pub fn lower_query_physical_dag( } QueryExpr::SQLWindowFunc { func, + partition_by, order_by, child, .. @@ -391,11 +466,20 @@ pub fn lower_query_physical_dag( self.lower_unary( query, occurrence, - PhysicalOperator::InMemoryOrderedWindow, + PhysicalOperator::InMemoryAnalyticWindow { + partition_key_count: u64::try_from(partition_by.keys().len()) + .map_err(|_| AnalyticalCostError::Overflow)?, + ordering_key_count: u64::try_from(order_by.len()) + .map_err(|_| AnalyticalCostError::Overflow)?, + function_operations_per_row: 1, + }, child, ) } - QueryExpr::TimeShift { child, .. } => { + QueryExpr::TimeShift { shift, child } => { + if !shift.is_identity() { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + } self.lower_unary(query, occurrence, PhysicalOperator::PassThrough, child) } QueryExpr::Concat { children } => { @@ -421,11 +505,15 @@ pub fn lower_query_physical_dag( left, right, } => { - if matches!(kind, asap_types::pre_asap::JoinKind::Cross) - || !is_hash_join_predicate(&pred.0, left, right) - { + let equality_key_count = + if matches!(kind, asap_types::pre_asap::JoinKind::Cross) { + None + } else { + hash_join_key_count(&pred.0, left, right) + }; + let Some(equality_key_count) = equality_key_count else { return Err(AnalyticalCostError::UnsupportedQueryOperator); - } + }; let left_id = self.lower(left)?; let right_id = self.lower(right)?; let children = vec![left_id.clone(), right_id.clone()]; @@ -437,7 +525,10 @@ pub fn lower_query_physical_dag( } else { HashJoinBuildSide::Right }; - let operator = PhysicalOperator::HashJoin { build_side }; + let operator = PhysicalOperator::HashJoin { + build_side, + equality_key_count, + }; let evidence = self.resolve(query, operator, occurrence, false, &children, None)?; let statistics = &evidence.statistics; @@ -630,15 +721,15 @@ fn bind_scan_coverage( Ok(coverage) } -fn is_hash_join_predicate( +fn hash_join_key_count( expr: &asap_types::pre_asap::QueryExpr, left: &asap_types::pre_asap::QueryExpr, right: &asap_types::pre_asap::QueryExpr, -) -> bool { +) -> Option { use asap_types::pre_asap::{CompareOpKind, QueryExpr}; let (Ok(left_schema), Ok(right_schema)) = (left.output_schema(), right.output_schema()) else { - return false; + return None; }; let left_width = left_schema.columns.len(); let total_width = left_width.saturating_add(right_schema.columns.len()); @@ -653,35 +744,106 @@ fn is_hash_join_predicate( } } - fn predicate(expr: &QueryExpr, left_width: usize, total_width: usize) -> bool { + fn predicate(expr: &QueryExpr, left_width: usize, total_width: usize) -> Option { match expr { QueryExpr::Compare { left, op: CompareOpKind::Eq, right, } => match (left.as_ref(), right.as_ref()) { - (QueryExpr::Column(left), QueryExpr::Column(right)) => matches!( - ( - column_side(*left, left_width, total_width), - column_side(*right, left_width, total_width) - ), - (Some(false), Some(true)) | (Some(true), Some(false)) - ), - _ => false, + (QueryExpr::Column(left), QueryExpr::Column(right)) => match ( + column_side(*left, left_width, total_width), + column_side(*right, left_width, total_width), + ) { + (Some(false), Some(true)) | (Some(true), Some(false)) => Some(1), + _ => None, + }, + _ => None, }, - QueryExpr::BoolAnd(parts) => { - !parts.is_empty() - && parts - .iter() - .all(|part| predicate(part, left_width, total_width)) + QueryExpr::BoolAnd(parts) if !parts.is_empty() => { + parts.iter().try_fold(0_u64, |count, part| { + count.checked_add(predicate(part, left_width, total_width)?) + }) } - _ => false, + _ => None, } } predicate(expr, left_width, total_width) } +fn scalar_operation_count( + expr: &asap_types::pre_asap::QueryExpr, +) -> Result { + use asap_types::pre_asap::QueryExpr; + + let add = |parts: &[&QueryExpr]| { + parts.iter().try_fold(0_u64, |total, part| { + total + .checked_add(scalar_operation_count(part)?) + .ok_or(AnalyticalCostError::Overflow) + }) + }; + let with_local = |children| { + add(children)? + .checked_add(1) + .ok_or(AnalyticalCostError::Overflow) + }; + match expr { + QueryExpr::Column(_) + | QueryExpr::Literal(_) + | QueryExpr::EvalTimestamp + | QueryExpr::CurrentTimestamp => Ok(0), + QueryExpr::Compare { left, right, .. } | QueryExpr::Arithmetic { left, right, .. } => { + with_local(&[left, right]) + } + QueryExpr::BoolAnd(parts) | QueryExpr::BoolOr(parts) => { + let children = parts.iter().collect::>(); + add(&children)? + .checked_add( + u64::try_from(parts.len().saturating_sub(1)) + .map_err(|_| AnalyticalCostError::Overflow)?, + ) + .ok_or(AnalyticalCostError::Overflow) + } + QueryExpr::Not(child) + | QueryExpr::IsNull(child) + | QueryExpr::IsNotNull(child) + | QueryExpr::PromqlScalarBridge(child) => with_local(&[child]), + QueryExpr::Cast { expr, .. } => with_local(&[expr]), + QueryExpr::InList { expr, list, .. } => { + let mut children = Vec::with_capacity(list.len() + 1); + children.push(expr.as_ref()); + children.extend(list.iter()); + add(&children)? + .checked_add(u64::try_from(list.len()).map_err(|_| AnalyticalCostError::Overflow)?) + .ok_or(AnalyticalCostError::Overflow) + } + QueryExpr::FunctionCall { args, .. } => { + let children = args.iter().collect::>(); + with_local(&children) + } + QueryExpr::Case { + operand, + branches, + else_expr, + } => { + let mut children = Vec::new(); + if let Some(operand) = operand { + children.push(operand.as_ref()); + } + for (when, then) in branches { + children.extend([when, then]); + } + if let Some(else_expr) = else_expr { + children.push(else_expr.as_ref()); + } + with_local(&children) + } + _ => Err(AnalyticalCostError::UnsupportedQueryOperator), + } +} + fn supports_hash_aggregate( reduction: &asap_types::pre_asap::Reduction, measures: &[asap_types::pre_asap::AggIntent], @@ -713,7 +875,7 @@ mod tests { estimate_physical_dag, estimate_physical_dag_comparison, PhysicalDagEstimateRequest, }; use crate::analytical_statistics::{ - validate_comparison_scopes, BinaryEdgeStatistics, UnaryEdgeStatistics, + validate_comparison_scopes, BinaryEdgeStatistics, PartitionStatistics, UnaryEdgeStatistics, }; use asap_types::workload::{ DataArrival, DurationMs, QueryRecurrence, QueryTimeScope, TimeSelection, TimestampMs, @@ -742,14 +904,30 @@ mod tests { ) -> OperatorStatistics { let edges = unary_edges(input, output); match operator { - PhysicalOperator::Filter => OperatorStatistics::Filter { edges }, - PhysicalOperator::Project => OperatorStatistics::Project { edges }, - PhysicalOperator::InMemoryComparisonSort => { - OperatorStatistics::InMemoryComparisonSort { edges } + PhysicalOperator::Filter { .. } => OperatorStatistics::Filter { edges }, + PhysicalOperator::Project { .. } => OperatorStatistics::Project { edges }, + PhysicalOperator::InMemoryComparisonSort { .. } => { + OperatorStatistics::InMemoryComparisonSort { + edges, + input_partitioning: PartitionStatistics { + partitions: (!input.eq(&edge(0, 0))) + .then_some(input) + .into_iter() + .collect(), + }, + } } PhysicalOperator::TopK { .. } => OperatorStatistics::TopK { edges }, - PhysicalOperator::InMemoryOrderedWindow => { - OperatorStatistics::InMemoryOrderedWindow { edges } + PhysicalOperator::InMemoryAnalyticWindow { .. } => { + OperatorStatistics::InMemoryAnalyticWindow { + edges, + input_partitioning: PartitionStatistics { + partitions: (!input.eq(&edge(0, 0))) + .then_some(input) + .into_iter() + .collect(), + }, + } } PhysicalOperator::Limit { .. } => OperatorStatistics::Limit { edges }, PhysicalOperator::PassThrough => OperatorStatistics::PassThrough { edges }, @@ -872,6 +1050,7 @@ mod tests { PhysicalOperator::TopK { limit: 10, offset: 5, + ordering_key_count: 1, }, edge(100, 4_000), edge(10, 400), @@ -882,7 +1061,9 @@ mod tests { ( "query-2".into(), evidence(unary_statistics( - PhysicalOperator::Filter, + PhysicalOperator::Filter { + predicate_operations_per_row: 1, + }, edge(1_000, 64_000), edge(400, 25_600), )), @@ -899,11 +1080,17 @@ mod tests { .collect::>(), vec![ PhysicalOperator::Scan, - PhysicalOperator::Filter, - PhysicalOperator::HashAggregate, + PhysicalOperator::Filter { + predicate_operations_per_row: 1, + }, + PhysicalOperator::HashAggregate { + grouping_key_count: 1, + accumulator_count: 1, + }, PhysicalOperator::TopK { limit: 10, offset: 5, + ordering_key_count: 1, }, ] ); @@ -1178,7 +1365,9 @@ mod tests { ( "query-6".into(), evidence(unary_statistics( - PhysicalOperator::Filter, + PhysicalOperator::Filter { + predicate_operations_per_row: 1, + }, edge(1_000, 8_000), edge(800, 6_400), )), @@ -1186,7 +1375,9 @@ mod tests { ( "query-5".into(), evidence(unary_statistics( - PhysicalOperator::Project, + PhysicalOperator::Project { + expression_operations_per_row: 1, + }, edge(800, 6_400), edge(800, 3_200), )), @@ -1195,7 +1386,11 @@ mod tests { ( "query-3".into(), evidence(unary_statistics( - PhysicalOperator::InMemoryOrderedWindow, + PhysicalOperator::InMemoryAnalyticWindow { + partition_key_count: 0, + ordering_key_count: 1, + function_operations_per_row: 1, + }, edge(500, 2_000), edge(500, 6_000), )), @@ -1203,7 +1398,10 @@ mod tests { ( "query-2".into(), evidence(unary_statistics( - PhysicalOperator::InMemoryComparisonSort, + PhysicalOperator::InMemoryComparisonSort { + ordering_key_count: 1, + partitioned: true, + }, edge(500, 6_000), edge(500, 6_000), )), @@ -1237,11 +1435,22 @@ mod tests { .collect::>(), vec![ PhysicalOperator::Scan, - PhysicalOperator::Filter, - PhysicalOperator::Project, - PhysicalOperator::HashDeduplicate, - PhysicalOperator::InMemoryOrderedWindow, - PhysicalOperator::InMemoryComparisonSort, + PhysicalOperator::Filter { + predicate_operations_per_row: 1, + }, + PhysicalOperator::Project { + expression_operations_per_row: 1, + }, + PhysicalOperator::HashDeduplicate { key_count: 1 }, + PhysicalOperator::InMemoryAnalyticWindow { + partition_key_count: 0, + ordering_key_count: 1, + function_operations_per_row: 1, + }, + PhysicalOperator::InMemoryComparisonSort { + ordering_key_count: 1, + partitioned: true, + }, PhysicalOperator::Limit { limit: 20, offset: 0, @@ -1382,7 +1591,9 @@ mod tests { ( "query-0".into(), evidence(unary_statistics( - PhysicalOperator::Project, + PhysicalOperator::Project { + expression_operations_per_row: 1, + }, edge(99, 792), edge(99, 396), )), @@ -1421,8 +1632,13 @@ mod tests { ); let mut complete = conflicting.clone(); - complete.get_mut("query-0").unwrap().statistics = - unary_statistics(PhysicalOperator::Project, edge(100, 800), edge(100, 400)); + complete.get_mut("query-0").unwrap().statistics = unary_statistics( + PhysicalOperator::Project { + expression_operations_per_row: 1, + }, + edge(100, 800), + edge(100, 400), + ); let extra_scope = scope(vec![ comparison_scope.sources[0].clone(), coverage( @@ -1475,7 +1691,9 @@ mod tests { ( "query-1".into(), evidence(unary_statistics( - PhysicalOperator::Filter, + PhysicalOperator::Filter { + predicate_operations_per_row: 1, + }, edge(100, 800), edge(0, 0), )), @@ -1552,6 +1770,13 @@ mod tests { output_name: "lag".into(), child: scan(), }); + let shifted = Rc::new(QueryExpr::TimeShift { + shift: asap_types::pre_asap::TimeShift { + offset_ms: 60_000, + at: None, + }, + child: scan(), + }); let scope = scope(vec![coverage( Source::Table { table_ref: "events".into(), @@ -1564,6 +1789,7 @@ mod tests { &per_entity, &empty_sort_limit, &unsupported_window, + &shifted, ] { assert_eq!( lower_query_physical_dag(query, &scope, &scripted(&unavailable)), @@ -1571,4 +1797,18 @@ mod tests { ); } } + + #[test] + fn scalar_work_counts_every_local_predicate_operation() { + use asap_types::pre_asap::{CompareOpKind, QueryExpr, ScalarValue}; + + let comparison = || QueryExpr::Compare { + left: Rc::new(QueryExpr::Column(0)), + op: CompareOpKind::Eq, + right: Rc::new(QueryExpr::Literal(ScalarValue::Int64(1))), + }; + let predicate = QueryExpr::BoolAnd(vec![comparison(), comparison()]); + + assert_eq!(scalar_operation_count(&predicate), Ok(3)); + } } diff --git a/crates/asap-aware-mapping/src/analytical_statistics.rs b/crates/asap-aware-mapping/src/analytical_statistics.rs index a15cbeda..13fe38dc 100644 --- a/crates/asap-aware-mapping/src/analytical_statistics.rs +++ b/crates/asap-aware-mapping/src/analytical_statistics.rs @@ -218,6 +218,14 @@ pub struct BinaryEdgeStatistics { pub output: EdgeStatistics, } +/// Input distribution for an algorithm that independently orders partitions. +/// The checked sum of these edges must equal the operator input. A global sort +/// has exactly one partition; an empty input has no partitions. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PartitionStatistics { + pub partitions: Vec, +} + /// Workload-dependent evidence for one operator in an already-lowered /// physical DAG. [`PhysicalOperator`](crate::analytical_cost::PhysicalOperator) /// is the authoritative operator vocabulary: every one of its variants has a @@ -254,6 +262,7 @@ pub enum OperatorStatistics { }, InMemoryComparisonSort { edges: UnaryEdgeStatistics, + input_partitioning: PartitionStatistics, }, TopK { edges: UnaryEdgeStatistics, @@ -270,8 +279,9 @@ pub enum OperatorStatistics { inputs: Vec, output: EdgeStatistics, }, - InMemoryOrderedWindow { + InMemoryAnalyticWindow { edges: UnaryEdgeStatistics, + input_partitioning: PartitionStatistics, }, Limit { edges: UnaryEdgeStatistics, @@ -291,7 +301,7 @@ impl OperatorStatistics { | Self::InMemoryComparisonSort { .. } | Self::TopK { .. } | Self::HashDeduplicate { .. } - | Self::InMemoryOrderedWindow { .. } + | Self::InMemoryAnalyticWindow { .. } | Self::Limit { .. } | Self::PassThrough { .. } => 1, Self::HashJoin { .. } => 2, @@ -306,9 +316,9 @@ impl OperatorStatistics { | Self::HashDeduplicate { edges, .. } => (index == 0).then_some(edges.input), Self::Filter { edges } | Self::Project { edges } - | Self::InMemoryComparisonSort { edges } + | Self::InMemoryComparisonSort { edges, .. } | Self::TopK { edges } - | Self::InMemoryOrderedWindow { edges } + | Self::InMemoryAnalyticWindow { edges, .. } | Self::Limit { edges } | Self::PassThrough { edges } => (index == 0).then_some(edges.input), Self::HashJoin { edges } => edges.inputs.get(index).copied(), @@ -323,9 +333,9 @@ impl OperatorStatistics { | Self::HashDeduplicate { edges, .. } => edges.output, Self::Filter { edges } | Self::Project { edges } - | Self::InMemoryComparisonSort { edges } + | Self::InMemoryComparisonSort { edges, .. } | Self::TopK { edges } - | Self::InMemoryOrderedWindow { edges } + | Self::InMemoryAnalyticWindow { edges, .. } | Self::Limit { edges } | Self::PassThrough { edges } => edges.output, Self::HashJoin { edges } => edges.output, diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 1d48cf15..d200de07 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -204,7 +204,10 @@ OperatorStatistics = key_bytes, accumulator_bytes_per_group, } - | InMemoryComparisonSort { edges: UnaryEdgeStatistics } + | InMemoryComparisonSort { + edges: UnaryEdgeStatistics, + input_partitioning: PartitionStatistics, + } | TopK { edges: UnaryEdgeStatistics } | HashJoin { edges: BinaryEdgeStatistics } | HashDeduplicate { @@ -213,7 +216,10 @@ OperatorStatistics = key_bytes, } | Concat { inputs, output } - | InMemoryOrderedWindow { edges: UnaryEdgeStatistics } + | InMemoryAnalyticWindow { + edges: UnaryEdgeStatistics, + input_partitioning: PartitionStatistics, + } | Limit { edges: UnaryEdgeStatistics } | PassThrough { edges: UnaryEdgeStatistics } } @@ -236,9 +242,15 @@ Physical configuration is not catalog evidence and therefore lives on | Physical operator | Configuration owned by the plan | |---|---| -| `TopK` | output `limit` and `offset`; heap capacity is `limit + offset` | +| `Filter` | predicate operations per input row | +| `Project` | expression/copy operations per input row | +| `HashAggregate` | grouping-key and accumulator counts | +| `InMemoryComparisonSort` | ordering-key count and whether ordering is partitioned | +| `TopK` | output `limit`, `offset`, and ordering-key count; heap capacity is `limit + offset` | | `Limit` | output `limit` and `offset` | -| `HashJoin` | left or right build side | +| `HashJoin` | left or right build side and equality-key count | +| `HashDeduplicate` | deduplication-key count | +| `InMemoryAnalyticWindow` | partition/order-key counts and window-function work per row | This distinction removes the former flat optional `k` field. A Top-K bound is part of the chosen algorithm, while input/output cardinality and width are @@ -276,7 +288,16 @@ DAG-child counts must both be defined. Compression, column pruning, or encoded storage can therefore make these values different; neither is inferred from the other. Other statistics variants have no source-read field, so charging source I/O at a non-scan node -is not representable. +is not representable. A non-empty Scan must report positive source-read bytes; +zero cannot stand in for missing I/O evidence. + +Hash-aggregate validation distinguishes logical grouping from the workload's +observed number of groups. An ungrouped aggregate has +`grouping_key_count = 0`, `key_bytes = 0`, and `group_count = 1`, including on +empty input because SQL scalar aggregation still emits one row. A grouped +aggregate has at least one grouping key and positive encoded key width, but it +may report `group_count = 0` when its input is empty. In both cases output rows +must equal `group_count`. ### Composition rules @@ -386,14 +407,14 @@ The supported mappings are: | Filter | Filter | | Project | Project | | Reducing Count/Sum/Min/Max/Avg/StdDev/Variance/Group/CountValues without HAVING | HashAggregate | -| Dedup | Deduplicate | -| Equi-Join | HashJoin with an evidence-selected build side | +| Dedup | HashDeduplicate | +| Equi-Join | HashJoin whose build side is selected from child output evidence | | Concat or `UNION ALL` | Concat | | Sort with at least one ordering key | in-memory Sort | | global non-empty-key Sort followed by Limit | heap TopK, with `k = offset + n` from the query IR | | partitioned Sort followed by Limit | Sort → Limit | -| RowNumber/Rank/DenseRank SQLWindowFunc with non-empty order_by | ordered in-memory Window | -| TimeShift | PassThrough | +| RowNumber/Rank/DenseRank SQLWindowFunc with non-empty order_by | InMemoryAnalyticWindow | +| identity TimeShift only | PassThrough | Per-entity reductions, HAVING, ordered/distribution-dependent intents such as exact quantile or cardinality, Top-K aggregate intents, and extensions remain @@ -430,19 +451,20 @@ the DAG rules above. | Physical operator | CPU operations | Local memory | Source/disk reads | |---|---:|---:|---:| | Scan | `input_rows` | one decoded input row/batch | `source_read_bytes` | -| Filter | `input_rows` | one output row/batch | `0` | -| Project or scalar pass-through | `input_rows` | one output row/batch | `0` | -| Hash aggregate | `input_rows` | `groups × (key_bytes + accumulator_bytes_per_group + hash metadata)` | `0` | -| Hash deduplicate | `input_rows` | `distinct_key_count × (key_bytes + hash metadata)` | `0` | -| In-memory comparison sort | `rows × ceil(log2(rows))` | `input_bytes` | `0` | -| Heap Top-K | `rows × ceil(log2(max(min(limit + offset, rows), 2)))` | `min(limit + offset, rows) × row_bytes` | `0` | -| Hash join | `left_rows + right_rows + output_rows` | selected build-side logical bytes plus 16 bytes of hash metadata per build row | `0` beyond children | +| Filter | `input_rows × predicate_operations_per_row` | one output row/batch | `0` | +| Project | `input_rows × expression_operations_per_row` | one output row/batch | `0` | +| Scalar pass-through | `input_rows` | one output row/batch | `0` | +| Hash aggregate | `input_rows × (grouping_key_count + accumulator_count)` | `groups × (key_bytes + accumulator_bytes_per_group + hash metadata)` | `0` | +| Hash deduplicate | `input_rows × key_count` | `distinct_key_count × (key_bytes + hash metadata)` | `0` | +| In-memory comparison sort | `sum(n_i × ceil(log2(n_i)) × ordering_key_count)` | largest partition bytes | `0` | +| Heap Top-K | `rows × ceil(log2(max(min(limit + offset, rows), 2))) × ordering_key_count` | `min(limit + offset, rows) × row_bytes` | `0` | +| Hash join | `(left_rows + right_rows) × equality_key_count + output_rows` | selected build-side logical bytes plus 16 bytes of hash metadata per build row | `0` beyond children | | Concat | `output_rows` | one output row/batch | `0` | -| In-memory ordered window | `rows × ceil(log2(rows))` | live partition/input bytes | `0` | +| In-memory analytic window | per-partition ordering work plus per-row function work | largest partition bytes | `0` | | Limit | `min(input_rows, limit + offset)` | one output row/batch | `0` | These formulas name physical implementations. The enum uses names such as -`InMemoryComparisonSort`, `HashDeduplicate`, and `InMemoryOrderedWindow` so a +`InMemoryComparisonSort`, `HashDeduplicate`, and `InMemoryAnalyticWindow` so a new algorithm cannot silently inherit a formula merely because it has the same logical purpose. An external sort must add spill writes and reads; a nested-loop join must not use the hash-join formula. diff --git a/docs/design_docs/asap-aware-mapping/physical-plan-integration.md b/docs/design_docs/asap-aware-mapping/physical-plan-integration.md index 843d79ed..8af1105b 100644 --- a/docs/design_docs/asap-aware-mapping/physical-plan-integration.md +++ b/docs/design_docs/asap-aware-mapping/physical-plan-integration.md @@ -148,6 +148,129 @@ requires a corresponding statistics variant and exhaustive integration into: - CPU, memory, and I/O formulas; - lowering and coverage tests. +## Physical operator catalog + +This catalog defines every physical node currently accepted by the analytical +estimator. SQL examples describe the logical shape; the physical node is the +algorithm selected when lowering that shape. A query outside the stated shape +is unavailable until another physical operator is defined. + +### `Scan` + +Reads a physical source snapshot and emits decoded logical rows. For example, +`SELECT * FROM metrics` lowers its source leaf to `Scan`. Its evidence contains +the external/output edge and `source_read_bytes`. A non-empty scan must report +positive physical read bytes. CPU is one decode/visit per input row; memory is +one decoded row or batch; I/O is `source_read_bytes`. + +### `Filter { predicate_operations_per_row }` + +Evaluates a scalar predicate and retains matching rows. For example, +`SELECT * FROM metrics WHERE latency > 100 AND status = 500` lowers to +`Scan -> Filter`. The configuration counts the comparison/boolean operations +in the predicate; the evidence provides input and filtered output edges. CPU +is `input_rows * predicate_operations_per_row`; memory is one output row or +batch; it adds no source I/O. + +### `Project { expression_operations_per_row }` + +Computes and copies a SELECT list without changing row cardinality. For +example, `SELECT latency * 1000 AS latency_us, service FROM metrics` lowers to +`Scan -> Project`. The configuration counts expression and output-copy work; +the evidence supplies the changed logical row width. CPU is +`input_rows * expression_operations_per_row`; memory is one output row or +batch; it adds no source I/O. + +### `HashAggregate { grouping_key_count, accumulator_count }` + +Builds hash-group state and updates one or more accumulators. For example, +`SELECT service, COUNT(*), SUM(bytes) FROM metrics GROUP BY service` uses one +grouping key and two accumulators. Evidence supplies `group_count`, encoded +key bytes, and total accumulator bytes per group. CPU is +`input_rows * (grouping_key_count + accumulator_count)`; memory is +`group_count * (key_bytes + accumulator_bytes_per_group + hash metadata)`. +An ungrouped aggregate has zero keys, zero key bytes, and exactly one output +group even for empty input. A grouped aggregate may have zero groups when its +input is empty. + +### `InMemoryComparisonSort { ordering_key_count, partitioned }` + +Comparison-sorts rows without spilling. A global example is +`SELECT * FROM metrics ORDER BY latency`; a partitioned logical shape is +`ORDER BY latency` within each region. Evidence lists the observed input edge +for every independently sorted partition. CPU is +`sum(n_i * ceil(log2(n_i)) * ordering_key_count)` and peak local memory is the +largest partition bytes. A global sort must provide exactly one partition. +If a provider cannot prove an in-memory implementation or its partition +distribution, the candidate is unavailable rather than silently using a +global-sort estimate. + +### `TopK { limit, offset, ordering_key_count }` + +Maintains a bounded comparison heap for a global `ORDER BY ... LIMIT/OFFSET`. +For example, `SELECT service, count FROM counts ORDER BY count DESC LIMIT 10` +uses a heap of at most ten rows. CPU is +`input_rows * ceil(log2(min(limit + offset, input_rows))) * ordering_key_count`; +memory is the bounded heap rows times logical row width. Partitioned Top-K is +not this operator and requires its own supported physical realization. + +### `HashJoin { build_side, equality_key_count }` + +Builds a hash table on the selected side and probes it with the other side. +For example, `SELECT * FROM requests r JOIN services s ON r.service_id = s.id` +uses one equality key. Evidence supplies ordered left/right edges and the join +output. CPU charges equality-key hashing/probing for both inputs plus emitted +output rows; memory is the selected build-side bytes plus hash metadata. A +cross join, non-equality predicate, or unknown join algorithm is unavailable. + +### `HashDeduplicate { key_count }` + +Retains one hash-table entry per distinct key. For example, +`SELECT DISTINCT service, region FROM metrics` has two deduplication keys. +Evidence supplies distinct-key count and encoded key bytes. CPU is +`input_rows * key_count`; memory is +`distinct_key_count * (key_bytes + hash metadata)`. Empty input legitimately +has zero distinct keys. + +### `Concat` + +Concatenates one or more union-compatible inputs without deduplication. For +example, `SELECT * FROM east UNION ALL SELECT * FROM west` lowers both scans +into one `Concat`. Its output rows and bytes must equal the checked sum of all +input edges. CPU is one append/forward operation per output row; memory is one +output row or batch. A zero-input Concat is invalid. + +### `InMemoryAnalyticWindow` + +Evaluates an ordered SQL analytic function over in-memory partitions. For +example, +`ROW_NUMBER() OVER (PARTITION BY region ORDER BY latency DESC)` partitions by +region, orders each partition, and appends the row-number column. Its physical +configuration records partition keys, ordering keys, and function work per +row; evidence supplies the actual partition distribution. Ordering CPU and +memory use the same per-partition calculation as comparison sort, plus window +function work for each row. This is **not** a streaming tumbling window, +sliding window, pane layout, or exponential-histogram window framework. + +### `Limit { limit, offset }` + +Stops after consuming enough rows to satisfy an unordered limit. For example, +`SELECT * FROM metrics LIMIT 10 OFFSET 5` consumes at most fifteen rows and +emits at most ten. CPU is the number of consumed rows; memory is one output row +or batch. An ordered limit is represented by Sort plus Limit or a supported +Top-K implementation. + +### `PassThrough` + +Represents a proven row- and byte-preserving physical boundary with per-row +forwarding work. The current lowerer uses it only for a programmatically +constructed identity `TimeShift(default, Scan(metrics))`, whose query semantics +are the same as `SELECT * FROM metrics`; normal front ends omit that identity +wrapper. A non-identity PromQL `offset` or `@` changes source time coverage and +is unavailable until lowering propagates that temporal context into descendant +Scan evidence and physical identity. `PassThrough` must never hide an +unsupported operation. + ## Comparison and failure behavior Raw and post-ASAP alternatives must cover the same `ComparisonScope`: arrival From ce80e5f4b1eb9e48a2e2a1f8d8f5fc4204cd90f3 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 14:14:36 -0600 Subject: [PATCH 39/48] fix(cost): make PromQL physical estimates fail closed --- .../asap-aware-mapping/src/analytical_cost.rs | 1715 +++++++++++------ .../src/analytical_lowering.rs | 877 ++++++++- .../src/analytical_statistics.rs | 204 +- .../analytical-resource-cost.md | 82 +- .../physical-plan-integration.md | 106 +- 5 files changed, 2335 insertions(+), 649 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 87a1b0fd..e2bc0b6d 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize}; use crate::analytical_statistics::{ validate_comparison_scopes, ComparisonScope, EdgeStatistics, OperatorStatistics, - OperatorStatisticsProvider, PromqlBinaryOperandMode, SourceCoverage, + OperatorStatisticsProvider, PromqlEdgeStatistics, PromqlValueKind, SourceCoverage, }; pub const ANALYTICAL_MODEL_VERSION: &str = "analytical-resource-at-rest-v1"; @@ -105,17 +105,76 @@ pub enum PhysicalOperator { offset: u64, }, PassThrough, - PromqlRange, - PromqlSubquery, - PromqlVectorBinary, - PromqlRelabel, - PromqlInfoEnrich, - PromqlSeriesSample, + PromqlRange { + range_millis: u64, + }, + PromqlSubquery { + range_millis: u64, + resolution_millis: Option, + }, + PromqlBinary { + operation: PromqlBinaryOperation, + operand_mode: PromqlBinaryOperandMode, + cardinality: PromqlVectorCardinality, + build_side: Option, + }, + PromqlRelabel { + expression_operations_per_row: u64, + }, + PromqlInfoEnrich { + matcher_operations_per_info_row: u64, + }, + PromqlSeriesSample { + kind: PromqlSeriesSampleKind, + grouping_key_count: u64, + }, PromqlScalarToVector, PromqlVectorToScalar, PromqlScalarLeaf, - PromqlPerSeries, - PromqlPresence, + PromqlPerSeries { + operations_per_row: u64, + accumulator_count: u64, + }, + PromqlPresence { + kind: PromqlPresenceKind, + operations_per_row: u64, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum PromqlBinaryOperandMode { + VectorVector, + VectorScalar, + ScalarVector, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum PromqlBinaryOperation { + ArithmeticOrComparison, + And, + Or, + Unless, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum PromqlVectorCardinality { + OneToOne, + ManyToOne, + OneToMany, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum PromqlSeriesSampleKind { + LimitK { k: u64 }, + LimitRatio { ratio_bits: u64 }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum PromqlPresenceKind { + /// `absent` and `absent_over_time`: synthesize at most one output series. + Absent, + /// `present_over_time`: emit independently for each input series. + PresentPerSeries, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -415,220 +474,57 @@ fn validate_operator_statistics( } } validate_operator_semantics(node.operator, node_statistics)?; - if let Some(promql) = node_statistics.promql.as_ref() { - if promql.input_series.len() != expected_inputs { + validate_promql_child_edges(node, node_statistics, statistics)?; + Ok(()) +} + +fn validate_promql_child_edges( + node: &PhysicalDagNode, + node_statistics: &OperatorStatistics, + statistics: &HashMap<&str, OperatorStatistics>, +) -> Result<(), AnalyticalCostError> { + let output = node_statistics.promql_output(); + let child_has_promql = node + .children + .iter() + .any(|child_id| statistics[child_id.as_str()].promql_output().is_some()); + if output.is_none() && child_has_promql { + return Err(AnalyticalCostError::InvalidOperatorStatistics { + node: node.id.clone(), + reason: "operator drops child PromQL edge statistics", + }); + } + let Some(output) = output else { return Ok(()) }; + validate_promql_edge(output)?; + for (index, child_id) in node.children.iter().enumerate() { + let input = + node_statistics + .promql_input(index) + .ok_or(AnalyticalCostError::MissingOrStale( + "promql_input_edge_statistics", + ))?; + let child = statistics[child_id.as_str()].promql_output().ok_or( + AnalyticalCostError::MissingOrStale("promql_child_output_statistics"), + )?; + validate_promql_edge(input)?; + if input != child { return Err(AnalyticalCostError::InvalidOperatorStatistics { node: node.id.clone(), - reason: "PromQL input-series arity does not match physical inputs", + reason: "PromQL parent input does not match child output", }); } - for (index, child_id) in node.children.iter().enumerate() { - let child = &statistics[child_id.as_str()]; - let child_promql = child.promql.as_ref().ok_or_else(|| { - AnalyticalCostError::InvalidOperatorStatistics { - node: child_id.clone(), - reason: "PromQL child is missing series statistics", - } - })?; - if promql.input_series[index] != child_promql.output_series - || (!matches!(node.operator, PhysicalOperator::PromqlSubquery) - && promql.evaluation_steps != child_promql.evaluation_steps) - { - return Err(AnalyticalCostError::InvalidOperatorStatistics { - node: node.id.clone(), - reason: "PromQL edge series or evaluation steps conflict", - }); - } - } - if matches!(node.operator, PhysicalOperator::PromqlSubquery) { - let subquery_steps = promql - .subquery_steps - .filter(|steps| *steps > 0) - .ok_or(AnalyticalCostError::MissingOrZero("subquery_steps"))?; - let child = &statistics[node.children[0].as_str()]; - let child_steps = child - .promql - .as_ref() - .ok_or(AnalyticalCostError::MissingOrStale( - "child_promql_operator_statistics", - ))? - .evaluation_steps; - let expected = promql - .evaluation_steps - .checked_mul(subquery_steps) - .ok_or(AnalyticalCostError::Overflow)?; - if child_steps != expected { - return Err(AnalyticalCostError::InvalidOperatorStatistics { - node: node.id.clone(), - reason: "subquery child steps do not equal parent steps times subquery steps", - }); - } - } } Ok(()) } -pub(crate) fn validate_operator_semantics( - operator: PhysicalOperator, - statistics: &OperatorStatistics, -) -> Result<(), AnalyticalCostError> { - let invalid = |reason| Err(AnalyticalCostError::InconsistentOperatorStatistics(reason)); - if !matches!(operator, PhysicalOperator::Scan) && statistics.source_scan_bytes != 0 { - return invalid("only Scan may charge source bytes"); - } - if matches!(operator, PhysicalOperator::PromqlScalarLeaf) { - let promql = require_promql_statistics(statistics, 0)?; - if promql.output_series != 0 || statistics.output.rows != promql.evaluation_steps { - return invalid("PromQL scalar leaf must emit one scalar row per evaluation step"); - } - return Ok(()); +fn validate_promql_edge(edge: PromqlEdgeStatistics) -> Result<(), AnalyticalCostError> { + if edge.evaluation_steps == 0 { + return Err(AnalyticalCostError::MissingOrZero("evaluation_steps")); } - let input = statistics.inputs.first().copied().ok_or( - AnalyticalCostError::InconsistentOperatorStatistics("operator input is missing"), - )?; - let output = statistics.output; - match operator { - PhysicalOperator::Scan => { - if input != output { - return invalid("Scan external input edge does not match its output edge"); - } - } - PhysicalOperator::Filter => { - if output.rows > input.rows || output.bytes > input.bytes { - return invalid("filter output expands its input"); - } - } - PhysicalOperator::Project => { - if output.rows != input.rows { - return invalid("projection changes row cardinality"); - } - } - PhysicalOperator::HashAggregate => { - let groups = statistics - .group_count - .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - if groups == 0 && (input.rows != 0 || output.rows != 0) { - return invalid("zero groups require an empty grouped input and output"); - } - if output.rows != groups { - return invalid("aggregate output does not equal group cardinality"); - } - } - PhysicalOperator::Deduplicate => { - let groups = statistics - .group_count - .ok_or(AnalyticalCostError::MissingOrZero("group_count"))?; - if output.rows != groups || output.rows > input.rows { - return invalid("deduplicate output does not equal distinct cardinality"); - } - } - PhysicalOperator::Sort => { - if output != input { - return invalid("sort changes its input cardinality or width"); - } - } - PhysicalOperator::TopK => { - let k = statistics - .k - .filter(|k| *k > 0) - .ok_or(AnalyticalCostError::MissingOrZero("k"))?; - let offset = statistics - .topk_output_offset - .ok_or(AnalyticalCostError::MissingOrZero("topk_output_offset"))?; - if offset > k || output.rows != input.rows.min(k).saturating_sub(offset) { - return invalid("top-k output does not equal its cardinality bound"); - } - } - PhysicalOperator::Limit => { - if output.rows > input.rows { - return invalid("bounded output exceeds its input cardinality"); - } - } - PhysicalOperator::Window => { - if output.rows != input.rows { - return invalid("SQL window changes row cardinality"); - } - } - PhysicalOperator::PassThrough => { - if output != input { - return invalid("pass-through wrapper changes its edge statistics"); - } - } - PhysicalOperator::Concat => { - let totals = statistics.inputs.iter().try_fold( - EdgeStatistics { rows: 0, bytes: 0 }, - |total, edge| { - Ok::<_, AnalyticalCostError>(EdgeStatistics { - rows: total - .rows - .checked_add(edge.rows) - .ok_or(AnalyticalCostError::Overflow)?, - bytes: total - .bytes - .checked_add(edge.bytes) - .ok_or(AnalyticalCostError::Overflow)?, - }) - }, - )?; - if output != totals { - return invalid("concat output does not equal the sum of its inputs"); - } - } - PhysicalOperator::PromqlRange => { - let promql = require_promql_statistics(statistics, 1)?; - if !matches!(promql.window_samples_per_series, Some(value) if value > 0) { - return Err(AnalyticalCostError::MissingOrZero( - "window_samples_per_series", - )); - } - } - PhysicalOperator::PromqlSubquery => { - require_promql_statistics(statistics, 1)?; - } - PhysicalOperator::PromqlScalarToVector => { - validate_promql_bridge(operator, statistics)?; - } - PhysicalOperator::PromqlVectorToScalar => { - validate_promql_bridge(operator, statistics)?; - } - PhysicalOperator::PromqlVectorBinary => { - validate_promql_binary(statistics)?; - } - PhysicalOperator::PromqlInfoEnrich => { - let promql = require_promql_statistics(statistics, 2)?; - if output.rows != input.rows || promql.output_series != promql.input_series[0] { - return invalid("info enrichment changes left sample or series cardinality"); - } - if statistics.hash_join_build_side != Some(HashJoinBuildSide::Right) { - return invalid("info enrichment must build its label index from the right side"); - } - } - PhysicalOperator::PromqlRelabel => { - let promql = require_promql_statistics(statistics, 1)?; - if output.rows != input.rows || promql.output_series != promql.input_series[0] { - return invalid("relabel changes row or series cardinality"); - } - } - PhysicalOperator::PromqlSeriesSample => { - let promql = require_promql_statistics(statistics, 1)?; - if output.rows > input.rows || promql.output_series > promql.input_series[0] { - return invalid("series sampling expands its input"); - } - } - PhysicalOperator::PromqlPerSeries => { - let promql = require_promql_statistics(statistics, 1)?; - if promql.output_series > promql.input_series[0] { - return invalid("per-series operator expands series cardinality"); - } - } - PhysicalOperator::PromqlPresence => { - let promql = require_promql_statistics(statistics, 1)?; - if promql.output_series > 1 { - return invalid("PromQL absence operator emits more than one series"); - } - } - PhysicalOperator::HashJoin => {} - PhysicalOperator::PromqlScalarLeaf => unreachable!(), + if matches!(edge.value_kind, PromqlValueKind::Scalar) && edge.series != 0 { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "PromQL scalar edge cannot carry series", + )); } Ok(()) } @@ -667,6 +563,39 @@ fn statistics_match_operator(operator: PhysicalOperator, statistics: &OperatorSt PhysicalOperator::PassThrough => { matches!(statistics, OperatorStatistics::PassThrough { .. }) } + PhysicalOperator::PromqlRange { .. } => { + matches!(statistics, OperatorStatistics::PromqlRange { .. }) + } + PhysicalOperator::PromqlSubquery { .. } => { + matches!(statistics, OperatorStatistics::PromqlSubquery { .. }) + } + PhysicalOperator::PromqlBinary { .. } => { + matches!(statistics, OperatorStatistics::PromqlBinary { .. }) + } + PhysicalOperator::PromqlRelabel { .. } => { + matches!(statistics, OperatorStatistics::PromqlRelabel { .. }) + } + PhysicalOperator::PromqlInfoEnrich { .. } => { + matches!(statistics, OperatorStatistics::PromqlInfoEnrich { .. }) + } + PhysicalOperator::PromqlSeriesSample { .. } => { + matches!(statistics, OperatorStatistics::PromqlSeriesSample { .. }) + } + PhysicalOperator::PromqlScalarToVector => { + matches!(statistics, OperatorStatistics::PromqlScalarToVector { .. }) + } + PhysicalOperator::PromqlVectorToScalar => { + matches!(statistics, OperatorStatistics::PromqlVectorToScalar { .. }) + } + PhysicalOperator::PromqlScalarLeaf => { + matches!(statistics, OperatorStatistics::PromqlScalarLeaf { .. }) + } + PhysicalOperator::PromqlPerSeries { .. } => { + matches!(statistics, OperatorStatistics::PromqlPerSeries { .. }) + } + PhysicalOperator::PromqlPresence { .. } => { + matches!(statistics, OperatorStatistics::PromqlPresence { .. }) + } } } @@ -705,11 +634,25 @@ fn expected_input_arity( | PhysicalOperator::HashDeduplicate { .. } | PhysicalOperator::InMemoryAnalyticWindow { .. } | PhysicalOperator::Limit { .. } - | PhysicalOperator::PassThrough => unary, - PhysicalOperator::HashJoin { .. } => OperatorInputArity { + | PhysicalOperator::PassThrough + | PhysicalOperator::PromqlRange { .. } + | PhysicalOperator::PromqlSubquery { .. } + | PhysicalOperator::PromqlRelabel { .. } + | PhysicalOperator::PromqlSeriesSample { .. } + | PhysicalOperator::PromqlScalarToVector + | PhysicalOperator::PromqlVectorToScalar + | PhysicalOperator::PromqlPerSeries { .. } + | PhysicalOperator::PromqlPresence { .. } => unary, + PhysicalOperator::HashJoin { .. } + | PhysicalOperator::PromqlBinary { .. } + | PhysicalOperator::PromqlInfoEnrich { .. } => OperatorInputArity { statistics_inputs: 2, dag_children: 2, }, + PhysicalOperator::PromqlScalarLeaf => OperatorInputArity { + statistics_inputs: 0, + dag_children: 0, + }, PhysicalOperator::Concat => OperatorInputArity { statistics_inputs: variadic_child_count, dag_children: variadic_child_count, @@ -815,8 +758,27 @@ pub fn estimate_operator( .input(index) .ok_or(AnalyticalCostError::MissingOrZero("operator input edge")) }; - let left = input(0)?; let output = statistics.output(); + if let ( + PhysicalOperator::PromqlScalarLeaf, + OperatorStatistics::PromqlScalarLeaf { promql_output, .. }, + ) = (operator, &statistics) + { + validate_promql_edge(*promql_output)?; + if promql_output.value_kind != PromqlValueKind::Scalar + || output.rows != promql_output.evaluation_steps + { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "PromQL scalar leaf must emit one scalar row per evaluation step", + )); + } + return Ok(ResourceEstimate { + cpu_ops: output.rows as f64, + peak_memory_bytes: per_row_width(output.rows, output.bytes)?, + scan_bytes: 0, + }); + } + let left = input(0)?; let estimate = match (operator, &statistics) { ( PhysicalOperator::Scan, @@ -994,6 +956,160 @@ pub fn estimate_operator( scan_bytes: 0, } } + ( + PhysicalOperator::PromqlRange { .. }, + OperatorStatistics::PromqlRange { + edges, + max_window_samples_per_series, + }, + ) => { + let promql = require_promql_unary(edges)?; + ResourceEstimate { + cpu_ops: left.rows as f64, + peak_memory_bytes: checked_bytes(&[ + promql.input.series, + *max_window_samples_per_series, + per_row_width(left.rows, left.bytes)?, + ])?, + scan_bytes: 0, + } + } + ( + PhysicalOperator::PromqlSubquery { .. }, + OperatorStatistics::PromqlSubquery { edges, .. }, + ) => ResourceEstimate { + cpu_ops: left.rows as f64 + output.rows as f64, + peak_memory_bytes: edges.input.bytes, + scan_bytes: 0, + }, + ( + PhysicalOperator::PromqlBinary { + operand_mode, + build_side, + .. + }, + OperatorStatistics::PromqlBinary { + edges, + matching_key_bytes, + }, + ) => { + let promql = require_promql_binary(edges)?; + let right = input(1)?; + let matching_bytes = match operand_mode { + PromqlBinaryOperandMode::VectorScalar | PromqlBinaryOperandMode::ScalarVector => { + per_row_width(output.rows, output.bytes)? + } + PromqlBinaryOperandMode::VectorVector => { + let build_series = match build_side.ok_or( + AnalyticalCostError::MissingOrStale("vector_match_build_side"), + )? { + HashJoinBuildSide::Left => promql.inputs[0].series, + HashJoinBuildSide::Right => promql.inputs[1].series, + }; + checked_bytes(&[ + build_series, + matching_key_bytes + .checked_add(16) + .ok_or(AnalyticalCostError::Overflow)?, + ])? + } + }; + ResourceEstimate { + cpu_ops: left.rows as f64 + right.rows as f64 + output.rows as f64, + peak_memory_bytes: matching_bytes, + scan_bytes: 0, + } + } + ( + PhysicalOperator::PromqlRelabel { + expression_operations_per_row, + }, + OperatorStatistics::PromqlRelabel { .. }, + ) => ResourceEstimate { + cpu_ops: checked_cpu_product(left.rows, expression_operations_per_row)?, + peak_memory_bytes: per_row_width(output.rows, output.bytes)?, + scan_bytes: 0, + }, + ( + PhysicalOperator::PromqlInfoEnrich { + matcher_operations_per_info_row, + }, + OperatorStatistics::PromqlInfoEnrich { + edges, + matching_key_bytes, + }, + ) => { + let promql = require_promql_binary(edges)?; + let right = input(1)?; + ResourceEstimate { + cpu_ops: left.rows as f64 + + right.rows as f64 + + output.rows as f64 + + checked_cpu_product(right.rows, matcher_operations_per_info_row)?, + peak_memory_bytes: checked_bytes(&[ + promql.inputs[1].series, + matching_key_bytes + .checked_add(16) + .ok_or(AnalyticalCostError::Overflow)?, + ])?, + scan_bytes: 0, + } + } + ( + PhysicalOperator::PromqlSeriesSample { .. }, + OperatorStatistics::PromqlSeriesSample { + edges, key_bytes, .. + }, + ) => { + let promql = require_promql_unary(edges)?; + ResourceEstimate { + cpu_ops: left.rows as f64 + promql.input.series as f64, + peak_memory_bytes: checked_bytes(&[ + promql.output.series, + key_bytes + .checked_add(16) + .ok_or(AnalyticalCostError::Overflow)?, + ])?, + scan_bytes: 0, + } + } + (PhysicalOperator::PromqlScalarToVector | PhysicalOperator::PromqlVectorToScalar, _) => { + ResourceEstimate { + cpu_ops: left.rows as f64 + output.rows as f64, + peak_memory_bytes: per_row_width(output.rows, output.bytes)?, + scan_bytes: 0, + } + } + ( + PhysicalOperator::PromqlPerSeries { + operations_per_row, .. + }, + OperatorStatistics::PromqlPerSeries { + edges, + accumulator_bytes_per_series, + }, + ) => { + let promql = require_promql_unary(edges)?; + ResourceEstimate { + cpu_ops: checked_cpu_product(left.rows, operations_per_row)?, + peak_memory_bytes: checked_bytes(&[ + promql.input.series, + *accumulator_bytes_per_series, + ])?, + scan_bytes: 0, + } + } + ( + PhysicalOperator::PromqlPresence { + operations_per_row, .. + }, + OperatorStatistics::PromqlPresence { .. }, + ) => ResourceEstimate { + cpu_ops: checked_cpu_product(left.rows, operations_per_row)? + output.rows as f64, + peak_memory_bytes: per_row_width(output.rows, output.bytes)?, + scan_bytes: 0, + }, + (PhysicalOperator::PromqlScalarLeaf, _) => unreachable!(), _ => { return Err(AnalyticalCostError::InconsistentOperatorStatistics( "statistics variant does not match physical operator", @@ -1024,6 +1140,21 @@ pub(crate) fn validate_operator_semantics( ) { return inconsistent("Concat must have at least one input"); } + if let ( + PhysicalOperator::PromqlScalarLeaf, + OperatorStatistics::PromqlScalarLeaf { + output, + promql_output, + }, + ) = (operator, statistics) + { + validate_promql_edge_shape(*output, *promql_output)?; + return if promql_output.value_kind == PromqlValueKind::Scalar { + Ok(()) + } else { + inconsistent("PromQL scalar leaf output is not scalar") + }; + } let input = statistics .input(0) .ok_or(AnalyticalCostError::InconsistentOperatorStatistics( @@ -1034,7 +1165,8 @@ pub(crate) fn validate_operator_semantics( ( PhysicalOperator::Scan, OperatorStatistics::Scan { - source_read_bytes, .. + edges, + source_read_bytes, }, ) => { if input != output { @@ -1043,6 +1175,13 @@ pub(crate) fn validate_operator_semantics( if input.rows > 0 && *source_read_bytes == 0 { return inconsistent("non-empty Scan has zero source-read bytes"); } + if let Some(promql) = edges.promql { + validate_promql_edge_shape(edges.input, promql.input)?; + validate_promql_edge_shape(edges.output, promql.output)?; + if promql.input != promql.output { + return inconsistent("Scan changes its PromQL edge shape"); + } + } } ( PhysicalOperator::Filter { @@ -1056,6 +1195,9 @@ pub(crate) fn validate_operator_semantics( if output.rows > input.rows || output.bytes > input.bytes { return inconsistent("Filter output expands its input"); } + if let Some(promql) = statistics.unary_promql() { + validate_promql_filter_shape(promql)?; + } } ( PhysicalOperator::Project { @@ -1069,6 +1211,9 @@ pub(crate) fn validate_operator_semantics( if output.rows != input.rows { return inconsistent("Project changes row cardinality"); } + if let Some(promql) = statistics.unary_promql() { + validate_promql_cardinality_preserving_shape(promql)?; + } } ( PhysicalOperator::HashAggregate { @@ -1082,6 +1227,11 @@ pub(crate) fn validate_operator_semantics( .. }, ) => { + if statistics.promql_output().is_some() { + return inconsistent( + "PromQL cross-series aggregation needs an explicit physical operator", + ); + } if accumulator_count == 0 || *accumulator_bytes_per_group == 0 { return inconsistent("HashAggregate accumulator work and width must be positive"); } @@ -1134,6 +1284,9 @@ pub(crate) fn validate_operator_semantics( if input != output { return inconsistent("cardinality-preserving operator changes its edge"); } + if let Some(promql) = statistics.unary_promql() { + validate_promql_cardinality_preserving_shape(promql)?; + } } ( PhysicalOperator::InMemoryAnalyticWindow { @@ -1175,6 +1328,34 @@ pub(crate) fn validate_operator_semantics( if output != total { return inconsistent("Concat output differs from the sum of its inputs"); } + if let OperatorStatistics::Concat { + promql: Some(promql), + .. + } = statistics + { + if promql.inputs.len() != inputs.len() || promql.inputs.is_empty() { + return inconsistent("PromQL Concat edge arity is invalid"); + } + let first = promql.inputs[0]; + let series_bound = promql.inputs.iter().try_fold(0_u64, |total, edge| { + if edge.evaluation_steps != first.evaluation_steps + || edge.value_kind != first.value_kind + { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "PromQL Concat inputs have different shapes", + )); + } + total + .checked_add(edge.series) + .ok_or(AnalyticalCostError::Overflow) + })?; + if promql.output.evaluation_steps != first.evaluation_steps + || promql.output.value_kind != first.value_kind + || promql.output.series > series_bound + { + return inconsistent("PromQL Concat output exceeds its input-series bound"); + } + } } ( PhysicalOperator::TopK { @@ -1208,73 +1389,473 @@ pub(crate) fn validate_operator_semantics( return inconsistent("HashJoin has no equality keys"); } } - _ => return inconsistent("statistics variant does not match physical operator"), - } - Ok(()) -} - -fn validate_promql_binary( - statistics: &OperatorStatistics, -) -> Result { - let promql = require_promql_statistics(statistics, 2)?; - let mode = promql - .binary_operand_mode - .ok_or(AnalyticalCostError::MissingOrStale( - "promql_binary_operand_mode", - ))?; - match mode { - PromqlBinaryOperandMode::VectorVector => { - if statistics.hash_join_build_side.is_none() || statistics.key_bytes.is_none() { - return Err(AnalyticalCostError::MissingOrStale( - "vector_binary_label_match_statistics", - )); + ( + PhysicalOperator::PromqlRange { range_millis }, + OperatorStatistics::PromqlRange { + edges, + max_window_samples_per_series, + }, + ) => { + let promql = require_promql_unary(edges)?; + if range_millis == 0 || *max_window_samples_per_series == 0 { + return inconsistent("PromQL range needs a positive range and window bound"); } - } - PromqlBinaryOperandMode::VectorScalar => { - if promql.input_series[1] != 0 - || statistics.inputs[1].rows != promql.evaluation_steps - || statistics.hash_join_build_side.is_some() - || statistics.key_bytes.is_some() + if promql.input.value_kind != PromqlValueKind::Vector + || promql.output.value_kind != PromqlValueKind::RangeVector + || promql.input.series != promql.output.series + || promql.input.evaluation_steps != promql.output.evaluation_steps { - return Err(AnalyticalCostError::InconsistentOperatorStatistics( - "vector/scalar binary evidence has an invalid scalar edge or label-match state", - )); + return inconsistent("PromQL range edge shape is invalid"); } } - PromqlBinaryOperandMode::ScalarVector => { - if promql.input_series[0] != 0 - || statistics.inputs[0].rows != promql.evaluation_steps - || statistics.hash_join_build_side.is_some() - || statistics.key_bytes.is_some() + ( + PhysicalOperator::PromqlSubquery { + range_millis, + resolution_millis, + }, + OperatorStatistics::PromqlSubquery { + edges, + subquery_steps, + }, + ) => { + let promql = require_promql_unary(edges)?; + if range_millis == 0 + || resolution_millis == Some(0) + || *subquery_steps == 0 + || promql.input.value_kind != PromqlValueKind::RangeVector + || promql.output.value_kind != PromqlValueKind::RangeVector + || promql.input.series != promql.output.series { - return Err(AnalyticalCostError::InconsistentOperatorStatistics( - "scalar/vector binary evidence has an invalid scalar edge or label-match state", - )); + return inconsistent("PromQL subquery configuration or edge shape is invalid"); + } + let expected = promql + .output + .evaluation_steps + .checked_mul(*subquery_steps) + .ok_or(AnalyticalCostError::Overflow)?; + if promql.input.evaluation_steps != expected { + return inconsistent( + "subquery child steps do not equal output steps times subquery steps", + ); + } + } + ( + PhysicalOperator::PromqlBinary { + operation, + operand_mode, + cardinality, + build_side, + }, + OperatorStatistics::PromqlBinary { + edges, + matching_key_bytes, + }, + ) => validate_promql_binary( + operation, + operand_mode, + cardinality, + build_side, + *matching_key_bytes, + edges, + )?, + ( + PhysicalOperator::PromqlRelabel { + expression_operations_per_row, + }, + OperatorStatistics::PromqlRelabel { edges }, + ) => { + let promql = require_promql_unary(edges)?; + validate_instant_vector_rows(output, promql.output)?; + if expression_operations_per_row == 0 + || input.rows != output.rows + || promql.input.series != promql.output.series + || promql.input.evaluation_steps != promql.output.evaluation_steps + || promql.input.value_kind != PromqlValueKind::Vector + || promql.output.value_kind != PromqlValueKind::Vector + { + return inconsistent("PromQL relabel configuration or edge shape is invalid"); + } + } + ( + PhysicalOperator::PromqlInfoEnrich { + matcher_operations_per_info_row, + }, + OperatorStatistics::PromqlInfoEnrich { + edges, + matching_key_bytes, + }, + ) => { + let promql = require_promql_binary(edges)?; + validate_instant_vector_rows(output, promql.output)?; + if matcher_operations_per_info_row == 0 + || *matching_key_bytes == 0 + || input.rows != output.rows + || promql.inputs[0].series != promql.output.series + || promql.inputs[0].evaluation_steps != promql.output.evaluation_steps + || promql.inputs[1].evaluation_steps != promql.output.evaluation_steps + || promql + .inputs + .iter() + .any(|edge| edge.value_kind != PromqlValueKind::Vector) + || promql.output.value_kind != PromqlValueKind::Vector + { + return inconsistent( + "PromQL info enrichment configuration or edge shape is invalid", + ); + } + } + ( + PhysicalOperator::PromqlSeriesSample { + kind, + grouping_key_count, + }, + OperatorStatistics::PromqlSeriesSample { + edges, + group_count, + key_bytes, + }, + ) => validate_promql_series_sample( + kind, + grouping_key_count, + *group_count, + *key_bytes, + edges, + )?, + ( + PhysicalOperator::PromqlScalarToVector, + OperatorStatistics::PromqlScalarToVector { edges }, + ) + | ( + PhysicalOperator::PromqlVectorToScalar, + OperatorStatistics::PromqlVectorToScalar { edges }, + ) => { + validate_promql_bridge(operator, edges)?; + } + (PhysicalOperator::PromqlScalarLeaf, _) => unreachable!(), + ( + PhysicalOperator::PromqlPerSeries { + operations_per_row, + accumulator_count, + }, + OperatorStatistics::PromqlPerSeries { + edges, + accumulator_bytes_per_series, + }, + ) => { + let promql = require_promql_unary(edges)?; + validate_instant_vector_rows(output, promql.output)?; + if operations_per_row == 0 + || accumulator_count == 0 + || *accumulator_bytes_per_series == 0 + || promql.output.value_kind != PromqlValueKind::Vector + || promql.output.series > promql.input.series + || promql.output.evaluation_steps != promql.input.evaluation_steps + { + return inconsistent("PromQL per-series configuration or edge shape is invalid"); + } + } + ( + PhysicalOperator::PromqlPresence { + kind, + operations_per_row, + }, + OperatorStatistics::PromqlPresence { edges }, + ) => { + let promql = require_promql_unary(edges)?; + validate_instant_vector_rows(output, promql.output)?; + if operations_per_row == 0 || promql.output.value_kind != PromqlValueKind::Vector { + return inconsistent("PromQL presence output violates its per-step bound"); + } + match kind { + PromqlPresenceKind::Absent + if promql.output.series > 1 + || output.rows > promql.output.evaluation_steps + || (input.rows == 0 && output.rows != promql.output.evaluation_steps) + || (output.rows == 0 && promql.output.series != 0) + || (output.rows > 0 && promql.output.series != 1) => + { + return inconsistent("PromQL absence output violates its per-step bound"); + } + PromqlPresenceKind::PresentPerSeries + if output.rows > input.rows + || promql.output.series > promql.input.series + || promql.output.evaluation_steps != promql.input.evaluation_steps + || (input.rows == 0 && (output.rows != 0 || promql.output.series != 0)) => + { + return inconsistent( + "PromQL present-over-time output exceeds its per-series bound", + ); + } + _ => {} } } + _ => return inconsistent("statistics variant does not match physical operator"), } - Ok(mode) + Ok(()) } -fn require_promql_statistics( - statistics: &OperatorStatistics, - inputs: usize, -) -> Result<&crate::analytical_statistics::PromqlOperatorStatistics, AnalyticalCostError> { - let promql = statistics - .promql - .as_ref() - .ok_or(AnalyticalCostError::MissingOrStale( - "promql_operator_statistics", - ))?; - if promql.evaluation_steps == 0 { - return Err(AnalyticalCostError::MissingOrZero("evaluation_steps")); +fn require_promql_unary( + edges: &crate::analytical_statistics::UnaryEdgeStatistics, +) -> Result { + let promql = edges.promql.ok_or(AnalyticalCostError::MissingOrStale( + "promql_edge_statistics", + ))?; + validate_promql_edge_shape(edges.input, promql.input)?; + validate_promql_edge_shape(edges.output, promql.output)?; + Ok(promql) +} + +fn require_promql_binary( + edges: &crate::analytical_statistics::BinaryEdgeStatistics, +) -> Result { + let promql = edges.promql.ok_or(AnalyticalCostError::MissingOrStale( + "promql_edge_statistics", + ))?; + validate_promql_edge_shape(edges.inputs[0], promql.inputs[0])?; + validate_promql_edge_shape(edges.inputs[1], promql.inputs[1])?; + validate_promql_edge_shape(edges.output, promql.output)?; + Ok(promql) +} + +fn validate_promql_cardinality_preserving_shape( + promql: crate::analytical_statistics::PromqlUnaryEdgeStatistics, +) -> Result<(), AnalyticalCostError> { + validate_promql_edge(promql.input)?; + validate_promql_edge(promql.output)?; + if promql.input == promql.output { + Ok(()) + } else { + Err(AnalyticalCostError::InconsistentOperatorStatistics( + "cardinality-preserving operator changes its PromQL edge shape", + )) } - if promql.input_series.len() != inputs { +} + +fn validate_promql_filter_shape( + promql: crate::analytical_statistics::PromqlUnaryEdgeStatistics, +) -> Result<(), AnalyticalCostError> { + validate_promql_edge(promql.input)?; + validate_promql_edge(promql.output)?; + if promql.input.evaluation_steps == promql.output.evaluation_steps + && promql.input.value_kind == promql.output.value_kind + && promql.output.series <= promql.input.series + { + Ok(()) + } else { + Err(AnalyticalCostError::InconsistentOperatorStatistics( + "Filter changes PromQL steps/kind or expands its series", + )) + } +} + +fn validate_instant_vector_rows( + logical: EdgeStatistics, + promql: PromqlEdgeStatistics, +) -> Result<(), AnalyticalCostError> { + if promql.value_kind != PromqlValueKind::Vector { return Err(AnalyticalCostError::InconsistentOperatorStatistics( - "PromQL input-series arity does not match physical inputs", + "instant-vector result has a non-vector value kind", )); } - Ok(promql) + let bound = promql + .series + .checked_mul(promql.evaluation_steps) + .ok_or(AnalyticalCostError::Overflow)?; + if logical.rows <= bound { + Ok(()) + } else { + Err(AnalyticalCostError::InconsistentOperatorStatistics( + "instant-vector rows exceed series times evaluation steps", + )) + } +} + +fn validate_promql_edge_shape( + logical: EdgeStatistics, + promql: PromqlEdgeStatistics, +) -> Result<(), AnalyticalCostError> { + validate_promql_edge(promql)?; + let max_rows = promql.series.checked_mul(promql.evaluation_steps); + match promql.value_kind { + PromqlValueKind::Scalar if logical.rows != promql.evaluation_steps => { + Err(AnalyticalCostError::InconsistentOperatorStatistics( + "PromQL scalar rows do not equal evaluation steps", + )) + } + PromqlValueKind::RangeVector if max_rows.is_none_or(|bound| logical.rows > bound) => { + Err(AnalyticalCostError::InconsistentOperatorStatistics( + "PromQL vector rows exceed series times evaluation steps", + )) + } + PromqlValueKind::Vector if logical.rows > 0 && promql.series == 0 => { + Err(AnalyticalCostError::InconsistentOperatorStatistics( + "PromQL samples have no source series", + )) + } + _ => Ok(()), + } +} + +fn validate_promql_bridge( + operator: PhysicalOperator, + edges: &crate::analytical_statistics::UnaryEdgeStatistics, +) -> Result<(), AnalyticalCostError> { + let promql = require_promql_unary(edges)?; + let valid = match operator { + PhysicalOperator::PromqlScalarToVector => { + validate_instant_vector_rows(edges.output, promql.output)?; + promql.input.value_kind == PromqlValueKind::Scalar + && promql.output.value_kind == PromqlValueKind::Vector + && promql.output.series == 1 + && promql.input.evaluation_steps == promql.output.evaluation_steps + && edges.input.rows == edges.output.rows + } + PhysicalOperator::PromqlVectorToScalar => { + promql.input.value_kind == PromqlValueKind::Vector + && promql.output.value_kind == PromqlValueKind::Scalar + && promql.input.evaluation_steps == promql.output.evaluation_steps + } + _ => false, + }; + if valid { + Ok(()) + } else { + Err(AnalyticalCostError::InconsistentOperatorStatistics( + "PromQL scalar/vector bridge edge shape is invalid", + )) + } +} + +fn validate_promql_binary( + operation: PromqlBinaryOperation, + operand_mode: PromqlBinaryOperandMode, + cardinality: PromqlVectorCardinality, + build_side: Option, + matching_key_bytes: u64, + edges: &crate::analytical_statistics::BinaryEdgeStatistics, +) -> Result<(), AnalyticalCostError> { + let promql = require_promql_binary(edges)?; + validate_instant_vector_rows(edges.output, promql.output)?; + let invalid = |reason| Err(AnalyticalCostError::InconsistentOperatorStatistics(reason)); + if matches!( + operation, + PromqlBinaryOperation::And | PromqlBinaryOperation::Or | PromqlBinaryOperation::Unless + ) && cardinality != PromqlVectorCardinality::OneToOne + { + return invalid("PromQL set operations cannot use group_left or group_right"); + } + if promql.output.value_kind != PromqlValueKind::Vector + || promql.inputs[0].evaluation_steps != promql.output.evaluation_steps + || promql.inputs[1].evaluation_steps != promql.output.evaluation_steps + { + return invalid("PromQL binary evaluation steps or output kind are invalid"); + } + let (row_bound, series_bound) = match operand_mode { + PromqlBinaryOperandMode::VectorScalar => { + if promql.inputs[0].value_kind != PromqlValueKind::Vector + || promql.inputs[1].value_kind != PromqlValueKind::Scalar + || cardinality != PromqlVectorCardinality::OneToOne + || build_side.is_some() + || matching_key_bytes != 0 + { + return invalid("vector/scalar binary configuration is invalid"); + } + (edges.inputs[0].rows, promql.inputs[0].series) + } + PromqlBinaryOperandMode::ScalarVector => { + if promql.inputs[0].value_kind != PromqlValueKind::Scalar + || promql.inputs[1].value_kind != PromqlValueKind::Vector + || cardinality != PromqlVectorCardinality::OneToOne + || build_side.is_some() + || matching_key_bytes != 0 + { + return invalid("scalar/vector binary configuration is invalid"); + } + (edges.inputs[1].rows, promql.inputs[1].series) + } + PromqlBinaryOperandMode::VectorVector => { + if promql + .inputs + .iter() + .any(|input| input.value_kind != PromqlValueKind::Vector) + || build_side.is_none() + || matching_key_bytes == 0 + { + return invalid("vector/vector binary label-match configuration is invalid"); + } + match operation { + PromqlBinaryOperation::And | PromqlBinaryOperation::Unless => { + (edges.inputs[0].rows, promql.inputs[0].series) + } + PromqlBinaryOperation::Or => ( + edges.inputs[0] + .rows + .checked_add(edges.inputs[1].rows) + .ok_or(AnalyticalCostError::Overflow)?, + promql.inputs[0] + .series + .checked_add(promql.inputs[1].series) + .ok_or(AnalyticalCostError::Overflow)?, + ), + PromqlBinaryOperation::ArithmeticOrComparison => match cardinality { + PromqlVectorCardinality::OneToOne => ( + edges.inputs[0].rows.min(edges.inputs[1].rows), + promql.inputs[0].series.min(promql.inputs[1].series), + ), + PromqlVectorCardinality::ManyToOne => { + (edges.inputs[0].rows, promql.inputs[0].series) + } + PromqlVectorCardinality::OneToMany => { + (edges.inputs[1].rows, promql.inputs[1].series) + } + }, + } + } + }; + if edges.output.rows > row_bound || promql.output.series > series_bound { + return invalid("PromQL binary output exceeds its semantic cardinality bound"); + } + Ok(()) +} + +fn validate_promql_series_sample( + kind: PromqlSeriesSampleKind, + grouping_key_count: u64, + group_count: u64, + key_bytes: u64, + edges: &crate::analytical_statistics::UnaryEdgeStatistics, +) -> Result<(), AnalyticalCostError> { + let promql = require_promql_unary(edges)?; + validate_instant_vector_rows(edges.output, promql.output)?; + let invalid = |reason| Err(AnalyticalCostError::InconsistentOperatorStatistics(reason)); + if key_bytes == 0 + || promql.input.value_kind != PromqlValueKind::Vector + || promql.output.value_kind != PromqlValueKind::Vector + || promql.input.evaluation_steps != promql.output.evaluation_steps + || edges.output.rows > edges.input.rows + || promql.output.series > promql.input.series + || (grouping_key_count == 0 && group_count != 1) + || (grouping_key_count > 0 && group_count == 0 && promql.input.series > 0) + { + return invalid("PromQL series-sample configuration or edge shape is invalid"); + } + if let PromqlSeriesSampleKind::LimitK { k } = kind { + if k == 0 { + return invalid("PromQL limitk must be positive"); + } + let bound = group_count + .checked_mul(k) + .ok_or(AnalyticalCostError::Overflow)?; + if promql.output.series > promql.input.series.min(bound) { + return invalid("PromQL limitk output exceeds its group bound"); + } + } else if let PromqlSeriesSampleKind::LimitRatio { ratio_bits } = kind { + let ratio = f64::from_bits(ratio_bits); + if !ratio.is_finite() || !(-1.0..=1.0).contains(&ratio) { + return invalid("PromQL limit_ratio is outside [-1, 1]"); + } + } + Ok(()) } impl ResourceEstimate { @@ -1360,7 +1941,8 @@ mod tests { use super::*; use crate::analytical_statistics::{ validate_comparison_scopes, BinaryEdgeStatistics, ComparisonScope, EdgeStatistics, - OperatorStatistics, PartitionStatistics, SourceCoverage, UnaryEdgeStatistics, + OperatorStatistics, PartitionStatistics, PromqlBinaryEdgeStatistics, PromqlEdgeStatistics, + PromqlUnaryEdgeStatistics, PromqlValueKind, SourceCoverage, UnaryEdgeStatistics, }; fn filter_operator() -> PhysicalOperator { @@ -1392,6 +1974,48 @@ mod tests { rows: output_rows, bytes: output_rows.saturating_mul(8), }, + promql: None, + } + } + + fn promql_edge( + series: u64, + evaluation_steps: u64, + value_kind: PromqlValueKind, + ) -> PromqlEdgeStatistics { + PromqlEdgeStatistics { + series, + evaluation_steps, + value_kind, + } + } + + fn promql_binary_statistics(output_rows: u64, output_series: u64) -> OperatorStatistics { + let left = EdgeStatistics { + rows: 100, + bytes: 1_600, + }; + let right = EdgeStatistics { + rows: 50, + bytes: 800, + }; + let output = EdgeStatistics { + rows: output_rows, + bytes: output_rows.saturating_mul(16), + }; + OperatorStatistics::PromqlBinary { + edges: BinaryEdgeStatistics { + inputs: [left, right], + output, + promql: Some(PromqlBinaryEdgeStatistics { + inputs: [ + promql_edge(10, 10, PromqlValueKind::Vector), + promql_edge(5, 10, PromqlValueKind::Vector), + ], + output: promql_edge(output_series, 10, PromqlValueKind::Vector), + }), + }, + matching_key_bytes: 8, } } @@ -1433,6 +2057,219 @@ mod tests { ); } + #[test] + fn promql_binary_operation_controls_cardinality_bounds() { + let operator = |operation| PhysicalOperator::PromqlBinary { + operation, + operand_mode: PromqlBinaryOperandMode::VectorVector, + cardinality: PromqlVectorCardinality::OneToOne, + build_side: Some(HashJoinBuildSide::Right), + }; + + assert!(estimate_operator( + operator(PromqlBinaryOperation::Or), + promql_binary_statistics(150, 15), + ) + .is_ok()); + assert!(matches!( + estimate_operator( + operator(PromqlBinaryOperation::ArithmeticOrComparison), + promql_binary_statistics(60, 5), + ), + Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) + )); + assert!(matches!( + estimate_operator( + operator(PromqlBinaryOperation::And), + promql_binary_statistics(101, 10), + ), + Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) + )); + } + + #[test] + fn promql_bridges_require_directional_edge_kinds() { + let scalar = promql_edge(0, 10, PromqlValueKind::Scalar); + let vector = promql_edge(1, 10, PromqlValueKind::Vector); + let edges = |input, output| UnaryEdgeStatistics { + input: EdgeStatistics { + rows: 10, + bytes: 80, + }, + output: EdgeStatistics { + rows: 10, + bytes: 80, + }, + promql: Some(PromqlUnaryEdgeStatistics { input, output }), + }; + assert!(estimate_operator( + PhysicalOperator::PromqlScalarToVector, + OperatorStatistics::PromqlScalarToVector { + edges: edges(scalar, vector), + }, + ) + .is_ok()); + assert!(matches!( + estimate_operator( + PhysicalOperator::PromqlScalarToVector, + OperatorStatistics::PromqlScalarToVector { + edges: edges(vector, scalar), + }, + ), + Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) + )); + } + + #[test] + fn promql_scalar_leaf_has_no_input_and_one_row_per_step() { + let statistics = OperatorStatistics::PromqlScalarLeaf { + output: EdgeStatistics { + rows: 10, + bytes: 80, + }, + promql_output: promql_edge(0, 10, PromqlValueKind::Scalar), + }; + let estimate = estimate_operator(PhysicalOperator::PromqlScalarLeaf, statistics).unwrap(); + + assert_eq!(estimate.cpu_ops, 10.0); + assert_eq!(estimate.peak_memory_bytes, 8); + } + + #[test] + fn physical_dag_cannot_drop_promql_edge_evidence() { + let edge = EdgeStatistics { + rows: 10, + bytes: 80, + }; + let promql = promql_edge(1, 10, PromqlValueKind::Vector); + let nodes = vec![ + PhysicalDagNode { + id: "scan".into(), + operator: PhysicalOperator::Scan, + children: vec![], + source_coverage: Some(comparison_scope().sources[0].clone()), + output_buffer_bytes: 8, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }, + PhysicalDagNode { + id: "filter".into(), + operator: filter_operator(), + children: vec!["scan".into()], + source_coverage: None, + output_buffer_bytes: 8, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }, + ]; + let provided = HashMap::from([ + ( + "scan".into(), + OperatorStatistics::Scan { + edges: UnaryEdgeStatistics { + input: edge, + output: edge, + promql: Some(PromqlUnaryEdgeStatistics { + input: promql, + output: promql, + }), + }, + source_read_bytes: 80, + }, + ), + ( + "filter".into(), + OperatorStatistics::Filter { + edges: unary_edges(edge, edge), + }, + ), + ]); + + assert!(matches!( + estimate_physical_dag(&nodes, "filter", &comparison_scope(), &provided), + Err(AnalyticalCostError::InvalidOperatorStatistics { + reason: "operator drops child PromQL edge statistics", + .. + }) + )); + } + + #[test] + fn generic_hash_aggregate_cannot_masquerade_as_promql_aggregation() { + let input = EdgeStatistics { + rows: 10, + bytes: 80, + }; + let output = EdgeStatistics { rows: 1, bytes: 8 }; + let statistics = OperatorStatistics::HashAggregate { + edges: UnaryEdgeStatistics { + input, + output, + promql: Some(PromqlUnaryEdgeStatistics { + input: promql_edge(1, 10, PromqlValueKind::Vector), + output: promql_edge(1, 10, PromqlValueKind::Vector), + }), + }, + group_count: 1, + key_bytes: 0, + accumulator_bytes_per_group: 8, + }; + + assert_eq!( + estimate_operator( + PhysicalOperator::HashAggregate { + grouping_key_count: 0, + accumulator_count: 1, + }, + statistics, + ), + Err(AnalyticalCostError::InconsistentOperatorStatistics( + "PromQL cross-series aggregation needs an explicit physical operator" + )) + ); + } + + #[test] + fn present_over_time_keeps_per_series_cardinality() { + let input = EdgeStatistics { + rows: 100, + bytes: 1_600, + }; + let output = EdgeStatistics { + rows: 80, + bytes: 1_280, + }; + let valid = OperatorStatistics::PromqlPresence { + edges: UnaryEdgeStatistics { + input, + output, + promql: Some(PromqlUnaryEdgeStatistics { + input: promql_edge(10, 10, PromqlValueKind::RangeVector), + output: promql_edge(8, 10, PromqlValueKind::Vector), + }), + }, + }; + + assert!(estimate_operator( + PhysicalOperator::PromqlPresence { + kind: PromqlPresenceKind::PresentPerSeries, + operations_per_row: 1, + }, + valid.clone(), + ) + .is_ok()); + assert!(matches!( + estimate_operator( + PhysicalOperator::PromqlPresence { + kind: PromqlPresenceKind::Absent, + operations_per_row: 1, + }, + valid, + ), + Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) + )); + } + #[test] fn operator_estimator_rejects_contradictory_cardinality_evidence() { assert!(matches!( @@ -1497,6 +2334,7 @@ mod tests { rows: 1_000, bytes: 64_000, }, + promql: None, }, }, ) @@ -1519,6 +2357,7 @@ mod tests { rows: 10, bytes: 400, }, + promql: None, }, }, ) @@ -1564,6 +2403,7 @@ mod tests { rows: 100, bytes: 12_800, }, + promql: None, }, }, ) @@ -1585,6 +2425,7 @@ mod tests { rows: 100, bytes: 4_000, }, + promql: None, }, }, ) @@ -1607,6 +2448,7 @@ mod tests { rows: 4, bytes: 160, }, + promql: None, }, }, ) @@ -1628,6 +2470,7 @@ mod tests { rows: 10, bytes: 400, }, + promql: None, }, }, ) @@ -1712,6 +2555,7 @@ mod tests { rows: 80, bytes: 800, }, + promql: None, }, ), ]); @@ -1821,333 +2665,19 @@ mod tests { }, source_snapshot_id: "catalog-version-42".into(), predicates: vec![], + info_matchers: vec![], }], } } fn unary_edges(input: EdgeStatistics, output: EdgeStatistics) -> UnaryEdgeStatistics { - UnaryEdgeStatistics { input, output } - } - - fn manual_unary_plan( - operator: PhysicalOperator, - operator_statistics: OperatorStatistics, - ) -> (Vec, HashMap) { - let coverage = comparison_scope().sources[0].clone(); - let scan_edge = operator_statistics.inputs[0]; - let nodes = vec![ - PhysicalDagNode { - id: "scan".into(), - operator: PhysicalOperator::Scan, - children: vec![], - source_coverage: Some(coverage), - output_buffer_bytes: 10, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - PhysicalDagNode { - id: "operator".into(), - operator, - children: vec!["scan".into()], - source_coverage: None, - output_buffer_bytes: 10, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - ]; - let provided = HashMap::from([ - ( - "scan".into(), - OperatorStatistics { - source_scan_bytes: scan_edge.bytes, - ..statistics(vec![scan_edge], scan_edge) - }, - ), - ("operator".into(), operator_statistics), - ]); - (nodes, provided) - } - - #[test] - fn estimator_requires_reachable_scans_to_cover_the_exact_scope_set() { - use asap_types::pre_asap::Source; - - let edge = EdgeStatistics { - rows: 100, - bytes: 1_000, - }; - let (nodes, provided) = - manual_unary_plan(PhysicalOperator::Filter, statistics(vec![edge], edge)); - let mut scope = comparison_scope(); - scope.sources.push(SourceCoverage { - source: Source::Table { - table_ref: "unread_metrics".into(), - }, - snapshot_id: "catalog-version-42".into(), - predicates: vec![], - }); - - assert_eq!( - estimate_physical_dag(&nodes, "operator", &scope, &provided), - Err(AnalyticalCostError::InvalidPhysicalDag( - "physical scans omit a comparison-scope source" - )) - ); - } - - #[test] - fn estimator_rejects_semantically_impossible_manual_operator_statistics() { - let input = EdgeStatistics { - rows: 100, - bytes: 1_000, - }; - let invalid_cases = [ - ( - PhysicalOperator::Filter, - statistics( - vec![input], - EdgeStatistics { - rows: 101, - bytes: 1_010, - }, - ), - ), - ( - PhysicalOperator::Sort, - statistics( - vec![input], - EdgeStatistics { - rows: 99, - bytes: 990, - }, - ), - ), - ( - PhysicalOperator::HashAggregate, - OperatorStatistics { - group_count: Some(10), - key_bytes: Some(8), - aggregate_value_bytes: Some(8), - ..statistics( - vec![input], - EdgeStatistics { - rows: 9, - bytes: 144, - }, - ) - }, - ), - ]; - - for (operator, invalid_statistics) in invalid_cases { - let (nodes, provided) = manual_unary_plan(operator, invalid_statistics); - assert!(matches!( - estimate_physical_dag(&nodes, "operator", &comparison_scope(), &provided), - Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) - )); + UnaryEdgeStatistics { + input, + output, + promql: None, } } - #[test] - fn estimator_validates_concat_totals_but_allows_duplicate_source_coverage() { - let coverage = comparison_scope().sources[0].clone(); - let edge = EdgeStatistics { - rows: 50, - bytes: 500, - }; - let nodes = vec![ - PhysicalDagNode { - id: "left".into(), - operator: PhysicalOperator::Scan, - children: vec![], - source_coverage: Some(coverage.clone()), - output_buffer_bytes: 10, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - PhysicalDagNode { - id: "right".into(), - operator: PhysicalOperator::Scan, - children: vec![], - source_coverage: Some(coverage), - output_buffer_bytes: 10, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - PhysicalDagNode { - id: "concat".into(), - operator: PhysicalOperator::Concat, - children: vec!["left".into(), "right".into()], - source_coverage: None, - output_buffer_bytes: 10, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - ]; - let mut provided = HashMap::from([ - ( - "left".into(), - OperatorStatistics { - source_scan_bytes: 500, - ..statistics(vec![edge], edge) - }, - ), - ( - "right".into(), - OperatorStatistics { - source_scan_bytes: 500, - ..statistics(vec![edge], edge) - }, - ), - ( - "concat".into(), - statistics( - vec![edge, edge], - EdgeStatistics { - rows: 99, - bytes: 990, - }, - ), - ), - ]); - - assert!(matches!( - estimate_physical_dag(&nodes, "concat", &comparison_scope(), &provided), - Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) - )); - - provided.insert( - "concat".into(), - statistics( - vec![edge, edge], - EdgeStatistics { - rows: 100, - bytes: 1_000, - }, - ), - ); - assert!(estimate_physical_dag(&nodes, "concat", &comparison_scope(), &provided).is_ok()); - } - - #[test] - fn directional_promql_bridges_fail_closed_on_wrong_cardinality() { - use crate::analytical_statistics::PromqlOperatorStatistics; - - let bridge_statistics = |input_series, output_series, input_rows, output_rows| { - let mut statistics = statistics( - vec![EdgeStatistics { - rows: input_rows, - bytes: input_rows * 8, - }], - EdgeStatistics { - rows: output_rows, - bytes: output_rows * 8, - }, - ); - statistics.promql = Some(PromqlOperatorStatistics { - input_series: vec![input_series], - output_series, - evaluation_steps: 10, - window_samples_per_series: None, - subquery_steps: None, - scalar_ops_per_row: None, - binary_operand_mode: None, - }); - statistics - }; - - assert!(estimate_operator( - PhysicalOperator::PromqlScalarToVector, - bridge_statistics(0, 1, 10, 10), - ) - .is_ok()); - assert!(estimate_operator( - PhysicalOperator::PromqlVectorToScalar, - bridge_statistics(3, 0, 30, 10), - ) - .is_ok()); - assert!(matches!( - estimate_operator( - PhysicalOperator::PromqlScalarToVector, - bridge_statistics(0, 0, 10, 10), - ), - Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) - )); - assert!(matches!( - estimate_operator( - PhysicalOperator::PromqlVectorToScalar, - bridge_statistics(3, 1, 30, 10), - ), - Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) - )); - assert!(matches!( - estimate_operator( - PhysicalOperator::PromqlScalarToVector, - bridge_statistics(0, 1, 9, 10), - ), - Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) - )); - assert!(matches!( - estimate_operator( - PhysicalOperator::PromqlVectorToScalar, - bridge_statistics(3, 0, 30, 9), - ), - Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) - )); - } - - #[test] - fn promql_binary_mode_distinguishes_empty_vectors_from_scalars() { - use crate::analytical_statistics::{PromqlBinaryOperandMode, PromqlOperatorStatistics}; - - let empty = EdgeStatistics { rows: 0, bytes: 0 }; - let mut vector_vector = OperatorStatistics { - key_bytes: Some(16), - hash_join_build_side: Some(HashJoinBuildSide::Right), - promql: Some(PromqlOperatorStatistics { - input_series: vec![0, 0], - output_series: 0, - evaluation_steps: 10, - window_samples_per_series: None, - subquery_steps: None, - scalar_ops_per_row: None, - binary_operand_mode: Some(PromqlBinaryOperandMode::VectorVector), - }), - ..statistics(vec![empty, empty], empty) - }; - assert!( - estimate_operator(PhysicalOperator::PromqlVectorBinary, vector_vector.clone()).is_ok() - ); - - vector_vector.hash_join_build_side = None; - assert_eq!( - estimate_operator(PhysicalOperator::PromqlVectorBinary, vector_vector), - Err(AnalyticalCostError::MissingOrStale( - "vector_binary_label_match_statistics" - )) - ); - } - - #[test] - fn promql_scalar_leaf_requires_one_row_per_evaluation_step() { - use crate::analytical_statistics::PromqlOperatorStatistics; - - let mut scalar = statistics(vec![], EdgeStatistics { rows: 9, bytes: 72 }); - scalar.promql = Some(PromqlOperatorStatistics { - input_series: vec![], - output_series: 0, - evaluation_steps: 10, - window_samples_per_series: None, - subquery_steps: None, - scalar_ops_per_row: None, - binary_operand_mode: None, - }); - assert!(matches!( - estimate_operator(PhysicalOperator::PromqlScalarLeaf, scalar), - Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) - )); - } - #[test] fn comparison_rejects_different_snapshot_predicate_time_or_horizon() { use std::rc::Rc; @@ -2393,6 +2923,7 @@ mod tests { }, source_snapshot_id: "catalog-version-42".into(), predicates: vec![], + info_matchers: vec![], }), output_buffer_bytes: 10, retained_bytes: 0, @@ -2459,6 +2990,7 @@ mod tests { }, source_snapshot_id: "catalog-version-42".into(), predicates: vec![], + info_matchers: vec![], }); let nodes = vec![PhysicalDagNode { id: "scan".into(), @@ -2737,6 +3269,7 @@ mod tests { OperatorStatistics::Concat { inputs: vec![], output: EdgeStatistics { rows: 0, bytes: 0 }, + promql: None, }, ), Err(AnalyticalCostError::InconsistentOperatorStatistics( diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index b661d7e3..23576776 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -6,7 +6,8 @@ use serde::{Deserialize, Serialize}; use crate::analytical_cost::{ validate_operator_semantics, AnalyticalCostError, ExecutionMultiplicity, HashJoinBuildSide, - PhysicalDagNode, PhysicalOperator, + PhysicalDagNode, PhysicalOperator, PromqlBinaryOperandMode, PromqlBinaryOperation, + PromqlPresenceKind, PromqlSeriesSampleKind, PromqlVectorCardinality, }; use crate::analytical_statistics::{ ComparisonScope, EdgeStatistics, OperatorStatistics, OperatorStatisticsProvider, SourceCoverage, @@ -201,10 +202,33 @@ pub fn lower_query_physical_dag( &child_id, child_statistics, )?; + require_promql_edge(statistics, 0, child_statistics)?; require_operator_statistics(operator, statistics)?; self.push(evidence, operator, children, None) } + fn lower_promql_unary( + &mut self, + query: &QueryExpr, + occurrence: usize, + operator: PhysicalOperator, + child: &QueryExpr, + ) -> Result { + self.lower_unary(query, occurrence, operator, child) + } + + fn lower_promql_scalar_leaf( + &mut self, + query: &QueryExpr, + occurrence: usize, + ) -> Result { + let operator = PhysicalOperator::PromqlScalarLeaf; + let evidence = self.resolve(query, operator, occurrence, false, &[], None)?; + require_statistics_shape(&evidence.physical_id, &evidence.statistics, 0)?; + require_operator_statistics(operator, &evidence.statistics)?; + self.push(evidence, operator, vec![], None) + } + fn node_statistics(&self, id: &str) -> Result<&OperatorStatistics, AnalyticalCostError> { self.evidence .get(id) @@ -321,13 +345,39 @@ pub fn lower_query_physical_dag( child, .. } => { - let asap_types::pre_asap::Reduction::Reduce(grouping) = reduction else { + if having.is_some() || measures.is_empty() { return Err(AnalyticalCostError::UnsupportedQueryOperator); + } + if matches!(reduction, asap_types::pre_asap::Reduction::PerEntity) { + if measures.len() != 1 { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + } + let accumulator_count = u64::try_from(measures.len()) + .map_err(|_| AnalyticalCostError::Overflow)?; + let operator = if measures.iter().all(presence_intent) { + PhysicalOperator::PromqlPresence { + kind: PromqlPresenceKind::Absent, + operations_per_row: accumulator_count, + } + } else if measures.iter().all(present_over_time_intent) { + PhysicalOperator::PromqlPresence { + kind: PromqlPresenceKind::PresentPerSeries, + operations_per_row: accumulator_count, + } + } else if measures.iter().all(fixed_state_per_series_intent) { + PhysicalOperator::PromqlPerSeries { + operations_per_row: accumulator_count, + accumulator_count, + } + } else { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + }; + return self.lower_promql_unary(query, occurrence, operator, child); + } + let asap_types::pre_asap::Reduction::Reduce(grouping) = reduction else { + unreachable!("per-entity reduction returned above") }; - if grouping.is_without() - || having.is_some() - || !supports_hash_aggregate(reduction, measures) - { + if grouping.is_without() || !supports_hash_aggregate(reduction, measures) { return Err(AnalyticalCostError::UnsupportedQueryOperator); } self.lower_unary( @@ -476,6 +526,225 @@ pub fn lower_query_physical_dag( child, ) } + QueryExpr::TimeRange { range, child } => { + let range_millis = duration_millis(*range, "range")?; + self.lower_promql_unary( + query, + occurrence, + PhysicalOperator::PromqlRange { range_millis }, + child, + ) + } + QueryExpr::PromqlSubquery { + range, + resolution, + child, + } => { + let range_millis = duration_millis(*range, "subquery range")?; + let resolution_millis = resolution + .map(|value| duration_millis(value, "subquery resolution")) + .transpose()?; + let operator = PhysicalOperator::PromqlSubquery { + range_millis, + resolution_millis, + }; + let id = self.lower_promql_unary(query, occurrence, operator, child)?; + let statistics = self.node_statistics(&id)?; + let OperatorStatistics::PromqlSubquery { subquery_steps, .. } = statistics + else { + unreachable!("operator/statistics matching was validated") + }; + if let Some(resolution_millis) = resolution_millis { + let expected = range_millis + .checked_div(resolution_millis) + .and_then(|steps| steps.checked_add(1)) + .ok_or(AnalyticalCostError::Overflow)?; + if *subquery_steps != expected { + return Err(AnalyticalCostError::InconsistentOperatorStatistics( + "subquery steps disagree with range and resolution", + )); + } + } + Ok(id) + } + QueryExpr::PromqlRelabel { value, child, .. } => self.lower_promql_unary( + query, + occurrence, + PhysicalOperator::PromqlRelabel { + expression_operations_per_row: scalar_operation_count(value)?.max(1), + }, + child, + ), + QueryExpr::PromqlSeriesSample { + by, kind, child, .. + } => { + if by.is_without() { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + } + let kind = match kind { + asap_types::pre_asap::SampleKind::LimitK(k) => { + let k = u64::try_from(*k).map_err(|_| AnalyticalCostError::Overflow)?; + if k == 0 { + return Err(AnalyticalCostError::MissingOrZero("limitk")); + } + PromqlSeriesSampleKind::LimitK { k } + } + asap_types::pre_asap::SampleKind::LimitRatio(ratio) + if ratio.is_finite() && (-1.0..=1.0).contains(ratio) => + { + PromqlSeriesSampleKind::LimitRatio { + ratio_bits: ratio.to_bits(), + } + } + asap_types::pre_asap::SampleKind::LimitRatio(_) => { + return Err(AnalyticalCostError::UnsupportedQueryOperator) + } + }; + self.lower_promql_unary( + query, + occurrence, + PhysicalOperator::PromqlSeriesSample { + kind, + grouping_key_count: u64::try_from(by.keys().len()) + .map_err(|_| AnalyticalCostError::Overflow)?, + }, + child, + ) + } + QueryExpr::PromqlInfoEnrich { selector, child } => { + let left_id = self.lower(child)?; + let coverage = bind_info_coverage( + &format!("occurrence-{occurrence}-info"), + selector, + self.scope, + )?; + let info_evidence = self.resolve( + query, + PhysicalOperator::Scan, + occurrence, + true, + &[], + Some(&coverage), + )?; + require_statistics_shape( + &info_evidence.physical_id, + &info_evidence.statistics, + 1, + )?; + require_scan_edges_equal(&info_evidence.statistics)?; + let info_statistics = info_evidence.statistics.clone(); + let info_id = self.push( + info_evidence, + PhysicalOperator::Scan, + vec![], + Some(coverage), + )?; + let children = vec![left_id.clone(), info_id.clone()]; + let operator = PhysicalOperator::PromqlInfoEnrich { + matcher_operations_per_info_row: u64::try_from( + selector + .iter() + .filter(|matcher| matcher.label != "__name__") + .count() + .max(1), + ) + .map_err(|_| AnalyticalCostError::Overflow)?, + }; + let evidence = + self.resolve(query, operator, occurrence, false, &children, None)?; + require_binary_edges( + &evidence.physical_id, + &evidence.statistics, + &left_id, + self.node_statistics(&left_id)?, + &info_id, + &info_statistics, + )?; + require_operator_statistics(operator, &evidence.statistics)?; + self.push(evidence, operator, children, None) + } + QueryExpr::BinaryOp { + op, + lhs, + rhs, + vector_match, + } => { + let left_scalar = is_promql_scalar(lhs); + let right_scalar = is_promql_scalar(rhs); + if left_scalar && right_scalar { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + } + let operation = promql_binary_operation(op); + if (left_scalar || right_scalar) + && !matches!(operation, PromqlBinaryOperation::ArithmeticOrComparison) + { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + } + let operand_mode = match (left_scalar, right_scalar) { + (false, false) => PromqlBinaryOperandMode::VectorVector, + (false, true) => PromqlBinaryOperandMode::VectorScalar, + (true, false) => PromqlBinaryOperandMode::ScalarVector, + (true, true) => unreachable!("scalar/scalar returned above"), + }; + if operand_mode != PromqlBinaryOperandMode::VectorVector + && vector_match.is_some() + { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + } + let cardinality = promql_vector_cardinality(vector_match.as_ref()); + let left_id = self.lower(lhs)?; + let right_id = self.lower(rhs)?; + let children = vec![left_id.clone(), right_id.clone()]; + let left_statistics = self.node_statistics(&left_id)?; + let right_statistics = self.node_statistics(&right_id)?; + let build_side = (operand_mode == PromqlBinaryOperandMode::VectorVector) + .then_some( + if left_statistics.output().bytes <= right_statistics.output().bytes { + HashJoinBuildSide::Left + } else { + HashJoinBuildSide::Right + }, + ); + let operator = PhysicalOperator::PromqlBinary { + operation, + operand_mode, + cardinality, + build_side, + }; + let evidence = + self.resolve(query, operator, occurrence, false, &children, None)?; + require_binary_edges( + &evidence.physical_id, + &evidence.statistics, + &left_id, + left_statistics, + &right_id, + right_statistics, + )?; + require_operator_statistics(operator, &evidence.statistics)?; + self.push(evidence, operator, children, None) + } + QueryExpr::PromqlVectorFromScalar(child) => self.lower_promql_unary( + query, + occurrence, + PhysicalOperator::PromqlScalarToVector, + child, + ), + QueryExpr::PromqlScalarFromVector(child) => self.lower_promql_unary( + query, + occurrence, + PhysicalOperator::PromqlVectorToScalar, + child, + ), + QueryExpr::PromqlScalarBridge(inner) + if matches!( + inner.as_ref(), + QueryExpr::Literal(asap_types::pre_asap::ScalarValue::Float64(_)) + ) => + { + self.lower_promql_scalar_leaf(query, occurrence) + } + QueryExpr::EvalTimestamp => self.lower_promql_scalar_leaf(query, occurrence), QueryExpr::TimeShift { shift, child } => { if !shift.is_identity() { return Err(AnalyticalCostError::UnsupportedQueryOperator); @@ -532,14 +801,14 @@ pub fn lower_query_physical_dag( let evidence = self.resolve(query, operator, occurrence, false, &children, None)?; let statistics = &evidence.statistics; - require_statistics_shape(&evidence.physical_id, statistics, 2)?; - if statistics.input(0) != Some(left_statistics.output()) - || statistics.input(1) != Some(right_statistics.output()) - { - return Err(AnalyticalCostError::InconsistentOperatorStatistics( - "join inputs do not match child outputs", - )); - } + require_binary_edges( + &evidence.physical_id, + statistics, + &left_id, + left_statistics, + &right_id, + right_statistics, + )?; require_operator_statistics(operator, statistics)?; self.push(evidence, operator, children, None) } @@ -579,6 +848,7 @@ pub fn lower_query_physical_dag( input_index: index, }); } + require_promql_edge(statistics, index, child_statistics)?; Ok::<_, AnalyticalCostError>(( rows.checked_add(child_statistics.output().rows) .ok_or(AnalyticalCostError::Overflow)?, @@ -692,6 +962,45 @@ fn require_unary_edge( Ok(()) } +fn require_binary_edges( + node: &str, + statistics: &OperatorStatistics, + left_id: &str, + left: &OperatorStatistics, + right_id: &str, + right: &OperatorStatistics, +) -> Result<(), AnalyticalCostError> { + require_statistics_shape(node, statistics, 2)?; + for (index, (child_id, child)) in [(left_id, left), (right_id, right)].into_iter().enumerate() { + if statistics.input(index) != Some(child.output()) { + return Err(AnalyticalCostError::ConflictingEdgeStatistics { + parent: node.into(), + child: child_id.into(), + input_index: index, + }); + } + require_promql_edge(statistics, index, child)?; + } + Ok(()) +} + +fn require_promql_edge( + parent: &OperatorStatistics, + input_index: usize, + child: &OperatorStatistics, +) -> Result<(), AnalyticalCostError> { + match (parent.promql_input(input_index), child.promql_output()) { + (None, None) => Ok(()), + (Some(input), Some(output)) if input == output => Ok(()), + (Some(_), Some(_)) => Err(AnalyticalCostError::InconsistentOperatorStatistics( + "PromQL parent input does not match child output", + )), + _ => Err(AnalyticalCostError::MissingOrStale( + "promql_edge_statistics", + )), + } +} + fn require_operator_statistics( operator: PhysicalOperator, statistics: &OperatorStatistics, @@ -705,10 +1014,11 @@ fn bind_scan_coverage( predicates: &[asap_types::pre_asap::Predicate], scope: &ComparisonScope, ) -> Result { - let mut matches = scope - .sources - .iter() - .filter(|coverage| coverage.source == *source && coverage.predicates == predicates); + let mut matches = scope.sources.iter().filter(|coverage| { + coverage.source == *source + && coverage.predicates == predicates + && coverage.info_matchers.is_empty() + }); let coverage = matches .next() .cloned() @@ -721,6 +1031,80 @@ fn bind_scan_coverage( Ok(coverage) } +fn bind_info_coverage( + node_id: &str, + selector: &[asap_types::pre_asap::InfoMatcher], + scope: &ComparisonScope, +) -> Result { + use asap_types::pre_asap::{CompareOpKind, Source}; + + let mut metric: Option<&str> = None; + for matcher in selector + .iter() + .filter(|matcher| matcher.label == "__name__") + { + if matcher.op != CompareOpKind::Eq || metric.is_some_and(|current| current != matcher.value) + { + return Err(AnalyticalCostError::UnsupportedQueryOperator); + } + metric = Some(&matcher.value); + } + let source = Source::TimeSeries { + metric: metric.unwrap_or("target_info").into(), + }; + let mut matches = scope.sources.iter().filter(|coverage| { + coverage.source == source + && coverage.predicates.is_empty() + && coverage.info_matchers == selector + }); + let coverage = matches + .next() + .cloned() + .ok_or_else(|| AnalyticalCostError::ScanOutsideComparisonScope(node_id.into()))?; + if matches.next().is_some() { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "info source coverage is ambiguous", + )); + } + Ok(coverage) +} + +fn duration_millis( + duration: std::time::Duration, + field: &'static str, +) -> Result { + if duration.is_zero() { + return Err(AnalyticalCostError::MissingOrZero(field)); + } + u64::try_from(duration.as_millis()).map_err(|_| AnalyticalCostError::Overflow) +} + +fn promql_binary_operation( + operation: &asap_types::pre_asap::BinaryOpKind, +) -> PromqlBinaryOperation { + use asap_types::pre_asap::BinaryOpKind; + match operation { + BinaryOpKind::And => PromqlBinaryOperation::And, + BinaryOpKind::Or => PromqlBinaryOperation::Or, + BinaryOpKind::Unless => PromqlBinaryOperation::Unless, + BinaryOpKind::Arithmetic(_) + | BinaryOpKind::Compare(_) + | BinaryOpKind::Pow + | BinaryOpKind::Atan2 => PromqlBinaryOperation::ArithmeticOrComparison, + } +} + +fn promql_vector_cardinality( + vector_match: Option<&asap_types::pre_asap::VectorMatch>, +) -> PromqlVectorCardinality { + use asap_types::pre_asap::GroupSide; + match vector_match.and_then(|matching| matching.grouping.as_ref()) { + Some(grouping) if grouping.side == GroupSide::Left => PromqlVectorCardinality::ManyToOne, + Some(_) => PromqlVectorCardinality::OneToMany, + None => PromqlVectorCardinality::OneToOne, + } +} + fn hash_join_key_count( expr: &asap_types::pre_asap::QueryExpr, left: &asap_types::pre_asap::QueryExpr, @@ -868,6 +1252,64 @@ fn supports_hash_aggregate( }) } +fn presence_intent(intent: &asap_types::pre_asap::AggIntent) -> bool { + matches!( + intent, + asap_types::pre_asap::AggIntent::Absent | asap_types::pre_asap::AggIntent::AbsentOverTime + ) +} + +fn present_over_time_intent(intent: &asap_types::pre_asap::AggIntent) -> bool { + matches!(intent, asap_types::pre_asap::AggIntent::PresentOverTime) +} + +fn fixed_state_per_series_intent(intent: &asap_types::pre_asap::AggIntent) -> bool { + use asap_types::pre_asap::AggIntent; + matches!( + intent, + AggIntent::Rate + | AggIntent::Count { .. } + | AggIntent::Sum { .. } + | AggIntent::Min { .. } + | AggIntent::Max { .. } + | AggIntent::Avg { .. } + | AggIntent::StdDev { .. } + | AggIntent::Variance { .. } + | AggIntent::Increase + | AggIntent::Changes + | AggIntent::Delta + | AggIntent::IDelta + | AggIntent::Deriv + | AggIntent::Resets + | AggIntent::PredictLinear { .. } + | AggIntent::DoubleExpSmoothing { .. } + | AggIntent::HistogramCount + | AggIntent::HistogramSum + | AggIntent::HistogramAvg + | AggIntent::HistogramStdDev + | AggIntent::HistogramStdVar + | AggIntent::HistogramFraction { .. } + | AggIntent::Math(_) + | AggIntent::TimeFn(_) + | AggIntent::LastOverTime + | AggIntent::FirstOverTime + | AggIntent::TsOfMinOverTime + | AggIntent::TsOfMaxOverTime + | AggIntent::TsOfFirstOverTime + | AggIntent::TsOfLastOverTime + ) +} + +fn is_promql_scalar(query: &asap_types::pre_asap::QueryExpr) -> bool { + use asap_types::pre_asap::QueryExpr; + matches!( + query, + QueryExpr::PromqlScalarBridge(_) + | QueryExpr::PromqlScalarFromVector(_) + | QueryExpr::EvalTimestamp + ) +} + #[cfg(test)] mod tests { use super::*; @@ -875,7 +1317,8 @@ mod tests { estimate_physical_dag, estimate_physical_dag_comparison, PhysicalDagEstimateRequest, }; use crate::analytical_statistics::{ - validate_comparison_scopes, BinaryEdgeStatistics, PartitionStatistics, UnaryEdgeStatistics, + validate_comparison_scopes, BinaryEdgeStatistics, PartitionStatistics, + PromqlEdgeStatistics, PromqlUnaryEdgeStatistics, PromqlValueKind, UnaryEdgeStatistics, }; use asap_types::workload::{ DataArrival, DurationMs, QueryRecurrence, QueryTimeScope, TimeSelection, TimestampMs, @@ -887,7 +1330,11 @@ mod tests { } fn unary_edges(input: EdgeStatistics, output: EdgeStatistics) -> UnaryEdgeStatistics { - UnaryEdgeStatistics { input, output } + UnaryEdgeStatistics { + input, + output, + promql: None, + } } fn scan_stats(edge: EdgeStatistics, source_read_bytes: u64) -> OperatorStatistics { @@ -897,6 +1344,34 @@ mod tests { } } + fn promql_edge( + series: u64, + evaluation_steps: u64, + value_kind: PromqlValueKind, + ) -> PromqlEdgeStatistics { + PromqlEdgeStatistics { + series, + evaluation_steps, + value_kind, + } + } + + fn promql_unary_edges( + input: EdgeStatistics, + output: EdgeStatistics, + promql_input: PromqlEdgeStatistics, + promql_output: PromqlEdgeStatistics, + ) -> UnaryEdgeStatistics { + UnaryEdgeStatistics { + input, + output, + promql: Some(PromqlUnaryEdgeStatistics { + input: promql_input, + output: promql_output, + }), + } + } + fn unary_statistics( operator: PhysicalOperator, input: EdgeStatistics, @@ -988,9 +1463,45 @@ mod tests { source, source_snapshot_id: "snapshot-1".into(), predicates, + info_matchers: vec![], } } + #[test] + fn info_source_coverage_includes_symbolic_selector_matchers() { + use asap_types::pre_asap::{CompareOpKind, InfoMatcher, Source}; + + let selector = vec![InfoMatcher { + label: "cluster".into(), + op: CompareOpKind::Eq, + value: "prod".into(), + }]; + let info_coverage = SourceCoverage { + source: Source::TimeSeries { + metric: "target_info".into(), + }, + source_snapshot_id: "snapshot-1".into(), + predicates: vec![], + info_matchers: selector.clone(), + }; + let matching_scope = scope(vec![info_coverage.clone()]); + assert_eq!( + bind_info_coverage("info", &selector, &matching_scope), + Ok(info_coverage) + ); + + let wrong_scope = scope(vec![coverage( + Source::TimeSeries { + metric: "target_info".into(), + }, + vec![], + )]); + assert!(matches!( + bind_info_coverage("info", &selector, &wrong_scope), + Err(AnalyticalCostError::ScanOutsideComparisonScope(_)) + )); + } + #[test] fn query_lowering_recurses_and_fuses_global_sort_limit() { use asap_types::pre_asap::{AggIntent, GroupKeys, QueryExpr, Reduction, SortKey, Source}; @@ -1164,6 +1675,7 @@ mod tests { edges: BinaryEdgeStatistics { inputs: [edge(100, 800), edge(100, 800)], output: edge(25, 400), + promql: None, }, }; let provided = HashMap::from([ @@ -1504,6 +2016,7 @@ mod tests { evidence(OperatorStatistics::Concat { inputs: vec![edge(10, 80), edge(20, 160)], output: edge(30, 240), + promql: None, }), ), ]); @@ -1745,13 +2258,6 @@ mod tests { having: None, child: scan(), }); - let per_entity = Rc::new(QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Rate], - output_names: vec![], - having: None, - child: scan(), - }); let empty_sort_limit = Rc::new(QueryExpr::Limit { n: 10, offset: 0, @@ -1786,7 +2292,6 @@ mod tests { let unavailable = HashMap::::new(); for query in [ &exact_quantile, - &per_entity, &empty_sort_limit, &unsupported_window, &shifted, @@ -1811,4 +2316,320 @@ mod tests { assert_eq!(scalar_operation_count(&predicate), Ok(3)); } + + #[test] + fn promql_presence_is_lowered_with_a_per_step_output_bound() { + use asap_types::pre_asap::{ + AggIntent, Column, DataType, QueryExpr, Reduction, Schema, Source, + }; + + let source = Source::TimeSeries { + metric: "missing".into(), + }; + let scan = Rc::new(QueryExpr::Scan { + source: source.clone(), + predicates: vec![], + schema: Schema::new(vec![Column::new("value", DataType::Float64, false)]), + }); + let root = Rc::new(QueryExpr::Aggregate { + reduction: Reduction::PerEntity, + measures: vec![AggIntent::Absent], + output_names: vec![], + having: None, + child: scan, + }); + let vector = promql_edge(0, 2, PromqlValueKind::Vector); + let scan_statistics = OperatorStatistics::Scan { + edges: promql_unary_edges(edge(0, 0), edge(0, 0), vector, vector), + source_read_bytes: 0, + }; + let output = promql_edge(1, 2, PromqlValueKind::Vector); + let presence_statistics = OperatorStatistics::PromqlPresence { + edges: promql_unary_edges(edge(0, 0), edge(2, 16), vector, output), + }; + let provided = HashMap::from([ + ("query-1".into(), evidence(scan_statistics)), + ("query-0".into(), evidence(presence_statistics.clone())), + ]); + let query_scope = scope(vec![coverage(source, vec![])]); + let dag = lower_query_physical_dag(&root, &query_scope, &scripted(&provided)).unwrap(); + assert!(matches!( + dag.nodes.last().map(|node| node.operator), + Some(PhysicalOperator::PromqlPresence { + kind: PromqlPresenceKind::Absent, + operations_per_row: 1 + }) + )); + + let OperatorStatistics::PromqlPresence { mut edges } = presence_statistics else { + unreachable!() + }; + edges.output = edge(3, 24); + let invalid = HashMap::from([ + ( + "query-1".into(), + evidence(OperatorStatistics::Scan { + edges: promql_unary_edges(edge(0, 0), edge(0, 0), vector, vector), + source_read_bytes: 0, + }), + ), + ( + "query-0".into(), + evidence(OperatorStatistics::PromqlPresence { edges }), + ), + ]); + assert!(matches!( + lower_query_physical_dag(&root, &query_scope, &scripted(&invalid)), + Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) + )); + } + + #[test] + fn promql_range_and_subquery_preserve_internal_steps() { + use asap_types::pre_asap::{Column, DataType, QueryExpr, Schema, Source}; + use std::time::Duration; + + let source = Source::TimeSeries { metric: "m".into() }; + let scan = Rc::new(QueryExpr::Scan { + source: source.clone(), + predicates: vec![], + schema: Schema::new(vec![Column::new("value", DataType::Float64, false)]), + }); + let range = Rc::new(QueryExpr::TimeRange { + range: Duration::from_secs(300), + child: scan, + }); + let root = Rc::new(QueryExpr::PromqlSubquery { + range: Duration::from_secs(300), + resolution: Some(Duration::from_secs(60)), + child: range, + }); + let vector = promql_edge(10, 6, PromqlValueKind::Vector); + let range_vector = promql_edge(10, 6, PromqlValueKind::RangeVector); + let outer_range = promql_edge(10, 1, PromqlValueKind::RangeVector); + let provided = HashMap::from([ + ( + "query-2".into(), + evidence(OperatorStatistics::Scan { + edges: promql_unary_edges(edge(600, 9_600), edge(600, 9_600), vector, vector), + source_read_bytes: 4_800, + }), + ), + ( + "query-1".into(), + evidence(OperatorStatistics::PromqlRange { + edges: promql_unary_edges( + edge(600, 9_600), + edge(60, 960), + vector, + range_vector, + ), + max_window_samples_per_series: 10, + }), + ), + ( + "query-0".into(), + evidence(OperatorStatistics::PromqlSubquery { + edges: promql_unary_edges( + edge(60, 960), + edge(10, 160), + range_vector, + outer_range, + ), + subquery_steps: 6, + }), + ), + ]); + let query_scope = scope(vec![coverage(source, vec![])]); + let dag = lower_query_physical_dag(&root, &query_scope, &scripted(&provided)).unwrap(); + assert!(matches!( + dag.nodes[1].operator, + PhysicalOperator::PromqlRange { + range_millis: 300_000 + } + )); + assert!(matches!( + dag.nodes[2].operator, + PhysicalOperator::PromqlSubquery { + range_millis: 300_000, + resolution_millis: Some(60_000) + } + )); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, &query_scope, &dag.evidence).is_ok()); + } + + #[test] + fn promql_binary_lowering_keeps_operation_and_matching_cardinality() { + use asap_types::pre_asap::{ + ArithmeticOpKind, BinaryOpKind, Column, DataType, GroupSide, QueryExpr, Schema, Source, + VectorGrouping, VectorMatch, VectorMatchKind, + }; + + let left_source = Source::TimeSeries { metric: "a".into() }; + let right_source = Source::TimeSeries { metric: "b".into() }; + let scan = |source| { + Rc::new(QueryExpr::Scan { + source, + predicates: vec![], + schema: Schema::new(vec![Column::new("value", DataType::Float64, false)]), + }) + }; + let root = Rc::new(QueryExpr::BinaryOp { + op: BinaryOpKind::Arithmetic(ArithmeticOpKind::Div), + lhs: scan(left_source.clone()), + rhs: scan(right_source.clone()), + vector_match: Some(VectorMatch { + kind: VectorMatchKind::On, + labels: vec!["service".into()], + grouping: Some(VectorGrouping { + side: GroupSide::Left, + labels: vec!["region".into()], + }), + }), + }); + let left_promql = promql_edge(10, 10, PromqlValueKind::Vector); + let right_promql = promql_edge(5, 10, PromqlValueKind::Vector); + let output_promql = promql_edge(8, 10, PromqlValueKind::Vector); + let left_edge = edge(100, 1_600); + let right_edge = edge(50, 800); + let output_edge = edge(80, 1_280); + let provided = HashMap::from([ + ( + "query-1".into(), + evidence(OperatorStatistics::Scan { + edges: promql_unary_edges(left_edge, left_edge, left_promql, left_promql), + source_read_bytes: 800, + }), + ), + ( + "query-2".into(), + evidence(OperatorStatistics::Scan { + edges: promql_unary_edges(right_edge, right_edge, right_promql, right_promql), + source_read_bytes: 400, + }), + ), + ( + "query-0".into(), + evidence(OperatorStatistics::PromqlBinary { + edges: BinaryEdgeStatistics { + inputs: [left_edge, right_edge], + output: output_edge, + promql: Some(crate::analytical_statistics::PromqlBinaryEdgeStatistics { + inputs: [left_promql, right_promql], + output: output_promql, + }), + }, + matching_key_bytes: 16, + }), + ), + ]); + let query_scope = scope(vec![ + coverage(left_source, vec![]), + coverage(right_source, vec![]), + ]); + let dag = lower_query_physical_dag(&root, &query_scope, &scripted(&provided)).unwrap(); + assert!(matches!( + dag.nodes.last().map(|node| node.operator), + Some(PhysicalOperator::PromqlBinary { + operation: PromqlBinaryOperation::ArithmeticOrComparison, + operand_mode: PromqlBinaryOperandMode::VectorVector, + cardinality: PromqlVectorCardinality::ManyToOne, + build_side: Some(HashJoinBuildSide::Right), + }) + )); + } + + #[test] + fn promql_relabel_sample_and_per_series_lower_as_a_complete_chain() { + use asap_types::pre_asap::{ + AggIntent, Column, DataType, GroupKeys, QueryExpr, Reduction, SampleKind, ScalarValue, + Schema, Source, + }; + + let source = Source::TimeSeries { + metric: "requests".into(), + }; + let scan = Rc::new(QueryExpr::Scan { + source: source.clone(), + predicates: vec![], + schema: Schema::new(vec![Column::new("value", DataType::Float64, false)]), + }); + let relabel = Rc::new(QueryExpr::PromqlRelabel { + dst: "service".into(), + value: Rc::new(QueryExpr::Literal(ScalarValue::Utf8("api".into()))), + child: scan, + }); + let sample = Rc::new(QueryExpr::PromqlSeriesSample { + by: GroupKeys::none(), + kind: SampleKind::LimitK(5), + child: relabel, + }); + let root = Rc::new(QueryExpr::Aggregate { + reduction: Reduction::PerEntity, + measures: vec![AggIntent::Sum { col: None }], + output_names: vec![], + having: None, + child: sample, + }); + + let input = edge(100, 1_600); + let sampled = edge(50, 800); + let input_promql = promql_edge(10, 10, PromqlValueKind::Vector); + let sampled_promql = promql_edge(5, 10, PromqlValueKind::Vector); + let provided = HashMap::from([ + ( + "query-3".into(), + evidence(OperatorStatistics::Scan { + edges: promql_unary_edges(input, input, input_promql, input_promql), + source_read_bytes: 800, + }), + ), + ( + "query-2".into(), + evidence(OperatorStatistics::PromqlRelabel { + edges: promql_unary_edges(input, input, input_promql, input_promql), + }), + ), + ( + "query-1".into(), + evidence(OperatorStatistics::PromqlSeriesSample { + edges: promql_unary_edges(input, sampled, input_promql, sampled_promql), + group_count: 1, + key_bytes: 8, + }), + ), + ( + "query-0".into(), + evidence(OperatorStatistics::PromqlPerSeries { + edges: promql_unary_edges(sampled, sampled, sampled_promql, sampled_promql), + accumulator_bytes_per_series: 8, + }), + ), + ]); + let query_scope = scope(vec![coverage(source, vec![])]); + + let dag = lower_query_physical_dag(&root, &query_scope, &scripted(&provided)).unwrap(); + assert!(matches!( + dag.nodes.as_slice(), + [ + PhysicalDagNode { + operator: PhysicalOperator::Scan, + .. + }, + PhysicalDagNode { + operator: PhysicalOperator::PromqlRelabel { .. }, + .. + }, + PhysicalDagNode { + operator: PhysicalOperator::PromqlSeriesSample { .. }, + .. + }, + PhysicalDagNode { + operator: PhysicalOperator::PromqlPerSeries { .. }, + .. + } + ] + )); + assert!(estimate_physical_dag(&dag.nodes, &dag.root, &query_scope, &dag.evidence).is_ok()); + } } diff --git a/crates/asap-aware-mapping/src/analytical_statistics.rs b/crates/asap-aware-mapping/src/analytical_statistics.rs index abf0c802..ecd3a9ef 100644 --- a/crates/asap-aware-mapping/src/analytical_statistics.rs +++ b/crates/asap-aware-mapping/src/analytical_statistics.rs @@ -6,7 +6,7 @@ use std::collections::HashMap; -use asap_types::pre_asap::query_expr::{Predicate, Source}; +use asap_types::pre_asap::query_expr::{InfoMatcher, Predicate, Source}; use asap_types::workload::{ DataArrival, DataWorkload, DurationMs, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, TimeSelection, TimestampMs, @@ -39,6 +39,10 @@ pub struct SourceCoverage { pub source_snapshot_id: String, /// Canonical predicates copied from the bound/canonicalized query IR. pub predicates: Vec, + /// Symbolic selectors on an info-metric source. Ordinary query scans leave + /// this empty because their selection is represented by `predicates`. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub info_matchers: Vec, } impl ComparisonScope { @@ -83,16 +87,6 @@ impl ComparisonScope { "duplicate source coverage", )); } - if self - .sources - .iter() - .enumerate() - .any(|(index, source)| self.sources[..index].contains(source)) - { - return Err(AnalyticalCostError::MissingComparisonScope( - "duplicate source coverage", - )); - } if self .sources .iter() @@ -215,6 +209,10 @@ impl EdgeStatistics { pub struct UnaryEdgeStatistics { pub input: EdgeStatistics, pub output: EdgeStatistics, + /// Time-series shape on the same logical edges. This is edge metadata, + /// not an operator-specific bag of optional cost parameters. + #[serde(default)] + pub promql: Option, } /// Input and output evidence for a binary physical operator. The input order @@ -223,6 +221,44 @@ pub struct UnaryEdgeStatistics { pub struct BinaryEdgeStatistics { pub inputs: [EdgeStatistics; 2], pub output: EdgeStatistics, + #[serde(default)] + pub promql: Option, +} + +/// PromQL value shape carried alongside rows/bytes on a physical edge. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct PromqlEdgeStatistics { + pub series: u64, + pub evaluation_steps: u64, + pub value_kind: PromqlValueKind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PromqlValueKind { + Scalar, + /// Vector/sample data. Operator semantics determine whether rows are + /// instant results or decoded source samples for a range evaluation. + Vector, + RangeVector, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct PromqlUnaryEdgeStatistics { + pub input: PromqlEdgeStatistics, + pub output: PromqlEdgeStatistics, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct PromqlBinaryEdgeStatistics { + pub inputs: [PromqlEdgeStatistics; 2], + pub output: PromqlEdgeStatistics, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PromqlNaryEdgeStatistics { + pub inputs: Vec, + pub output: PromqlEdgeStatistics, } /// Input distribution for an algorithm that independently orders partitions. @@ -285,6 +321,8 @@ pub enum OperatorStatistics { Concat { inputs: Vec, output: EdgeStatistics, + #[serde(default)] + promql: Option, }, InMemoryAnalyticWindow { edges: UnaryEdgeStatistics, @@ -296,6 +334,47 @@ pub enum OperatorStatistics { PassThrough { edges: UnaryEdgeStatistics, }, + PromqlRange { + edges: UnaryEdgeStatistics, + max_window_samples_per_series: u64, + }, + PromqlSubquery { + edges: UnaryEdgeStatistics, + subquery_steps: u64, + }, + PromqlBinary { + edges: BinaryEdgeStatistics, + matching_key_bytes: u64, + }, + PromqlRelabel { + edges: UnaryEdgeStatistics, + }, + PromqlInfoEnrich { + edges: BinaryEdgeStatistics, + matching_key_bytes: u64, + }, + PromqlSeriesSample { + edges: UnaryEdgeStatistics, + group_count: u64, + key_bytes: u64, + }, + PromqlScalarToVector { + edges: UnaryEdgeStatistics, + }, + PromqlVectorToScalar { + edges: UnaryEdgeStatistics, + }, + PromqlScalarLeaf { + output: EdgeStatistics, + promql_output: PromqlEdgeStatistics, + }, + PromqlPerSeries { + edges: UnaryEdgeStatistics, + accumulator_bytes_per_series: u64, + }, + PromqlPresence { + edges: UnaryEdgeStatistics, + }, } impl OperatorStatistics { @@ -310,9 +389,18 @@ impl OperatorStatistics { | Self::HashDeduplicate { .. } | Self::InMemoryAnalyticWindow { .. } | Self::Limit { .. } - | Self::PassThrough { .. } => 1, - Self::HashJoin { .. } => 2, + | Self::PassThrough { .. } + | Self::PromqlRange { .. } + | Self::PromqlSubquery { .. } + | Self::PromqlRelabel { .. } + | Self::PromqlSeriesSample { .. } + | Self::PromqlScalarToVector { .. } + | Self::PromqlVectorToScalar { .. } + | Self::PromqlPerSeries { .. } + | Self::PromqlPresence { .. } => 1, + Self::HashJoin { .. } | Self::PromqlBinary { .. } | Self::PromqlInfoEnrich { .. } => 2, Self::Concat { inputs, .. } => inputs.len(), + Self::PromqlScalarLeaf { .. } => 0, } } @@ -327,9 +415,20 @@ impl OperatorStatistics { | Self::TopK { edges } | Self::InMemoryAnalyticWindow { edges, .. } | Self::Limit { edges } - | Self::PassThrough { edges } => (index == 0).then_some(edges.input), - Self::HashJoin { edges } => edges.inputs.get(index).copied(), + | Self::PassThrough { edges } + | Self::PromqlRange { edges, .. } + | Self::PromqlSubquery { edges, .. } + | Self::PromqlRelabel { edges } + | Self::PromqlSeriesSample { edges, .. } + | Self::PromqlScalarToVector { edges } + | Self::PromqlVectorToScalar { edges } + | Self::PromqlPerSeries { edges, .. } + | Self::PromqlPresence { edges } => (index == 0).then_some(edges.input), + Self::HashJoin { edges } + | Self::PromqlBinary { edges, .. } + | Self::PromqlInfoEnrich { edges, .. } => edges.inputs.get(index).copied(), Self::Concat { inputs, .. } => inputs.get(index).copied(), + Self::PromqlScalarLeaf { .. } => None, } } @@ -344,9 +443,80 @@ impl OperatorStatistics { | Self::TopK { edges } | Self::InMemoryAnalyticWindow { edges, .. } | Self::Limit { edges } - | Self::PassThrough { edges } => edges.output, - Self::HashJoin { edges } => edges.output, + | Self::PassThrough { edges } + | Self::PromqlRange { edges, .. } + | Self::PromqlSubquery { edges, .. } + | Self::PromqlRelabel { edges } + | Self::PromqlSeriesSample { edges, .. } + | Self::PromqlScalarToVector { edges } + | Self::PromqlVectorToScalar { edges } + | Self::PromqlPerSeries { edges, .. } + | Self::PromqlPresence { edges } => edges.output, + Self::HashJoin { edges } + | Self::PromqlBinary { edges, .. } + | Self::PromqlInfoEnrich { edges, .. } => edges.output, Self::Concat { output, .. } => *output, + Self::PromqlScalarLeaf { output, .. } => *output, + } + } + + pub fn promql_input(&self, index: usize) -> Option { + match self { + Self::Concat { + promql: Some(edges), + .. + } => edges.inputs.get(index).copied(), + Self::PromqlScalarLeaf { .. } => None, + Self::HashJoin { edges } + | Self::PromqlBinary { edges, .. } + | Self::PromqlInfoEnrich { edges, .. } => edges.promql?.inputs.get(index).copied(), + _ => self.unary_promql().and_then( + |edges| { + if index == 0 { + Some(edges.input) + } else { + None + } + }, + ), + } + } + + pub fn promql_output(&self) -> Option { + match self { + Self::Concat { + promql: Some(edges), + .. + } => Some(edges.output), + Self::PromqlScalarLeaf { promql_output, .. } => Some(*promql_output), + Self::HashJoin { edges } + | Self::PromqlBinary { edges, .. } + | Self::PromqlInfoEnrich { edges, .. } => Some(edges.promql?.output), + _ => Some(self.unary_promql()?.output), + } + } + + pub(crate) fn unary_promql(&self) -> Option { + match self { + Self::Scan { edges, .. } + | Self::Filter { edges } + | Self::Project { edges } + | Self::HashAggregate { edges, .. } + | Self::InMemoryComparisonSort { edges, .. } + | Self::TopK { edges } + | Self::HashDeduplicate { edges, .. } + | Self::InMemoryAnalyticWindow { edges, .. } + | Self::Limit { edges } + | Self::PassThrough { edges } + | Self::PromqlRange { edges, .. } + | Self::PromqlSubquery { edges, .. } + | Self::PromqlRelabel { edges } + | Self::PromqlSeriesSample { edges, .. } + | Self::PromqlScalarToVector { edges } + | Self::PromqlVectorToScalar { edges } + | Self::PromqlPerSeries { edges, .. } + | Self::PromqlPresence { edges } => edges.promql, + _ => None, } } } diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 1cec8802..af6b0fa9 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -157,7 +157,7 @@ canonical workload and query terms rather than defining parallel strings: | Invocation schedule | `QueryRecurrence`; the evaluation count is derived, not copied. | | Event-time coverage | `TimeSelection`. | | Logical sources | the existing query-IR `Source`, one per scan. | -| Filters | canonical bound `Predicate` values copied from the query IR. | +| Source selection | canonical bound `Predicate` values for ordinary scans and symbolic `InfoMatcher` values for info-metric scans. | | Physical source contents | provider-owned `source_snapshot_id` per source. | The snapshot identifier is the only new scope concept. It is necessary because @@ -168,7 +168,7 @@ independent facts. `ComparisonScope::from_workload` copies arrival, recurrence, and time selection from `DataWorkload` and `QueryWorkloadEntry`; the catalog/lowering boundary adds -the source, snapshot identifier, and canonical predicates. Raw and candidate +the source, snapshot identifier, and canonical source selection. Raw and candidate scopes must match exactly in every field before their estimates are compared. Unknown subsumption such as "this wider retained summary covers the requested interval" is not guessed here; it requires a separate semantic coverage proof. @@ -180,7 +180,7 @@ identifiers, invalid recurrence, or a zero horizon also fail closed. Every reachable physical `Scan` carries one exact `SourceCoverage` copied from this scope. That coverage includes the existing `Source`, its provider-owned -snapshot ID, and canonical predicates. A scan with no coverage, or coverage +snapshot ID, and canonical ordinary predicates or info-metric matchers. A scan with no coverage, or coverage not present in `ComparisonScope.sources`, makes the plan unavailable. Other operators cannot declare source coverage. This prevents a DAG over source B from being estimated under source A's comparison scope. @@ -225,6 +225,17 @@ OperatorStatistics = } | Limit { edges: UnaryEdgeStatistics } | PassThrough { edges: UnaryEdgeStatistics } + | PromqlRange { edges, max_window_samples_per_series } + | PromqlSubquery { edges, subquery_steps } + | PromqlBinary { edges, matching_key_bytes } + | PromqlRelabel { edges } + | PromqlInfoEnrich { edges, matching_key_bytes } + | PromqlSeriesSample { edges, group_count, key_bytes } + | PromqlScalarToVector { edges } + | PromqlVectorToScalar { edges } + | PromqlScalarLeaf { output, promql_output } + | PromqlPerSeries { edges, accumulator_bytes_per_series } + | PromqlPresence { edges } } ``` @@ -235,6 +246,24 @@ remains operator-independent: an edge carries logical rows and bytes, and the same edge is the output of one node and an input of every consumer. Making the edge type depend on either endpoint would prevent direct consistency checks. +PromQL operators additionally attach a typed `PromqlEdgeStatistics` view to +the same physical edge: + +```text +PromqlEdgeStatistics { + series, + evaluation_steps, + value_kind: Scalar | Vector | RangeVector, +} +``` + +Rows/bytes still describe materialized logical data; PromQL metadata describes +its series and step shape. They are complementary, not alternative +cardinalities. Unary, binary, and variadic wrappers preserve the physical +arity, and parent input metadata must exactly match the corresponding child +output. Once a child carries PromQL metadata, a parent may not silently drop +it. Scalar leaves have zero inputs and emit exactly one scalar row per step. + The outer enum is operator-specific. A Filter cannot accidentally carry group cardinality, a Top-K statistics record cannot carry a join build side, and a non-scan record cannot carry source-read bytes. Serialized evidence is @@ -254,6 +283,14 @@ Physical configuration is not catalog evidence and therefore lives on | `HashJoin` | left or right build side and equality-key count | | `HashDeduplicate` | deduplication-key count | | `InMemoryAnalyticWindow` | partition/order-key counts and window-function work per row | +| `PromqlRange` | selector range duration | +| `PromqlSubquery` | range and optional explicit resolution | +| `PromqlBinary` | operation class, operand modes, match cardinality, and vector/vector hash-build side | +| `PromqlRelabel` | expression work per sample | +| `PromqlInfoEnrich` | info-selector matcher work per info row | +| `PromqlSeriesSample` | `limitk`/`limit_ratio` choice and grouping-key count | +| `PromqlPerSeries` | primitive update work and accumulator count | +| `PromqlPresence` | absence versus per-series-presence mode and test work per input row | This distinction removes the former flat optional `k` field. A Top-K bound is part of the chosen algorithm, while input/output cardinality and width are @@ -425,10 +462,21 @@ The supported mappings are: | partitioned Sort followed by Limit | Sort → Limit | | RowNumber/Rank/DenseRank SQLWindowFunc with non-empty order_by | InMemoryAnalyticWindow | | identity TimeShift only | PassThrough | - -Per-entity reductions, HAVING, ordered/distribution-dependent intents such as -exact quantile or cardinality, Top-K aggregate intents, and extensions remain -unavailable until they have an explicit physical algorithm. Hash-join lowering +| range selector `[duration]` | PromqlRange | +| PromQL subquery `[range:resolution]` | PromqlSubquery | +| PromQL scalar/vector or vector/vector binary expression | PromqlBinary | +| `label_replace`/`label_join` | PromqlRelabel | +| `info(...)` | info-series Scan → PromqlInfoEnrich | +| `limitk`/`limit_ratio` | PromqlSeriesSample | +| `vector(scalar)` / `scalar(vector)` | PromqlScalarToVector / PromqlVectorToScalar | +| scalar literal, `time()`, or `pi()` | PromqlScalarLeaf | +| supported fixed-state per-series range reduction | PromqlPerSeries | +| absence/presence reduction | PromqlPresence | + +HAVING, cross-series PromQL aggregation, and ordered/distribution-dependent +per-series intents such as exact quantile, cardinality, and Top-K aggregate +intents remain unavailable until they have an explicit physical algorithm. +Hash-join lowering also uses the bound left and right output schemas to prove that every equality compares one column from each side; same-side or out-of-range `ColumnId`s fail closed. @@ -447,11 +495,11 @@ embedded physical identity and that every node's buffer equals the provider snapshot before returning statistics, preventing the public node and evidence views from silently drifting apart. -Cross/non-equi joins, `INTERSECT`, `EXCEPT`, distinct `UNION`, PromQL -range/subquery execution, vector matching, and PromQL-specific enrichment/ -relabel/sample operators stay unavailable. Their cost requires a physical -implementation or multiplicity/state facts that the current physical-operator -vocabulary cannot represent; they are not treated as free pass-through work. +Cross/non-equi joins, `INTERSECT`, `EXCEPT`, distinct `UNION`, and unlisted +PromQL algorithms stay unavailable. They are not treated as free pass-through +work. In particular, generic `HashAggregate` cannot masquerade as a PromQL +cross-series aggregation: such a candidate needs a physical operator whose +per-step series grouping and memory semantics are explicit. ## Physical operator formulas @@ -472,6 +520,16 @@ the DAG rules above. | Concat | `output_rows` | one output row/batch | `0` | | In-memory analytic window | per-partition ordering work plus per-row function work | largest partition bytes | `0` | | Limit | `min(input_rows, limit + offset)` | one output row/batch | `0` | +| PromQL range | `input_rows` | `input_series × max_window_samples_per_series × sample_bytes` | `0` | +| PromQL subquery | `input_rows + output_rows` | materialized inner-step input bytes | `0` | +| PromQL vector binary | `left_rows + right_rows + output_rows` | vector/vector match hash state, or one output row for scalar/vector | `0` | +| PromQL relabel | `input_rows × expression_operations_per_row` | one output row/batch | `0` | +| PromQL info enrichment | input/output visits plus matcher work | info-side series hash state | `0` | +| PromQL series sample | `input_rows + input_series` | selected-series key state | `0` | +| PromQL scalar/vector bridge | `input_rows + output_rows` | one output row/batch | `0` | +| PromQL scalar leaf | `evaluation_steps` | one scalar row | `0` | +| PromQL fixed-state per-series operation | `input_rows × operations_per_row` | `input_series × accumulator_bytes_per_series` | `0` | +| PromQL presence | `input_rows × operations_per_row + output_rows` | one output row/batch | `0` | These formulas name physical implementations. The enum uses names such as `InMemoryComparisonSort`, `HashDeduplicate`, and `InMemoryAnalyticWindow` so a diff --git a/docs/design_docs/asap-aware-mapping/physical-plan-integration.md b/docs/design_docs/asap-aware-mapping/physical-plan-integration.md index 8af1105b..735d177a 100644 --- a/docs/design_docs/asap-aware-mapping/physical-plan-integration.md +++ b/docs/design_docs/asap-aware-mapping/physical-plan-integration.md @@ -271,11 +271,115 @@ is unavailable until lowering propagates that temporal context into descendant Scan evidence and physical identity. `PassThrough` must never hide an unsupported operation. +### `PromqlRange { range_millis }` + +Forms a range vector for each evaluation step and retains at most the observed +`max_window_samples_per_series` samples for each input series. For example, +the selector in `rate(http_requests_total[5m])` lowers to `PromqlRange` with +`range_millis = 300000`. Evidence carries series count, evaluation-step count, +value kind, and the maximum samples in one per-series range. CPU visits every +input sample; local memory is `series × max_window_samples_per_series × +sample_width`. This query-time range buffer is not a tumbling, sliding, pane, +or exponential-histogram layout for maintaining a summary. + +### `PromqlSubquery { range_millis, resolution_millis }` + +Evaluates a child expression at inner steps and groups those results into a +range vector at each outer step. For example, +`max_over_time(rate(http_requests_total[5m])[1h:1m])` contains a one-hour +subquery at one-minute resolution. Evidence supplies the realized +`subquery_steps`; child steps must equal `outer_steps × subquery_steps`. CPU +charges child-result visits plus emitted results, and memory retains the +materialized inner-step input for one invocation. + +### `PromqlBinary { operation, operand_mode, cardinality, build_side }` + +Evaluates scalar/vector arithmetic or comparison, or a PromQL `and`, `or`, or +`unless` set operation. For example, +`rate(errors_total[5m]) / on(service) group_left(region) service_info` is a +many-to-one vector match; `up or maintenance_mode` is a set union. The node +records whether each operand is scalar or vector, vector-match cardinality, +operation class, and a hash-build side for vector/vector matching. Evidence +supplies both edge shapes and encoded matching-key width. CPU visits both +inputs and the output; vector/vector memory is the selected build-side series +times key width plus hash metadata. Operation-specific cardinality bounds are +validated (`or` may use the sum, while `and`/`unless` cannot exceed the left). + +### `PromqlRelabel { expression_operations_per_row }` + +Evaluates a PromQL label transformation without changing series or sample +cardinality. For example, +`label_replace(up, "host", "$1", "instance", "(.*):.*")` lowers to this +operator. The physical configuration counts expression work per sample; CPU +is rows times that count and memory is one output row or batch. + +### `PromqlInfoEnrich { matcher_operations_per_info_row }` + +Builds a lookup over an info metric and enriches the data vector while +preserving its left-side cardinality. For example, +`info(rate(http_server_request_duration_seconds_count[2m]))` lowers the data +expression and a separately scoped info-series `Scan`, then joins them here. +Evidence supplies both vector edges and matching-key width. CPU visits data, +info, and output rows plus selector-matcher work; memory is the info-side hash +state. Both scans must be present in the comparison scope. + +### `PromqlSeriesSample { kind, grouping_key_count }` + +Selects series by deterministic sampling within optional label groups. For +example, `limitk(10, up)` records `LimitK { k: 10 }`, while +`limit_ratio(0.1, up)` records the exact ratio bits. Evidence supplies group +count and encoded selection-key width. CPU visits samples and hashes input +series; memory retains selected series keys. `limitk` output is bounded by +`min(input_series, group_count × k)`. The current lowerer rejects the +unsupported `without` grouping form instead of changing its semantics. + +### `PromqlScalarToVector` + +Materializes one vector sample per evaluation step from a scalar, as in +`vector(1)`. Its edge contract is Scalar to a one-series Vector with the same +steps. CPU visits input and output rows; memory is one output row or batch. + +### `PromqlVectorToScalar` + +Converts a vector to a scalar at each evaluation step, as in `scalar(up)`. +Its edge contract is Vector to Scalar with the same steps. CPU visits input +and output rows; memory is one output row or batch. + +### `PromqlScalarLeaf` + +Produces a source-free scalar for each evaluation step. Number literals, +`time()`, and `pi()` are examples. It has no child or external input edge, and +its output must contain exactly one scalar row per step. CPU is one operation +per emitted scalar and memory is one scalar row. + +### `PromqlPerSeries { operations_per_row, accumulator_count }` + +Updates fixed-size state independently for each series and emits at most one +instant-vector sample per series and step. Examples include +`rate(http_requests_total[5m])` after `PromqlRange`, and +`avg_over_time(temperature_celsius[10m])`. The configuration records primitive +update work and accumulator count; evidence supplies accumulator bytes per +series. CPU is `input_rows × operations_per_row`; memory is +`input_series × accumulator_bytes_per_series`. Ordered or +distribution-dependent functions are unavailable until a distinct physical +algorithm and formula exist. + +### `PromqlPresence { kind, operations_per_row }` + +Implements presence semantics while preserving their two different +cardinality rules. `Absent` covers `absent(up)` and +`absent_over_time(up[5m])`; it may synthesize at most one series and one row +per evaluation step, and an entirely empty input emits one row per step. +`PresentPerSeries` covers `present_over_time(up[5m])`; its output is bounded by +the input series and input rows. CPU visits input and output rows; memory is +one output row or batch. + ## Comparison and failure behavior Raw and post-ASAP alternatives must cover the same `ComparisonScope`: arrival mode, planning time, workload horizon, recurrence, event-time selection, -logical sources, predicates, and physical source snapshots. +logical sources, ordinary predicates or info-metric matchers, and physical +source snapshots. Missing statistics, stale evidence, an unsupported physical algorithm, unlowered logical operations, inconsistent edges, or different comparison From 8ca8310d22d0b8c00f6f033bed59bcf740594ca1 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 11:04:46 -0600 Subject: [PATCH 40/48] feat(cost): rank complete physical plan alternatives --- .../src/analytical_planner.rs | 480 ++++++++++++++++++ crates/asap-aware-mapping/src/cost_model.rs | 12 + crates/asap-aware-mapping/src/lib.rs | 1 + crates/asap-aware-mapping/src/replacement.rs | 82 +++ .../analytical-resource-cost.md | 48 +- 5 files changed, 615 insertions(+), 8 deletions(-) create mode 100644 crates/asap-aware-mapping/src/analytical_planner.rs diff --git a/crates/asap-aware-mapping/src/analytical_planner.rs b/crates/asap-aware-mapping/src/analytical_planner.rs new file mode 100644 index 00000000..e95d1ae8 --- /dev/null +++ b/crates/asap-aware-mapping/src/analytical_planner.rs @@ -0,0 +1,480 @@ +//! Evidence-backed physical-plan costing at the planner selection boundary. + +use std::rc::Rc; + +use asap_types::post_asap::{SketchAlgorithm, SummaryExpr, SummaryNode}; +use asap_types::pre_asap::AggIntent; + +use crate::analytical_cost::{ + estimate_physical_dag_comparison, AnalyticalCostError, PhysicalDagComparisonEstimate, + PhysicalDagEstimateRequest, ResourceCalibration, +}; +use crate::analytical_lowering::{ + lower_query_physical_dag, PhysicalDag, PhysicalNodeEvidence, PhysicalNodeEvidenceProvider, + PhysicalNodeRequest, +}; +use crate::analytical_statistics::ComparisonScope; +use crate::cost_model::{Cost, CostModel, DefaultCostModel}; +use crate::replacement::{Replacement, ReplacementSubDAG, TargetSubDAG}; + +/// Deployment evidence needed to price one planner alternative. +/// +/// The planner lowers raw queries and logical rewrites itself. A deployment +/// supplies the comparison scope and atomic evidence for each selected query +/// operator. Post-ASAP summary operators need a physical binder because their +/// implementation, placement, and retained-state layout are deployment +/// choices; that binder must return the complete summary DAG, including any +/// embedded `KeepPreAsap` work. +pub trait PlannerPhysicalPlanProvider { + fn comparison_scope( + &self, + target: &TargetSubDAG<'_>, + ) -> Result; + + fn query_node_evidence( + &self, + request: PhysicalNodeRequest<'_>, + ) -> Result; + + fn summary_physical_dag( + &self, + summary: &Rc, + target: &TargetSubDAG<'_>, + scope: &ComparisonScope, + ) -> Result; +} + +/// Dimensional comparison retained for explanations and verification. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct PlannerCandidateEstimate { + pub resources: PhysicalDagComparisonEstimate, + pub raw_cost: Cost, + pub candidate_cost: Cost, +} + +/// Planner cost model that admits only complete, cheaper physical plans. +/// +/// There is deliberately no structural or compact-formula fallback. Failure +/// to lower either alternative, missing statistics, an unknown physical +/// algorithm, or invalid source/horizon evidence makes the candidate +/// unavailable and leaves the target on its raw path. +pub struct AnalyticalPlannerCostModel<'a> { + provider: &'a dyn PlannerPhysicalPlanProvider, + calibration: ResourceCalibration, +} + +impl<'a> AnalyticalPlannerCostModel<'a> { + pub fn new( + provider: &'a dyn PlannerPhysicalPlanProvider, + calibration: ResourceCalibration, + ) -> Result { + calibration.validate()?; + Ok(Self { + provider, + calibration, + }) + } + + pub fn estimate_candidate( + &self, + candidate: &ReplacementSubDAG, + target: &TargetSubDAG<'_>, + ) -> Result { + struct QueryEvidence<'a>(&'a dyn PlannerPhysicalPlanProvider); + impl PhysicalNodeEvidenceProvider for QueryEvidence<'_> { + fn evidence( + &self, + request: PhysicalNodeRequest<'_>, + ) -> Result { + self.0.query_node_evidence(request) + } + } + + let scope = self.provider.comparison_scope(target)?; + let evidence = QueryEvidence(self.provider); + let raw = lower_query_physical_dag(target.root, &scope, &evidence)?; + let replacement = match &candidate.replacement { + Replacement::Rewrite(query) => lower_query_physical_dag(query, &scope, &evidence)?, + Replacement::Summary(summary) => match &summary.expr { + SummaryExpr::KeepPreAsap(query) => { + lower_query_physical_dag(query, &scope, &evidence)? + } + _ => self + .provider + .summary_physical_dag(summary, target, &scope)?, + }, + }; + let resources = estimate_physical_dag_comparison( + PhysicalDagEstimateRequest { + nodes: &raw.nodes, + root: &raw.root, + scope: &scope, + statistics: &raw, + }, + PhysicalDagEstimateRequest { + nodes: &replacement.nodes, + root: &replacement.root, + scope: &scope, + statistics: &replacement, + }, + )?; + let raw_cost = Cost(resources.raw.calibrated_cost(&self.calibration)?); + let candidate_cost = Cost(resources.candidate.calibrated_cost(&self.calibration)?); + Ok(PlannerCandidateEstimate { + resources, + raw_cost, + candidate_cost, + }) + } +} + +impl CostModel for AnalyticalPlannerCostModel<'_> { + fn candidate_cost_covers_complete_plan(&self) -> bool { + true + } + + fn candidate_cost( + &self, + candidate: &ReplacementSubDAG, + target: &TargetSubDAG<'_>, + ) -> Option { + self.estimate_candidate(candidate, target) + .ok() + .filter(|estimate| estimate.candidate_cost < estimate.raw_cost) + .map(|estimate| estimate.candidate_cost) + } + + fn rank_candidates( + &self, + intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + // Physical plans do not exist at algorithm enumeration time. Final + // ranking happens after binding, through `candidate_cost` above. + DefaultCostModel.rank_candidates(intent, candidates) + } + + fn estimate_cost(&self, candidate: &ReplacementSubDAG, target: &TargetSubDAG<'_>) -> f64 { + self.candidate_cost(candidate, target) + .map_or(f64::NAN, |cost| cost.0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + use asap_types::pre_asap::{Column, DataType, QueryExpr, Reduction, Schema, Source}; + use asap_types::types::AccuracyTarget; + use asap_types::workload::{ + DataArrival, DurationMs, QueryRecurrence, QueryTimeScope, TimeSelection, TimestampMs, + }; + + use crate::analytical_cost::{ExecutionMultiplicity, PhysicalDagNode, PhysicalOperator}; + use crate::analytical_statistics::{EdgeStatistics, OperatorStatistics, SourceCoverage}; + use crate::replacement::ReplacementStrategy; + + fn edge(rows: u64, bytes: u64) -> EdgeStatistics { + EdgeStatistics { rows, bytes } + } + + fn statistics( + source_scan_bytes: u64, + inputs: Vec, + output: EdgeStatistics, + ) -> OperatorStatistics { + OperatorStatistics { + source_scan_bytes, + inputs, + output, + group_count: None, + key_bytes: None, + aggregate_value_bytes: None, + k: None, + hash_join_build_side: None, + promql: None, + } + } + + fn query() -> Rc { + Rc::new(QueryExpr::Aggregate { + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::Count { + accuracy: AccuracyTarget::Epsilon(0.01), + }], + output_names: vec![], + having: None, + child: Rc::new(QueryExpr::Scan { + source: Source::Table { + table_ref: "events".into(), + }, + predicates: vec![], + schema: Schema::new(vec![Column::new("value", DataType::Float64, false)]), + }), + }) + } + + fn scope() -> ComparisonScope { + ComparisonScope { + data_arrival: DataArrival::AtRest, + planning_time: TimestampMs(1_000), + horizon: DurationMs(10_000), + recurrence: QueryRecurrence::OneTime { + invocations: 10, + execute_at: None, + }, + time_selection: TimeSelection { + scope: QueryTimeScope::Longitudinal, + lookback: Some(DurationMs(10_000)), + as_of: Some(TimestampMs(1_000)), + }, + sources: vec![SourceCoverage { + source: Source::Table { + table_ref: "events".into(), + }, + snapshot_id: "snapshot-1".into(), + predicates: vec![], + }], + } + } + + struct TestProvider { + summary_available: bool, + candidate_scan_bytes: u64, + } + + impl TestProvider { + fn summary_dag(&self, scope: &ComparisonScope) -> PhysicalDag { + let scan_statistics = statistics( + self.candidate_scan_bytes, + vec![edge(100, 800)], + edge(100, 800), + ); + let mut aggregate_statistics = statistics(0, vec![edge(100, 800)], edge(1, 8)); + aggregate_statistics.group_count = Some(1); + aggregate_statistics.key_bytes = Some(8); + aggregate_statistics.aggregate_value_bytes = Some(8); + let read_statistics = statistics(0, vec![edge(1, 8)], edge(1, 8)); + let evidence = HashMap::from([ + ( + "candidate-scan".into(), + PhysicalNodeEvidence { + physical_id: "candidate-scan".into(), + statistics: scan_statistics, + output_buffer_bytes: 8, + }, + ), + ( + "candidate-state".into(), + PhysicalNodeEvidence { + physical_id: "candidate-state".into(), + statistics: aggregate_statistics, + output_buffer_bytes: 8, + }, + ), + ( + "candidate-read".into(), + PhysicalNodeEvidence { + physical_id: "candidate-read".into(), + statistics: read_statistics, + output_buffer_bytes: 8, + }, + ), + ]); + PhysicalDag { + nodes: vec![ + PhysicalDagNode { + id: "candidate-scan".into(), + operator: PhysicalOperator::Scan, + children: vec![], + source_coverage: Some(scope.sources[0].clone()), + output_buffer_bytes: 8, + retained_bytes: 0, + execution: ExecutionMultiplicity::Once, + }, + PhysicalDagNode { + id: "candidate-state".into(), + operator: PhysicalOperator::HashAggregate, + children: vec!["candidate-scan".into()], + source_coverage: None, + output_buffer_bytes: 8, + retained_bytes: 8, + execution: ExecutionMultiplicity::Once, + }, + PhysicalDagNode { + id: "candidate-read".into(), + operator: PhysicalOperator::PassThrough, + children: vec!["candidate-state".into()], + source_coverage: None, + output_buffer_bytes: 8, + retained_bytes: 0, + execution: ExecutionMultiplicity::PerEvaluation, + }, + ], + root: "candidate-read".into(), + evidence, + } + } + } + + impl PlannerPhysicalPlanProvider for TestProvider { + fn comparison_scope( + &self, + _target: &TargetSubDAG<'_>, + ) -> Result { + Ok(scope()) + } + + fn query_node_evidence( + &self, + request: PhysicalNodeRequest<'_>, + ) -> Result { + let (physical_id, mut statistics) = match request.operator { + PhysicalOperator::Scan => ( + "raw-scan", + statistics(800, vec![edge(100, 800)], edge(100, 800)), + ), + PhysicalOperator::HashAggregate => ( + "raw-aggregate", + statistics(0, vec![edge(100, 800)], edge(1, 8)), + ), + _ => return Err(AnalyticalCostError::UnsupportedQueryOperator), + }; + if request.operator == PhysicalOperator::HashAggregate { + statistics.group_count = Some(1); + statistics.key_bytes = Some(8); + statistics.aggregate_value_bytes = Some(8); + } + Ok(PhysicalNodeEvidence { + physical_id: physical_id.into(), + output_buffer_bytes: 8, + statistics, + }) + } + + fn summary_physical_dag( + &self, + _summary: &Rc, + _target: &TargetSubDAG<'_>, + scope: &ComparisonScope, + ) -> Result { + self.summary_available + .then(|| self.summary_dag(scope)) + .ok_or(AnalyticalCostError::MissingOrStale("summary_physical_plan")) + } + } + + fn calibration() -> ResourceCalibration { + ResourceCalibration { + cost_per_cpu_op: 1.0, + cost_per_scan_byte: 1.0, + cost_per_retained_byte: 1.0, + version: "test-v1".into(), + } + } + + #[test] + fn global_selection_uses_complete_physical_comparison() { + let root = query(); + let space = crate::replacement::search_workload_with( + vec![("q", Rc::clone(&root))], + &crate::replacement::default_strategies(), + ); + let planned_root = Rc::clone(&space.roots[0].1); + let provider = TestProvider { + summary_available: true, + candidate_scan_bytes: 800, + }; + let model = AnalyticalPlannerCostModel::new(&provider, calibration()).unwrap(); + + let selected = space.global_selection(&model); + assert!( + selected.for_target(&planned_root).unwrap().chosen.is_some(), + "a fully bound build-once summary cheaper than ten raw scans must be selected" + ); + } + + #[test] + fn missing_summary_evidence_keeps_the_raw_target() { + let root = query(); + let space = crate::replacement::search_workload_with( + vec![("q", Rc::clone(&root))], + &crate::replacement::default_strategies(), + ); + let planned_root = Rc::clone(&space.roots[0].1); + let provider = TestProvider { + summary_available: false, + candidate_scan_bytes: 800, + }; + let model = AnalyticalPlannerCostModel::new(&provider, calibration()).unwrap(); + + let selected = space.global_selection(&model); + assert!( + selected.for_target(&planned_root).unwrap().chosen.is_none(), + "missing physical summary evidence must not fall back to a structural estimate" + ); + } + + #[test] + fn candidate_with_incomplete_scope_is_unavailable() { + struct WrongScope(TestProvider); + impl PlannerPhysicalPlanProvider for WrongScope { + fn comparison_scope( + &self, + target: &TargetSubDAG<'_>, + ) -> Result { + self.0.comparison_scope(target) + } + + fn query_node_evidence( + &self, + request: PhysicalNodeRequest<'_>, + ) -> Result { + self.0.query_node_evidence(request) + } + + fn summary_physical_dag( + &self, + summary: &Rc, + target: &TargetSubDAG<'_>, + scope: &ComparisonScope, + ) -> Result { + let mut dag = self.0.summary_physical_dag(summary, target, scope)?; + dag.nodes[0].source_coverage.as_mut().unwrap().snapshot_id = "other".into(); + Ok(dag) + } + } + + let root = query(); + let candidates = crate::replacement::SketchAlgorithmStrategy::default_cost_model() + .replacements(&TargetSubDAG::new(&root)); + let provider = WrongScope(TestProvider { + summary_available: true, + candidate_scan_bytes: 800, + }); + let model = AnalyticalPlannerCostModel::new(&provider, calibration()).unwrap(); + assert_eq!( + model.candidate_cost(&candidates[0], &TargetSubDAG::new(&root)), + None + ); + } + + #[test] + fn complete_candidate_that_costs_more_than_raw_is_not_selected() { + let root = query(); + let space = crate::replacement::search_workload_with( + vec![("q", Rc::clone(&root))], + &crate::replacement::default_strategies(), + ); + let planned_root = Rc::clone(&space.roots[0].1); + let provider = TestProvider { + summary_available: true, + candidate_scan_bytes: 100_000, + }; + let model = AnalyticalPlannerCostModel::new(&provider, calibration()).unwrap(); + + let selected = space.global_selection(&model); + assert!(selected.for_target(&planned_root).unwrap().chosen.is_none()); + } +} diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index 7f3407d3..bb65c776 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -190,6 +190,18 @@ pub fn default_cse_shared_maintenance_cost(family: &SummaryFamilyType) -> Cost { /// `replacement::implementations_for_with` constructs every candidate in the /// resulting order. pub trait CostModel { + /// Whether [`Self::candidate_cost`] prices a complete physical + /// alternative, including its raw baseline, rather than a local + /// heuristic for one memo-group node. + /// + /// Complete-plan models make every CSE alternative available to final + /// ranking. Choosing a share/recompute arm first through the legacy + /// structural hooks would discard a physical alternative before its + /// evidence-backed cost was compared. + fn candidate_cost_covers_complete_plan(&self) -> bool { + false + } + /// Candidate-level availability for final selection. The default keeps /// every candidate, including legacy models whose numeric estimate is a /// display-only placeholder. Models that require evidence override this diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 9f1a16cb..e7098b61 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -185,6 +185,7 @@ pub mod accuracy; pub mod accuracy_reconciliation; pub mod analytical_cost; pub mod analytical_lowering; +pub mod analytical_planner; pub mod analytical_statistics; pub mod cost_model; pub mod explanation; diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index 402b5267..e3a4deec 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -3083,6 +3083,18 @@ impl PlanSpace { }); let chosen = if lifecycle_choice.is_some() { lifecycle_choice + } else if cost_model.candidate_cost_covers_complete_plan() { + let effective_target = TargetSubDAG::with_consumer_count(&group.target, effective); + group + .candidates + .iter() + .filter_map(|candidate| { + cost_model + .candidate_cost(candidate, &effective_target) + .map(|cost| (candidate, cost)) + }) + .min_by(|(_, left), (_, right)| left.0.total_cmp(&right.0)) + .map(|(candidate, _)| candidate) } else if effective >= 2 && cse_candidate_pair(group).is_some() { let decision = if let Some(profiles) = profiles { decide_group_with_recurrence( @@ -3175,6 +3187,22 @@ impl PlanSpace { }) }; + if cost_model.candidate_cost_covers_complete_plan() { + if let Some(candidate) = chosen.filter(|candidate| is_cse_candidate(candidate)) { + let Replacement::Rewrite(rewrite) = &candidate.replacement else { + unreachable!("CSE candidates are logical rewrites") + }; + chosen_share.insert( + *ptr, + if Rc::ptr_eq(rewrite, &group.target) { + ShareDecision::Share + } else { + ShareDecision::RecomputeIndependently + }, + ); + } + } + let outgoing_multiplier = multiplier(*ptr, &effective_uses, &chosen_share); match chosen { Some(ReplacementSubDAG { @@ -5823,6 +5851,60 @@ mod tests { ); } + #[test] + fn complete_plan_costs_rank_both_cse_arms_without_structural_preselection() { + struct CompletePlanCost; + impl CostModel for CompletePlanCost { + fn candidate_cost_covers_complete_plan(&self) -> bool { + true + } + + fn candidate_cost( + &self, + candidate: &ReplacementSubDAG, + target: &TargetSubDAG<'_>, + ) -> Option { + let Replacement::Rewrite(rewrite) = &candidate.replacement else { + return None; + }; + Some(if Rc::ptr_eq(rewrite, target.root) { + Cost(1.0) + } else { + Cost(10.0) + }) + } + + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + candidates.to_vec() + } + + fn cse_share_decision(&self, _candidate: &CseCandidate) -> ShareDecision { + panic!("complete physical costs must bypass structural CSE preselection") + } + } + + let shared = Rc::new(QueryExpr::Dedup { + cols: vec![0], + child: Rc::new(metric_scan(&["job"])), + }); + let space = search_workload(vec![ + ("left", Rc::clone(&shared)), + ("right", Rc::clone(&shared)), + ]); + let planned = &space.roots[0].1; + + let selected = space.global_selection(&CompletePlanCost); + let chosen = selected.for_target(planned).unwrap().chosen.unwrap(); + assert!(matches!( + &chosen.replacement, + Replacement::Rewrite(rewrite) if Rc::ptr_eq(rewrite, planned) + )); + } + #[test] fn effective_repetition_materializes_a_cse_choice_for_a_single_edge_child() { use asap_types::pre_asap::expr_ir::ScalarValue; diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index af6b0fa9..c6609868 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -13,14 +13,20 @@ and guarantee composition run first; costing ranks only the candidates that survive. Missing evidence produces an unavailable estimate, never an assumed zero or a structural-cost fallback. -This document distinguishes three implementation layers: +This document distinguishes four implementation layers: - the physical-DAG estimator, which can compose any DAG whose nodes have supported physical operators and complete `OperatorStatistics`; and - query-DAG lowering, which recursively maps supported resolved `QueryExpr` operators to that physical representation; and -- replacement lowering and ranking, which must compare complete alternatives. - +- the deployment summary binder, which maps a selected `SummaryExpr` DAG to + physical summary operators and snapshots their evidence; and +- the planner-ranking adapter, which compares the complete raw and replacement + DAGs before making a candidate available to global selection. + +Support in the estimator does not imply that a deployment has selected and +bound that physical algorithm. Unknown query lowering or summary binding makes +the entire alternative unavailable. No layer may substitute a shape-specific shortcut or structural node count. An estimate has physical dimensions: @@ -588,8 +594,32 @@ children would undercount the plan. complete physical DAG and per-node evidence. Filters, projections, joins, windows, nested aggregates, Top-K, and shared sub-DAGs therefore use the same estimation path. The generic query lowerer recursively maps the supported raw -query operators into that representation. Replacement lowering remains a -separate layer and must include all summary-maintenance work. +query operators into that representation. + +`AnalyticalPlannerCostModel` is the final-selection adapter. For every +candidate it obtains one canonical `ComparisonScope`, recursively lowers the +actual raw target, and then lowers a logical rewrite or requests the fully +bound physical DAG for a `SummaryExpr` candidate. The deployment implements +`PlannerPhysicalPlanProvider`: query-node evidence is consumed atomically by +the generic query lowerer, while summary binding returns a complete +`PhysicalDag`, including embedded raw work, build/read operators, retained +state, execution multiplicity, and source coverage. The adapter calls +`estimate_physical_dag_comparison`; it never calls `DefaultCostModel` or a +structural-node-count fallback for final cost. + +A candidate is exposed to global selection only when both complete DAGs are +valid and its calibrated cost is strictly below the raw baseline. Missing or +stale evidence, an unknown physical algorithm, invalid edges, incomplete +source coverage, or a candidate that is not cheaper yields `None`. When no +candidate remains, `chosen = None` preserves the raw pre-ASAP target. + +Complete-plan costing also changes CSE selection order. Global selection sends +all share and recompute alternatives through `candidate_cost`; it does not use +the legacy structural CSE hook to discard one arm first. Once an arm is chosen, +the existing consumer-count propagation records whether that physical +alternative shares or recomputes. Within each returned physical DAG, stable +provider-owned physical IDs deduplicate shared scans, builds, and retained +states. Logical `Rc` identity is never substituted for physical identity. DDSketch is unavailable because occupied bins depend on value range and distribution. The model does not invent a bin count. Algorithm/parameter @@ -641,9 +671,11 @@ The intended end-to-end selection pipeline is: 6. applies calibration and ranks candidates by ascending cost. The query lowerer and physical estimator cover the supported raw-query shapes -listed above. Replacement evidence resolution, legality checks, and planner -integration remain separate layers; each preserves the complete-plan and -fail-closed requirements above. +listed above. `AnalyticalPlannerCostModel` executes this pipeline for every +candidate supplied to `PlanSpace::global_selection`. Logical rewrites are +lowered recursively. Summary candidates participate only after the deployment +has bound their complete `SummaryExpr` DAG; there is no optimistic generic +summary fallback. Before applying the following arithmetic, callers validate exact equality of the raw and selected alternative's `ComparisonScope`, and use the same From b8ba76ea22bdf07162119635c210b400096a2e3e Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 2 Sep 2026 20:41:23 -0600 Subject: [PATCH 41/48] test(cost): populate bounded operator evidence --- crates/asap-aware-mapping/src/analytical_planner.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/asap-aware-mapping/src/analytical_planner.rs b/crates/asap-aware-mapping/src/analytical_planner.rs index e95d1ae8..96314b57 100644 --- a/crates/asap-aware-mapping/src/analytical_planner.rs +++ b/crates/asap-aware-mapping/src/analytical_planner.rs @@ -192,6 +192,8 @@ mod tests { key_bytes: None, aggregate_value_bytes: None, k: None, + topk_output_offset: None, + limit_rows_consumed: None, hash_join_build_side: None, promql: None, } From a9454971111e4c55dba5a09fe6d9b816376381be Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 10:02:48 -0600 Subject: [PATCH 42/48] fix(cost): snapshot planner evidence per target --- .../src/analytical_planner.rs | 194 +++++++++++++----- 1 file changed, 144 insertions(+), 50 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_planner.rs b/crates/asap-aware-mapping/src/analytical_planner.rs index 96314b57..47d9b43c 100644 --- a/crates/asap-aware-mapping/src/analytical_planner.rs +++ b/crates/asap-aware-mapping/src/analytical_planner.rs @@ -1,9 +1,9 @@ //! Evidence-backed physical-plan costing at the planner selection boundary. -use std::rc::Rc; +use std::{cell::RefCell, rc::Rc}; use asap_types::post_asap::{SketchAlgorithm, SummaryExpr, SummaryNode}; -use asap_types::pre_asap::AggIntent; +use asap_types::pre_asap::{AggIntent, QueryExpr}; use crate::analytical_cost::{ estimate_physical_dag_comparison, AnalyticalCostError, PhysicalDagComparisonEstimate, @@ -17,6 +17,18 @@ use crate::analytical_statistics::ComparisonScope; use crate::cost_model::{Cost, CostModel, DefaultCostModel}; use crate::replacement::{Replacement, ReplacementSubDAG, TargetSubDAG}; +/// One immutable generation of deployment evidence for a planner target. +/// +/// The opaque version lets a provider bind every subsequent query and summary +/// lookup to the same catalog/runtime snapshot. A cost-model instance retains +/// this value for the target, so a caller that needs fresher evidence creates a +/// new model instead of mixing generations in one ranking decision. +#[derive(Debug, Clone, PartialEq)] +pub struct PlannerEvidenceSnapshot { + pub version: String, + pub scope: ComparisonScope, +} + /// Deployment evidence needed to price one planner alternative. /// /// The planner lowers raw queries and logical rewrites itself. A deployment @@ -26,21 +38,23 @@ use crate::replacement::{Replacement, ReplacementSubDAG, TargetSubDAG}; /// choices; that binder must return the complete summary DAG, including any /// embedded `KeepPreAsap` work. pub trait PlannerPhysicalPlanProvider { - fn comparison_scope( + /// Atomically captures the comparison scope and evidence generation. + fn capture_evidence_snapshot( &self, target: &TargetSubDAG<'_>, - ) -> Result; + ) -> Result; fn query_node_evidence( &self, + snapshot: &PlannerEvidenceSnapshot, request: PhysicalNodeRequest<'_>, ) -> Result; fn summary_physical_dag( &self, + snapshot: &PlannerEvidenceSnapshot, summary: &Rc, target: &TargetSubDAG<'_>, - scope: &ComparisonScope, ) -> Result; } @@ -61,6 +75,14 @@ pub struct PlannerCandidateEstimate { pub struct AnalyticalPlannerCostModel<'a> { provider: &'a dyn PlannerPhysicalPlanProvider, calibration: ResourceCalibration, + target_evidence: RefCell>, +} + +struct CachedTargetEvidence { + root: Rc, + consumer_count: usize, + snapshot: PlannerEvidenceSnapshot, + raw: PhysicalDag, } impl<'a> AnalyticalPlannerCostModel<'a> { @@ -72,49 +94,76 @@ impl<'a> AnalyticalPlannerCostModel<'a> { Ok(Self { provider, calibration, + target_evidence: RefCell::new(Vec::new()), }) } + fn target_evidence( + &self, + target: &TargetSubDAG<'_>, + ) -> Result<(PlannerEvidenceSnapshot, PhysicalDag), AnalyticalCostError> { + if let Some(cached) = self.target_evidence.borrow().iter().find(|cached| { + Rc::ptr_eq(&cached.root, target.root) && cached.consumer_count == target.consumer_count + }) { + return Ok((cached.snapshot.clone(), cached.raw.clone())); + } + + let snapshot = self.provider.capture_evidence_snapshot(target)?; + if snapshot.version.is_empty() { + return Err(AnalyticalCostError::MissingOrStale( + "planner_evidence_snapshot.version", + )); + } + snapshot.scope.validate()?; + let evidence = QueryEvidence { + provider: self.provider, + snapshot: &snapshot, + }; + let raw = lower_query_physical_dag(target.root, &snapshot.scope, &evidence)?; + self.target_evidence + .borrow_mut() + .push(CachedTargetEvidence { + root: Rc::clone(target.root), + consumer_count: target.consumer_count, + snapshot: snapshot.clone(), + raw: raw.clone(), + }); + Ok((snapshot, raw)) + } + pub fn estimate_candidate( &self, candidate: &ReplacementSubDAG, target: &TargetSubDAG<'_>, ) -> Result { - struct QueryEvidence<'a>(&'a dyn PlannerPhysicalPlanProvider); - impl PhysicalNodeEvidenceProvider for QueryEvidence<'_> { - fn evidence( - &self, - request: PhysicalNodeRequest<'_>, - ) -> Result { - self.0.query_node_evidence(request) - } - } - - let scope = self.provider.comparison_scope(target)?; - let evidence = QueryEvidence(self.provider); - let raw = lower_query_physical_dag(target.root, &scope, &evidence)?; + let (snapshot, raw) = self.target_evidence(target)?; + let scope = &snapshot.scope; + let evidence = QueryEvidence { + provider: self.provider, + snapshot: &snapshot, + }; let replacement = match &candidate.replacement { - Replacement::Rewrite(query) => lower_query_physical_dag(query, &scope, &evidence)?, + Replacement::Rewrite(query) => lower_query_physical_dag(query, scope, &evidence)?, Replacement::Summary(summary) => match &summary.expr { SummaryExpr::KeepPreAsap(query) => { - lower_query_physical_dag(query, &scope, &evidence)? + lower_query_physical_dag(query, scope, &evidence)? } _ => self .provider - .summary_physical_dag(summary, target, &scope)?, + .summary_physical_dag(&snapshot, summary, target)?, }, }; let resources = estimate_physical_dag_comparison( PhysicalDagEstimateRequest { nodes: &raw.nodes, root: &raw.root, - scope: &scope, + scope, statistics: &raw, }, PhysicalDagEstimateRequest { nodes: &replacement.nodes, root: &replacement.root, - scope: &scope, + scope, statistics: &replacement, }, )?; @@ -128,6 +177,20 @@ impl<'a> AnalyticalPlannerCostModel<'a> { } } +struct QueryEvidence<'a> { + provider: &'a dyn PlannerPhysicalPlanProvider, + snapshot: &'a PlannerEvidenceSnapshot, +} + +impl PhysicalNodeEvidenceProvider for QueryEvidence<'_> { + fn evidence( + &self, + request: PhysicalNodeRequest<'_>, + ) -> Result { + self.provider.query_node_evidence(self.snapshot, request) + } +} + impl CostModel for AnalyticalPlannerCostModel<'_> { fn candidate_cost_covers_complete_plan(&self) -> bool { true @@ -163,6 +226,7 @@ impl CostModel for AnalyticalPlannerCostModel<'_> { #[cfg(test)] mod tests { use super::*; + use std::cell::Cell; use std::collections::HashMap; use asap_types::pre_asap::{Column, DataType, QueryExpr, Reduction, Schema, Source}; @@ -244,9 +308,20 @@ mod tests { struct TestProvider { summary_available: bool, candidate_scan_bytes: u64, + snapshot_calls: Cell, + raw_evidence_calls: Cell, } impl TestProvider { + fn new(summary_available: bool, candidate_scan_bytes: u64) -> Self { + Self { + summary_available, + candidate_scan_bytes, + snapshot_calls: Cell::new(0), + raw_evidence_calls: Cell::new(0), + } + } + fn summary_dag(&self, scope: &ComparisonScope) -> PhysicalDag { let scan_statistics = statistics( self.candidate_scan_bytes, @@ -321,17 +396,25 @@ mod tests { } impl PlannerPhysicalPlanProvider for TestProvider { - fn comparison_scope( + fn capture_evidence_snapshot( &self, _target: &TargetSubDAG<'_>, - ) -> Result { - Ok(scope()) + ) -> Result { + self.snapshot_calls.set(self.snapshot_calls.get() + 1); + Ok(PlannerEvidenceSnapshot { + version: "test-snapshot-1".into(), + scope: scope(), + }) } fn query_node_evidence( &self, + snapshot: &PlannerEvidenceSnapshot, request: PhysicalNodeRequest<'_>, ) -> Result { + assert_eq!(snapshot.version, "test-snapshot-1"); + self.raw_evidence_calls + .set(self.raw_evidence_calls.get() + 1); let (physical_id, mut statistics) = match request.operator { PhysicalOperator::Scan => ( "raw-scan", @@ -357,12 +440,13 @@ mod tests { fn summary_physical_dag( &self, + snapshot: &PlannerEvidenceSnapshot, _summary: &Rc, _target: &TargetSubDAG<'_>, - scope: &ComparisonScope, ) -> Result { + assert_eq!(snapshot.version, "test-snapshot-1"); self.summary_available - .then(|| self.summary_dag(scope)) + .then(|| self.summary_dag(&snapshot.scope)) .ok_or(AnalyticalCostError::MissingOrStale("summary_physical_plan")) } } @@ -384,10 +468,7 @@ mod tests { &crate::replacement::default_strategies(), ); let planned_root = Rc::clone(&space.roots[0].1); - let provider = TestProvider { - summary_available: true, - candidate_scan_bytes: 800, - }; + let provider = TestProvider::new(true, 800); let model = AnalyticalPlannerCostModel::new(&provider, calibration()).unwrap(); let selected = space.global_selection(&model); @@ -405,10 +486,7 @@ mod tests { &crate::replacement::default_strategies(), ); let planned_root = Rc::clone(&space.roots[0].1); - let provider = TestProvider { - summary_available: false, - candidate_scan_bytes: 800, - }; + let provider = TestProvider::new(false, 800); let model = AnalyticalPlannerCostModel::new(&provider, calibration()).unwrap(); let selected = space.global_selection(&model); @@ -422,27 +500,28 @@ mod tests { fn candidate_with_incomplete_scope_is_unavailable() { struct WrongScope(TestProvider); impl PlannerPhysicalPlanProvider for WrongScope { - fn comparison_scope( + fn capture_evidence_snapshot( &self, target: &TargetSubDAG<'_>, - ) -> Result { - self.0.comparison_scope(target) + ) -> Result { + self.0.capture_evidence_snapshot(target) } fn query_node_evidence( &self, + snapshot: &PlannerEvidenceSnapshot, request: PhysicalNodeRequest<'_>, ) -> Result { - self.0.query_node_evidence(request) + self.0.query_node_evidence(snapshot, request) } fn summary_physical_dag( &self, + snapshot: &PlannerEvidenceSnapshot, summary: &Rc, target: &TargetSubDAG<'_>, - scope: &ComparisonScope, ) -> Result { - let mut dag = self.0.summary_physical_dag(summary, target, scope)?; + let mut dag = self.0.summary_physical_dag(snapshot, summary, target)?; dag.nodes[0].source_coverage.as_mut().unwrap().snapshot_id = "other".into(); Ok(dag) } @@ -451,10 +530,7 @@ mod tests { let root = query(); let candidates = crate::replacement::SketchAlgorithmStrategy::default_cost_model() .replacements(&TargetSubDAG::new(&root)); - let provider = WrongScope(TestProvider { - summary_available: true, - candidate_scan_bytes: 800, - }); + let provider = WrongScope(TestProvider::new(true, 800)); let model = AnalyticalPlannerCostModel::new(&provider, calibration()).unwrap(); assert_eq!( model.candidate_cost(&candidates[0], &TargetSubDAG::new(&root)), @@ -470,13 +546,31 @@ mod tests { &crate::replacement::default_strategies(), ); let planned_root = Rc::clone(&space.roots[0].1); - let provider = TestProvider { - summary_available: true, - candidate_scan_bytes: 100_000, - }; + let provider = TestProvider::new(true, 100_000); let model = AnalyticalPlannerCostModel::new(&provider, calibration()).unwrap(); let selected = space.global_selection(&model); assert!(selected.for_target(&planned_root).unwrap().chosen.is_none()); } + + #[test] + fn sibling_candidates_share_one_scope_and_raw_baseline() { + let root = query(); + let candidates = crate::replacement::SketchAlgorithmStrategy::default_cost_model() + .replacements(&TargetSubDAG::new(&root)); + assert!(candidates.len() >= 2); + let provider = TestProvider::new(true, 800); + let model = AnalyticalPlannerCostModel::new(&provider, calibration()).unwrap(); + let target = TargetSubDAG::new(&root); + + model.estimate_candidate(&candidates[0], &target).unwrap(); + model.estimate_candidate(&candidates[1], &target).unwrap(); + + assert_eq!(provider.snapshot_calls.get(), 1); + assert_eq!( + provider.raw_evidence_calls.get(), + 2, + "the scan and aggregate evidence for the raw baseline are captured once" + ); + } } From fa499a195dbb47d8064f5b8565865d80fa8f7129 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 17:53:50 -0600 Subject: [PATCH 43/48] fix(cost): use typed evidence in planner ranking tests --- .../src/analytical_planner.rs | 90 ++++++++++--------- 1 file changed, 49 insertions(+), 41 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_planner.rs b/crates/asap-aware-mapping/src/analytical_planner.rs index 47d9b43c..4a6701e8 100644 --- a/crates/asap-aware-mapping/src/analytical_planner.rs +++ b/crates/asap-aware-mapping/src/analytical_planner.rs @@ -236,33 +236,45 @@ mod tests { }; use crate::analytical_cost::{ExecutionMultiplicity, PhysicalDagNode, PhysicalOperator}; - use crate::analytical_statistics::{EdgeStatistics, OperatorStatistics, SourceCoverage}; + use crate::analytical_statistics::{ + EdgeStatistics, OperatorStatistics, SourceCoverage, UnaryEdgeStatistics, + }; use crate::replacement::ReplacementStrategy; fn edge(rows: u64, bytes: u64) -> EdgeStatistics { EdgeStatistics { rows, bytes } } - fn statistics( - source_scan_bytes: u64, - inputs: Vec, - output: EdgeStatistics, - ) -> OperatorStatistics { - OperatorStatistics { - source_scan_bytes, - inputs, + fn unary_edges(input: EdgeStatistics, output: EdgeStatistics) -> UnaryEdgeStatistics { + UnaryEdgeStatistics { + input, output, - group_count: None, - key_bytes: None, - aggregate_value_bytes: None, - k: None, - topk_output_offset: None, - limit_rows_consumed: None, - hash_join_build_side: None, promql: None, } } + fn scan_statistics(source_read_bytes: u64, edge: EdgeStatistics) -> OperatorStatistics { + OperatorStatistics::Scan { + edges: unary_edges(edge, edge), + source_read_bytes, + } + } + + fn aggregate_statistics(input: EdgeStatistics, output: EdgeStatistics) -> OperatorStatistics { + OperatorStatistics::HashAggregate { + edges: unary_edges(input, output), + group_count: 1, + key_bytes: 0, + accumulator_bytes_per_group: 8, + } + } + + fn pass_through_statistics(edge: EdgeStatistics) -> OperatorStatistics { + OperatorStatistics::PassThrough { + edges: unary_edges(edge, edge), + } + } + fn query() -> Rc { Rc::new(QueryExpr::Aggregate { reduction: Reduction::by(vec![]), @@ -299,8 +311,9 @@ mod tests { source: Source::Table { table_ref: "events".into(), }, - snapshot_id: "snapshot-1".into(), + source_snapshot_id: "snapshot-1".into(), predicates: vec![], + info_matchers: vec![], }], } } @@ -323,16 +336,9 @@ mod tests { } fn summary_dag(&self, scope: &ComparisonScope) -> PhysicalDag { - let scan_statistics = statistics( - self.candidate_scan_bytes, - vec![edge(100, 800)], - edge(100, 800), - ); - let mut aggregate_statistics = statistics(0, vec![edge(100, 800)], edge(1, 8)); - aggregate_statistics.group_count = Some(1); - aggregate_statistics.key_bytes = Some(8); - aggregate_statistics.aggregate_value_bytes = Some(8); - let read_statistics = statistics(0, vec![edge(1, 8)], edge(1, 8)); + let scan_statistics = scan_statistics(self.candidate_scan_bytes, edge(100, 800)); + let aggregate_statistics = aggregate_statistics(edge(100, 800), edge(1, 8)); + let read_statistics = pass_through_statistics(edge(1, 8)); let evidence = HashMap::from([ ( "candidate-scan".into(), @@ -372,7 +378,10 @@ mod tests { }, PhysicalDagNode { id: "candidate-state".into(), - operator: PhysicalOperator::HashAggregate, + operator: PhysicalOperator::HashAggregate { + grouping_key_count: 0, + accumulator_count: 1, + }, children: vec!["candidate-scan".into()], source_coverage: None, output_buffer_bytes: 8, @@ -415,22 +424,17 @@ mod tests { assert_eq!(snapshot.version, "test-snapshot-1"); self.raw_evidence_calls .set(self.raw_evidence_calls.get() + 1); - let (physical_id, mut statistics) = match request.operator { - PhysicalOperator::Scan => ( - "raw-scan", - statistics(800, vec![edge(100, 800)], edge(100, 800)), - ), - PhysicalOperator::HashAggregate => ( + let (physical_id, statistics) = match request.operator { + PhysicalOperator::Scan => ("raw-scan", scan_statistics(800, edge(100, 800))), + PhysicalOperator::HashAggregate { + grouping_key_count: 0, + accumulator_count: 1, + } => ( "raw-aggregate", - statistics(0, vec![edge(100, 800)], edge(1, 8)), + aggregate_statistics(edge(100, 800), edge(1, 8)), ), _ => return Err(AnalyticalCostError::UnsupportedQueryOperator), }; - if request.operator == PhysicalOperator::HashAggregate { - statistics.group_count = Some(1); - statistics.key_bytes = Some(8); - statistics.aggregate_value_bytes = Some(8); - } Ok(PhysicalNodeEvidence { physical_id: physical_id.into(), output_buffer_bytes: 8, @@ -522,7 +526,11 @@ mod tests { target: &TargetSubDAG<'_>, ) -> Result { let mut dag = self.0.summary_physical_dag(snapshot, summary, target)?; - dag.nodes[0].source_coverage.as_mut().unwrap().snapshot_id = "other".into(); + dag.nodes[0] + .source_coverage + .as_mut() + .unwrap() + .source_snapshot_id = "other".into(); Ok(dag) } } From 15dbef7a67330dc49f3263f81e703431f6e3f854 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 17:57:06 -0600 Subject: [PATCH 44/48] fix(cost): reject unbound CSE physical costs --- crates/asap-aware-mapping/src/replacement.rs | 59 ++++++++----------- .../analytical-resource-cost.md | 15 ++--- 2 files changed, 34 insertions(+), 40 deletions(-) diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index e3a4deec..7ac1dcc7 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -3085,16 +3085,34 @@ impl PlanSpace { lifecycle_choice } else if cost_model.candidate_cost_covers_complete_plan() { let effective_target = TargetSubDAG::with_consumer_count(&group.target, effective); - group + let bound_physical = group .candidates .iter() + // A logical CSE rewrite does not encode shared retained + // state or independent execution multiplicity. Until it + // is bound as a complete physical DAG, it must not enter + // evidence-backed ranking as though those costs were + // known. + .filter(|candidate| !is_cse_candidate(candidate)) .filter_map(|candidate| { cost_model .candidate_cost(candidate, &effective_target) .map(|cost| (candidate, cost)) }) .min_by(|(_, left), (_, right)| left.0.total_cmp(&right.0)) - .map(|(candidate, _)| candidate) + .map(|(candidate, _)| candidate); + bound_physical.or_else(|| { + (effective >= 2) + .then(|| { + decide_with_effective_count(group, effective, cost_model).and_then( + |decision| { + chosen_share.insert(*ptr, decision); + pick_shared_subtree_candidate(group, decision) + }, + ) + }) + .flatten() + }) } else if effective >= 2 && cse_candidate_pair(group).is_some() { let decision = if let Some(profiles) = profiles { decide_group_with_recurrence( @@ -3187,22 +3205,6 @@ impl PlanSpace { }) }; - if cost_model.candidate_cost_covers_complete_plan() { - if let Some(candidate) = chosen.filter(|candidate| is_cse_candidate(candidate)) { - let Replacement::Rewrite(rewrite) = &candidate.replacement else { - unreachable!("CSE candidates are logical rewrites") - }; - chosen_share.insert( - *ptr, - if Rc::ptr_eq(rewrite, &group.target) { - ShareDecision::Share - } else { - ShareDecision::RecomputeIndependently - }, - ); - } - } - let outgoing_multiplier = multiplier(*ptr, &effective_uses, &chosen_share); match chosen { Some(ReplacementSubDAG { @@ -5852,7 +5854,7 @@ mod tests { } #[test] - fn complete_plan_costs_rank_both_cse_arms_without_structural_preselection() { + fn complete_plan_costs_reject_unbound_cse_arms() { struct CompletePlanCost; impl CostModel for CompletePlanCost { fn candidate_cost_covers_complete_plan(&self) -> bool { @@ -5862,16 +5864,10 @@ mod tests { fn candidate_cost( &self, candidate: &ReplacementSubDAG, - target: &TargetSubDAG<'_>, + _target: &TargetSubDAG<'_>, ) -> Option { - let Replacement::Rewrite(rewrite) = &candidate.replacement else { - return None; - }; - Some(if Rc::ptr_eq(rewrite, target.root) { - Cost(1.0) - } else { - Cost(10.0) - }) + assert!(!is_cse_candidate(candidate)); + None } fn rank_candidates( @@ -5883,7 +5879,7 @@ mod tests { } fn cse_share_decision(&self, _candidate: &CseCandidate) -> ShareDecision { - panic!("complete physical costs must bypass structural CSE preselection") + ShareDecision::RecomputeIndependently } } @@ -5899,10 +5895,7 @@ mod tests { let selected = space.global_selection(&CompletePlanCost); let chosen = selected.for_target(planned).unwrap().chosen.unwrap(); - assert!(matches!( - &chosen.replacement, - Replacement::Rewrite(rewrite) if Rc::ptr_eq(rewrite, planned) - )); + assert_eq!(chosen.provenance, ReplacementProvenance::CseRecompute); } #[test] diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index c6609868..91dd9e62 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -613,13 +613,14 @@ stale evidence, an unknown physical algorithm, invalid edges, incomplete source coverage, or a candidate that is not cheaper yields `None`. When no candidate remains, `chosen = None` preserves the raw pre-ASAP target. -Complete-plan costing also changes CSE selection order. Global selection sends -all share and recompute alternatives through `candidate_cost`; it does not use -the legacy structural CSE hook to discard one arm first. Once an arm is chosen, -the existing consumer-count propagation records whether that physical -alternative shares or recomputes. Within each returned physical DAG, stable -provider-owned physical IDs deduplicate shared scans, builds, and retained -states. Logical `Rc` identity is never substituted for physical identity. +Logical CSE share/recompute rewrites are not complete physical alternatives: +they do not encode retained shared state or independent execution +multiplicity. The analytical adapter therefore does not assign them an +optimistic complete-plan cost. They continue through the existing CSE policy +until a physical binder can return explicit DAGs for both arms. Within every +bound physical DAG, stable provider-owned physical IDs deduplicate shared +scans, builds, and retained states; logical `Rc` identity is never substituted +for physical identity. DDSketch is unavailable because occupied bins depend on value range and distribution. The model does not invent a bin count. Algorithm/parameter From 3ec380311a32497f4f66788512d4ab8009275915 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 20:25:37 -0600 Subject: [PATCH 45/48] refactor(cost): name physical plan cost boundary --- crates/asap-aware-mapping/src/lib.rs | 2 +- ...planner.rs => physical_plan_cost_model.rs} | 50 +++++++++---------- .../analytical-resource-cost.md | 4 +- 3 files changed, 28 insertions(+), 28 deletions(-) rename crates/asap-aware-mapping/src/{analytical_planner.rs => physical_plan_cost_model.rs} (93%) diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index e7098b61..59e97e68 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -185,11 +185,11 @@ pub mod accuracy; pub mod accuracy_reconciliation; pub mod analytical_cost; pub mod analytical_lowering; -pub mod analytical_planner; pub mod analytical_statistics; pub mod cost_model; pub mod explanation; pub mod grouping; +pub mod physical_plan_cost_model; pub mod recurrence; pub mod replacement; pub mod rewrite; diff --git a/crates/asap-aware-mapping/src/analytical_planner.rs b/crates/asap-aware-mapping/src/physical_plan_cost_model.rs similarity index 93% rename from crates/asap-aware-mapping/src/analytical_planner.rs rename to crates/asap-aware-mapping/src/physical_plan_cost_model.rs index 4a6701e8..b0f2571a 100644 --- a/crates/asap-aware-mapping/src/analytical_planner.rs +++ b/crates/asap-aware-mapping/src/physical_plan_cost_model.rs @@ -24,7 +24,7 @@ use crate::replacement::{Replacement, ReplacementSubDAG, TargetSubDAG}; /// this value for the target, so a caller that needs fresher evidence creates a /// new model instead of mixing generations in one ranking decision. #[derive(Debug, Clone, PartialEq)] -pub struct PlannerEvidenceSnapshot { +pub struct PhysicalEvidenceSnapshot { pub version: String, pub scope: ComparisonScope, } @@ -42,17 +42,17 @@ pub trait PlannerPhysicalPlanProvider { fn capture_evidence_snapshot( &self, target: &TargetSubDAG<'_>, - ) -> Result; + ) -> Result; fn query_node_evidence( &self, - snapshot: &PlannerEvidenceSnapshot, + snapshot: &PhysicalEvidenceSnapshot, request: PhysicalNodeRequest<'_>, ) -> Result; fn summary_physical_dag( &self, - snapshot: &PlannerEvidenceSnapshot, + snapshot: &PhysicalEvidenceSnapshot, summary: &Rc, target: &TargetSubDAG<'_>, ) -> Result; @@ -60,7 +60,7 @@ pub trait PlannerPhysicalPlanProvider { /// Dimensional comparison retained for explanations and verification. #[derive(Debug, Clone, Copy, PartialEq)] -pub struct PlannerCandidateEstimate { +pub struct PhysicalPlanComparison { pub resources: PhysicalDagComparisonEstimate, pub raw_cost: Cost, pub candidate_cost: Cost, @@ -72,7 +72,7 @@ pub struct PlannerCandidateEstimate { /// to lower either alternative, missing statistics, an unknown physical /// algorithm, or invalid source/horizon evidence makes the candidate /// unavailable and leaves the target on its raw path. -pub struct AnalyticalPlannerCostModel<'a> { +pub struct PhysicalPlanCostModel<'a> { provider: &'a dyn PlannerPhysicalPlanProvider, calibration: ResourceCalibration, target_evidence: RefCell>, @@ -81,11 +81,11 @@ pub struct AnalyticalPlannerCostModel<'a> { struct CachedTargetEvidence { root: Rc, consumer_count: usize, - snapshot: PlannerEvidenceSnapshot, + snapshot: PhysicalEvidenceSnapshot, raw: PhysicalDag, } -impl<'a> AnalyticalPlannerCostModel<'a> { +impl<'a> PhysicalPlanCostModel<'a> { pub fn new( provider: &'a dyn PlannerPhysicalPlanProvider, calibration: ResourceCalibration, @@ -101,7 +101,7 @@ impl<'a> AnalyticalPlannerCostModel<'a> { fn target_evidence( &self, target: &TargetSubDAG<'_>, - ) -> Result<(PlannerEvidenceSnapshot, PhysicalDag), AnalyticalCostError> { + ) -> Result<(PhysicalEvidenceSnapshot, PhysicalDag), AnalyticalCostError> { if let Some(cached) = self.target_evidence.borrow().iter().find(|cached| { Rc::ptr_eq(&cached.root, target.root) && cached.consumer_count == target.consumer_count }) { @@ -135,7 +135,7 @@ impl<'a> AnalyticalPlannerCostModel<'a> { &self, candidate: &ReplacementSubDAG, target: &TargetSubDAG<'_>, - ) -> Result { + ) -> Result { let (snapshot, raw) = self.target_evidence(target)?; let scope = &snapshot.scope; let evidence = QueryEvidence { @@ -169,7 +169,7 @@ impl<'a> AnalyticalPlannerCostModel<'a> { )?; let raw_cost = Cost(resources.raw.calibrated_cost(&self.calibration)?); let candidate_cost = Cost(resources.candidate.calibrated_cost(&self.calibration)?); - Ok(PlannerCandidateEstimate { + Ok(PhysicalPlanComparison { resources, raw_cost, candidate_cost, @@ -179,7 +179,7 @@ impl<'a> AnalyticalPlannerCostModel<'a> { struct QueryEvidence<'a> { provider: &'a dyn PlannerPhysicalPlanProvider, - snapshot: &'a PlannerEvidenceSnapshot, + snapshot: &'a PhysicalEvidenceSnapshot, } impl PhysicalNodeEvidenceProvider for QueryEvidence<'_> { @@ -191,7 +191,7 @@ impl PhysicalNodeEvidenceProvider for QueryEvidence<'_> { } } -impl CostModel for AnalyticalPlannerCostModel<'_> { +impl CostModel for PhysicalPlanCostModel<'_> { fn candidate_cost_covers_complete_plan(&self) -> bool { true } @@ -408,9 +408,9 @@ mod tests { fn capture_evidence_snapshot( &self, _target: &TargetSubDAG<'_>, - ) -> Result { + ) -> Result { self.snapshot_calls.set(self.snapshot_calls.get() + 1); - Ok(PlannerEvidenceSnapshot { + Ok(PhysicalEvidenceSnapshot { version: "test-snapshot-1".into(), scope: scope(), }) @@ -418,7 +418,7 @@ mod tests { fn query_node_evidence( &self, - snapshot: &PlannerEvidenceSnapshot, + snapshot: &PhysicalEvidenceSnapshot, request: PhysicalNodeRequest<'_>, ) -> Result { assert_eq!(snapshot.version, "test-snapshot-1"); @@ -444,7 +444,7 @@ mod tests { fn summary_physical_dag( &self, - snapshot: &PlannerEvidenceSnapshot, + snapshot: &PhysicalEvidenceSnapshot, _summary: &Rc, _target: &TargetSubDAG<'_>, ) -> Result { @@ -473,7 +473,7 @@ mod tests { ); let planned_root = Rc::clone(&space.roots[0].1); let provider = TestProvider::new(true, 800); - let model = AnalyticalPlannerCostModel::new(&provider, calibration()).unwrap(); + let model = PhysicalPlanCostModel::new(&provider, calibration()).unwrap(); let selected = space.global_selection(&model); assert!( @@ -491,7 +491,7 @@ mod tests { ); let planned_root = Rc::clone(&space.roots[0].1); let provider = TestProvider::new(false, 800); - let model = AnalyticalPlannerCostModel::new(&provider, calibration()).unwrap(); + let model = PhysicalPlanCostModel::new(&provider, calibration()).unwrap(); let selected = space.global_selection(&model); assert!( @@ -507,13 +507,13 @@ mod tests { fn capture_evidence_snapshot( &self, target: &TargetSubDAG<'_>, - ) -> Result { + ) -> Result { self.0.capture_evidence_snapshot(target) } fn query_node_evidence( &self, - snapshot: &PlannerEvidenceSnapshot, + snapshot: &PhysicalEvidenceSnapshot, request: PhysicalNodeRequest<'_>, ) -> Result { self.0.query_node_evidence(snapshot, request) @@ -521,7 +521,7 @@ mod tests { fn summary_physical_dag( &self, - snapshot: &PlannerEvidenceSnapshot, + snapshot: &PhysicalEvidenceSnapshot, summary: &Rc, target: &TargetSubDAG<'_>, ) -> Result { @@ -539,7 +539,7 @@ mod tests { let candidates = crate::replacement::SketchAlgorithmStrategy::default_cost_model() .replacements(&TargetSubDAG::new(&root)); let provider = WrongScope(TestProvider::new(true, 800)); - let model = AnalyticalPlannerCostModel::new(&provider, calibration()).unwrap(); + let model = PhysicalPlanCostModel::new(&provider, calibration()).unwrap(); assert_eq!( model.candidate_cost(&candidates[0], &TargetSubDAG::new(&root)), None @@ -555,7 +555,7 @@ mod tests { ); let planned_root = Rc::clone(&space.roots[0].1); let provider = TestProvider::new(true, 100_000); - let model = AnalyticalPlannerCostModel::new(&provider, calibration()).unwrap(); + let model = PhysicalPlanCostModel::new(&provider, calibration()).unwrap(); let selected = space.global_selection(&model); assert!(selected.for_target(&planned_root).unwrap().chosen.is_none()); @@ -568,7 +568,7 @@ mod tests { .replacements(&TargetSubDAG::new(&root)); assert!(candidates.len() >= 2); let provider = TestProvider::new(true, 800); - let model = AnalyticalPlannerCostModel::new(&provider, calibration()).unwrap(); + let model = PhysicalPlanCostModel::new(&provider, calibration()).unwrap(); let target = TargetSubDAG::new(&root); model.estimate_candidate(&candidates[0], &target).unwrap(); diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 91dd9e62..2a3f6693 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -596,7 +596,7 @@ windows, nested aggregates, Top-K, and shared sub-DAGs therefore use the same estimation path. The generic query lowerer recursively maps the supported raw query operators into that representation. -`AnalyticalPlannerCostModel` is the final-selection adapter. For every +`PhysicalPlanCostModel` is the final-selection adapter. For every candidate it obtains one canonical `ComparisonScope`, recursively lowers the actual raw target, and then lowers a logical rewrite or requests the fully bound physical DAG for a `SummaryExpr` candidate. The deployment implements @@ -672,7 +672,7 @@ The intended end-to-end selection pipeline is: 6. applies calibration and ranks candidates by ascending cost. The query lowerer and physical estimator cover the supported raw-query shapes -listed above. `AnalyticalPlannerCostModel` executes this pipeline for every +listed above. `PhysicalPlanCostModel` executes this pipeline for every candidate supplied to `PlanSpace::global_selection`. Logical rewrites are lowered recursively. Summary candidates participate only after the deployment has bound their complete `SummaryExpr` DAG; there is no optimistic generic From 6bafaf219ca5d1af5e83ce4653c1a45095c0bb8a Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 20:47:31 -0600 Subject: [PATCH 46/48] refactor(cost): separate physical resource core --- .../src/analytical_lowering.rs | 79 +++---------------- .../src/analytical_statistics.rs | 4 +- crates/asap-aware-mapping/src/lib.rs | 2 +- .../src/physical_plan_cost_model.rs | 14 ++-- ...ical_cost.rs => physical_resource_cost.rs} | 61 +++++++++++++- 5 files changed, 82 insertions(+), 78 deletions(-) rename crates/asap-aware-mapping/src/{analytical_cost.rs => physical_resource_cost.rs} (97%) diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/analytical_lowering.rs index 23576776..9cdc5f5c 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/analytical_lowering.rs @@ -2,36 +2,18 @@ use std::rc::Rc; -use serde::{Deserialize, Serialize}; - -use crate::analytical_cost::{ - validate_operator_semantics, AnalyticalCostError, ExecutionMultiplicity, HashJoinBuildSide, - PhysicalDagNode, PhysicalOperator, PromqlBinaryOperandMode, PromqlBinaryOperation, - PromqlPresenceKind, PromqlSeriesSampleKind, PromqlVectorCardinality, -}; use crate::analytical_statistics::{ - ComparisonScope, EdgeStatistics, OperatorStatistics, OperatorStatisticsProvider, SourceCoverage, + ComparisonScope, EdgeStatistics, OperatorStatistics, SourceCoverage, +}; +use crate::physical_resource_cost::{ + validate_operator_semantics, AnalyticalCostError, EvidenceBackedPhysicalDag, + ExecutionMultiplicity, HashJoinBuildSide, PhysicalDagNode, PhysicalNodeEvidence, + PhysicalOperator, PromqlBinaryOperandMode, PromqlBinaryOperation, PromqlPresenceKind, + PromqlSeriesSampleKind, PromqlVectorCardinality, }; -/// A lowered physical DAG and the node whose output is the query result. -/// Keeping the root beside its nodes prevents callers from accidentally -/// estimating a valid node list from the wrong entry point. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct PhysicalDag { - pub nodes: Vec, - pub root: String, - pub evidence: std::collections::HashMap, -} - -/// Atomic evidence for one lowered physical node. The statistics contract is -/// reused unchanged; the separate buffer field is necessary because logical -/// edge bytes cannot stand in for an allocation. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct PhysicalNodeEvidence { - pub physical_id: String, - pub statistics: OperatorStatistics, - pub output_buffer_bytes: u64, -} +/// The canonical evidenced DAG returned by query lowering. +pub type PhysicalDag = EvidenceBackedPhysicalDag; pub struct PhysicalNodeRequest<'a> { pub logical_node: &'a asap_types::pre_asap::QueryExpr, @@ -61,43 +43,6 @@ where } } -impl OperatorStatisticsProvider for std::collections::HashMap { - fn statistics(&self, node_id: &str) -> Result { - let evidence = self - .get(node_id) - .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(node_id.into()))?; - if evidence.physical_id != node_id { - return Err(AnalyticalCostError::InvalidPhysicalDag( - "evidence map key differs from embedded physical identity", - )); - } - Ok(evidence.statistics.clone()) - } -} - -impl OperatorStatisticsProvider for PhysicalDag { - fn statistics(&self, node_id: &str) -> Result { - let evidence = self - .evidence - .get(node_id) - .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(node_id.into()))?; - let node = self.nodes.iter().find(|node| node.id == node_id).ok_or( - AnalyticalCostError::InvalidPhysicalDag("evidence has no matching physical node"), - )?; - if evidence.physical_id != node_id { - return Err(AnalyticalCostError::InvalidPhysicalDag( - "evidence map key differs from embedded physical identity", - )); - } - if node.output_buffer_bytes != evidence.output_buffer_bytes { - return Err(AnalyticalCostError::InvalidPhysicalDag( - "physical node buffer differs from evidence snapshot", - )); - } - Ok(evidence.statistics.clone()) - } -} - /// Lower a resolved query operator DAG to the physical operators understood by /// this cost model. The authoritative provider supplies statistics by the /// stable physical IDs owned by that provider; missing evidence makes the @@ -1313,13 +1258,13 @@ fn is_promql_scalar(query: &asap_types::pre_asap::QueryExpr) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::analytical_cost::{ - estimate_physical_dag, estimate_physical_dag_comparison, PhysicalDagEstimateRequest, - }; use crate::analytical_statistics::{ validate_comparison_scopes, BinaryEdgeStatistics, PartitionStatistics, PromqlEdgeStatistics, PromqlUnaryEdgeStatistics, PromqlValueKind, UnaryEdgeStatistics, }; + use crate::physical_resource_cost::{ + estimate_physical_dag, estimate_physical_dag_comparison, PhysicalDagEstimateRequest, + }; use asap_types::workload::{ DataArrival, DurationMs, QueryRecurrence, QueryTimeScope, TimeSelection, TimestampMs, }; diff --git a/crates/asap-aware-mapping/src/analytical_statistics.rs b/crates/asap-aware-mapping/src/analytical_statistics.rs index ecd3a9ef..73b2fe52 100644 --- a/crates/asap-aware-mapping/src/analytical_statistics.rs +++ b/crates/asap-aware-mapping/src/analytical_statistics.rs @@ -13,7 +13,7 @@ use asap_types::workload::{ }; use serde::{Deserialize, Serialize}; -use crate::analytical_cost::AnalyticalCostError; +use crate::physical_resource_cost::AnalyticalCostError; /// The semantic and workload boundary within which two resource estimates /// may be compared. Canonical workload and query-IR types remain authoritative; @@ -270,7 +270,7 @@ pub struct PartitionStatistics { } /// Workload-dependent evidence for one operator in an already-lowered -/// physical DAG. [`PhysicalOperator`](crate::analytical_cost::PhysicalOperator) +/// physical DAG. [`PhysicalOperator`](crate::physical_resource_cost::PhysicalOperator) /// is the authoritative operator vocabulary: every one of its variants has a /// matching statistics variant here. /// diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 59e97e68..58dd1ee2 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -183,13 +183,13 @@ pub mod accuracy; pub mod accuracy_reconciliation; -pub mod analytical_cost; pub mod analytical_lowering; pub mod analytical_statistics; pub mod cost_model; pub mod explanation; pub mod grouping; pub mod physical_plan_cost_model; +pub mod physical_resource_cost; pub mod recurrence; pub mod replacement; pub mod rewrite; diff --git a/crates/asap-aware-mapping/src/physical_plan_cost_model.rs b/crates/asap-aware-mapping/src/physical_plan_cost_model.rs index b0f2571a..a72b2692 100644 --- a/crates/asap-aware-mapping/src/physical_plan_cost_model.rs +++ b/crates/asap-aware-mapping/src/physical_plan_cost_model.rs @@ -5,16 +5,16 @@ use std::{cell::RefCell, rc::Rc}; use asap_types::post_asap::{SketchAlgorithm, SummaryExpr, SummaryNode}; use asap_types::pre_asap::{AggIntent, QueryExpr}; -use crate::analytical_cost::{ - estimate_physical_dag_comparison, AnalyticalCostError, PhysicalDagComparisonEstimate, - PhysicalDagEstimateRequest, ResourceCalibration, -}; use crate::analytical_lowering::{ - lower_query_physical_dag, PhysicalDag, PhysicalNodeEvidence, PhysicalNodeEvidenceProvider, - PhysicalNodeRequest, + lower_query_physical_dag, PhysicalNodeEvidenceProvider, PhysicalNodeRequest, }; use crate::analytical_statistics::ComparisonScope; use crate::cost_model::{Cost, CostModel, DefaultCostModel}; +use crate::physical_resource_cost::{ + estimate_physical_dag_comparison, AnalyticalCostError, + EvidenceBackedPhysicalDag as PhysicalDag, PhysicalDagComparisonEstimate, + PhysicalDagEstimateRequest, PhysicalNodeEvidence, ResourceCalibration, +}; use crate::replacement::{Replacement, ReplacementSubDAG, TargetSubDAG}; /// One immutable generation of deployment evidence for a planner target. @@ -235,10 +235,10 @@ mod tests { DataArrival, DurationMs, QueryRecurrence, QueryTimeScope, TimeSelection, TimestampMs, }; - use crate::analytical_cost::{ExecutionMultiplicity, PhysicalDagNode, PhysicalOperator}; use crate::analytical_statistics::{ EdgeStatistics, OperatorStatistics, SourceCoverage, UnaryEdgeStatistics, }; + use crate::physical_resource_cost::{ExecutionMultiplicity, PhysicalDagNode, PhysicalOperator}; use crate::replacement::ReplacementStrategy; fn edge(rows: u64, bytes: u64) -> EdgeStatistics { diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/physical_resource_cost.rs similarity index 97% rename from crates/asap-aware-mapping/src/analytical_cost.rs rename to crates/asap-aware-mapping/src/physical_resource_cost.rs index e2bc0b6d..a4922c96 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/physical_resource_cost.rs @@ -16,7 +16,7 @@ use crate::analytical_statistics::{ OperatorStatisticsProvider, PromqlEdgeStatistics, PromqlValueKind, SourceCoverage, }; -pub const ANALYTICAL_MODEL_VERSION: &str = "analytical-resource-at-rest-v1"; +pub const PHYSICAL_RESOURCE_MODEL_VERSION: &str = "physical-resource-v1"; /// Conversion from physical dimensions to one deployment-specific objective. /// Memory's coefficient means cost units per retained byte over this model's @@ -203,6 +203,65 @@ pub struct PhysicalDagNode { pub execution: ExecutionMultiplicity, } +/// Authoritative statistics and allocation evidence for one physical node. +/// The identity is repeated so a map entry cannot be accidentally rebound to +/// a different node while an estimate is in progress. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PhysicalNodeEvidence { + pub physical_id: String, + pub statistics: OperatorStatistics, + pub output_buffer_bytes: u64, +} + +/// A complete physical DAG together with the evidence used to cost it. +/// +/// This representation is shared by query lowering and summary-maintenance +/// lowering. Keeping the root and evidence beside the nodes prevents callers +/// from estimating a valid node list with a different entry point or snapshot. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct EvidenceBackedPhysicalDag { + pub nodes: Vec, + pub root: String, + pub evidence: HashMap, +} + +impl OperatorStatisticsProvider for HashMap { + fn statistics(&self, node_id: &str) -> Result { + let evidence = self + .get(node_id) + .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(node_id.into()))?; + if evidence.physical_id != node_id { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "evidence map key differs from embedded physical identity", + )); + } + Ok(evidence.statistics.clone()) + } +} + +impl OperatorStatisticsProvider for EvidenceBackedPhysicalDag { + fn statistics(&self, node_id: &str) -> Result { + let evidence = self + .evidence + .get(node_id) + .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(node_id.into()))?; + let node = self.nodes.iter().find(|node| node.id == node_id).ok_or( + AnalyticalCostError::InvalidPhysicalDag("evidence has no matching physical node"), + )?; + if evidence.physical_id != node_id { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "evidence map key differs from embedded physical identity", + )); + } + if node.output_buffer_bytes != evidence.output_buffer_bytes { + return Err(AnalyticalCostError::InvalidPhysicalDag( + "physical node buffer differs from evidence snapshot", + )); + } + Ok(evidence.statistics.clone()) + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum ExecutionMultiplicity { Once, From e69b45ac4bb82a5e655dab2ef28c48b7ac97432f Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 21:23:49 -0600 Subject: [PATCH 47/48] refactor(cost): separate physical evidence from analytical estimation --- ...al_resource_cost.rs => analytical_cost.rs} | 32 ++++++++------- crates/asap-aware-mapping/src/lib.rs | 6 +-- ...ics.rs => physical_operator_statistics.rs} | 4 +- .../src/physical_plan_cost_model.rs | 16 ++++---- ...lowering.rs => query_physical_lowering.rs} | 35 ++++++++-------- .../analytical-resource-cost.md | 40 ++++++++++++------- .../physical-plan-integration.md | 2 +- 7 files changed, 74 insertions(+), 61 deletions(-) rename crates/asap-aware-mapping/src/{physical_resource_cost.rs => analytical_cost.rs} (99%) rename crates/asap-aware-mapping/src/{analytical_statistics.rs => physical_operator_statistics.rs} (99%) rename crates/asap-aware-mapping/src/{analytical_lowering.rs => query_physical_lowering.rs} (99%) diff --git a/crates/asap-aware-mapping/src/physical_resource_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs similarity index 99% rename from crates/asap-aware-mapping/src/physical_resource_cost.rs rename to crates/asap-aware-mapping/src/analytical_cost.rs index a4922c96..4b766082 100644 --- a/crates/asap-aware-mapping/src/physical_resource_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -11,12 +11,16 @@ use asap_types::post_asap::{SketchAlgorithm, SketchParams}; use asap_types::workload::DataArrival; use serde::{Deserialize, Serialize}; -use crate::analytical_statistics::{ +use crate::physical_operator_statistics::{ validate_comparison_scopes, ComparisonScope, EdgeStatistics, OperatorStatistics, OperatorStatisticsProvider, PromqlEdgeStatistics, PromqlValueKind, SourceCoverage, }; -pub const PHYSICAL_RESOURCE_MODEL_VERSION: &str = "physical-resource-v1"; +/// Version of the analytical formulas applied to evidenced physical plans. +/// +/// This identifies the estimation method, not a physical executor or runtime +/// implementation version. +pub const ANALYTICAL_COST_MODEL_VERSION: &str = "analytical-cost-v1"; /// Conversion from physical dimensions to one deployment-specific objective. /// Memory's coefficient means cost units per retained byte over this model's @@ -726,7 +730,7 @@ fn checked_cpu_product(rows: u64, operations_per_row: u64) -> Result Result { @@ -751,7 +755,7 @@ fn partitioned_order_estimate( fn validate_partitioning( input: EdgeStatistics, - partitioning: &crate::analytical_statistics::PartitionStatistics, + partitioning: &crate::physical_operator_statistics::PartitionStatistics, partitioned: bool, ) -> Result<(), AnalyticalCostError> { let inconsistent = |reason| Err(AnalyticalCostError::InconsistentOperatorStatistics(reason)); @@ -1654,8 +1658,8 @@ pub(crate) fn validate_operator_semantics( } fn require_promql_unary( - edges: &crate::analytical_statistics::UnaryEdgeStatistics, -) -> Result { + edges: &crate::physical_operator_statistics::UnaryEdgeStatistics, +) -> Result { let promql = edges.promql.ok_or(AnalyticalCostError::MissingOrStale( "promql_edge_statistics", ))?; @@ -1665,8 +1669,8 @@ fn require_promql_unary( } fn require_promql_binary( - edges: &crate::analytical_statistics::BinaryEdgeStatistics, -) -> Result { + edges: &crate::physical_operator_statistics::BinaryEdgeStatistics, +) -> Result { let promql = edges.promql.ok_or(AnalyticalCostError::MissingOrStale( "promql_edge_statistics", ))?; @@ -1677,7 +1681,7 @@ fn require_promql_binary( } fn validate_promql_cardinality_preserving_shape( - promql: crate::analytical_statistics::PromqlUnaryEdgeStatistics, + promql: crate::physical_operator_statistics::PromqlUnaryEdgeStatistics, ) -> Result<(), AnalyticalCostError> { validate_promql_edge(promql.input)?; validate_promql_edge(promql.output)?; @@ -1691,7 +1695,7 @@ fn validate_promql_cardinality_preserving_shape( } fn validate_promql_filter_shape( - promql: crate::analytical_statistics::PromqlUnaryEdgeStatistics, + promql: crate::physical_operator_statistics::PromqlUnaryEdgeStatistics, ) -> Result<(), AnalyticalCostError> { validate_promql_edge(promql.input)?; validate_promql_edge(promql.output)?; @@ -1757,7 +1761,7 @@ fn validate_promql_edge_shape( fn validate_promql_bridge( operator: PhysicalOperator, - edges: &crate::analytical_statistics::UnaryEdgeStatistics, + edges: &crate::physical_operator_statistics::UnaryEdgeStatistics, ) -> Result<(), AnalyticalCostError> { let promql = require_promql_unary(edges)?; let valid = match operator { @@ -1791,7 +1795,7 @@ fn validate_promql_binary( cardinality: PromqlVectorCardinality, build_side: Option, matching_key_bytes: u64, - edges: &crate::analytical_statistics::BinaryEdgeStatistics, + edges: &crate::physical_operator_statistics::BinaryEdgeStatistics, ) -> Result<(), AnalyticalCostError> { let promql = require_promql_binary(edges)?; validate_instant_vector_rows(edges.output, promql.output)?; @@ -1882,7 +1886,7 @@ fn validate_promql_series_sample( grouping_key_count: u64, group_count: u64, key_bytes: u64, - edges: &crate::analytical_statistics::UnaryEdgeStatistics, + edges: &crate::physical_operator_statistics::UnaryEdgeStatistics, ) -> Result<(), AnalyticalCostError> { let promql = require_promql_unary(edges)?; validate_instant_vector_rows(edges.output, promql.output)?; @@ -1998,7 +2002,7 @@ fn checked_bytes(parts: &[u64]) -> Result { #[cfg(test)] mod tests { use super::*; - use crate::analytical_statistics::{ + use crate::physical_operator_statistics::{ validate_comparison_scopes, BinaryEdgeStatistics, ComparisonScope, EdgeStatistics, OperatorStatistics, PartitionStatistics, PromqlBinaryEdgeStatistics, PromqlEdgeStatistics, PromqlUnaryEdgeStatistics, PromqlValueKind, SourceCoverage, UnaryEdgeStatistics, diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 58dd1ee2..3e4fa92b 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -183,13 +183,13 @@ pub mod accuracy; pub mod accuracy_reconciliation; -pub mod analytical_lowering; -pub mod analytical_statistics; +pub mod analytical_cost; pub mod cost_model; pub mod explanation; pub mod grouping; +pub mod physical_operator_statistics; pub mod physical_plan_cost_model; -pub mod physical_resource_cost; +pub mod query_physical_lowering; pub mod recurrence; pub mod replacement; pub mod rewrite; diff --git a/crates/asap-aware-mapping/src/analytical_statistics.rs b/crates/asap-aware-mapping/src/physical_operator_statistics.rs similarity index 99% rename from crates/asap-aware-mapping/src/analytical_statistics.rs rename to crates/asap-aware-mapping/src/physical_operator_statistics.rs index 73b2fe52..ecd3a9ef 100644 --- a/crates/asap-aware-mapping/src/analytical_statistics.rs +++ b/crates/asap-aware-mapping/src/physical_operator_statistics.rs @@ -13,7 +13,7 @@ use asap_types::workload::{ }; use serde::{Deserialize, Serialize}; -use crate::physical_resource_cost::AnalyticalCostError; +use crate::analytical_cost::AnalyticalCostError; /// The semantic and workload boundary within which two resource estimates /// may be compared. Canonical workload and query-IR types remain authoritative; @@ -270,7 +270,7 @@ pub struct PartitionStatistics { } /// Workload-dependent evidence for one operator in an already-lowered -/// physical DAG. [`PhysicalOperator`](crate::physical_resource_cost::PhysicalOperator) +/// physical DAG. [`PhysicalOperator`](crate::analytical_cost::PhysicalOperator) /// is the authoritative operator vocabulary: every one of its variants has a /// matching statistics variant here. /// diff --git a/crates/asap-aware-mapping/src/physical_plan_cost_model.rs b/crates/asap-aware-mapping/src/physical_plan_cost_model.rs index a72b2692..ac5f3858 100644 --- a/crates/asap-aware-mapping/src/physical_plan_cost_model.rs +++ b/crates/asap-aware-mapping/src/physical_plan_cost_model.rs @@ -5,16 +5,16 @@ use std::{cell::RefCell, rc::Rc}; use asap_types::post_asap::{SketchAlgorithm, SummaryExpr, SummaryNode}; use asap_types::pre_asap::{AggIntent, QueryExpr}; -use crate::analytical_lowering::{ - lower_query_physical_dag, PhysicalNodeEvidenceProvider, PhysicalNodeRequest, -}; -use crate::analytical_statistics::ComparisonScope; -use crate::cost_model::{Cost, CostModel, DefaultCostModel}; -use crate::physical_resource_cost::{ +use crate::analytical_cost::{ estimate_physical_dag_comparison, AnalyticalCostError, EvidenceBackedPhysicalDag as PhysicalDag, PhysicalDagComparisonEstimate, PhysicalDagEstimateRequest, PhysicalNodeEvidence, ResourceCalibration, }; +use crate::cost_model::{Cost, CostModel, DefaultCostModel}; +use crate::physical_operator_statistics::ComparisonScope; +use crate::query_physical_lowering::{ + lower_query_physical_dag, PhysicalNodeEvidenceProvider, PhysicalNodeRequest, +}; use crate::replacement::{Replacement, ReplacementSubDAG, TargetSubDAG}; /// One immutable generation of deployment evidence for a planner target. @@ -235,10 +235,10 @@ mod tests { DataArrival, DurationMs, QueryRecurrence, QueryTimeScope, TimeSelection, TimestampMs, }; - use crate::analytical_statistics::{ + use crate::analytical_cost::{ExecutionMultiplicity, PhysicalDagNode, PhysicalOperator}; + use crate::physical_operator_statistics::{ EdgeStatistics, OperatorStatistics, SourceCoverage, UnaryEdgeStatistics, }; - use crate::physical_resource_cost::{ExecutionMultiplicity, PhysicalDagNode, PhysicalOperator}; use crate::replacement::ReplacementStrategy; fn edge(rows: u64, bytes: u64) -> EdgeStatistics { diff --git a/crates/asap-aware-mapping/src/analytical_lowering.rs b/crates/asap-aware-mapping/src/query_physical_lowering.rs similarity index 99% rename from crates/asap-aware-mapping/src/analytical_lowering.rs rename to crates/asap-aware-mapping/src/query_physical_lowering.rs index 9cdc5f5c..be89847a 100644 --- a/crates/asap-aware-mapping/src/analytical_lowering.rs +++ b/crates/asap-aware-mapping/src/query_physical_lowering.rs @@ -1,19 +1,16 @@ -//! Recursive lowering from the canonical query IR to analytical physical DAGs. +//! Recursive lowering from the canonical query IR to evidenced physical DAGs. use std::rc::Rc; -use crate::analytical_statistics::{ - ComparisonScope, EdgeStatistics, OperatorStatistics, SourceCoverage, -}; -use crate::physical_resource_cost::{ +use crate::analytical_cost::{ validate_operator_semantics, AnalyticalCostError, EvidenceBackedPhysicalDag, ExecutionMultiplicity, HashJoinBuildSide, PhysicalDagNode, PhysicalNodeEvidence, PhysicalOperator, PromqlBinaryOperandMode, PromqlBinaryOperation, PromqlPresenceKind, PromqlSeriesSampleKind, PromqlVectorCardinality, }; - -/// The canonical evidenced DAG returned by query lowering. -pub type PhysicalDag = EvidenceBackedPhysicalDag; +use crate::physical_operator_statistics::{ + ComparisonScope, EdgeStatistics, OperatorStatistics, SourceCoverage, +}; pub struct PhysicalNodeRequest<'a> { pub logical_node: &'a asap_types::pre_asap::QueryExpr, @@ -52,7 +49,7 @@ pub fn lower_query_physical_dag( root: &Rc, scope: &ComparisonScope, evidence: &dyn PhysicalNodeEvidenceProvider, -) -> Result { +) -> Result { use std::collections::HashMap; use asap_types::pre_asap::{GroupKeys, QueryExpr, SetOpKind}; @@ -822,7 +819,7 @@ pub fn lower_query_physical_dag( }; let root = lowerer.lower(root)?; validate_source_consumption(&lowerer.nodes, scope)?; - Ok(PhysicalDag { + Ok(EvidenceBackedPhysicalDag { nodes: lowerer.nodes, root, evidence: lowerer.evidence, @@ -1258,13 +1255,13 @@ fn is_promql_scalar(query: &asap_types::pre_asap::QueryExpr) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::analytical_statistics::{ + use crate::analytical_cost::{ + estimate_physical_dag, estimate_physical_dag_comparison, PhysicalDagEstimateRequest, + }; + use crate::physical_operator_statistics::{ validate_comparison_scopes, BinaryEdgeStatistics, PartitionStatistics, PromqlEdgeStatistics, PromqlUnaryEdgeStatistics, PromqlValueKind, UnaryEdgeStatistics, }; - use crate::physical_resource_cost::{ - estimate_physical_dag, estimate_physical_dag_comparison, PhysicalDagEstimateRequest, - }; use asap_types::workload::{ DataArrival, DurationMs, QueryRecurrence, QueryTimeScope, TimeSelection, TimestampMs, }; @@ -2459,10 +2456,12 @@ mod tests { edges: BinaryEdgeStatistics { inputs: [left_edge, right_edge], output: output_edge, - promql: Some(crate::analytical_statistics::PromqlBinaryEdgeStatistics { - inputs: [left_promql, right_promql], - output: output_promql, - }), + promql: Some( + crate::physical_operator_statistics::PromqlBinaryEdgeStatistics { + inputs: [left_promql, right_promql], + output: output_promql, + }, + ), }, matching_key_bytes: 16, }), diff --git a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md index 2a3f6693..ec6756a3 100644 --- a/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/asap-aware-mapping/analytical-resource-cost.md @@ -2,23 +2,33 @@ ## Purpose and boundaries -The analytical resource cost model version implemented here is explicitly for -`DataArrival::AtRest`. It compares legal physical plans using CPU work, peak -memory, and source/disk I/O. It replaces dimensionless plan-node counts with -estimates derived from operator complexity, cardinality, row width, and -concrete summary parameters. +This document defines an analytical estimation method over evidenced physical +plans. These are separate concerns: + +- the **physical input layer** names the physical operators and DAG edges being + estimated and supplies catalog or observed statistics for them; and +- the **analytical estimation layer** applies algorithmic formulas to that + evidence to estimate CPU work, peak memory, and source/disk I/O. + +The model version implemented here is explicitly for `DataArrival::AtRest`. +It replaces dimensionless plan-node counts with estimates derived from +operator complexity, cardinality, row width, and concrete summary parameters. +The estimates are predictions; they are not measurements reported by a +physical executor. The model does not decide semantic or accuracy legality. Candidate generation and guarantee composition run first; costing ranks only the candidates that survive. Missing evidence produces an unavailable estimate, never an assumed zero or a structural-cost fallback. -This document distinguishes four implementation layers: +The implementation keeps five layers distinct: -- the physical-DAG estimator, which can compose any DAG whose nodes have - supported physical operators and complete `OperatorStatistics`; and -- query-DAG lowering, which recursively maps supported resolved `QueryExpr` - operators to that physical representation; and +- physical query lowering (`query_physical_lowering`), which recursively maps + supported resolved `QueryExpr` operators to the physical representation; +- physical evidence (`physical_operator_statistics`), which pairs every + physical operator with the statistics required by its formula; +- analytical estimation (`analytical_cost`), which composes any evidenced DAG + whose operators have supported formulas; - the deployment summary binder, which maps a selected `SummaryExpr` DAG to physical summary operators and snapshots their evidence; and - the planner-ranking adapter, which compares the complete raw and replacement @@ -410,9 +420,9 @@ deduplicated by physical identity. ### Query-DAG lowering and statistics contract `lower_query_physical_dag` recursively lowers a resolved `Rc` and -returns a `PhysicalDag` containing both its nodes and root ID. It consumes the -existing query and physical-operator enums; it does not introduce a parallel -logical operator vocabulary. For every occurrence, the lowerer sends a +returns an `EvidenceBackedPhysicalDag` containing both its nodes and root ID. +It consumes the existing query and physical-operator enums; it does not +introduce a parallel logical operator vocabulary. For every occurrence, the lowerer sends a `PhysicalNodeRequest` containing the logical node, selected existing `PhysicalOperator`, occurrence and synthetic-role metadata, already-lowered child physical IDs, and any source coverage to a @@ -420,7 +430,7 @@ child physical IDs, and any source coverage to a `physical_id`, the authoritative `OperatorStatistics`, and explicit `output_buffer_bytes`; logical edge bytes are never substituted for an allocation. Missing evidence makes the entire query unavailable. The returned -`PhysicalDag` snapshots this evidence so costing does not re-read a live +`EvidenceBackedPhysicalDag` snapshots this evidence so costing does not re-read a live catalog after lowering. Each lowered Scan is bound to exactly one `SourceCoverage` in the comparison @@ -602,7 +612,7 @@ actual raw target, and then lowers a logical rewrite or requests the fully bound physical DAG for a `SummaryExpr` candidate. The deployment implements `PlannerPhysicalPlanProvider`: query-node evidence is consumed atomically by the generic query lowerer, while summary binding returns a complete -`PhysicalDag`, including embedded raw work, build/read operators, retained +`EvidenceBackedPhysicalDag`, including embedded raw work, build/read operators, retained state, execution multiplicity, and source coverage. The adapter calls `estimate_physical_dag_comparison`; it never calls `DefaultCostModel` or a structural-node-count fallback for final cost. diff --git a/docs/design_docs/asap-aware-mapping/physical-plan-integration.md b/docs/design_docs/asap-aware-mapping/physical-plan-integration.md index 735d177a..fbc4a455 100644 --- a/docs/design_docs/asap-aware-mapping/physical-plan-integration.md +++ b/docs/design_docs/asap-aware-mapping/physical-plan-integration.md @@ -3,7 +3,7 @@ ## Purpose This document defines the boundary between ASAPPlanner's logical plans, -physical lowering, statistics resolution, and analytical resource costing. +physical lowering, statistics resolution, and analytical resource estimation. It answers which representation is authoritative at each stage and prevents the cost model from being coupled directly to either logical IR. From e4cb66665ae3989930f1b9632ec3fe7a452ea29b Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 22:00:51 -0600 Subject: [PATCH 48/48] fix(cost): reject blank evidence snapshot versions --- .../src/physical_plan_cost_model.rs | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/crates/asap-aware-mapping/src/physical_plan_cost_model.rs b/crates/asap-aware-mapping/src/physical_plan_cost_model.rs index ac5f3858..0761e326 100644 --- a/crates/asap-aware-mapping/src/physical_plan_cost_model.rs +++ b/crates/asap-aware-mapping/src/physical_plan_cost_model.rs @@ -109,7 +109,7 @@ impl<'a> PhysicalPlanCostModel<'a> { } let snapshot = self.provider.capture_evidence_snapshot(target)?; - if snapshot.version.is_empty() { + if snapshot.version.trim().is_empty() { return Err(AnalyticalCostError::MissingOrStale( "planner_evidence_snapshot.version", )); @@ -546,6 +546,48 @@ mod tests { ); } + #[test] + fn blank_snapshot_version_is_unavailable_before_evidence_lookup() { + struct BlankVersionProvider; + impl PlannerPhysicalPlanProvider for BlankVersionProvider { + fn capture_evidence_snapshot( + &self, + _target: &TargetSubDAG<'_>, + ) -> Result { + Ok(PhysicalEvidenceSnapshot { + version: " \t".into(), + scope: scope(), + }) + } + + fn query_node_evidence( + &self, + _snapshot: &PhysicalEvidenceSnapshot, + _request: PhysicalNodeRequest<'_>, + ) -> Result { + panic!("blank snapshot versions must fail before evidence lookup") + } + + fn summary_physical_dag( + &self, + _snapshot: &PhysicalEvidenceSnapshot, + _summary: &Rc, + _target: &TargetSubDAG<'_>, + ) -> Result { + panic!("blank snapshot versions must fail before summary binding") + } + } + + let root = query(); + let candidates = crate::replacement::SketchAlgorithmStrategy::default_cost_model() + .replacements(&TargetSubDAG::new(&root)); + let model = PhysicalPlanCostModel::new(&BlankVersionProvider, calibration()).unwrap(); + assert_eq!( + model.candidate_cost(&candidates[0], &TargetSubDAG::new(&root)), + None + ); + } + #[test] fn complete_candidate_that_costs_more_than_raw_is_not_selected() { let root = query();