From cd9388664e8858bad4dbb605eb480f2e2b8eb810 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 20:13:59 -0600 Subject: [PATCH 01/28] feat(erp): define catalog-scoped population observation envelope --- crates/asap_types/src/erp_observation.rs | 79 ++++++++++++++++++++++++ crates/asap_types/src/lib.rs | 1 + 2 files changed, 80 insertions(+) create mode 100644 crates/asap_types/src/erp_observation.rs diff --git a/crates/asap_types/src/erp_observation.rs b/crates/asap_types/src/erp_observation.rs new file mode 100644 index 000000000..9bdde9b77 --- /dev/null +++ b/crates/asap_types/src/erp_observation.rs @@ -0,0 +1,79 @@ +//! Versioned runtime evidence about the inputs of one installed summary. +//! Shape is generic so the transport contract does not depend on a planner. +use crate::sds::{CatalogGeneration, SummaryDefinitionId, SummaryInstanceId}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ErpObservationInputSemantics { + ScalarSampleValue, + UnitSampleFrequency, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ErpPopulationObservation { + pub population_id: SummaryInstanceId, + pub shape: Shape, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ErpPopulationObservations { + pub schema_version: u32, + pub catalog_generation: CatalogGeneration, + pub summary_definition_id: SummaryDefinitionId, + pub observed_at_unix_ms: u64, + pub window_start_ms: i64, + pub window_end_ms: i64, + pub input_semantics: ErpObservationInputSemantics, + /// Any overflow or unsupported input invalidates the entire envelope. + /// Invalid envelopes contain no population fits. + pub invalid_reason: Option, + pub populations: Vec>, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ErpObservationFreshness { + pub max_age_ms: u64, + pub max_future_skew_ms: u64, +} + +impl ErpPopulationObservations { + pub fn validate_identity_and_freshness( + &self, + expected_generation: &CatalogGeneration, + expected_definition: SummaryDefinitionId, + now_ms: u64, + freshness: ErpObservationFreshness, + ) -> Result<(), &'static str> { + if self.schema_version != 1 + || &self.catalog_generation != expected_generation + || self.summary_definition_id != expected_definition + { + return Err("ERP observation belongs to a different catalog or summary"); + } + if self.invalid_reason.is_some() + || self.populations.is_empty() + || self.window_start_ms >= self.window_end_ms + { + return Err("ERP observation is incomplete or invalid"); + } + if freshness.max_age_ms == 0 + || self.observed_at_unix_ms > now_ms.saturating_add(freshness.max_future_skew_ms) + || now_ms.saturating_sub(self.observed_at_unix_ms) > freshness.max_age_ms + { + return Err("ERP observation is stale or future-dated"); + } + let mut seen = std::collections::BTreeSet::new(); + if self + .populations + .iter() + .any(|p| !seen.insert(&p.population_id)) + { + return Err("ERP observation repeats a population"); + } + Ok(()) + } +} diff --git a/crates/asap_types/src/lib.rs b/crates/asap_types/src/lib.rs index ccf828726..5b82e68e8 100644 --- a/crates/asap_types/src/lib.rs +++ b/crates/asap_types/src/lib.rs @@ -3,6 +3,7 @@ pub mod accuracy; pub mod aggregation_config; pub mod aggregation_type; pub mod enums; +pub mod erp_observation; pub mod executable_plan; pub mod key_by_label_names; pub mod monitor_spec; From eb304610420203317ea7afb1cf7a1748bfe98134 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 20:00:11 -0600 Subject: [PATCH 02/28] Validate ERP readouts against the existing state parameters --- control_plane/src/physical/erp.rs | 109 ++++++++++++++++++++++++++++-- 1 file changed, 103 insertions(+), 6 deletions(-) diff --git a/control_plane/src/physical/erp.rs b/control_plane/src/physical/erp.rs index 646162e21..a638e38a8 100644 --- a/control_plane/src/physical/erp.rs +++ b/control_plane/src/physical/erp.rs @@ -273,11 +273,11 @@ impl asap_aware_mapping::AccuracyModel for ErpAccuracyModel<'_> { _ => theoretical, }; }; - match policy.select_readout( + match policy.evidence_for_readout( kind.algorithm().clone(), readout, self.max_error, - kind.params().clone(), + kind.params(), ) { ErpParameterDecision::ExactFallback { .. } => None, ErpParameterDecision::TheoreticalFallback { .. } => theoretical, @@ -471,7 +471,7 @@ impl ErpPlanningInput { max_error: f64, theoretical: SketchParams, ) -> ErpParameterDecision { - self.select_metric(algorithm, &self.error_metric, max_error, theoretical) + self.select_metric(algorithm, &self.error_metric, max_error, theoretical, None) } pub(crate) fn select_readout( @@ -481,7 +481,31 @@ impl ErpPlanningInput { max_error: f64, theoretical: SketchParams, ) -> ErpParameterDecision { - self.select_metric(algorithm, readout.metric_key(), max_error, theoretical) + self.select_metric( + algorithm, + readout.metric_key(), + max_error, + theoretical, + None, + ) + } + + /// Validate an existing state's readout independently of which new state + /// would be cheapest to allocate for this one consumer. + fn evidence_for_readout( + &self, + algorithm: SketchAlgorithm, + readout: ReadoutEvidence, + max_error: f64, + actual: &SketchParams, + ) -> ErpParameterDecision { + self.select_metric( + algorithm, + readout.metric_key(), + max_error, + actual.clone(), + Some(actual), + ) } fn select_metric( @@ -490,6 +514,7 @@ impl ErpPlanningInput { error_metric: &str, max_error: f64, theoretical: SketchParams, + required_params: Option<&SketchParams>, ) -> ErpParameterDecision { // Runtime admissibility belongs before ranking: an unusable cheap // profile must not hide a more expensive executable alternative. @@ -497,8 +522,12 @@ impl ErpPlanningInput { artifact.records.retain(|row| { sketch_name_matches(&row.sketch, &algorithm) && parse_params(&algorithm, &row.parameters).is_some_and(|params| { - self.runtime - .supports(&algorithm, ¶ms, Some(row.resources.memory_bytes)) + required_params.is_none_or(|required| required == ¶ms) + && self.runtime.supports( + &algorithm, + ¶ms, + Some(row.resources.memory_bytes), + ) }) }); let allowed_sketches = artifact @@ -811,6 +840,74 @@ mod tests { } } + /// Structural evidence fixture, not measured calibration data. + #[test] + fn existing_univmon_readout_uses_its_own_evidence_despite_cheaper_sibling() { + use asap_aware_mapping::AccuracyModel; + use planner_types::post_asap::*; + let mut policy = input(ErpAccuracyMode::Empirical); + let small = &mut policy.artifact.records[0]; + small.sketch = "univmon".into(); + small.parameters = + serde_json::json!({"heap_size":32,"sketch_rows":5,"sketch_cols":128,"layers":4}); + small.error_metrics = + BTreeMap::from([("max_frequency_entropy_absolute_bits_error".into(), 0.1)]); + let mut large = small.clone(); + large.id = "larger-shared-state".into(); + large.parameters["heap_size"] = 128.into(); + large.resources.memory_bytes *= 2.0; + large + .error_metrics + .insert("max_frequency_entropy_absolute_bits_error".into(), 0.02); + policy.artifact.records.push(large); + let small_params = SketchParams::UnivMon { + heap_size: 32, + sketch_rows: 5, + sketch_cols: 128, + layers: 4, + }; + let actual = SketchParams::UnivMon { + heap_size: 128, + sketch_rows: 5, + sketch_cols: 128, + layers: 4, + }; + assert_eq!( + policy + .select_readout( + SketchAlgorithm::UnivMon, + ReadoutEvidence::EntropyAbsoluteBits, + 0.2, + actual.clone() + ) + .params(), + Some(&small_params) + ); + let family = SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::UnivMon, actual), + GroupingStrategy::PerSubpopulationInstance, + ); + let model = ErpAccuracyModel { + policy: Some(&policy), + max_error: 0.2, + }; + let guarantee = model + .local_guarantee(&family, &SketchQuery::FrequencyEntropy) + .unwrap(); + assert_eq!(guarantee.bound, BoundExpr::Constant { value: 0.02 }); + assert!(matches!( + guarantee.failure_probability, + ProbabilityExpr::Unknown { .. } + )); + policy.artifact.records[1].error_metrics.clear(); + assert!(ErpAccuracyModel { + policy: Some(&policy), + max_error: 0.2 + } + .local_guarantee(&family, &SketchQuery::FrequencyEntropy) + .is_none()); + } + #[test] fn hll_cardinality_uses_measured_relative_error_not_rse() { use asap_aware_mapping::AccuracyModel; From 1a7cdbcedc22c75edb1f0ac54100659a8511c43c Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 20:16:16 -0600 Subject: [PATCH 03/28] feat(erp): require common configuration evidence across populations --- control_plane/src/physical/compiler.rs | 2 + control_plane/src/physical/erp.rs | 73 ++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index e4eaa85c6..8816d4842 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -4350,6 +4350,7 @@ mod tests { byte_second_weight: 1e-9, mode: super::super::erp::ErpAccuracyMode::Hybrid, observed_shape: None, + observed_populations: None, observed_shape_source: None, shape_match: None, runtime: super::super::erp::ErpRuntimeCapabilities { @@ -4399,6 +4400,7 @@ mod tests { byte_second_weight: 1e-9, mode: ErpAccuracyMode::Hybrid, observed_shape: None, + observed_populations: None, observed_shape_source: None, shape_match: None, runtime: ErpRuntimeCapabilities { diff --git a/control_plane/src/physical/erp.rs b/control_plane/src/physical/erp.rs index a638e38a8..973cf3d89 100644 --- a/control_plane/src/physical/erp.rs +++ b/control_plane/src/physical/erp.rs @@ -362,6 +362,9 @@ pub struct ErpPlanningInput { /// replaced by bounded nearest-profile matching. #[serde(default)] pub observed_shape: Option, + #[serde(default)] + pub observed_populations: + Option>, /// Runtime-samples ring key from which the backend resolves the freshest /// `erp_observed_shape` payload before compiling a plan. #[serde(default)] @@ -516,6 +519,75 @@ impl ErpPlanningInput { theoretical: SketchParams, required_params: Option<&SketchParams>, ) -> ErpParameterDecision { + if let Some(populations) = &self.observed_populations { + // Every installed partition must support the same configuration. + // Never aggregate their frequency maps into a fictitious population. + let mut best: Option = None; + if populations.invalid_reason.is_none() && !populations.populations.is_empty() { + for row in &self.artifact.records { + if !sketch_name_matches(&row.sketch, &algorithm) { + continue; + } + let Some(params) = parse_params(&algorithm, &row.parameters) else { + continue; + }; + if required_params.is_some_and(|required| required != ¶ms) { + continue; + } + let mut policy = self.clone(); + policy.observed_populations = None; + policy.mode = ErpAccuracyMode::Empirical; + let mut errors: f64 = 0.0; + let mut cost = 0.0; + let mut evidence = Vec::new(); + let valid = populations.populations.iter().all(|population| { + policy.observed_shape = Some(population.shape.observation.clone()); + match policy.select_metric( + algorithm.clone(), + error_metric, + max_error, + theoretical.clone(), + Some(¶ms), + ) { + ErpParameterDecision::Empirical { + record_id, + observed_error, + estimated_cost, + .. + } => { + errors = errors.max(observed_error); + cost += estimated_cost; + evidence.push(record_id); + true + } + _ => false, + } + }); + if valid && cost.is_finite() && best.as_ref().is_none_or(|previous| { + matches!(previous, ErpParameterDecision::Empirical { estimated_cost, .. } if cost < *estimated_cost) + }) { + best = Some(ErpParameterDecision::Empirical { + params, record_id: serde_json::to_string(&evidence).expect("string IDs serialize"), + observed_error: errors, estimated_cost: cost, + }); + } + } + } + return best.unwrap_or_else(|| { + let reason = + "ERP has no configuration valid for every observed population".to_owned(); + if self.mode == ErpAccuracyMode::Hybrid + && self.runtime.supports(&algorithm, &theoretical, None) + { + ErpParameterDecision::TheoreticalFallback { + params: theoretical, + reason, + } + } else { + ErpParameterDecision::ExactFallback { reason } + } + }); + } // Runtime admissibility belongs before ranking: an unusable cheap // profile must not hide a more expensive executable alternative. let mut artifact = self.artifact.clone(); @@ -834,6 +906,7 @@ mod tests { byte_second_weight: 1e-9, mode, observed_shape: None, + observed_populations: None, observed_shape_source: None, shape_match: None, runtime: ErpRuntimeCapabilities::default(), From 218715c86284d65efede7bb92d91d68c808074d9 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 20:16:54 -0600 Subject: [PATCH 04/28] fix(erp): preserve comparable population observation contracts --- crates/asap_types/src/erp_observation.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/asap_types/src/erp_observation.rs b/crates/asap_types/src/erp_observation.rs index 9bdde9b77..9f543102d 100644 --- a/crates/asap_types/src/erp_observation.rs +++ b/crates/asap_types/src/erp_observation.rs @@ -10,14 +10,14 @@ pub enum ErpObservationInputSemantics { UnitSampleFrequency, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ErpPopulationObservation { pub population_id: SummaryInstanceId, pub shape: Shape, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ErpPopulationObservations { pub schema_version: u32, From 38066e569d8fd285be79c627bfd235414521490a Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 20:16:54 -0600 Subject: [PATCH 05/28] fix(erp): preserve comparable population observation contracts --- crates/asap_types/src/erp_observation.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/asap_types/src/erp_observation.rs b/crates/asap_types/src/erp_observation.rs index 9bdde9b77..9f543102d 100644 --- a/crates/asap_types/src/erp_observation.rs +++ b/crates/asap_types/src/erp_observation.rs @@ -10,14 +10,14 @@ pub enum ErpObservationInputSemantics { UnitSampleFrequency, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ErpPopulationObservation { pub population_id: SummaryInstanceId, pub shape: Shape, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ErpPopulationObservations { pub schema_version: u32, From 8bd9d233341d75c7cd96cbe45bed2193f26542a9 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 20:24:23 -0600 Subject: [PATCH 06/28] feat(erp): retain authoritative observed data descriptor --- control_plane/src/physical/compiler.rs | 2 ++ control_plane/src/physical/erp.rs | 3 +++ 2 files changed, 5 insertions(+) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 8816d4842..1bc6ab9fd 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -4351,6 +4351,7 @@ mod tests { mode: super::super::erp::ErpAccuracyMode::Hybrid, observed_shape: None, observed_populations: None, + resolved_data_descriptor: None, observed_shape_source: None, shape_match: None, runtime: super::super::erp::ErpRuntimeCapabilities { @@ -4401,6 +4402,7 @@ mod tests { mode: ErpAccuracyMode::Hybrid, observed_shape: None, observed_populations: None, + resolved_data_descriptor: None, observed_shape_source: None, shape_match: None, runtime: ErpRuntimeCapabilities { diff --git a/control_plane/src/physical/erp.rs b/control_plane/src/physical/erp.rs index 973cf3d89..8d2b1dd8d 100644 --- a/control_plane/src/physical/erp.rs +++ b/control_plane/src/physical/erp.rs @@ -365,6 +365,8 @@ pub struct ErpPlanningInput { #[serde(default)] pub observed_populations: Option>, + #[serde(skip)] + pub resolved_data_descriptor: Option>, /// Runtime-samples ring key from which the backend resolves the freshest /// `erp_observed_shape` payload before compiling a plan. #[serde(default)] @@ -907,6 +909,7 @@ mod tests { mode, observed_shape: None, observed_populations: None, + resolved_data_descriptor: None, observed_shape_source: None, shape_match: None, runtime: ErpRuntimeCapabilities::default(), From 2d6999006f934884bcd88662e019f0ee462fe286 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 20:26:13 -0600 Subject: [PATCH 07/28] feat(erp): scope live evidence to catalog input semantics --- control_plane/src/physical/compiler.rs | 86 ++++++++++++++++--- crates/asap_types/src/sds.rs | 14 +++ .../storage_engines/sketch_db/index/mod.rs | 16 ++-- 3 files changed, 96 insertions(+), 20 deletions(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 1bc6ab9fd..9aa011bdc 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -2464,6 +2464,52 @@ pub fn select_workload_roots( select_workload_roots_with_erp(queries, roots, evidence, exact_costs, None) } +fn observed_population_matches_root( + policy: &super::erp::ErpPlanningInput, + root: &QueryExpr, +) -> bool { + use asap_types::sds::{PopulationPartitioning, ValueProjectionIdentity}; + use planner_types::pre_asap::{AggIntent, Reduction}; + let (Some(data), Some(observed)) = ( + &policy.resolved_data_descriptor, + &policy.observed_populations, + ) else { + return false; + }; + let QueryExpr::Aggregate { + reduction: Reduction::PerEntity, + measures, + having: None, + child, + .. + } = root + else { + return false; + }; + if measures.is_empty() + || !measures.iter().all(|intent| { + matches!( + intent, + AggIntent::Cardinality { col: None, .. } + | AggIntent::FrequencyL2 { col: None, .. } + | AggIntent::FrequencyEntropy { col: None, .. } + ) + }) + { + return false; + } + let Ok((metric, Some(window), filter)) = raw_time_series_input_contract(child, false) else { + return false; + }; + data.time_series_metric() == Some(metric.as_str()) + && data.population_filter_canonical == filter + && data.value_projection == ValueProjectionIdentity::SampleValue + && data.partitioning == Some(PopulationPartitioning::PerEntity) + && data.group_by_keys.is_empty() + && observed.window_end_ms.checked_sub(observed.window_start_ms) + == i64::try_from(window.saturating_mul(1000)).ok() +} + pub fn select_workload_roots_with_erp( queries: &mut [PlanningQuery], roots: Vec>, @@ -2501,6 +2547,17 @@ pub fn select_workload_roots_with_erp( if !matches!(accuracy, AccuracyTarget::Epsilon(_)) { policy.artifact.records.clear(); } + if policy.observed_populations.is_some() + && !roots + .iter() + .all(|(_, root)| observed_population_matches_root(&policy, root)) + { + if let Some(observed) = &mut policy.observed_populations { + observed.invalid_reason = + Some("candidate input differs from observed catalog data semantics".into()); + observed.populations.clear(); + } + } // A benchmark of a different KLL implementation is not evidence // for the collector's sketchlib KLL, even with the same k. policy.artifact.records.retain(|row| { @@ -3109,22 +3166,31 @@ pub(crate) fn materialization_leaf_contract( let SummaryExpr::KeepPreAsap(expr) = &child.expr else { return Err("materialization input is not a raw source".into()); }; - let (source, window_secs) = match expr.as_ref() { + raw_time_series_input_contract( + expr, + matches!( + &node.expr, + SummaryExpr::SummaryAgg { + family: SummaryFamilyType::ExactAggregate(..), + .. + } + ), + ) +} + +fn raw_time_series_input_contract( + expr: &QueryExpr, + exact: bool, +) -> Result<(String, Option, String), String> { + use planner_types::pre_asap::{CompareOpKind, ScalarValue}; + let (source, window_secs) = match expr { QueryExpr::TimeRange { child, range } => { if range.as_millis() == 0 || range.as_millis() % 1000 != 0 { return Err("warm producer requires a positive whole-second range".into()); } (child.as_ref(), Some(range.as_secs())) } - QueryExpr::Scan { .. } - if matches!( - &node.expr, - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(..), - .. - } - ) => - { + QueryExpr::Scan { .. } if exact => { return Err( "instantaneous sample selection is not a temporal accumulator readout".into(), ); diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index 24fe2a6d9..7720cf60a 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -115,6 +115,20 @@ pub struct SummaryInstanceCoordinates { pub group_values: BTreeMap, } +impl SummaryInstanceCoordinates { + pub fn instance_id(&self) -> Result { + let bytes = + serde_json::to_vec(&self.group_values).map_err(|error| SdsError(error.to_string()))?; + SummaryInstanceId::new(format!( + "summary-instance:v1:{}:{}:{}:{}", + self.summary_definition_id.as_u64(), + self.time_range.start_ms, + self.time_range.end_ms, + xxhash_rust::xxh64::xxh64(&bytes, 0) + )) + } +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SummaryWindowCompletion { diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index 6e383d520..60411e9bd 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -1062,16 +1062,12 @@ impl SketchStore { .map_err(|_| "summary instance start exceeds signed timestamp range")?; let end_ms = i64::try_from(window.1) .map_err(|_| "summary instance end exceeds signed timestamp range")?; - let group_bytes = - serde_json::to_vec(&group_values).map_err(|error| error.to_string())?; - let group_fingerprint = xxhash_rust::xxh64::xxh64(&group_bytes, 0); - let instance_id = SummaryInstanceId::new(format!( - "summary-instance:v1:{}:{}:{}:{}", - summary_definition_id.as_u64(), - window.0, - window.1, - group_fingerprint - )) + let instance_id = asap_types::sds::SummaryInstanceCoordinates { + summary_definition_id, + time_range: asap_types::sds::HalfOpenTimeRange { start_ms, end_ms }, + group_values: group_values.clone(), + } + .instance_id() .map_err(|error| error.to_string())?; let instance = SummaryInstance { instance_id: instance_id.clone(), From a503d10c93ad181bb3efa7e8404b2f3e7ec8d7a6 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 20:31:51 -0600 Subject: [PATCH 08/28] feat(erp): observe bounded precompute inputs and publish after finite drain --- control_plane/src/physical/erp.rs | 4 + .../drivers/ingest/prometheus_remote_write.rs | 7 + data_plane/src/main.rs | 10 + data_plane/src/precompute_engine/engine.rs | 3 +- .../src/precompute_engine/erp_observer.rs | 223 ++++++++++++++++++ data_plane/src/precompute_engine/mod.rs | 1 + .../src/precompute_engine/series_router.rs | 11 + data_plane/src/precompute_engine/worker.rs | 43 ++++ 8 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 data_plane/src/precompute_engine/erp_observer.rs diff --git a/control_plane/src/physical/erp.rs b/control_plane/src/physical/erp.rs index 8d2b1dd8d..646a7f010 100644 --- a/control_plane/src/physical/erp.rs +++ b/control_plane/src/physical/erp.rs @@ -31,6 +31,10 @@ pub struct ErpShapeObserver { } impl ErpShapeObserver { + pub fn observed_key_count(&self) -> usize { + self.frequencies.len() + } + pub fn new(max_observed_keys: usize) -> Result { Self::with_limits(max_observed_keys, 256) } diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index 2841bf626..7deb61e22 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -201,6 +201,13 @@ impl PrometheusRemoteWriteReceiver { .ingest .sketch_index .seal_finite_summary_input(&generation)?; + if let Some(observer) = self.inner.ingest.router.erp_observer() { + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|error| error.to_string())? + .as_millis() as u64; + observer.publish_finite(&generation, now_ms).await?; + } trim_process_allocator(); Ok(()) } diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 345479b29..208b7db50 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -44,6 +44,9 @@ fn unix_time_ms() -> u64 { #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] struct Args { + /// Publish bounded materialization-input ERP observations after a verified finite-source drain. + #[arg(long)] + erp_runtime_samples_endpoint: Option, /// Runtime component profile. `asapquery` enables backend ingest-time /// materialization and rejects Collector/OTLP-only components. #[arg(long, value_enum, default_value = "distributed")] @@ -874,6 +877,13 @@ async fn main() -> Result<()> { series_resolver.clone(), sketch_index.clone(), ); + if let Some(endpoint) = args.erp_runtime_samples_endpoint.clone() { + engine + .ingest_state() + .router + .enable_erp_observation(endpoint) + .map_err(std::io::Error::other)?; + } let worker_diagnostics = engine.diagnostics(); let ingest_state = engine.ingest_state(); info!("Starting precompute engine (ingest adapters share its bounded worker queues)"); diff --git a/data_plane/src/precompute_engine/engine.rs b/data_plane/src/precompute_engine/engine.rs index 15f44c40f..8d59efe94 100644 --- a/data_plane/src/precompute_engine/engine.rs +++ b/data_plane/src/precompute_engine/engine.rs @@ -129,7 +129,7 @@ impl PrecomputeEngine { // ConfigReload messages needed. let mut worker_handles = Vec::with_capacity(num_workers); for (id, rx) in receivers.into_iter().enumerate() { - let worker = Worker::new( + let mut worker = Worker::new( id, rx, output_sink.clone(), @@ -148,6 +148,7 @@ impl PrecomputeEngine { self.diagnostics.worker_group_counts[id].clone(), self.diagnostics.worker_watermarks[id].clone(), ); + worker.set_erp_observer(self.ingest_state.router.erp_observer()); let handle = tokio::spawn(async move { worker.run().await; }); diff --git a/data_plane/src/precompute_engine/erp_observer.rs b/data_plane/src/precompute_engine/erp_observer.rs new file mode 100644 index 000000000..4c05aa617 --- /dev/null +++ b/data_plane/src/precompute_engine/erp_observer.rs @@ -0,0 +1,223 @@ +//! Bounded observations of actual materialization inputs, published only after +//! an explicit finite-source completion barrier. No worker timestamp is a seal. +use asap_types::erp_observation::*; +use asap_types::sds::*; +use control_plane::physical::erp::{ErpObservedShape, ErpShapeObserver}; +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +const MAX_POPULATIONS: usize = 128; +const MAX_KEYS: usize = 65_536; + +struct Population { + coordinates: SummaryInstanceCoordinates, + semantics: ErpObservationInputSemantics, + source: String, + sketch: String, + implementation: String, + observer: ErpShapeObserver, +} +#[derive(Default)] +struct Observations { + generation: Option, + populations: BTreeMap, + extent: BTreeMap, + total_keys: usize, + invalid: Option, +} + +pub struct RuntimeErpObserver { + endpoint: String, + observations: Mutex, +} +impl RuntimeErpObserver { + pub fn new(endpoint: String) -> Arc { + Arc::new(Self { + endpoint, + observations: Mutex::new(Observations::default()), + }) + } + pub fn observe( + &self, + generation: &CatalogGeneration, + coordinates: SummaryInstanceCoordinates, + config: &asap_types::AggregationConfig, + timestamp_ms: i64, + value: f64, + ) { + use asap_types::{AggregationType, SampleUpdateRule}; + let (semantics, sketch, implementation) = match config.aggregation_type { + AggregationType::HLL => ( + ErpObservationInputSemantics::ScalarSampleValue, + "hll", + "asap-sketchlib-hll-regular-v1", + ), + AggregationType::UnivMon => ( + ErpObservationInputSemantics::UnitSampleFrequency, + "univmon", + "asap-sketchlib-univmon-standard-v1", + ), + _ => return, + }; + let mut state = self.observations.lock().unwrap(); + if state.generation.as_ref() != Some(generation) { + *state = Observations { + generation: Some(generation.clone()), + ..Default::default() + }; + } + if state.invalid.is_some() { + return; + } + if !value.is_finite() + || !matches!( + config.sample_update_rule(), + SampleUpdateRule::Value { scale: 1.0 } + ) + { + state.invalid = Some("unsupported non-finite or transformed summary input".into()); + for population in state.populations.values_mut() { + population.observer = ErpShapeObserver::new(1).unwrap(); + } + return; + } + let extent = state + .extent + .entry(coordinates.summary_definition_id) + .or_insert((timestamp_ms, timestamp_ms)); + extent.0 = extent.0.min(timestamp_ms); + extent.1 = extent.1.max(timestamp_ms); + let Ok(id) = coordinates.instance_id() else { + return; + }; + if !state.populations.contains_key(&id) && state.populations.len() >= MAX_POPULATIONS { + state.invalid = Some("ERP population observation budget exceeded".into()); + for population in state.populations.values_mut() { + population.observer = ErpShapeObserver::new(1).unwrap(); + } + return; + } + let population = state.populations.entry(id).or_insert_with(|| Population { + source: format!( + "summary-definition:{}", + coordinates.summary_definition_id.as_u64() + ), + coordinates, + semantics, + sketch: sketch.into(), + implementation: implementation.into(), + observer: ErpShapeObserver::new(MAX_KEYS).unwrap(), + }); + let before = population.observer.observed_key_count(); + // Canonical fixed-width numeric identity; no raw sample or arbitrary label copy. + let key = format!("{:016x}", if value == 0.0 { 0 } else { value.to_bits() }); + let result = population.observer.observe(&key, 0); + let added = population + .observer + .observed_key_count() + .saturating_sub(before); + state.total_keys = state.total_keys.saturating_add(added); + if result.is_err() || state.total_keys > MAX_KEYS { + state.invalid = Some("ERP key observation budget exceeded".into()); + for population in state.populations.values_mut() { + population.observer = ErpShapeObserver::new(1).unwrap(); + } + } + } + + pub async fn publish_finite( + &self, + generation: &CatalogGeneration, + now_ms: u64, + ) -> Result<(), String> { + use control_plane::runtime_samples::feedback::{ + runtime_samples_client::RuntimeSamplesClient, PushBatch, RuntimeRecord, + }; + let records = { + let state = self.observations.lock().unwrap(); + if state.generation.as_ref() != Some(generation) { + return Ok(()); + } + let mut groups: BTreeMap< + (SummaryDefinitionId, i64, i64), + ( + String, + String, + String, + ErpPopulationObservations, + ), + > = BTreeMap::new(); + for (id, population) in &state.populations { + let c = &population.coordinates; + let Some((first, last)) = state.extent.get(&c.summary_definition_id) else { + continue; + }; + if c.time_range.start_ms < *first || c.time_range.end_ms > *last { + continue; + } + let key = ( + c.summary_definition_id, + c.time_range.start_ms, + c.time_range.end_ms, + ); + let entry = groups.entry(key).or_insert_with(|| { + ( + population.source.clone(), + population.sketch.clone(), + population.implementation.clone(), + ErpPopulationObservations { + schema_version: 1, + catalog_generation: generation.clone(), + summary_definition_id: c.summary_definition_id, + observed_at_unix_ms: now_ms, + window_start_ms: c.time_range.start_ms, + window_end_ms: c.time_range.end_ms, + input_semantics: population.semantics, + invalid_reason: state.invalid.clone(), + populations: Vec::new(), + }, + ) + }); + match population.observer.snapshot() { + Some(shape) => entry.3.populations.push(ErpPopulationObservation { + population_id: id.clone(), + shape, + }), + None => entry.3.invalid_reason = Some("ERP population fit unavailable".into()), + } + } + groups + .into_values() + .map(|(source, sketch, impl_name, mut envelope)| { + if envelope.invalid_reason.is_some() { + envelope.populations.clear(); + } + RuntimeRecord { + source, + sketch, + impl_name, + schema_version: 1, + payload_json: serde_json::json!({"erp_population_observations":envelope}) + .to_string(), + } + }) + .collect::>() + }; + if records.is_empty() { + return Ok(()); + } + let expected = records.len() as u64; + let mut client = RuntimeSamplesClient::connect(self.endpoint.clone()) + .await + .map_err(|e| e.to_string())?; + let ack = client + .push(PushBatch { records }) + .await + .map_err(|e| e.to_string())? + .into_inner(); + if ack.accepted != expected { + return Err("control plane rejected ERP observation records".into()); + } + Ok(()) + } +} diff --git a/data_plane/src/precompute_engine/mod.rs b/data_plane/src/precompute_engine/mod.rs index c10b4bb1f..068ac13b7 100644 --- a/data_plane/src/precompute_engine/mod.rs +++ b/data_plane/src/precompute_engine/mod.rs @@ -2,6 +2,7 @@ pub mod accumulator_factory; pub mod config; pub mod coordination_checkpoint; mod engine; +pub mod erp_observer; pub mod frame_lineage; pub mod group_key; pub mod ingest_handler; diff --git a/data_plane/src/precompute_engine/series_router.rs b/data_plane/src/precompute_engine/series_router.rs index b0de9df62..3a90705d8 100644 --- a/data_plane/src/precompute_engine/series_router.rs +++ b/data_plane/src/precompute_engine/series_router.rs @@ -148,6 +148,7 @@ impl fmt::Debug for WorkerMessage { /// Routes incoming samples to one of N workers based on a consistent hash. pub struct SeriesRouter { + erp_observer: std::sync::OnceLock>, senders: Vec>, num_workers: usize, } @@ -156,11 +157,21 @@ impl SeriesRouter { pub fn new(senders: Vec>) -> Self { let num_workers = senders.len(); Self { + erp_observer: std::sync::OnceLock::new(), senders, num_workers, } } + pub fn enable_erp_observation(&self, endpoint: String) -> Result<(), String> { + self.erp_observer + .set(super::erp_observer::RuntimeErpObserver::new(endpoint)) + .map_err(|_| "ERP observer already configured".into()) + } + pub fn erp_observer(&self) -> Option> { + self.erp_observer.get().cloned() + } + /// Route a pre-grouped batch of group messages to workers concurrently. /// /// Each `GroupSamples` / `AccumulatorInput` message is routed by diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index 6fdbeed50..845c72d71 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -148,6 +148,7 @@ pub struct WorkerRuntimeConfig { /// `(metric, attrs_fingerprint, agg_kind_canonical)` identity contract on /// `SeriesIdResolver`, so one sid uniquely names one bucket. pub struct Worker { + erp_observer: Option>, current_input_revision: Option>, id: usize, receiver: mpsc::Receiver, @@ -184,6 +185,12 @@ pub struct Worker { } impl Worker { + pub fn set_erp_observer( + &mut self, + observer: Option>, + ) { + self.erp_observer = observer; + } #[allow(clippy::too_many_arguments)] pub fn new( id: usize, @@ -204,6 +211,7 @@ impl Worker { wall_clock_max_open_grace_period_ms, } = runtime_config; Self { + erp_observer: None, current_input_revision: None, id, receiver, @@ -574,6 +582,24 @@ impl Worker { record_late_input("append_correction", "raw_sample"); let mut updater = create_accumulator_updater(&state.config); apply_sample(&mut *updater, series_key, *val, *ts, &state.config); + if let (Some(observer), Some(revision)) = + (&self.erp_observer, &input_revision) + { + observer.observe( + &revision.generation, + asap_types::sds::SummaryInstanceCoordinates { + summary_definition_id: policy_fp.into(), + time_range: asap_types::sds::HalfOpenTimeRange { + start_ms: bucket_start, + end_ms: bucket_end, + }, + group_values: group_key.as_population_labels(), + }, + &state.config, + *ts, + *val, + ); + } let key = build_group_key_label_values(group_key); let output = precomputed_output_for_group( window_start as u64, @@ -604,6 +630,23 @@ impl Worker { .or_insert_with(|| create_accumulator_updater(&state.config)); if let Some(value) = value { apply_sample(&mut **updater, series_key, value, *ts, &state.config); + if let (Some(observer), Some(revision)) = (&self.erp_observer, &input_revision) + { + observer.observe( + &revision.generation, + asap_types::sds::SummaryInstanceCoordinates { + summary_definition_id: policy_fp.into(), + time_range: asap_types::sds::HalfOpenTimeRange { + start_ms: bucket_start, + end_ms: bucket_end, + }, + group_values: group_key.as_population_labels(), + }, + &state.config, + *ts, + value, + ); + } } } } From 1826bc8d29e822fe3ed2628bb843c8af6cbd79f2 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 20:32:28 -0600 Subject: [PATCH 09/28] feat(erp): validate live evidence against the activated catalog --- control_plane/src/main.rs | 87 ++++- control_plane/src/physical/erp.rs | 299 +++++++++++++++++- crates/asap_types/src/erp_observation.rs | 2 +- .../summary-catalog-sds-architecture.md | 32 ++ 4 files changed, 411 insertions(+), 9 deletions(-) diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index a79236381..dbf63519d 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -69,6 +69,10 @@ struct AppState { /// agents' `sketch-runtime::PushExporter`. Read by decision /// loops in the replanner. runtime_samples: Arc, + /// The successfully activated typed catalog is authoritative for live ERP + /// input identity; incoming telemetry cannot supply its own descriptors. + active_summary_catalog: + Arc>>>, /// Phase C (MVP v6): shared `BackendClient` for posting /// `StreamingConfig` JSON / YAML to the ASAPQuery-backend's /// `POST /api/v1/streaming-config` endpoint. Phase B had this @@ -487,6 +491,7 @@ async fn main() { opamp_endpoint: opamp_ep, workload_registry: Arc::clone(&workload_registry), runtime_samples: Arc::clone(&runtime_samples_store), + active_summary_catalog: Arc::new(tokio::sync::Mutex::new(None)), backend_client: backend_client_shared, backend_routing_cache: Arc::clone(&backend_routing_cache), }; @@ -611,6 +616,8 @@ struct PhysicalPlanQueryRequest { #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct CompileAndPublishPhysicalPlanRequest { + #[serde(default = "default_physical_deployment_target")] + target: physical::compiler::PhysicalDeploymentTarget, #[serde(default)] workload_cost_evidence: Option, queries: Vec, @@ -635,6 +642,10 @@ struct CompileAndPublishPhysicalPlanRequest { apply_timeout_ms: u64, } +fn default_physical_deployment_target() -> physical::compiler::PhysicalDeploymentTarget { + physical::compiler::PhysicalDeploymentTarget::DistributedCollectors +} + fn default_physical_plan_timeout_ms() -> u64 { 10_000 } @@ -704,10 +715,15 @@ async fn compile_and_publish_physical_plan( mut request: CompileAndPublishPhysicalPlanRequest, frontend: PhysicalQueryFrontend, ) -> Response { + // Serialize typed activations so an older response cannot overwrite the + // catalog recorded after a newer backend activation. + let mut active_catalog = st.active_summary_catalog.lock().await; if let Some(erp) = &mut request.erp { if let Err(error) = erp.hydrate_observed_shape(&st.runtime_samples) { return (StatusCode::UNPROCESSABLE_ENTITY, error).into_response(); } + let catalog = active_catalog.clone(); + erp.resolve_population_data_descriptor(catalog.as_deref()); } let (bundle, collector_ids, apply_timeout, adaptation_evidence, _) = match compile_physical_plan_request(request, false, frontend) { @@ -802,6 +818,8 @@ async fn compile_and_publish_physical_plan( .into_response(); } + *active_catalog = Some(Arc::new(bundle.summary_catalog)); + Json(CompileAndPublishPhysicalPlanResponse { cost_comparison: bundle.cost_comparison, plan_id: bundle.envelope.plan_id, @@ -842,6 +860,7 @@ async fn publish_clickhouse_plan( publication: physical::publication::PhysicalPlanPublication, selection_trace: Option, ) -> axum::response::Response { + let mut active_catalog = state.active_summary_catalog.lock().await; let plan_id = publication.summary_catalog.plan_id; let plan_version = publication.summary_catalog.plan_version; let Some(client) = state.backend_client.as_ref() else { @@ -863,6 +882,7 @@ async fn publish_clickhouse_plan( .await; return (StatusCode::BAD_GATEWAY, error.to_string()).into_response(); } + *active_catalog = Some(Arc::new(publication.summary_catalog)); Json(serde_json::json!({ "plan_id": plan_id, "plan_version": plan_version, @@ -888,10 +908,15 @@ fn compile_physical_plan_request( ), (StatusCode, String), > { - if request.queries.is_empty() || request.collector_ids.is_empty() { + if request.queries.is_empty() + || (request.target == physical::compiler::PhysicalDeploymentTarget::DistributedCollectors + && request.collector_ids.is_empty()) + || (request.target == physical::compiler::PhysicalDeploymentTarget::BackendLocalRemoteWrite + && !request.collector_ids.is_empty()) + { return Err(( StatusCode::UNPROCESSABLE_ENTITY, - "queries and collector_ids must both be non-empty".to_string(), + "queries must be non-empty; distributed deployment requires collectors and backend-local deployment requires none".to_string(), )); } if request.max_evidence_age_ms == 0 || request.apply_timeout_ms == 0 { @@ -967,7 +992,8 @@ fn compile_physical_plan_request( let planning_request = physical::compiler::PlanningRequest { query_workload: None, queries, - hybrid_execution: false, + hybrid_execution: request.target + == physical::compiler::PhysicalDeploymentTarget::BackendLocalRemoteWrite, materialization_policy: None, evidence: request.evidence, exact_composition_costs: request.exact_composition_costs, @@ -978,7 +1004,7 @@ fn compile_physical_plan_request( retained_summary_memory_budget_bytes: None, }; let environment = physical::compiler::DeploymentEnvironment { - target: physical::compiler::PhysicalDeploymentTarget::DistributedCollectors, + target: request.target, collector_ids: request.collector_ids.clone(), capability_snapshot_id: request.capability_snapshot_id, observed_at_unix_ms: now, @@ -2242,6 +2268,7 @@ fn test_app_with_backend(backend_url: Option) -> (AppState, axum::Router opamp_endpoint: "ws://ctrl:4320/v1/opamp".into(), workload_registry: Arc::new(WorkloadRegistry::empty()), runtime_samples: runtime_samples::RuntimeSamplesStore::new(64), + active_summary_catalog: Arc::new(tokio::sync::Mutex::new(None)), backend_client, backend_routing_cache: Arc::new(Mutex::new(HashMap::new())), }; @@ -2309,6 +2336,58 @@ mod api_tests { assert_eq!(manifests.as_array().unwrap().len(), 1); } + #[test] + fn backend_local_typed_request_compiles_without_collectors() { + let snapshot: physical::compiler::BackendLocalPlanningSnapshot = serde_json::from_str( + include_str!("../../docs/examples/asapquery-compatibility-demo-snapshot.json"), + ) + .unwrap(); + let (planning, _) = snapshot.planning_request().unwrap(); + let mut query = planning.queries[0].clone(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64; + query.lifecycle.evidence_observed_at_unix_ms = now; + for implementation in &mut query.window_implementations { + implementation.cost.observed_at_unix_ms = now; + } + let planner_types::pre_asap::Source::TimeSeries { metric } = &query.source else { + panic!("expected time series fixture"); + }; + let value = serde_json::json!({ + "target": "backend_local_remote_write", + "queries": [{ + "query_id": query.query_id, "query_string": query.query_string, + "metric": metric, "window_secs": query.window_secs, "accuracy": query.accuracy, + "lifecycle": query.lifecycle, "window_implementations": query.window_implementations + }], + "collector_ids": [], "capability_snapshot_id": "test", + "planner_revision": physical::compiler::PLANNER_REVISION, + "max_evidence_age_ms": 60000, "plan_version": 1, + "activation_unix_ms": now, "backend_compat": physical::compiler::BACKEND_COMPAT + }); + let request = serde_json::from_value(value.clone()).unwrap(); + let (plan, collectors, _, _, _) = + compile_physical_plan_request(request, false, PhysicalQueryFrontend::PromQl).unwrap(); + let plan = plan.unwrap(); + assert!(collectors.is_empty()); + assert!(plan.collector_plans.is_empty()); + assert_eq!( + plan.precompute_plan.ingest.protocol, + physical::compiler::IngestProtocol::PrometheusRemoteWriteV1 + ); + assert!(!plan.precompute_plan.materializations.is_empty()); + let mut distributed = value; + distributed["target"] = serde_json::json!("distributed_collectors"); + assert!(compile_physical_plan_request( + serde_json::from_value(distributed).unwrap(), + false, + PhysicalQueryFrontend::PromQl + ) + .is_err()); + } + async fn body_json(resp: axum::response::Response) -> serde_json::Value { let bytes = resp.into_body().collect().await.unwrap().to_bytes(); serde_json::from_slice(&bytes).unwrap() diff --git a/control_plane/src/physical/erp.rs b/control_plane/src/physical/erp.rs index 8d2b1dd8d..a57ddbc05 100644 --- a/control_plane/src/physical/erp.rs +++ b/control_plane/src/physical/erp.rs @@ -383,25 +383,134 @@ pub struct ErpObservedShapeSource { pub source: String, pub sketch: String, pub implementation: String, + /// Required for catalog-scoped online evidence; absent only for legacy + /// offline runtime records that contain a single shape. + #[serde(default)] + pub population_scope: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct ErpPopulationObservationScope { + pub catalog_generation: asap_types::sds::CatalogGeneration, + pub summary_definition_id: asap_types::sds::SummaryDefinitionId, + pub input_semantics: asap_types::erp_observation::ErpObservationInputSemantics, + pub freshness: asap_types::erp_observation::ErpObservationFreshness, } impl ErpPlanningInput { + /// Resolve evidence against the control plane's accepted catalog, never + /// descriptors supplied by the observation producer. + pub fn resolve_population_data_descriptor( + &mut self, + catalog: Option<&asap_types::summary_catalog::SummaryCatalog>, + ) { + self.resolved_data_descriptor = None; + let Some(populations) = self.observed_populations.as_mut() else { + return; + }; + if populations.invalid_reason.is_some() { + return; + } + let resolved = (|| -> Result<_, String> { + let catalog = catalog.ok_or("no active authoritative catalog for ERP evidence")?; + let generation = catalog.reference().map_err(|error| error.to_string())?; + if generation != populations.catalog_generation { + return Err("ERP evidence catalog differs from the active catalog".into()); + } + let materialization = catalog + .materializations + .get(&populations.summary_definition_id) + .ok_or("ERP evidence summary is absent from the active catalog")?; + let data = catalog + .data_descriptors + .get(&materialization.data_descriptor_id) + .ok_or("ERP evidence data descriptor is absent from the active catalog")?; + Ok(std::sync::Arc::new(data.clone())) + })(); + match resolved { + Ok(data) => self.resolved_data_descriptor = Some(data), + Err(reason) => { + populations.invalid_reason = Some(reason); + populations.populations.clear(); + } + } + } + pub fn hydrate_observed_shape( &mut self, samples: &crate::runtime_samples::RuntimeSamplesStore, ) -> Result<(), String> { - if self.observed_shape.is_some() { - return Ok(()); - } + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|error| format!("cannot timestamp ERP observation: {error}"))? + .as_millis() + .try_into() + .map_err(|_| "ERP observation timestamp overflows milliseconds")?; + self.hydrate_observed_shape_at(samples, now_ms) + } + + pub fn hydrate_observed_shape_at( + &mut self, + samples: &crate::runtime_samples::RuntimeSamplesStore, + now_ms: u64, + ) -> Result<(), String> { let Some(source) = &self.observed_shape_source else { return Ok(()); }; + // A reused planning request must resolve new evidence on every compile. + // Never keep an old inline or previously hydrated fit after drift. + self.observed_shape = None; + self.observed_populations = None; let key = crate::runtime_samples::SampleKey { source: source.source.clone(), sketch: source.sketch.clone(), impl_name: source.implementation.clone(), }; - let record = samples.latest(&key).ok_or_else(|| { + let record = samples.latest(&key); + if let Some(scope) = &source.population_scope { + let resolved = (|| -> Result<_, String> { + let record = record + .as_ref() + .ok_or("no live ERP population observation")?; + let value = record + .payload + .get("erp_population_observations") + .ok_or("latest runtime sample has no erp_population_observations")?; + let observed: asap_types::erp_observation::ErpPopulationObservations< + ErpObservedShape, + > = serde_json::from_value(value.clone()) + .map_err(|error| format!("invalid ERP population observations: {error}"))?; + observed.validate_identity_and_freshness( + &scope.catalog_generation, + scope.summary_definition_id, + now_ms, + scope.freshness, + )?; + if observed.input_semantics != scope.input_semantics { + return Err("ERP observation input semantics do not match the summary".into()); + } + Ok(observed) + })(); + // An explicit invalid envelope prevents selection from borrowing an + // older fit or falling back to the artifact's distribution identity. + // The normal ERP -> theoretical -> exact policy handles this miss. + self.observed_populations = Some(resolved.unwrap_or_else(|reason| { + asap_types::erp_observation::ErpPopulationObservations { + schema_version: 1, + catalog_generation: scope.catalog_generation.clone(), + summary_definition_id: scope.summary_definition_id, + observed_at_unix_ms: now_ms, + window_start_ms: 0, + window_end_ms: 0, + input_semantics: scope.input_semantics, + invalid_reason: Some(reason), + populations: Vec::new(), + } + })); + return Ok(()); + } + let record = record.ok_or_else(|| { format!( "no runtime shape sample for {}/{}/{}", source.source, source.sketch, source.implementation @@ -1211,11 +1320,193 @@ mod tests { source: "edge-a".into(), sketch: "cms".into(), implementation: "oxide".into(), + population_scope: None, }); policy.hydrate_observed_shape(&samples).unwrap(); assert_eq!(policy.observed_shape.unwrap().cardinality, 1000); } + fn online_population_fixture() -> ( + ErpPlanningInput, + asap_types::erp_observation::ErpPopulationObservations, + ) { + use asap_types::erp_observation::*; + let generation = asap_types::sds::CatalogGeneration { + schema_version: 1, + plan_id: 7, + plan_version: 3, + snapshot_sha256: "test-catalog".into(), + }; + let definition = asap_types::PolicyFingerprint(7).into(); + let mut observer = ErpShapeObserver::new(4).unwrap(); + for key in ["a", "a", "b", "b"] { + observer.observe(key, 0).unwrap(); + } + let observed = ErpPopulationObservations { + schema_version: 1, + catalog_generation: generation.clone(), + summary_definition_id: definition, + observed_at_unix_ms: 1_000, + window_start_ms: 0, + window_end_ms: 1_000, + input_semantics: ErpObservationInputSemantics::UnitSampleFrequency, + invalid_reason: None, + populations: vec![ErpPopulationObservation { + population_id: asap_types::sds::SummaryInstanceId::new("partition-a").unwrap(), + shape: observer.snapshot().unwrap(), + }], + }; + let mut policy = input(ErpAccuracyMode::Hybrid); + policy.observed_shape_source = Some(ErpObservedShapeSource { + source: "backend-a".into(), + sketch: "univmon".into(), + implementation: "asap_sketchlib".into(), + population_scope: Some(ErpPopulationObservationScope { + catalog_generation: generation, + summary_definition_id: definition, + input_semantics: ErpObservationInputSemantics::UnitSampleFrequency, + freshness: ErpObservationFreshness { + max_age_ms: 100, + max_future_skew_ms: 5, + }, + }), + }); + (policy, observed) + } + + fn publish_population_fixture( + samples: &crate::runtime_samples::RuntimeSamplesStore, + payload: serde_json::Value, + ) { + samples.append_for_test(crate::runtime_samples::RuntimeRecord { + source: "backend-a".into(), + sketch: "univmon".into(), + impl_name: "asap_sketchlib".into(), + schema_version: 1, + payload, + }); + } + + /// New live evidence replaces hydrated fits; expiry invalidates them even + /// when a caller reuses the same planning request. + #[test] + fn online_hydration_refreshes_and_expires_population_evidence() { + let (mut policy, mut observed) = online_population_fixture(); + let samples = crate::runtime_samples::RuntimeSamplesStore::new(4); + publish_population_fixture( + &samples, + serde_json::json!({"erp_population_observations": observed}), + ); + policy.hydrate_observed_shape_at(&samples, 1_000).unwrap(); + assert_eq!( + policy.observed_populations.as_ref().unwrap().populations[0] + .shape + .observation + .cardinality, + 2 + ); + observed.populations[0].shape.observation.cardinality = 3; + observed.observed_at_unix_ms = 1_010; + publish_population_fixture( + &samples, + serde_json::json!({"erp_population_observations": observed}), + ); + policy.hydrate_observed_shape_at(&samples, 1_010).unwrap(); + assert_eq!( + policy.observed_populations.as_ref().unwrap().populations[0] + .shape + .observation + .cardinality, + 3 + ); + policy.hydrate_observed_shape_at(&samples, 1_111).unwrap(); + let invalid = policy.observed_populations.as_ref().unwrap(); + assert!(invalid.populations.is_empty()); + assert!(invalid.invalid_reason.as_deref().unwrap().contains("stale")); + assert!(policy.observed_shape.is_none()); + } + + /// Only the activated catalog may supply a candidate's data contract. + #[test] + fn online_population_descriptor_requires_authoritative_catalog() { + let snapshot: crate::physical::compiler::BackendLocalPlanningSnapshot = + serde_json::from_str(include_str!( + "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let plan = snapshot.compile().unwrap(); + let (mut policy, mut observed) = online_population_fixture(); + observed.catalog_generation = plan.summary_catalog.reference().unwrap(); + observed.summary_definition_id = + *plan.summary_catalog.materializations.keys().next().unwrap(); + policy.observed_populations = Some(observed.clone()); + policy.resolve_population_data_descriptor(Some(&plan.summary_catalog)); + let expected = &plan.summary_catalog.materializations[&observed.summary_definition_id] + .data_descriptor_id; + assert_eq!( + &policy.resolved_data_descriptor.as_ref().unwrap().id, + expected + ); + policy.resolve_population_data_descriptor(None); + assert!(policy.resolved_data_descriptor.is_none()); + assert!(policy + .observed_populations + .as_ref() + .unwrap() + .invalid_reason + .is_some()); + assert!(policy + .observed_populations + .as_ref() + .unwrap() + .populations + .is_empty()); + } + + /// Missing, corrupt, foreign and overflow evidence must never recover a + /// previously accepted fit or the artifact's legacy distribution match. + #[test] + fn online_hydration_invalidates_unusable_latest_evidence() { + let (template, observed) = online_population_fixture(); + let mut foreign = observed.clone(); + foreign.catalog_generation.plan_version += 1; + let mut wrong_input = observed.clone(); + wrong_input.input_semantics = + asap_types::erp_observation::ErpObservationInputSemantics::ScalarSampleValue; + let mut overflow = observed.clone(); + overflow.invalid_reason = Some("population limit".into()); + let mut future = observed.clone(); + future.observed_at_unix_ms = 1_006; + for payload in [ + None, + Some(serde_json::json!({})), + Some(serde_json::json!({"erp_population_observations": foreign})), + Some(serde_json::json!({"erp_population_observations": wrong_input})), + Some(serde_json::json!({"erp_population_observations": overflow})), + Some(serde_json::json!({"erp_population_observations": future})), + ] { + let mut policy = template.clone(); + policy.observed_shape = Some(observed.populations[0].shape.observation.clone()); + let samples = crate::runtime_samples::RuntimeSamplesStore::new(4); + if let Some(payload) = payload { + publish_population_fixture(&samples, payload); + } + policy.hydrate_observed_shape_at(&samples, 1_000).unwrap(); + let invalid = policy.observed_populations.as_ref().unwrap(); + assert!(invalid.invalid_reason.is_some()); + assert!(invalid.populations.is_empty()); + assert!(policy.observed_shape.is_none()); + assert!(matches!( + policy.select( + SketchAlgorithm::Hll, + 0.05, + SketchParams::Hll { precision: 12 } + ), + ErpParameterDecision::TheoreticalFallback { .. } + )); + } + } + #[test] fn observer_fails_loud_at_cardinality_cap() { let mut observer = ErpShapeObserver::new(1).unwrap(); diff --git a/crates/asap_types/src/erp_observation.rs b/crates/asap_types/src/erp_observation.rs index 9f543102d..efb693c05 100644 --- a/crates/asap_types/src/erp_observation.rs +++ b/crates/asap_types/src/erp_observation.rs @@ -33,7 +33,7 @@ pub struct ErpPopulationObservations { pub populations: Vec>, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ErpObservationFreshness { pub max_age_ms: u64, diff --git a/docs/design_docs/summary-catalog-sds-architecture.md b/docs/design_docs/summary-catalog-sds-architecture.md index baf28e179..4cf38d01c 100644 --- a/docs/design_docs/summary-catalog-sds-architecture.md +++ b/docs/design_docs/summary-catalog-sds-architecture.md @@ -375,3 +375,35 @@ creates a new Data Descriptor. Advancing the time range creates a new Summary Instance. Merge compatibility additionally requires the operator's merge rules, compatible data scopes and valid instance coverage; sharing descriptors alone does not authorize merging overlapping observations. + +### Catalog-scoped runtime ERP evidence + +A runtime observation describes the input of one allocated summary, not an +entire deployment. `ErpPopulationObservations` identifies its catalog generation, +summary definition, observation time, input window and separate summary-instance +populations. The control plane resolves the `DataDescriptor` from its successfully +activated catalog; a telemetry payload cannot provide replacement descriptors. +Alternative sketch parameters may use this evidence only when the compiler +verifies the same data and update semantics. + +The typed physical-plan HTTP endpoints accept `target: backend_local_remote_write` +with an empty `collector_ids` list. Omitting `target` preserves the distributed +collector deployment. Both paths use catalog publication and activation. Typed +activations are serialized, and the accepted catalog is retained only after the +backend acknowledges activation, including ClickHouse publications. + +An ERP `observed_shape_source.population_scope` supplies the expected catalog +and definition, input semantics, and explicit `max_age_ms` / +`max_future_skew_ms` bounds. Each compilation reads the latest runtime record +again. Missing, stale, malformed, foreign or incomplete observations invalidate +all population fits. This is an ERP miss handled by theoretical sizing or exact +execution; it must not restore an older fit or match the artifact's legacy +distribution descriptor. Offline single-shape inputs remain a separate path. + +The initial eligibility is deliberately limited to verified raw per-series +frequency/cardinality readouts over a complete matching window. A 30-second pane +observation does not certify a one-hour input distribution. These checks do not +implement an autonomous drift-triggered replan scheduler, continuous source +completion, or durable restoration of the control plane's active catalog. After +a control-plane restart, live evidence remains ineligible until an authoritative +catalog has been activated again. From c13ba4f00c93f0eb8e32040d593acec8fc4c15da Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 20:33:06 -0600 Subject: [PATCH 10/28] test(erp): cover bounded partition observations and catalog reset --- .../src/precompute_engine/erp_observer.rs | 89 ++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) diff --git a/data_plane/src/precompute_engine/erp_observer.rs b/data_plane/src/precompute_engine/erp_observer.rs index 4c05aa617..e8bc37485 100644 --- a/data_plane/src/precompute_engine/erp_observer.rs +++ b/data_plane/src/precompute_engine/erp_observer.rs @@ -111,7 +111,13 @@ impl RuntimeErpObserver { let before = population.observer.observed_key_count(); // Canonical fixed-width numeric identity; no raw sample or arbitrary label copy. let key = format!("{:016x}", if value == 0.0 { 0 } else { value.to_bits() }); - let result = population.observer.observe(&key, 0); + let range = population.coordinates.time_range; + let duration = range.end_ms.saturating_sub(range.start_ms).max(1); + let offset = timestamp_ms + .saturating_sub(range.start_ms) + .clamp(0, duration); + let interval = ((offset as u128 * 64) / duration as u128).min(63) as usize; + let result = population.observer.observe(&key, interval); let added = population .observer .observed_key_count() @@ -221,3 +227,84 @@ impl RuntimeErpObserver { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + fn fixture() -> (CatalogGeneration, asap_types::AggregationConfig) { + let config = asap_types::AggregationConfig::new( + asap_types::AggregationType::HLL, + String::new(), + Default::default(), + asap_types::KeyByLabelNames::new(vec![]), + asap_types::KeyByLabelNames::new(vec![]), + asap_types::KeyByLabelNames::new(vec![]), + String::new(), + 60, + 60, + planner_types::pre_asap::WindowKind::Tumbling, + String::new(), + "m".into(), + None, + None, + None, + ); + ( + CatalogGeneration { + schema_version: 1, + plan_id: 1, + plan_version: 1, + snapshot_sha256: "a".repeat(64), + }, + config, + ) + } + fn coordinate(group: usize) -> SummaryInstanceCoordinates { + SummaryInstanceCoordinates { + summary_definition_id: asap_types::PolicyFingerprint(1).into(), + time_range: HalfOpenTimeRange { + start_ms: 0, + end_ms: 60_000, + }, + group_values: BTreeMap::from([("job".into(), group.to_string())]), + } + } + #[test] + fn observations_keep_partitions_separate_and_invalidate_on_overflow() { + let observer = RuntimeErpObserver::new("http://127.0.0.1:1".into()); + let (generation, config) = fixture(); + observer.observe(&generation, coordinate(0), &config, 1, 1.0); + observer.observe(&generation, coordinate(0), &config, 2, 1.0); + observer.observe(&generation, coordinate(1), &config, 3, 2.0); + { + let state = observer.observations.lock().unwrap(); + assert_eq!(state.populations.len(), 2); + assert_eq!(state.total_keys, 2); + assert!(state + .populations + .values() + .all(|p| p.observer.observed_key_count() == 1)); + } + for group in 2..=MAX_POPULATIONS { + observer.observe(&generation, coordinate(group), &config, 4, 3.0); + } + let state = observer.observations.lock().unwrap(); + assert!(state.invalid.is_some()); + assert!(state + .populations + .values() + .all(|p| p.observer.snapshot().is_none())); + } + #[test] + fn changed_catalog_resets_invalid_observation_without_cross_generation_counts() { + let observer = RuntimeErpObserver::new("http://127.0.0.1:1".into()); + let (mut generation, config) = fixture(); + observer.observe(&generation, coordinate(0), &config, 1, f64::NAN); + generation.plan_version = 2; + observer.observe(&generation, coordinate(0), &config, 2, 9.0); + let state = observer.observations.lock().unwrap(); + assert!(state.invalid.is_none()); + assert_eq!(state.total_keys, 1); + assert_eq!(state.generation.as_ref(), Some(&generation)); + } +} From f033f5b24bccf810dee3dc72d4196df5d765a927 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 20:40:21 -0600 Subject: [PATCH 11/28] test(erp): exercise live worker observations through control-plane selection --- control_plane/src/physical/compiler.rs | 1 - .../src/precompute_engine/erp_observer.rs | 2 +- .../storage_engines/sketch_db/index/mod.rs | 4 + .../tests/support/univmon_erp_process.rs | 82 ++++++++++++++++++- 4 files changed, 86 insertions(+), 3 deletions(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 9aa011bdc..ab977d1ab 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -3159,7 +3159,6 @@ fn select_lifecycle( pub(crate) fn materialization_leaf_contract( node: &SummaryNode, ) -> Result<(String, Option, String), String> { - use planner_types::pre_asap::{CompareOpKind, QueryExpr, ScalarValue}; let SummaryExpr::SummaryAgg { child, .. } = &node.expr else { return Err("materialization requires a SummaryAgg leaf".into()); }; diff --git a/data_plane/src/precompute_engine/erp_observer.rs b/data_plane/src/precompute_engine/erp_observer.rs index e8bc37485..e31dca828 100644 --- a/data_plane/src/precompute_engine/erp_observer.rs +++ b/data_plane/src/precompute_engine/erp_observer.rs @@ -242,7 +242,7 @@ mod tests { String::new(), 60, 60, - planner_types::pre_asap::WindowKind::Tumbling, + asap_types::enums::WindowKind::Tumbling, String::new(), "m".into(), None, diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index 60411e9bd..ee127988f 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -1062,6 +1062,10 @@ impl SketchStore { .map_err(|_| "summary instance start exceeds signed timestamp range")?; let end_ms = i64::try_from(window.1) .map_err(|_| "summary instance end exceeds signed timestamp range")?; + let group_fingerprint = xxhash_rust::xxh64::xxh64( + &serde_json::to_vec(&group_values).map_err(|error| error.to_string())?, + 0, + ); let instance_id = asap_types::sds::SummaryInstanceCoordinates { summary_definition_id, time_range: asap_types::sds::HalfOpenTimeRange { start_ms, end_ms }, diff --git a/data_plane/tests/support/univmon_erp_process.rs b/data_plane/tests/support/univmon_erp_process.rs index a37bc601e..4737db8f1 100644 --- a/data_plane/tests/support/univmon_erp_process.rs +++ b/data_plane/tests/support/univmon_erp_process.rs @@ -195,12 +195,26 @@ async fn measured_readout_evidence_selects_and_executes_univmon() { .await .unwrap(); }); + let runtime_samples = control_plane::runtime_samples::RuntimeSamplesStore::new(8); + let runtime_port = unused_port(); + let runtime_endpoint = format!("http://127.0.0.1:{runtime_port}"); + let runtime_service = + control_plane::runtime_samples::RuntimeSamplesService::new(runtime_samples.clone()) + .into_server(); + let runtime_task = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service(runtime_service) + .serve(([127, 0, 0, 1], runtime_port).into()) + .await + .unwrap(); + }); let output = tempfile::tempdir().unwrap(); let path = output.path().join("planning.json"); std::fs::write(&path, serde_json::to_vec(&fixture).unwrap()).unwrap(); let port = unused_port(); let mut child = ChildGuard( Command::new(env!("CARGO_BIN_EXE_data_plane")) + .args(["--erp-runtime-samples-endpoint", &runtime_endpoint]) .args(["--profile", "asapquery", "--planning-snapshot"]) .arg(&path) .args([ @@ -231,11 +245,14 @@ async fn measured_readout_evidence_selects_and_executes_univmon() { .unwrap() .as_millis() as i64; let base = now - now.rem_euclid(5000) - 20_000; - let samples: Vec<_> = raw + let mut samples: Vec<_> = raw .iter() .enumerate() .map(|(i, v)| (base + 1 + i as i64, *v)) .collect(); + // Declared finite source includes the preceding boundary; this sample is + // outside the query's left-open range and does not alter its truth. + samples.insert(0, (base, 0.0)); assert_eq!( remote_write( &client, @@ -259,6 +276,69 @@ async fn measured_readout_evidence_selects_and_executes_univmon() { 204 ); drain_precompute(&client, &backend).await; + let keys = runtime_samples.keys(); + assert!( + !keys.is_empty(), + "real worker inputs must reach RuntimeSamples after finite drain" + ); + for key in keys { + let record = runtime_samples.latest(&key).unwrap(); + let observed: asap_types::erp_observation::ErpPopulationObservations< + control_plane::physical::erp::ErpObservedShape, + > = serde_json::from_value(record.payload["erp_population_observations"].clone()).unwrap(); + assert!(observed.invalid_reason.is_none(), "{observed:?}"); + assert!(!observed.populations.is_empty()); + assert_eq!(observed.window_end_ms - observed.window_start_ms, 5000); + assert!(plan + .summary_catalog + .materializations + .contains_key(&observed.summary_definition_id)); + assert_eq!( + observed.catalog_generation, + plan.summary_catalog.reference().unwrap() + ); + if key.sketch == "univmon" { + let mut live_snapshot: BackendLocalPlanningSnapshot = + serde_json::from_value(fixture.clone()).unwrap(); + let policy = live_snapshot.implementation.erp.as_mut().unwrap(); + policy.observed_shape_source = + Some(control_plane::physical::erp::ErpObservedShapeSource { + source: key.source.clone(), + sketch: key.sketch.clone(), + implementation: key.impl_name.clone(), + population_scope: Some( + control_plane::physical::erp::ErpPopulationObservationScope { + catalog_generation: observed.catalog_generation.clone(), + summary_definition_id: observed.summary_definition_id, + input_semantics: observed.input_semantics, + freshness: asap_types::erp_observation::ErpObservationFreshness { + max_age_ms: 60_000, + max_future_skew_ms: 1000, + }, + }, + ), + }); + policy.hydrate_observed_shape(&runtime_samples).unwrap(); + policy.resolve_population_data_descriptor(Some(&plan.summary_catalog)); + assert!(policy + .observed_populations + .as_ref() + .unwrap() + .invalid_reason + .is_none()); + let replanned = live_snapshot.compile().unwrap(); + assert!( + replanned + .precompute_plan + .materializations + .iter() + .any(|m| m.aggregation_type == asap_types::AggregationType::UnivMon), + "actual producer evidence should reach normal Planner selection" + ); + } + } + runtime_task.abort(); + for (i, query) in queries.iter().enumerate() { let result = wait_for_warm_instant( &client, From 190ee925181045b608827395ca3809a73e52d0df Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 20:42:04 -0600 Subject: [PATCH 12/28] fix(erp): bound retained population metadata bytes --- .../src/precompute_engine/erp_observer.rs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/data_plane/src/precompute_engine/erp_observer.rs b/data_plane/src/precompute_engine/erp_observer.rs index e31dca828..766bf4103 100644 --- a/data_plane/src/precompute_engine/erp_observer.rs +++ b/data_plane/src/precompute_engine/erp_observer.rs @@ -8,6 +8,7 @@ use std::sync::{Arc, Mutex}; const MAX_POPULATIONS: usize = 128; const MAX_KEYS: usize = 65_536; +const MAX_POPULATION_METADATA_BYTES: usize = 1_048_576; struct Population { coordinates: SummaryInstanceCoordinates, @@ -23,6 +24,7 @@ struct Observations { populations: BTreeMap, extent: BTreeMap, total_keys: usize, + metadata_bytes: usize, invalid: Option, } @@ -97,6 +99,25 @@ impl RuntimeErpObserver { } return; } + if !state.populations.contains_key(&id) { + let bytes = coordinates + .group_values + .iter() + .try_fold(0usize, |total, (key, value)| { + total + .checked_add(key.len()) + .and_then(|n| n.checked_add(value.len())) + }); + let total = bytes.and_then(|bytes| state.metadata_bytes.checked_add(bytes)); + if total.is_none_or(|bytes| bytes > MAX_POPULATION_METADATA_BYTES) { + state.invalid = Some("ERP population metadata budget exceeded".into()); + for population in state.populations.values_mut() { + population.observer = ErpShapeObserver::new(1).unwrap(); + } + return; + } + state.metadata_bytes = total.unwrap(); + } let population = state.populations.entry(id).or_insert_with(|| Population { source: format!( "summary-definition:{}", From 559f6ca5a777216f42fe69f3099bac94f24ce03c Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 20:45:16 -0600 Subject: [PATCH 13/28] fix(erp): validate observation semantics against catalog operators --- control_plane/src/physical/compiler.rs | 1 + control_plane/src/physical/erp.rs | 24 ++++++++++++++++++++++++ crates/asap_types/src/sds.rs | 2 ++ crates/asap_types/src/summary_catalog.rs | 2 +- 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index ab977d1ab..b3482731d 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -2506,6 +2506,7 @@ fn observed_population_matches_root( && data.value_projection == ValueProjectionIdentity::SampleValue && data.partitioning == Some(PopulationPartitioning::PerEntity) && data.group_by_keys.is_empty() + && data.observation_semantics == asap_types::sds::TIMESTAMPED_OBSERVATION_SEMANTICS && observed.window_end_ms.checked_sub(observed.window_start_ms) == i64::try_from(window.saturating_mul(1000)).ok() } diff --git a/control_plane/src/physical/erp.rs b/control_plane/src/physical/erp.rs index ec4e035fb..90ad0d069 100644 --- a/control_plane/src/physical/erp.rs +++ b/control_plane/src/physical/erp.rs @@ -426,6 +426,30 @@ impl ErpPlanningInput { .materializations .get(&populations.summary_definition_id) .ok_or("ERP evidence summary is absent from the active catalog")?; + let summary = catalog + .summary_descriptors + .get(&materialization.summary_descriptor_id) + .ok_or("ERP summary descriptor is absent from the active catalog")?; + let actual_semantics = match &summary.operator { + asap_types::sds::SummaryOperator::Configured { + aggregation_type: asap_types::AggregationType::HLL, + .. + } => Some( + asap_types::erp_observation::ErpObservationInputSemantics::ScalarSampleValue, + ), + asap_types::sds::SummaryOperator::Configured { + aggregation_type: asap_types::AggregationType::UnivMon, + .. + } => Some( + asap_types::erp_observation::ErpObservationInputSemantics::UnitSampleFrequency, + ), + _ => None, + }; + if actual_semantics != Some(populations.input_semantics) { + return Err( + "ERP observation semantics differ from the installed summary operator".into(), + ); + } let data = catalog .data_descriptors .get(&materialization.data_descriptor_id) diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index 7720cf60a..94a7924fd 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -1,5 +1,7 @@ //! Shared SDS metadata contracts. Summary payload bytes remain storage-engine //! owned; catalogs and inventories contain identities and state references only. +pub const TIMESTAMPED_OBSERVATION_SEMANTICS: &str = "asap.timestamped-observations.v2"; + use crate::{AggregationType, PrecomputeMaterialization}; use planner_types::post_asap::{SketchAlgorithm, SketchParams, SummaryFamilyType}; use serde::{Deserialize, Serialize}; diff --git a/crates/asap_types/src/summary_catalog.rs b/crates/asap_types/src/summary_catalog.rs index 4febef66b..c7bc75c1e 100644 --- a/crates/asap_types/src/summary_catalog.rs +++ b/crates/asap_types/src/summary_catalog.rs @@ -124,7 +124,7 @@ impl SummaryCatalog { .population_filter_canonical() .map_err(SummaryCatalogError::Descriptor)?, config.grouping_labels.labels.clone(), - "asap.timestamped-observations.v2", + crate::sds::TIMESTAMPED_OBSERVATION_SEMANTICS, ) .with_partitioning(config.partitioning) .with_timestamp_column(config.table_timestamp_column.clone()); From 6168be5f313f926e9e587aac036cb6c33720e5f2 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 20:50:45 -0600 Subject: [PATCH 14/28] fix(erp): apportion update demand across observed populations --- control_plane/src/physical/erp.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/control_plane/src/physical/erp.rs b/control_plane/src/physical/erp.rs index 90ad0d069..9ca65e0e2 100644 --- a/control_plane/src/physical/erp.rs +++ b/control_plane/src/physical/erp.rs @@ -676,11 +676,22 @@ impl ErpPlanningInput { let mut policy = self.clone(); policy.observed_populations = None; policy.mode = ErpAccuracyMode::Empirical; + let total_events: f64 = populations + .populations + .iter() + .map(|p| p.shape.observation.observed_events as f64) + .sum(); let mut errors: f64 = 0.0; let mut cost = 0.0; let mut evidence = Vec::new(); let valid = populations.populations.iter().all(|population| { policy.observed_shape = Some(population.shape.observation.clone()); + // expected_updates is the definition-wide demand; distribute + // it across partitions instead of charging it once per series. + policy.expected_updates = self.expected_updates + * population.shape.observation.observed_events as f64 + / total_events; + match policy.select_metric( algorithm.clone(), error_metric, From d87d30a2d8589035e5f15e342c231c72d753be84 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 21:03:12 -0600 Subject: [PATCH 15/28] fix(erp): bind observations to catalog generations and fit only in control plane --- control_plane/src/physical/erp.rs | 140 ++++++++-------- crates/asap_types/src/erp_observation.rs | 150 ++++++++++++++++++ data_plane/src/main.rs | 9 +- .../src/precompute_engine/erp_observer.rs | 64 +++++--- .../src/precompute_engine/series_router.rs | 10 +- .../tests/support/univmon_erp_process.rs | 2 +- 6 files changed, 271 insertions(+), 104 deletions(-) diff --git a/control_plane/src/physical/erp.rs b/control_plane/src/physical/erp.rs index 9ca65e0e2..a679a607a 100644 --- a/control_plane/src/physical/erp.rs +++ b/control_plane/src/physical/erp.rs @@ -7,7 +7,7 @@ use asap_aware_mapping::erp::{ use planner_types::post_asap::{SketchAlgorithm, SketchParams}; use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::collections::{BTreeMap, HashMap}; +use std::collections::BTreeMap; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] @@ -22,74 +22,40 @@ pub struct ErpObservedShape { /// instead of silently under-reporting cardinality. #[derive(Debug)] pub struct ErpShapeObserver { - frequencies: HashMap, - interval_updates: HashMap, - max_observed_keys: usize, - max_observed_intervals: usize, - updates: u64, - invalid: bool, + inner: asap_types::erp_observation::BoundedFrequencyObserver, } - impl ErpShapeObserver { pub fn observed_key_count(&self) -> usize { - self.frequencies.len() + self.inner.observed_key_count() } - - pub fn new(max_observed_keys: usize) -> Result { - Self::with_limits(max_observed_keys, 256) + pub fn new(max_keys: usize) -> Result { + Self::with_limits(max_keys, 256) } - - pub fn with_limits( - max_observed_keys: usize, - max_observed_intervals: usize, - ) -> Result { - if max_observed_keys == 0 || max_observed_intervals == 0 { - return Err("observation limits must be positive"); - } + pub fn with_limits(max_keys: usize, max_intervals: usize) -> Result { Ok(Self { - frequencies: HashMap::new(), - interval_updates: HashMap::new(), - max_observed_keys, - max_observed_intervals, - updates: 0, - invalid: false, + inner: asap_types::erp_observation::BoundedFrequencyObserver::new( + max_keys, + max_intervals, + )?, }) } - pub fn observe(&mut self, key: &str, interval: usize) -> Result<(), &'static str> { - if self.invalid { - return Err("ERP observation was invalidated; start a fresh observation window"); - } if key.len() > 4096 { - self.invalid = true; + self.inner.invalidate(); return Err("ERP shape observer key size exceeded"); } - if !self.frequencies.contains_key(key) && self.frequencies.len() == self.max_observed_keys { - self.invalid = true; - return Err("ERP shape observer cardinality cap exceeded"); - } - if !self.interval_updates.contains_key(&interval) - && self.interval_updates.len() == self.max_observed_intervals - { - self.invalid = true; - return Err("ERP shape observer interval cap exceeded"); - } - let Some(updates) = self.updates.checked_add(1) else { - self.invalid = true; - return Err("ERP shape observer count overflow"); - }; - *self.frequencies.entry(key.to_owned()).or_default() += 1; - *self.interval_updates.entry(interval).or_default() += 1; - self.updates = updates; - Ok(()) + self.inner.observe(key.to_owned(), interval) } - pub fn snapshot(&self) -> Option { - if self.frequencies.is_empty() || self.invalid { - return None; - } - let mut counts: Vec<_> = self.frequencies.values().copied().collect(); - counts.sort_unstable_by(|left, right| right.cmp(left)); + ErpObservedShape::from_empirical(&self.inner.snapshot()?) + } +} +impl ErpObservedShape { + pub fn from_empirical( + empirical: &asap_types::erp_observation::EmpiricalFrequencyObservation, + ) -> Option { + let events = empirical.event_count()?; + let counts = &empirical.sorted_counts; let exponent = fit_zipf_exponent(&counts); let fits = [("uniform", 0.0), ("zipf", exponent)] .into_iter() @@ -104,7 +70,7 @@ impl ErpShapeObserver { .iter() .zip(expected) .map(|(count, expected)| { - (*count as f64 / self.updates as f64 - expected / total).abs() + (*count as f64 / events as f64 - expected / total).abs() }) .sum::() / 2.0; @@ -117,13 +83,13 @@ impl ErpShapeObserver { }, goodness_of_fit: distance, // A fit-quality score, not an estimator tail-probability claim. - confidence: (1.0 - distance) * (1.0 - 1.0 / (self.updates as f64).sqrt()), + confidence: (1.0 - distance) * (1.0 - 1.0 / (events as f64).sqrt()), } }) .collect(); - let mut nonzero: Vec<_> = self - .interval_updates - .values() + let mut nonzero: Vec<_> = empirical + .interval_counts + .iter() .copied() .filter(|count| *count > 0) .collect(); @@ -132,8 +98,8 @@ impl ErpShapeObserver { let peak = nonzero.last().copied().unwrap_or(median); Some(ErpObservedShape { observation: ErpShapeObservation { - cardinality: self.frequencies.len() as u64, - observed_events: self.updates, + cardinality: counts.len() as u64, + observed_events: events, fits, empirical_fingerprint: None, }, @@ -506,7 +472,7 @@ impl ErpPlanningInput { .get("erp_population_observations") .ok_or("latest runtime sample has no erp_population_observations")?; let observed: asap_types::erp_observation::ErpPopulationObservations< - ErpObservedShape, + asap_types::erp_observation::EmpiricalFrequencyObservation, > = serde_json::from_value(value.clone()) .map_err(|error| format!("invalid ERP population observations: {error}"))?; observed.validate_identity_and_freshness( @@ -518,7 +484,9 @@ impl ErpPlanningInput { if observed.input_semantics != scope.input_semantics { return Err("ERP observation input semantics do not match the summary".into()); } - Ok(observed) + observed + .try_map_shapes(|empirical| ErpObservedShape::from_empirical(&empirical)) + .ok_or_else(|| "invalid bounded empirical frequency observations".to_owned()) })(); // An explicit invalid envelope prevents selection from borrowing an // older fit or falling back to the artifact's distribution identity. @@ -1415,8 +1383,21 @@ mod tests { fn publish_population_fixture( samples: &crate::runtime_samples::RuntimeSamplesStore, - payload: serde_json::Value, + mut payload: serde_json::Value, ) { + if let Some(populations) = payload + .pointer_mut("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/erp_population_observations/populations") + .and_then(Value::as_array_mut) + { + for population in populations { + if let Some(cardinality) = population + .pointer("/shape/observation/cardinality") + .and_then(Value::as_u64) + { + population["shape"] = serde_json::json!({"sorted_counts":vec![2;cardinality as usize],"interval_counts":[cardinality*2]}); + } + } + } samples.append_for_test(crate::runtime_samples::RuntimeRecord { source: "backend-a".into(), sketch: "univmon".into(), @@ -1468,16 +1449,23 @@ mod tests { /// Only the activated catalog may supply a candidate's data contract. #[test] fn online_population_descriptor_requires_authoritative_catalog() { + let mut fixture: Value = serde_json::from_str(include_str!( + "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let mut query = fixture["query_workload"]["repeating_queries"][3].clone(); + query["query"] = "distinct_over_time(asap_demo_latency_ms[5s])".into(); + query["requirements"]["accuracy"] = serde_json::json!({"explicit":{"Epsilon":0.05}}); + fixture["query_workload"]["repeating_queries"] = serde_json::json!([query]); let snapshot: crate::physical::compiler::BackendLocalPlanningSnapshot = - serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" - )) - .unwrap(); + serde_json::from_value(fixture).unwrap(); let plan = snapshot.compile().unwrap(); let (mut policy, mut observed) = online_population_fixture(); observed.catalog_generation = plan.summary_catalog.reference().unwrap(); observed.summary_definition_id = *plan.summary_catalog.materializations.keys().next().unwrap(); + observed.input_semantics = + asap_types::erp_observation::ErpObservationInputSemantics::ScalarSampleValue; policy.observed_populations = Some(observed.clone()); policy.resolve_population_data_descriptor(Some(&plan.summary_catalog)); let expected = &plan.summary_catalog.materializations[&observed.summary_definition_id] @@ -1563,14 +1551,16 @@ mod tests { let mut observer = ErpShapeObserver::with_limits(2, 2).unwrap(); observer.observe("a", usize::MAX).unwrap(); observer.observe("a", 0).unwrap(); - assert_eq!(observer.interval_updates.len(), 2); + assert_eq!(observer.inner.snapshot().unwrap().interval_counts.len(), 2); assert!(observer.observe("a", 1).is_err()); assert!(observer.snapshot().is_none()); - let mut observer = ErpShapeObserver::new(2).unwrap(); - observer.observe("a", 0).unwrap(); - observer.updates = u64::MAX; - assert!(observer.observe("a", 0).is_err()); - assert!(observer.snapshot().is_none()); + assert!(ErpObservedShape::from_empirical( + &asap_types::erp_observation::EmpiricalFrequencyObservation { + sorted_counts: vec![u64::MAX, 1], + interval_counts: vec![u64::MAX, 1], + } + ) + .is_none()); } #[test] diff --git a/crates/asap_types/src/erp_observation.rs b/crates/asap_types/src/erp_observation.rs index efb693c05..7693a3c72 100644 --- a/crates/asap_types/src/erp_observation.rs +++ b/crates/asap_types/src/erp_observation.rs @@ -77,3 +77,153 @@ impl ErpPopulationObservations { Ok(()) } } + +/// Empirical sufficient statistics for frequency-shape fitting. No raw keys, +/// raw samples, distribution-family assertion, or planner configuration crosses the wire. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EmpiricalFrequencyObservation { + pub sorted_counts: Vec, + pub interval_counts: Vec, +} +impl EmpiricalFrequencyObservation { + pub fn event_count(&self) -> Option { + if self.sorted_counts.len() > 65_536 + || self.interval_counts.len() > 256 + || self.sorted_counts.is_empty() + || self.sorted_counts.iter().any(|n| *n == 0) + || self.sorted_counts.windows(2).any(|w| w[0] < w[1]) + { + return None; + } + let total = self + .sorted_counts + .iter() + .try_fold(0u64, |sum, n| sum.checked_add(*n))?; + let intervals = self + .interval_counts + .iter() + .try_fold(0u64, |sum, n| sum.checked_add(*n))?; + (total == intervals).then_some(total) + } +} + +/// Bounded exact frequency counting. Fitting and ERP selection remain control-plane work. +#[derive(Debug)] +pub struct BoundedFrequencyObserver { + frequencies: std::collections::HashMap, + intervals: std::collections::HashMap, + max_keys: usize, + max_intervals: usize, + events: u64, + invalid: bool, +} +impl BoundedFrequencyObserver { + pub fn new(max_keys: usize, max_intervals: usize) -> Result { + if max_keys == 0 || max_intervals == 0 { + return Err("observation limits must be positive"); + } + Ok(Self { + frequencies: Default::default(), + intervals: Default::default(), + max_keys, + max_intervals, + events: 0, + invalid: false, + }) + } + pub fn observed_key_count(&self) -> usize { + self.frequencies.len() + } + pub fn invalidate(&mut self) { + self.invalid = true; + } + pub fn observe(&mut self, key: K, interval: usize) -> Result<(), &'static str> { + if self.invalid { + return Err("ERP observation was invalidated"); + } + if (!self.frequencies.contains_key(&key) && self.frequencies.len() >= self.max_keys) + || (!self.intervals.contains_key(&interval) + && self.intervals.len() >= self.max_intervals) + { + self.invalid = true; + return Err("ERP observation key or interval budget exceeded"); + } + let Some(events) = self.events.checked_add(1) else { + self.invalid = true; + return Err("ERP observation count overflow"); + }; + *self.frequencies.entry(key).or_default() += 1; + *self.intervals.entry(interval).or_default() += 1; + self.events = events; + Ok(()) + } + pub fn snapshot(&self) -> Option { + if self.invalid || self.frequencies.is_empty() { + return None; + } + let mut sorted_counts: Vec<_> = self.frequencies.values().copied().collect(); + sorted_counts.sort_unstable_by(|a, b| b.cmp(a)); + Some(EmpiricalFrequencyObservation { + sorted_counts, + interval_counts: self.intervals.values().copied().collect(), + }) + } +} + +impl ErpPopulationObservations { + pub fn try_map_shapes( + self, + mut convert: impl FnMut(S) -> Option, + ) -> Option> { + let populations = self + .populations + .into_iter() + .map(|population| { + Some(ErpPopulationObservation { + population_id: population.population_id, + shape: convert(population.shape)?, + }) + }) + .collect::>>()?; + Some(ErpPopulationObservations { + schema_version: self.schema_version, + catalog_generation: self.catalog_generation, + summary_definition_id: self.summary_definition_id, + observed_at_unix_ms: self.observed_at_unix_ms, + window_start_ms: self.window_start_ms, + window_end_ms: self.window_end_ms, + input_semantics: self.input_semantics, + invalid_reason: self.invalid_reason, + populations, + }) + } +} + +#[cfg(test)] +mod empirical_tests { + use super::*; + #[test] + fn count_overflow_invalidates_instead_of_publishing_prefix() { + let mut observer = BoundedFrequencyObserver::new(2, 2).unwrap(); + observer.observe(1u64, 0).unwrap(); + observer.events = u64::MAX; + assert!(observer.observe(1, 0).is_err()); + assert!(observer.snapshot().is_none()); + } + #[test] + fn malformed_frequency_counts_fail_closed() { + assert!(EmpiricalFrequencyObservation { + sorted_counts: vec![2, 1], + interval_counts: vec![2] + } + .event_count() + .is_none()); + assert!(EmpiricalFrequencyObservation { + sorted_counts: vec![1, 2], + interval_counts: vec![3] + } + .event_count() + .is_none()); + } +} diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 208b7db50..4c8204104 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -878,10 +878,17 @@ async fn main() -> Result<()> { sketch_index.clone(), ); if let Some(endpoint) = args.erp_runtime_samples_endpoint.clone() { + let generation = engine + .ingest_state() + .physical_plan_snapshot() + .and_then(|plan| plan.precompute_plan.summary_catalog.clone()) + .ok_or_else(|| { + std::io::Error::other("ERP observation requires an installed catalog") + })?; engine .ingest_state() .router - .enable_erp_observation(endpoint) + .enable_erp_observation(endpoint, generation) .map_err(std::io::Error::other)?; } let worker_diagnostics = engine.diagnostics(); diff --git a/data_plane/src/precompute_engine/erp_observer.rs b/data_plane/src/precompute_engine/erp_observer.rs index 766bf4103..25fb6ffcf 100644 --- a/data_plane/src/precompute_engine/erp_observer.rs +++ b/data_plane/src/precompute_engine/erp_observer.rs @@ -2,7 +2,6 @@ //! an explicit finite-source completion barrier. No worker timestamp is a seal. use asap_types::erp_observation::*; use asap_types::sds::*; -use control_plane::physical::erp::{ErpObservedShape, ErpShapeObserver}; use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; @@ -16,7 +15,7 @@ struct Population { source: String, sketch: String, implementation: String, - observer: ErpShapeObserver, + observer: BoundedFrequencyObserver, } #[derive(Default)] struct Observations { @@ -33,10 +32,13 @@ pub struct RuntimeErpObserver { observations: Mutex, } impl RuntimeErpObserver { - pub fn new(endpoint: String) -> Arc { + pub fn new(endpoint: String, generation: CatalogGeneration) -> Arc { Arc::new(Self { endpoint, - observations: Mutex::new(Observations::default()), + observations: Mutex::new(Observations { + generation: Some(generation), + ..Default::default() + }), }) } pub fn observe( @@ -62,11 +64,10 @@ impl RuntimeErpObserver { _ => return, }; let mut state = self.observations.lock().unwrap(); + // Catalog installation establishes identity. Delayed old worker input + // cannot reset the current generation or create a partial replacement fit. if state.generation.as_ref() != Some(generation) { - *state = Observations { - generation: Some(generation.clone()), - ..Default::default() - }; + return; } if state.invalid.is_some() { return; @@ -79,7 +80,7 @@ impl RuntimeErpObserver { { state.invalid = Some("unsupported non-finite or transformed summary input".into()); for population in state.populations.values_mut() { - population.observer = ErpShapeObserver::new(1).unwrap(); + population.observer = BoundedFrequencyObserver::new(1, 64).unwrap(); } return; } @@ -95,7 +96,7 @@ impl RuntimeErpObserver { if !state.populations.contains_key(&id) && state.populations.len() >= MAX_POPULATIONS { state.invalid = Some("ERP population observation budget exceeded".into()); for population in state.populations.values_mut() { - population.observer = ErpShapeObserver::new(1).unwrap(); + population.observer = BoundedFrequencyObserver::new(1, 64).unwrap(); } return; } @@ -112,7 +113,7 @@ impl RuntimeErpObserver { if total.is_none_or(|bytes| bytes > MAX_POPULATION_METADATA_BYTES) { state.invalid = Some("ERP population metadata budget exceeded".into()); for population in state.populations.values_mut() { - population.observer = ErpShapeObserver::new(1).unwrap(); + population.observer = BoundedFrequencyObserver::new(1, 64).unwrap(); } return; } @@ -127,18 +128,18 @@ impl RuntimeErpObserver { semantics, sketch: sketch.into(), implementation: implementation.into(), - observer: ErpShapeObserver::new(MAX_KEYS).unwrap(), + observer: BoundedFrequencyObserver::new(MAX_KEYS, 64).unwrap(), }); let before = population.observer.observed_key_count(); // Canonical fixed-width numeric identity; no raw sample or arbitrary label copy. - let key = format!("{:016x}", if value == 0.0 { 0 } else { value.to_bits() }); + let key = if value == 0.0 { 0 } else { value.to_bits() }; let range = population.coordinates.time_range; let duration = range.end_ms.saturating_sub(range.start_ms).max(1); let offset = timestamp_ms .saturating_sub(range.start_ms) .clamp(0, duration); let interval = ((offset as u128 * 64) / duration as u128).min(63) as usize; - let result = population.observer.observe(&key, interval); + let result = population.observer.observe(key, interval); let added = population .observer .observed_key_count() @@ -147,7 +148,7 @@ impl RuntimeErpObserver { if result.is_err() || state.total_keys > MAX_KEYS { state.invalid = Some("ERP key observation budget exceeded".into()); for population in state.populations.values_mut() { - population.observer = ErpShapeObserver::new(1).unwrap(); + population.observer = BoundedFrequencyObserver::new(1, 64).unwrap(); } } } @@ -171,7 +172,7 @@ impl RuntimeErpObserver { String, String, String, - ErpPopulationObservations, + ErpPopulationObservations, ), > = BTreeMap::new(); for (id, population) in &state.populations { @@ -292,8 +293,8 @@ mod tests { } #[test] fn observations_keep_partitions_separate_and_invalidate_on_overflow() { - let observer = RuntimeErpObserver::new("http://127.0.0.1:1".into()); let (generation, config) = fixture(); + let observer = RuntimeErpObserver::new("http://127.0.0.1:1".into(), generation.clone()); observer.observe(&generation, coordinate(0), &config, 1, 1.0); observer.observe(&generation, coordinate(0), &config, 2, 1.0); observer.observe(&generation, coordinate(1), &config, 3, 2.0); @@ -317,15 +318,28 @@ mod tests { .all(|p| p.observer.snapshot().is_none())); } #[test] - fn changed_catalog_resets_invalid_observation_without_cross_generation_counts() { - let observer = RuntimeErpObserver::new("http://127.0.0.1:1".into()); - let (mut generation, config) = fixture(); - observer.observe(&generation, coordinate(0), &config, 1, f64::NAN); - generation.plan_version = 2; - observer.observe(&generation, coordinate(0), &config, 2, 9.0); + fn delayed_generation_cannot_reset_current_population_counts() { + let (generation, config) = fixture(); + let observer = RuntimeErpObserver::new("http://127.0.0.1:1".into(), generation.clone()); + observer.observe(&generation, coordinate(0), &config, 1, 1.0); + let mut stale = generation.clone(); + stale.plan_version = 0; + observer.observe(&stale, coordinate(0), &config, 2, 99.0); + observer.observe(&generation, coordinate(0), &config, 3, 2.0); let state = observer.observations.lock().unwrap(); - assert!(state.invalid.is_none()); - assert_eq!(state.total_keys, 1); + assert_eq!(state.total_keys, 2); assert_eq!(state.generation.as_ref(), Some(&generation)); + assert_eq!( + state + .populations + .values() + .next() + .unwrap() + .observer + .snapshot() + .unwrap() + .sorted_counts, + vec![1, 1] + ); } } diff --git a/data_plane/src/precompute_engine/series_router.rs b/data_plane/src/precompute_engine/series_router.rs index 3a90705d8..b709d9e6f 100644 --- a/data_plane/src/precompute_engine/series_router.rs +++ b/data_plane/src/precompute_engine/series_router.rs @@ -163,9 +163,15 @@ impl SeriesRouter { } } - pub fn enable_erp_observation(&self, endpoint: String) -> Result<(), String> { + pub fn enable_erp_observation( + &self, + endpoint: String, + generation: asap_types::sds::CatalogGeneration, + ) -> Result<(), String> { self.erp_observer - .set(super::erp_observer::RuntimeErpObserver::new(endpoint)) + .set(super::erp_observer::RuntimeErpObserver::new( + endpoint, generation, + )) .map_err(|_| "ERP observer already configured".into()) } pub fn erp_observer(&self) -> Option> { diff --git a/data_plane/tests/support/univmon_erp_process.rs b/data_plane/tests/support/univmon_erp_process.rs index 4737db8f1..9c933b8cb 100644 --- a/data_plane/tests/support/univmon_erp_process.rs +++ b/data_plane/tests/support/univmon_erp_process.rs @@ -284,7 +284,7 @@ async fn measured_readout_evidence_selects_and_executes_univmon() { for key in keys { let record = runtime_samples.latest(&key).unwrap(); let observed: asap_types::erp_observation::ErpPopulationObservations< - control_plane::physical::erp::ErpObservedShape, + asap_types::erp_observation::EmpiricalFrequencyObservation, > = serde_json::from_value(record.payload["erp_population_observations"].clone()).unwrap(); assert!(observed.invalid_reason.is_none(), "{observed:?}"); assert!(!observed.populations.is_empty()); From 9d98460347226ea1187dc8a45adb8cea4530c017 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 21:10:51 -0600 Subject: [PATCH 16/28] fix(erp): reset observation epoch only on catalog activation --- control_plane/src/physical/erp.rs | 2 +- .../drivers/ingest/prometheus_remote_write.rs | 9 +++++++++ data_plane/src/drivers/query/servers/http.rs | 15 ++++++++++++--- data_plane/src/precompute_engine/erp_observer.rs | 16 ++++++++++++++++ .../univmon-erp-process-validation.md | 7 +++++++ 5 files changed, 45 insertions(+), 4 deletions(-) diff --git a/control_plane/src/physical/erp.rs b/control_plane/src/physical/erp.rs index a679a607a..c41fa2fe2 100644 --- a/control_plane/src/physical/erp.rs +++ b/control_plane/src/physical/erp.rs @@ -1540,7 +1540,7 @@ mod tests { observer.observe("a", 0).unwrap(); assert_eq!( observer.observe("b", 0), - Err("ERP shape observer cardinality cap exceeded") + Err("ERP observation key or interval budget exceeded") ); assert!(observer.snapshot().is_none()); assert!(observer.observe("a", 0).is_err()); diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index 7deb61e22..bb8181d29 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -177,6 +177,15 @@ impl PrometheusRemoteWriteReceiver { self.inner.stats.clone() } + pub(crate) fn install_erp_observation_generation( + &self, + generation: asap_types::sds::CatalogGeneration, + ) { + if let Some(observer) = self.inner.ingest.router.erp_observer() { + observer.install_generation(generation); + } + } + /// Permanently seal this finite source before queuing worker barriers. pub async fn drain(&self) -> Result<(), String> { { diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 087b3d586..ff66bc12f 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -6263,14 +6263,23 @@ async fn handle_activate_physical_plan( }; let _guard = state.physical_plan_lock.lock().await; let store = Arc::clone(&state.sketch_index); + let remote_write = state.remote_write.clone(); let old = match lifecycle.activate_with_prepare( request.plan_id, request.plan_version, unix_time_ms(), move |plan| match plan.summary_catalog.as_ref() { - Some(catalog) => store - .install_summary_catalog(Arc::clone(catalog)) - .map_err(|error| format!("SummaryCatalog install error: {error}")), + Some(catalog) => { + store + .install_summary_catalog(Arc::clone(catalog)) + .map_err(|error| format!("SummaryCatalog install error: {error}"))?; + if let Some(receiver) = &remote_write { + receiver.install_erp_observation_generation( + catalog.reference().map_err(|error| error.to_string())?, + ); + } + Ok(()) + } None => Err("authoritative SummaryCatalog is unavailable".to_string()), }, ) { diff --git a/data_plane/src/precompute_engine/erp_observer.rs b/data_plane/src/precompute_engine/erp_observer.rs index 25fb6ffcf..47593276b 100644 --- a/data_plane/src/precompute_engine/erp_observer.rs +++ b/data_plane/src/precompute_engine/erp_observer.rs @@ -41,6 +41,17 @@ impl RuntimeErpObserver { }), }) } + /// Only the accepted catalog activation path may reset an observation epoch. + pub(crate) fn install_generation(&self, generation: CatalogGeneration) { + let mut state = self.observations.lock().unwrap(); + if state.generation.as_ref() != Some(&generation) { + *state = Observations { + generation: Some(generation), + ..Default::default() + }; + } + } + pub fn observe( &self, generation: &CatalogGeneration, @@ -321,6 +332,11 @@ mod tests { fn delayed_generation_cannot_reset_current_population_counts() { let (generation, config) = fixture(); let observer = RuntimeErpObserver::new("http://127.0.0.1:1".into(), generation.clone()); + let mut previous = generation.clone(); + previous.plan_version = 0; + observer.install_generation(previous.clone()); + observer.observe(&previous, coordinate(0), &config, 0, 77.0); + observer.install_generation(generation.clone()); observer.observe(&generation, coordinate(0), &config, 1, 1.0); let mut stale = generation.clone(); stale.plan_version = 0; diff --git a/docs/developer_docs/univmon-erp-process-validation.md b/docs/developer_docs/univmon-erp-process-validation.md index b3e5849ef..51e65f914 100644 --- a/docs/developer_docs/univmon-erp-process-validation.md +++ b/docs/developer_docs/univmon-erp-process-validation.md @@ -69,3 +69,10 @@ The three queries need different physical configurations in this fixture; they do not demonstrate a single shared state. Planner may share compatible equal- parameter states. The finite replay barrier also does not establish continuous multi-worker population completeness, which is a separate publication contract. + +Finite runtime observations carry bounded empirical frequency counts and temporal +interval counts through the shared metadata contract. The data plane does not +classify distributions or depend on Planner fitting types; the control plane +fits these counts before profile matching. Raw keys are absent from feedback. +An accepted catalog activation establishes the observer generation. Delayed +samples from another generation are ignored and cannot reset current counts. From 911a472c1e8e0d7941b023fb55145ba7408c9c0a Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 11 Sep 2026 06:56:53 -0600 Subject: [PATCH 17/28] fix(storage): make completed summary windows immutable --- .../sketch_db/index/admission.rs | 38 +++-- .../storage_engines/sketch_db/index/mod.rs | 140 +++++++++++++++++- .../sketch_db/persistence/metadata.rs | 10 ++ .../summary-catalog-sds-architecture.md | 20 +++ 4 files changed, 193 insertions(+), 15 deletions(-) diff --git a/data_plane/src/storage_engines/sketch_db/index/admission.rs b/data_plane/src/storage_engines/sketch_db/index/admission.rs index f0d6eceed..19c325108 100644 --- a/data_plane/src/storage_engines/sketch_db/index/admission.rs +++ b/data_plane/src/storage_engines/sketch_db/index/admission.rs @@ -22,7 +22,7 @@ pub(super) struct AdmissionInventory { replay_floors: BTreeMap, observed_extent: Option, finite_complete: bool, - published_series: BTreeSet, + published_series: BTreeMap, pending_revisions: usize, } @@ -182,7 +182,7 @@ impl AdmissionInventory { if self.generation.as_ref() != Some(generation) { return Err("summary series publication catalog generation differs".into()); } - if !self.published_series.contains(&series_id) + if !self.published_series.contains_key(&series_id) && self.published_series.len() >= Self::MAX_WINDOWS { return Err("summary admission series capacity exceeded".into()); @@ -198,11 +198,16 @@ impl AdmissionInventory { return Err("summary coordinate changed series identity".into()); } window.series_id = Some(series_id); - self.published_series.insert(series_id); + let end = u64::try_from(coordinate.time_range.end_ms) + .map_err(|_| "published window end is outside storage timestamp range")?; + self.published_series + .entry(series_id) + .and_modify(|current| *current = (*current).max(end)) + .or_insert(end); Ok(()) } - pub(super) fn seal_finite(&mut self, generation: &CatalogGeneration) -> Result<(), String> { + pub(super) fn validate_finite(&self, generation: &CatalogGeneration) -> Result { if self.generation.as_ref() != Some(generation) { return Err("finite completion catalog generation differs".into()); } @@ -213,11 +218,15 @@ impl AdmissionInventory { { return Err("finite source has unpublished summary windows".into()); } - self.finite_complete = true; - self.revision = self - .revision + self.revision .checked_add(1) - .ok_or("summary admission revision exhausted")?; + .ok_or_else(|| "summary admission revision exhausted".into()) + } + + pub(super) fn seal_finite(&mut self, generation: &CatalogGeneration) -> Result<(), String> { + let revision = self.validate_finite(generation)?; + self.finite_complete = true; + self.revision = revision; Ok(()) } @@ -228,7 +237,7 @@ impl AdmissionInventory { range: HalfOpenTimeRange, ) -> bool { self.finite_complete - && self.published_series.contains(&series_id) + && self.published_series.contains_key(&series_id) && self.observed_extent.is_some_and(|extent| { range.start_ms >= extent.start_ms && range.end_ms <= extent.end_ms }) @@ -244,6 +253,10 @@ impl AdmissionInventory { }) } + pub(super) fn published_frontiers(&self) -> &BTreeMap { + &self.published_series + } + pub(super) fn revision(&self) -> u64 { self.revision } @@ -358,11 +371,18 @@ mod tests { assert!(inventory.has_pending(coordinate.summary_definition_id, coordinate.time_range)); inventory.retire_completed_before(coordinate.summary_definition_id, 1000); assert_eq!(inventory.windows.len(), 1); + inventory + .record_series(&generation, &coordinate, 42) + .unwrap(); inventory .acknowledge(&generation, &coordinate, second) .unwrap(); inventory.retire_completed_before(coordinate.summary_definition_id, 1000); assert!(inventory.windows.is_empty()); + assert_eq!( + inventory.published_frontiers().get(&42), + Some(&(coordinate.time_range.end_ms as u64)) + ); } #[test] diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index 72d172468..a7175b2e1 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -605,6 +605,8 @@ impl Drop for StateMutation<'_> { #[derive(Default)] pub struct SketchStore { + /// Held through each state append; completion takes the exclusive guard. + completed_windows: RwLock>, admission: RwLock, mutation_revision: std::sync::atomic::AtomicU64, active_mutations: std::sync::atomic::AtomicUsize, @@ -881,13 +883,43 @@ impl SketchStore { generation: &CatalogGeneration, ) -> Result<(), String> { use std::sync::atomic::Ordering::SeqCst; + // Same order as admitted publication: admission -> metadata -> append fence. + // Closing a receiver alone is insufficient: the fence also rejects writes + // from every other producer once these physical windows are complete. + let mut inventory = self.admission.write().unwrap(); + let frontiers = inventory.published_frontiers().clone(); + let instances = self.instances.read().unwrap(); + let mut records = Vec::new(); + for (sid, end) in &frontiers { + let instance = instances + .get(sid) + .ok_or("completed series has no identity")?; + let mut record = self + .metadata_record(instance) + .ok_or("completed series has no catalog provenance")?; + record.completed_through_ms = record.completed_through_ms.max(Some(*end)); + records.push(record); + } + let mut completed = self.completed_windows.write().unwrap(); let mutation = self.mutation_revision.load(SeqCst); if self.active_mutations.load(SeqCst) != 0 || mutation != self.admitted_mutations.load(SeqCst) { return Err("finite summary completion cannot certify untracked state writes".into()); } - self.admission.write().unwrap().seal_finite(generation)?; + inventory.validate_finite(generation)?; + if let Some(writer) = self.persistence_metadata.read().unwrap().as_ref() { + writer + .upsert_all(&records) + .map_err(|error| error.to_string())?; + } + inventory.seal_finite(generation)?; + for (sid, end) in frontiers { + completed + .entry(sid) + .and_modify(|value| *value = (*value).max(end)) + .or_insert(end); + } self.finite_mutation_revision.store(mutation, SeqCst); Ok(()) } @@ -1206,7 +1238,11 @@ impl SketchStore { series_label_values: BTreeMap, window: TimestampRange, sample: SketchSampleState, - ) { + ) -> bool { + let completed = self.completed_windows.read().unwrap(); + if completed.get(&sid).is_some_and(|end| window.1 <= *end) { + return false; + } let _mutation = self.begin_state_mutation(); let store = self .series @@ -1216,6 +1252,7 @@ impl SketchStore { let mut guard = store.write().unwrap(); guard.insert(window, series_label_values, AggPayload::Sketch(sample)); guard.last_write_unix_ms = now_ms(); + true } /// Build a `SidStoreData` pre-configured for the store's current @@ -1249,7 +1286,11 @@ impl SketchStore { series_label_values: BTreeMap, window: TimestampRange, payload: Box, - ) { + ) -> bool { + let completed = self.completed_windows.read().unwrap(); + if completed.get(&sid).is_some_and(|end| window.1 <= *end) { + return false; + } let _mutation = self.begin_state_mutation(); let max_value = payload .as_any() @@ -1280,6 +1321,7 @@ impl SketchStore { retention_horizon_ms, ); } + true } /// Read a category through the derived in-memory rollup. Returns `None` @@ -2416,6 +2458,7 @@ impl SketchStore { record.summary_definition_id = Some(SummaryDefinitionId::from(m.policy_fp)); record.catalog_generation = Some(Arc::clone(m.catalog_generation.as_ref()?)); } + record.completed_through_ms = self.completed_windows.read().unwrap().get(&m.sid).copied(); record.retired_at_ms = m.retired_at_ms; record.expires_at_ms = m.expires_at_ms; Some(record) @@ -2793,7 +2836,7 @@ impl SketchStore { } let window = (output.start_timestamp, output.end_timestamp); - match crate::storage_engines::sketch_db::data::agg_kind_for_config(agg_cfg) { + let accepted = match crate::storage_engines::sketch_db::data::agg_kind_for_config(agg_cfg) { AggKind::Sketch { .. } => self.append_sample( sid, label_values_map, @@ -2809,8 +2852,8 @@ impl SketchStore { window, accumulator.clone_boxed_core(), ), - } - Some(sid) + }; + accepted.then_some(sid) } /// Phase 5 M2.3.6d — eviction-side helper. Removes every sid in the @@ -3006,6 +3049,14 @@ impl SketchStore { let mut registered = 0usize; for rec in records { + if let Some(end) = rec.completed_through_ms { + self.completed_windows + .write() + .unwrap() + .entry(rec.sid) + .and_modify(|current| *current = (*current).max(end)) + .or_insert(end); + } if rec.removed || rec.expires_at_ms.is_some_and(|expiry| expiry <= now_ms()) { continue; } @@ -4746,6 +4797,83 @@ mod tests { assert_eq!(expired.expires_at_ms, Some(2)); } + #[test] + fn completed_windows_reject_late_updates_after_restart() { + // Completion is a storage admission rule, including legacy producers, + // and survives restart without allowing a correction into consumed state. + let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + serde_json::from_str(include_str!( + "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let plan = snapshot.compile().unwrap(); + let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); + let directory = tempfile::tempdir().unwrap(); + let store = SketchStore::new(); + store + .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) + .unwrap(); + store.register(meta_with_policy(850, fingerprint)); + let generation = store.active_catalog_generation().unwrap(); + let writer = Arc::new(persistence::metadata::SidMetadataStore::new( + directory.path(), + )); + *store.persistence_metadata.write().unwrap() = Some(writer.clone()); + let coordinate = asap_types::sds::SummaryInstanceCoordinates { + summary_definition_id: fingerprint.into(), + time_range: HalfOpenTimeRange { + start_ms: 0, + end_ms: 30_000, + }, + group_values: BTreeMap::new(), + }; + let revision = store + .admit_summary_updates(&generation, [coordinate.clone()].into()) + .unwrap(); + assert!(store.seal_finite_summary_input(&generation).is_err()); + store + .publish_admitted_summary_update( + &generation, + &coordinate, + revision, + revision, + 120_000, + || { + store + .append_sample(850, BTreeMap::new(), (0, 30_000), sample(1)) + .then_some(850) + }, + ) + .unwrap(); + let stale_record = store + .metadata_record(&store.instances.read().unwrap()[&850]) + .unwrap(); + let before_failed_seal = store.summary_update_revision(); + std::fs::create_dir(writer.path()).unwrap(); + assert!(store.seal_finite_summary_input(&generation).is_err()); + assert_eq!(store.summary_update_revision(), before_failed_seal); + assert!(!store.completed_windows.read().unwrap().contains_key(&850)); + std::fs::remove_dir(writer.path()).unwrap(); + store.seal_finite_summary_input(&generation).unwrap(); + assert!(!store.append_sample(850, BTreeMap::new(), (0, 30_000), sample(2))); + assert!(!store.append_precompute( + 850, + BTreeMap::new(), + (0, 30_000), + Box::new(crate::precompute_engine::operators::SumAccumulator::new()) + )); + // A flusher that captured metadata before completion cannot reopen it. + writer.upsert_all(&[stale_record]).unwrap(); + assert_eq!(writer.load().unwrap()[0].completed_through_ms, Some(30_000)); + let restored = SketchStore::new(); + restored + .install_summary_catalog(Arc::new(plan.summary_catalog)) + .unwrap(); + restored.register_recovered_disk_series(directory.path()); + assert!(!restored.append_sample(850, BTreeMap::new(), (0, 30_000), sample(3))); + assert!(restored.append_sample(850, BTreeMap::new(), (30_000, 60_000), sample(4))); + } + #[test] fn failed_durable_lifecycle_write_preserves_live_instance() { let store = SketchStore::new(); diff --git a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs index 51e6e683d..93caef534 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs @@ -277,6 +277,9 @@ pub struct SidMetaRecord { pub expires_at_ms: Option, #[serde(default)] pub removed: bool, + /// No further publication may change a window ending at or before this bound. + #[serde(default)] + pub completed_through_ms: Option, } impl SidMetaRecord { @@ -301,6 +304,7 @@ impl SidMetaRecord { retired_at_ms: None, expires_at_ms: None, removed: false, + completed_through_ms: None, } } @@ -368,6 +372,8 @@ struct SidBindingRec { expires_at_ms: Option, #[serde(default)] removed: bool, + #[serde(default)] + completed_through_ms: Option, } /// Version-3 normalized sidecar with authoritative catalog provenance. Descriptors appear once and SeriesId bindings hold @@ -438,6 +444,7 @@ impl SdsSidecar { retired_at_ms: record.retired_at_ms, expires_at_ms: record.expires_at_ms, removed: record.removed, + completed_through_ms: record.completed_through_ms, }, ); } @@ -491,6 +498,7 @@ impl SdsSidecar { retired_at_ms: binding.retired_at_ms, expires_at_ms: binding.expires_at_ms, removed: binding.removed, + completed_through_ms: binding.completed_through_ms, }) }) .collect() @@ -608,6 +616,8 @@ impl SidMetadataStore { // Lifecycle is monotone for a SeriesId. An older flush snapshot // must not resurrect a retired or removed persisted instance. next.removed |= existing.removed; + next.completed_through_ms = + existing.completed_through_ms.max(next.completed_through_ms); next.retired_at_ms = existing.retired_at_ms.or(next.retired_at_ms); next.expires_at_ms = match (existing.expires_at_ms, next.expires_at_ms) { (Some(a), Some(b)) => Some(a.min(b)), diff --git a/docs/design_docs/summary-catalog-sds-architecture.md b/docs/design_docs/summary-catalog-sds-architecture.md index abc38b97b..0ef4724c6 100644 --- a/docs/design_docs/summary-catalog-sds-architecture.md +++ b/docs/design_docs/summary-catalog-sds-architecture.md @@ -403,3 +403,23 @@ This is an explicit lifetime transition, not cross-generation recovery of arbitr summary state. Legacy records without trustworthy catalog provenance remain unbound. Tombstone reclamation still requires coordinated removal of old physical parts and is not implemented by this transition. + +### Immutable completed windows + +Finite Remote Write completion now fences the SummaryStore append boundary, +not just the receiver queue. After all admitted outputs are published, the store +records the greatest published window end for each physical SeriesId. Sketch and +exact-state writes ending at or before that boundary are rejected, including +writes arriving through other producers. A later window remains writable. + +The boundary is monotone in the existing SeriesId metadata sidecar and is restored +before recovered identities become writable. A stale background metadata flush +cannot reopen a completed window. The guard belongs to the physical lifetime; +a catalog-authorized replacement SeriesId has its own boundary. + +This is an immutability guarantee, not a promise that every payload has reached +disk. Maintenance consumers must separately verify durable state availability and +atomically publish their output identity before claiming replay-safe consumption. +The existing finite-source completeness proof still rejects untracked writes or +pending admitted work. Continuous producer watermarks and derived-state commit +transactions are separate from this finite-input boundary. From 7312e244d56ea39ddaadba3632bfa8cd42ced383 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 11 Sep 2026 06:57:47 -0600 Subject: [PATCH 18/28] test(storage): check unchanged completion revision with snapshot fence --- data_plane/src/storage_engines/sketch_db/index/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index a7175b2e1..014ea8e75 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -4851,7 +4851,7 @@ mod tests { let before_failed_seal = store.summary_update_revision(); std::fs::create_dir(writer.path()).unwrap(); assert!(store.seal_finite_summary_input(&generation).is_err()); - assert_eq!(store.summary_update_revision(), before_failed_seal); + assert!(store.summary_update_revision().matches(before_failed_seal)); assert!(!store.completed_windows.read().unwrap().contains_key(&850)); std::fs::remove_dir(writer.path()).unwrap(); store.seal_finite_summary_input(&generation).unwrap(); From 875486e3c68e47e1f62853f91a522fa35356c8d8 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 11 Sep 2026 07:05:05 -0600 Subject: [PATCH 19/28] fix(storage): flush completed payloads before sealing their windows --- data_plane/src/drivers/ingest/otel.rs | 22 ++-- .../drivers/ingest/prometheus_remote_write.rs | 21 +++- .../sketch_db/index/epoch_columnar.rs | 28 +++++ .../storage_engines/sketch_db/index/mod.rs | 106 +++++++++++++++++- .../sketch_db/persistence/flusher.rs | 10 +- .../sketch_db/persistence/source.rs | 5 + .../summary-catalog-sds-architecture.md | 9 +- 7 files changed, 178 insertions(+), 23 deletions(-) diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index 1419df0b6..cc38990ad 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -1753,15 +1753,6 @@ async fn route_modified_otlp_sketches_to_precompute( } } - ingest_state.sketch_snapshots.insert( - series_key.clone(), - crate::precompute_engine::ingest_handler::SnapshotCacheEntry { - core: accumulator.clone_boxed_core(), - window_start: dp.start_time_unix_nano, - }, - ); - ingest_state.note_window_and_sweep(dp.start_time_unix_nano); - use crate::storage_engines::sketch_db::index::{ SketchEncoding, SketchSampleState, }; @@ -1777,7 +1768,7 @@ async fn route_modified_otlp_sketches_to_precompute( ); let encoding = encoding_to_handle(dp.encoding).unwrap_or(SketchEncoding::ProtoFull); - ingest_state.sketch_index.append_sample( + if !ingest_state.sketch_index.append_sample( sid, label_values, window, @@ -1785,7 +1776,18 @@ async fn route_modified_otlp_sketches_to_precompute( bytes: dp.sketch.clone(), encoding, }, + ) { + return Err("summary window is immutable after completion".into()); + } + + ingest_state.sketch_snapshots.insert( + series_key.clone(), + crate::precompute_engine::ingest_handler::SnapshotCacheEntry { + core: accumulator.clone_boxed_core(), + window_start: dp.start_time_unix_nano, + }, ); + ingest_state.note_window_and_sweep(dp.start_time_unix_nano); // Collect the configs whose metric matches this DP. // Detection is independent of the legacy dual-write diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index 367ac15f1..3808c9468 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -199,10 +199,23 @@ impl PrometheusRemoteWriteReceiver { .and_then(|plan| plan.precompute_plan.summary_catalog.clone()) .ok_or("finite completion requires a catalog generation")?; self.inner.ingest.router.drain().await?; - self.inner - .ingest - .sketch_index - .seal_finite_summary_input(&generation)?; + let flush_deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30); + loop { + if self + .inner + .ingest + .sketch_index + .seal_finite_summary_input(&generation)? + { + break; + } + if tokio::time::Instant::now() >= flush_deadline { + return Err( + "finite completion is waiting for durable summary payloads; retry drain".into(), + ); + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } trim_process_allocator(); Ok(()) } diff --git a/data_plane/src/storage_engines/sketch_db/index/epoch_columnar.rs b/data_plane/src/storage_engines/sketch_db/index/epoch_columnar.rs index 4442dfcbc..695c75052 100644 --- a/data_plane/src/storage_engines/sketch_db/index/epoch_columnar.rs +++ b/data_plane/src/storage_engines/sketch_db/index/epoch_columnar.rs @@ -956,6 +956,19 @@ impl SidStoreData { } } + /// Whether any hot or queued-for-flush state belongs to this completion prefix. + pub(crate) fn contains_window_ending_at_or_before(&self, end_ms: u64) -> bool { + self.current_epoch + .iter_entries() + .any(|(window, _, _)| window.1 <= end_ms) + || self.sealed_epochs.values().any(|epoch| { + epoch + .entries + .iter() + .any(|(window, _, _)| window.1 <= end_ms) + }) + } + /// Time-driven seal for the persistence tier: roll every window in /// `current_epoch` whose END is at or before `cutoff_end` into a /// freshly-sealed epoch, leaving the more-recent windows in @@ -1372,6 +1385,21 @@ mod tests { assert_eq!(s.current_epoch.distinct_windows(), 0); } + #[test] + fn completion_prefix_does_not_wait_for_future_windows() { + let mut store = SidStoreData::, i32>::new(); + store.insert((30, 60), vec![], 1); + assert!(!store.contains_window_ending_at_or_before(30)); + store.insert((0, 30), vec![], 2); + assert!(store.contains_window_ending_at_or_before(30)); + store.persistence_enabled = true; + store.seal_aged_windows(31); + assert!(store.contains_window_ending_at_or_before(30)); + store.sealed_epochs.clear(); + assert!(!store.contains_window_ending_at_or_before(30)); + assert!(store.contains_window_ending_at_or_before(60)); + } + #[test] fn seal_aged_windows_noop_when_persistence_disabled() { let mut s = SidStoreData::::new(); diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index 014ea8e75..556c9749c 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -607,6 +607,7 @@ impl Drop for StateMutation<'_> { pub struct SketchStore { /// Held through each state append; completion takes the exclusive guard. completed_windows: RwLock>, + completion_flush_before: std::sync::atomic::AtomicU64, admission: RwLock, mutation_revision: std::sync::atomic::AtomicU64, active_mutations: std::sync::atomic::AtomicUsize, @@ -881,7 +882,7 @@ impl SketchStore { pub(crate) fn seal_finite_summary_input( &self, generation: &CatalogGeneration, - ) -> Result<(), String> { + ) -> Result { use std::sync::atomic::Ordering::SeqCst; // Same order as admitted publication: admission -> metadata -> append fence. // Closing a receiver alone is insufficient: the fence also rejects writes @@ -908,6 +909,24 @@ impl SketchStore { return Err("finite summary completion cannot certify untracked state writes".into()); } inventory.validate_finite(generation)?; + if self.persistence_read.read().unwrap().is_some() { + if let Some(end) = frontiers.values().max() { + self.completion_flush_before + .fetch_max(end.saturating_add(1), SeqCst); + } + // The flusher evicts an epoch only after its payload and manifest + // are durable. Until then a restart must remain able to replay it. + let pending = frontiers.iter().any(|(sid, end)| { + self.series.get(sid).is_some_and(|data| { + data.read() + .unwrap() + .contains_window_ending_at_or_before(*end) + }) + }); + if pending { + return Ok(false); + } + } if let Some(writer) = self.persistence_metadata.read().unwrap().as_ref() { writer .upsert_all(&records) @@ -921,7 +940,7 @@ impl SketchStore { .or_insert(end); } self.finite_mutation_revision.store(mutation, SeqCst); - Ok(()) + Ok(true) } pub(crate) fn summary_window_known_empty( @@ -3154,6 +3173,13 @@ impl SketchStore { // the trait keeps the historical name so the flusher / manifest / // part-writer stay untouched. impl crate::storage_engines::sketch_db::index::persistence::EpochSource for SketchStore { + fn flush_before_ms(&self) -> Option { + let cutoff = self + .completion_flush_before + .load(std::sync::atomic::Ordering::SeqCst); + (cutoff != 0).then_some(cutoff) + } + fn list_sealed_epochs( &self, ) -> Vec { @@ -4874,6 +4900,82 @@ mod tests { assert!(restored.append_sample(850, BTreeMap::new(), (30_000, 60_000), sample(4))); } + #[test] + fn finite_completion_flushes_payload_before_persisting_immutability() { + // With neither memory pressure nor a hot-tier deadline, completion must + // explicitly flush its payload before persisting a non-replayable window. + let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + serde_json::from_str(include_str!( + "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let plan = snapshot.compile().unwrap(); + let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); + let directory = tempfile::tempdir().unwrap(); + { + let store = Arc::new(SketchStore::new()); + store + .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) + .unwrap(); + store.register(meta_with_policy(851, fingerprint)); + let mut config = durable_cfg(directory.path().to_path_buf()); + config.hot_window_ms = None; + config.seal_window_count = 100; + let mut persistence = store.start_persistence(config).unwrap(); + let generation = store.active_catalog_generation().unwrap(); + let coordinate = asap_types::sds::SummaryInstanceCoordinates { + summary_definition_id: fingerprint.into(), + time_range: HalfOpenTimeRange { + start_ms: 0, + end_ms: 30_000, + }, + group_values: BTreeMap::new(), + }; + let revision = store + .admit_summary_updates(&generation, [coordinate.clone()].into()) + .unwrap(); + store + .publish_admitted_summary_update( + &generation, + &coordinate, + revision, + revision, + 120_000, + || { + store + .append_sample(851, BTreeMap::new(), (0, 30_000), sample(1)) + .then_some(851) + }, + ) + .unwrap(); + assert!(!store.seal_finite_summary_input(&generation).unwrap()); + assert!(!store.completed_windows.read().unwrap().contains_key(&851)); + assert!(wait_until( + || store.seal_finite_summary_input(&generation).unwrap(), + Duration::from_secs(5) + )); + assert!(!persistence.manifest.live_parts().is_empty()); + persistence.shutdown(); + } + let restored = Arc::new(SketchStore::new()); + restored + .install_summary_catalog(Arc::new(plan.summary_catalog)) + .unwrap(); + let _persistence = restored + .start_persistence(durable_cfg(directory.path().to_path_buf())) + .unwrap(); + assert!(!restored.append_sample(851, BTreeMap::new(), (0, 30_000), sample(2))); + let rows = restored.query_range(851, 0, 30_000); + assert_eq!(rows.len(), 1); + let payloads: Vec<_> = rows + .iter() + .flat_map(|row| row.samples.values()) + .flatten() + .collect(); + assert_eq!(payloads.len(), 1); + assert_eq!(payloads[0].bytes, vec![1]); + } + #[test] fn failed_durable_lifecycle_write_preserves_live_instance() { let store = SketchStore::new(); diff --git a/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs b/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs index c2aad589b..e774b39e6 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs @@ -296,8 +296,11 @@ fn run_tick(shared: &Arc, source: &S) -> PersistR // it becomes flushable this same tick. (Without `hot_window_ms` the // durable tier is purely memory-pressure driven and phase 1 below // handles eviction.) - if let Some(hot) = cfg.hot_window_ms { - let cutoff = now.saturating_sub(hot); + let flush_cutoff = cfg + .hot_window_ms + .map(|hot| now.saturating_sub(hot)) + .max(source.flush_before_ms()); + if let Some(cutoff) = flush_cutoff { source.seal_aged_epochs(cutoff); } @@ -325,8 +328,7 @@ fn run_tick(shared: &Arc, source: &S) -> PersistR } // Phase 2: time watermark (any epoch older than now - hot_window). - if let Some(hot) = cfg.hot_window_ms { - let cutoff = now.saturating_sub(hot); + if let Some(cutoff) = flush_cutoff { for r in &all { if r.end_ts < cutoff && !selected diff --git a/data_plane/src/storage_engines/sketch_db/persistence/source.rs b/data_plane/src/storage_engines/sketch_db/persistence/source.rs index 117c68544..734cfaf6a 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/source.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/source.rs @@ -97,6 +97,11 @@ pub struct EpochSnapshotEntry { /// * [`approx_memory_bytes`] is cheap (atomic load) and is kept in sync /// with what the flusher has evicted. pub trait EpochSource: Send + Sync { + /// Explicit durability demand from a completed source, independent of the hot tier. + fn flush_before_ms(&self) -> Option { + None + } + fn list_sealed_epochs(&self) -> Vec; /// Time-driven seal: roll every un-sealed `current_epoch` window diff --git a/docs/design_docs/summary-catalog-sds-architecture.md b/docs/design_docs/summary-catalog-sds-architecture.md index 0ef4724c6..4b25499a3 100644 --- a/docs/design_docs/summary-catalog-sds-architecture.md +++ b/docs/design_docs/summary-catalog-sds-architecture.md @@ -417,9 +417,12 @@ before recovered identities become writable. A stale background metadata flush cannot reopen a completed window. The guard belongs to the physical lifetime; a catalog-authorized replacement SeriesId has its own boundary. -This is an immutability guarantee, not a promise that every payload has reached -disk. Maintenance consumers must separately verify durable state availability and -atomically publish their output identity before claiming replay-safe consumption. +With persistence enabled, completion explicitly requests the existing flusher to +make the completed prefix durable, even if it is still inside the hot tier. +Completion waits until the corresponding epochs have been evicted after part and +manifest publication; only then does it persist the immutable boundary. An +in-memory deployment provides no restart guarantee. Maintenance consumers still +must atomically publish their output identity before claiming replay-safe consumption. The existing finite-source completeness proof still rejects untracked writes or pending admitted work. Continuous producer watermarks and derived-state commit transactions are separate from this finite-input boundary. From c5ae4a606d04296b6d0e3f49f23e016d8932c29d Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 11 Sep 2026 07:16:12 -0600 Subject: [PATCH 20/28] fix(storage): report proven immutable instances as complete --- .../storage_engines/sketch_db/index/mod.rs | 24 +++++++++++++++---- .../summary-catalog-sds-architecture.md | 3 ++- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index f2fbe573d..e6f355a4b 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -1109,6 +1109,12 @@ impl SketchStore { .series .get(series_id) .map(|entry| Arc::clone(entry.value())); + let completed_through = self + .completed_windows + .read() + .unwrap() + .get(series_id) + .copied(); let status = match binding.metadata.status() { AggStatus::Active => SummaryInstanceStatus::Ready, AggStatus::Retired | AggStatus::Expired => SummaryInstanceStatus::Retiring, @@ -1155,10 +1161,11 @@ impl SketchStore { checksum: None, }, status: status.clone(), - // The current payload row does not distinguish a normal - // pane close from a late standalone correction. Report the - // concrete instance without inventing a completeness proof. - completeness: InstanceCompleteness::Unknown, + completeness: if completed_through.is_some_and(|end| window.1 <= end) { + InstanceCompleteness::Complete + } else { + InstanceCompleteness::Unknown + }, lifecycle: InstanceLifecycle::Persistent, observed_at_ms, }; @@ -4880,6 +4887,15 @@ mod tests { assert!(!store.completed_windows.read().unwrap().contains_key(&850)); std::fs::remove_dir(writer.path()).unwrap(); store.seal_finite_summary_input(&generation).unwrap(); + let producers = BTreeMap::from([(fingerprint.into(), "producer".to_string())]); + let inventory = store + .observed_summary_inventory("backend", "store", &producers, 1, 30_000) + .unwrap(); + assert_eq!(inventory.instances.len(), 1); + assert_eq!( + inventory.instances.values().next().unwrap().completeness, + InstanceCompleteness::Complete + ); assert!(!store.append_sample(850, BTreeMap::new(), (0, 30_000), sample(2))); assert!(!store.append_precompute( 850, diff --git a/docs/design_docs/summary-catalog-sds-architecture.md b/docs/design_docs/summary-catalog-sds-architecture.md index 9534cb31b..59e1fb72f 100644 --- a/docs/design_docs/summary-catalog-sds-architecture.md +++ b/docs/design_docs/summary-catalog-sds-architecture.md @@ -436,7 +436,8 @@ Finite Remote Write completion now fences the SummaryStore append boundary, not just the receiver queue. After all admitted outputs are published, the store records the greatest published window end for each physical SeriesId. Sketch and exact-state writes ending at or before that boundary are rejected, including -writes arriving through other producers. A later window remains writable. +writes arriving through other producers. A later window remains writable. Observed SDS inventory reports only these frozen +instances as `Complete`; ordinary emitted panes remain `Unknown`. The boundary is monotone in the existing SeriesId metadata sidecar and is restored before recovered identities become writable. A stale background metadata flush From 2ee7a223cd1dea1cd7a0cb2796fd0a16b04dafbf Mon Sep 17 00:00:00 2001 From: Zeying Zhu <50204836+zzylol@users.noreply.github.com> Date: Fri, 11 Sep 2026 10:55:01 -0400 Subject: [PATCH 21/28] refactor(clickhouse): remove unused relational tree executor (#652) * refactor(clickhouse): remove unused relational tree executor * test(clickhouse): construct the shared predicate wrapper --- .../relational_adapter.rs | 216 +++--------------- data_plane/src/query_engines/canonical/mod.rs | 9 +- .../src/query_engines/canonical/relational.rs | 62 ----- 3 files changed, 33 insertions(+), 254 deletions(-) delete mode 100644 data_plane/src/query_engines/canonical/relational.rs diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs index 0bfc9fc91..15945be21 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs @@ -14,18 +14,10 @@ use arrow::{ }; use chrono::{DateTime, NaiveDateTime, TimeZone}; use planner_types::{ - post_asap::{SummaryFamilyType, SummaryNode, SummarySchema, ValueOperation}, + post_asap::{SummaryFamilyType, SummarySchema, ValueOperation}, pre_asap::{ArithmeticOpKind, CompareOpKind, DataType, QueryExpr, ScalarValue, SortKey}, }; -use crate::query_engines::{ - asap_query_engine::{ - summary_exec::ExecOutcome, - summary_executor::{GroupState, SummaryValue}, - }, - canonical::relational::RelationalAdapter, -}; - use super::clickhouse_result_adapter::ClickHouseQueryResult; #[derive(Debug, thiserror::Error, PartialEq)] @@ -456,81 +448,6 @@ impl ClickHouseRelationalAdapter { } } -impl RelationalAdapter for ClickHouseRelationalAdapter -where - E: crate::query_engines::canonical::executor::SummaryExecutor< - GroupKey = BTreeMap, - State = GroupState, - Value = SummaryValue, - >, -{ - type Relation = ClickHouseRelation; - type Error = ClickHouseRelationalError; - - fn relation_from_outcome( - &self, - node: &SummaryNode, - outcome: ExecOutcome, - ) -> Result { - let fields = fields(node); - let mut rows = Vec::new(); - let mut coverage = None; - let mut coverage_complete = true; - match outcome { - ExecOutcome::Value(groups) => { - for (group, value) in groups { - match value.coverage() { - Some(next) if coverage_complete => { - coverage = intersect_coverage(coverage, Some(next)); - } - None => { - coverage = None; - coverage_complete = false; - } - Some(_) => {} - } - match value { - SummaryValue::Points(points, _) => { - for (timestamp, value) in points { - rows.push(row_from_value(&fields, &group, timestamp, value)?); - } - } - SummaryValue::TopK(_, _) => { - return Err(ClickHouseRelationalError::Unsupported( - "TopK summary rows".into(), - )); - } - } - } - } - ExecOutcome::State(groups) => { - for (group, state, family) in groups { - let value = exact_value(&state, &family)?; - rows.push(row_from_value(&fields, &group, 0, value)?); - } - } - } - Ok(ClickHouseRelation { - rows, - fields, - coverage, - }) - } - - fn apply( - &self, - node: &SummaryNode, - operation: &ValueOperation, - input: Self::Relation, - ) -> Result { - self.apply_operation(operation, &node.schema, input) - } -} - -fn fields(node: &SummaryNode) -> Vec<(String, DataType, bool)> { - fields_from_schema(&node.schema) -} - fn fields_from_schema(schema: &SummarySchema) -> Vec<(String, DataType, bool)> { schema .fields @@ -545,20 +462,6 @@ fn fields_from_schema(schema: &SummarySchema) -> Vec<(String, DataType, bool)> { .collect() } -fn exact_value( - state: &GroupState, - family: &SummaryFamilyType, -) -> Result { - if !matches!(family, SummaryFamilyType::ExactAggregate(..)) { - return Err(ClickHouseRelationalError::Unsupported( - "unfinalized non-exact summary state".into(), - )); - } - state.exact_value(&None).ok_or_else(|| { - ClickHouseRelationalError::Invalid("exact accumulator cannot be finalized".into()) - }) -} - fn row_from_value( fields: &[(String, DataType, bool)], group: &BTreeMap, @@ -923,14 +826,6 @@ fn cell_cmp(left: &Cell, right: &Cell) -> Option { } } -fn intersect_coverage(current: Option<(u64, u64)>, next: Option<(u64, u64)>) -> Option<(u64, u64)> { - match (current, next) { - (None, next) => next, - (current, None) => current, - (Some(left), Some(right)) => Some((left.0.max(right.0), left.1.min(right.1))), - } -} - fn arrow_type(dtype: &DataType) -> ArrowDataType { match dtype { DataType::Null => ArrowDataType::Null, @@ -1073,54 +968,12 @@ fn build_array( #[cfg(test)] mod tests { use super::*; - use crate::query_engines::canonical::{ - executor::SummaryExecutor, relational::execute_relational, - }; use planner_types::{ - post_asap::{ExecutionTiming, SketchQuery, SummaryExpr, SummaryField, SummarySchema}, - pre_asap::{ColumnRef, GroupKeys, Predicate, ProjectItem, Reduction}, + post_asap::{SummaryField, SummarySchema}, + pre_asap::{GroupKeys, Predicate, ProjectItem}, }; use std::rc::Rc; - struct MockExecutor; - - impl SummaryExecutor for MockExecutor { - type Handle = (); - type State = GroupState; - type Value = SummaryValue; - type Error = (); - type GroupKey = BTreeMap; - - fn find_candidates( - &self, - _: &SummaryFamilyType, - _: &ColumnRef, - _: &Reduction, - _: &SummaryNode, - ) -> Result, Self::Error> { - unreachable!() - } - - fn fetch_state(&self, _: &Self::Handle) -> Result { - unreachable!() - } - - fn merge_states(&self, _: Vec) -> Result { - unreachable!() - } - - fn readout(&self, _: &Self::State, _: &SketchQuery) -> Result { - unreachable!() - } - - fn logical(&self, _: &QueryExpr) -> Result { - Ok(SummaryValue::Points( - vec![(10, 2.0), (20, 3.0), (30, 1.0)], - Some((0, 40)), - )) - } - } - fn schema(fields: &[(&str, DataType)]) -> SummarySchema { SummarySchema { fields: fields @@ -1137,22 +990,6 @@ mod tests { } } - fn value_node( - child: Rc, - operation: ValueOperation, - schema: SummarySchema, - ) -> Rc { - Rc::new(SummaryNode { - expr: SummaryExpr::ValueOperation { - child, - operation, - timing: ExecutionTiming::ReadTime, - }, - schema, - guarantee: None, - }) - } - /// Map entries remain ordered pairs, including duplicate keys and null values. #[test] fn map_transport_retains_duplicate_keys_and_null_values() { @@ -1199,17 +1036,32 @@ mod tests { #[test] fn executes_filter_project_arithmetic_sort_and_limit_chain() { let input_schema = schema(&[("ts", DataType::Timestamp), ("sum", DataType::Float64)]); - let leaf = Rc::new(SummaryNode { - expr: SummaryExpr::KeepPreAsap(Rc::new(QueryExpr::Literal(ScalarValue::Int64(0)))), - schema: input_schema.clone(), - guarantee: None, - }); + let adapter = ClickHouseRelationalAdapter; + let input = ClickHouseRelation { + rows: vec![ + vec![Cell::Timestamp(10), Cell::Float64(2.0)], + vec![Cell::Timestamp(20), Cell::Float64(3.0)], + vec![Cell::Timestamp(30), Cell::Float64(1.0)], + ], + fields: fields_from_schema(&input_schema), + coverage: Some((0, 40)), + }; + let mut relation = adapter + .apply_filter( + &Predicate(Rc::new(QueryExpr::Compare { + left: Rc::new(QueryExpr::Column(1)), + op: CompareOpKind::Gt, + right: Rc::new(QueryExpr::Literal(ScalarValue::Float64(1.0))), + })), + input, + ) + .unwrap(); + assert_eq!(relation.rows.len(), 2); let projected_schema = schema(&[ ("bucket", DataType::Timestamp), ("score", DataType::Float64), ]); - let projected = value_node( - leaf, + for operation in [ ValueOperation::Project { cols: vec![ ProjectItem { @@ -1227,10 +1079,6 @@ mod tests { ], qualifier: None, }, - projected_schema.clone(), - ); - let sorted = value_node( - projected, ValueOperation::Sort { keys: vec![SortKey { expr: QueryExpr::Column(1), @@ -1239,16 +1087,12 @@ mod tests { }], partition_by: GroupKeys::none(), }, - projected_schema.clone(), - ); - let limited = value_node( - sorted, ValueOperation::Limit { n: 1, offset: 0 }, - projected_schema, - ); - - let relation = execute_relational(&limited, &MockExecutor, &ClickHouseRelationalAdapter) - .expect("supported SQL chain should execute"); + ] { + relation = adapter + .apply_operation(&operation, &projected_schema, relation) + .expect("installed SQL operators should execute"); + } assert_eq!(relation.coverage, Some((0, 40))); let result = relation.into_result().unwrap(); let batch = &result.batches[0]; diff --git a/data_plane/src/query_engines/canonical/mod.rs b/data_plane/src/query_engines/canonical/mod.rs index 61290336b..c170e5d0e 100644 --- a/data_plane/src/query_engines/canonical/mod.rs +++ b/data_plane/src/query_engines/canonical/mod.rs @@ -1,10 +1,7 @@ -//! Language-independent execution of ASAPPlanner's post-ASAP DAG. +//! Compatibility re-exports for the existing summary executor. //! -//! The implementation remains in its compatibility location while PromQL -//! callers migrate. These re-exports give SQL and PromQL one execution API -//! without changing the existing PromQL types or behavior. - -pub mod relational; +//! Installed SQL serving executes QueryPlan nodes through its typed relational +//! adapter. These aliases remain for callers of the summary execution API. pub mod executor { pub use crate::query_engines::asap_query_engine::summary_exec::{ diff --git a/data_plane/src/query_engines/canonical/relational.rs b/data_plane/src/query_engines/canonical/relational.rs deleted file mode 100644 index 816e97ba6..000000000 --- a/data_plane/src/query_engines/canonical/relational.rs +++ /dev/null @@ -1,62 +0,0 @@ -//! Relational query-time operators over the shared summary executor. -//! -//! The legacy executor intentionally keeps `Value` opaque. This companion -//! adapter lets a language runtime interpret planner-owned `ValueOperation` -//! nodes without widening that trait or changing existing PromQL behavior. - -use planner_types::post_asap::{SummaryExpr, SummaryNode, ValueOperation}; - -use super::executor::{execute, ExecError, ExecOutcome, SummaryExecutor}; - -pub trait RelationalAdapter { - type Relation; - type Error; - - fn relation_from_outcome( - &self, - node: &SummaryNode, - outcome: ExecOutcome, - ) -> Result; - - fn apply( - &self, - node: &SummaryNode, - operation: &ValueOperation, - input: Self::Relation, - ) -> Result; -} - -#[derive(Debug)] -pub enum RelationalExecError { - Executor(ExecError), - Adapter(AdapterError), -} - -/// Recursively evaluates query-time relational nodes and delegates every -/// summary/state node to the unchanged shared summary executor. -pub fn execute_relational( - node: &SummaryNode, - executor: &E, - adapter: &A, -) -> Result> -where - E: SummaryExecutor, - A: RelationalAdapter, -{ - match &node.expr { - SummaryExpr::ValueOperation { - child, operation, .. - } => { - let input = execute_relational(child, executor, adapter)?; - adapter - .apply(node, operation, input) - .map_err(RelationalExecError::Adapter) - } - _ => { - let outcome = execute(node, executor).map_err(RelationalExecError::Executor)?; - adapter - .relation_from_outcome(node, outcome) - .map_err(RelationalExecError::Adapter) - } - } -} From d0e3ea31b0446af6190dc80db8aa1b7fa5360e86 Mon Sep 17 00:00:00 2001 From: Zeying Zhu <50204836+zzylol@users.noreply.github.com> Date: Fri, 11 Sep 2026 10:56:14 -0400 Subject: [PATCH 22/28] refactor: share installed QueryPlan contracts across components (#634) * refactor: share installed QueryPlan contracts across components * refactor(query): use shared identity contract in MetricsQL adapter * style: format shared query identity fixture --- Cargo.lock | 1 + control_plane/src/clickhouse.rs | 2 +- control_plane/src/physical/compiler.rs | 4 +- control_plane/src/query_plan.rs | 1051 ++++------------- .../src/query_plan/clickhouse_exact.rs | 6 +- control_plane/src/query_plan/logical.rs | 235 +--- control_plane/tests/o11y_exact_fallback.rs | 2 +- control_plane/tests/offline_evidence.rs | 6 +- crates/asap_types/Cargo.toml | 1 + crates/asap_types/src/lib.rs | 2 + crates/asap_types/src/query_plan.rs | 677 +++++++++++ crates/asap_types/src/query_plan/logical.rs | 155 +++ .../drivers/ingest/prometheus_remote_write.rs | 8 +- .../query/adapters/victoriametrics_http.rs | 2 +- data_plane/src/drivers/query/servers/http.rs | 28 +- data_plane/src/main.rs | 2 +- .../precompute_engine/maintenance_runtime.rs | 14 +- .../src/precompute_engine/subdag_scheduler.rs | 8 +- .../accelerator.rs | 14 +- .../asap_clickhouse_query_engine/execution.rs | 7 +- .../asap_query_engine/catalog_resolver.rs | 2 +- .../query_engines/asap_query_engine/engine.rs | 29 +- .../asap_query_engine/exact_subqueries.rs | 16 +- .../asap_query_engine/live_serve.rs | 32 +- .../asap_query_engine/logical_dag.rs | 16 +- .../asap_query_engine/physical_dag.rs | 8 +- .../asap_query_engine/post_asap_readout.rs | 78 +- .../asap_query_engine/summary_executor.rs | 30 +- .../types/hot_reload_config.rs | 4 +- .../tests/clickhouse_differential_e2e.rs | 4 +- ...e2e_controller_plans_and_backend_serves.rs | 3 +- data_plane/tests/support/physical_fixture.rs | 2 +- docs/design_docs/asapplanner-integration.md | 3 +- .../summary-catalog-sds-architecture.md | 8 +- 34 files changed, 1230 insertions(+), 1230 deletions(-) create mode 100644 crates/asap_types/src/query_plan.rs create mode 100644 crates/asap_types/src/query_plan/logical.rs diff --git a/Cargo.lock b/Cargo.lock index 5b68bc67f..cd77ff936 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -462,6 +462,7 @@ dependencies = [ "asap-types", "base64 0.21.7", "clap 4.6.1", + "promql-parser", "serde", "serde_json", "serde_yaml", diff --git a/control_plane/src/clickhouse.rs b/control_plane/src/clickhouse.rs index 19fe715cf..533219378 100644 --- a/control_plane/src/clickhouse.rs +++ b/control_plane/src/clickhouse.rs @@ -360,7 +360,7 @@ where .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; let mut materialization_nodes = std::collections::BTreeMap::new(); let mut query_nodes = std::collections::BTreeMap::new(); - let executable = QueryPlanEntry::compile_bound_relational_mapped( + let executable = crate::query_plan::compile_bound_relational_mapped( query.sql.clone(), planned.canonical_sql.clone(), &root, diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 460b5e531..6026ac86a 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -2207,7 +2207,7 @@ impl PhysicalCompiler { cumulative_readout: true, }; let mut entry = if request.hybrid_execution { - QueryPlanEntry::compile_bound_composable_mapped( + crate::query_plan::compile_bound_composable_mapped( query.query_id.clone(), canonical.clone(), &query.post_asap, @@ -2224,7 +2224,7 @@ impl PhysicalCompiler { }, ) } else { - QueryPlanEntry::compile_bound_mapped( + crate::query_plan::compile_bound_mapped( query.query_id.clone(), canonical.clone(), &query.post_asap, diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index 17943b88e..9ffe17971 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -1,854 +1,217 @@ -//! Authoritative backend-executable query DAG. -//! -//! ASAPPlanner owns semantic post-ASAP IR. Physical compilation binds every -//! maintained-summary leaf to one materialization and lowers edges to stable -//! node IDs. Serving executes this graph without reconstructing Planner IR or -//! searching for compatible materializations. +//! Control-plane lowering from Planner IR to the shared installed query DAG. +//! Serving consumes asap_types::query_plan; compilation stays in this component. mod clickhouse_exact; pub mod logical; - -use std::collections::{BTreeMap, BTreeSet}; -use std::rc::Rc; - -use planner_types::post_asap::{SketchQuery, SummaryExpr, SummaryFamilyType, SummaryNode}; +pub use asap_types::query_plan::*; +#[cfg(test)] +use asap_types::PolicyFingerprint; +use planner_types::post_asap::{SummaryExpr, SummaryFamilyType, SummaryNode}; use planner_types::pre_asap::Reduction; -use serde::{Deserialize, Serialize}; -use thiserror::Error; - -pub use asap_types::QueryLanguage; -use asap_types::{sds::SummaryDefinitionId, PolicyFingerprint}; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct QueryPlan { - pub plan_id: u64, - pub plan_version: u64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub clickhouse_context: Option, - pub entries: BTreeMap, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct ClickHousePlanningContext { - pub tables: std::collections::HashMap, - pub accuracy: planner_types::types::AccuracyTarget, -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct FixedEvaluationRange { - pub start_ms: u64, - pub end_ms: u64, - pub cumulative: bool, -} - -impl QueryPlan { - pub fn empty() -> Self { - Self { - plan_id: 0, - plan_version: 0, - clickhouse_context: None, - entries: BTreeMap::new(), - } - } - - pub fn lookup(&self, promql: &str) -> Result<&QueryPlanEntry, QueryPlanError> { - let identity = canonical_promql(promql)?; - self.lookup_canonical(QueryLanguage::PromQl, &identity) - } - - pub fn lookup_canonical( - &self, - language: QueryLanguage, - identity: &str, - ) -> Result<&QueryPlanEntry, QueryPlanError> { - let key = Self::catalog_key(language, identity); - self.entries - .get(&key) - .filter(|entry| entry.language == language) - .ok_or_else(|| QueryPlanError::QueryNotPlanned(identity.into())) - } - - pub fn catalog_key(language: QueryLanguage, identity: &str) -> String { - match language { - QueryLanguage::PromQl => identity.to_owned(), - QueryLanguage::MetricsQl => format!("metricsql:{identity}"), - QueryLanguage::ClickHouseSql => format!("clickhouse:{identity}"), - } - } - - pub fn lookup_clickhouse( - &self, - canonical_sql: &str, - ) -> Result<&QueryPlanEntry, QueryPlanError> { - self.lookup_canonical(QueryLanguage::ClickHouseSql, canonical_sql) - } - - /// Validate semantic bindings against the authoritative snapshot before use. - pub fn validate_against_catalog( - &self, - catalog: &crate::physical::summary_catalog::SummaryCatalog, - ) -> Result<(), QueryPlanError> { - catalog - .validate() - .map_err(|error| QueryPlanError::Invalid(error.to_string()))?; - if self.plan_id != catalog.plan_id || self.plan_version != catalog.plan_version { - return Err(QueryPlanError::Invalid( - "QueryPlan and SummaryCatalog have different plan identity/version".into(), - )); - } - let available = catalog - .materializations - .keys() - .copied() - .map(Into::into) - .collect(); - self.validate(&available)?; - for entry in self.entries.values() { - for binding in entry.materialization_bindings() { - let identity = catalog - .materializations - .get(&binding.materialization) - .ok_or_else(|| { - QueryPlanError::Invalid( - "query binding references absent catalog materialization".into(), - ) - })?; - let _data = &catalog.data_descriptors[&identity.data_descriptor_id]; - if binding.window_ms == 0 { - return Err(QueryPlanError::Invalid( - "zero physical pane duration".into(), - )); - } - if binding.pane_origin_ms != identity.pane_origin_ms { - return Err(QueryPlanError::Invalid( - "query pane origin differs from catalog definition".into(), - )); - } - } - for node in entry.nodes.values() { - let QueryPlanNode::ExactReadout { input, readout } = node else { - continue; - }; - if !matches!(readout, ExactReadout::Increase | ExactReadout::Rate) { - continue; - } - let Some(QueryPlanNode::ReadMaterialization { binding }) = entry.nodes.get(input) - else { - return Err(QueryPlanError::Invalid( - "counter readout must directly consume one catalog materialization".into(), - )); - }; - let identity = &catalog.materializations[&binding.materialization]; - let descriptor = &catalog.summary_descriptors[&identity.summary_descriptor_id]; - if !matches!( - descriptor.fidelity, - asap_types::sds::FidelityGuarantee::ExactCounter { - full_pane_coverage_required: true, - .. - } - ) { - return Err(QueryPlanError::Invalid( - "rate/increase binding does not reference an exact counter SDS".into(), - )); - } - } - } - Ok(()) - } - - pub fn validate(&self, available: &BTreeSet) -> Result<(), QueryPlanError> { - if self.plan_id != 0 && self.plan_version == 0 { - return Err(QueryPlanError::Invalid( - "non-bootstrap QueryPlan has zero plan_version".into(), - )); - } - for (identity, entry) in &self.entries { - let expected = Self::catalog_key(entry.language, &entry.canonical_query); - if identity != &expected { - return Err(QueryPlanError::Invalid(format!( - "query map key `{identity}` differs from entry identity `{}`", - entry.canonical_query - ))); - } - match entry.language { - QueryLanguage::PromQl | QueryLanguage::MetricsQl - if entry.fixed_evaluation.is_some() => - { - return Err(QueryPlanError::Invalid( - "PromQL query entry carries a ClickHouse fixed evaluation range".into(), - )); - } - QueryLanguage::ClickHouseSql => { - if self.clickhouse_context.is_none() { - return Err(QueryPlanError::Invalid( - "ClickHouse query entry has no planning context".into(), - )); - } - let Some(range) = entry.fixed_evaluation else { - return Err(QueryPlanError::Invalid( - "ClickHouse query entry has no fixed evaluation range".into(), - )); - }; - if range.end_ms <= range.start_ms { - return Err(QueryPlanError::Invalid( - "ClickHouse query entry has an empty evaluation range".into(), - )); - } - } - QueryLanguage::PromQl | QueryLanguage::MetricsQl => {} - } - entry.validate(available)?; - } - Ok(()) - } -} - -pub use asap_types::executable_plan::QueryNodeId; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct QueryPlanEntry { - #[serde(default)] - pub language: QueryLanguage, - pub query_id: String, - #[serde(alias = "canonical_promql")] - pub canonical_query: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub fixed_evaluation: Option, - pub root: QueryNodeId, - pub nodes: BTreeMap, - pub instant: InstantExecution, - pub fallback: FallbackPolicy, -} +use std::collections::BTreeMap; +#[cfg(test)] +use std::collections::BTreeSet; +use std::rc::Rc; -fn topological_order( - root: QueryNodeId, - nodes: &BTreeMap, -) -> Result, QueryPlanError> { - fn visit( - id: QueryNodeId, - nodes: &BTreeMap, - visiting: &mut BTreeSet, - visited: &mut BTreeSet, - out: &mut Vec, - ) -> Result<(), QueryPlanError> { - if visited.contains(&id) { - return Ok(()); - } - if !visiting.insert(id) { - return Err(QueryPlanError::Invalid(format!( - "cycle detected at query node {}", - id.0 - ))); - } - let node = nodes - .get(&id) - .ok_or_else(|| QueryPlanError::Invalid(format!("missing query node {}", id.0)))?; - for input in node.inputs() { - visit(*input, nodes, visiting, visited, out)?; - } - visiting.remove(&id); - visited.insert(id); - out.push(id); - Ok(()) - } - let mut out = Vec::with_capacity(nodes.len()); - visit( +pub fn compile_bound( + query_id: String, + canonical_query: String, + root: &Rc, + instant: InstantExecution, + fallback: FallbackPolicy, + bind: F, +) -> Result +where + F: FnMut( + &Rc, + &SummaryFamilyType, + ) -> Result, +{ + compile_bound_mapped( + query_id, + canonical_query, root, - nodes, - &mut BTreeSet::new(), - &mut BTreeSet::new(), - &mut out, - )?; - Ok(out) -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct InstantExecution { - pub lookback_ms: u64, - pub full_history: bool, - pub cumulative_readout: bool, -} - -impl QueryPlanEntry { - /// Replace an explicit planner fallback cut with a typed external-exact leaf. - /// The control plane chooses the cut; serving only executes the published DAG. - pub fn bind_external_exact_leaf( - &mut self, - node_id: QueryNodeId, - request: ExternalExactRequest, - ) -> Result<(), QueryPlanError> { - if request.language != self.language { - return Err(QueryPlanError::Invalid( - "external exact language differs from its query plan".into(), - )); - } - if !request.input_contracts.is_empty() { - return Err(QueryPlanError::Invalid( - "leaf binding cannot declare DAG input contracts".into(), - )); - } - match self.nodes.get(&node_id) { - Some(QueryPlanNode::ExactFallback { .. }) => {} - Some(_) => { - return Err(QueryPlanError::Invalid( - "external exact binding must replace a planner fallback cut".into(), - )) - } - None => { - return Err(QueryPlanError::Invalid( - "external exact cut node is absent".into(), - )) - } - } - self.nodes.insert( - node_id, - QueryPlanNode::ExternalExact { - request, - inputs: Vec::new(), - }, - ); - Ok(()) - } - - /// Materializations this executable DAG reads, in stable node order. - /// Serving uses this set for readiness accounting; it never performs a - /// catalog candidate search to reconstruct dependencies. - pub fn materialization_bindings(&self) -> Vec<&MaterializationBinding> { - self.nodes - .values() - .filter_map(|node| match node { - QueryPlanNode::ReadMaterialization { binding } => Some(binding), - _ => None, - }) - .collect() - } - - pub fn topological_order(&self) -> Result, QueryPlanError> { - topological_order(self.root, &self.nodes) - } - - pub fn topological_order_from( - &self, - root: QueryNodeId, - ) -> Result, QueryPlanError> { - topological_order(root, &self.nodes) - } - - pub fn compile_bound( - query_id: String, - canonical_query: String, - root: &Rc, - instant: InstantExecution, - fallback: FallbackPolicy, - bind: F, - ) -> Result - where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - { - Self::compile_bound_mapped( - query_id, - canonical_query, - root, - instant, - fallback, - bind, - |_, _| {}, - ) - } - - pub fn compile_bound_mapped( - query_id: String, - canonical_query: String, - root: &Rc, - instant: InstantExecution, - fallback: FallbackPolicy, - mut bind: F, - mut lowered: G, - ) -> Result - where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - G: FnMut(&Rc, QueryNodeId), - { - let mut compiler = DagCompiler { - next_id: 0, - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - bind: &mut bind, - logical_source: None, - preserve_relational: false, - lowered: Some(&mut lowered), - }; - let root = compiler.lower(root)?; - Ok(Self { - language: QueryLanguage::PromQl, - query_id, - canonical_query, - fixed_evaluation: None, - root, - nodes: compiler.nodes, - instant, - fallback, - }) - } - - pub fn compile_bound_relational( - query_id: String, - canonical_query: String, - root: &Rc, - fixed_evaluation: FixedEvaluationRange, - instant: InstantExecution, - fallback: FallbackPolicy, - bind: F, - ) -> Result - where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - { - Self::compile_bound_relational_mapped( - query_id, - canonical_query, - root, - fixed_evaluation, - instant, - fallback, - bind, - |_, _| {}, - ) - } - - /// Preserve Planner-to-runtime node identities for installed SQL DAGs. - pub fn compile_bound_relational_mapped( - query_id: String, - canonical_query: String, - root: &Rc, - fixed_evaluation: FixedEvaluationRange, - instant: InstantExecution, - fallback: FallbackPolicy, - mut bind: F, - mut lowered: G, - ) -> Result - where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - G: FnMut(&Rc, QueryNodeId), - { - let mut compiler = DagCompiler { - next_id: 0, - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - bind: &mut bind, - logical_source: None, - preserve_relational: true, - lowered: Some(&mut lowered), - }; - let root = compiler.lower(root)?; - Ok(Self { - language: QueryLanguage::ClickHouseSql, - query_id, - canonical_query, - fixed_evaluation: Some(fixed_evaluation), - root, - nodes: compiler.nodes, - instant, - fallback, - }) - } - - /// Compile selected summary nodes and verified native residuals into one DAG. - /// This is a distinct physical alternative; native execution remains available. - pub fn compile_bound_composable( - query_id: String, - canonical_query: String, - root: &Rc, - instant: InstantExecution, - fallback: FallbackPolicy, - bind: F, - ) -> Result - where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - { - Self::compile_bound_composable_mapped( - query_id, - canonical_query, - root, - instant, - fallback, - bind, - |_, _| {}, - ) - } - - /// Compile a composable query while exposing the stable mapping from - /// Planner semantic nodes to installed query nodes. The control-plane - /// physical compiler uses this to persist backend placement without - /// relying on pointer values or reconstructing query shape later. - pub fn compile_bound_composable_mapped( - query_id: String, - canonical_query: String, - root: &Rc, - instant: InstantExecution, - fallback: FallbackPolicy, - mut bind: F, - mut lowered: G, - ) -> Result - where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - G: FnMut(&Rc, QueryNodeId), - { - let mut compiler = DagCompiler { - next_id: 0, - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - bind: &mut bind, - logical_source: Some(canonical_query.clone()), - preserve_relational: false, - lowered: Some(&mut lowered), - }; - let root = compiler.lower(root)?; - let mut entry = Self { - language: QueryLanguage::PromQl, - query_id, - canonical_query, - fixed_evaluation: None, - root, - nodes: compiler.nodes, - instant, - fallback, - }; - logical::finalize_residuals(&mut entry)?; - Ok(entry) - } - - /// Validate references, bindings, reachability, and cycles before activation. - pub fn validate(&self, available: &BTreeSet) -> Result<(), QueryPlanError> { - if !self.nodes.contains_key(&self.root) { - return Err(QueryPlanError::Invalid(format!( - "query `{}` has missing root {}", - self.query_id, self.root.0 - ))); - } - for (id, node) in &self.nodes { - if let QueryPlanNode::Logical { operator, inputs } = node { - operator.validate(inputs.len())?; - } - if matches!(node, QueryPlanNode::Scalar { value } if !value.is_finite()) { - return Err(QueryPlanError::Invalid("non-finite scalar constant".into())); - } - if let QueryPlanNode::ExternalExact { request, inputs } = node { - if request.expression.trim().is_empty() { - return Err(QueryPlanError::Invalid( - "external exact expression must not be empty".into(), - )); - } - if request.input_contracts.len() != inputs.len() { - return Err(QueryPlanError::Invalid( - "external exact input contracts must match DAG inputs".into(), - )); - } - if request.input_contracts.iter().any(|contract| { - matches!(contract, ExternalExactInput::CandidateMembership { item_label } if item_label.is_empty()) - }) { - return Err(QueryPlanError::Invalid( - "external exact candidate item label must not be empty".into(), - )); - } - } - if let QueryPlanNode::CandidateTopK { - k, completeness, .. - } = node - { - if *k == 0 { - return Err(QueryPlanError::Invalid( - "CandidateTopK requires k > 0".into(), - )); - } - if matches!( - completeness, - CandidateCompleteness::Certified { guarantee } - if guarantee.metric - != planner_types::post_asap::ErrorMetric::TopKMembership - || guarantee.bound.evaluate().is_none() - || guarantee.failure_probability.evaluate().is_none() - ) { - return Err(QueryPlanError::Invalid( - "invalid CandidateTopK completeness certificate".into(), - )); - } - } - for input in node.inputs() { - if !self.nodes.contains_key(input) { - return Err(QueryPlanError::Invalid(format!( - "query `{}` node {} references missing input {}", - self.query_id, id.0, input.0 - ))); - } - } - if let QueryPlanNode::ReadMaterialization { binding } = node { - if binding.readout_lookback_ms == Some(0) { - return Err(QueryPlanError::Invalid( - "zero semantic readout lookback".into(), - )); - } - if !available.contains(&binding.materialization.fingerprint()) { - return Err(QueryPlanError::Invalid(format!( - "query `{}` node {} references absent materialization {}", - self.query_id, - id.0, - binding.materialization.as_u64() - ))); - } - } - } - let order = self.topological_order()?; - if order.len() != self.nodes.len() { - return Err(QueryPlanError::Invalid(format!( - "query `{}` contains unreachable nodes", - self.query_id - ))); - } - Ok(()) - } -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum FallbackPolicy { - ExactBackend, - Reject, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct MaterializationBinding { - pub materialization: SummaryDefinitionId, - /// Query operator grouping applied while folding those SIDs. - pub output_grouping: PhysicalGrouping, - /// Labels whose values form an item identity inside a keyed sketch. - #[serde(default, alias = "itemLabels", skip_serializing_if = "Vec::is_empty")] - pub item_labels: Vec, - pub window_ms: u64, - /// Unix millisecond timestamp on the materialized pane-boundary grid. - /// Legacy plans deserialize this as unknown and fall back at read time. - #[serde( - default, - alias = "paneOriginMs", - skip_serializing_if = "Option::is_none" - )] - pub pane_origin_ms: Option, - /// Semantic query lookback, independent of the physical pane duration. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub readout_lookback_ms: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "mode", content = "keys", rename_all = "snake_case")] -pub enum PhysicalGrouping { - PerEntity, - Reduce(Vec), + instant, + fallback, + bind, + |_, _| {}, + ) } -/// Result shape promised by an external exact engine. The backend uses this -/// contract to type-check downstream DAG nodes without depending on an -/// engine-specific response envelope. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum ExternalExactOutput { - Scalar, - InstantVector, - RangeVector, - Relation { schema: serde_json::Value }, -} - -/// How an ordinary DAG input constrains an external exact evaluation. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum ExternalExactInput { - CandidateMembership { item_label: String }, -} - -/// Language-neutral request contract for an exact subtree. Evaluation time is -/// inherited from the containing QueryPlanEntry, avoiding a second time-range -/// envelope that could drift from the installed query plan. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct ExternalExactRequest { - pub language: QueryLanguage, - pub expression: String, - pub output: ExternalExactOutput, - /// Engine parameters forwarded without embedding transport details in the DAG. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub parameters: BTreeMap, - /// Optional parameter names populated from the query entry's evaluation range. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub start_parameter: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub end_parameter: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub input_contracts: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(tag = "op", rename_all = "snake_case", deny_unknown_fields)] -pub enum QueryPlanNode { - RelationalJoin { - inputs: [QueryNodeId; 2], - join_kind: planner_types::pre_asap::JoinKind, - pred: serde_json::Value, - left_schema: planner_types::post_asap::SummarySchema, - right_schema: planner_types::post_asap::SummarySchema, - output_schema: planner_types::post_asap::SummarySchema, - }, - Relational { - input: QueryNodeId, - /// Serialized planner-owned operation. Keeping the wire form here makes - /// the published catalog Send + Sync even though the planner AST uses Rc. - operation: serde_json::Value, - input_schema: planner_types::post_asap::SummarySchema, - output_schema: planner_types::post_asap::SummarySchema, - }, - Logical { - operator: logical::LogicalOperator, - inputs: Vec, - }, - Scalar { - value: f64, - }, - Binary { - inputs: [QueryNodeId; 2], - operator: planner_types::pre_asap::ArithmeticOpKind, - }, - ReduceSum { - input: QueryNodeId, - grouping: PhysicalGrouping, - }, - ReadMaterialization { - binding: MaterializationBinding, - }, - SummaryEstimate { - input: QueryNodeId, - query: QueryReadout, - }, - ExactReadout { - input: QueryNodeId, - readout: ExactReadout, - }, - SummaryMerge { - inputs: Vec, - }, - /// Use an approximate heap only as a membership sidecar, then rerank the - /// matching exact counter readouts. `inputs[0]` is candidate membership; - /// `inputs[1]` is the authoritative exact value vector. - CandidateTopK { - inputs: [QueryNodeId; 2], - k: u64, - grouping: logical::Grouping, - completeness: CandidateCompleteness, - }, - /// An exact subtree evaluated outside ASAP. Its results enter the query DAG - /// like any other node output and may depend on summary-produced inputs. - ExternalExact { - request: ExternalExactRequest, - inputs: Vec, - }, - ExactFallback { - reason: String, - }, -} - -impl QueryPlanNode { - pub fn inputs(&self) -> &[QueryNodeId] { - match self { - Self::Scalar { .. } | Self::ReadMaterialization { .. } | Self::ExactFallback { .. } => { - &[] - } - Self::Binary { inputs, .. } | Self::RelationalJoin { inputs, .. } => inputs, - Self::ReduceSum { input, .. } - | Self::Relational { input, .. } - | Self::SummaryEstimate { input, .. } - | Self::ExactReadout { input, .. } => std::slice::from_ref(input), - Self::SummaryMerge { inputs } - | Self::Logical { inputs, .. } - | Self::ExternalExact { inputs, .. } => inputs, - Self::CandidateTopK { inputs, .. } => inputs, - } - } +pub fn compile_bound_mapped( + query_id: String, + canonical_query: String, + root: &Rc, + instant: InstantExecution, + fallback: FallbackPolicy, + mut bind: F, + mut lowered: G, +) -> Result +where + F: FnMut( + &Rc, + &SummaryFamilyType, + ) -> Result, + G: FnMut(&Rc, QueryNodeId), +{ + let mut compiler = DagCompiler { + next_id: 0, + nodes: BTreeMap::new(), + seen: BTreeMap::new(), + bind: &mut bind, + logical_source: None, + preserve_relational: false, + lowered: Some(&mut lowered), + }; + let root = compiler.lower(root)?; + Ok(QueryPlanEntry { + language: QueryLanguage::PromQl, + query_id, + canonical_query, + fixed_evaluation: None, + root, + nodes: compiler.nodes, + instant, + fallback, + }) } -pub use planner_types::post_asap::CandidateCompleteness; - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum ExactReadout { - Sum, - Count, - Increase, - Rate, - Max, +pub fn compile_bound_relational( + query_id: String, + canonical_query: String, + root: &Rc, + fixed_evaluation: FixedEvaluationRange, + instant: InstantExecution, + fallback: FallbackPolicy, + bind: F, +) -> Result +where + F: FnMut( + &Rc, + &SummaryFamilyType, + ) -> Result, +{ + compile_bound_relational_mapped( + query_id, + canonical_query, + root, + fixed_evaluation, + instant, + fallback, + bind, + |_, _| {}, + ) } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum QueryReadout { - FrequencyL2, - FrequencyEntropy, - Quantile { - q: f64, - }, - PointCount { - key: planner_types::pre_asap::ColumnRef, - value: Option, - }, - Cardinality, - TopK { - k: usize, - }, +/// Preserve Planner-to-runtime node identities for installed SQL DAGs. +pub fn compile_bound_relational_mapped( + query_id: String, + canonical_query: String, + root: &Rc, + fixed_evaluation: FixedEvaluationRange, + instant: InstantExecution, + fallback: FallbackPolicy, + mut bind: F, + mut lowered: G, +) -> Result +where + F: FnMut( + &Rc, + &SummaryFamilyType, + ) -> Result, + G: FnMut(&Rc, QueryNodeId), +{ + let mut compiler = DagCompiler { + next_id: 0, + nodes: BTreeMap::new(), + seen: BTreeMap::new(), + bind: &mut bind, + logical_source: None, + preserve_relational: true, + lowered: Some(&mut lowered), + }; + let root = compiler.lower(root)?; + Ok(QueryPlanEntry { + language: QueryLanguage::ClickHouseSql, + query_id, + canonical_query, + fixed_evaluation: Some(fixed_evaluation), + root, + nodes: compiler.nodes, + instant, + fallback, + }) } -impl From for QueryReadout { - fn from(query: SketchQuery) -> Self { - match query { - SketchQuery::FrequencyL2 => Self::FrequencyL2, - SketchQuery::FrequencyEntropy => Self::FrequencyEntropy, - SketchQuery::Quantile { q } => Self::Quantile { q }, - SketchQuery::PointCount { key, value } => Self::PointCount { key, value }, - SketchQuery::Cardinality => Self::Cardinality, - SketchQuery::TopK { k } => Self::TopK { k }, - } - } +/// Compile selected summary nodes and verified native residuals into one DAG. +/// This is a distinct physical alternative; native execution remains available. +pub fn compile_bound_composable( + query_id: String, + canonical_query: String, + root: &Rc, + instant: InstantExecution, + fallback: FallbackPolicy, + bind: F, +) -> Result +where + F: FnMut( + &Rc, + &SummaryFamilyType, + ) -> Result, +{ + compile_bound_composable_mapped( + query_id, + canonical_query, + root, + instant, + fallback, + bind, + |_, _| {}, + ) } -impl From for SketchQuery { - fn from(query: QueryReadout) -> Self { - match query { - QueryReadout::FrequencyL2 => Self::FrequencyL2, - QueryReadout::FrequencyEntropy => Self::FrequencyEntropy, - QueryReadout::Quantile { q } => Self::Quantile { q }, - QueryReadout::PointCount { key, value } => Self::PointCount { key, value }, - QueryReadout::Cardinality => Self::Cardinality, - QueryReadout::TopK { k } => Self::TopK { k }, - } - } +/// Compile a composable query while exposing the stable mapping from +/// Planner semantic nodes to installed query nodes. The control-plane +/// physical compiler uses this to persist backend placement without +/// relying on pointer values or reconstructing query shape later. +pub fn compile_bound_composable_mapped( + query_id: String, + canonical_query: String, + root: &Rc, + instant: InstantExecution, + fallback: FallbackPolicy, + mut bind: F, + mut lowered: G, +) -> Result +where + F: FnMut( + &Rc, + &SummaryFamilyType, + ) -> Result, + G: FnMut(&Rc, QueryNodeId), +{ + let mut compiler = DagCompiler { + next_id: 0, + nodes: BTreeMap::new(), + seen: BTreeMap::new(), + bind: &mut bind, + logical_source: Some(canonical_query.clone()), + preserve_relational: false, + lowered: Some(&mut lowered), + }; + let root = compiler.lower(root)?; + let mut entry = QueryPlanEntry { + language: QueryLanguage::PromQl, + query_id, + canonical_query, + fixed_evaluation: None, + root, + nodes: compiler.nodes, + instant, + fallback, + }; + logical::finalize_residuals(&mut entry)?; + Ok(entry) } struct DagCompiler<'a, F> { @@ -1614,24 +977,6 @@ fn physical_grouping( Ok(PhysicalGrouping::Reduce(names)) } -#[derive(Debug, Error)] -pub enum QueryPlanError { - #[error("invalid PromQL query identity: {0}")] - InvalidPromql(String), - #[error("query is absent from the active QueryPlan: {0}")] - QueryNotPlanned(String), - #[error("post-ASAP DAG cannot be represented by the query executor: {0}")] - UnsupportedNode(String), - #[error("invalid QueryPlan: {0}")] - Invalid(String), -} - -pub fn canonical_promql(query: &str) -> Result { - promql_parser::parser::parse(query.trim()) - .map(|expr| expr.to_string()) - .map_err(|error| QueryPlanError::InvalidPromql(error.to_string())) -} - #[cfg(test)] mod tests { use super::*; diff --git a/control_plane/src/query_plan/clickhouse_exact.rs b/control_plane/src/query_plan/clickhouse_exact.rs index 9c8455429..85f476093 100644 --- a/control_plane/src/query_plan/clickhouse_exact.rs +++ b/control_plane/src/query_plan/clickhouse_exact.rs @@ -361,8 +361,8 @@ mod original_tests { if sql.contains("sum(value)") && sql.contains("argMax") { use crate::physical::post_asap::{PhysicalExpr, PostAsapPlan}; use crate::query_plan::{ - FallbackPolicy, FixedEvaluationRange, InstantExecution, QueryPlanEntry, - QueryPlanError, QueryPlanNode, + FallbackPolicy, FixedEvaluationRange, InstantExecution, QueryPlanError, + QueryPlanNode, }; let planned = crate::clickhouse::plan_clickhouse_sql(sql, &catalog, AccuracyTarget::Exact) @@ -371,7 +371,7 @@ mod original_tests { let PhysicalExpr::Committed(PostAsapPlan::Summary(root)) = planned.physical else { panic!("missing selected SQL DAG") }; - let entry = QueryPlanEntry::compile_bound_relational( + let entry = crate::query_plan::compile_bound_relational( "test".into(), planned.canonical_sql, &root, diff --git a/control_plane/src/query_plan/logical.rs b/control_plane/src/query_plan/logical.rs index 05d474715..6b20a5a84 100644 --- a/control_plane/src/query_plan/logical.rs +++ b/control_plane/src/query_plan/logical.rs @@ -6,62 +6,13 @@ use promql_parser::{ label::MatchOp, parser::{self, Expr, LabelModifier, Offset, VectorSelector}, }; -use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum LogicalOperator { - /// A maximal exact scalar/vector subtree evaluated by Prometheus. - ExactSubquery { - query: String, - }, - /// Prometheus exact subtree whose selectors are restricted at runtime by - /// the candidate vector produced by its single input. - CandidateExactSubquery { - query: String, - item_label: String, - }, - Scan { - metric: Option, - matchers: Vec, - range_ms: Option, - offset_ms: i64, - }, - UnaryNegate, - VectorToScalar, - Aggregate { - operation: Aggregation, - grouping: Grouping, - }, - /// PromQL `topk(k, vector)` selection over values produced by the child. - /// This is distinct from a frequency-sketch TopK readout: any exact or - /// summary-backed instant-vector child may feed this query-time operator. - TopKSelection { - k: u64, - grouping: Grouping, - }, - Binary { - operation: BinaryOperation, - return_bool: bool, - }, - Temporal { - operation: TemporalOperation, - }, - Sort { - descending: bool, - }, - HistogramQuantile, - Subquery { - range_ms: u64, - step_ms: u64, - offset_ms: i64, - }, -} +pub use asap_types::query_plan::logical::*; /// Stable identity of a Planner-authorized materializable DAG leaf. This is a /// workload-selection key, not another physical materialization definition. -#[derive(Debug, Clone, Serialize, PartialEq)] +#[derive(Debug, Clone, serde::Serialize, PartialEq)] struct MaterializationCandidateIdentity { metric: String, matchers: Vec, @@ -69,63 +20,6 @@ struct MaterializationCandidateIdentity { offset_ms: i64, operation: TemporalOperation, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct Grouping { - pub labels: Vec, - pub without: bool, -} -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct LabelMatcher { - pub name: String, - pub value: String, - pub operation: LabelMatch, -} -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum LabelMatch { - Equal, - NotEqual, - Regex, - NotRegex, -} -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum Aggregation { - Sum, - Max, - Min, - Avg, - Count, -} -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum BinaryOperation { - Add, - Sub, - Mul, - Div, - Mod, - Pow, - Equal, - NotEqual, - Less, - LessEqual, - Greater, - GreaterEqual, -} -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum TemporalOperation { - Rate, - Increase, - Avg, - Max, - Min, - Sum, - Count, -} fn invalid(message: impl Into) -> QueryPlanError { QueryPlanError::Invalid(message.into()) @@ -143,46 +37,6 @@ fn offset(value: &Option) -> Result { } } -impl LogicalOperator { - pub fn validate(&self, inputs: usize) -> Result<(), QueryPlanError> { - let expected = match self { - Self::Scan { .. } | Self::ExactSubquery { .. } => 0, - Self::CandidateExactSubquery { .. } => 1, - Self::Binary { .. } | Self::HistogramQuantile => 2, - _ => 1, - }; - if inputs != expected { - return Err(invalid("logical operator input arity mismatch")); - } - if matches!( - self, - Self::Scan { - range_ms: Some(0), - .. - } - ) { - return Err(invalid("zero range")); - } - if let Self::ExactSubquery { query } | Self::CandidateExactSubquery { query, .. } = self { - let parsed = parser::parse(query).map_err(|e| invalid(e.to_string()))?; - if matches!(parsed, Expr::MatrixSelector(_) | Expr::Subquery(_)) { - return Err(invalid( - "exact subtree boundary must return scalar or instant vector", - )); - } - } - if let Self::Subquery { - range_ms, step_ms, .. - } = self - { - if *range_ms == 0 || *step_ms == 0 || range_ms / step_ms > 100_000 { - return Err(invalid("invalid or excessive subquery grid")); - } - } - Ok(()) - } -} - struct Lower { nodes: BTreeMap, seen: BTreeMap, @@ -409,53 +263,32 @@ impl Lower { } } -impl QueryPlanEntry { - /// Lower a Planner-authorized native residual into typed backend operations. - /// Callers retain a separate external-native alternative for cost comparison. - pub fn compile_logical( - query_id: String, - canonical_query: String, - instant: InstantExecution, - fallback: FallbackPolicy, - ) -> Result { - let expr = parser::parse(&canonical_query).map_err(|e| invalid(e.to_string()))?; - let mut lower = Lower { - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - }; - let root = lower.lower(&expr)?; - let entry = Self { - language: super::QueryLanguage::PromQl, - query_id, - canonical_query, - fixed_evaluation: None, - root, - nodes: lower.nodes, - instant, - fallback, - }; - entry.validate(&Default::default())?; - Ok(entry) - } - /// Promote only a wholly native entry; never discard selected summary bindings. - pub fn lower_native_residual(&self) -> Result { - if self.nodes.len() != 1 - || !matches!( - self.nodes.get(&self.root), - Some(QueryPlanNode::ExactFallback { .. }) - ) - { - return Err(invalid( - "logical residual promotion requires a whole native root", - )); - } - Self::compile_logical( - self.query_id.clone(), - self.canonical_query.clone(), - self.instant, - self.fallback, - ) - } +/// Lower a Planner-authorized native residual into typed backend operations. +/// Callers retain a separate external-native alternative for cost comparison. +pub fn compile_logical( + query_id: String, + canonical_query: String, + instant: InstantExecution, + fallback: FallbackPolicy, +) -> Result { + let expr = parser::parse(&canonical_query).map_err(|e| invalid(e.to_string()))?; + let mut lower = Lower { + nodes: BTreeMap::new(), + seen: BTreeMap::new(), + }; + let root = lower.lower(&expr)?; + let entry = QueryPlanEntry { + language: super::QueryLanguage::PromQl, + query_id, + canonical_query, + fixed_evaluation: None, + root, + nodes: lower.nodes, + instant, + fallback, + }; + entry.validate(&Default::default())?; + Ok(entry) } /// Match residuals by semantic IR equality, not display text or source names. @@ -555,7 +388,7 @@ mod tests { serde_json::from_str(include_str!("../../tests/fixtures/o11y_queries.json")).unwrap(); for row in corpus["queries"].as_array().unwrap() { let query = row["query"].as_str().unwrap(); - let entry = QueryPlanEntry::compile_logical( + let entry = crate::query_plan::logical::compile_logical( row["id"].as_str().unwrap().into(), query.into(), instant(), @@ -587,7 +420,7 @@ mod tests { #[test] fn repeated_subexpressions_share_node_identity() { // Serialized edges must retain CSE rather than duplicating raw work. - let entry = QueryPlanEntry::compile_logical( + let entry = crate::query_plan::logical::compile_logical( "q".into(), "sum(up) / sum(up)".into(), instant(), @@ -633,7 +466,7 @@ mod tests { 3, ), ] { - let entry = QueryPlanEntry::compile_logical( + let entry = crate::query_plan::logical::compile_logical( "topk".into(), query.into(), instant(), @@ -652,7 +485,7 @@ mod tests { #[test] fn topk_keeps_unsupported_child_as_exact_leaf() { - let entry = QueryPlanEntry::compile_logical( + let entry = crate::query_plan::logical::compile_logical( "topk-subquery".into(), "topk(3, label_replace(memory_bytes, \"dst\", \"$1\", \"src\", \"(.*)\"))".into(), instant(), @@ -681,7 +514,7 @@ mod tests { ("topk by (cluster) (2, m)", vec!["cluster"], false), ("topk without (pod) (2, m)", vec!["pod"], true), ] { - let entry = QueryPlanEntry::compile_logical( + let entry = crate::query_plan::logical::compile_logical( "topk-group".into(), query.into(), instant(), @@ -799,7 +632,7 @@ mod hybrid_tests { .unwrap(); let selected = crate::planner_selection::select_summary_default(&canonical).unwrap(); let entry = - QueryPlanEntry::compile_bound_composable( + crate::query_plan::compile_bound_composable( "hybrid".into(), query.into(), &selected, diff --git a/control_plane/tests/o11y_exact_fallback.rs b/control_plane/tests/o11y_exact_fallback.rs index 6f0b12d80..e43508a03 100644 --- a/control_plane/tests/o11y_exact_fallback.rs +++ b/control_plane/tests/o11y_exact_fallback.rs @@ -26,7 +26,7 @@ fn o11y_non_summary_roots_preserve_exact_query_semantics() { }; assert_eq!(original.as_ref(), &expr, "query semantics changed: {query}"); assert_eq!(node.schema.fields.len(), expr.output_schema().unwrap().columns.len()); - let executable = control_plane::query_plan::QueryPlanEntry::compile_bound( + let executable = control_plane::query_plan::compile_bound( "fixture".into(), query.into(), &node, control_plane::query_plan::InstantExecution { lookback_ms: 300_000, full_history: false, cumulative_readout: false, diff --git a/control_plane/tests/offline_evidence.rs b/control_plane/tests/offline_evidence.rs index 769bdd6bd..b36bd3fdb 100644 --- a/control_plane/tests/offline_evidence.rs +++ b/control_plane/tests/offline_evidence.rs @@ -346,9 +346,7 @@ fn incompatible_evidence_preserves_deployment_behavior() { /// fallback even though the warm tier now supports exact additive binaries. #[test] fn binary_summary_has_explicit_warm_tier_fallback() { - use control_plane::query_plan::{ - FallbackPolicy, InstantExecution, QueryPlanEntry, QueryPlanNode, - }; + use control_plane::query_plan::{FallbackPolicy, InstantExecution, QueryPlanNode}; use planner_types::{post_asap::BinaryOperator, pre_asap::BinaryOpKind}; let child = bound(&model()); let root = std::rc::Rc::new(SummaryNode { @@ -363,7 +361,7 @@ fn binary_summary_has_explicit_warm_tier_fallback() { schema: child.schema.clone(), guarantee: None, }); - let plan = QueryPlanEntry::compile_bound( + let plan = control_plane::query_plan::compile_bound( "test".into(), "left / right".into(), &root, diff --git a/crates/asap_types/Cargo.toml b/crates/asap_types/Cargo.toml index a83ea156c..671d34ca3 100644 --- a/crates/asap_types/Cargo.toml +++ b/crates/asap_types/Cargo.toml @@ -4,6 +4,7 @@ version.workspace = true edition.workspace = true [dependencies] +promql-parser.workspace = true base64 = "0.21" tracing.workspace = true serde.workspace = true diff --git a/crates/asap_types/src/lib.rs b/crates/asap_types/src/lib.rs index 3c0a23997..0b4f40eee 100644 --- a/crates/asap_types/src/lib.rs +++ b/crates/asap_types/src/lib.rs @@ -33,3 +33,5 @@ pub use routing_index::RoutingIndex; pub use storage_backend::*; pub mod precompute_plan; + +pub mod query_plan; diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs new file mode 100644 index 000000000..945608afe --- /dev/null +++ b/crates/asap_types/src/query_plan.rs @@ -0,0 +1,677 @@ +//! Shared installed QueryPlan contract and activation validation. +//! +//! ASAPPlanner owns semantic post-ASAP IR. Physical compilation binds every +//! maintained-summary leaf to one materialization and lowers edges to stable +//! node IDs. Serving executes this graph without reconstructing Planner IR or +//! searching for compatible materializations. + +pub mod logical; + +use std::collections::{BTreeMap, BTreeSet}; + +use planner_types::post_asap::SketchQuery; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +pub use crate::QueryLanguage; +use crate::{sds::SummaryDefinitionId, PolicyFingerprint}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct QueryPlan { + pub plan_id: u64, + pub plan_version: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub clickhouse_context: Option, + pub entries: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ClickHousePlanningContext { + pub tables: std::collections::HashMap, + pub accuracy: planner_types::types::AccuracyTarget, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct FixedEvaluationRange { + pub start_ms: u64, + pub end_ms: u64, + pub cumulative: bool, +} + +impl QueryPlan { + pub fn empty() -> Self { + Self { + plan_id: 0, + plan_version: 0, + clickhouse_context: None, + entries: BTreeMap::new(), + } + } + + pub fn lookup(&self, promql: &str) -> Result<&QueryPlanEntry, QueryPlanError> { + let identity = canonical_promql(promql)?; + self.lookup_canonical(QueryLanguage::PromQl, &identity) + } + + pub fn lookup_canonical( + &self, + language: QueryLanguage, + identity: &str, + ) -> Result<&QueryPlanEntry, QueryPlanError> { + let key = Self::catalog_key(language, identity); + self.entries + .get(&key) + .filter(|entry| entry.language == language) + .ok_or_else(|| QueryPlanError::QueryNotPlanned(identity.into())) + } + + pub fn catalog_key(language: QueryLanguage, identity: &str) -> String { + match language { + QueryLanguage::PromQl => identity.to_owned(), + QueryLanguage::MetricsQl => format!("metricsql:{identity}"), + QueryLanguage::ClickHouseSql => format!("clickhouse:{identity}"), + } + } + + pub fn lookup_clickhouse( + &self, + canonical_sql: &str, + ) -> Result<&QueryPlanEntry, QueryPlanError> { + self.lookup_canonical(QueryLanguage::ClickHouseSql, canonical_sql) + } + + /// Validate semantic bindings against the authoritative snapshot before use. + pub fn validate_against_catalog( + &self, + catalog: &crate::summary_catalog::SummaryCatalog, + ) -> Result<(), QueryPlanError> { + catalog + .validate() + .map_err(|error| QueryPlanError::Invalid(error.to_string()))?; + if self.plan_id != catalog.plan_id || self.plan_version != catalog.plan_version { + return Err(QueryPlanError::Invalid( + "QueryPlan and SummaryCatalog have different plan identity/version".into(), + )); + } + let available = catalog + .materializations + .keys() + .copied() + .map(Into::into) + .collect(); + self.validate(&available)?; + for entry in self.entries.values() { + for binding in entry.materialization_bindings() { + let identity = catalog + .materializations + .get(&binding.materialization) + .ok_or_else(|| { + QueryPlanError::Invalid( + "query binding references absent catalog materialization".into(), + ) + })?; + let _data = &catalog.data_descriptors[&identity.data_descriptor_id]; + if binding.window_ms == 0 { + return Err(QueryPlanError::Invalid( + "zero physical pane duration".into(), + )); + } + if binding.pane_origin_ms != identity.pane_origin_ms { + return Err(QueryPlanError::Invalid( + "query pane origin differs from catalog definition".into(), + )); + } + } + for node in entry.nodes.values() { + let QueryPlanNode::ExactReadout { input, readout } = node else { + continue; + }; + if !matches!(readout, ExactReadout::Increase | ExactReadout::Rate) { + continue; + } + let Some(QueryPlanNode::ReadMaterialization { binding }) = entry.nodes.get(input) + else { + return Err(QueryPlanError::Invalid( + "counter readout must directly consume one catalog materialization".into(), + )); + }; + let identity = &catalog.materializations[&binding.materialization]; + let descriptor = &catalog.summary_descriptors[&identity.summary_descriptor_id]; + if !matches!( + descriptor.fidelity, + crate::sds::FidelityGuarantee::ExactCounter { + full_pane_coverage_required: true, + .. + } + ) { + return Err(QueryPlanError::Invalid( + "rate/increase binding does not reference an exact counter SDS".into(), + )); + } + } + } + Ok(()) + } + + pub fn validate(&self, available: &BTreeSet) -> Result<(), QueryPlanError> { + if self.plan_id != 0 && self.plan_version == 0 { + return Err(QueryPlanError::Invalid( + "non-bootstrap QueryPlan has zero plan_version".into(), + )); + } + for (identity, entry) in &self.entries { + let expected = Self::catalog_key(entry.language, &entry.canonical_query); + if identity != &expected { + return Err(QueryPlanError::Invalid(format!( + "query map key `{identity}` differs from entry identity `{}`", + entry.canonical_query + ))); + } + match entry.language { + QueryLanguage::PromQl | QueryLanguage::MetricsQl + if entry.fixed_evaluation.is_some() => + { + return Err(QueryPlanError::Invalid( + "PromQL query entry carries a ClickHouse fixed evaluation range".into(), + )); + } + QueryLanguage::ClickHouseSql => { + if self.clickhouse_context.is_none() { + return Err(QueryPlanError::Invalid( + "ClickHouse query entry has no planning context".into(), + )); + } + let Some(range) = entry.fixed_evaluation else { + return Err(QueryPlanError::Invalid( + "ClickHouse query entry has no fixed evaluation range".into(), + )); + }; + if range.end_ms <= range.start_ms { + return Err(QueryPlanError::Invalid( + "ClickHouse query entry has an empty evaluation range".into(), + )); + } + } + QueryLanguage::PromQl | QueryLanguage::MetricsQl => {} + } + entry.validate(available)?; + } + Ok(()) + } +} + +pub use crate::executable_plan::QueryNodeId; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct QueryPlanEntry { + #[serde(default)] + pub language: QueryLanguage, + pub query_id: String, + #[serde(alias = "canonical_promql")] + pub canonical_query: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fixed_evaluation: Option, + pub root: QueryNodeId, + pub nodes: BTreeMap, + pub instant: InstantExecution, + pub fallback: FallbackPolicy, +} + +fn topological_order( + root: QueryNodeId, + nodes: &BTreeMap, +) -> Result, QueryPlanError> { + fn visit( + id: QueryNodeId, + nodes: &BTreeMap, + visiting: &mut BTreeSet, + visited: &mut BTreeSet, + out: &mut Vec, + ) -> Result<(), QueryPlanError> { + if visited.contains(&id) { + return Ok(()); + } + if !visiting.insert(id) { + return Err(QueryPlanError::Invalid(format!( + "cycle detected at query node {}", + id.0 + ))); + } + let node = nodes + .get(&id) + .ok_or_else(|| QueryPlanError::Invalid(format!("missing query node {}", id.0)))?; + for input in node.inputs() { + visit(*input, nodes, visiting, visited, out)?; + } + visiting.remove(&id); + visited.insert(id); + out.push(id); + Ok(()) + } + let mut out = Vec::with_capacity(nodes.len()); + visit( + root, + nodes, + &mut BTreeSet::new(), + &mut BTreeSet::new(), + &mut out, + )?; + Ok(out) +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct InstantExecution { + pub lookback_ms: u64, + pub full_history: bool, + pub cumulative_readout: bool, +} + +impl QueryPlanEntry { + /// Replace an explicit planner fallback cut with a typed external-exact leaf. + /// The control plane chooses the cut; serving only executes the published DAG. + pub fn bind_external_exact_leaf( + &mut self, + node_id: QueryNodeId, + request: ExternalExactRequest, + ) -> Result<(), QueryPlanError> { + if request.language != self.language { + return Err(QueryPlanError::Invalid( + "external exact language differs from its query plan".into(), + )); + } + if !request.input_contracts.is_empty() { + return Err(QueryPlanError::Invalid( + "leaf binding cannot declare DAG input contracts".into(), + )); + } + match self.nodes.get(&node_id) { + Some(QueryPlanNode::ExactFallback { .. }) => {} + Some(_) => { + return Err(QueryPlanError::Invalid( + "external exact binding must replace a planner fallback cut".into(), + )) + } + None => { + return Err(QueryPlanError::Invalid( + "external exact cut node is absent".into(), + )) + } + } + self.nodes.insert( + node_id, + QueryPlanNode::ExternalExact { + request, + inputs: Vec::new(), + }, + ); + Ok(()) + } + + /// Materializations this executable DAG reads, in stable node order. + /// Serving uses this set for readiness accounting; it never performs a + /// catalog candidate search to reconstruct dependencies. + pub fn materialization_bindings(&self) -> Vec<&MaterializationBinding> { + self.nodes + .values() + .filter_map(|node| match node { + QueryPlanNode::ReadMaterialization { binding } => Some(binding), + _ => None, + }) + .collect() + } + + pub fn topological_order(&self) -> Result, QueryPlanError> { + topological_order(self.root, &self.nodes) + } + + pub fn topological_order_from( + &self, + root: QueryNodeId, + ) -> Result, QueryPlanError> { + topological_order(root, &self.nodes) + } + + /// Validate references, bindings, reachability, and cycles before activation. + pub fn validate(&self, available: &BTreeSet) -> Result<(), QueryPlanError> { + if !self.nodes.contains_key(&self.root) { + return Err(QueryPlanError::Invalid(format!( + "query `{}` has missing root {}", + self.query_id, self.root.0 + ))); + } + for (id, node) in &self.nodes { + if let QueryPlanNode::Logical { operator, inputs } = node { + operator.validate(inputs.len())?; + } + if matches!(node, QueryPlanNode::Scalar { value } if !value.is_finite()) { + return Err(QueryPlanError::Invalid("non-finite scalar constant".into())); + } + if let QueryPlanNode::ExternalExact { request, inputs } = node { + if request.expression.trim().is_empty() { + return Err(QueryPlanError::Invalid( + "external exact expression must not be empty".into(), + )); + } + if request.input_contracts.len() != inputs.len() { + return Err(QueryPlanError::Invalid( + "external exact input contracts must match DAG inputs".into(), + )); + } + if request.input_contracts.iter().any(|contract| { + matches!(contract, ExternalExactInput::CandidateMembership { item_label } if item_label.is_empty()) + }) { + return Err(QueryPlanError::Invalid( + "external exact candidate item label must not be empty".into(), + )); + } + } + if let QueryPlanNode::CandidateTopK { + k, completeness, .. + } = node + { + if *k == 0 { + return Err(QueryPlanError::Invalid( + "CandidateTopK requires k > 0".into(), + )); + } + if matches!( + completeness, + CandidateCompleteness::Certified { guarantee } + if guarantee.metric + != planner_types::post_asap::ErrorMetric::TopKMembership + || guarantee.bound.evaluate().is_none() + || guarantee.failure_probability.evaluate().is_none() + ) { + return Err(QueryPlanError::Invalid( + "invalid CandidateTopK completeness certificate".into(), + )); + } + } + for input in node.inputs() { + if !self.nodes.contains_key(input) { + return Err(QueryPlanError::Invalid(format!( + "query `{}` node {} references missing input {}", + self.query_id, id.0, input.0 + ))); + } + } + if let QueryPlanNode::ReadMaterialization { binding } = node { + if binding.readout_lookback_ms == Some(0) { + return Err(QueryPlanError::Invalid( + "zero semantic readout lookback".into(), + )); + } + if !available.contains(&binding.materialization.fingerprint()) { + return Err(QueryPlanError::Invalid(format!( + "query `{}` node {} references absent materialization {}", + self.query_id, + id.0, + binding.materialization.as_u64() + ))); + } + } + } + let order = self.topological_order()?; + if order.len() != self.nodes.len() { + return Err(QueryPlanError::Invalid(format!( + "query `{}` contains unreachable nodes", + self.query_id + ))); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum FallbackPolicy { + ExactBackend, + Reject, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct MaterializationBinding { + pub materialization: SummaryDefinitionId, + /// Query operator grouping applied while folding those SIDs. + pub output_grouping: PhysicalGrouping, + /// Labels whose values form an item identity inside a keyed sketch. + #[serde(default, alias = "itemLabels", skip_serializing_if = "Vec::is_empty")] + pub item_labels: Vec, + pub window_ms: u64, + /// Unix millisecond timestamp on the materialized pane-boundary grid. + /// Legacy plans deserialize this as unknown and fall back at read time. + #[serde( + default, + alias = "paneOriginMs", + skip_serializing_if = "Option::is_none" + )] + pub pane_origin_ms: Option, + /// Semantic query lookback, independent of the physical pane duration. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub readout_lookback_ms: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "mode", content = "keys", rename_all = "snake_case")] +pub enum PhysicalGrouping { + PerEntity, + Reduce(Vec), +} + +/// Result shape promised by an external exact engine. The backend uses this +/// contract to type-check downstream DAG nodes without depending on an +/// engine-specific response envelope. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum ExternalExactOutput { + Scalar, + InstantVector, + RangeVector, + Relation { schema: serde_json::Value }, +} + +/// How an ordinary DAG input constrains an external exact evaluation. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum ExternalExactInput { + CandidateMembership { item_label: String }, +} + +/// Language-neutral request contract for an exact subtree. Evaluation time is +/// inherited from the containing QueryPlanEntry, avoiding a second time-range +/// envelope that could drift from the installed query plan. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct ExternalExactRequest { + pub language: QueryLanguage, + pub expression: String, + pub output: ExternalExactOutput, + /// Engine parameters forwarded without embedding transport details in the DAG. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub parameters: BTreeMap, + /// Optional parameter names populated from the query entry's evaluation range. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub start_parameter: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub end_parameter: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub input_contracts: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "op", rename_all = "snake_case", deny_unknown_fields)] +pub enum QueryPlanNode { + RelationalJoin { + inputs: [QueryNodeId; 2], + join_kind: planner_types::pre_asap::JoinKind, + pred: serde_json::Value, + left_schema: planner_types::post_asap::SummarySchema, + right_schema: planner_types::post_asap::SummarySchema, + output_schema: planner_types::post_asap::SummarySchema, + }, + Relational { + input: QueryNodeId, + /// Serialized planner-owned operation. Keeping the wire form here makes + /// the published catalog Send + Sync even though the planner AST uses Rc. + operation: serde_json::Value, + input_schema: planner_types::post_asap::SummarySchema, + output_schema: planner_types::post_asap::SummarySchema, + }, + Logical { + operator: logical::LogicalOperator, + inputs: Vec, + }, + Scalar { + value: f64, + }, + Binary { + inputs: [QueryNodeId; 2], + operator: planner_types::pre_asap::ArithmeticOpKind, + }, + ReduceSum { + input: QueryNodeId, + grouping: PhysicalGrouping, + }, + ReadMaterialization { + binding: MaterializationBinding, + }, + SummaryEstimate { + input: QueryNodeId, + query: QueryReadout, + }, + ExactReadout { + input: QueryNodeId, + readout: ExactReadout, + }, + SummaryMerge { + inputs: Vec, + }, + /// Use an approximate heap only as a membership sidecar, then rerank the + /// matching exact counter readouts. `inputs[0]` is candidate membership; + /// `inputs[1]` is the authoritative exact value vector. + CandidateTopK { + inputs: [QueryNodeId; 2], + k: u64, + grouping: logical::Grouping, + completeness: CandidateCompleteness, + }, + /// An exact subtree evaluated outside ASAP. Its results enter the query DAG + /// like any other node output and may depend on summary-produced inputs. + ExternalExact { + request: ExternalExactRequest, + inputs: Vec, + }, + ExactFallback { + reason: String, + }, +} + +impl QueryPlanNode { + pub fn inputs(&self) -> &[QueryNodeId] { + match self { + Self::Scalar { .. } | Self::ReadMaterialization { .. } | Self::ExactFallback { .. } => { + &[] + } + Self::Binary { inputs, .. } | Self::RelationalJoin { inputs, .. } => inputs, + Self::ReduceSum { input, .. } + | Self::Relational { input, .. } + | Self::SummaryEstimate { input, .. } + | Self::ExactReadout { input, .. } => std::slice::from_ref(input), + Self::SummaryMerge { inputs } + | Self::Logical { inputs, .. } + | Self::ExternalExact { inputs, .. } => inputs, + Self::CandidateTopK { inputs, .. } => inputs, + } + } +} + +pub use planner_types::post_asap::CandidateCompleteness; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ExactReadout { + Sum, + Count, + Increase, + Rate, + Max, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum QueryReadout { + FrequencyL2, + FrequencyEntropy, + Quantile { + q: f64, + }, + PointCount { + key: planner_types::pre_asap::ColumnRef, + value: Option, + }, + Cardinality, + TopK { + k: usize, + }, +} + +impl From for QueryReadout { + fn from(query: SketchQuery) -> Self { + match query { + SketchQuery::FrequencyL2 => Self::FrequencyL2, + SketchQuery::FrequencyEntropy => Self::FrequencyEntropy, + SketchQuery::Quantile { q } => Self::Quantile { q }, + SketchQuery::PointCount { key, value } => Self::PointCount { key, value }, + SketchQuery::Cardinality => Self::Cardinality, + SketchQuery::TopK { k } => Self::TopK { k }, + } + } +} + +impl From for SketchQuery { + fn from(query: QueryReadout) -> Self { + match query { + QueryReadout::FrequencyL2 => Self::FrequencyL2, + QueryReadout::FrequencyEntropy => Self::FrequencyEntropy, + QueryReadout::Quantile { q } => Self::Quantile { q }, + QueryReadout::PointCount { key, value } => Self::PointCount { key, value }, + QueryReadout::Cardinality => Self::Cardinality, + QueryReadout::TopK { k } => Self::TopK { k }, + } + } +} + +#[derive(Debug, Error)] +pub enum QueryPlanError { + #[error("invalid PromQL query identity: {0}")] + InvalidPromql(String), + #[error("query is absent from the active QueryPlan: {0}")] + QueryNotPlanned(String), + #[error("post-ASAP DAG cannot be represented by the query executor: {0}")] + UnsupportedNode(String), + #[error("invalid QueryPlan: {0}")] + Invalid(String), +} + +pub fn canonical_promql(query: &str) -> Result { + promql_parser::parser::parse(query.trim()) + .map(|expr| expr.to_string()) + .map_err(|error| QueryPlanError::InvalidPromql(error.to_string())) +} + +#[cfg(test)] +mod contract_tests { + // Installed plans cross producer/query threads without Planner Rc state. + #[test] + fn installed_query_contract_is_send_sync() { + fn assert_send_sync() {} + assert_send_sync::(); + assert_send_sync::(); + } +} diff --git a/crates/asap_types/src/query_plan/logical.rs b/crates/asap_types/src/query_plan/logical.rs new file mode 100644 index 000000000..5c3f6e138 --- /dev/null +++ b/crates/asap_types/src/query_plan/logical.rs @@ -0,0 +1,155 @@ +//! Typed installed residual operators; no Planner selection or AST lowering. +use super::QueryPlanError; +use promql_parser::parser::{self, Expr}; +use serde::{Deserialize, Serialize}; +fn invalid(message: impl Into) -> QueryPlanError { + QueryPlanError::Invalid(message.into()) +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum LogicalOperator { + /// A maximal exact scalar/vector subtree evaluated by Prometheus. + ExactSubquery { + query: String, + }, + /// Prometheus exact subtree whose selectors are restricted at runtime by + /// the candidate vector produced by its single input. + CandidateExactSubquery { + query: String, + item_label: String, + }, + Scan { + metric: Option, + matchers: Vec, + range_ms: Option, + offset_ms: i64, + }, + UnaryNegate, + VectorToScalar, + Aggregate { + operation: Aggregation, + grouping: Grouping, + }, + /// PromQL `topk(k, vector)` selection over values produced by the child. + /// This is distinct from a frequency-sketch TopK readout: any exact or + /// summary-backed instant-vector child may feed this query-time operator. + TopKSelection { + k: u64, + grouping: Grouping, + }, + Binary { + operation: BinaryOperation, + return_bool: bool, + }, + Temporal { + operation: TemporalOperation, + }, + Sort { + descending: bool, + }, + HistogramQuantile, + Subquery { + range_ms: u64, + step_ms: u64, + offset_ms: i64, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct Grouping { + pub labels: Vec, + pub without: bool, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct LabelMatcher { + pub name: String, + pub value: String, + pub operation: LabelMatch, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum LabelMatch { + Equal, + NotEqual, + Regex, + NotRegex, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum Aggregation { + Sum, + Max, + Min, + Avg, + Count, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum BinaryOperation { + Add, + Sub, + Mul, + Div, + Mod, + Pow, + Equal, + NotEqual, + Less, + LessEqual, + Greater, + GreaterEqual, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum TemporalOperation { + Rate, + Increase, + Avg, + Max, + Min, + Sum, + Count, +} + +impl LogicalOperator { + pub fn validate(&self, inputs: usize) -> Result<(), QueryPlanError> { + let expected = match self { + Self::Scan { .. } | Self::ExactSubquery { .. } => 0, + Self::CandidateExactSubquery { .. } => 1, + Self::Binary { .. } | Self::HistogramQuantile => 2, + _ => 1, + }; + if inputs != expected { + return Err(invalid("logical operator input arity mismatch")); + } + if matches!( + self, + Self::Scan { + range_ms: Some(0), + .. + } + ) { + return Err(invalid("zero range")); + } + if let Self::ExactSubquery { query } | Self::CandidateExactSubquery { query, .. } = self { + let parsed = parser::parse(query).map_err(|e| invalid(e.to_string()))?; + if matches!(parsed, Expr::MatrixSelector(_) | Expr::Subquery(_)) { + return Err(invalid( + "exact subtree boundary must return scalar or instant vector", + )); + } + } + if let Self::Subquery { + range_ms, step_ms, .. + } = self + { + if *range_ms == 0 || *step_ms == 0 || range_ms / step_ms > 100_000 { + return Err(invalid("invalid or excessive subquery grid")); + } + } + Ok(()) + } +} diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index ef9f115f0..be9905bb5 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -956,7 +956,7 @@ mod tests { rules: Vec::new(), }, runtime_config: Arc::new(streaming), - query_plan: Arc::new(control_plane::query_plan::QueryPlan::empty()), + query_plan: Arc::new(asap_types::query_plan::QueryPlan::empty()), storage_routing: Arc::new(BackendStorageRouting::empty()), }; HotReloadStreamingConfig::from_active(HotReloadActivePhysicalPlan::new(active)) @@ -1496,11 +1496,9 @@ mod tests { .keys() .next() .unwrap(); - let binding = control_plane::query_plan::MaterializationBinding { + let binding = asap_types::query_plan::MaterializationBinding { materialization: asap_types::PolicyFingerprint(policy).into(), - output_grouping: control_plane::query_plan::PhysicalGrouping::Reduce( - vec!["job".into()], - ), + output_grouping: asap_types::query_plan::PhysicalGrouping::Reduce(vec!["job".into()]), item_labels: vec![], window_ms: 60_000, pane_origin_ms: Some(0), diff --git a/data_plane/src/drivers/query/adapters/victoriametrics_http.rs b/data_plane/src/drivers/query/adapters/victoriametrics_http.rs index a75bbd264..e23cf4a9b 100644 --- a/data_plane/src/drivers/query/adapters/victoriametrics_http.rs +++ b/data_plane/src/drivers/query/adapters/victoriametrics_http.rs @@ -104,7 +104,7 @@ impl HttpProtocolAdapter for VictoriaMetricsHttpAdapter { "VictoriaMetrics HTTP / MetricsQL" } fn canonical_plan_identity(&self, query: &str) -> Result, AdapterError> { - control_plane::query_plan::canonical_promql(query) + asap_types::query_plan::canonical_promql(query) .map(Some) .map_err(|error| AdapterError::ParseError(format!("promql-compatible subset: {error}"))) } diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index c456d8e3c..e219aca43 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -2734,7 +2734,7 @@ mod tests { rules: Vec::new(), }, runtime_config: streaming_config.clone(), - query_plan: Arc::new(control_plane::query_plan::QueryPlan { + query_plan: Arc::new(asap_types::query_plan::QueryPlan { plan_id: 7, plan_version: 1, clickhouse_context: None, @@ -6216,13 +6216,13 @@ async fn handle_post_physical_plan( .query_plan .entries .values() - .filter(|entry| entry.language == control_plane::query_plan::QueryLanguage::MetricsQl) + .filter(|entry| entry.language == asap_types::query_plan::QueryLanguage::MetricsQl) .count(); let clickhouse_plan_count = active .query_plan .entries .values() - .filter(|entry| entry.language == control_plane::query_plan::QueryLanguage::ClickHouseSql) + .filter(|entry| entry.language == asap_types::query_plan::QueryLanguage::ClickHouseSql) .count(); let plan_version = active.plan_version(); let now = unix_time_ms(); @@ -6298,7 +6298,7 @@ async fn handle_activate_physical_plan( .query_plan .entries .values() - .filter(|entry| entry.language == control_plane::query_plan::QueryLanguage::ClickHouseSql) + .filter(|entry| entry.language == asap_types::query_plan::QueryLanguage::ClickHouseSql) .count(); if old.plan_id() != 0 { let draining_id = old.plan_id(); @@ -7179,7 +7179,7 @@ mod catalog_install_tests { let before = handle.snapshot(); let mut candidate = request(); candidate.query_plan.clickhouse_context = - Some(control_plane::query_plan::ClickHousePlanningContext { + Some(asap_types::query_plan::ClickHousePlanningContext { tables: Default::default(), accuracy: planner_types::types::AccuracyTarget::Exact, }); @@ -7188,8 +7188,8 @@ mod catalog_install_tests { .entries .pop_first() .expect("fixture has a query entry"); - entry.language = control_plane::query_plan::QueryLanguage::ClickHouseSql; - entry.fixed_evaluation = Some(control_plane::query_plan::FixedEvaluationRange { + entry.language = asap_types::query_plan::QueryLanguage::ClickHouseSql; + entry.fixed_evaluation = Some(asap_types::query_plan::FixedEvaluationRange { start_ms: 0, end_ms: 1_000, cumulative: true, @@ -7199,7 +7199,7 @@ mod catalog_install_tests { .nodes .values_mut() .find_map(|node| match node { - control_plane::query_plan::QueryPlanNode::ReadMaterialization { binding } => { + asap_types::query_plan::QueryPlanNode::ReadMaterialization { binding } => { Some(binding) } _ => None, @@ -7272,7 +7272,7 @@ mod catalog_install_tests { .values_mut() .flat_map(|entry| entry.nodes.values_mut()) .find_map(|node| match node { - control_plane::query_plan::QueryPlanNode::ReadMaterialization { binding } => { + asap_types::query_plan::QueryPlanNode::ReadMaterialization { binding } => { Some(binding) } _ => None, @@ -7287,22 +7287,20 @@ mod catalog_install_tests { let mut request = request(); let key = request.query_plan.entries.keys().next().unwrap().clone(); let mut entry = request.query_plan.entries.remove(&key).unwrap(); - entry.language = control_plane::query_plan::QueryLanguage::MetricsQl; + entry.language = asap_types::query_plan::QueryLanguage::MetricsQl; let binding = entry .nodes .values_mut() .find_map(|node| match node { - control_plane::query_plan::QueryPlanNode::ReadMaterialization { binding } => { + asap_types::query_plan::QueryPlanNode::ReadMaterialization { binding } => { Some(binding) } _ => None, }) .expect("demo has maintained summaries"); binding.pane_origin_ms = Some(1); - let key = control_plane::query_plan::QueryPlan::catalog_key( - entry.language, - &entry.canonical_query, - ); + let key = + asap_types::query_plan::QueryPlan::catalog_key(entry.language, &entry.canonical_query); request.query_plan.entries.insert(key, entry); let error = install(request).unwrap_err(); assert!(error.contains("pane origin"), "{error}"); diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 345479b29..23409c3b1 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -787,7 +787,7 @@ async fn main() -> Result<()> { precompute_plan: initial_precompute_plan, transmission_plan: initial_transmission_plan, runtime_config: streaming_config.clone(), - query_plan: Arc::new(control_plane::query_plan::QueryPlan::empty()), + query_plan: Arc::new(asap_types::query_plan::QueryPlan::empty()), storage_routing: Arc::new( data_plane::storage_engines::types::BackendStorageRouting::empty(), ), diff --git a/data_plane/src/precompute_engine/maintenance_runtime.rs b/data_plane/src/precompute_engine/maintenance_runtime.rs index 3d64b8eeb..b49aadff4 100644 --- a/data_plane/src/precompute_engine/maintenance_runtime.rs +++ b/data_plane/src/precompute_engine/maintenance_runtime.rs @@ -654,7 +654,7 @@ mod tests { let binding = BackendExecutableBinding { nodes: BTreeMap::new(), query_sink: PostAsapNodeId(2), - query_plan_sink: control_plane::query_plan::QueryNodeId(2), + query_plan_sink: asap_types::query_plan::QueryNodeId(2), precompute_sinks: vec![PostAsapNodeId(1)], }; let adapter = OperatorAdapter { @@ -841,12 +841,12 @@ mod tests { ( PostAsapNodeId(2), BackendNodeBinding::Query { - query_node: control_plane::query_plan::QueryNodeId(9), + query_node: asap_types::query_plan::QueryNodeId(9), }, ), ]), query_sink: PostAsapNodeId(2), - query_plan_sink: control_plane::query_plan::QueryNodeId(9), + query_plan_sink: asap_types::query_plan::QueryNodeId(9), precompute_sinks: vec![PostAsapNodeId(1)], }; bundle.precompute_plan.executable_dags = BTreeMap::from([( @@ -970,12 +970,12 @@ mod tests { .chain([( PostAsapNodeId(4), BackendNodeBinding::Query { - query_node: control_plane::query_plan::QueryNodeId(9), + query_node: asap_types::query_plan::QueryNodeId(9), }, )]) .collect(), query_sink: PostAsapNodeId(4), - query_plan_sink: control_plane::query_plan::QueryNodeId(9), + query_plan_sink: asap_types::query_plan::QueryNodeId(9), precompute_sinks: vec![PostAsapNodeId(3)], }; let source = sum(2.0); @@ -1043,12 +1043,12 @@ mod tests { ( PostAsapNodeId(2), BackendNodeBinding::Query { - query_node: control_plane::query_plan::QueryNodeId(9), + query_node: asap_types::query_plan::QueryNodeId(9), }, ), ]), query_sink: PostAsapNodeId(2), - query_plan_sink: control_plane::query_plan::QueryNodeId(9), + query_plan_sink: asap_types::query_plan::QueryNodeId(9), precompute_sinks: vec![PostAsapNodeId(1)], }; let adapter = OperatorAdapter { diff --git a/data_plane/src/precompute_engine/subdag_scheduler.rs b/data_plane/src/precompute_engine/subdag_scheduler.rs index 6f632f44c..6a52b88df 100644 --- a/data_plane/src/precompute_engine/subdag_scheduler.rs +++ b/data_plane/src/precompute_engine/subdag_scheduler.rs @@ -185,12 +185,12 @@ mod tests { .chain([( PostAsapNodeId(4), BackendNodeBinding::Query { - query_node: control_plane::query_plan::QueryNodeId(9), + query_node: asap_types::query_plan::QueryNodeId(9), }, )]) .collect(), query_sink: PostAsapNodeId(4), - query_plan_sink: control_plane::query_plan::QueryNodeId(9), + query_plan_sink: asap_types::query_plan::QueryNodeId(9), precompute_sinks: vec![PostAsapNodeId(3)], } } @@ -376,7 +376,7 @@ mod tests { ( PostAsapNodeId(0), BackendNodeBinding::Query { - query_node: control_plane::query_plan::QueryNodeId(1), + query_node: asap_types::query_plan::QueryNodeId(1), }, ), ( @@ -389,7 +389,7 @@ mod tests { .into_iter() .collect(), query_sink: PostAsapNodeId(0), - query_plan_sink: control_plane::query_plan::QueryNodeId(1), + query_plan_sink: asap_types::query_plan::QueryNodeId(1), precompute_sinks: vec![PostAsapNodeId(1)], }; assert!(matches!( diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs index a69da5956..2d8b4a86d 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs @@ -64,14 +64,14 @@ impl CatalogClickHouseAccelerator { async fn prepare_external_exact( &self, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, start_ms: u64, end_ms: u64, request_context: &ClickHouseQueryRequest, ) -> Result { let mut prepared = PreparedExternalLeaves::new(); let leaves = entry.nodes.iter().filter_map(|(id, node)| match node { - control_plane::query_plan::QueryPlanNode::ExternalExact { request, inputs } + asap_types::query_plan::QueryPlanNode::ExternalExact { request, inputs } if request.language == asap_types::QueryLanguage::ClickHouseSql && inputs.is_empty() => { @@ -85,7 +85,7 @@ impl CatalogClickHouseAccelerator { .as_ref() .ok_or_else(|| "ClickHouse exact subtree endpoint unavailable".to_owned())?; let schema = match &bound.output { - control_plane::query_plan::ExternalExactOutput::Relation { schema } => { + asap_types::query_plan::ExternalExactOutput::Relation { schema } => { serde_json::from_value(schema.clone()).map_err(|error| error.to_string())? } _ => return Err("ClickHouse exact subtree must produce a relation".into()), @@ -310,14 +310,14 @@ mod tests { precompute_engine::operators::SumAccumulator, storage_engines::sketch_db::index::{AggKind, Capability, SketchInstanceMetadata}, }; - use asap_types::summary_catalog::SummaryCatalog; - use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; - use axum::http::Method; - use control_plane::query_plan::{ + use asap_types::query_plan::{ ClickHousePlanningContext, ExactReadout, ExternalExactOutput, ExternalExactRequest, FallbackPolicy, FixedEvaluationRange, InstantExecution, MaterializationBinding, PhysicalGrouping, QueryLanguage, QueryNodeId, QueryPlan, QueryPlanEntry, QueryPlanNode, }; + use asap_types::summary_catalog::SummaryCatalog; + use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + use axum::http::Method; struct FixedExactSubtree; diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs index aa9fd90cc..26183c1f6 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs @@ -9,8 +9,8 @@ use crate::{ }, storage_engines::sketch_db::index::SketchStore, }; +use asap_types::query_plan::{QueryNodeId, QueryPlanEntry, QueryPlanNode}; use asap_types::summary_catalog::SummaryCatalog; -use control_plane::query_plan::{QueryNodeId, QueryPlanEntry, QueryPlanNode}; use planner_types::post_asap::ValueOperation; use std::collections::{BTreeMap, BTreeSet}; @@ -65,8 +65,7 @@ fn execute_relation_subtree( if request.language != asap_types::QueryLanguage::ClickHouseSql { return Err("ClickHouse DAG contains an external leaf for another language".into()); } - let control_plane::query_plan::ExternalExactOutput::Relation { schema } = - &request.output + let asap_types::query_plan::ExternalExactOutput::Relation { schema } = &request.output else { return Err("ClickHouse external leaf must declare relation output".into()); }; @@ -276,7 +275,7 @@ fn execute_sql_dag_with_external_unfenced( Some(QueryPlanNode::Relational { output_schema, .. }) | Some(QueryPlanNode::RelationalJoin { output_schema, .. }) => output_schema.clone(), Some(QueryPlanNode::ExternalExact { request, .. }) => { - let control_plane::query_plan::ExternalExactOutput::Relation { schema } = + let asap_types::query_plan::ExternalExactOutput::Relation { schema } = &request.output else { return ClickHouseDagOutcome::Fallback(ClickHouseDagFallback::UnsupportedPlan( diff --git a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs index 4ae79947a..6bb5877e2 100644 --- a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs +++ b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs @@ -3,10 +3,10 @@ //! query execution checks only reachable IDs and their requested capabilities. use std::collections::{BTreeMap, BTreeSet}; +use asap_types::query_plan::{ExactReadout, QueryPlanEntry, QueryPlanNode, QueryReadout}; use asap_types::sds::{SummaryDefinitionId, SummaryDescriptor, SummaryOperator}; use asap_types::summary_catalog::SummaryCatalog; use asap_types::AggregationType; -use control_plane::query_plan::{ExactReadout, QueryPlanEntry, QueryPlanNode, QueryReadout}; use crate::query_engines::EngineError; diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 12d31e207..ccd3a2464 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -10,7 +10,7 @@ struct QueryReadinessRequirement { } fn readiness_requirement( - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, ) -> QueryReadinessRequirement { let bindings = entry.materialization_bindings(); let mut materializations = bindings @@ -128,10 +128,7 @@ impl ASAPQueryEngine { })?; let planned = physical .query_plan - .lookup_canonical( - control_plane::query_plan::QueryLanguage::MetricsQl, - identity, - ) + .lookup_canonical(asap_types::query_plan::QueryLanguage::MetricsQl, identity) .map_err(|error| { crate::query_engines::EngineError::capability_miss("query_plan", error.to_string()) })?; @@ -160,10 +157,7 @@ impl ASAPQueryEngine { })?; let planned = physical .query_plan - .lookup_canonical( - control_plane::query_plan::QueryLanguage::MetricsQl, - identity, - ) + .lookup_canonical(asap_types::query_plan::QueryLanguage::MetricsQl, identity) .map_err(|error| { crate::query_engines::EngineError::capability_miss("query_plan", error.to_string()) })?; @@ -201,7 +195,7 @@ impl ASAPQueryEngine { async fn prepare_logical( &self, physical: &crate::storage_engines::types::ActivePhysicalPlan, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, times: &[u64], ) -> Result { super::catalog_resolver::validate_entry( @@ -266,7 +260,7 @@ impl ASAPQueryEngine { fn execute_logical_entry( &self, physical: &crate::storage_engines::types::ActivePhysicalPlan, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, leaves: &super::logical_dag::PreparedLeaves, at: u64, ) -> Result< @@ -410,7 +404,7 @@ impl ASAPQueryEngine { async fn execute_logical_range( &self, physical: &crate::storage_engines::types::ActivePhysicalPlan, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, start: u64, end: u64, step: u64, @@ -667,10 +661,7 @@ impl ASAPQueryEngine { if let Some(physical) = self.physical_plan_snapshot() { if let Ok(entry) = physical.query_plan.lookup(query) { if entry.nodes.values().any(|node| { - matches!( - node, - control_plane::query_plan::QueryPlanNode::Logical { .. } - ) + matches!(node, asap_types::query_plan::QueryPlanNode::Logical { .. }) }) { return self .execute_logical_range(&physical, entry, start_ms, end_ms, step_ms) @@ -3622,7 +3613,7 @@ mod range_stitch_tests { #[tokio::test] async fn active_metricsql_entry_reaches_the_shared_dag_executor() { - use control_plane::query_plan::{ + use asap_types::query_plan::{ FallbackPolicy, InstantExecution, QueryLanguage, QueryNodeId, QueryPlanEntry, QueryPlanNode, }; @@ -3632,9 +3623,9 @@ mod range_stitch_tests { )) .unwrap(); let mut plan = snapshot.compile().unwrap(); - let identity = control_plane::query_plan::canonical_promql("1 + 2").unwrap(); + let identity = asap_types::query_plan::canonical_promql("1 + 2").unwrap(); plan.query_plan.entries.insert( - control_plane::query_plan::QueryPlan::catalog_key(QueryLanguage::MetricsQl, &identity), + asap_types::query_plan::QueryPlan::catalog_key(QueryLanguage::MetricsQl, &identity), QueryPlanEntry { language: QueryLanguage::MetricsQl, query_id: "vm-scalar".into(), diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index 70221144b..aa1537b3e 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -1,7 +1,7 @@ //! Fetch installed exact cuts from Prometheus before composing them with ASAP state. use super::logical_dag::{PreparedLeaf, PreparedLeaves, Value}; use crate::query_engines::EngineError; -use control_plane::query_plan::{ +use asap_types::query_plan::{ logical::LogicalOperator, ExternalExactInput, ExternalExactRequest, QueryLanguage, QueryNodeId, QueryPlanEntry, QueryPlanNode, }; @@ -441,10 +441,10 @@ pub(super) async fn prepare_external( #[cfg(test)] mod tests { use super::*; - use control_plane::query_plan::{FallbackPolicy, InstantExecution}; + use asap_types::query_plan::{FallbackPolicy, InstantExecution}; fn entry(nodes: BTreeMap) -> QueryPlanEntry { QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "remote-cut".into(), canonical_query: "a / b".into(), fixed_evaluation: None, @@ -491,7 +491,7 @@ mod tests { request: ExternalExactRequest { language: QueryLanguage::MetricsQl, expression: "sum(rate(m[5m]))".into(), - output: control_plane::query_plan::ExternalExactOutput::InstantVector, + output: asap_types::query_plan::ExternalExactOutput::InstantVector, parameters: BTreeMap::new(), start_parameter: None, end_parameter: None, @@ -526,7 +526,7 @@ mod tests { request: ExternalExactRequest { language: QueryLanguage::PromQl, expression: query.into(), - output: control_plane::query_plan::ExternalExactOutput::InstantVector, + output: asap_types::query_plan::ExternalExactOutput::InstantVector, parameters: BTreeMap::new(), start_parameter: None, end_parameter: None, @@ -630,7 +630,7 @@ mod tests { #[tokio::test] async fn candidate_exact_is_discovered_and_prepared_behind_candidate_topk_root() { - use control_plane::query_plan::{logical::Grouping, CandidateCompleteness}; + use asap_types::query_plan::{logical::Grouping, CandidateCompleteness}; let mut entry = candidate_entry("sum by (job) (rate(m[5m]))"); entry.nodes.insert( QueryNodeId(2), @@ -751,7 +751,7 @@ mod tests { #[tokio::test] async fn exact_leaf_calls_prometheus_and_combines_with_prepared_summary() { // A successful exact branch remains an intermediate, not a whole-root fallback. - use control_plane::query_plan::logical::BinaryOperation; + use asap_types::query_plan::logical::BinaryOperation; use std::sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -866,7 +866,7 @@ mod tests { index::{Capability, SketchInstanceMetadata}, }; use crate::storage_engines::types::{KeyByLabelValues, Measurement}; - use control_plane::query_plan::{ + use asap_types::query_plan::{ logical::BinaryOperation, ExactReadout, MaterializationBinding, PhysicalGrouping, }; use std::sync::{ diff --git a/data_plane/src/query_engines/asap_query_engine/live_serve.rs b/data_plane/src/query_engines/asap_query_engine/live_serve.rs index aac4fca19..ccfb7333a 100644 --- a/data_plane/src/query_engines/asap_query_engine/live_serve.rs +++ b/data_plane/src/query_engines/asap_query_engine/live_serve.rs @@ -160,7 +160,7 @@ pub fn serve_instant_from_summary_executor( pub fn serve_from_query_plan( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, t0_ms: u64, t1_ms: u64, is_cumulative: bool, @@ -181,7 +181,7 @@ pub fn serve_from_query_plan( /// timestamps never leak into the public range response. pub fn serve_range_steps_from_query_plan( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, start_ms: u64, end_ms: u64, step_ms: u64, @@ -263,7 +263,7 @@ fn coverage_covers_closed_windows( pub fn serve_instant_from_query_plan( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, now_ms: u64, ) -> Result<(ASAPTierResult, u64), LoweringSkip> { if !summary_executor_live_enabled() { @@ -447,27 +447,27 @@ mod tests { Box::new(crate::precompute_engine::operators::SumAccumulator::with_sum(value)), ); } - let entry = control_plane::query_plan::QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + let entry = asap_types::query_plan::QueryPlanEntry { + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "q-sum".into(), canonical_query: "sum_over_time(bytes[1s])".into(), fixed_evaluation: None, - root: control_plane::query_plan::QueryNodeId(0), + root: asap_types::query_plan::QueryNodeId(0), nodes: BTreeMap::from([ ( - control_plane::query_plan::QueryNodeId(0), - control_plane::query_plan::QueryPlanNode::ExactReadout { - input: control_plane::query_plan::QueryNodeId(1), - readout: control_plane::query_plan::ExactReadout::Sum, + asap_types::query_plan::QueryNodeId(0), + asap_types::query_plan::QueryPlanNode::ExactReadout { + input: asap_types::query_plan::QueryNodeId(1), + readout: asap_types::query_plan::ExactReadout::Sum, }, ), ( - control_plane::query_plan::QueryNodeId(1), - control_plane::query_plan::QueryPlanNode::ReadMaterialization { - binding: control_plane::query_plan::MaterializationBinding { + asap_types::query_plan::QueryNodeId(1), + asap_types::query_plan::QueryPlanNode::ReadMaterialization { + binding: asap_types::query_plan::MaterializationBinding { item_labels: Vec::new(), materialization: policy.into(), - output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, + output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 1_000, pane_origin_ms: Some(0), readout_lookback_ms: Some(1_000), @@ -475,12 +475,12 @@ mod tests { }, ), ]), - instant: control_plane::query_plan::InstantExecution { + instant: asap_types::query_plan::InstantExecution { lookback_ms: 1_000, full_history: false, cumulative_readout: true, }, - fallback: control_plane::query_plan::FallbackPolicy::ExactBackend, + fallback: asap_types::query_plan::FallbackPolicy::ExactBackend, }; let result = serve_range_steps_from_query_plan(&idx, &entry, 1_000, 3_000, 1_000) diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs index b07ecf64a..dbb5eee54 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs @@ -4,12 +4,10 @@ use crate::query_engines::{ EngineError, }; use crate::storage_engines::types::KeyByLabelValues; -use control_plane::query_plan::logical::{ +use asap_types::query_plan::logical::{ Aggregation, BinaryOperation, Grouping, LogicalOperator, TemporalOperation, }; -use control_plane::query_plan::{ - CandidateCompleteness, QueryNodeId, QueryPlanEntry, QueryPlanNode, -}; +use asap_types::query_plan::{CandidateCompleteness, QueryNodeId, QueryPlanEntry, QueryPlanNode}; use std::collections::{BTreeMap, BTreeSet}; type Labels = BTreeMap; @@ -713,7 +711,7 @@ fn bucket_quantile(q: f64, mut b: Vec<(f64, f64)>) -> f64 { #[cfg(test)] mod topk_tests { use super::*; - use control_plane::query_plan::{FallbackPolicy, InstantExecution}; + use asap_types::query_plan::{FallbackPolicy, InstantExecution}; fn labels(items: &[(&str, &str)]) -> Labels { items @@ -789,7 +787,7 @@ mod topk_tests { #[test] fn installed_topk_combines_with_prometheus_exact_child() { - let mut entry = QueryPlanEntry::compile_logical( + let mut entry = control_plane::query_plan::logical::compile_logical( "hybrid-topk".into(), "topk(2, m)".into(), InstantExecution { @@ -856,7 +854,7 @@ mod topk_tests { let summary = QueryNodeId(0); let root = QueryNodeId(1); let entry = QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "summary-rate-topk".into(), canonical_query: "topk(2, rate(requests_total[5m]))".into(), fixed_evaluation: None, @@ -866,7 +864,7 @@ mod topk_tests { summary, QueryPlanNode::ExactReadout { input: QueryNodeId(99), - readout: control_plane::query_plan::ExactReadout::Rate, + readout: asap_types::query_plan::ExactReadout::Rate, }, ), ( @@ -985,7 +983,7 @@ mod topk_tests { let value_id = QueryNodeId(1); let root = QueryNodeId(2); let entry = QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "candidate-topk".into(), canonical_query: "topk(1, rate(requests_total[5m]))".into(), fixed_evaluation: None, diff --git a/data_plane/src/query_engines/asap_query_engine/physical_dag.rs b/data_plane/src/query_engines/asap_query_engine/physical_dag.rs index a917ff23a..ea6d9f63b 100644 --- a/data_plane/src/query_engines/asap_query_engine/physical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/physical_dag.rs @@ -6,7 +6,7 @@ use std::collections::BTreeMap; -use control_plane::query_plan::{QueryNodeId, QueryPlanEntry, QueryPlanNode}; +use asap_types::query_plan::{QueryNodeId, QueryPlanEntry, QueryPlanNode}; use thiserror::Error; pub trait QueryNodeRuntime { @@ -146,7 +146,7 @@ mod tests { use std::cell::RefCell; use std::collections::BTreeMap; - use control_plane::query_plan::{FallbackPolicy, InstantExecution, QueryReadout}; + use asap_types::query_plan::{FallbackPolicy, InstantExecution, QueryReadout}; use super::*; @@ -204,7 +204,7 @@ mod tests { .into_iter() .collect(); let entry = QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "q".into(), canonical_query: "up".into(), fixed_evaluation: None, @@ -277,7 +277,7 @@ mod tests { .into_iter() .collect(); let entry = QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "q".into(), canonical_query: "up".into(), fixed_evaluation: None, diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 7a8e8bf52..eaa5a2ec3 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -5,7 +5,7 @@ use std::collections::BTreeMap; use crate::query_engines::asap_query_engine::summary_exec::{execute, ExecOutcome}; -use control_plane::query_plan::{QueryNodeId, QueryPlanNode}; +use asap_types::query_plan::{QueryNodeId, QueryPlanNode}; use control_plane::types_v2::AccuracyTarget; use crate::query_engines::asap_query_engine::physical_dag::{self, QueryNodeRuntime}; @@ -82,7 +82,7 @@ pub fn execute_post_asap_readout( /// Installed QueryPlan materialization resolution occurs before this legacy test helper. pub fn execute_query_plan_readout( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, t0_ms: u64, t1_ms: u64, is_cumulative: bool, @@ -92,8 +92,8 @@ pub fn execute_query_plan_readout( pub fn execute_query_plan_from_readout( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, - root: control_plane::query_plan::QueryNodeId, + entry: &asap_types::query_plan::QueryPlanEntry, + root: asap_types::query_plan::QueryNodeId, t0_ms: u64, t1_ms: u64, is_cumulative: bool, @@ -103,7 +103,7 @@ pub fn execute_query_plan_from_readout( pub fn execute_query_plan_instant( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, now_ms: u64, ) -> Result<(PostAsapReadoutOutcome, u64), LoweringSkip> { let t0_ms = if entry.instant.full_history { @@ -476,11 +476,11 @@ fn intersect_coverage(left: Option<(u64, u64)>, right: Option<(u64, u64)>) -> Op } fn reduce_sum_values( - grouping: &control_plane::query_plan::PhysicalGrouping, + grouping: &asap_types::query_plan::PhysicalGrouping, values: &[(BTreeMap, SummaryValue)], coverage: Option<(u64, u64)>, ) -> Result { - let control_plane::query_plan::PhysicalGrouping::Reduce(keys) = grouping else { + let asap_types::query_plan::PhysicalGrouping::Reduce(keys) = grouping else { return Ok(PhysicalQueryOutput::Value(values.to_vec(), coverage)); }; let mut groups = BTreeMap::new(); @@ -519,7 +519,7 @@ fn reduce_sum_values( fn execute_physical_query_plan( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, t0_ms: u64, t1_ms: u64, is_cumulative: bool, @@ -529,8 +529,8 @@ fn execute_physical_query_plan( fn execute_physical_query_payload( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, - root: control_plane::query_plan::QueryNodeId, + entry: &asap_types::query_plan::QueryPlanEntry, + root: asap_types::query_plan::QueryNodeId, t0_ms: u64, t1_ms: u64, is_cumulative: bool, @@ -793,7 +793,7 @@ mod tests { }) .collect::>(); let PhysicalQueryOutput::Value(result, _) = reduce_sum_values( - &control_plane::query_plan::PhysicalGrouping::Reduce(vec!["service".into()]), + &asap_types::query_plan::PhysicalGrouping::Reduce(vec!["service".into()]), &values, Some((1000, 2000)), ) @@ -907,22 +907,22 @@ mod tests { register_hll(&idx, 2, "worker", &["b", "c"]); let node = plan_promql_to_post_asap(&idx, "count(unique_users)", accuracy()) .expect("compile-stage fixture"); - let canonical = control_plane::query_plan::canonical_promql("count(unique_users)").unwrap(); - let entry = control_plane::query_plan::QueryPlanEntry::compile_bound( + let canonical = asap_types::query_plan::canonical_promql("count(unique_users)").unwrap(); + let entry = control_plane::query_plan::compile_bound( "q-cardinality".into(), canonical, &node, - control_plane::query_plan::InstantExecution { + asap_types::query_plan::InstantExecution { lookback_ms: 60_000, full_history: false, cumulative_readout: true, }, - control_plane::query_plan::FallbackPolicy::ExactBackend, + asap_types::query_plan::FallbackPolicy::ExactBackend, |_node, _family| { - Ok(control_plane::query_plan::MaterializationBinding { + Ok(asap_types::query_plan::MaterializationBinding { item_labels: Vec::new(), materialization: asap_types::PolicyFingerprint(123).into(), - output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, + output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 60_000, pane_origin_ms: Some(2_000), readout_lookback_ms: Some(60_000), @@ -1058,27 +1058,27 @@ mod tests { ); } - let entry = control_plane::query_plan::QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + let entry = asap_types::query_plan::QueryPlanEntry { + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "q-rate".into(), canonical_query: "rate(requests_total[1m])".into(), fixed_evaluation: None, - root: control_plane::query_plan::QueryNodeId(0), + root: asap_types::query_plan::QueryNodeId(0), nodes: BTreeMap::from([ ( - control_plane::query_plan::QueryNodeId(0), + asap_types::query_plan::QueryNodeId(0), QueryPlanNode::ExactReadout { - input: control_plane::query_plan::QueryNodeId(1), - readout: control_plane::query_plan::ExactReadout::Sum, + input: asap_types::query_plan::QueryNodeId(1), + readout: asap_types::query_plan::ExactReadout::Sum, }, ), ( - control_plane::query_plan::QueryNodeId(1), + asap_types::query_plan::QueryNodeId(1), QueryPlanNode::ReadMaterialization { - binding: control_plane::query_plan::MaterializationBinding { + binding: asap_types::query_plan::MaterializationBinding { item_labels: Vec::new(), materialization: policy.into(), - output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, + output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 10_000, pane_origin_ms: Some(0), readout_lookback_ms: Some(60_000), @@ -1086,12 +1086,12 @@ mod tests { }, ), ]), - instant: control_plane::query_plan::InstantExecution { + instant: asap_types::query_plan::InstantExecution { lookback_ms: 60_000, full_history: false, cumulative_readout: true, }, - fallback: control_plane::query_plan::FallbackPolicy::ExactBackend, + fallback: asap_types::query_plan::FallbackPolicy::ExactBackend, }; for (now, expected) in [(60_000, 21.0), (70_000, 27.0), (80_000, 33.0)] { let (outcome, _) = execute_query_plan_instant(&idx, &entry, now).unwrap(); @@ -1154,27 +1154,27 @@ mod tests { accumulator.update(Measurement::new(13.0), 50_000); idx.append_precompute(7, BTreeMap::new(), (0, 60_000), Box::new(accumulator)); - let entry = control_plane::query_plan::QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + let entry = asap_types::query_plan::QueryPlanEntry { + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "q-rate".into(), canonical_query: "rate(requests_total[1m])".into(), fixed_evaluation: None, - root: control_plane::query_plan::QueryNodeId(0), + root: asap_types::query_plan::QueryNodeId(0), nodes: BTreeMap::from([ ( - control_plane::query_plan::QueryNodeId(0), + asap_types::query_plan::QueryNodeId(0), QueryPlanNode::ExactReadout { - input: control_plane::query_plan::QueryNodeId(1), - readout: control_plane::query_plan::ExactReadout::Rate, + input: asap_types::query_plan::QueryNodeId(1), + readout: asap_types::query_plan::ExactReadout::Rate, }, ), ( - control_plane::query_plan::QueryNodeId(1), + asap_types::query_plan::QueryNodeId(1), QueryPlanNode::ReadMaterialization { - binding: control_plane::query_plan::MaterializationBinding { + binding: asap_types::query_plan::MaterializationBinding { item_labels: Vec::new(), materialization: policy.into(), - output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, + output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 60_000, pane_origin_ms: Some(0), readout_lookback_ms: Some(60_000), @@ -1182,12 +1182,12 @@ mod tests { }, ), ]), - instant: control_plane::query_plan::InstantExecution { + instant: asap_types::query_plan::InstantExecution { lookback_ms: 60_000, full_history: false, cumulative_readout: true, }, - fallback: control_plane::query_plan::FallbackPolicy::ExactBackend, + fallback: asap_types::query_plan::FallbackPolicy::ExactBackend, }; let outcome = execute_query_plan_readout(&idx, &entry, 0, 60_000, true) .expect("execute exact rate DAG"); diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index 75ce06ad1..ab3389e05 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -228,7 +228,7 @@ impl GroupState { /// in QueryPlan so serving never infers semantics from PromQL text. pub fn exact_value_for( &self, - readout: control_plane::query_plan::ExactReadout, + readout: asap_types::query_plan::ExactReadout, key: &Option, range_start_ms: u64, range_end_ms: u64, @@ -237,23 +237,23 @@ impl GroupState { return None; }; let stat = match (readout, agg_type) { - (control_plane::query_plan::ExactReadout::Count, AggregationType::Sum) => { + (asap_types::query_plan::ExactReadout::Count, AggregationType::Sum) => { asap_types::Statistic::Count } ( - control_plane::query_plan::ExactReadout::Sum, + asap_types::query_plan::ExactReadout::Sum, AggregationType::Sum | AggregationType::MultipleSum, ) => asap_types::Statistic::Sum, ( - control_plane::query_plan::ExactReadout::Increase, + asap_types::query_plan::ExactReadout::Increase, AggregationType::Increase | AggregationType::MultipleIncrease, ) => asap_types::Statistic::Increase, ( - control_plane::query_plan::ExactReadout::Rate, + asap_types::query_plan::ExactReadout::Rate, AggregationType::Increase | AggregationType::MultipleIncrease, ) => asap_types::Statistic::Rate, ( - control_plane::query_plan::ExactReadout::Max, + asap_types::query_plan::ExactReadout::Max, AggregationType::MinMax | AggregationType::MultipleMinMax, ) => asap_types::Statistic::Max, _ => return None, @@ -284,7 +284,7 @@ impl GroupState { if matches!( agg_type, AggregationType::MinMax | AggregationType::MultipleMinMax - ) && readout == control_plane::query_plan::ExactReadout::Max + ) && readout == asap_types::query_plan::ExactReadout::Max { return entries .iter() @@ -312,7 +312,7 @@ impl GroupState { ("range_end_ms".to_string(), range_end_ms.to_string()), ]); let merged = merged?; - if readout == control_plane::query_plan::ExactReadout::Count { + if readout == asap_types::query_plan::ExactReadout::Count { return merged.aux_stats().count.map(|count| count as f64); } merged.query_statistic(stat, key, &query_kwargs).ok() @@ -406,7 +406,7 @@ impl SummaryValue { } fn validate_binding_phase( - binding: &control_plane::query_plan::MaterializationBinding, + binding: &asap_types::query_plan::MaterializationBinding, evaluation_ms: u64, ) -> Result<(), SummaryExecutorError> { if i64::try_from(binding.window_ms).is_err() { @@ -435,9 +435,9 @@ impl QueryExecutionContext<'_> { /// are integrity checks and never broaden the candidate set. pub fn read_bound_materialization( &self, - binding: &control_plane::query_plan::MaterializationBinding, + binding: &asap_types::query_plan::MaterializationBinding, ) -> Result, GroupState)>, SummaryExecutorError> { - use control_plane::query_plan::PhysicalGrouping; + use asap_types::query_plan::PhysicalGrouping; let inventory_revision = self.index.summary_update_revision(); let query_range = asap_types::sds::HalfOpenTimeRange { @@ -1410,10 +1410,10 @@ mod tests { #[test] fn pane_only_reads_require_the_planned_evaluation_phase() { - let binding = control_plane::query_plan::MaterializationBinding { + let binding = asap_types::query_plan::MaterializationBinding { item_labels: Vec::new(), materialization: asap_types::PolicyFingerprint(7).into(), - output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, + output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 60_000, pane_origin_ms: Some(7_000), readout_lookback_ms: Some(60_000), @@ -1421,7 +1421,7 @@ mod tests { validate_binding_phase(&binding, 67_000).unwrap(); assert!(validate_binding_phase(&binding, 68_000).is_err()); - let legacy = control_plane::query_plan::MaterializationBinding { + let legacy = asap_types::query_plan::MaterializationBinding { item_labels: Vec::new(), pane_origin_ms: None, ..binding @@ -1819,7 +1819,7 @@ mod tests { use crate::precompute_engine::operators::univmon_accumulator::UnivMonAccumulator; use crate::storage_engines::sketch_db::index::SketchEncoding; use crate::storage_engines::types::SerializableToSink; - use control_plane::query_plan::{MaterializationBinding, PhysicalGrouping}; + use asap_types::query_plan::{MaterializationBinding, PhysicalGrouping}; let index = SketchStore::new(); let fp = asap_types::PolicyFingerprint(701); let mut meta = kll_meta(1, "m", &["job"]); diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index c007f8f5b..9cba552f9 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -94,7 +94,7 @@ pub struct ActivePhysicalPlan { pub precompute_plan: asap_types::precompute_plan::PrecomputePlan, pub transmission_plan: control_plane::physical::compiler::TransmissionPlan, pub runtime_config: Arc, - pub query_plan: Arc, + pub query_plan: Arc, pub storage_routing: Arc, } @@ -707,7 +707,7 @@ mod tests { rules: Vec::new(), }, runtime_config: Arc::new(StreamingConfig::new(HashMap::new())), - query_plan: Arc::new(control_plane::query_plan::QueryPlan { + query_plan: Arc::new(asap_types::query_plan::QueryPlan { plan_id, plan_version, clickhouse_context: None, diff --git a/data_plane/tests/clickhouse_differential_e2e.rs b/data_plane/tests/clickhouse_differential_e2e.rs index 871bb0b1b..75a07fd46 100644 --- a/data_plane/tests/clickhouse_differential_e2e.rs +++ b/data_plane/tests/clickhouse_differential_e2e.rs @@ -270,13 +270,13 @@ async fn run_mixed_aggregate(aggregate: &str) { assert_eq!( entry.nodes.values().any(|node| matches!( node, - control_plane::query_plan::QueryPlanNode::ExternalExact { .. } + asap_types::query_plan::QueryPlanNode::ExternalExact { .. } )), !grouped ); assert!(entry.nodes.values().any(|node| matches!( node, - control_plane::query_plan::QueryPlanNode::ReadMaterialization { .. } + asap_types::query_plan::QueryPlanNode::ReadMaterialization { .. } ))); let install = publication.install_request(None, Vec::new()).unwrap(); diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index a6d9366fd..0bf5ff300 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -90,8 +90,7 @@ async fn post_full_config(client: &reqwest::Client, stack: &FullStack, json: &Js { entry.instant.lookback_ms = 1000; for node in entry.nodes.values_mut() { - if let control_plane::query_plan::QueryPlanNode::ReadMaterialization { binding } = node - { + if let asap_types::query_plan::QueryPlanNode::ReadMaterialization { binding } = node { binding.readout_lookback_ms = Some(1000); } } diff --git a/data_plane/tests/support/physical_fixture.rs b/data_plane/tests/support/physical_fixture.rs index 30de3d05d..00ae0e862 100644 --- a/data_plane/tests/support/physical_fixture.rs +++ b/data_plane/tests/support/physical_fixture.rs @@ -104,7 +104,7 @@ pub fn artifact(config: &StreamingConfig) -> PhysicalPlanInstallRequest { query_plan.entries.insert( canonical.clone(), QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: canonical.clone(), canonical_query: canonical, fixed_evaluation: None, diff --git a/docs/design_docs/asapplanner-integration.md b/docs/design_docs/asapplanner-integration.md index f1f1d082b..c6cc83be6 100644 --- a/docs/design_docs/asapplanner-integration.md +++ b/docs/design_docs/asapplanner-integration.md @@ -106,7 +106,8 @@ Evidence: [selection adapter](../../control_plane/src/planner_selection.rs), [physical compiler](../../control_plane/src/physical/compiler.rs), [legacy workload adapter](../../control_plane/src/physical/workload_planner.rs), -[QueryPlan](../../control_plane/src/query_plan.rs), and +[shared QueryPlan](../../crates/asap_types/src/query_plan.rs), +[query lowering](../../control_plane/src/query_plan.rs), and [bound serving executor](../../data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs). The selection adapter explicitly commits a ranked Planner candidate downstream. Consequently, the figure does not imply that the Planner library deploys or diff --git a/docs/design_docs/summary-catalog-sds-architecture.md b/docs/design_docs/summary-catalog-sds-architecture.md index 59e1fb72f..b712c0d94 100644 --- a/docs/design_docs/summary-catalog-sds-architecture.md +++ b/docs/design_docs/summary-catalog-sds-architecture.md @@ -162,8 +162,12 @@ bindings against QueryPlan; precompute execution consumes the shared contract. `PrecomputePlan`, its envelope, ingest, producer, state schema, and catalog consistency checks live in `asap_types::precompute_plan`. The compiler chooses materializations and placement; data-plane installation uses the shared -contract. `QueryPlan` definitions still reside in the control-plane -crate while their remaining compilation methods are separated from wire types. +contract. `asap_types::query_plan` owns QueryPlan, materialization bindings, +logical operator DTOs, and activation validation. The control plane reexports +those types for existing callers and owns the `compile_bound*` and +`logical::compile_logical` functions; Planner traversal and AST lowering do not +move into the shared contract. Data-plane engines import the shared types +directly. No wrapper plan or second wire definition is introduced. The implemented ownership split is: From 3dd6ae2bf9535cde1a8c543c38a21cb552d47ba3 Mon Sep 17 00:00:00 2001 From: Zeying Zhu <50204836+zzylol@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:10:46 -0400 Subject: [PATCH 23/28] refactor: share installed producer and publication contracts (#643) * refactor: share installed QueryPlan contracts across components * refactor(query): use shared identity contract in MetricsQL adapter * style: format shared query identity fixture * refactor(plans): share producer and publication contracts --- .../examples/audit_clickhouse_corpus.rs | 10 +- control_plane/src/clickhouse.rs | 13 +- control_plane/src/emit/backend_push.rs | 2 +- control_plane/src/physical/compiler.rs | 1158 +++-------------- control_plane/src/physical/publication.rs | 127 +- crates/asap_types/src/lib.rs | 2 + crates/asap_types/src/plan_publication.rs | 126 ++ crates/asap_types/src/producer_plan.rs | 879 +++++++++++++ .../examples/audit_clickhouse_fallback.rs | 12 +- data_plane/src/drivers/ingest/otel.rs | 13 +- .../drivers/ingest/prometheus_remote_write.rs | 5 +- data_plane/src/drivers/query/servers/http.rs | 7 +- data_plane/src/main.rs | 6 +- .../src/precompute_engine/frame_lineage.rs | 2 +- .../accelerator.rs | 2 +- .../storage_engines/sketch_db/index/mod.rs | 13 +- .../types/hot_reload_config.rs | 17 +- ...e2e_controller_plans_and_backend_serves.rs | 4 +- data_plane/tests/support/physical_fixture.rs | 8 +- .../summary-catalog-sds-architecture.md | 7 + 20 files changed, 1244 insertions(+), 1169 deletions(-) create mode 100644 crates/asap_types/src/plan_publication.rs create mode 100644 crates/asap_types/src/producer_plan.rs diff --git a/control_plane/examples/audit_clickhouse_corpus.rs b/control_plane/examples/audit_clickhouse_corpus.rs index 71fedf79e..e7497b8d1 100644 --- a/control_plane/examples/audit_clickhouse_corpus.rs +++ b/control_plane/examples/audit_clickhouse_corpus.rs @@ -4,7 +4,7 @@ use asap_frontend_sql::SqlCatalog; use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; use control_plane::clickhouse::{ClickHouseSqlWorkload, ClickHouseSqlWorkloadEntry}; use control_plane::physical::compiler::{ - PlanEnvelope, PrecomputePlan, TransmissionPlan, BACKEND_COMPAT, PLANNER_REVISION, + PlanEnvelope, PrecomputePlan, BACKEND_COMPAT, PLANNER_REVISION, }; use planner_types::pre_asap::{Column, DataType, Schema}; use planner_types::types::AccuracyTarget; @@ -56,8 +56,12 @@ fn publication_inputs(schema: &Schema, sql: String) -> ClickHouseSqlWorkload { let mut precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![materialization.clone()]) .unwrap(); - let mut transmission_plan = - TransmissionPlan::build(envelope, &precompute_plan, &Default::default()).unwrap(); + let mut transmission_plan = control_plane::physical::compiler::compile_transmission_plan( + envelope, + &precompute_plan, + &Default::default(), + ) + .unwrap(); let sds = asap_types::summary_catalog::SummaryCatalog::from_materializations( 27, 1, diff --git a/control_plane/src/clickhouse.rs b/control_plane/src/clickhouse.rs index 533219378..f1334574f 100644 --- a/control_plane/src/clickhouse.rs +++ b/control_plane/src/clickhouse.rs @@ -183,7 +183,7 @@ pub async fn compile_automatic_clickhouse_workload( .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?, ); precompute.executable_dags = installed_dags; - let mut transmission = TransmissionPlan::build( + let mut transmission = crate::physical::compiler::compile_transmission_plan( request.envelope.clone(), &precompute, &std::collections::BTreeMap::new(), @@ -1010,9 +1010,12 @@ mod tests { let mut precompute = PrecomputePlan::build_backend_local(envelope.clone(), vec![config]).unwrap(); precompute.summary_catalog = Some(sds.reference().unwrap()); - let mut transmission = - TransmissionPlan::build(envelope, &precompute, &std::collections::BTreeMap::new()) - .unwrap(); + let mut transmission = crate::physical::compiler::compile_transmission_plan( + envelope, + &precompute, + &std::collections::BTreeMap::new(), + ) + .unwrap(); transmission.summary_catalog = Some(sds.reference().unwrap()); let timestamped = |time_name: &str, value_name: &str| { Schema::with_time_index( @@ -1204,7 +1207,7 @@ mod tests { request.precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![config]).unwrap(); request.precompute_plan.summary_catalog = Some(request.sds.reference().unwrap()); - request.transmission_plan = TransmissionPlan::build( + request.transmission_plan = crate::physical::compiler::compile_transmission_plan( envelope, &request.precompute_plan, &std::collections::BTreeMap::new(), diff --git a/control_plane/src/emit/backend_push.rs b/control_plane/src/emit/backend_push.rs index d3f4861de..9be93c761 100644 --- a/control_plane/src/emit/backend_push.rs +++ b/control_plane/src/emit/backend_push.rs @@ -260,7 +260,7 @@ async fn push_documents_coupled( warn!(%error, "failed to bind compatibility PrecomputePlan to SummaryCatalog"); return (false, false, 0); } - let transmission_plan = match crate::physical::compiler::TransmissionPlan::build( + let transmission_plan = match crate::physical::compiler::compile_transmission_plan( precompute_plan.envelope.clone(), &precompute_plan, &Default::default(), diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 6026ac86a..26482cc23 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -258,47 +258,136 @@ pub use asap_types::precompute_plan::{ StateWindowContract, TimestampUnit, }; -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct CollectorMaterialization { - pub query_id: String, - pub materialization: asap_types::sds::SummaryDefinitionId, - pub metric: String, - pub algorithm: String, - pub parameters: Value, - pub group_by: Vec, - pub window_secs: u64, - pub abstract_window_framework: SummaryWindowFramework, - pub window_implementation_id: String, - pub slide_secs: u64, - #[serde( - default, - alias = "paneOriginMs", - skip_serializing_if = "Option::is_none" - )] - pub pane_origin_ms: Option, - pub window_layout: asap_types::WindowMaterializationLayout, - pub evidence_source: Option, - pub lifecycle: CollectorLifecycle, +pub use asap_types::producer_plan::{ + AdaptiveF64Bounds, AdaptiveU64Bounds, CollectorLifecycle, CollectorMaterialization, + CollectorPlan, DeltaPolicy, FrameIdentityContract, GosPolicy, GosThresholdMode, + RuntimeAdaptationEvidence, RuntimeAdaptationPolicy, RuntimeRulePolicy, SamplingEstimator, + SamplingPolicy, SequenceScope, SummaryFrameIdentity, SummaryFrameKind, TransmissionMode, + TransmissionPlan, TransmissionPlanError, TransmissionRule, +}; + +/// Build the fixed physical knob from the controller's canonical +/// epsilon-floor allocator. Degenerate budgets/rates disable sampling. +pub fn sampling_policy_from_accuracy_budget( + epsilon_sampling: f64, + updates_per_window: f64, + estimator: SamplingEstimator, +) -> SamplingPolicy { + if !epsilon_sampling.is_finite() + || epsilon_sampling <= 0.0 + || !updates_per_window.is_finite() + || updates_per_window <= 0.0 + { + return SamplingPolicy::Disabled; + } + let probability = crate::epsilon_alloc::derive_sample_p(epsilon_sampling, updates_per_window); + if probability >= 1.0 { + SamplingPolicy::Disabled + } else { + SamplingPolicy::Fixed { + probability, + estimator, + } + } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct CollectorLifecycle { - pub kind: String, - pub maintenance_mode: String, - pub evaluation_schedule: String, - pub output_representation: String, +/// Allocate the deterministic staleness share with the same linear-peel +/// composition used by `epsilon_alloc`. `None` means the selected sketch +/// already consumes the budget or communication has no allocated weight. +pub fn gos_policy_from_accuracy_budget( + epsilon_total: f64, + epsilon_sketch: f64, + sites: u32, + edge_cpu_weight: f64, + communication_weight: f64, + threshold_mode: GosThresholdMode, +) -> Option { + if !epsilon_total.is_finite() + || !epsilon_sketch.is_finite() + || !(0.0..=1.0).contains(&epsilon_total) + || epsilon_sketch < 0.0 + || !edge_cpu_weight.is_finite() + || edge_cpu_weight < 0.0 + || !communication_weight.is_finite() + || communication_weight <= 0.0 + { + return None; + } + let (_, epsilon_staleness) = crate::epsilon_alloc::split_budget( + epsilon_total, + epsilon_sketch, + edge_cpu_weight, + communication_weight, + ); + (epsilon_staleness.is_finite() && epsilon_staleness > 0.0).then_some(GosPolicy { + epsilon_staleness, + sites: sites.max(1), + threshold_mode, + }) } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct CollectorPlan { - /// Absent only in legacy artifacts; catalog-aware validation requires it. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub summary_catalog: Option, - pub collector_id: String, - pub envelope: PlanEnvelope, - pub materializations: Vec, - pub transmission_rules: Vec, +pub fn compile_transmission_plan( + envelope: PlanEnvelope, + precompute: &PrecomputePlan, + runtime_policies: &BTreeMap, +) -> Result { + if envelope != precompute.envelope { + return Err(TransmissionPlanError::EnvelopeMismatch); + } + let schemas: BTreeMap<_, _> = precompute + .schemas + .iter() + .map(|schema| (schema.materialization, schema)) + .collect(); + let rules = precompute + .producers + .iter() + .map(|producer| { + let schema = schemas + .get(&producer.materialization) + .expect("validated PrecomputePlan schema binding"); + let materialization = precompute + .materializations + .iter() + .find(|m| m.policy_fingerprint() == producer.materialization.fingerprint()) + .expect("validated PrecomputePlan materialization binding"); + let runtime_policy = runtime_policies + .get(&producer.materialization.fingerprint()) + .cloned() + .unwrap_or_default(); + let mode = if runtime_policy.delta.is_some() { + TransmissionMode::Delta + } else { + TransmissionMode::Full + }; + let emit_every_ms = materialization.window_size.saturating_mul(1_000); + TransmissionRule { + materialization: producer.materialization, + producer_id: producer.producer_id.clone(), + schema_id: producer.schema_id.clone(), + mode, + encoding: schema.encodings[0].clone(), + emit_every_ms, + full_checkpoint_every_ms: (mode == TransmissionMode::Delta) + .then(|| emit_every_ms.saturating_mul(10)), + destination_ref: "asapquery-backend".into(), + runtime_policy, + } + }) + .collect(); + let plan = TransmissionPlan { + summary_catalog: precompute.summary_catalog.clone(), + envelope, + frame_identity: FrameIdentityContract { + identity_version: 1, + sequence_scope: SequenceScope::MaterializationSeriesProducerEpoch, + require_checkpoint_for_full: true, + require_base_checkpoint_for_delta: true, + }, + rules, + }; + plan.validate(precompute)?; + Ok(plan) } /// Complete physical projection of one post-ASAP planning decision. @@ -327,905 +416,6 @@ pub struct MaterializationLifecycleEstimate { pub lifecycle_cost: f64, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -#[serde(rename_all = "snake_case")] -pub enum TransmissionMode { - Full, - Delta, -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum SequenceScope { - MaterializationSeriesProducerEpoch, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct FrameIdentityContract { - pub identity_version: u32, - pub sequence_scope: SequenceScope, - pub require_checkpoint_for_full: bool, - pub require_base_checkpoint_for_delta: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct TransmissionRule { - pub materialization: asap_types::sds::SummaryDefinitionId, - pub producer_id: String, - pub schema_id: String, - pub mode: TransmissionMode, - pub encoding: StateEncoding, - pub emit_every_ms: u64, - pub full_checkpoint_every_ms: Option, - pub destination_ref: String, - /// Plan-owned runtime knobs. These values are part of the immutable plan - /// generation; live feedback may only change them by publishing a - /// successor generation accepted by [`TransmissionPlan::authorize_successor`]. - #[serde(default)] - pub runtime_policy: RuntimeRulePolicy, -} - -/// How the collector admits updates before sketch maintenance. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(tag = "mode", rename_all = "snake_case", deny_unknown_fields)] -pub enum SamplingPolicy { - Disabled, - Fixed { - /// Probability in `(0, 1]`; `1` is valid but should normally be - /// represented by `Disabled`. - probability: f64, - estimator: SamplingEstimator, - }, -} - -impl Default for SamplingPolicy { - fn default() -> Self { - Self::Disabled - } -} - -impl SamplingPolicy { - /// Build the fixed physical knob from the controller's canonical - /// epsilon-floor allocator. Degenerate budgets/rates disable sampling. - pub fn from_accuracy_budget( - epsilon_sampling: f64, - updates_per_window: f64, - estimator: SamplingEstimator, - ) -> Self { - if !epsilon_sampling.is_finite() - || epsilon_sampling <= 0.0 - || !updates_per_window.is_finite() - || updates_per_window <= 0.0 - { - return Self::Disabled; - } - let probability = - crate::epsilon_alloc::derive_sample_p(epsilon_sampling, updates_per_window); - if probability >= 1.0 { - Self::Disabled - } else { - Self::Fixed { - probability, - estimator, - } - } - } -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum SamplingEstimator { - /// Hash-threshold element sampling, used by cardinality summaries. - HashThreshold, - /// Geometric admission/Nitro-style update sampling, used by frequency - /// summaries. The sketch readout carries the corresponding correction. - GeometricAdmission, -} - -/// Norm-adaptive Group-of-Sketches delta gating. GOS is meaningful only for -/// CountSketch families and only when the transmission rule is in delta mode. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct GosPolicy { - pub epsilon_staleness: f64, - pub sites: u32, - pub threshold_mode: GosThresholdMode, -} - -impl GosPolicy { - /// Allocate the deterministic staleness share with the same linear-peel - /// composition used by `epsilon_alloc`. `None` means the selected sketch - /// already consumes the budget or communication has no allocated weight. - pub fn from_accuracy_budget( - epsilon_total: f64, - epsilon_sketch: f64, - sites: u32, - edge_cpu_weight: f64, - communication_weight: f64, - threshold_mode: GosThresholdMode, - ) -> Option { - if !epsilon_total.is_finite() - || !epsilon_sketch.is_finite() - || !(0.0..=1.0).contains(&epsilon_total) - || epsilon_sketch < 0.0 - || !edge_cpu_weight.is_finite() - || edge_cpu_weight < 0.0 - || !communication_weight.is_finite() - || communication_weight <= 0.0 - { - return None; - } - let (_, epsilon_staleness) = crate::epsilon_alloc::split_budget( - epsilon_total, - epsilon_sketch, - edge_cpu_weight, - communication_weight, - ); - (epsilon_staleness.is_finite() && epsilon_staleness > 0.0).then_some(Self { - epsilon_staleness, - sites: sites.max(1), - threshold_mode, - }) - } -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum GosThresholdMode { - Isotropic, - Anisotropic, -} - -/// Sparse-delta semantics within a delta transmission rule. An absolute -/// threshold of zero sends every changed cell. When `gos` is present it -/// replaces the fixed threshold with the GOS norm-adaptive threshold. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct DeltaPolicy { - pub absolute_threshold: f64, - pub gos: Option, -} - -/// Inclusive bounds for one floating-point adaptation knob. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct AdaptiveF64Bounds { - pub min: f64, - pub max: f64, - pub max_step: f64, -} - -/// Inclusive bounds for one integer adaptation knob. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct AdaptiveU64Bounds { - pub min: u64, - pub max: u64, - pub max_step: u64, -} - -/// Guardrails for telemetry-driven runtime adaptation. This is an -/// authorization contract, not an instruction to mutate the active plan. -/// Every accepted change becomes a staged successor PhysicalPlan. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct RuntimeAdaptationPolicy { - pub enabled: bool, - pub not_before_unix_ms: u64, - pub max_evidence_age_ms: u64, - pub min_evidence_samples: u64, - pub sample_probability: Option, - pub emit_every_ms: Option, - pub delta_threshold: Option, - pub gos_epsilon_staleness: Option, -} - -impl Default for RuntimeAdaptationPolicy { - fn default() -> Self { - Self { - enabled: false, - not_before_unix_ms: 0, - max_evidence_age_ms: 0, - min_evidence_samples: 0, - sample_probability: None, - emit_every_ms: None, - delta_threshold: None, - gos_epsilon_staleness: None, - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] -#[serde(deny_unknown_fields)] -pub struct RuntimeRulePolicy { - #[serde(default)] - pub sampling: SamplingPolicy, - pub delta: Option, - #[serde(default)] - pub adaptation: RuntimeAdaptationPolicy, -} - -/// Identity and sufficiency information for evidence authorizing one rule's -/// successor knobs. Raw measurements remain in the runtime-samples store; the -/// authorization boundary needs only their exact provenance and sample count. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct RuntimeAdaptationEvidence { - pub plan_id: u64, - pub plan_version: u64, - pub materialization: asap_types::sds::SummaryDefinitionId, - pub producer_id: String, - pub schema_id: String, - pub producer_version: String, - pub observed_at_unix_ms: u64, - pub sample_count: u64, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct TransmissionPlan { - /// Absent only in legacy artifacts; catalog-aware validation requires it. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub summary_catalog: Option, - pub envelope: PlanEnvelope, - pub frame_identity: FrameIdentityContract, - pub rules: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum SummaryFrameKind { - Full, - Delta, -} - -/// Identity attached to every summary record. Window bounds come from the -/// data point; the remaining fields are carried as reserved `asap.frame.*` -/// attributes until the modified-OTLP schema gains a dedicated message. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct SummaryFrameIdentity { - pub identity_version: u32, - pub plan_id: u64, - pub plan_version: u64, - pub backend_compat: String, - pub materialization: asap_types::sds::SummaryDefinitionId, - /// Canonical producer-side identity for one concrete retained-label group. - pub series_identity: String, - pub schema_id: String, - pub producer_id: String, - pub producer_epoch: String, - pub window_start_unix_nano: u64, - pub window_end_unix_nano: u64, - pub sequence: u64, - pub kind: SummaryFrameKind, - pub encoding: StateEncoding, - pub checkpoint_id: Option, - pub base_checkpoint_id: Option, -} - -#[derive(Debug, Error, PartialEq, Eq)] -pub enum TransmissionPlanError { - #[error("summary catalog mismatch: {0}")] - Catalog(String), - #[error("TransmissionPlan envelope differs from PrecomputePlan")] - EnvelopeMismatch, - #[error("transmission rules do not exactly match precompute producer bindings")] - ProducerSetMismatch, - #[error("invalid transmission rule for producer {0}")] - InvalidRule(String), - #[error("frame identity is invalid: {0}")] - InvalidFrame(String), - #[error("frame has no matching transmission rule")] - UnknownFrame, - #[error("runtime policy for producer {producer_id} is invalid: {reason}")] - InvalidRuntimePolicy { producer_id: String, reason: String }, - #[error("runtime adaptation successor is invalid: {0}")] - InvalidSuccessor(String), - #[error("runtime adaptation evidence for producer {0} is missing or invalid")] - InvalidAdaptationEvidence(String), - #[error("runtime adaptation for producer {producer_id} exceeds guardrails: {knob}")] - AdaptationOutOfBounds { - producer_id: String, - knob: &'static str, - }, -} - -fn validate_catalog_projection( - reference: Option<&asap_types::sds::CatalogGeneration>, - envelope: &PlanEnvelope, - materializations: impl IntoIterator, - catalog: &super::summary_catalog::SummaryCatalog, -) -> Result<(), TransmissionPlanError> { - let expected = catalog - .reference() - .map_err(|error| TransmissionPlanError::Catalog(error.to_string()))?; - if reference != Some(&expected) - || envelope.plan_id != catalog.plan_id - || envelope.plan_version != catalog.plan_version - { - return Err(TransmissionPlanError::Catalog( - "missing or different snapshot reference".into(), - )); - } - for id in materializations { - if !catalog.materializations.contains_key(&id) { - return Err(TransmissionPlanError::Catalog(format!( - "unknown materialization {}", - id.as_u64() - ))); - } - } - Ok(()) -} - -impl CollectorPlan { - pub fn validate_against_catalog( - &self, - catalog: &super::summary_catalog::SummaryCatalog, - ) -> Result<(), TransmissionPlanError> { - validate_catalog_projection( - self.summary_catalog.as_ref(), - &self.envelope, - self.materializations - .iter() - .map(|m| m.materialization) - .chain(self.transmission_rules.iter().map(|r| r.materialization)), - catalog, - ) - } -} - -impl TransmissionPlan { - pub fn validate_against_catalog( - &self, - catalog: &super::summary_catalog::SummaryCatalog, - ) -> Result<(), TransmissionPlanError> { - validate_catalog_projection( - self.summary_catalog.as_ref(), - &self.envelope, - self.rules.iter().map(|r| r.materialization), - catalog, - ) - } - - pub fn build( - envelope: PlanEnvelope, - precompute: &PrecomputePlan, - runtime_policies: &BTreeMap, - ) -> Result { - if envelope != precompute.envelope { - return Err(TransmissionPlanError::EnvelopeMismatch); - } - let schemas: BTreeMap<_, _> = precompute - .schemas - .iter() - .map(|schema| (schema.materialization, schema)) - .collect(); - let rules = precompute - .producers - .iter() - .map(|producer| { - let schema = schemas - .get(&producer.materialization) - .expect("validated PrecomputePlan schema binding"); - let materialization = precompute - .materializations - .iter() - .find(|m| m.policy_fingerprint() == producer.materialization.fingerprint()) - .expect("validated PrecomputePlan materialization binding"); - let runtime_policy = runtime_policies - .get(&producer.materialization.fingerprint()) - .cloned() - .unwrap_or_default(); - let mode = if runtime_policy.delta.is_some() { - TransmissionMode::Delta - } else { - TransmissionMode::Full - }; - let emit_every_ms = materialization.window_size.saturating_mul(1_000); - TransmissionRule { - materialization: producer.materialization, - producer_id: producer.producer_id.clone(), - schema_id: producer.schema_id.clone(), - mode, - encoding: schema.encodings[0].clone(), - emit_every_ms, - full_checkpoint_every_ms: (mode == TransmissionMode::Delta) - .then(|| emit_every_ms.saturating_mul(10)), - destination_ref: "asapquery-backend".into(), - runtime_policy, - } - }) - .collect(); - let plan = Self { - summary_catalog: precompute.summary_catalog.clone(), - envelope, - frame_identity: FrameIdentityContract { - identity_version: 1, - sequence_scope: SequenceScope::MaterializationSeriesProducerEpoch, - require_checkpoint_for_full: true, - require_base_checkpoint_for_delta: true, - }, - rules, - }; - plan.validate(precompute)?; - Ok(plan) - } - - pub fn validate(&self, precompute: &PrecomputePlan) -> Result<(), TransmissionPlanError> { - if self.summary_catalog != precompute.summary_catalog { - return Err(TransmissionPlanError::Catalog( - "transmission and precompute plans reference different catalog snapshots".into(), - )); - } - if self.envelope != precompute.envelope { - return Err(TransmissionPlanError::EnvelopeMismatch); - } - let expected: BTreeSet<_> = precompute - .producers - .iter() - .map(|producer| { - ( - producer.materialization, - producer.producer_id.as_str(), - producer.schema_id.as_str(), - ) - }) - .collect(); - let actual: BTreeSet<_> = self - .rules - .iter() - .map(|rule| { - ( - rule.materialization, - rule.producer_id.as_str(), - rule.schema_id.as_str(), - ) - }) - .collect(); - if expected != actual || actual.len() != self.rules.len() { - return Err(TransmissionPlanError::ProducerSetMismatch); - } - for rule in &self.rules { - let valid_checkpoint_cadence = match (rule.mode, rule.full_checkpoint_every_ms) { - (TransmissionMode::Full, None) => true, - (TransmissionMode::Delta, Some(full_every)) => { - rule.emit_every_ms > 0 - && full_every >= rule.emit_every_ms - && full_every % rule.emit_every_ms == 0 - } - _ => false, - }; - if rule.emit_every_ms == 0 - || rule.destination_ref.is_empty() - || !valid_checkpoint_cadence - { - return Err(TransmissionPlanError::InvalidRule(rule.producer_id.clone())); - } - let schema = precompute - .schemas - .iter() - .find(|schema| schema.materialization == rule.materialization) - .expect("producer set validation guarantees a matching schema"); - if !schema.encodings.contains(&rule.encoding) { - return Err(TransmissionPlanError::InvalidRule(rule.producer_id.clone())); - } - validate_runtime_rule_policy(rule, &schema.family)?; - } - Ok(()) - } - - /// Authorize a telemetry-driven successor without mutating this active - /// plan. Semantic identity, codecs, destination, transmission mode and - /// checkpoint cadence remain fixed. Only explicitly bounded runtime knobs - /// may move, and each changed rule needs fresh evidence attributed to the - /// exact active generation. - pub fn authorize_successor( - &self, - successor: &TransmissionPlan, - evidence: &[RuntimeAdaptationEvidence], - now_unix_ms: u64, - ) -> Result<(), TransmissionPlanError> { - if successor.envelope.plan_id != self.envelope.plan_id - || successor.envelope.plan_version != self.envelope.plan_version.saturating_add(1) - || successor.envelope.backend_compat != self.envelope.backend_compat - || successor.envelope.planner_revision != self.envelope.planner_revision - || successor.envelope.capability_snapshot_id != self.envelope.capability_snapshot_id - || successor.envelope.generated_at_unix_ms < self.envelope.generated_at_unix_ms - || successor.envelope.activation_unix_ms < successor.envelope.generated_at_unix_ms - || successor.rules.len() != self.rules.len() - || successor.frame_identity != self.frame_identity - { - return Err(TransmissionPlanError::InvalidSuccessor( - "successor must be the next version of the same semantic/capability generation" - .into(), - )); - } - - for current in &self.rules { - let Some(next) = successor.rules.iter().find(|candidate| { - candidate.materialization == current.materialization - && candidate.producer_id == current.producer_id - && candidate.schema_id == current.schema_id - }) else { - return Err(TransmissionPlanError::InvalidSuccessor(format!( - "missing rule for producer {}", - current.producer_id - ))); - }; - if current.mode != next.mode - || current.encoding != next.encoding - || current.full_checkpoint_every_ms != next.full_checkpoint_every_ms - || current.destination_ref != next.destination_ref - || current.runtime_policy.adaptation != next.runtime_policy.adaptation - || sampling_estimator(¤t.runtime_policy.sampling) - != sampling_estimator(&next.runtime_policy.sampling) - || delta_shape(¤t.runtime_policy.delta) - != delta_shape(&next.runtime_policy.delta) - { - return Err(TransmissionPlanError::InvalidSuccessor(format!( - "rule identity/codec/mode/guardrails drifted for producer {}", - current.producer_id - ))); - } - if current.emit_every_ms == next.emit_every_ms - && current.runtime_policy.sampling == next.runtime_policy.sampling - && current.runtime_policy.delta == next.runtime_policy.delta - { - continue; - } - let policy = ¤t.runtime_policy.adaptation; - if !policy.enabled || now_unix_ms < policy.not_before_unix_ms { - return Err(TransmissionPlanError::AdaptationOutOfBounds { - producer_id: current.producer_id.clone(), - knob: "adaptation_disabled_or_in_cooldown", - }); - } - let has_evidence = evidence.iter().any(|item| { - item.plan_id == self.envelope.plan_id - && item.plan_version == self.envelope.plan_version - && item.materialization == current.materialization - && item.producer_id == current.producer_id - && item.schema_id == current.schema_id - && !item.producer_version.trim().is_empty() - && item.sample_count >= policy.min_evidence_samples - && item.observed_at_unix_ms <= now_unix_ms - && now_unix_ms.saturating_sub(item.observed_at_unix_ms) - <= policy.max_evidence_age_ms - }); - if !has_evidence { - return Err(TransmissionPlanError::InvalidAdaptationEvidence( - current.producer_id.clone(), - )); - } - authorize_f64_change( - sampling_probability(¤t.runtime_policy.sampling), - sampling_probability(&next.runtime_policy.sampling), - policy.sample_probability.as_ref(), - ¤t.producer_id, - "sample_probability", - )?; - authorize_u64_change( - current.emit_every_ms, - next.emit_every_ms, - policy.emit_every_ms.as_ref(), - ¤t.producer_id, - "emit_every_ms", - )?; - authorize_f64_change( - delta_threshold(¤t.runtime_policy.delta), - delta_threshold(&next.runtime_policy.delta), - policy.delta_threshold.as_ref(), - ¤t.producer_id, - "delta_threshold", - )?; - authorize_f64_change( - gos_epsilon(¤t.runtime_policy.delta), - gos_epsilon(&next.runtime_policy.delta), - policy.gos_epsilon_staleness.as_ref(), - ¤t.producer_id, - "gos_epsilon_staleness", - )?; - } - Ok(()) - } - - pub fn validate_frame( - &self, - frame: &SummaryFrameIdentity, - ) -> Result<(), TransmissionPlanError> { - if frame.identity_version != self.frame_identity.identity_version - || frame.plan_id != self.envelope.plan_id - || frame.plan_version != self.envelope.plan_version - || frame.backend_compat != self.envelope.backend_compat - || frame.series_identity.is_empty() - || frame.producer_epoch.is_empty() - || frame.sequence == 0 - || frame.window_start_unix_nano >= frame.window_end_unix_nano - || (frame.kind == SummaryFrameKind::Full - && self.frame_identity.require_checkpoint_for_full - && frame.checkpoint_id.is_none()) - || (frame.kind == SummaryFrameKind::Delta - && self.frame_identity.require_base_checkpoint_for_delta - && frame.base_checkpoint_id.is_none()) - { - return Err(TransmissionPlanError::InvalidFrame( - "identity/lifecycle/window/checkpoint fields do not satisfy the active contract" - .into(), - )); - } - if self.rules.iter().any(|rule| { - rule.materialization == frame.materialization - && rule.producer_id == frame.producer_id - && rule.schema_id == frame.schema_id - // A delta rule necessarily emits periodic full checkpoints; - // a full-only rule must never emit deltas. - && (frame.kind == SummaryFrameKind::Full - || rule.mode == TransmissionMode::Delta) - && rule.encoding == frame.encoding - }) { - Ok(()) - } else { - Err(TransmissionPlanError::UnknownFrame) - } - } -} - -fn validate_runtime_rule_policy( - rule: &TransmissionRule, - family: &StateFamilyContract, -) -> Result<(), TransmissionPlanError> { - let invalid = |reason: &str| TransmissionPlanError::InvalidRuntimePolicy { - producer_id: rule.producer_id.clone(), - reason: reason.into(), - }; - if let SamplingPolicy::Fixed { - probability, - estimator, - } = &rule.runtime_policy.sampling - { - if !probability.is_finite() || !(0.0..=1.0).contains(probability) || *probability == 0.0 { - return Err(invalid("sample probability must be finite and in (0, 1]")); - } - let supported = matches!( - (family, *estimator), - ( - StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::Hll, - .. - }, - SamplingEstimator::HashThreshold, - ) | ( - StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap, - .. - }, - SamplingEstimator::GeometricAdmission, - ) - ); - if !supported { - return Err(invalid( - "sampling estimator is not implemented for the materialization family", - )); - } - } - - if (rule.mode == TransmissionMode::Delta) != rule.runtime_policy.delta.is_some() { - return Err(invalid( - "delta policy must be present exactly when transmission mode is delta", - )); - } - if let Some(delta) = &rule.runtime_policy.delta { - if !delta.absolute_threshold.is_finite() || delta.absolute_threshold < 0.0 { - return Err(invalid("delta threshold must be finite and non-negative")); - } - if !matches!( - family, - StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::DDSketch - | SketchAlgorithm::Hll - | SketchAlgorithm::Cms - | SketchAlgorithm::CmsWithHeap - | SketchAlgorithm::CountSketch - | SketchAlgorithm::CountSketchWithHeap, - .. - } - ) { - return Err(invalid( - "delta transmission is not implemented for the materialization family", - )); - } - if let Some(gos) = &delta.gos { - if !matches!( - family, - StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap, - .. - } - ) || !gos.epsilon_staleness.is_finite() - || !(0.0..=1.0).contains(&gos.epsilon_staleness) - || gos.epsilon_staleness == 0.0 - || gos.sites == 0 - { - return Err(invalid( - "GOS requires a CountSketch family, epsilon in (0, 1], and at least one site", - )); - } - } - } - - let adaptation = &rule.runtime_policy.adaptation; - if adaptation.enabled - && (adaptation.max_evidence_age_ms == 0 || adaptation.min_evidence_samples == 0) - { - return Err(invalid( - "enabled adaptation requires non-zero evidence age and sample-count requirements", - )); - } - validate_f64_bounds(adaptation.sample_probability.as_ref(), 0.0, 1.0) - .map_err(|reason| invalid(reason))?; - validate_f64_bounds(adaptation.delta_threshold.as_ref(), 0.0, f64::MAX) - .map_err(|reason| invalid(reason))?; - validate_f64_bounds(adaptation.gos_epsilon_staleness.as_ref(), 0.0, 1.0) - .map_err(|reason| invalid(reason))?; - if let Some(bounds) = &adaptation.emit_every_ms { - if bounds.min == 0 - || bounds.min > bounds.max - || bounds.max_step == 0 - || !(bounds.min..=bounds.max).contains(&rule.emit_every_ms) - { - return Err(invalid("emit interval guardrails are invalid")); - } - } - if let Some(bounds) = &adaptation.sample_probability { - let current = sampling_probability(&rule.runtime_policy.sampling); - if current < bounds.min || current > bounds.max { - return Err(invalid( - "current sampling probability is outside guardrails", - )); - } - } - if let Some(bounds) = &adaptation.delta_threshold { - let Some(current) = rule - .runtime_policy - .delta - .as_ref() - .map(|policy| policy.absolute_threshold) - else { - return Err(invalid("delta guardrails require an active delta policy")); - }; - if current < bounds.min || current > bounds.max { - return Err(invalid("current delta threshold is outside guardrails")); - } - } - if let Some(bounds) = &adaptation.gos_epsilon_staleness { - let Some(current) = rule - .runtime_policy - .delta - .as_ref() - .and_then(|policy| policy.gos.as_ref()) - .map(|gos| gos.epsilon_staleness) - else { - return Err(invalid("GOS guardrails require an active GOS policy")); - }; - if current < bounds.min || current > bounds.max { - return Err(invalid("current GOS epsilon is outside guardrails")); - } - } - Ok(()) -} - -fn validate_f64_bounds( - bounds: Option<&AdaptiveF64Bounds>, - domain_min: f64, - domain_max: f64, -) -> Result<(), &'static str> { - let Some(bounds) = bounds else { - return Ok(()); - }; - if !bounds.min.is_finite() - || !bounds.max.is_finite() - || !bounds.max_step.is_finite() - || bounds.min < domain_min - || bounds.max > domain_max - || bounds.min > bounds.max - || bounds.max_step <= 0.0 - { - Err("floating-point adaptation guardrails are invalid") - } else { - Ok(()) - } -} - -fn sampling_probability(policy: &SamplingPolicy) -> f64 { - match policy { - SamplingPolicy::Disabled => 1.0, - SamplingPolicy::Fixed { probability, .. } => *probability, - } -} - -fn sampling_estimator(policy: &SamplingPolicy) -> Option { - match policy { - SamplingPolicy::Disabled => None, - SamplingPolicy::Fixed { estimator, .. } => Some(*estimator), - } -} - -fn delta_threshold(policy: &Option) -> f64 { - policy - .as_ref() - .map(|policy| policy.absolute_threshold) - .unwrap_or(0.0) -} - -fn gos_epsilon(policy: &Option) -> f64 { - policy - .as_ref() - .and_then(|policy| policy.gos.as_ref()) - .map(|gos| gos.epsilon_staleness) - .unwrap_or(0.0) -} - -fn delta_shape(policy: &Option) -> Option<(Option<(u32, GosThresholdMode)>,)> { - policy.as_ref().map(|policy| { - (policy - .gos - .as_ref() - .map(|gos| (gos.sites, gos.threshold_mode)),) - }) -} - -fn authorize_f64_change( - current: f64, - next: f64, - bounds: Option<&AdaptiveF64Bounds>, - producer_id: &str, - knob: &'static str, -) -> Result<(), TransmissionPlanError> { - if current == next { - return Ok(()); - } - let allowed = bounds.is_some_and(|bounds| { - next.is_finite() - && (bounds.min..=bounds.max).contains(&next) - && (next - current).abs() <= bounds.max_step - }); - if allowed { - Ok(()) - } else { - Err(TransmissionPlanError::AdaptationOutOfBounds { - producer_id: producer_id.into(), - knob, - }) - } -} - -fn authorize_u64_change( - current: u64, - next: u64, - bounds: Option<&AdaptiveU64Bounds>, - producer_id: &str, - knob: &'static str, -) -> Result<(), TransmissionPlanError> { - if current == next { - return Ok(()); - } - let allowed = bounds.is_some_and(|bounds| { - (bounds.min..=bounds.max).contains(&next) && current.abs_diff(next) <= bounds.max_step - }); - if allowed { - Ok(()) - } else { - Err(TransmissionPlanError::AdaptationOutOfBounds { - producer_id: producer_id.into(), - knob, - }) - } -} - #[derive(Debug, Error)] pub enum CompileError { #[error("invalid backend-local workload snapshot: {0}")] @@ -2089,12 +1279,15 @@ impl PhysicalCompiler { query_id: "precompute-plan".into(), reason: error.to_string(), })?; - let mut transmission_plan = - TransmissionPlan::build(envelope.clone(), &precompute_plan, &runtime_policies) - .map_err(|error| CompileError::Query { - query_id: "transmission-plan".into(), - reason: error.to_string(), - })?; + let mut transmission_plan = crate::physical::compiler::compile_transmission_plan( + envelope.clone(), + &precompute_plan, + &runtime_policies, + ) + .map_err(|error| CompileError::Query { + query_id: "transmission-plan".into(), + reason: error.to_string(), + })?; let mut collector_plans = producer_ids .into_iter() .map(|collector_id| CollectorPlan { @@ -5222,13 +4415,21 @@ mod tests { collector.validate_against_catalog(catalog).unwrap(); collector.envelope.plan_version += 1; assert!(collector.validate_against_catalog(catalog).is_err()); - assert!(validate_catalog_projection( - Some(&catalog.reference().unwrap()), - &bundle.envelope, - [asap_types::PolicyFingerprint(u64::MAX).into()], - catalog - ) - .is_err()); + let mut unknown_materialization = bundle.transmission_plan.clone(); + unknown_materialization.rules.push(TransmissionRule { + materialization: asap_types::PolicyFingerprint(u64::MAX).into(), + producer_id: "foreign".into(), + schema_id: "foreign".into(), + mode: TransmissionMode::Full, + encoding: StateEncoding::ExactAccumulatorV1, + emit_every_ms: 60_000, + full_checkpoint_every_ms: None, + destination_ref: "backend".into(), + runtime_policy: Default::default(), + }); + assert!(unknown_materialization + .validate_against_catalog(catalog) + .is_err()); let encoded = serde_json::to_value(&collector).unwrap(); assert!(encoded["summary_catalog"] .get("summary_descriptors") @@ -5542,10 +4743,14 @@ mod tests { assert_eq!(plan.ingest.timestamp_unit, TimestampUnit::UnixMilliseconds); assert!(plan.producers.is_empty()); plan.validate().expect("valid backend-local projection"); - TransmissionPlan::build(bundle.envelope, &plan, &BTreeMap::new()) - .expect("empty backend-local transmission contract") - .validate(&plan) - .expect("valid empty transmission plan"); + crate::physical::compiler::compile_transmission_plan( + bundle.envelope, + &plan, + &BTreeMap::new(), + ) + .expect("empty backend-local transmission contract") + .validate(&plan) + .expect("valid empty transmission plan"); } #[test] @@ -6005,6 +5210,9 @@ mod tests { ), full_selected ); + // Publication validates stored extent independently of slide cadence. + // In particular the full-window candidate stores 60s at a 10s slide. + bundle.publication().unwrap().validate().unwrap(); assert_eq!(bundle.precompute_plan.materializations[0].window_size, 60); assert_eq!( bundle.precompute_plan.materializations[0].slide_interval, @@ -6213,7 +5421,7 @@ mod tests { #[test] fn runtime_policy_uses_canonical_sampling_and_gos_allocators() { - let sampling = SamplingPolicy::from_accuracy_budget( + let sampling = sampling_policy_from_accuracy_budget( 0.05, 1_000.0, SamplingEstimator::GeometricAdmission, @@ -6224,64 +5432,26 @@ mod tests { assert!((probability - 1.0 / 3.5).abs() < 1e-9); let gos = - GosPolicy::from_accuracy_budget(0.1, 0.03, 4, 1.0, 1.0, GosThresholdMode::Isotropic) + gos_policy_from_accuracy_budget(0.1, 0.03, 4, 1.0, 1.0, GosThresholdMode::Isotropic) .expect("positive communication allocation"); assert!((gos.epsilon_staleness - 0.035).abs() < 1e-12); assert_eq!(gos.sites, 4); } #[test] - fn runtime_policy_is_family_and_mode_checked() { + fn runtime_policy_encoding_is_checked() { let bundle = PhysicalCompiler .compile( request("q", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) .expect("compile"); - let mut rule = bundle.transmission_plan.rules[0].clone(); let mut invalid_encoding = bundle.transmission_plan.clone(); invalid_encoding.rules[0].encoding = StateEncoding::ExactAccumulatorV1; assert!(matches!( invalid_encoding.validate(&bundle.precompute_plan), Err(TransmissionPlanError::InvalidRule(_)) )); - rule.runtime_policy.sampling = SamplingPolicy::Fixed { - probability: 0.5, - estimator: SamplingEstimator::HashThreshold, - }; - let hll = StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::Hll, - parameters: SketchParams::Hll { precision: 14 }, - }; - let count_sketch = StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::CountSketch, - parameters: SketchParams::CountSketch { - width: 128, - depth: 4, - }, - }; - validate_runtime_rule_policy(&rule, &hll).expect("HLL supports hash-threshold sampling"); - assert!(matches!( - validate_runtime_rule_policy(&rule, &count_sketch), - Err(TransmissionPlanError::InvalidRuntimePolicy { .. }) - )); - - rule.runtime_policy.sampling = SamplingPolicy::Disabled; - rule.mode = TransmissionMode::Delta; - rule.full_checkpoint_every_ms = Some(300_000); - rule.runtime_policy.delta = Some(DeltaPolicy { - absolute_threshold: 0.0, - gos: Some(GosPolicy { - epsilon_staleness: 0.02, - sites: 2, - threshold_mode: GosThresholdMode::Isotropic, - }), - }); - validate_runtime_rule_policy(&rule, &count_sketch).expect("CountSketch supports delta GOS"); - assert!(matches!( - validate_runtime_rule_policy(&rule, &hll), - Err(TransmissionPlanError::InvalidRuntimePolicy { .. }) - )); } #[test] diff --git a/control_plane/src/physical/publication.rs b/control_plane/src/physical/publication.rs index 42e28d086..63081fa5b 100644 --- a/control_plane/src/physical/publication.rs +++ b/control_plane/src/physical/publication.rs @@ -1,128 +1,7 @@ -//! Canonical publication document for one catalog generation. -use super::compiler::{CollectorPlan, PhysicalPlan, PrecomputePlan, TransmissionPlan}; -use super::summary_catalog::SummaryCatalog; -use crate::query_plan::QueryPlan; -use serde::{Deserialize, Serialize}; +//! Control-plane construction of the shared catalog publication contract. +use super::compiler::PhysicalPlan; +pub use asap_types::plan_publication::{PhysicalPlanInstallRequest, PhysicalPlanPublication}; -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PhysicalPlanPublication { - pub summary_catalog: SummaryCatalog, - pub precompute_plan: PrecomputePlan, - pub collector_plans: Vec, - pub transmission_plan: TransmissionPlan, - pub query_plan: QueryPlan, -} - -/// Complete typed envelope accepted by the data-plane install endpoint. -/// -/// Runtime-only routing and adaptation evidence decorate the compiler-owned -/// publication without changing its catalog generation. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PhysicalPlanInstallRequest { - pub summary_catalog: SummaryCatalog, - #[serde(default)] - pub collector_plans: Vec, - pub precompute_plan: PrecomputePlan, - pub transmission_plan: TransmissionPlan, - pub query_plan: QueryPlan, - pub storage_routing: Option, - #[serde(default)] - pub adaptation_evidence: Vec, -} - -impl PhysicalPlanPublication { - /// Validate every plan against the shared catalog snapshot. - pub fn validate(&self) -> Result<(), String> { - let catalog = &self.summary_catalog; - self.precompute_plan - .validate_against_catalog(catalog) - .map_err(|e| e.to_string())?; - self.transmission_plan - .validate(&self.precompute_plan) - .map_err(|e| e.to_string())?; - self.transmission_plan - .validate_against_catalog(catalog) - .map_err(|e| e.to_string())?; - self.query_plan - .validate_against_catalog(catalog) - .map_err(|e| e.to_string())?; - let materializations = self - .precompute_plan - .materializations - .iter() - .map(|config| (config.policy_fingerprint(), config)) - .collect::>(); - for entry in self.query_plan.entries.values() { - for binding in entry.materialization_bindings() { - let config = materializations - .get(&binding.materialization.fingerprint()) - .copied() - .ok_or("query binding has no precompute materialization")?; - if config.slide_interval.checked_mul(1000) != Some(binding.window_ms) { - return Err("query pane differs from precompute emission interval".into()); - } - if config.pane_origin_ms != binding.pane_origin_ms { - return Err("query pane origin differs from precompute definition".into()); - } - } - } - let mut collectors = std::collections::BTreeSet::new(); - for collector in &self.collector_plans { - if collector.envelope != self.precompute_plan.envelope - || !collectors.insert(&collector.collector_id) - { - return Err("collector envelope mismatch or duplicate collector".into()); - } - collector - .validate_against_catalog(catalog) - .map_err(|e| e.to_string())?; - for rule in &collector.transmission_rules { - if !self.transmission_plan.rules.contains(rule) { - return Err( - "collector transmission rule absent from published transmission plan" - .into(), - ); - } - } - } - for producer in &self.precompute_plan.producers { - let collector = self - .collector_plans - .iter() - .find(|c| c.collector_id == producer.collector_id) - .ok_or("precompute producer has no published CollectorPlan")?; - if !collector - .materializations - .iter() - .any(|m| m.materialization == producer.materialization) - { - return Err( - "collector does not produce referenced precompute materialization".into(), - ); - } - } - Ok(()) - } - - pub fn install_request( - &self, - storage_routing: Option, - adaptation_evidence: Vec, - ) -> Result { - self.validate()?; - Ok(PhysicalPlanInstallRequest { - summary_catalog: self.summary_catalog.clone(), - collector_plans: self.collector_plans.clone(), - precompute_plan: self.precompute_plan.clone(), - transmission_plan: self.transmission_plan.clone(), - query_plan: self.query_plan.clone(), - storage_routing, - adaptation_evidence, - }) - } -} impl PhysicalPlan { pub fn publication(&self) -> Result { let artifact = PhysicalPlanPublication { diff --git a/crates/asap_types/src/lib.rs b/crates/asap_types/src/lib.rs index 0b4f40eee..7f4673837 100644 --- a/crates/asap_types/src/lib.rs +++ b/crates/asap_types/src/lib.rs @@ -7,8 +7,10 @@ pub mod executable_plan; pub mod grouping_projection; pub mod key_by_label_names; pub mod monitor_spec; +pub mod plan_publication; pub mod policy_fingerprint; pub mod policy_registry; +pub mod producer_plan; pub mod query_requirements; pub mod routing_index; pub mod sds; diff --git a/crates/asap_types/src/plan_publication.rs b/crates/asap_types/src/plan_publication.rs new file mode 100644 index 000000000..35ce812f5 --- /dev/null +++ b/crates/asap_types/src/plan_publication.rs @@ -0,0 +1,126 @@ +//! Canonical publication document for one catalog generation. +use crate::precompute_plan::PrecomputePlan; +use crate::producer_plan::{CollectorPlan, TransmissionPlan}; +use crate::query_plan::QueryPlan; +use crate::summary_catalog::SummaryCatalog; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PhysicalPlanPublication { + pub summary_catalog: SummaryCatalog, + pub precompute_plan: PrecomputePlan, + pub collector_plans: Vec, + pub transmission_plan: TransmissionPlan, + pub query_plan: QueryPlan, +} + +/// Complete typed envelope accepted by the data-plane install endpoint. +/// +/// Runtime-only routing and adaptation evidence decorate the shared +/// publication without changing its catalog generation. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PhysicalPlanInstallRequest { + pub summary_catalog: SummaryCatalog, + #[serde(default)] + pub collector_plans: Vec, + pub precompute_plan: PrecomputePlan, + pub transmission_plan: TransmissionPlan, + pub query_plan: QueryPlan, + pub storage_routing: Option, + #[serde(default)] + pub adaptation_evidence: Vec, +} + +impl PhysicalPlanPublication { + /// Validate every plan against the shared catalog snapshot. + pub fn validate(&self) -> Result<(), String> { + let catalog = &self.summary_catalog; + self.precompute_plan + .validate_against_catalog(catalog) + .map_err(|e| e.to_string())?; + self.transmission_plan + .validate(&self.precompute_plan) + .map_err(|e| e.to_string())?; + self.transmission_plan + .validate_against_catalog(catalog) + .map_err(|e| e.to_string())?; + self.query_plan + .validate_against_catalog(catalog) + .map_err(|e| e.to_string())?; + let materializations = self + .precompute_plan + .materializations + .iter() + .map(|config| (config.policy_fingerprint(), config)) + .collect::>(); + for entry in self.query_plan.entries.values() { + for binding in entry.materialization_bindings() { + let config = materializations + .get(&binding.materialization.fingerprint()) + .copied() + .ok_or("query binding has no precompute materialization")?; + if config.stored_window_ms() != binding.window_ms { + return Err("query pane differs from precompute stored window".into()); + } + if config.pane_origin_ms != binding.pane_origin_ms { + return Err("query pane origin differs from precompute definition".into()); + } + } + } + let mut collectors = std::collections::BTreeSet::new(); + for collector in &self.collector_plans { + if collector.envelope != self.precompute_plan.envelope + || !collectors.insert(&collector.collector_id) + { + return Err("collector envelope mismatch or duplicate collector".into()); + } + collector + .validate_against_catalog(catalog) + .map_err(|e| e.to_string())?; + for rule in &collector.transmission_rules { + if !self.transmission_plan.rules.contains(rule) { + return Err( + "collector transmission rule absent from published transmission plan" + .into(), + ); + } + } + } + for producer in &self.precompute_plan.producers { + let collector = self + .collector_plans + .iter() + .find(|c| c.collector_id == producer.collector_id) + .ok_or("precompute producer has no published CollectorPlan")?; + if !collector + .materializations + .iter() + .any(|m| m.materialization == producer.materialization) + { + return Err( + "collector does not produce referenced precompute materialization".into(), + ); + } + } + Ok(()) + } + + pub fn install_request( + &self, + storage_routing: Option, + adaptation_evidence: Vec, + ) -> Result { + self.validate()?; + Ok(PhysicalPlanInstallRequest { + summary_catalog: self.summary_catalog.clone(), + collector_plans: self.collector_plans.clone(), + precompute_plan: self.precompute_plan.clone(), + transmission_plan: self.transmission_plan.clone(), + query_plan: self.query_plan.clone(), + storage_routing, + adaptation_evidence, + }) + } +} diff --git a/crates/asap_types/src/producer_plan.rs b/crates/asap_types/src/producer_plan.rs new file mode 100644 index 000000000..81d4b36e6 --- /dev/null +++ b/crates/asap_types/src/producer_plan.rs @@ -0,0 +1,879 @@ +//! Installed collector and transmission contracts shared across producers and consumers. +//! Deployment choices and accuracy-budget allocation remain in the control plane. +use crate::precompute_plan::{PlanEnvelope, PrecomputePlan, StateEncoding, StateFamilyContract}; +use planner_types::post_asap::{SketchAlgorithm, SummaryWindowFramework}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::BTreeSet; +use thiserror::Error; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct CollectorMaterialization { + pub query_id: String, + pub materialization: crate::sds::SummaryDefinitionId, + pub metric: String, + pub algorithm: String, + pub parameters: Value, + pub group_by: Vec, + pub window_secs: u64, + pub abstract_window_framework: SummaryWindowFramework, + pub window_implementation_id: String, + pub slide_secs: u64, + #[serde( + default, + alias = "paneOriginMs", + skip_serializing_if = "Option::is_none" + )] + pub pane_origin_ms: Option, + pub window_layout: crate::WindowMaterializationLayout, + pub evidence_source: Option, + pub lifecycle: CollectorLifecycle, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct CollectorLifecycle { + pub kind: String, + pub maintenance_mode: String, + pub evaluation_schedule: String, + pub output_representation: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct CollectorPlan { + /// Absent only in legacy artifacts; catalog-aware validation requires it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub summary_catalog: Option, + pub collector_id: String, + pub envelope: PlanEnvelope, + pub materializations: Vec, + pub transmission_rules: Vec, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum TransmissionMode { + Full, + Delta, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum SequenceScope { + MaterializationSeriesProducerEpoch, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct FrameIdentityContract { + pub identity_version: u32, + pub sequence_scope: SequenceScope, + pub require_checkpoint_for_full: bool, + pub require_base_checkpoint_for_delta: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TransmissionRule { + pub materialization: crate::sds::SummaryDefinitionId, + pub producer_id: String, + pub schema_id: String, + pub mode: TransmissionMode, + pub encoding: StateEncoding, + pub emit_every_ms: u64, + pub full_checkpoint_every_ms: Option, + pub destination_ref: String, + /// Plan-owned runtime knobs. These values are part of the immutable plan + /// generation; live feedback may only change them by publishing a + /// successor generation accepted by [`TransmissionPlan::authorize_successor`]. + #[serde(default)] + pub runtime_policy: RuntimeRulePolicy, +} + +/// How the collector admits updates before sketch maintenance. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "mode", rename_all = "snake_case", deny_unknown_fields)] +pub enum SamplingPolicy { + Disabled, + Fixed { + /// Probability in `(0, 1]`; `1` is valid but should normally be + /// represented by `Disabled`. + probability: f64, + estimator: SamplingEstimator, + }, +} + +impl Default for SamplingPolicy { + fn default() -> Self { + Self::Disabled + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum SamplingEstimator { + /// Hash-threshold element sampling, used by cardinality summaries. + HashThreshold, + /// Geometric admission/Nitro-style update sampling, used by frequency + /// summaries. The sketch readout carries the corresponding correction. + GeometricAdmission, +} + +/// Norm-adaptive Group-of-Sketches delta gating. GOS is meaningful only for +/// CountSketch families and only when the transmission rule is in delta mode. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct GosPolicy { + pub epsilon_staleness: f64, + pub sites: u32, + pub threshold_mode: GosThresholdMode, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum GosThresholdMode { + Isotropic, + Anisotropic, +} + +/// Sparse-delta semantics within a delta transmission rule. An absolute +/// threshold of zero sends every changed cell. When `gos` is present it +/// replaces the fixed threshold with the GOS norm-adaptive threshold. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct DeltaPolicy { + pub absolute_threshold: f64, + pub gos: Option, +} + +/// Inclusive bounds for one floating-point adaptation knob. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct AdaptiveF64Bounds { + pub min: f64, + pub max: f64, + pub max_step: f64, +} + +/// Inclusive bounds for one integer adaptation knob. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdaptiveU64Bounds { + pub min: u64, + pub max: u64, + pub max_step: u64, +} + +/// Guardrails for telemetry-driven runtime adaptation. This is an +/// authorization contract, not an instruction to mutate the active plan. +/// Every accepted change becomes a staged successor PhysicalPlan. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct RuntimeAdaptationPolicy { + pub enabled: bool, + pub not_before_unix_ms: u64, + pub max_evidence_age_ms: u64, + pub min_evidence_samples: u64, + pub sample_probability: Option, + pub emit_every_ms: Option, + pub delta_threshold: Option, + pub gos_epsilon_staleness: Option, +} + +impl Default for RuntimeAdaptationPolicy { + fn default() -> Self { + Self { + enabled: false, + not_before_unix_ms: 0, + max_evidence_age_ms: 0, + min_evidence_samples: 0, + sample_probability: None, + emit_every_ms: None, + delta_threshold: None, + gos_epsilon_staleness: None, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] +#[serde(deny_unknown_fields)] +pub struct RuntimeRulePolicy { + #[serde(default)] + pub sampling: SamplingPolicy, + pub delta: Option, + #[serde(default)] + pub adaptation: RuntimeAdaptationPolicy, +} + +/// Identity and sufficiency information for evidence authorizing one rule's +/// successor knobs. Raw measurements remain in the runtime-samples store; the +/// authorization boundary needs only their exact provenance and sample count. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct RuntimeAdaptationEvidence { + pub plan_id: u64, + pub plan_version: u64, + pub materialization: crate::sds::SummaryDefinitionId, + pub producer_id: String, + pub schema_id: String, + pub producer_version: String, + pub observed_at_unix_ms: u64, + pub sample_count: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TransmissionPlan { + /// Absent only in legacy artifacts; catalog-aware validation requires it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub summary_catalog: Option, + pub envelope: PlanEnvelope, + pub frame_identity: FrameIdentityContract, + pub rules: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum SummaryFrameKind { + Full, + Delta, +} + +/// Identity attached to every summary record. Window bounds come from the +/// data point; the remaining fields are carried as reserved `asap.frame.*` +/// attributes until the modified-OTLP schema gains a dedicated message. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct SummaryFrameIdentity { + pub identity_version: u32, + pub plan_id: u64, + pub plan_version: u64, + pub backend_compat: String, + pub materialization: crate::sds::SummaryDefinitionId, + /// Canonical producer-side identity for one concrete retained-label group. + pub series_identity: String, + pub schema_id: String, + pub producer_id: String, + pub producer_epoch: String, + pub window_start_unix_nano: u64, + pub window_end_unix_nano: u64, + pub sequence: u64, + pub kind: SummaryFrameKind, + pub encoding: StateEncoding, + pub checkpoint_id: Option, + pub base_checkpoint_id: Option, +} + +#[derive(Debug, Error, PartialEq, Eq)] +pub enum TransmissionPlanError { + #[error("summary catalog mismatch: {0}")] + Catalog(String), + #[error("TransmissionPlan envelope differs from PrecomputePlan")] + EnvelopeMismatch, + #[error("transmission rules do not exactly match precompute producer bindings")] + ProducerSetMismatch, + #[error("invalid transmission rule for producer {0}")] + InvalidRule(String), + #[error("frame identity is invalid: {0}")] + InvalidFrame(String), + #[error("frame has no matching transmission rule")] + UnknownFrame, + #[error("runtime policy for producer {producer_id} is invalid: {reason}")] + InvalidRuntimePolicy { producer_id: String, reason: String }, + #[error("runtime adaptation successor is invalid: {0}")] + InvalidSuccessor(String), + #[error("runtime adaptation evidence for producer {0} is missing or invalid")] + InvalidAdaptationEvidence(String), + #[error("runtime adaptation for producer {producer_id} exceeds guardrails: {knob}")] + AdaptationOutOfBounds { + producer_id: String, + knob: &'static str, + }, +} + +fn validate_catalog_projection( + reference: Option<&crate::sds::CatalogGeneration>, + envelope: &PlanEnvelope, + materializations: impl IntoIterator, + catalog: &crate::summary_catalog::SummaryCatalog, +) -> Result<(), TransmissionPlanError> { + let expected = catalog + .reference() + .map_err(|error| TransmissionPlanError::Catalog(error.to_string()))?; + if reference != Some(&expected) + || envelope.plan_id != catalog.plan_id + || envelope.plan_version != catalog.plan_version + { + return Err(TransmissionPlanError::Catalog( + "missing or different snapshot reference".into(), + )); + } + for id in materializations { + if !catalog.materializations.contains_key(&id) { + return Err(TransmissionPlanError::Catalog(format!( + "unknown materialization {}", + id.as_u64() + ))); + } + } + Ok(()) +} + +impl CollectorPlan { + pub fn validate_against_catalog( + &self, + catalog: &crate::summary_catalog::SummaryCatalog, + ) -> Result<(), TransmissionPlanError> { + validate_catalog_projection( + self.summary_catalog.as_ref(), + &self.envelope, + self.materializations + .iter() + .map(|m| m.materialization) + .chain(self.transmission_rules.iter().map(|r| r.materialization)), + catalog, + ) + } +} + +impl TransmissionPlan { + pub fn validate_against_catalog( + &self, + catalog: &crate::summary_catalog::SummaryCatalog, + ) -> Result<(), TransmissionPlanError> { + validate_catalog_projection( + self.summary_catalog.as_ref(), + &self.envelope, + self.rules.iter().map(|r| r.materialization), + catalog, + ) + } + + pub fn validate(&self, precompute: &PrecomputePlan) -> Result<(), TransmissionPlanError> { + if self.summary_catalog != precompute.summary_catalog { + return Err(TransmissionPlanError::Catalog( + "transmission and precompute plans reference different catalog snapshots".into(), + )); + } + if self.envelope != precompute.envelope { + return Err(TransmissionPlanError::EnvelopeMismatch); + } + let expected: BTreeSet<_> = precompute + .producers + .iter() + .map(|producer| { + ( + producer.materialization, + producer.producer_id.as_str(), + producer.schema_id.as_str(), + ) + }) + .collect(); + let actual: BTreeSet<_> = self + .rules + .iter() + .map(|rule| { + ( + rule.materialization, + rule.producer_id.as_str(), + rule.schema_id.as_str(), + ) + }) + .collect(); + if expected != actual || actual.len() != self.rules.len() { + return Err(TransmissionPlanError::ProducerSetMismatch); + } + for rule in &self.rules { + let valid_checkpoint_cadence = match (rule.mode, rule.full_checkpoint_every_ms) { + (TransmissionMode::Full, None) => true, + (TransmissionMode::Delta, Some(full_every)) => { + rule.emit_every_ms > 0 + && full_every >= rule.emit_every_ms + && full_every % rule.emit_every_ms == 0 + } + _ => false, + }; + if rule.emit_every_ms == 0 + || rule.destination_ref.is_empty() + || !valid_checkpoint_cadence + { + return Err(TransmissionPlanError::InvalidRule(rule.producer_id.clone())); + } + let schema = precompute + .schemas + .iter() + .find(|schema| schema.materialization == rule.materialization) + .expect("producer set validation guarantees a matching schema"); + if !schema.encodings.contains(&rule.encoding) { + return Err(TransmissionPlanError::InvalidRule(rule.producer_id.clone())); + } + validate_runtime_rule_policy(rule, &schema.family)?; + } + Ok(()) + } + + /// Authorize a telemetry-driven successor without mutating this active + /// plan. Semantic identity, codecs, destination, transmission mode and + /// checkpoint cadence remain fixed. Only explicitly bounded runtime knobs + /// may move, and each changed rule needs fresh evidence attributed to the + /// exact active generation. + pub fn authorize_successor( + &self, + successor: &TransmissionPlan, + evidence: &[RuntimeAdaptationEvidence], + now_unix_ms: u64, + ) -> Result<(), TransmissionPlanError> { + if successor.envelope.plan_id != self.envelope.plan_id + || successor.envelope.plan_version != self.envelope.plan_version.saturating_add(1) + || successor.envelope.backend_compat != self.envelope.backend_compat + || successor.envelope.planner_revision != self.envelope.planner_revision + || successor.envelope.capability_snapshot_id != self.envelope.capability_snapshot_id + || successor.envelope.generated_at_unix_ms < self.envelope.generated_at_unix_ms + || successor.envelope.activation_unix_ms < successor.envelope.generated_at_unix_ms + || successor.rules.len() != self.rules.len() + || successor.frame_identity != self.frame_identity + { + return Err(TransmissionPlanError::InvalidSuccessor( + "successor must be the next version of the same semantic/capability generation" + .into(), + )); + } + + for current in &self.rules { + let Some(next) = successor.rules.iter().find(|candidate| { + candidate.materialization == current.materialization + && candidate.producer_id == current.producer_id + && candidate.schema_id == current.schema_id + }) else { + return Err(TransmissionPlanError::InvalidSuccessor(format!( + "missing rule for producer {}", + current.producer_id + ))); + }; + if current.mode != next.mode + || current.encoding != next.encoding + || current.full_checkpoint_every_ms != next.full_checkpoint_every_ms + || current.destination_ref != next.destination_ref + || current.runtime_policy.adaptation != next.runtime_policy.adaptation + || sampling_estimator(¤t.runtime_policy.sampling) + != sampling_estimator(&next.runtime_policy.sampling) + || delta_shape(¤t.runtime_policy.delta) + != delta_shape(&next.runtime_policy.delta) + { + return Err(TransmissionPlanError::InvalidSuccessor(format!( + "rule identity/codec/mode/guardrails drifted for producer {}", + current.producer_id + ))); + } + if current.emit_every_ms == next.emit_every_ms + && current.runtime_policy.sampling == next.runtime_policy.sampling + && current.runtime_policy.delta == next.runtime_policy.delta + { + continue; + } + let policy = ¤t.runtime_policy.adaptation; + if !policy.enabled || now_unix_ms < policy.not_before_unix_ms { + return Err(TransmissionPlanError::AdaptationOutOfBounds { + producer_id: current.producer_id.clone(), + knob: "adaptation_disabled_or_in_cooldown", + }); + } + let has_evidence = evidence.iter().any(|item| { + item.plan_id == self.envelope.plan_id + && item.plan_version == self.envelope.plan_version + && item.materialization == current.materialization + && item.producer_id == current.producer_id + && item.schema_id == current.schema_id + && !item.producer_version.trim().is_empty() + && item.sample_count >= policy.min_evidence_samples + && item.observed_at_unix_ms <= now_unix_ms + && now_unix_ms.saturating_sub(item.observed_at_unix_ms) + <= policy.max_evidence_age_ms + }); + if !has_evidence { + return Err(TransmissionPlanError::InvalidAdaptationEvidence( + current.producer_id.clone(), + )); + } + authorize_f64_change( + sampling_probability(¤t.runtime_policy.sampling), + sampling_probability(&next.runtime_policy.sampling), + policy.sample_probability.as_ref(), + ¤t.producer_id, + "sample_probability", + )?; + authorize_u64_change( + current.emit_every_ms, + next.emit_every_ms, + policy.emit_every_ms.as_ref(), + ¤t.producer_id, + "emit_every_ms", + )?; + authorize_f64_change( + delta_threshold(¤t.runtime_policy.delta), + delta_threshold(&next.runtime_policy.delta), + policy.delta_threshold.as_ref(), + ¤t.producer_id, + "delta_threshold", + )?; + authorize_f64_change( + gos_epsilon(¤t.runtime_policy.delta), + gos_epsilon(&next.runtime_policy.delta), + policy.gos_epsilon_staleness.as_ref(), + ¤t.producer_id, + "gos_epsilon_staleness", + )?; + } + Ok(()) + } + + pub fn validate_frame( + &self, + frame: &SummaryFrameIdentity, + ) -> Result<(), TransmissionPlanError> { + if frame.identity_version != self.frame_identity.identity_version + || frame.plan_id != self.envelope.plan_id + || frame.plan_version != self.envelope.plan_version + || frame.backend_compat != self.envelope.backend_compat + || frame.series_identity.is_empty() + || frame.producer_epoch.is_empty() + || frame.sequence == 0 + || frame.window_start_unix_nano >= frame.window_end_unix_nano + || (frame.kind == SummaryFrameKind::Full + && self.frame_identity.require_checkpoint_for_full + && frame.checkpoint_id.is_none()) + || (frame.kind == SummaryFrameKind::Delta + && self.frame_identity.require_base_checkpoint_for_delta + && frame.base_checkpoint_id.is_none()) + { + return Err(TransmissionPlanError::InvalidFrame( + "identity/lifecycle/window/checkpoint fields do not satisfy the active contract" + .into(), + )); + } + if self.rules.iter().any(|rule| { + rule.materialization == frame.materialization + && rule.producer_id == frame.producer_id + && rule.schema_id == frame.schema_id + // A delta rule necessarily emits periodic full checkpoints; + // a full-only rule must never emit deltas. + && (frame.kind == SummaryFrameKind::Full + || rule.mode == TransmissionMode::Delta) + && rule.encoding == frame.encoding + }) { + Ok(()) + } else { + Err(TransmissionPlanError::UnknownFrame) + } + } +} + +fn validate_runtime_rule_policy( + rule: &TransmissionRule, + family: &StateFamilyContract, +) -> Result<(), TransmissionPlanError> { + let invalid = |reason: &str| TransmissionPlanError::InvalidRuntimePolicy { + producer_id: rule.producer_id.clone(), + reason: reason.into(), + }; + if let SamplingPolicy::Fixed { + probability, + estimator, + } = &rule.runtime_policy.sampling + { + if !probability.is_finite() || !(0.0..=1.0).contains(probability) || *probability == 0.0 { + return Err(invalid("sample probability must be finite and in (0, 1]")); + } + let supported = matches!( + (family, *estimator), + ( + StateFamilyContract::Sketch { + algorithm: SketchAlgorithm::Hll, + .. + }, + SamplingEstimator::HashThreshold, + ) | ( + StateFamilyContract::Sketch { + algorithm: SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap, + .. + }, + SamplingEstimator::GeometricAdmission, + ) + ); + if !supported { + return Err(invalid( + "sampling estimator is not implemented for the materialization family", + )); + } + } + + if (rule.mode == TransmissionMode::Delta) != rule.runtime_policy.delta.is_some() { + return Err(invalid( + "delta policy must be present exactly when transmission mode is delta", + )); + } + if let Some(delta) = &rule.runtime_policy.delta { + if !delta.absolute_threshold.is_finite() || delta.absolute_threshold < 0.0 { + return Err(invalid("delta threshold must be finite and non-negative")); + } + if !matches!( + family, + StateFamilyContract::Sketch { + algorithm: SketchAlgorithm::DDSketch + | SketchAlgorithm::Hll + | SketchAlgorithm::Cms + | SketchAlgorithm::CmsWithHeap + | SketchAlgorithm::CountSketch + | SketchAlgorithm::CountSketchWithHeap, + .. + } + ) { + return Err(invalid( + "delta transmission is not implemented for the materialization family", + )); + } + if let Some(gos) = &delta.gos { + if !matches!( + family, + StateFamilyContract::Sketch { + algorithm: SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap, + .. + } + ) || !gos.epsilon_staleness.is_finite() + || !(0.0..=1.0).contains(&gos.epsilon_staleness) + || gos.epsilon_staleness == 0.0 + || gos.sites == 0 + { + return Err(invalid( + "GOS requires a CountSketch family, epsilon in (0, 1], and at least one site", + )); + } + } + } + + let adaptation = &rule.runtime_policy.adaptation; + if adaptation.enabled + && (adaptation.max_evidence_age_ms == 0 || adaptation.min_evidence_samples == 0) + { + return Err(invalid( + "enabled adaptation requires non-zero evidence age and sample-count requirements", + )); + } + validate_f64_bounds(adaptation.sample_probability.as_ref(), 0.0, 1.0) + .map_err(|reason| invalid(reason))?; + validate_f64_bounds(adaptation.delta_threshold.as_ref(), 0.0, f64::MAX) + .map_err(|reason| invalid(reason))?; + validate_f64_bounds(adaptation.gos_epsilon_staleness.as_ref(), 0.0, 1.0) + .map_err(|reason| invalid(reason))?; + if let Some(bounds) = &adaptation.emit_every_ms { + if bounds.min == 0 + || bounds.min > bounds.max + || bounds.max_step == 0 + || !(bounds.min..=bounds.max).contains(&rule.emit_every_ms) + { + return Err(invalid("emit interval guardrails are invalid")); + } + } + if let Some(bounds) = &adaptation.sample_probability { + let current = sampling_probability(&rule.runtime_policy.sampling); + if current < bounds.min || current > bounds.max { + return Err(invalid( + "current sampling probability is outside guardrails", + )); + } + } + if let Some(bounds) = &adaptation.delta_threshold { + let Some(current) = rule + .runtime_policy + .delta + .as_ref() + .map(|policy| policy.absolute_threshold) + else { + return Err(invalid("delta guardrails require an active delta policy")); + }; + if current < bounds.min || current > bounds.max { + return Err(invalid("current delta threshold is outside guardrails")); + } + } + if let Some(bounds) = &adaptation.gos_epsilon_staleness { + let Some(current) = rule + .runtime_policy + .delta + .as_ref() + .and_then(|policy| policy.gos.as_ref()) + .map(|gos| gos.epsilon_staleness) + else { + return Err(invalid("GOS guardrails require an active GOS policy")); + }; + if current < bounds.min || current > bounds.max { + return Err(invalid("current GOS epsilon is outside guardrails")); + } + } + Ok(()) +} + +fn validate_f64_bounds( + bounds: Option<&AdaptiveF64Bounds>, + domain_min: f64, + domain_max: f64, +) -> Result<(), &'static str> { + let Some(bounds) = bounds else { + return Ok(()); + }; + if !bounds.min.is_finite() + || !bounds.max.is_finite() + || !bounds.max_step.is_finite() + || bounds.min < domain_min + || bounds.max > domain_max + || bounds.min > bounds.max + || bounds.max_step <= 0.0 + { + Err("floating-point adaptation guardrails are invalid") + } else { + Ok(()) + } +} + +fn sampling_probability(policy: &SamplingPolicy) -> f64 { + match policy { + SamplingPolicy::Disabled => 1.0, + SamplingPolicy::Fixed { probability, .. } => *probability, + } +} + +fn sampling_estimator(policy: &SamplingPolicy) -> Option { + match policy { + SamplingPolicy::Disabled => None, + SamplingPolicy::Fixed { estimator, .. } => Some(*estimator), + } +} + +fn delta_threshold(policy: &Option) -> f64 { + policy + .as_ref() + .map(|policy| policy.absolute_threshold) + .unwrap_or(0.0) +} + +fn gos_epsilon(policy: &Option) -> f64 { + policy + .as_ref() + .and_then(|policy| policy.gos.as_ref()) + .map(|gos| gos.epsilon_staleness) + .unwrap_or(0.0) +} + +fn delta_shape(policy: &Option) -> Option<(Option<(u32, GosThresholdMode)>,)> { + policy.as_ref().map(|policy| { + (policy + .gos + .as_ref() + .map(|gos| (gos.sites, gos.threshold_mode)),) + }) +} + +fn authorize_f64_change( + current: f64, + next: f64, + bounds: Option<&AdaptiveF64Bounds>, + producer_id: &str, + knob: &'static str, +) -> Result<(), TransmissionPlanError> { + if current == next { + return Ok(()); + } + let allowed = bounds.is_some_and(|bounds| { + next.is_finite() + && (bounds.min..=bounds.max).contains(&next) + && (next - current).abs() <= bounds.max_step + }); + if allowed { + Ok(()) + } else { + Err(TransmissionPlanError::AdaptationOutOfBounds { + producer_id: producer_id.into(), + knob, + }) + } +} + +fn authorize_u64_change( + current: u64, + next: u64, + bounds: Option<&AdaptiveU64Bounds>, + producer_id: &str, + knob: &'static str, +) -> Result<(), TransmissionPlanError> { + if current == next { + return Ok(()); + } + let allowed = bounds.is_some_and(|bounds| { + (bounds.min..=bounds.max).contains(&next) && current.abs_diff(next) <= bounds.max_step + }); + if allowed { + Ok(()) + } else { + Err(TransmissionPlanError::AdaptationOutOfBounds { + producer_id: producer_id.into(), + knob, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use planner_types::post_asap::SketchParams; + + #[test] + fn runtime_policy_is_family_and_mode_checked() { + let mut rule = TransmissionRule { + materialization: crate::PolicyFingerprint(1).into(), + producer_id: "test".into(), + schema_id: "test".into(), + mode: TransmissionMode::Full, + encoding: StateEncoding::SketchlibProtobufV1, + emit_every_ms: 60_000, + full_checkpoint_every_ms: None, + destination_ref: "backend".into(), + runtime_policy: Default::default(), + }; + rule.runtime_policy.sampling = SamplingPolicy::Fixed { + probability: 0.5, + estimator: SamplingEstimator::HashThreshold, + }; + let hll = StateFamilyContract::Sketch { + algorithm: SketchAlgorithm::Hll, + parameters: SketchParams::Hll { precision: 14 }, + }; + let count_sketch = StateFamilyContract::Sketch { + algorithm: SketchAlgorithm::CountSketch, + parameters: SketchParams::CountSketch { + width: 128, + depth: 4, + }, + }; + validate_runtime_rule_policy(&rule, &hll).expect("HLL supports hash-threshold sampling"); + assert!(matches!( + validate_runtime_rule_policy(&rule, &count_sketch), + Err(TransmissionPlanError::InvalidRuntimePolicy { .. }) + )); + + rule.runtime_policy.sampling = SamplingPolicy::Disabled; + rule.mode = TransmissionMode::Delta; + rule.full_checkpoint_every_ms = Some(300_000); + rule.runtime_policy.delta = Some(DeltaPolicy { + absolute_threshold: 0.0, + gos: Some(GosPolicy { + epsilon_staleness: 0.02, + sites: 2, + threshold_mode: GosThresholdMode::Isotropic, + }), + }); + validate_runtime_rule_policy(&rule, &count_sketch).expect("CountSketch supports delta GOS"); + assert!(matches!( + validate_runtime_rule_policy(&rule, &hll), + Err(TransmissionPlanError::InvalidRuntimePolicy { .. }) + )); + } +} diff --git a/data_plane/examples/audit_clickhouse_fallback.rs b/data_plane/examples/audit_clickhouse_fallback.rs index 2ada8ebaf..a5bdd1ac0 100644 --- a/data_plane/examples/audit_clickhouse_fallback.rs +++ b/data_plane/examples/audit_clickhouse_fallback.rs @@ -3,9 +3,7 @@ use axum::{ http::{HeaderMap, Method}, }; use control_plane::{ - physical::compiler::{ - PlanEnvelope, PrecomputePlan, TransmissionPlan, BACKEND_COMPAT, PLANNER_REVISION, - }, + physical::compiler::{PlanEnvelope, PrecomputePlan, BACKEND_COMPAT, PLANNER_REVISION}, query_plan::{ClickHousePlanningContext, QueryPlan}, }; use data_plane::{ @@ -76,8 +74,12 @@ async fn main() { let mut precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![]).unwrap(); precompute_plan.summary_catalog = Some(reference.clone()); - let mut transmission_plan = - TransmissionPlan::build(envelope, &precompute_plan, &BTreeMap::new()).unwrap(); + let mut transmission_plan = control_plane::physical::compiler::compile_transmission_plan( + envelope, + &precompute_plan, + &BTreeMap::new(), + ) + .unwrap(); transmission_plan.summary_catalog = Some(reference); let query_plan = QueryPlan { plan_id: 27, diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index 7e65edadc..2961bb7aa 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -1714,7 +1714,7 @@ async fn route_modified_otlp_sketches_to_precompute( crate::precompute_engine::frame_lineage::FrameLineageDecision::Apply, ) => { if frame.kind - == control_plane::physical::compiler::SummaryFrameKind::Full + == asap_types::producer_plan::SummaryFrameKind::Full { ingest_state .sketch_index @@ -2253,7 +2253,7 @@ fn preflight_summary_frames( mut dp: ModifiedOtlpSketchDp, ingest_state: &IngestState, active: &crate::storage_engines::types::ActivePhysicalPlan, - ) -> Result { + ) -> Result { let canonical_name = canonical_sketch_metric_name(metric_name, dp.algorithm.clone()); let frame = take_summary_frame_identity(&mut dp.attrs, dp.start_time_unix_nano, dp.time_unix_nano)?; @@ -2296,7 +2296,7 @@ fn preflight_summary_frames( // A malformed full snapshot must not be discovered after an earlier // frame in the request has already reached SketchStore. - if frame.kind == control_plane::physical::compiler::SummaryFrameKind::Full { + if frame.kind == asap_types::producer_plan::SummaryFrameKind::Full { decode_modified_otlp_sketch_bytes(dp.algorithm.clone(), dp.encoding, &dp.sketch) .map_err(|error| format!("invalid full frame for {metric_name}: {error}"))?; } else { @@ -2454,10 +2454,9 @@ fn take_summary_frame_identity( attrs: &mut HashMap, window_start_unix_nano: u64, window_end_unix_nano: u64, -) -> Result { - use control_plane::physical::compiler::{ - StateEncoding, SummaryFrameIdentity, SummaryFrameKind, - }; +) -> Result { + use asap_types::producer_plan::{SummaryFrameIdentity, SummaryFrameKind}; + use control_plane::physical::compiler::StateEncoding; fn required(attrs: &mut HashMap, key: &str) -> Result { attrs diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index be9905bb5..bbe1fbcad 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -895,9 +895,10 @@ mod tests { } fn physical_config(streaming: StreamingConfig) -> HotReloadStreamingConfig { + use asap_types::producer_plan::{FrameIdentityContract, SequenceScope, TransmissionPlan}; use control_plane::physical::compiler::{ - FrameIdentityContract, IngestContract, IngestProtocol, PlanEnvelope, PrecomputePlan, - SequenceScope, TimestampUnit, TransmissionPlan, PLANNER_REVISION, + IngestContract, IngestProtocol, PlanEnvelope, PrecomputePlan, TimestampUnit, + PLANNER_REVISION, }; let envelope = PlanEnvelope { plan_id: 7, diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index e219aca43..998a96ed3 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -2679,9 +2679,10 @@ mod tests { } async fn setup_remote_write_test_server() -> (u16, PrometheusRemoteWriteReceiver) { + use asap_types::producer_plan::{FrameIdentityContract, SequenceScope, TransmissionPlan}; use control_plane::physical::compiler::{ - FrameIdentityContract, IngestContract, IngestProtocol, PlanEnvelope, PrecomputePlan, - SequenceScope, TimestampUnit, TransmissionPlan, PLANNER_REVISION, + IngestContract, IngestProtocol, PlanEnvelope, PrecomputePlan, TimestampUnit, + PLANNER_REVISION, }; let streaming_config = Arc::new(StreamingConfig::default()); let envelope = PlanEnvelope { @@ -6045,7 +6046,7 @@ async fn handle_post_streaming_config( (StatusCode::OK, axum::Json(body)).into_response() } -pub use control_plane::physical::publication::PhysicalPlanInstallRequest; +pub use asap_types::plan_publication::PhysicalPlanInstallRequest; /// Decode and cross-validate every backend view before it can become visible. /// Used by both startup artifact loading and the staged HTTP install path. diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 23409c3b1..97f6386d8 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -768,13 +768,13 @@ async fn main() -> Result<()> { .collect(), executable_dags: Default::default(), }; - let initial_transmission_plan = control_plane::physical::compiler::TransmissionPlan { + let initial_transmission_plan = asap_types::producer_plan::TransmissionPlan { summary_catalog: None, envelope: initial_precompute_plan.envelope.clone(), - frame_identity: control_plane::physical::compiler::FrameIdentityContract { + frame_identity: asap_types::producer_plan::FrameIdentityContract { identity_version: 1, sequence_scope: - control_plane::physical::compiler::SequenceScope::MaterializationSeriesProducerEpoch, + asap_types::producer_plan::SequenceScope::MaterializationSeriesProducerEpoch, require_checkpoint_for_full: true, require_base_checkpoint_for_delta: true, }, diff --git a/data_plane/src/precompute_engine/frame_lineage.rs b/data_plane/src/precompute_engine/frame_lineage.rs index 6598a1384..7f5e0a847 100644 --- a/data_plane/src/precompute_engine/frame_lineage.rs +++ b/data_plane/src/precompute_engine/frame_lineage.rs @@ -5,7 +5,7 @@ //! the wire contract: deltas are only safe to apply after an observed full //! checkpoint, in sequence, within the exact producer/window lineage. -use control_plane::physical::compiler::{SummaryFrameIdentity, SummaryFrameKind}; +use asap_types::producer_plan::{SummaryFrameIdentity, SummaryFrameKind}; use dashmap::mapref::entry::Entry; use thiserror::Error; diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs index 2d8b4a86d..907b7fea5 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs @@ -703,7 +703,7 @@ mod tests { ) .unwrap(); precompute.summary_catalog = Some(sds.reference().unwrap()); - let mut transmission = control_plane::physical::compiler::TransmissionPlan::build( + let mut transmission = control_plane::physical::compiler::compile_transmission_plan( envelope.clone(), &precompute, &BTreeMap::new(), diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index e6f355a4b..1066e9f70 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -537,8 +537,8 @@ struct IncompleteSummaryLineage { window_end_unix_ms: u64, } -impl From<&control_plane::physical::compiler::SummaryFrameIdentity> for IncompleteSummaryLineage { - fn from(frame: &control_plane::physical::compiler::SummaryFrameIdentity) -> Self { +impl From<&asap_types::producer_plan::SummaryFrameIdentity> for IncompleteSummaryLineage { + fn from(frame: &asap_types::producer_plan::SummaryFrameIdentity) -> Self { Self { plan_id: frame.plan_id, plan_version: frame.plan_version, @@ -2679,7 +2679,7 @@ impl SketchStore { pub fn mark_summary_lineage_incomplete( &self, sid: u64, - frame: &control_plane::physical::compiler::SummaryFrameIdentity, + frame: &asap_types::producer_plan::SummaryFrameIdentity, ) { self.incomplete_summary_lineages .entry(sid) @@ -2691,7 +2691,7 @@ impl SketchStore { pub fn clear_summary_lineage_incomplete( &self, sid: u64, - frame: &control_plane::physical::compiler::SummaryFrameIdentity, + frame: &asap_types::producer_plan::SummaryFrameIdentity, ) { let key = IncompleteSummaryLineage::from(frame); if let Some(mut lineages) = self.incomplete_summary_lineages.get_mut(&sid) { @@ -3613,9 +3613,8 @@ mod tests { #[test] fn incomplete_delta_window_fails_closed_until_matching_full_checkpoint() { - use control_plane::physical::compiler::{ - StateEncoding, SummaryFrameIdentity, SummaryFrameKind, - }; + use asap_types::producer_plan::{SummaryFrameIdentity, SummaryFrameKind}; + use control_plane::physical::compiler::StateEncoding; let idx = SketchStore::new(); idx.register(meta(12)); diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index 9cba552f9..88786343c 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -92,7 +92,7 @@ pub struct ActivePhysicalPlan { /// Present for authoritative installations; legacy bootstrap has no catalog. pub summary_catalog: Option>, pub precompute_plan: asap_types::precompute_plan::PrecomputePlan, - pub transmission_plan: control_plane::physical::compiler::TransmissionPlan, + pub transmission_plan: asap_types::producer_plan::TransmissionPlan, pub runtime_config: Arc, pub query_plan: Arc, pub storage_routing: Arc, @@ -672,11 +672,9 @@ mod tests { summary_catalog: None, envelope: envelope.clone(), ingest: asap_types::precompute_plan::IngestContract { - protocol: - asap_types::precompute_plan::IngestProtocol::ModifiedOtlpMetricsV1, + protocol: asap_types::precompute_plan::IngestProtocol::ModifiedOtlpMetricsV1, endpoint_path: "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/v1/metrics".into(), - timestamp_unit: - asap_types::precompute_plan::TimestampUnit::UnixNanoseconds, + timestamp_unit: asap_types::precompute_plan::TimestampUnit::UnixNanoseconds, require_plan_identity: true, require_summary_definition_identity: true, require_registered_producer: true, @@ -684,9 +682,9 @@ mod tests { schemas: Vec::new(), producers: Vec::new(), executable_dags: Default::default(), - materializations: Vec::new(), + materializations: Vec::new(), }, - transmission_plan: control_plane::physical::compiler::TransmissionPlan { + transmission_plan: asap_types::producer_plan::TransmissionPlan { summary_catalog: None, envelope: asap_types::precompute_plan::PlanEnvelope { plan_id, @@ -698,9 +696,10 @@ mod tests { planner_revision: control_plane::physical::compiler::PLANNER_REVISION.into(), capability_snapshot_id: "test".into(), }, - frame_identity: control_plane::physical::compiler::FrameIdentityContract { + frame_identity: asap_types::producer_plan::FrameIdentityContract { identity_version: 1, - sequence_scope: control_plane::physical::compiler::SequenceScope::MaterializationSeriesProducerEpoch, + sequence_scope: + asap_types::producer_plan::SequenceScope::MaterializationSeriesProducerEpoch, require_checkpoint_for_full: true, require_base_checkpoint_for_delta: true, }, diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index 0bf5ff300..cc1b733fa 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -74,9 +74,9 @@ async fn post_full_config(client: &reqwest::Client, stack: &FullStack, json: &Js .any(|c| c.metric == "http_requests_total_latency_ms") { for rule in &mut artifact.transmission_plan.rules { - rule.mode = control_plane::physical::compiler::TransmissionMode::Delta; + rule.mode = asap_types::producer_plan::TransmissionMode::Delta; rule.full_checkpoint_every_ms = Some(rule.emit_every_ms); - rule.runtime_policy.delta = Some(control_plane::physical::compiler::DeltaPolicy { + rule.runtime_policy.delta = Some(asap_types::producer_plan::DeltaPolicy { absolute_threshold: 0.0, gos: None, }); diff --git a/data_plane/tests/support/physical_fixture.rs b/data_plane/tests/support/physical_fixture.rs index 00ae0e862..984fc0d16 100644 --- a/data_plane/tests/support/physical_fixture.rs +++ b/data_plane/tests/support/physical_fixture.rs @@ -37,8 +37,12 @@ pub fn artifact(config: &StreamingConfig) -> PhysicalPlanInstallRequest { let mut precompute = PrecomputePlan::build(envelope.clone(), configs, &["fixture".into()]).unwrap(); precompute.summary_catalog = Some(catalog.reference().unwrap()); - let mut transmission = - TransmissionPlan::build(envelope, &precompute, &BTreeMap::new()).unwrap(); + let mut transmission = control_plane::physical::compiler::compile_transmission_plan( + envelope, + &precompute, + &BTreeMap::new(), + ) + .unwrap(); transmission.summary_catalog = Some(catalog.reference().unwrap()); let mut query_plan = QueryPlan { plan_id: 1, diff --git a/docs/design_docs/summary-catalog-sds-architecture.md b/docs/design_docs/summary-catalog-sds-architecture.md index b712c0d94..09f2fab20 100644 --- a/docs/design_docs/summary-catalog-sds-architecture.md +++ b/docs/design_docs/summary-catalog-sds-architecture.md @@ -169,6 +169,13 @@ those types for existing callers and owns the `compile_bound*` and move into the shared contract. Data-plane engines import the shared types directly. No wrapper plan or second wire definition is introduced. +`asap_types::producer_plan` owns the installed collector and transmission +contracts, frame identities, runtime policy bounds and their validation. The +control plane allocates sampling/GOS budgets and constructs transmission rules +through `sampling_policy_from_accuracy_budget`, `gos_policy_from_accuracy_budget` +and `compile_transmission_plan`. Producers and the data plane import the shared +contracts directly; compilation is not a runtime dependency of those contracts. + The implemented ownership split is: 1. Move the SDS catalog contract into `asap_types`. From 624e4506971782e89e817709844cb788f937a785 Mon Sep 17 00:00:00 2001 From: Zeying Zhu <50204836+zzylol@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:23:13 -0400 Subject: [PATCH 24/28] feat(clickhouse): execute typed array element access (#653) * feat(clickhouse): execute typed array element access * style(clickhouse): format collection default helper * fix(clickhouse): distinguish nonfinite exact values from nulls * test(clickhouse): verify exact denormal transport policy --- Cargo.lock | 10 +- control_plane/Cargo.toml | 8 +- .../src/query_plan/clickhouse_exact.rs | 24 +- crates/asap_types/Cargo.toml | 2 +- data_plane/Cargo.toml | 6 +- .../accelerator.rs | 7 + .../relational_adapter.rs | 231 +++++++++++++++++- 7 files changed, 265 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cd77ff936..c0dfaf2b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -373,7 +373,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=0402384e589df6e087d6d2d22b463ddc2eea0774#0402384e589df6e087d6d2d22b463ddc2eea0774" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=5e9396033a5a5d5f350bfa675d12770c972ee991#5e9396033a5a5d5f350bfa675d12770c972ee991" dependencies = [ "asap-types", "serde", @@ -384,7 +384,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=0402384e589df6e087d6d2d22b463ddc2eea0774#0402384e589df6e087d6d2d22b463ddc2eea0774" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=5e9396033a5a5d5f350bfa675d12770c972ee991#5e9396033a5a5d5f350bfa675d12770c972ee991" dependencies = [ "asap-types", "promql-parser", @@ -393,7 +393,7 @@ dependencies = [ [[package]] name = "asap-frontend-sql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=0402384e589df6e087d6d2d22b463ddc2eea0774#0402384e589df6e087d6d2d22b463ddc2eea0774" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=5e9396033a5a5d5f350bfa675d12770c972ee991#5e9396033a5a5d5f350bfa675d12770c972ee991" dependencies = [ "asap-sql-function-catalog", "asap-types", @@ -415,12 +415,12 @@ dependencies = [ [[package]] name = "asap-sql-function-catalog" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=0402384e589df6e087d6d2d22b463ddc2eea0774#0402384e589df6e087d6d2d22b463ddc2eea0774" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=5e9396033a5a5d5f350bfa675d12770c972ee991#5e9396033a5a5d5f350bfa675d12770c972ee991" [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=0402384e589df6e087d6d2d22b463ddc2eea0774#0402384e589df6e087d6d2d22b463ddc2eea0774" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=5e9396033a5a5d5f350bfa675d12770c972ee991#5e9396033a5a5d5f350bfa675d12770c972ee991" dependencies = [ "serde", "serde_json", diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index 7cb29606b..264534341 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -76,8 +76,8 @@ asap_types.workspace = true # scaffolding, unaware that `data_plane`'s `summary_executor.rs` in *this* # repo is a real one. Vendored locally instead of chased upstream -- see # `data_plane/src/query_engines/asap_query_engine/summary_exec.rs`. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "0402384e589df6e087d6d2d22b463ddc2eea0774" } -asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "0402384e589df6e087d6d2d22b463ddc2eea0774" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "5e9396033a5a5d5f350bfa675d12770c972ee991" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "5e9396033a5a5d5f350bfa675d12770c972ee991" } # L1 adoption (design-target-architecture.md Part B): the PromQL front # end itself, replacing control_plane's own query_parser/promql.rs. @@ -85,8 +85,8 @@ asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = # `planner-types`/`asap-aware-mapping` above -- these three MUST move # together (two revs of the same upstream repo's types in one workspace # resolve to distinct Rust types that won't unify). -asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "0402384e589df6e087d6d2d22b463ddc2eea0774" } -asap-frontend-sql = { git = "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/ProjectASAP/ASAPPlanner", rev = "0402384e589df6e087d6d2d22b463ddc2eea0774" } +asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "5e9396033a5a5d5f350bfa675d12770c972ee991" } +asap-frontend-sql = { git = "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/ProjectASAP/ASAPPlanner", rev = "5e9396033a5a5d5f350bfa675d12770c972ee991" } [dev-dependencies] tokio = { version = "1", features = ["full", "test-util"] } diff --git a/control_plane/src/query_plan/clickhouse_exact.rs b/control_plane/src/query_plan/clickhouse_exact.rs index 85f476093..ea7c7c551 100644 --- a/control_plane/src/query_plan/clickhouse_exact.rs +++ b/control_plane/src/query_plan/clickhouse_exact.rs @@ -77,7 +77,7 @@ fn scalar(expr: &QueryExpr, schema: &Schema) -> Result { let function = match name.as_str() { "map" => "map", "mapconcat" => "mapConcat", - "asap_map_access" => "arrayElement", + "asap_map_access" | "asap_element_access" => "arrayElement", _ => return Err(format!("unsupported exact scalar function {name}")), }; expr.scalar_type(schema).map_err(|e| e.to_string())?; @@ -301,6 +301,28 @@ mod tests { assert!(!sql.contains("{from:")); assert!(!sql.contains("{to:")); } + #[test] + fn typed_list_access_renders_native_element_lookup() { + let schema = Schema::new(vec![Column::new( + "samples", + DataType::List { + element: Box::new(Column::new("item", DataType::Float64, false)), + }, + false, + )]); + let expr = QueryExpr::FunctionCall { + name: "asap_element_access".into(), + args: vec![ + QueryExpr::Column(0), + QueryExpr::Literal(ScalarValue::Int64(-1)), + ], + }; + assert_eq!( + scalar(&expr, &schema).unwrap(), + "arrayElement(`samples`, -1)" + ); + } + #[test] fn unsupported_scalar_is_not_forwarded_as_arbitrary_native_code() { let schema = Schema::new(vec![]); diff --git a/crates/asap_types/Cargo.toml b/crates/asap_types/Cargo.toml index 671d34ca3..d6480e602 100644 --- a/crates/asap_types/Cargo.toml +++ b/crates/asap_types/Cargo.toml @@ -34,4 +34,4 @@ sha2 = "0.10" # exactly (`control_plane/Cargo.toml`) -- two different revs of the same # git dependency in one workspace resolve to two distinct Rust types that # won't unify. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "0402384e589df6e087d6d2d22b463ddc2eea0774" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "5e9396033a5a5d5f350bfa675d12770c972ee991" } diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index 60ae2ef00..cc5310495 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -39,8 +39,8 @@ sha2 = "0.10" # reduction: Reduction, .. }`) are `pre_asap` types, in the same crate now # (not a separate `asap-ir` import). Query serving consumes the compiled # QueryPlan; these types are used at physical-plan compilation boundaries. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "0402384e589df6e087d6d2d22b463ddc2eea0774" } -asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "0402384e589df6e087d6d2d22b463ddc2eea0774" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "5e9396033a5a5d5f350bfa675d12770c972ee991" } +asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "5e9396033a5a5d5f350bfa675d12770c972ee991" } # Shared external (workspace) serde.workspace = true @@ -133,7 +133,7 @@ fs2 = "0.4" # none of them. [dev-dependencies] -asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "0402384e589df6e087d6d2d22b463ddc2eea0774" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "5e9396033a5a5d5f350bfa675d12770c972ee991" } tempfile = "3.20.0" criterion = { version = "0.5", features = ["html_reports"] } tokio-tungstenite = "0.21" diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs index 907b7fea5..8e34ceafd 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs @@ -111,6 +111,9 @@ impl CatalogClickHouseAccelerator { "0".into(), ); parameters.insert("output_format_json_quote_64bit_integers".into(), "0".into()); + // Preserve the distinction between NULL and unsupported NaN/Inf. + // The typed decoder rejects quoted non-finite values and falls back. + parameters.insert("output_format_json_quote_denormals".into(), "1".into()); if let Some(database) = request_context.database() { parameters.insert("database".into(), database.into()); } @@ -328,6 +331,10 @@ mod tests { request: &ClickHouseQueryRequest, ) -> Result { + assert_eq!( + request.parameters.get("output_format_json_quote_denormals"), + Some(&"1".into()) + ); assert_eq!(request.parameters.get("param_from"), Some(&"0".into())); assert_eq!(request.parameters.get("param_to"), Some(&"2000".into())); Ok(ClickHouseRawResponse { diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs index 15945be21..cdecb8791 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs @@ -41,6 +41,7 @@ enum Cell { Bool(bool), Timestamp(i64), Map(Vec<(Cell, Cell)>), + List(Arc<[Cell]>), } fn json_cell( @@ -60,9 +61,23 @@ fn json_cell( match dtype { DataType::Null if value.is_null() => Ok(Cell::Null), DataType::Null => Err(invalid()), - DataType::List { .. } | DataType::Struct { .. } => Err( - ClickHouseRelationalError::Unsupported("collection value transport".into()), - ), + DataType::List { element } => { + let item_type = clickhouse_type + .strip_prefix("Array(") + .and_then(|inner| inner.strip_suffix(')')) + .ok_or_else(invalid)?; + let items = value.as_array().ok_or_else(invalid)?; + Ok(Cell::List( + items + .iter() + .map(|item| json_cell(item, &element.dtype, element.nullable, item_type)) + .collect::, _>>()? + .into(), + )) + } + DataType::Struct { .. } => Err(ClickHouseRelationalError::Unsupported( + "struct value transport".into(), + )), DataType::Int64 => value.as_i64().map(Cell::Int64).ok_or_else(invalid), DataType::Float64 => value.as_f64().map(Cell::Float64).ok_or_else(invalid), DataType::Utf8 => value @@ -307,7 +322,16 @@ fn clickhouse_type_matches(actual: Option<&str>, expected: &DataType, nullable: } match expected { DataType::Null => actual == "Nothing", - DataType::List { .. } | DataType::Struct { .. } => false, + DataType::List { element } => { + !nullable + && actual + .strip_prefix("Array(") + .and_then(|inner| inner.strip_suffix(')')) + .is_some_and(|inner| { + clickhouse_type_matches(Some(inner), &element.dtype, element.nullable) + }) + } + DataType::Struct { .. } => false, DataType::Int64 => actual == "Int64", DataType::Float64 => actual == "Float64", DataType::Utf8 => actual == "String", @@ -428,11 +452,17 @@ impl ClickHouseRelationalAdapter { } for row in &input.rows { for key in keys { - if contains_nan(&eval(&key.expr, row, &schema)?) { + let value = eval(&key.expr, row, &schema)?; + if contains_nan(&value) { return Err(ClickHouseRelationalError::Unsupported( "NaN sort key".into(), )); } + if !matches!(value, Cell::Null) && cell_cmp(&value, &value).is_none() { + return Err(ClickHouseRelationalError::Unsupported( + "unsupported sort key value type".into(), + )); + } } } input @@ -548,7 +578,50 @@ fn eval( } QueryExpr::FunctionCall { name, args } => { use planner_types::pre_asap::scalar_signature::MapScalarFunction; - let function = MapScalarFunction::from_name(name).ok_or_else(|| { + if name.eq_ignore_ascii_case("asap_element_access") { + let (output_type, _) = expr + .scalar_type(schema) + .map_err(|error| ClickHouseRelationalError::Invalid(error.to_string()))?; + if let DataType::List { element } = args[0] + .scalar_type(schema) + .map_err(|error| ClickHouseRelationalError::Invalid(error.to_string()))? + .0 + { + let Cell::List(values) = eval(&args[0], row, schema)? else { + return Err(ClickHouseRelationalError::Invalid( + "array access input".into(), + )); + }; + let index = match eval(&args[1], row, schema)? { + Cell::Null => return Ok(Cell::Null), + Cell::Int64(index) => index, + _ => { + return Err(ClickHouseRelationalError::Invalid( + "array access index".into(), + )) + } + }; + let offset = if index > 0 { + usize::try_from(index - 1).ok() + } else if index < 0 { + usize::try_from(index.unsigned_abs()) + .ok() + .and_then(|distance| values.len().checked_sub(distance)) + } else { + None + }; + return match offset.and_then(|offset| values.get(offset)) { + Some(value) => Ok(value.clone()), + None => default_collection_element(&output_type, element.nullable), + }; + } + } + let function = (if name.eq_ignore_ascii_case("asap_element_access") { + Some(MapScalarFunction::Access) + } else { + MapScalarFunction::from_name(name) + }) + .ok_or_else(|| { ClickHouseRelationalError::Unsupported(format!("scalar function {name}")) })?; expr.scalar_type(schema) @@ -619,7 +692,7 @@ fn eval( else { unreachable!() }; - default_map_value(&value, value_nullable) + default_collection_element(&value, value_nullable) } } } @@ -629,7 +702,10 @@ fn eval( } } -fn default_map_value(dtype: &DataType, nullable: bool) -> Result { +fn default_collection_element( + dtype: &DataType, + nullable: bool, +) -> Result { if nullable { return Ok(Cell::Null); } @@ -640,9 +716,10 @@ fn default_map_value(dtype: &DataType, nullable: bool) -> Result Cell::Utf8(String::new()), DataType::Bool => Cell::Bool(false), DataType::Map { .. } => Cell::Map(Vec::new()), + DataType::List { .. } => Cell::List(Arc::from([])), _ => { return Err(ClickHouseRelationalError::Unsupported( - "map missing-key default type".into(), + "collection missing-element default type".into(), )) } }) @@ -768,6 +845,7 @@ fn compare_sort_keys( fn contains_nan(value: &Cell) -> bool { match value { Cell::Float64(value) => value.is_nan(), + Cell::List(values) => values.iter().any(contains_nan), Cell::Map(entries) => entries .iter() .any(|(key, value)| contains_nan(key) || contains_nan(value)), @@ -974,6 +1052,141 @@ mod tests { }; use std::rc::Rc; + #[test] + fn decodes_declared_array_elements_without_losing_nullability() { + use planner_types::pre_asap::Column; + let dtype = DataType::List { + element: Box::new(Column { + name: "item".into(), + dtype: DataType::Int64, + nullable: true, + table: None, + }), + }; + assert!(clickhouse_type_matches( + Some("Array(Nullable(Int64))"), + &dtype, + false + )); + assert!(!clickhouse_type_matches( + Some("Array(Int64)"), + &dtype, + false + )); + assert!(!clickhouse_type_matches( + Some("Nullable(Array(Nullable(Int64)))"), + &dtype, + true + )); + let value = json_cell( + &serde_json::json!([9007199254740993_i64, null, -7]), + &dtype, + false, + "Array(Nullable(Int64))", + ) + .unwrap(); + let Cell::List(items) = &value else { + panic!("expected list") + }; + assert_eq!( + items.as_ref(), + &[Cell::Int64(9007199254740993), Cell::Null, Cell::Int64(-7)] + ); + let Cell::List(copy) = value.clone() else { + unreachable!() + }; + assert!(Arc::ptr_eq(items, ©)); + assert!(json_cell( + &serde_json::json!(["wrong"]), + &dtype, + false, + "Array(Nullable(Int64))" + ) + .is_err()); + } + + #[test] + fn nonfinite_external_array_values_cannot_become_nulls() { + use planner_types::pre_asap::Column; + let dtype = DataType::List { + element: Box::new(Column::new("item", DataType::Float64, true)), + }; + for value in ["inf", "-inf", "nan"] { + assert!(json_cell( + &serde_json::json!(value), + &DataType::Float64, + true, + "Nullable(Float64)" + ) + .is_err()); + assert!(json_cell( + &serde_json::json!([value]), + &dtype, + false, + "Array(Nullable(Float64))" + ) + .is_err()); + } + assert_eq!( + json_cell( + &serde_json::json!([null]), + &dtype, + false, + "Array(Nullable(Float64))" + ) + .unwrap(), + Cell::List(vec![Cell::Null].into()) + ); + } + + #[test] + fn array_access_uses_signed_indices_and_element_defaults() { + use planner_types::pre_asap::{Column, Schema}; + let function = |name: &str, args| QueryExpr::FunctionCall { + name: name.into(), + args, + }; + let dtype = DataType::List { + element: Box::new(Column::new("item", DataType::Int64, false)), + }; + let schema = Schema::new(vec![ + Column::new("items", dtype, false), + Column::new("index", DataType::Int64, true), + ]); + let access = function( + "asap_element_access", + vec![QueryExpr::Column(0), QueryExpr::Column(1)], + ); + let items = Cell::List(vec![Cell::Int64(10), Cell::Int64(20)].into()); + for (index, expected) in [ + (1, 10), + (2, 20), + (-1, 20), + (-2, 10), + (0, 0), + (3, 0), + (i64::MIN, 0), + (i64::MAX, 0), + ] { + assert_eq!( + eval(&access, &[items.clone(), Cell::Int64(index)], &schema).unwrap(), + Cell::Int64(expected) + ); + } + assert_eq!( + eval(&access, &[items, Cell::Null], &schema).unwrap(), + Cell::Null + ); + let zero = function( + "asap_element_access", + vec![ + QueryExpr::Column(0), + QueryExpr::Literal(ScalarValue::Int64(0)), + ], + ); + assert!(eval(&zero, &[Cell::List(Arc::from([])), Cell::Int64(0)], &schema).is_err()); + } + fn schema(fields: &[(&str, DataType)]) -> SummarySchema { SummarySchema { fields: fields From fb98a40e53b9f720c93a53193d6bf4ccee196e6c Mon Sep 17 00:00:00 2001 From: Zeying Zhu <50204836+zzylol@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:32:53 -0400 Subject: [PATCH 25/28] Identify derived summary inputs in the shared catalog (#650) * feat(catalog): identify derived summary inputs * fix(catalog): separate derived policy domain from raw metric names * docs(sds): separate immutable window section --- crates/asap_types/src/aggregation_config.rs | 48 ++- crates/asap_types/src/derived_input.rs | 383 ++++++++++++++++++ crates/asap_types/src/lib.rs | 1 + crates/asap_types/src/policy_fingerprint.rs | 11 +- crates/asap_types/src/precompute_plan.rs | 11 + .../asap_types/src/precompute_plan/catalog.rs | 11 +- crates/asap_types/src/sds.rs | 16 +- crates/asap_types/src/summary_catalog.rs | 57 ++- .../drivers/ingest/prometheus_remote_write.rs | 2 + data_plane/src/drivers/query/servers/http.rs | 1 + .../src/precompute_engine/output_sink.rs | 1 + .../sketch_db/lifecycle/eviction.rs | 1 + .../storage_engines/types/streaming_config.rs | 16 + .../tests/test_utilities/engine_factories.rs | 8 + .../summary-catalog-sds-architecture.md | 22 + 15 files changed, 567 insertions(+), 22 deletions(-) create mode 100644 crates/asap_types/src/derived_input.rs diff --git a/crates/asap_types/src/aggregation_config.rs b/crates/asap_types/src/aggregation_config.rs index 3e9f5cab3..8254b65b1 100644 --- a/crates/asap_types/src/aggregation_config.rs +++ b/crates/asap_types/src/aggregation_config.rs @@ -103,6 +103,8 @@ pub struct PrecomputeMaterialization { pub grouping_labels: crate::GroupingProjection, #[serde(default, skip_serializing_if = "Option::is_none")] pub partitioning: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub derived_input: Option, pub aggregated_labels: KeyByLabelNames, pub rollup_labels: KeyByLabelNames, pub original_yaml: String, @@ -200,9 +202,39 @@ impl PrecomputeMaterialization { .saturating_mul(1_000) } + pub fn source_identity(&self) -> crate::sds::DataSourceIdentity { + use crate::sds::DataSourceIdentity; + if let Some(input) = &self.derived_input { + DataSourceIdentity::Derived { + input: input.clone(), + } + } else if let Some(table_ref) = &self.table_name { + DataSourceIdentity::Table { + table_ref: table_ref.clone(), + } + } else { + DataSourceIdentity::TimeSeries { + metric: self.metric.clone(), + } + } + } + pub fn population_filter_canonical(&self) -> Result { + if let Some(input) = &self.derived_input { + input.validate()?; + if self.table_name.is_some() + || self.table_population.is_some() + || self.table_timestamp_column.is_some() + || !self.spatial_filter.is_empty() + { + return Err("derived inputs cannot also declare a raw source/filter".into()); + } + } self.effective_value_projection().validate()?; - if self.value_projection.is_some() && self.table_name.is_none() { + if self.value_projection.is_some() + && self.table_name.is_none() + && self.derived_input.is_none() + { return Err("explicit table value projection requires a table source".into()); } if let Some(column) = &self.table_timestamp_column { @@ -255,6 +287,7 @@ impl PrecomputeMaterialization { parameters, grouping_labels: grouping_labels.into(), partitioning: None, + derived_input: None, aggregated_labels, rollup_labels, original_yaml, @@ -400,6 +433,11 @@ impl PrecomputeMaterialization { table_name, value_column, ); + config.derived_input = data + .get("derived_input") + .filter(|v| !v.is_null()) + .map(|v| serde_json::from_value(v.clone())) + .transpose()?; config.partitioning = data .get("partitioning") .filter(|value| !value.is_null()) @@ -595,6 +633,11 @@ impl PrecomputeMaterialization { table_name, value_column, ); + config.derived_input = aggregation_data + .get("derived_input") + .filter(|v| !v.is_null()) + .map(|v| serde_yaml::from_value(v.clone())) + .transpose()?; config.partitioning = aggregation_data .get("partitioning") .filter(|value| !value.is_null()) @@ -640,6 +683,9 @@ impl SerializableToSink for PrecomputeMaterialization { "metric": self.metric, }); + if let Some(input) = &self.derived_input { + json["derived_input"] = serde_json::json!(input); + } // Only include numAggregatesToRetain if it's Some if let Some(num_aggregates) = self.num_aggregates_to_retain { json["numAggregatesToRetain"] = serde_json::json!(num_aggregates); diff --git a/crates/asap_types/src/derived_input.rs b/crates/asap_types/src/derived_input.rs new file mode 100644 index 000000000..a5f6d6dce --- /dev/null +++ b/crates/asap_types/src/derived_input.rs @@ -0,0 +1,383 @@ +//! Content identity for a maintenance sub-DAG cut at existing summary inputs. +//! The executable program remains in OwnedPostAsapDag; this is its catalog key. +use std::collections::{BTreeMap, BTreeSet}; + +use planner_types::post_asap::PostAsapNodeId; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::{executable_plan::OwnedPostAsapDag, sds::SummaryDefinitionId}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DerivedInputIdentity { + pub inputs: BTreeSet, + pub program_sha256: String, +} + +impl DerivedInputIdentity { + pub fn validate(&self) -> Result<(), String> { + if self.inputs.is_empty() + || self.inputs.iter().any(|id| id.fingerprint().is_unset()) + || self.program_sha256.len() != 64 + || !self + .program_sha256 + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + { + return Err("derived input requires summary references and a canonical SHA-256".into()); + } + Ok(()) + } + + /// Hash semantic nodes and edge roles, replacing input frontiers with stable + /// catalog IDs. Query IDs, node numbering, and catalog generations are absent. + pub fn from_dag( + document: &OwnedPostAsapDag, + root: PostAsapNodeId, + frontiers: &BTreeMap, + ) -> Result { + if document.schema_version != crate::executable_plan::OWNED_POST_ASAP_DAG_SCHEMA_VERSION { + return Err("unsupported derived program document version".into()); + } + let decoded = document.decode()?; + let literals: BTreeSet<_> = decoded + .nodes + .iter() + .filter_map(|node| { + matches!( + &node.payload, + planner_types::post_asap::ExecutableOperatorPayload::Fallback { + expression: planner_types::pre_asap::QueryExpr::Literal(_), + } + ) + .then_some(node.id) + }) + .collect(); + let mut incoming: BTreeMap<_, Vec<_>> = BTreeMap::new(); + for edge in &document.edges { + incoming.entry(edge.consumer).or_default().push(edge); + } + let nodes: BTreeMap<_, _> = document.nodes.iter().map(|n| (n.id, n)).collect(); + if frontiers.contains_key(&root) { + return Err("derived program requires a non-frontier root".into()); + } + let mut hashes = BTreeMap::new(); + let mut visiting = BTreeSet::new(); + let mut inputs = BTreeSet::new(); + let mut stack = vec![(root, false)]; + while let Some((id, finish)) = stack.pop() { + if hashes.contains_key(&id) { + continue; + } + let node = nodes + .get(&id) + .ok_or("derived program references missing node")?; + if let Some(summary) = frontiers.get(&id) { + inputs.insert(*summary); + hashes.insert(id, serde_json::json!({"summary": summary})); + continue; + } + if !finish { + if !visiting.insert(id) { + return Err("derived program has a cycle".into()); + } + stack.push((id, true)); + if !incoming.contains_key(&id) && !literals.contains(&id) { + return Err("derived program has an unbound input leaf".into()); + } + for edge in incoming.get(&id).into_iter().flatten() { + stack.push((edge.producer, false)); + } + continue; + } + let mut edges = Vec::new(); + for edge in incoming.get(&id).into_iter().flatten() { + let input = hashes + .get(&edge.producer) + .ok_or("derived input was not evaluated")?; + edges.push( + serde_json::to_vec(&serde_json::json!({ + "input": input, "role": edge.role, "schema": edge.intermediate_schema, + "state": edge.data_state, "grouping": edge.grouping, "window": edge.window, + })) + .map_err(|e| e.to_string())?, + ); + } + edges.sort(); + let bytes = serde_json::to_vec(&serde_json::json!({ + "version": 1, "operator": node.operator, "payload": node.payload, + "state": node.output_state, "schema": node.output_schema, + "guarantee": node.guarantee, "inputs": edges, + })) + .map_err(|e| e.to_string())?; + hashes.insert( + id, + serde_json::json!(format!("{:x}", Sha256::digest(bytes))), + ); + visiting.remove(&id); + } + let identity = Self { + inputs, + program_sha256: hashes[&root] + .as_str() + .ok_or("derived root has no semantic hash")? + .into(), + }; + identity.validate()?; + Ok(identity) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::summary_catalog::SummaryCatalog; + use crate::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + + fn config() -> PrecomputeMaterialization { + PrecomputeMaterialization::new( + AggregationType::Sum, + String::new(), + Default::default(), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + 10, + 10, + WindowKind::Tumbling, + String::new(), + "m".into(), + None, + None, + None, + ) + } + + #[test] + fn derived_source_is_distinct_and_generation_independent() { + let raw = config(); + let raw_id = SummaryDefinitionId::from(raw.policy_fingerprint()); + let mut derived = raw.clone(); + derived.derived_input = Some(DerivedInputIdentity { + inputs: BTreeSet::from([raw_id]), + program_sha256: "a".repeat(64), + }); + assert_ne!(raw.policy_fingerprint(), derived.policy_fingerprint()); + let a = + SummaryCatalog::from_materializations(1, 1, &[raw.clone(), derived.clone()]).unwrap(); + let b = SummaryCatalog::from_materializations(2, 9, &[raw, derived.clone()]).unwrap(); + assert_eq!(a.materializations, b.materializations); + assert_eq!(a.data_descriptors, b.data_descriptors); + let mut renamed = derived.clone(); + renamed.metric = "output_alias".into(); + assert_eq!(renamed.policy_fingerprint(), derived.policy_fingerprint()); + let json = serde_json::to_value(&derived).unwrap(); + let decoded: PrecomputeMaterialization = serde_json::from_value(json).unwrap(); + assert_eq!(decoded.policy_fingerprint(), derived.policy_fingerprint()); + } + + #[test] + fn raw_utf8_metric_cannot_impersonate_derived_policy_domain() { + let mut derived = config(); + derived.derived_input = Some(DerivedInputIdentity { + inputs: BTreeSet::from([SummaryDefinitionId::from(derived.policy_fingerprint())]), + program_sha256: "d".repeat(64), + }); + let mut raw = derived.clone(); + raw.metric = format!( + "derived-input-v1:{}", + serde_json::to_string(&derived.source_identity()).unwrap() + ); + raw.derived_input = None; + assert_ne!(raw.policy_fingerprint(), derived.policy_fingerprint()); + } + + #[test] + fn catalog_rejects_missing_derived_dependencies_and_raw_source_conflicts() { + let mut derived = config(); + derived.derived_input = Some(DerivedInputIdentity { + inputs: BTreeSet::from([SummaryDefinitionId::from(derived.policy_fingerprint())]), + program_sha256: "b".repeat(64), + }); + assert!(SummaryCatalog::from_materializations(1, 1, &[derived.clone()]).is_err()); + derived.table_name = Some("table".into()); + assert!(derived.population_filter_canonical().is_err()); + } + fn program(source: u32, root: u32) -> OwnedPostAsapDag { + use crate::executable_plan::{OwnedPostAsapEdge, OwnedPostAsapNode}; + use planner_types::post_asap::{ + EdgeRole, ExecutableOperator, ExecutionDataState, GroupingEdgeCompatibility, + WindowEdgeCompatibility, + }; + let state = ExecutionDataState::MAINTENANCE_SUMMARY; + OwnedPostAsapDag { + schema_version: 1, + query_id: "query-a".into(), + root: PostAsapNodeId(root), + nodes: [source, root] + .into_iter() + .map(|id| OwnedPostAsapNode { + id: PostAsapNodeId(id), + operator: ExecutableOperator::SummaryMerge, + payload: serde_json::json!({"kind":"summary_merge"}), + output_state: state, + output_schema: serde_json::json!({"fields":[],"time_index":null}), + guarantee: None, + }) + .collect(), + edges: vec![OwnedPostAsapEdge { + producer: PostAsapNodeId(source), + consumer: PostAsapNodeId(root), + role: EdgeRole::Input, + intermediate_schema: serde_json::json!({"fields":[],"time_index":null}), + data_state: state, + grouping: GroupingEdgeCompatibility::Identical, + window: WindowEdgeCompatibility::NotApplicable, + }], + } + } + + #[test] + fn semantic_signature_ignores_node_and_query_numbering_but_not_inputs() { + let source = SummaryDefinitionId::from(config().policy_fingerprint()); + let a = program(1, 2); + let first = DerivedInputIdentity::from_dag( + &a, + a.root, + &BTreeMap::from([(PostAsapNodeId(1), source)]), + ) + .unwrap(); + let mut b = program(900, 42); + b.query_id = "another-query".into(); + b.nodes.reverse(); + let second = DerivedInputIdentity::from_dag( + &b, + b.root, + &BTreeMap::from([(PostAsapNodeId(900), source)]), + ) + .unwrap(); + assert_eq!(first, second); + b.nodes + .iter_mut() + .find(|n| n.id == b.root) + .unwrap() + .output_schema = serde_json::json!({"fields":[],"time_index":0}); + assert_ne!( + first, + DerivedInputIdentity::from_dag( + &b, + b.root, + &BTreeMap::from([(PostAsapNodeId(900), source)]) + ) + .unwrap() + ); + assert!(DerivedInputIdentity::from_dag(&a, a.root, &BTreeMap::new()).is_err()); + let mut unsupported = a.clone(); + unsupported.schema_version = 999; + assert!(DerivedInputIdentity::from_dag( + &unsupported, + unsupported.root, + &BTreeMap::from([(PostAsapNodeId(1), source)]) + ) + .is_err()); + unsupported = a.clone(); + unsupported.nodes[1].payload = serde_json::json!({"kind":"unknown_operator"}); + assert!(DerivedInputIdentity::from_dag( + &unsupported, + unsupported.root, + &BTreeMap::from([(PostAsapNodeId(1), source)]) + ) + .is_err()); + let mut cycle = a.clone(); + cycle.edges[0].producer = cycle.root; + assert!(DerivedInputIdentity::from_dag(&cycle, cycle.root, &BTreeMap::new()).is_err()); + cycle.edges[0].producer = PostAsapNodeId(999); + assert!(DerivedInputIdentity::from_dag(&cycle, cycle.root, &BTreeMap::new()).is_err()); + } + + #[test] + fn catalog_rejects_self_referential_summary() { + use crate::sds::{ + DataDescriptor, DataSourceIdentity, SummaryDescriptor, ValueProjectionIdentity, + }; + let config = config(); + let input = DerivedInputIdentity { + inputs: BTreeSet::from([SummaryDefinitionId::from(config.policy_fingerprint())]), + program_sha256: "f".repeat(64), + }; + let data = DataDescriptor::new_typed( + DataSourceIdentity::Derived { input }, + ValueProjectionIdentity::SampleValue, + "", + Vec::::new(), + "derived", + ); + assert!(SummaryCatalog::build( + 1, + 1, + [( + config.policy_fingerprint(), + SummaryDescriptor::from_config(&config).unwrap(), + data, + config.window_layout + )] + ) + .is_err()); + } + #[test] + fn installation_rejects_derived_inputs_until_immutable_consumer_is_enabled() { + use crate::precompute_plan::{PlanEnvelope, PrecomputePlan}; + let mut config = config(); + config.derived_input = Some(DerivedInputIdentity { + inputs: BTreeSet::from([SummaryDefinitionId::from(config.policy_fingerprint())]), + program_sha256: "c".repeat(64), + }); + let envelope = PlanEnvelope { + plan_id: 1, + plan_version: 1, + generated_at_unix_ms: 0, + activation_unix_ms: 0, + expiry_unix_ms: None, + backend_compat: "asap-query-backend.v1".into(), + planner_revision: "test".into(), + capability_snapshot_id: "test".into(), + }; + let error = + PrecomputePlan::build(envelope, vec![config], &["producer".into()]).unwrap_err(); + assert!(error.to_string().contains("immutable maintenance consumer")); + } + #[test] + fn literal_leaves_are_hashed_without_inventing_materialization_references() { + use planner_types::{ + post_asap::{ExecutableOperator, ExecutableOperatorPayload}, + pre_asap::{QueryExpr, ScalarValue}, + }; + let mut dag = program(1, 2); + let mut literal = dag.nodes[0].clone(); + literal.id = PostAsapNodeId(3); + literal.operator = ExecutableOperator::Fallback; + literal.payload = serde_json::to_value(ExecutableOperatorPayload::Fallback { + expression: QueryExpr::Literal(ScalarValue::Int64(2)), + }) + .unwrap(); + dag.nodes.push(literal); + let mut edge = dag.edges[0].clone(); + edge.producer = PostAsapNodeId(3); + dag.edges.push(edge); + let source = SummaryDefinitionId::from(config().policy_fingerprint()); + let frontiers = BTreeMap::from([(PostAsapNodeId(1), source)]); + let first = DerivedInputIdentity::from_dag(&dag, dag.root, &frontiers).unwrap(); + assert_eq!(first.inputs, BTreeSet::from([source])); + dag.nodes[2].payload = serde_json::to_value(ExecutableOperatorPayload::Fallback { + expression: QueryExpr::Literal(ScalarValue::Int64(3)), + }) + .unwrap(); + assert_ne!( + first, + DerivedInputIdentity::from_dag(&dag, dag.root, &frontiers).unwrap() + ); + assert!(DerivedInputIdentity::from_dag(&dag, PostAsapNodeId(3), &frontiers).is_err()); + } +} diff --git a/crates/asap_types/src/lib.rs b/crates/asap_types/src/lib.rs index 7f4673837..dfa826609 100644 --- a/crates/asap_types/src/lib.rs +++ b/crates/asap_types/src/lib.rs @@ -2,6 +2,7 @@ pub mod accumulator_spec; pub mod accuracy; pub mod aggregation_config; pub mod aggregation_type; +pub mod derived_input; pub mod enums; pub mod executable_plan; pub mod grouping_projection; diff --git a/crates/asap_types/src/policy_fingerprint.rs b/crates/asap_types/src/policy_fingerprint.rs index f9e751651..38afc45ff 100644 --- a/crates/asap_types/src/policy_fingerprint.rs +++ b/crates/asap_types/src/policy_fingerprint.rs @@ -90,7 +90,16 @@ impl PolicyFingerprint { let mut buf: Vec = Vec::with_capacity(512); // 1. metric name - buf.extend_from_slice(cfg.metric.as_bytes()); + if cfg.derived_input.is_some() { + // Raw policies start with UTF-8 metric bytes; 0xff is impossible + // there, so a metric cannot impersonate this source domain. + buf.extend_from_slice(b"\xffderived-input-v1:"); + buf.extend_from_slice( + &serde_json::to_vec(&cfg.source_identity()).expect("typed source identity"), + ); + } else { + buf.extend_from_slice(cfg.metric.as_bytes()); + } buf.push(0); // 2. aggregation_type (Serialize impl is the stable form) diff --git a/crates/asap_types/src/precompute_plan.rs b/crates/asap_types/src/precompute_plan.rs index 257d99dd4..da7b07b0c 100644 --- a/crates/asap_types/src/precompute_plan.rs +++ b/crates/asap_types/src/precompute_plan.rs @@ -334,6 +334,17 @@ impl PrecomputePlan { if !valid_ingest { return Err(PrecomputePlanError::UnsupportedIngestEndpoint); } + // Derived input identity is portable, but raw ingress cannot execute it. + // Installation stays closed until the immutable maintenance consumer is wired. + if self + .materializations + .iter() + .any(|config| config.derived_input.is_some()) + { + return Err(PrecomputePlanError::CatalogContract( + "derived summary input requires an immutable maintenance consumer".into(), + )); + } for (query_id, installed) in &self.executable_dags { if query_id != &installed.document.query_id { return Err(PrecomputePlanError::CatalogContract( diff --git a/crates/asap_types/src/precompute_plan/catalog.rs b/crates/asap_types/src/precompute_plan/catalog.rs index 9ee907798..e9dd2a599 100644 --- a/crates/asap_types/src/precompute_plan/catalog.rs +++ b/crates/asap_types/src/precompute_plan/catalog.rs @@ -1,6 +1,6 @@ //! Catalog consistency checks for the precompute execution plan. use super::*; -use crate::sds::{DataSourceIdentity, SummaryDefinitionId, SummaryDescriptor}; +use crate::sds::{SummaryDefinitionId, SummaryDescriptor}; use crate::summary_catalog::SummaryCatalog; use planner_types::pre_asap::Source; use std::collections::BTreeSet; @@ -70,14 +70,7 @@ impl PrecomputePlan { return Err(invalid("pane origin differs from catalog definition")); } let data = &catalog.data_descriptors[&binding.data_descriptor_id]; - let expected_source = config.table_name.as_ref().map_or_else( - || DataSourceIdentity::TimeSeries { - metric: config.metric.clone(), - }, - |table_ref| DataSourceIdentity::Table { - table_ref: table_ref.clone(), - }, - ); + let expected_source = config.source_identity(); if config.table_name.is_some() && !config.grouping_labels.is_empty() && data.observation_semantics diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index 23c3cc48b..b75fd88ca 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -729,8 +729,15 @@ impl FidelityGuarantee { #[serde(deny_unknown_fields)] #[serde(rename_all = "snake_case")] pub enum DataSourceIdentity { - TimeSeries { metric: String }, - Table { table_ref: String }, + TimeSeries { + metric: String, + }, + Table { + table_ref: String, + }, + Derived { + input: crate::derived_input::DerivedInputIdentity, + }, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -829,7 +836,7 @@ impl DataDescriptor { pub fn time_series_metric(&self) -> Option<&str> { match &self.source { DataSourceIdentity::TimeSeries { metric } => Some(metric), - DataSourceIdentity::Table { .. } => None, + DataSourceIdentity::Table { .. } | DataSourceIdentity::Derived { .. } => None, } } @@ -929,6 +936,9 @@ impl DataDescriptor { &self.id } pub fn validate(&self) -> Result<(), SdsError> { + if let DataSourceIdentity::Derived { input } = &self.source { + input.validate().map_err(SdsError)?; + } self.value_projection.validate().map_err(SdsError)?; self.group_by_keys.validate().map_err(SdsError)?; if matches!(self.source, DataSourceIdentity::TimeSeries { .. }) diff --git a/crates/asap_types/src/summary_catalog.rs b/crates/asap_types/src/summary_catalog.rs index 5f31ee43d..36468724d 100644 --- a/crates/asap_types/src/summary_catalog.rs +++ b/crates/asap_types/src/summary_catalog.rs @@ -103,14 +103,7 @@ impl SummaryCatalog { .map(|config| { let summary = SummaryDescriptor::from_config(config) .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; - let source = config.table_name.as_ref().map_or_else( - || DataSourceIdentity::TimeSeries { - metric: config.metric.clone(), - }, - |table_ref| DataSourceIdentity::Table { - table_ref: table_ref.clone(), - }, - ); + let source = config.source_identity(); let value_projection = config.effective_value_projection().clone(); let data = DataDescriptor::new_typed( source, @@ -253,6 +246,54 @@ impl SummaryCatalog { return Err(SummaryCatalogError::MissingDescriptor(id.as_u64())); } } + if !self + .data_descriptors + .values() + .any(|data| matches!(data.source, DataSourceIdentity::Derived { .. })) + { + return Ok(()); + } + // Dependencies must refer to this snapshot and form an acyclic graph. + let mut pending = std::collections::BTreeMap::new(); + let mut consumers: std::collections::BTreeMap<_, Vec<_>> = + std::collections::BTreeMap::new(); + for (id, binding) in &self.materializations { + let dependencies = match &self.data_descriptors[&binding.data_descriptor_id].source { + DataSourceIdentity::Derived { input } => input.inputs.clone(), + _ => Default::default(), + }; + for source in &dependencies { + if !self.materializations.contains_key(source) { + return Err(SummaryCatalogError::Descriptor( + "derived input references missing summary".into(), + )); + } + consumers.entry(*source).or_default().push(*id); + } + pending.insert(*id, dependencies.len()); + } + let mut ready: Vec<_> = pending + .iter() + .filter_map(|(id, count)| (*count == 0).then_some(*id)) + .collect(); + let mut visited = 0; + while let Some(id) = ready.pop() { + visited += 1; + for consumer in consumers.get(&id).into_iter().flatten() { + let count = pending + .get_mut(consumer) + .expect("catalog dependency target"); + *count -= 1; + if *count == 0 { + ready.push(*consumer); + } + } + } + if visited != pending.len() { + return Err(SummaryCatalogError::Descriptor( + "derived summary dependencies have a cycle".into(), + )); + } Ok(()) } } diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index bbe1fbcad..d7306b40f 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -1002,6 +1002,7 @@ mod tests { table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; @@ -1073,6 +1074,7 @@ mod tests { table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 998a96ed3..aa2b259b4 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -3682,6 +3682,7 @@ aggregations: table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; diff --git a/data_plane/src/precompute_engine/output_sink.rs b/data_plane/src/precompute_engine/output_sink.rs index b908f2a8b..46a83a42c 100644 --- a/data_plane/src/precompute_engine/output_sink.rs +++ b/data_plane/src/precompute_engine/output_sink.rs @@ -422,6 +422,7 @@ mod tests { table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, } diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs index 174c4ae5c..30713e75a 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs @@ -276,6 +276,7 @@ mod tests { table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, } diff --git a/data_plane/src/storage_engines/types/streaming_config.rs b/data_plane/src/storage_engines/types/streaming_config.rs index 91a2dc1b3..2298955db 100644 --- a/data_plane/src/storage_engines/types/streaming_config.rs +++ b/data_plane/src/storage_engines/types/streaming_config.rs @@ -133,6 +133,11 @@ impl StreamingConfig { num_aggregates_to_retain, QueryLanguage::PromQl, )?; + if config.derived_input.is_some() { + anyhow::bail!( + "legacy streaming input cannot execute a derived summary program" + ); + } // PR 5: the map key IS the policy-fingerprint u64. // `AggregationConfig::policy_fp_u64()` is the canonical // accessor for this value. @@ -191,6 +196,17 @@ mod tests { assert_eq!(cfg.storage_backend(), StorageBackend::GorillaObjectStore); } + #[test] + fn legacy_yaml_rejects_derived_summary_input() { + let data = serde_yaml::from_str::(&format!( + "aggregations:\n- aggregationType: Sum\n aggregationSubType: ''\n metric: outer\n labels: {{grouping: [], rollup: [], aggregated: []}}\n parameters: {{}}\n windowSize: 10\n windowType: tumbling\n spatialFilter: ''\n derived_input:\n inputs: [1]\n program_sha256: '{}'\n", "a".repeat(64) + )).unwrap(); + let error = StreamingConfig::from_yaml_data(&data).unwrap_err(); + assert!(error + .to_string() + .contains("legacy streaming input cannot execute")); + } + /// PR 5: a streaming-config YAML that omits `aggregationId` /// parses correctly — the backend derives identity from content /// via `PolicyFingerprint::from_config`. The map key is the diff --git a/data_plane/src/tests/test_utilities/engine_factories.rs b/data_plane/src/tests/test_utilities/engine_factories.rs index 3fce2d44b..564d21d7c 100644 --- a/data_plane/src/tests/test_utilities/engine_factories.rs +++ b/data_plane/src/tests/test_utilities/engine_factories.rs @@ -109,6 +109,7 @@ pub fn create_engine_single_pop_with_aggregated( table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; @@ -206,6 +207,7 @@ pub fn create_engine_dual_input( table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; @@ -233,6 +235,7 @@ pub fn create_engine_dual_input( table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; @@ -325,6 +328,7 @@ pub fn create_engine_two_metrics( table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; @@ -351,6 +355,7 @@ pub fn create_engine_two_metrics( table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; @@ -453,6 +458,7 @@ pub fn create_engine_three_metrics( table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; @@ -532,6 +538,7 @@ pub fn create_engine_multi_timestamp( table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; @@ -603,6 +610,7 @@ pub fn create_engine_multi_timestamp_with_window( table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; diff --git a/docs/design_docs/summary-catalog-sds-architecture.md b/docs/design_docs/summary-catalog-sds-architecture.md index 09f2fab20..14b388e5c 100644 --- a/docs/design_docs/summary-catalog-sds-architecture.md +++ b/docs/design_docs/summary-catalog-sds-architecture.md @@ -441,6 +441,28 @@ summary state. Legacy records without trustworthy catalog provenance remain unbound. Tombstone reclamation still requires coordinated removal of old physical parts and is not implemented by this transition. +### Derived summary input identity + +A summary computed from another summary has a different data source from the +original raw table or metric. `PrecomputeMaterialization.derived_input` and +`DataSourceIdentity::Derived` use the same `DerivedInputIdentity`: the referenced +`SummaryDefinitionId`s and a SHA-256 of the maintenance program. The executable +program remains in `OwnedPostAsapDag`; the catalog does not retain another copy. + +The signature replaces materialized input frontiers with stable summary IDs and +hashes the remaining node payloads, schemas, guarantees, and edge semantics. It +excludes query names, plan-local node numbering, and catalog generations. Literal +leaves are hashed directly; raw input leaves still require catalog frontiers. A changed +input definition or transformation creates a new identity. Existing raw-source +identities retain their previous byte representation. Catalog validation rejects +missing input definitions and dependency cycles. + +This contract is a prerequisite, not enabled summary-over-summary execution. +Installation currently rejects derived inputs so they cannot accidentally receive +raw samples through the legacy metric router. Enabling them requires the immutable +maintenance consumer and durable output deduplication protocol; neither raw-table +substitution nor treating late correction fragments as new observations is valid. + ### Immutable completed windows Finite Remote Write completion now fences the SummaryStore append boundary, From 5e85d753c3547a2f2407ea3d64d29ce404a20e08 Mon Sep 17 00:00:00 2001 From: Zeying Zhu <50204836+zzylol@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:40:50 -0400 Subject: [PATCH 26/28] feat(clickhouse): read tuple fields inside query DAGs (#655) * feat(clickhouse): execute typed array element access * style(clickhouse): format collection default helper * feat(clickhouse): read typed tuple fields inside query DAGs * refactor(clickhouse): share nested type argument parsing * fix(clickhouse): distinguish nonfinite exact values from nulls * test(clickhouse): verify exact denormal transport policy * test(clickhouse): verify native tuple field rendering --- .../src/query_plan/clickhouse_exact.rs | 26 ++++ .../relational_adapter.rs | 144 +++++++++++++++--- .../relational_adapter/collection.rs | 120 +++++++++++++++ 3 files changed, 271 insertions(+), 19 deletions(-) create mode 100644 data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter/collection.rs diff --git a/control_plane/src/query_plan/clickhouse_exact.rs b/control_plane/src/query_plan/clickhouse_exact.rs index ea7c7c551..fa227841f 100644 --- a/control_plane/src/query_plan/clickhouse_exact.rs +++ b/control_plane/src/query_plan/clickhouse_exact.rs @@ -78,6 +78,7 @@ fn scalar(expr: &QueryExpr, schema: &Schema) -> Result { "map" => "map", "mapconcat" => "mapConcat", "asap_map_access" | "asap_element_access" => "arrayElement", + "asap_struct_field" => "tupleElement", _ => return Err(format!("unsupported exact scalar function {name}")), }; expr.scalar_type(schema).map_err(|e| e.to_string())?; @@ -323,6 +324,31 @@ mod tests { ); } + #[test] + fn typed_struct_field_renders_native_lookup() { + let schema = Schema::new(vec![Column::new( + "sample", + DataType::Struct { + fields: vec![ + Column::new("ts", DataType::Int64, false), + Column::new("value", DataType::Float64, true), + ], + }, + false, + )]); + let expr = QueryExpr::FunctionCall { + name: "asap_struct_field".into(), + args: vec![ + QueryExpr::Column(0), + QueryExpr::Literal(ScalarValue::Utf8("value".into())), + ], + }; + assert_eq!( + scalar(&expr, &schema).unwrap(), + "tupleElement(`sample`, 'value')" + ); + } + #[test] fn unsupported_scalar_is_not_forwarded_as_arbitrary_native_code() { let schema = Schema::new(vec![]); diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs index cdecb8791..8ddb70e9c 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs @@ -1,6 +1,7 @@ //! ClickHouse row semantics for planner-owned relational wrappers. mod aggregate; +mod collection; use std::{cmp::Ordering, collections::BTreeMap, sync::Arc}; @@ -42,6 +43,7 @@ enum Cell { Timestamp(i64), Map(Vec<(Cell, Cell)>), List(Arc<[Cell]>), + Struct(Arc<[Cell]>), } fn json_cell( @@ -75,9 +77,25 @@ fn json_cell( .into(), )) } - DataType::Struct { .. } => Err(ClickHouseRelationalError::Unsupported( - "struct value transport".into(), - )), + DataType::Struct { fields } => { + let types = + collection::tuple_field_types(clickhouse_type, fields).ok_or_else(invalid)?; + let items = value + .as_array() + .filter(|items| items.len() == fields.len()) + .ok_or_else(invalid)?; + Ok(Cell::Struct( + items + .iter() + .zip(fields) + .zip(types) + .map(|((item, field), native)| { + json_cell(item, &field.dtype, field.nullable, native) + }) + .collect::, _>>()? + .into(), + )) + } DataType::Int64 => value.as_i64().map(Cell::Int64).ok_or_else(invalid), DataType::Float64 => value.as_f64().map(Cell::Float64).ok_or_else(invalid), DataType::Utf8 => value @@ -289,20 +307,11 @@ impl ClickHouseRelation { fn map_type_parts(actual: &str) -> Option<(&str, &str)> { let inner = actual.trim().strip_prefix("Map(")?.strip_suffix(')')?; - let mut depth = 0_i32; - let mut quoted = false; - for (index, ch) in inner.char_indices() { - match ch { - '\'' => quoted = !quoted, - '(' if !quoted => depth += 1, - ')' if !quoted => depth -= 1, - ',' if !quoted && depth == 0 => { - return Some((inner[..index].trim(), inner[index + 1..].trim())) - } - _ => {} - } - } - None + let args = collection::arguments(inner)?; + let [key, value] = args.as_slice() else { + return None; + }; + Some((*key, *value)) } fn clickhouse_type_matches(actual: Option<&str>, expected: &DataType, nullable: bool) -> bool { @@ -331,7 +340,9 @@ fn clickhouse_type_matches(actual: Option<&str>, expected: &DataType, nullable: clickhouse_type_matches(Some(inner), &element.dtype, element.nullable) }) } - DataType::Struct { .. } => false, + DataType::Struct { fields } => { + !nullable && collection::tuple_field_types(actual, fields).is_some() + } DataType::Int64 => actual == "Int64", DataType::Float64 => actual == "Float64", DataType::Utf8 => actual == "String", @@ -578,6 +589,37 @@ fn eval( } QueryExpr::FunctionCall { name, args } => { use planner_types::pre_asap::scalar_signature::MapScalarFunction; + if name.eq_ignore_ascii_case("asap_struct_field") { + expr.scalar_type(schema) + .map_err(|error| ClickHouseRelationalError::Invalid(error.to_string()))?; + let DataType::Struct { fields } = args[0] + .scalar_type(schema) + .map_err(|error| ClickHouseRelationalError::Invalid(error.to_string()))? + .0 + else { + unreachable!() + }; + let offset = match &args[1] { + QueryExpr::Literal(ScalarValue::Int64(index)) => { + usize::try_from(index - 1).ok() + } + QueryExpr::Literal(ScalarValue::Utf8(name)) => { + fields.iter().position(|field| &field.name == name) + } + _ => None, + } + .ok_or_else(|| { + ClickHouseRelationalError::Invalid("struct field selector".into()) + })?; + let Cell::Struct(values) = eval(&args[0], row, schema)? else { + return Err(ClickHouseRelationalError::Invalid( + "struct field input".into(), + )); + }; + return values.get(offset).cloned().ok_or_else(|| { + ClickHouseRelationalError::Invalid("struct field value".into()) + }); + } if name.eq_ignore_ascii_case("asap_element_access") { let (output_type, _) = expr .scalar_type(schema) @@ -717,6 +759,13 @@ fn default_collection_element( DataType::Bool => Cell::Bool(false), DataType::Map { .. } => Cell::Map(Vec::new()), DataType::List { .. } => Cell::List(Arc::from([])), + DataType::Struct { fields } => Cell::Struct( + fields + .iter() + .map(|field| default_collection_element(&field.dtype, field.nullable)) + .collect::, _>>()? + .into(), + ), _ => { return Err(ClickHouseRelationalError::Unsupported( "collection missing-element default type".into(), @@ -845,7 +894,7 @@ fn compare_sort_keys( fn contains_nan(value: &Cell) -> bool { match value { Cell::Float64(value) => value.is_nan(), - Cell::List(values) => values.iter().any(contains_nan), + Cell::List(values) | Cell::Struct(values) => values.iter().any(contains_nan), Cell::Map(entries) => entries .iter() .any(|(key, value)| contains_nan(key) || contains_nan(value)), @@ -1187,6 +1236,63 @@ mod tests { assert!(eval(&zero, &[Cell::List(Arc::from([])), Cell::Int64(0)], &schema).is_err()); } + #[test] + fn nested_array_tuple_access_preserves_fields_and_defaults() { + use planner_types::pre_asap::{Column, Schema}; + let tuple = DataType::Struct { + fields: vec![ + Column::new("ts", DataType::Int64, false), + Column::new("value", DataType::Float64, true), + ], + }; + let dtype = DataType::List { + element: Box::new(Column::new("item", tuple, false)), + }; + let native = "Array(Tuple(ts Int64, value Nullable(Float64)))"; + assert!(clickhouse_type_matches(Some(native), &dtype, false)); + let samples = json_cell( + &serde_json::json!([[9007199254740993_i64, 2.5], [7, null]]), + &dtype, + false, + native, + ) + .unwrap(); + let schema = Schema::new(vec![Column::new("samples", dtype, false)]); + let field = |index, name: &str| QueryExpr::FunctionCall { + name: "asap_struct_field".into(), + args: vec![ + QueryExpr::FunctionCall { + name: "asap_element_access".into(), + args: vec![ + QueryExpr::Column(0), + QueryExpr::Literal(ScalarValue::Int64(index)), + ], + }, + QueryExpr::Literal(ScalarValue::Utf8(name.into())), + ], + }; + assert_eq!( + eval(&field(1, "ts"), std::slice::from_ref(&samples), &schema).unwrap(), + Cell::Int64(9007199254740993) + ); + assert_eq!( + eval(&field(1, "value"), std::slice::from_ref(&samples), &schema).unwrap(), + Cell::Float64(2.5) + ); + assert_eq!( + eval(&field(-1, "value"), std::slice::from_ref(&samples), &schema).unwrap(), + Cell::Null + ); + assert_eq!( + eval(&field(99, "ts"), std::slice::from_ref(&samples), &schema).unwrap(), + Cell::Int64(0) + ); + assert_eq!( + eval(&field(99, "value"), std::slice::from_ref(&samples), &schema).unwrap(), + Cell::Null + ); + } + fn schema(fields: &[(&str, DataType)]) -> SummarySchema { SummarySchema { fields: fields diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter/collection.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter/collection.rs new file mode 100644 index 000000000..2bb43e7bb --- /dev/null +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter/collection.rs @@ -0,0 +1,120 @@ +//! Native collection metadata is checked against the existing shared schema. + +use planner_types::pre_asap::Column; + +/// Split native type arguments without splitting nested types or quoted names. +pub(super) fn arguments(input: &str) -> Option> { + let mut result = Vec::new(); + let mut start = 0; + let mut depth = 0_usize; + let mut quote = None; + let mut chars = input.char_indices().peekable(); + while let Some((position, ch)) = chars.next() { + if let Some(delimiter) = quote { + if ch == '\\' { + chars.next()?; + } else if ch == delimiter { + if chars.peek().is_some_and(|(_, next)| *next == delimiter) { + chars.next(); + } else { + quote = None; + } + } + continue; + } + match ch { + '\'' | '`' | '"' => quote = Some(ch), + '(' => depth = depth.checked_add(1)?, + ')' => depth = depth.checked_sub(1)?, + ',' if depth == 0 => { + result.push(input[start..position].trim()); + start = position + 1; + } + _ => {} + } + } + if depth != 0 || quote.is_some() { + return None; + } + if !input.is_empty() { + result.push(input[start..].trim()); + } + (!result.iter().any(|argument| argument.is_empty())).then_some(result) +} + +/// Anonymous native Tuple fields have explicit one-based names in the shared +/// Struct schema. Named fields must match their native names exactly. Other +/// Arrow names are never interpreted as an anonymous Tuple. +pub(super) fn tuple_field_types<'a>(native: &'a str, fields: &[Column]) -> Option> { + let inner = native.strip_prefix("Tuple(")?.strip_suffix(')')?; + let args = arguments(inner)?; + if args.len() != fields.len() { + return None; + } + let mut types = Vec::with_capacity(fields.len()); + for (index, (argument, field)) in args.into_iter().zip(fields).enumerate() { + if field.table.is_some() { + return None; + } + if field.name == (index + 1).to_string() + && super::clickhouse_type_matches(Some(argument), &field.dtype, field.nullable) + { + types.push(argument); + continue; + } + // Initial named transport accepts ordinary identifiers. Quoted names + // remain unsupported until a native identifier-decoding contract exists. + let (name, native_type) = argument.split_once(char::is_whitespace)?; + if name != field.name + || name.is_empty() + || !name + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_') + || !super::clickhouse_type_matches( + Some(native_type.trim()), + &field.dtype, + field.nullable, + ) + { + return None; + } + types.push(native_type.trim()); + } + Some(types) +} + +#[cfg(test)] +mod tests { + use super::*; + use planner_types::pre_asap::DataType; + + #[test] + fn tuple_metadata_preserves_names_order_and_nested_types() { + let fields = vec![ + Column::new("ts", DataType::Int64, false), + Column::new( + "samples", + DataType::List { + element: Box::new(Column::new("item", DataType::Float64, true)), + }, + false, + ), + ]; + assert_eq!( + tuple_field_types("Tuple(ts Int64, samples Array(Nullable(Float64)))", &fields), + Some(vec!["Int64", "Array(Nullable(Float64))"]) + ); + assert!( + tuple_field_types("Tuple(samples Int64, ts Array(Nullable(Float64)))", &fields) + .is_none() + ); + assert!(tuple_field_types("Tuple(Int64, Array(Nullable(Float64)))", &fields).is_none()); + let anonymous = vec![ + Column::new("1", DataType::Int64, false), + Column::new("2", DataType::Utf8, false), + ]; + assert!(tuple_field_types("Tuple(Int64, String)", &anonymous).is_some()); + assert!(arguments("Map(String, Tuple(Int64, String)), DateTime64(3, 'UTC')").is_some()); + assert!(arguments("Map(String, Tuple(Int64, String)").is_none()); + } +} From eaad52898722dfb0f8e7e8d60163ca1fe754e878 Mon Sep 17 00:00:00 2001 From: Zeying Zhu <50204836+zzylol@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:46:44 -0400 Subject: [PATCH 27/28] fix(storage): publish immutable windows with durable lineage (#654) * feat(catalog): identify derived summary inputs * fix(catalog): separate derived policy domain from raw metric names * fix(storage): reserve immutable output publication durably * fix(storage): reject retry after manifest retirement * docs(sds): separate immutable window section * fix(storage): look up committed lineage before sketch evaluation * fix(storage): match pending lineage atomically during recovery --- .../sketch_db/persistence/flusher.rs | 37 +- .../sketch_db/persistence/immutable_output.rs | 659 ++++++++++++++++++ .../sketch_db/persistence/metadata.rs | 75 +- .../sketch_db/persistence/mod.rs | 1 + .../sketch_db/persistence/recovery.rs | 12 +- 5 files changed, 776 insertions(+), 8 deletions(-) create mode 100644 data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs diff --git a/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs b/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs index e774b39e6..4c8307db2 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs @@ -32,7 +32,7 @@ use super::{PersistError, PersistResult}; /// Handle to a running flusher thread. Dropping the handle signals /// shutdown and joins the thread. pub struct FlusherHandle { - inner: Arc, + pub(super) inner: Arc, thread: Option>, } @@ -58,6 +58,14 @@ pub(crate) struct FlusherShared { pub back_pressure_wait_count: AtomicU64, } +impl FlusherShared { + pub(super) fn allocate_part_id(&self) -> PersistResult { + self.next_part_id + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |id| id.checked_add(1)) + .map_err(|_| PersistError::Internal("part ID exhausted".into())) + } +} + impl FlusherHandle { /// Start a flusher thread. Takes an `EpochSource` (typically the /// store itself, wrapped in `Arc`). @@ -84,12 +92,23 @@ impl FlusherHandle { { // Pick a starting part_id: one past the max currently in the // manifest (so IDs are monotonically increasing across restarts). + let reserved = sid_metadata.load_strict()?.into_iter().flat_map(|r| { + [r.pending_immutable, r.last_immutable] + .into_iter() + .flatten() + .map(|p| p.part_id) + }); let next_id = manifest .live_parts() .iter() .map(|p| p.part_id) + .chain(reserved) .max() - .map(|m| m + 1) + .map(|m| { + m.checked_add(1) + .ok_or_else(|| PersistError::Internal("part ID exhausted".into())) + }) + .transpose()? .unwrap_or(1); let shared = Arc::new(FlusherShared { @@ -369,7 +388,7 @@ fn run_tick(shared: &Arc, source: &S) -> PersistR if !snapshots.is_empty() { // Build one part for the tick. - let part_id = shared.next_part_id.fetch_add(1, Ordering::Relaxed); + let part_id = shared.allocate_part_id()?; let part_dir = part_dir_path(&parts_root(&cfg.disk_path), part_id); let entries_total: usize = snapshots.iter().map(|s| s.len()).sum(); let size_bytes_estimate: u64 = snapshots.iter().map(|s| s.approx_bytes as u64).sum(); @@ -428,6 +447,18 @@ fn run_tick(shared: &Arc, source: &S) -> PersistR .into_iter() .filter(|p| p.max_ts < cutoff) .collect(); + // Read reservations after capturing candidates: a newly published part + // cannot enter the older candidate set after this check. + let reserved: std::collections::HashSet<_> = shared + .sid_metadata + .load_strict()? + .into_iter() + .filter_map(|record| record.pending_immutable.map(|pending| pending.part_id)) + .collect(); + let expired: Vec<_> = expired + .into_iter() + .filter(|part| !reserved.contains(&part.part_id)) + .collect(); for p in &expired { shared.manifest.append_delete(p.part_id)?; let dir = part_dir_path(&parts_root(&cfg.disk_path), p.part_id); diff --git a/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs b/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs new file mode 100644 index 000000000..0ac85e833 --- /dev/null +++ b/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs @@ -0,0 +1,659 @@ +//! Crash-resumable publication of one immutable output window. The SID sidecar +//! reserves the existing part ID before writing; no second payload store is used. +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use super::flusher::{FlusherHandle, FlusherShared}; +use super::manifest::PartEntry; +use super::metadata::SidMetaRecord; +use super::part::{part_dir_path, PartReader, PartWriter}; +use super::source::{EpochSnapshot, EpochSnapshotEntry}; +use super::{PersistError, PersistResult}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ImmutableOutputReservation { + pub part_id: u64, + pub input_digest: [u8; 32], + pub payload_digest: [u8; 32], + pub start_ms: u64, + pub end_ms: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ImmutableWindowPublication { + pub part_id: u64, + pub already_published: bool, +} + +fn invalid(message: &str) -> PersistError { + PersistError::Format(message.into()) +} + +fn validate_identity(current: &SidMetaRecord, record: &SidMetaRecord) -> PersistResult<()> { + if current.sid != record.sid + || current.removed + || current.retired_at_ms.is_some() + || current.expires_at_ms.is_some() + || current.summary_definition_id != record.summary_definition_id + || current.catalog_generation != record.catalog_generation + || current.metric_name != record.metric_name + || current.group_by_keys != record.group_by_keys + || !current.same_kind(record) + { + return Err(invalid("immutable publication metadata identity changed")); + } + Ok(()) +} + +fn fingerprint(snapshot: &EpochSnapshot) -> PersistResult<[u8; 32]> { + if snapshot.entries.is_empty() || snapshot.min_ts >= snapshot.max_ts { + return Err(invalid("immutable publication requires a nonempty window")); + } + let mut rows = Vec::with_capacity(snapshot.entries.len()); + for entry in &snapshot.entries { + if entry.start_ts != snapshot.min_ts || entry.end_ts != snapshot.max_ts { + return Err(invalid("immutable publication spans multiple windows")); + } + let label = entry.label.as_ref().map(|label| label.serialize_to_bytes()); + rows.push(( + label, + &entry.sketch_type_name, + entry.encoding_tag, + &entry.sketch_bytes, + )); + } + rows.sort(); + if rows.windows(2).any(|pair| pair[0].0 == pair[1].0) { + return Err(invalid( + "immutable publication contains duplicate label populations", + )); + } + let mut digest = Sha256::new(); + digest.update(b"asap-immutable-window-v1"); + for value in [ + snapshot.agg_id, + snapshot.min_ts, + snapshot.max_ts, + rows.len() as u64, + ] { + digest.update(value.to_le_bytes()); + } + for (label, kind, encoding, payload) in rows { + digest.update([u8::from(label.is_some())]); + let label = label.as_deref().unwrap_or_default(); + for bytes in [label, kind.as_bytes()] { + digest.update((bytes.len() as u64).to_le_bytes()); + digest.update(bytes); + } + digest.update([encoding]); + digest.update((payload.len() as u64).to_le_bytes()); + digest.update(payload); + } + Ok(digest.finalize().into()) +} + +impl FlusherHandle { + /// Reuse the existing persistence state without transferring ownership of + /// its worker thread. The store can retain a Weak reference to this Arc. + pub(crate) fn publication_handle(&self) -> std::sync::Arc { + std::sync::Arc::clone(&self.inner) + } + pub fn publish_immutable_window( + &self, + record: &SidMetaRecord, + input_digest: [u8; 32], + snapshot: &EpochSnapshot, + ) -> PersistResult { + self.inner + .publish_immutable_window(record, input_digest, snapshot) + } + pub fn resume_pending_immutable_window( + &self, + sid: u64, + ) -> PersistResult> { + self.inner.resume_pending_immutable_window(sid) + } +} + +impl FlusherShared { + /// Find the latest committed result before evaluating a potentially + /// randomized sketch again. This never creates or replaces a payload. + pub fn lookup_immutable_window( + &self, + record: &SidMetaRecord, + input_digest: [u8; 32], + start_ms: u64, + end_ms: u64, + ) -> PersistResult> { + self.sid_metadata.transaction(|records, _| { + let Some(current) = records.get(&record.sid.to_string()) else { + return Ok(None); + }; + validate_identity(current, record)?; + let Some(previous) = ¤t.last_immutable else { + return Ok(None); + }; + if previous.start_ms != start_ms || previous.end_ms != end_ms { + return Ok(None); + } + if previous.input_digest != input_digest { + return Err(invalid("immutable lookup input lineage differs")); + } + self.validate_reserved_part(record.sid, previous)?; + if !self + .manifest + .live_parts() + .iter() + .any(|part| part.part_id == previous.part_id) + { + return Err(invalid("completed immutable part is no longer published")); + } + Ok(Some(ImmutableWindowPublication { + part_id: previous.part_id, + already_published: true, + })) + }) + } + + /// Publish a finalized window. A retry must present exactly the reserved + /// input and payload. This does not guarantee source availability after GC. + pub fn publish_immutable_window( + &self, + record: &SidMetaRecord, + input_digest: [u8; 32], + snapshot: &EpochSnapshot, + ) -> PersistResult { + if snapshot.agg_id != record.sid || record.removed { + return Err(invalid("immutable publication SID is invalid or removed")); + } + let payload_digest = fingerprint(snapshot)?; + self.sid_metadata.transaction(|records, metadata| { + let key = record.sid.to_string(); + let mut current = records.get(&key).cloned().unwrap_or_else(|| record.clone()); + validate_identity(¤t, record)?; + if current + .completed_through_ms + .is_some_and(|end| snapshot.min_ts < end) + { + let Some(previous) = ¤t.last_immutable else { + return Err(invalid( + "completed immutable window has no retained lineage proof", + )); + }; + if previous.input_digest != input_digest + || previous.payload_digest != payload_digest + || previous.start_ms != snapshot.min_ts + || previous.end_ms != snapshot.max_ts + { + return Err(invalid( + "completed immutable retry differs from latest publication", + )); + } + self.validate_reserved_part(record.sid, previous)?; + if !self + .manifest + .live_parts() + .iter() + .any(|part| part.part_id == previous.part_id) + { + return Err(invalid("completed immutable part is no longer published")); + } + return Ok(ImmutableWindowPublication { + part_id: previous.part_id, + already_published: true, + }); + } + let reservation = match current.pending_immutable.clone() { + Some(pending) => { + if pending.input_digest != input_digest + || pending.payload_digest != payload_digest + || pending.start_ms != snapshot.min_ts + || pending.end_ms != snapshot.max_ts + { + return Err(invalid( + "another immutable publication is pending for this SID", + )); + } + pending + } + None => { + let pending = ImmutableOutputReservation { + part_id: self.allocate_part_id()?, + input_digest, + payload_digest, + start_ms: snapshot.min_ts, + end_ms: snapshot.max_ts, + }; + current.pending_immutable = Some(pending.clone()); + records.insert(key.clone(), current.clone()); + metadata.write_records(records)?; + pending + } + }; + let path = part_dir_path(&self.cfg.disk_path.join("parts"), reservation.part_id); + if path.exists() { + // Never overwrite a reserved part silently: callers can recover a + // durable part without sources; damaged/partial parts fail closed. + self.validate_reserved_part(record.sid, &reservation)?; + } else { + PartWriter::write_part(&path, reservation.part_id, std::slice::from_ref(snapshot))?; + std::fs::File::open( + path.parent() + .ok_or_else(|| invalid("missing parts parent"))?, + )? + .sync_all()?; + self.validate_reserved_part(record.sid, &reservation)?; + } + self.finish_reserved_part(&mut current, &reservation)?; + records.insert(key, current); + metadata.write_records(records)?; + Ok(ImmutableWindowPublication { + part_id: reservation.part_id, + already_published: false, + }) + }) + } + + /// Resume only the caller's pending lineage, atomically with metadata + /// validation. A different pending input cannot be acknowledged by this call. + pub fn resume_matching_immutable_window( + &self, + record: &SidMetaRecord, + input_digest: [u8; 32], + start_ms: u64, + end_ms: u64, + ) -> PersistResult> { + self.resume_immutable_window(record.sid, Some((record, input_digest, start_ms, end_ms))) + } + + /// Complete a pending durable part without reconstructing its source input. + /// A reservation whose part was never completed returns an error, preserving + /// the reservation for an explicit recovery policy rather than losing data. + pub fn resume_pending_immutable_window( + &self, + sid: u64, + ) -> PersistResult> { + self.resume_immutable_window(sid, None) + } + + fn resume_immutable_window( + &self, + sid: u64, + expected: Option<(&SidMetaRecord, [u8; 32], u64, u64)>, + ) -> PersistResult> { + self.sid_metadata.transaction(|records, metadata| { + let key = sid.to_string(); + let Some(mut current) = records.get(&key).cloned() else { + return Ok(None); + }; + let Some(pending) = current.pending_immutable.clone() else { + return Ok(None); + }; + if current.removed || current.retired_at_ms.is_some() || current.expires_at_ms.is_some() + { + return Err(invalid("pending immutable SID was removed")); + } + if let Some((record, digest, start, end)) = expected { + validate_identity(¤t, record)?; + if pending.input_digest != digest + || pending.start_ms != start + || pending.end_ms != end + { + return Err(invalid("pending immutable recovery lineage differs")); + } + } + self.validate_reserved_part(sid, &pending)?; + self.finish_reserved_part(&mut current, &pending)?; + records.insert(key, current); + metadata.write_records(records)?; + Ok(Some(ImmutableWindowPublication { + part_id: pending.part_id, + already_published: true, + })) + }) + } + + fn validate_reserved_part( + &self, + sid: u64, + pending: &ImmutableOutputReservation, + ) -> PersistResult<()> { + let reader = PartReader::open(&part_dir_path( + &self.cfg.disk_path.join("parts"), + pending.part_id, + ))?; + if reader.meta.part_id != pending.part_id { + return Err(invalid("reserved part ID mismatch")); + } + let mut entries = Vec::new(); + for index in reader.index_records() { + let row = reader.load_entry(&index)?; + if row.agg_id != sid { + return Err(invalid("reserved part contains another SID")); + } + entries.push(EpochSnapshotEntry { + start_ts: row.start_ts, + end_ts: row.end_ts, + label: row.label, + sketch_type_name: row.sketch_type_name, + encoding_tag: row.encoding_tag, + sketch_bytes: row.sketch_bytes, + }); + } + let snapshot = EpochSnapshot { + agg_id: sid, + epoch_id: 0, + min_ts: pending.start_ms, + max_ts: pending.end_ms, + entries, + approx_bytes: 0, + }; + if fingerprint(&snapshot)? != pending.payload_digest { + return Err(invalid("reserved immutable payload digest mismatch")); + } + Ok(()) + } + + fn finish_reserved_part( + &self, + current: &mut SidMetaRecord, + pending: &ImmutableOutputReservation, + ) -> PersistResult<()> { + let path = part_dir_path(&self.cfg.disk_path.join("parts"), pending.part_id); + // The reservation may have been recovered before its parent-directory + // entry was durable; make that durable before manifest publication too. + std::fs::File::open( + path.parent() + .ok_or_else(|| invalid("missing parts parent"))?, + )? + .sync_all()?; + if !self + .manifest + .live_parts() + .iter() + .any(|part| part.part_id == pending.part_id) + { + let size_bytes = std::fs::read_dir(&path)?.try_fold(0u64, |sum, entry| { + let length = entry?.metadata()?.len(); + sum.checked_add(length) + .ok_or_else(|| std::io::Error::other("part size overflow")) + })?; + self.manifest.append_add(PartEntry { + part_id: pending.part_id, + min_ts: pending.start_ms, + max_ts: pending.end_ms, + size_bytes, + })?; + } + current.completed_through_ms = Some( + current + .completed_through_ms + .unwrap_or(0) + .max(pending.end_ms), + ); + current.last_immutable = Some(pending.clone()); + current.pending_immutable = None; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::super::{ + config::SketchStorePersistenceConfig, + manifest::Manifest, + source::{EpochSource, SealedEpochRef}, + }; + use super::*; + use crate::storage_engines::{sketch_db::data::AggKind, types::AggregationType}; + use std::sync::Arc; + + struct EmptySource; + impl EpochSource for EmptySource { + fn list_sealed_epochs(&self) -> Vec { + vec![] + } + fn snapshot_sealed_epoch(&self, _: u64, _: u64) -> PersistResult> { + Ok(None) + } + fn evict_sealed_epoch(&self, _: u64, _: u64) {} + fn approx_memory_bytes(&self) -> usize { + 0 + } + } + fn handle(path: &std::path::Path) -> FlusherHandle { + let mut config = SketchStorePersistenceConfig::with_memory_limit(1024, path.into()); + config.hot_window_ms = None; + config.delete_older_than_ms = None; + FlusherHandle::start( + config, + Arc::new(Manifest::open_or_init(path).unwrap()), + Arc::new(EmptySource), + ) + .unwrap() + } + fn record() -> SidMetaRecord { + SidMetaRecord::new( + 1, + "derived".into(), + vec![], + &AggKind::ExactAgg { + agg_type: AggregationType::Sum, + parameters_canonical: String::new(), + spatial_filter_canonical: String::new(), + }, + 0, + ) + } + fn snapshot() -> EpochSnapshot { + EpochSnapshot { + agg_id: 1, + epoch_id: 0, + min_ts: 10, + max_ts: 20, + approx_bytes: 8, + entries: vec![EpochSnapshotEntry { + start_ts: 10, + end_ts: 20, + label: None, + sketch_type_name: "SumAccumulator".into(), + encoding_tag: 0, + sketch_bytes: vec![1, 2, 3], + }], + } + } + fn reserve(handle: &FlusherHandle, snapshot: &EpochSnapshot) -> ImmutableOutputReservation { + let pending = ImmutableOutputReservation { + part_id: handle.inner.allocate_part_id().unwrap(), + input_digest: [7; 32], + payload_digest: fingerprint(snapshot).unwrap(), + start_ms: 10, + end_ms: 20, + }; + handle + .inner + .sid_metadata + .transaction(|records, metadata| { + let mut record = record(); + record.pending_immutable = Some(pending.clone()); + records.insert("1".into(), record); + metadata.write_records(records) + }) + .unwrap(); + pending + } + #[test] + fn completed_retry_and_concurrent_publications_do_not_duplicate_parts() { + let temp = tempfile::tempdir().unwrap(); + let handle = handle(temp.path()); + std::thread::scope(|scope| { + let workers: Vec<_> = (0..4) + .map(|_| { + scope.spawn(|| { + handle + .publish_immutable_window(&record(), [7; 32], &snapshot()) + .unwrap() + }) + }) + .collect(); + let ids: Vec<_> = workers + .into_iter() + .map(|worker| worker.join().unwrap().part_id) + .collect(); + assert!(ids.iter().all(|id| *id == ids[0])); + }); + assert_eq!(handle.manifest().live_parts().len(), 1); + assert!(handle + .publish_immutable_window(&record(), [8; 32], &snapshot()) + .is_err()); + let metadata = handle.inner.sid_metadata.load_strict().unwrap(); + assert_eq!(metadata[0].completed_through_ms, Some(20)); + assert!(metadata[0].pending_immutable.is_none()); + let mut different = snapshot(); + different.entries[0].sketch_bytes.push(4); + assert!(handle + .publish_immutable_window(&record(), [7; 32], &different) + .is_err()); + let part_id = handle.manifest().live_parts()[0].part_id; + handle.manifest().append_delete(part_id).unwrap(); + assert!(handle + .publish_immutable_window(&record(), [7; 32], &snapshot()) + .is_err()); + } + #[test] + fn restart_preserves_reserved_part_and_resumes_without_source() { + let temp = tempfile::tempdir().unwrap(); + let mut first = handle(temp.path()); + let state = snapshot(); + let pending = reserve(&first, &state); + let path = part_dir_path(&temp.path().join("parts"), pending.part_id); + PartWriter::write_part(&path, pending.part_id, &[state]).unwrap(); + first.shutdown(); + drop(first); + let (_, recovery) = super::super::recovery::recover(temp.path()).unwrap(); + assert_eq!(recovery.orphan_parts_removed, 0); + let resumed = handle(temp.path()); + assert!(resumed.inner.allocate_part_id().unwrap() > pending.part_id); + let result = resumed.resume_pending_immutable_window(1).unwrap().unwrap(); + assert_eq!(result.part_id, pending.part_id); + assert_eq!(resumed.manifest().live_parts().len(), 1); + assert!(resumed + .resume_pending_immutable_window(1) + .unwrap() + .is_none()); + } + #[test] + fn pending_missing_part_requires_source_and_stale_upsert_preserves_reservation() { + let temp = tempfile::tempdir().unwrap(); + let handle = handle(temp.path()); + let pending = reserve(&handle, &snapshot()); + handle.inner.sid_metadata.upsert_all(&[record()]).unwrap(); + assert_eq!( + handle.inner.sid_metadata.load_strict().unwrap()[0].pending_immutable, + Some(pending.clone()) + ); + assert!(handle.resume_pending_immutable_window(1).is_err()); + assert_eq!( + handle + .publish_immutable_window(&record(), [7; 32], &snapshot()) + .unwrap() + .part_id, + pending.part_id + ); + handle.inner.sid_metadata.upsert_all(&[record()]).unwrap(); + assert!(handle.inner.sid_metadata.load_strict().unwrap()[0] + .pending_immutable + .is_none()); + } + #[test] + fn malformed_metadata_fails_before_part_publication() { + let temp = tempfile::tempdir().unwrap(); + let handle = handle(temp.path()); + std::fs::write(handle.inner.sid_metadata.path(), b"broken").unwrap(); + assert!(handle + .publish_immutable_window(&record(), [7; 32], &snapshot()) + .is_err()); + assert!(handle.manifest().live_parts().is_empty()); + } + #[test] + fn failed_manifest_leaves_recoverable_payload_and_no_completion() { + let temp = tempfile::tempdir().unwrap(); + let handle = handle(temp.path()); + let log = handle.manifest().log_path(); + let backup = log.with_extension("saved"); + std::fs::rename(&log, &backup).unwrap(); + std::fs::create_dir(&log).unwrap(); + assert!(handle + .publish_immutable_window(&record(), [7; 32], &snapshot()) + .is_err()); + let records = handle.inner.sid_metadata.load_strict().unwrap(); + let pending = records[0].pending_immutable.clone().unwrap(); + assert_eq!(records[0].completed_through_ms, None); + assert!(handle.manifest().live_parts().is_empty()); + std::fs::remove_dir(&log).unwrap(); + std::fs::rename(&backup, &log).unwrap(); + let published = handle.resume_pending_immutable_window(1).unwrap().unwrap(); + assert_eq!(published.part_id, pending.part_id); + assert_eq!(handle.manifest().live_parts().len(), 1); + } + #[test] + fn committed_lookup_reuses_payload_without_recomputing_randomized_state() { + let temp = tempfile::tempdir().unwrap(); + let mut first = handle(temp.path()); + let publication = first + .publish_immutable_window(&record(), [7; 32], &snapshot()) + .unwrap(); + first.shutdown(); + drop(first); + let reopened = handle(temp.path()); + let found = reopened + .inner + .lookup_immutable_window(&record(), [7; 32], 10, 20) + .unwrap() + .unwrap(); + assert_eq!(found.part_id, publication.part_id); + assert!(reopened + .inner + .lookup_immutable_window(&record(), [8; 32], 10, 20) + .is_err()); + reopened + .manifest() + .append_delete(publication.part_id) + .unwrap(); + assert!(reopened + .inner + .lookup_immutable_window(&record(), [7; 32], 10, 20) + .is_err()); + } + #[test] + fn matching_resume_rejects_other_lineage_without_mutating_reservation() { + let temp = tempfile::tempdir().unwrap(); + let handle = handle(temp.path()); + let state = snapshot(); + let pending = reserve(&handle, &state); + let path = part_dir_path(&temp.path().join("parts"), pending.part_id); + PartWriter::write_part(&path, pending.part_id, &[state]).unwrap(); + assert!(handle + .inner + .resume_matching_immutable_window(&record(), [8; 32], 10, 20) + .is_err()); + assert!(handle + .inner + .resume_matching_immutable_window(&record(), [7; 32], 20, 30) + .is_err()); + assert!(handle.manifest().live_parts().is_empty()); + assert_eq!( + handle.inner.sid_metadata.load_strict().unwrap()[0].pending_immutable, + Some(pending.clone()) + ); + assert_eq!( + handle + .inner + .resume_matching_immutable_window(&record(), [7; 32], 10, 20) + .unwrap() + .unwrap() + .part_id, + pending.part_id + ); + } +} diff --git a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs index c432e07f2..dfdf7b4bc 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs @@ -322,6 +322,10 @@ pub struct SidMetaRecord { /// No further publication may change a window ending at or before this bound. #[serde(default)] pub completed_through_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pending_immutable: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_immutable: Option, } impl SidMetaRecord { @@ -347,9 +351,15 @@ impl SidMetaRecord { expires_at_ms: None, removed: false, completed_through_ms: None, + pending_immutable: None, + last_immutable: None, } } + pub(super) fn same_kind(&self, other: &Self) -> bool { + self.agg_kind == other.agg_kind + } + /// Reconstruct the structured [`AggKind`], or `None` for an /// unrecognized sketch kind. pub fn agg_kind(&self) -> Option { @@ -416,6 +426,10 @@ struct SidBindingRec { removed: bool, #[serde(default)] completed_through_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pending_immutable: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + last_immutable: Option, } /// Version-3 normalized sidecar with authoritative catalog provenance. Descriptors appear once and SeriesId bindings hold @@ -487,6 +501,8 @@ impl SdsSidecar { expires_at_ms: record.expires_at_ms, removed: record.removed, completed_through_ms: record.completed_through_ms, + pending_immutable: record.pending_immutable, + last_immutable: record.last_immutable, }, ); } @@ -541,6 +557,8 @@ impl SdsSidecar { expires_at_ms: binding.expires_at_ms, removed: binding.removed, completed_through_ms: binding.completed_through_ms, + pending_immutable: binding.pending_immutable, + last_immutable: binding.last_immutable, }) }) .collect() @@ -646,7 +664,7 @@ impl SidMetadataStore { return Ok(()); } let mut map: HashMap = self - .load()? + .load_strict()? .into_iter() .map(|r| (r.sid.to_string(), r)) .collect(); @@ -657,6 +675,8 @@ impl SidMetadataStore { if let Some(existing) = map.get(&key) { // Lifecycle is monotone for a SeriesId. An older flush snapshot // must not resurrect a retired or removed persisted instance. + next.pending_immutable = existing.pending_immutable.clone(); + next.last_immutable = existing.last_immutable.clone(); next.removed |= existing.removed; next.completed_through_ms = existing.completed_through_ms.max(next.completed_through_ms); @@ -680,6 +700,55 @@ impl SidMetadataStore { self.write_atomic(json.as_bytes()) } + pub fn load_strict(&self) -> PersistResult> { + let bytes = match fs::read(&self.path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(error.into()), + }; + let value: serde_json::Value = serde_json::from_slice(&bytes) + .map_err(|error| PersistError::Format(format!("invalid SID metadata: {error}")))?; + if let Some(version) = value.get("schema_version") { + if !matches!(version.as_u64(), Some(2 | 3)) { + return Err(PersistError::Format( + "unsupported SID metadata version".into(), + )); + } + let sidecar: SdsSidecar = serde_json::from_value(value) + .map_err(|error| PersistError::Format(error.to_string()))?; + sidecar.into_records() + } else { + let records: HashMap = serde_json::from_value(value) + .map_err(|error| PersistError::Format(error.to_string()))?; + Ok(records.into_values().collect()) + } + } + + pub(super) fn transaction( + &self, + operation: impl FnOnce(&mut HashMap, &Self) -> PersistResult, + ) -> PersistResult { + let _writer = self + .writer + .lock() + .map_err(|_| PersistError::Internal("SID metadata writer poisoned".into()))?; + let mut records = self + .load_strict()? + .into_iter() + .map(|r| (r.sid.to_string(), r)) + .collect(); + operation(&mut records, self) + } + pub(super) fn write_records( + &self, + records: &HashMap, + ) -> PersistResult<()> { + let sidecar = SdsSidecar::from_records(records.values().cloned()); + let bytes = + serde_json::to_vec(&sidecar).map_err(|e| PersistError::Serialize(e.to_string()))?; + self.write_atomic(&bytes) + } + fn write_atomic(&self, bytes: &[u8]) -> PersistResult<()> { if let Some(parent) = self.path.parent() { fs::create_dir_all(parent)?; @@ -696,9 +765,7 @@ impl SidMetadataStore { } fs::rename(&tmp, &self.path)?; if let Some(parent) = self.path.parent() { - if let Ok(dir) = File::open(parent) { - let _ = dir.sync_all(); - } + File::open(parent)?.sync_all()?; } Ok(()) } diff --git a/data_plane/src/storage_engines/sketch_db/persistence/mod.rs b/data_plane/src/storage_engines/sketch_db/persistence/mod.rs index 349feb92e..552fe4e03 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/mod.rs @@ -27,6 +27,7 @@ pub mod source; pub mod cache; pub mod flusher; +pub mod immutable_output; pub mod recovery; pub use config::SketchStorePersistenceConfig; diff --git a/data_plane/src/storage_engines/sketch_db/persistence/recovery.rs b/data_plane/src/storage_engines/sketch_db/persistence/recovery.rs index 7fb800946..a21402608 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/recovery.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/recovery.rs @@ -34,7 +34,12 @@ pub fn recover(disk_path: &Path) -> PersistResult<(Manifest, RecoveryReport)> { // Validate every live part by reading its meta.bin header. let mut to_drop: Vec = Vec::new(); - let mut referenced: HashSet = HashSet::new(); + let pending: HashSet = super::metadata::SidMetadataStore::new(disk_path) + .load_strict()? + .into_iter() + .filter_map(|record| record.pending_immutable.map(|pending| pending.part_id)) + .collect(); + let mut referenced = pending.clone(); for entry in manifest.live_parts() { referenced.insert(entry.part_id); let part_dir = super::part::part_dir_path(&parts_root, entry.part_id); @@ -61,6 +66,11 @@ pub fn recover(disk_path: &Path) -> PersistResult<(Manifest, RecoveryReport)> { } } + if to_drop.iter().any(|part_id| pending.contains(part_id)) { + return Err(super::PersistError::Format( + "corrupt reserved immutable part".into(), + )); + } for part_id in to_drop { manifest.append_delete(part_id)?; let dir = super::part::part_dir_path(&parts_root, part_id); From aa0b020cc99ae532b82d0bbdda21c37a8beee5da Mon Sep 17 00:00:00 2001 From: Zeying Zhu <50204836+zzylol@users.noreply.github.com> Date: Fri, 11 Sep 2026 12:12:33 -0400 Subject: [PATCH 28/28] fix(clickhouse): preserve nulls in query results (#657) * fix(clickhouse): preserve SQL nulls in result formats * test(clickhouse): distinguish literal null marker strings --- .../accelerator.rs | 9 ++-- .../clickhouse_result_adapter.rs | 46 +++++++++++++++++-- 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs index 8e34ceafd..dc7740561 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs @@ -147,11 +147,9 @@ impl CatalogClickHouseAccelerator { } fn requested_format(request: &ClickHouseQueryRequest) -> Result { - if let Some(setting) = request - .parameters - .keys() - .find(|key| key.starts_with("output_format_")) - { + if let Some(setting) = request.parameters.keys().find(|key| { + key.starts_with("output_format_") || key.as_str() == "format_tsv_null_representation" + }) { return Err(format!("unsupported output setting {setting}")); } match request @@ -302,6 +300,7 @@ mod tests { for setting in [ "output_format_json_map_as_array_of_tuples", "output_format_json_quote_64bit_integers", + "format_tsv_null_representation", ] { request.parameters.insert(setting.into(), "1".into()); assert!(requested_format(&request).is_err()); diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/clickhouse_result_adapter.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/clickhouse_result_adapter.rs index a829c548f..cdc46d262 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/clickhouse_result_adapter.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/clickhouse_result_adapter.rs @@ -2,7 +2,7 @@ use super::fallback::ClickHouseRawResponse; use arrow::{ array::{Array, Float64Array, StringArray, TimestampMillisecondArray}, datatypes::{DataType, Field, Schema}, - json::LineDelimitedWriter, + json::{LineDelimitedWriter, WriterBuilder}, record_batch::RecordBatch, util::display::array_value_to_string, }; @@ -118,6 +118,10 @@ impl ClickHouseQueryResult { if column > 0 { output.push(b'\t'); } + if batch.column(column).is_null(row) { + output.extend_from_slice(b"\\N"); + continue; + } if matches!(batch.column(column).data_type(), DataType::Map(..)) { output.extend_from_slice( map_literal(batch.column(column).as_ref(), row)?.as_bytes(), @@ -145,7 +149,9 @@ impl ClickHouseQueryResult { } let row = batch.slice(row, 1); - let mut writer = LineDelimitedWriter::new(&mut output); + let mut writer: LineDelimitedWriter<_> = WriterBuilder::new() + .with_explicit_nulls(true) + .build(&mut output); writer .write_batches(&[&row]) .map_err(|error| ClickHouseResultError::Arrow(error.to_string()))?; @@ -204,7 +210,9 @@ impl ClickHouseQueryResult { let mut rows = Vec::new(); for batch in &self.batches { let mut encoded = Vec::new(); - let mut writer = LineDelimitedWriter::new(&mut encoded); + let mut writer: LineDelimitedWriter<_> = WriterBuilder::new() + .with_explicit_nulls(true) + .build(&mut encoded); writer .write_batches(&[batch]) .map_err(|error| ClickHouseResultError::Arrow(error.to_string()))?; @@ -309,6 +317,38 @@ mod tests { }; use std::sync::Arc; + #[test] + fn nullable_fields_remain_explicit_in_json_and_tsv() { + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("value", DataType::Float64, true), + Field::new("label", DataType::Utf8, true), + ])), + vec![ + Arc::new(Float64Array::from(vec![Some(1.25), None, Some(2.5)])), + Arc::new(StringArray::from(vec![Some(""), None, Some("\\N")])), + ], + ) + .unwrap(); + let result = ClickHouseQueryResult { + batches: vec![batch], + }; + let json: serde_json::Value = + serde_json::from_slice(&result.encode(ClickHouseFormat::Json).unwrap()).unwrap(); + assert_eq!( + json["data"], + serde_json::json!([{ "value":1.25,"label":"" }, { "value":null,"label":null }, {"value":2.5,"label":"\\N"}]) + ); + let lines = result.encode(ClickHouseFormat::JsonEachRow).unwrap(); + let null_row: serde_json::Value = + serde_json::from_slice(lines.split(|byte| *byte == b'\n').nth(1).unwrap()).unwrap(); + assert_eq!(null_row, serde_json::json!({"value":null,"label":null})); + assert_eq!( + result.encode(ClickHouseFormat::TabSeparated).unwrap(), + b"1.25\t\n\\N\t\\N\n2.5\t\\\\N\n" + ); + } + #[test] fn empty_map_bottom_type_uses_clickhouse_nothing() { let entries = DataType::Struct(