From 29c0fc2f3472bd845c533ab4bad2c40867975ad2 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 20:03:18 -0600 Subject: [PATCH 1/4] refactor: share installed QueryPlan contracts across components --- Cargo.lock | 1 + control_plane/src/clickhouse.rs | 2 +- control_plane/src/physical/compiler.rs | 4 +- control_plane/src/query_plan.rs | 1052 ++++------------- control_plane/src/query_plan/logical.rs | 235 +--- control_plane/tests/o11y_exact_fallback.rs | 2 +- control_plane/tests/offline_evidence.rs | 6 +- crates/asap_types/Cargo.toml | 1 + crates/asap_types/src/lib.rs | 2 + crates/asap_types/src/query_plan.rs | 677 +++++++++++ crates/asap_types/src/query_plan/logical.rs | 155 +++ .../drivers/ingest/prometheus_remote_write.rs | 2 +- data_plane/src/drivers/query/servers/http.rs | 28 +- data_plane/src/main.rs | 2 +- .../precompute_engine/maintenance_runtime.rs | 14 +- .../src/precompute_engine/subdag_scheduler.rs | 8 +- .../accelerator.rs | 14 +- .../asap_clickhouse_query_engine/execution.rs | 7 +- .../asap_query_engine/catalog_resolver.rs | 2 +- .../query_engines/asap_query_engine/engine.rs | 27 +- .../asap_query_engine/exact_subqueries.rs | 16 +- .../asap_query_engine/live_serve.rs | 32 +- .../asap_query_engine/logical_dag.rs | 16 +- .../asap_query_engine/physical_dag.rs | 8 +- .../asap_query_engine/post_asap_readout.rs | 76 +- .../asap_query_engine/summary_executor.rs | 30 +- .../types/hot_reload_config.rs | 4 +- .../tests/clickhouse_differential_e2e.rs | 4 +- ...e2e_controller_plans_and_backend_serves.rs | 3 +- data_plane/tests/support/physical_fixture.rs | 2 +- docs/design_docs/asapplanner-integration.md | 3 +- .../summary-catalog-sds-architecture.md | 8 +- 32 files changed, 1222 insertions(+), 1221 deletions(-) create mode 100644 crates/asap_types/src/query_plan.rs create mode 100644 crates/asap_types/src/query_plan/logical.rs diff --git a/Cargo.lock b/Cargo.lock index 7d5b88241..40064b43a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -461,6 +461,7 @@ dependencies = [ "anyhow", "asap-types", "clap 4.6.1", + "promql-parser", "serde", "serde_json", "serde_yaml", diff --git a/control_plane/src/clickhouse.rs b/control_plane/src/clickhouse.rs index 7cd49accb..e2a6e8c21 100644 --- a/control_plane/src/clickhouse.rs +++ b/control_plane/src/clickhouse.rs @@ -342,7 +342,7 @@ where .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; let mut materialization_nodes = std::collections::BTreeMap::new(); let mut query_nodes = std::collections::BTreeMap::new(); - let executable = QueryPlanEntry::compile_bound_relational_mapped( + let executable = crate::query_plan::compile_bound_relational_mapped( query.sql.clone(), planned.canonical_sql.clone(), &root, diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index eff38ae7b..3269007dd 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -2207,7 +2207,7 @@ impl PhysicalCompiler { cumulative_readout: true, }; let mut entry = if request.hybrid_execution { - QueryPlanEntry::compile_bound_composable_mapped( + crate::query_plan::compile_bound_composable_mapped( query.query_id.clone(), canonical.clone(), &query.post_asap, @@ -2224,7 +2224,7 @@ impl PhysicalCompiler { }, ) } else { - QueryPlanEntry::compile_bound_mapped( + crate::query_plan::compile_bound_mapped( query.query_id.clone(), canonical.clone(), &query.post_asap, diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index 0e2d1ceae..06bde49a0 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -1,853 +1,215 @@ -//! Authoritative backend-executable query DAG. -//! -//! ASAPPlanner owns semantic post-ASAP IR. Physical compilation binds every -//! maintained-summary leaf to one materialization and lowers edges to stable -//! node IDs. Serving executes this graph without reconstructing Planner IR or -//! searching for compatible materializations. - +//! Control-plane lowering from Planner IR to the shared installed query DAG. +//! Serving consumes asap_types::query_plan; compilation stays in this component. pub mod logical; - -use std::collections::{BTreeMap, BTreeSet}; -use std::rc::Rc; - -use planner_types::post_asap::{SketchQuery, SummaryExpr, SummaryFamilyType, SummaryNode}; +pub use asap_types::query_plan::*; +#[cfg(test)] +use asap_types::PolicyFingerprint; +use planner_types::post_asap::{SummaryExpr, SummaryFamilyType, SummaryNode}; use planner_types::pre_asap::Reduction; -use serde::{Deserialize, Serialize}; -use thiserror::Error; - -pub use asap_types::QueryLanguage; -use asap_types::{sds::SummaryDefinitionId, PolicyFingerprint}; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct QueryPlan { - pub plan_id: u64, - pub plan_version: u64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub clickhouse_context: Option, - pub entries: BTreeMap, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct ClickHousePlanningContext { - pub tables: std::collections::HashMap, - pub accuracy: planner_types::types::AccuracyTarget, -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct FixedEvaluationRange { - pub start_ms: u64, - pub end_ms: u64, - pub cumulative: bool, -} - -impl QueryPlan { - pub fn empty() -> Self { - Self { - plan_id: 0, - plan_version: 0, - clickhouse_context: None, - entries: BTreeMap::new(), - } - } - - pub fn lookup(&self, promql: &str) -> Result<&QueryPlanEntry, QueryPlanError> { - let identity = canonical_promql(promql)?; - self.lookup_canonical(QueryLanguage::PromQl, &identity) - } - - pub fn lookup_canonical( - &self, - language: QueryLanguage, - identity: &str, - ) -> Result<&QueryPlanEntry, QueryPlanError> { - let key = Self::catalog_key(language, identity); - self.entries - .get(&key) - .filter(|entry| entry.language == language) - .ok_or_else(|| QueryPlanError::QueryNotPlanned(identity.into())) - } - - pub fn catalog_key(language: QueryLanguage, identity: &str) -> String { - match language { - QueryLanguage::PromQl => identity.to_owned(), - QueryLanguage::MetricsQl => format!("metricsql:{identity}"), - QueryLanguage::ClickHouseSql => format!("clickhouse:{identity}"), - } - } - - pub fn lookup_clickhouse( - &self, - canonical_sql: &str, - ) -> Result<&QueryPlanEntry, QueryPlanError> { - self.lookup_canonical(QueryLanguage::ClickHouseSql, canonical_sql) - } - - /// Validate semantic bindings against the authoritative snapshot before use. - pub fn validate_against_catalog( - &self, - catalog: &crate::physical::summary_catalog::SummaryCatalog, - ) -> Result<(), QueryPlanError> { - catalog - .validate() - .map_err(|error| QueryPlanError::Invalid(error.to_string()))?; - if self.plan_id != catalog.plan_id || self.plan_version != catalog.plan_version { - return Err(QueryPlanError::Invalid( - "QueryPlan and SummaryCatalog have different plan identity/version".into(), - )); - } - let available = catalog - .materializations - .keys() - .copied() - .map(Into::into) - .collect(); - self.validate(&available)?; - for entry in self.entries.values() { - for binding in entry.materialization_bindings() { - let identity = catalog - .materializations - .get(&binding.materialization) - .ok_or_else(|| { - QueryPlanError::Invalid( - "query binding references absent catalog materialization".into(), - ) - })?; - let _data = &catalog.data_descriptors[&identity.data_descriptor_id]; - if binding.window_ms == 0 { - return Err(QueryPlanError::Invalid( - "zero physical pane duration".into(), - )); - } - if binding.pane_origin_ms != identity.pane_origin_ms { - return Err(QueryPlanError::Invalid( - "query pane origin differs from catalog definition".into(), - )); - } - } - for node in entry.nodes.values() { - let QueryPlanNode::ExactReadout { input, readout } = node else { - continue; - }; - if !matches!(readout, ExactReadout::Increase | ExactReadout::Rate) { - continue; - } - let Some(QueryPlanNode::ReadMaterialization { binding }) = entry.nodes.get(input) - else { - return Err(QueryPlanError::Invalid( - "counter readout must directly consume one catalog materialization".into(), - )); - }; - let identity = &catalog.materializations[&binding.materialization]; - let descriptor = &catalog.summary_descriptors[&identity.summary_descriptor_id]; - if !matches!( - descriptor.fidelity, - asap_types::sds::FidelityGuarantee::ExactCounter { - full_pane_coverage_required: true, - .. - } - ) { - return Err(QueryPlanError::Invalid( - "rate/increase binding does not reference an exact counter SDS".into(), - )); - } - } - } - Ok(()) - } - - pub fn validate(&self, available: &BTreeSet) -> Result<(), QueryPlanError> { - if self.plan_id != 0 && self.plan_version == 0 { - return Err(QueryPlanError::Invalid( - "non-bootstrap QueryPlan has zero plan_version".into(), - )); - } - for (identity, entry) in &self.entries { - let expected = Self::catalog_key(entry.language, &entry.canonical_query); - if identity != &expected { - return Err(QueryPlanError::Invalid(format!( - "query map key `{identity}` differs from entry identity `{}`", - entry.canonical_query - ))); - } - match entry.language { - QueryLanguage::PromQl | QueryLanguage::MetricsQl - if entry.fixed_evaluation.is_some() => - { - return Err(QueryPlanError::Invalid( - "PromQL query entry carries a ClickHouse fixed evaluation range".into(), - )); - } - QueryLanguage::ClickHouseSql => { - if self.clickhouse_context.is_none() { - return Err(QueryPlanError::Invalid( - "ClickHouse query entry has no planning context".into(), - )); - } - let Some(range) = entry.fixed_evaluation else { - return Err(QueryPlanError::Invalid( - "ClickHouse query entry has no fixed evaluation range".into(), - )); - }; - if range.end_ms <= range.start_ms { - return Err(QueryPlanError::Invalid( - "ClickHouse query entry has an empty evaluation range".into(), - )); - } - } - QueryLanguage::PromQl | QueryLanguage::MetricsQl => {} - } - entry.validate(available)?; - } - Ok(()) - } -} - -pub use asap_types::executable_plan::QueryNodeId; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct QueryPlanEntry { - #[serde(default)] - pub language: QueryLanguage, - pub query_id: String, - #[serde(alias = "canonical_promql")] - pub canonical_query: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub fixed_evaluation: Option, - pub root: QueryNodeId, - pub nodes: BTreeMap, - pub instant: InstantExecution, - pub fallback: FallbackPolicy, -} +use std::collections::BTreeMap; +#[cfg(test)] +use std::collections::BTreeSet; +use std::rc::Rc; -fn topological_order( - root: QueryNodeId, - nodes: &BTreeMap, -) -> Result, QueryPlanError> { - fn visit( - id: QueryNodeId, - nodes: &BTreeMap, - visiting: &mut BTreeSet, - visited: &mut BTreeSet, - out: &mut Vec, - ) -> Result<(), QueryPlanError> { - if visited.contains(&id) { - return Ok(()); - } - if !visiting.insert(id) { - return Err(QueryPlanError::Invalid(format!( - "cycle detected at query node {}", - id.0 - ))); - } - let node = nodes - .get(&id) - .ok_or_else(|| QueryPlanError::Invalid(format!("missing query node {}", id.0)))?; - for input in node.inputs() { - visit(*input, nodes, visiting, visited, out)?; - } - visiting.remove(&id); - visited.insert(id); - out.push(id); - Ok(()) - } - let mut out = Vec::with_capacity(nodes.len()); - visit( +pub fn compile_bound( + query_id: String, + canonical_query: String, + root: &Rc, + instant: InstantExecution, + fallback: FallbackPolicy, + bind: F, +) -> Result +where + F: FnMut( + &Rc, + &SummaryFamilyType, + ) -> Result, +{ + compile_bound_mapped( + query_id, + canonical_query, root, - nodes, - &mut BTreeSet::new(), - &mut BTreeSet::new(), - &mut out, - )?; - Ok(out) -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct InstantExecution { - pub lookback_ms: u64, - pub full_history: bool, - pub cumulative_readout: bool, -} - -impl QueryPlanEntry { - /// Replace an explicit planner fallback cut with a typed external-exact leaf. - /// The control plane chooses the cut; serving only executes the published DAG. - pub fn bind_external_exact_leaf( - &mut self, - node_id: QueryNodeId, - request: ExternalExactRequest, - ) -> Result<(), QueryPlanError> { - if request.language != self.language { - return Err(QueryPlanError::Invalid( - "external exact language differs from its query plan".into(), - )); - } - if !request.input_contracts.is_empty() { - return Err(QueryPlanError::Invalid( - "leaf binding cannot declare DAG input contracts".into(), - )); - } - match self.nodes.get(&node_id) { - Some(QueryPlanNode::ExactFallback { .. }) => {} - Some(_) => { - return Err(QueryPlanError::Invalid( - "external exact binding must replace a planner fallback cut".into(), - )) - } - None => { - return Err(QueryPlanError::Invalid( - "external exact cut node is absent".into(), - )) - } - } - self.nodes.insert( - node_id, - QueryPlanNode::ExternalExact { - request, - inputs: Vec::new(), - }, - ); - Ok(()) - } - - /// Materializations this executable DAG reads, in stable node order. - /// Serving uses this set for readiness accounting; it never performs a - /// catalog candidate search to reconstruct dependencies. - pub fn materialization_bindings(&self) -> Vec<&MaterializationBinding> { - self.nodes - .values() - .filter_map(|node| match node { - QueryPlanNode::ReadMaterialization { binding } => Some(binding), - _ => None, - }) - .collect() - } - - pub fn topological_order(&self) -> Result, QueryPlanError> { - topological_order(self.root, &self.nodes) - } - - pub fn topological_order_from( - &self, - root: QueryNodeId, - ) -> Result, QueryPlanError> { - topological_order(root, &self.nodes) - } - - pub fn compile_bound( - query_id: String, - canonical_query: String, - root: &Rc, - instant: InstantExecution, - fallback: FallbackPolicy, - bind: F, - ) -> Result - where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - { - Self::compile_bound_mapped( - query_id, - canonical_query, - root, - instant, - fallback, - bind, - |_, _| {}, - ) - } - - pub fn compile_bound_mapped( - query_id: String, - canonical_query: String, - root: &Rc, - instant: InstantExecution, - fallback: FallbackPolicy, - mut bind: F, - mut lowered: G, - ) -> Result - where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - G: FnMut(&Rc, QueryNodeId), - { - let mut compiler = DagCompiler { - next_id: 0, - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - bind: &mut bind, - logical_source: None, - preserve_relational: false, - lowered: Some(&mut lowered), - }; - let root = compiler.lower(root)?; - Ok(Self { - language: QueryLanguage::PromQl, - query_id, - canonical_query, - fixed_evaluation: None, - root, - nodes: compiler.nodes, - instant, - fallback, - }) - } - - pub fn compile_bound_relational( - query_id: String, - canonical_query: String, - root: &Rc, - fixed_evaluation: FixedEvaluationRange, - instant: InstantExecution, - fallback: FallbackPolicy, - bind: F, - ) -> Result - where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - { - Self::compile_bound_relational_mapped( - query_id, - canonical_query, - root, - fixed_evaluation, - instant, - fallback, - bind, - |_, _| {}, - ) - } - - /// Preserve Planner-to-runtime node identities for installed SQL DAGs. - pub fn compile_bound_relational_mapped( - query_id: String, - canonical_query: String, - root: &Rc, - fixed_evaluation: FixedEvaluationRange, - instant: InstantExecution, - fallback: FallbackPolicy, - mut bind: F, - mut lowered: G, - ) -> Result - where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - G: FnMut(&Rc, QueryNodeId), - { - let mut compiler = DagCompiler { - next_id: 0, - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - bind: &mut bind, - logical_source: None, - preserve_relational: true, - lowered: Some(&mut lowered), - }; - let root = compiler.lower(root)?; - Ok(Self { - language: QueryLanguage::ClickHouseSql, - query_id, - canonical_query, - fixed_evaluation: Some(fixed_evaluation), - root, - nodes: compiler.nodes, - instant, - fallback, - }) - } - - /// Compile selected summary nodes and verified native residuals into one DAG. - /// This is a distinct physical alternative; native execution remains available. - pub fn compile_bound_composable( - query_id: String, - canonical_query: String, - root: &Rc, - instant: InstantExecution, - fallback: FallbackPolicy, - bind: F, - ) -> Result - where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - { - Self::compile_bound_composable_mapped( - query_id, - canonical_query, - root, - instant, - fallback, - bind, - |_, _| {}, - ) - } - - /// Compile a composable query while exposing the stable mapping from - /// Planner semantic nodes to installed query nodes. The control-plane - /// physical compiler uses this to persist backend placement without - /// relying on pointer values or reconstructing query shape later. - pub fn compile_bound_composable_mapped( - query_id: String, - canonical_query: String, - root: &Rc, - instant: InstantExecution, - fallback: FallbackPolicy, - mut bind: F, - mut lowered: G, - ) -> Result - where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - G: FnMut(&Rc, QueryNodeId), - { - let mut compiler = DagCompiler { - next_id: 0, - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - bind: &mut bind, - logical_source: Some(canonical_query.clone()), - preserve_relational: false, - lowered: Some(&mut lowered), - }; - let root = compiler.lower(root)?; - let mut entry = Self { - language: QueryLanguage::PromQl, - query_id, - canonical_query, - fixed_evaluation: None, - root, - nodes: compiler.nodes, - instant, - fallback, - }; - logical::finalize_residuals(&mut entry)?; - Ok(entry) - } - - /// Validate references, bindings, reachability, and cycles before activation. - pub fn validate(&self, available: &BTreeSet) -> Result<(), QueryPlanError> { - if !self.nodes.contains_key(&self.root) { - return Err(QueryPlanError::Invalid(format!( - "query `{}` has missing root {}", - self.query_id, self.root.0 - ))); - } - for (id, node) in &self.nodes { - if let QueryPlanNode::Logical { operator, inputs } = node { - operator.validate(inputs.len())?; - } - if matches!(node, QueryPlanNode::Scalar { value } if !value.is_finite()) { - return Err(QueryPlanError::Invalid("non-finite scalar constant".into())); - } - if let QueryPlanNode::ExternalExact { request, inputs } = node { - if request.expression.trim().is_empty() { - return Err(QueryPlanError::Invalid( - "external exact expression must not be empty".into(), - )); - } - if request.input_contracts.len() != inputs.len() { - return Err(QueryPlanError::Invalid( - "external exact input contracts must match DAG inputs".into(), - )); - } - if request.input_contracts.iter().any(|contract| { - matches!(contract, ExternalExactInput::CandidateMembership { item_label } if item_label.is_empty()) - }) { - return Err(QueryPlanError::Invalid( - "external exact candidate item label must not be empty".into(), - )); - } - } - if let QueryPlanNode::CandidateTopK { - k, completeness, .. - } = node - { - if *k == 0 { - return Err(QueryPlanError::Invalid( - "CandidateTopK requires k > 0".into(), - )); - } - if matches!( - completeness, - CandidateCompleteness::Certified { guarantee } - if guarantee.metric - != planner_types::post_asap::ErrorMetric::TopKMembership - || guarantee.bound.evaluate().is_none() - || guarantee.failure_probability.evaluate().is_none() - ) { - return Err(QueryPlanError::Invalid( - "invalid CandidateTopK completeness certificate".into(), - )); - } - } - for input in node.inputs() { - if !self.nodes.contains_key(input) { - return Err(QueryPlanError::Invalid(format!( - "query `{}` node {} references missing input {}", - self.query_id, id.0, input.0 - ))); - } - } - if let QueryPlanNode::ReadMaterialization { binding } = node { - if binding.readout_lookback_ms == Some(0) { - return Err(QueryPlanError::Invalid( - "zero semantic readout lookback".into(), - )); - } - if !available.contains(&binding.materialization.fingerprint()) { - return Err(QueryPlanError::Invalid(format!( - "query `{}` node {} references absent materialization {}", - self.query_id, - id.0, - binding.materialization.as_u64() - ))); - } - } - } - let order = self.topological_order()?; - if order.len() != self.nodes.len() { - return Err(QueryPlanError::Invalid(format!( - "query `{}` contains unreachable nodes", - self.query_id - ))); - } - Ok(()) - } -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum FallbackPolicy { - ExactBackend, - Reject, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct MaterializationBinding { - pub materialization: SummaryDefinitionId, - /// Query operator grouping applied while folding those SIDs. - pub output_grouping: PhysicalGrouping, - /// Labels whose values form an item identity inside a keyed sketch. - #[serde(default, alias = "itemLabels", skip_serializing_if = "Vec::is_empty")] - pub item_labels: Vec, - pub window_ms: u64, - /// Unix millisecond timestamp on the materialized pane-boundary grid. - /// Legacy plans deserialize this as unknown and fall back at read time. - #[serde( - default, - alias = "paneOriginMs", - skip_serializing_if = "Option::is_none" - )] - pub pane_origin_ms: Option, - /// Semantic query lookback, independent of the physical pane duration. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub readout_lookback_ms: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "mode", content = "keys", rename_all = "snake_case")] -pub enum PhysicalGrouping { - PerEntity, - Reduce(Vec), + instant, + fallback, + bind, + |_, _| {}, + ) } -/// Result shape promised by an external exact engine. The backend uses this -/// contract to type-check downstream DAG nodes without depending on an -/// engine-specific response envelope. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum ExternalExactOutput { - Scalar, - InstantVector, - RangeVector, - Relation { schema: serde_json::Value }, -} - -/// How an ordinary DAG input constrains an external exact evaluation. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum ExternalExactInput { - CandidateMembership { item_label: String }, -} - -/// Language-neutral request contract for an exact subtree. Evaluation time is -/// inherited from the containing QueryPlanEntry, avoiding a second time-range -/// envelope that could drift from the installed query plan. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct ExternalExactRequest { - pub language: QueryLanguage, - pub expression: String, - pub output: ExternalExactOutput, - /// Engine parameters forwarded without embedding transport details in the DAG. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub parameters: BTreeMap, - /// Optional parameter names populated from the query entry's evaluation range. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub start_parameter: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub end_parameter: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub input_contracts: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(tag = "op", rename_all = "snake_case", deny_unknown_fields)] -pub enum QueryPlanNode { - RelationalJoin { - inputs: [QueryNodeId; 2], - join_kind: planner_types::pre_asap::JoinKind, - pred: serde_json::Value, - left_schema: planner_types::post_asap::SummarySchema, - right_schema: planner_types::post_asap::SummarySchema, - output_schema: planner_types::post_asap::SummarySchema, - }, - Relational { - input: QueryNodeId, - /// Serialized planner-owned operation. Keeping the wire form here makes - /// the published catalog Send + Sync even though the planner AST uses Rc. - operation: serde_json::Value, - input_schema: planner_types::post_asap::SummarySchema, - output_schema: planner_types::post_asap::SummarySchema, - }, - Logical { - operator: logical::LogicalOperator, - inputs: Vec, - }, - Scalar { - value: f64, - }, - Binary { - inputs: [QueryNodeId; 2], - operator: planner_types::pre_asap::ArithmeticOpKind, - }, - ReduceSum { - input: QueryNodeId, - grouping: PhysicalGrouping, - }, - ReadMaterialization { - binding: MaterializationBinding, - }, - SummaryEstimate { - input: QueryNodeId, - query: QueryReadout, - }, - ExactReadout { - input: QueryNodeId, - readout: ExactReadout, - }, - SummaryMerge { - inputs: Vec, - }, - /// Use an approximate heap only as a membership sidecar, then rerank the - /// matching exact counter readouts. `inputs[0]` is candidate membership; - /// `inputs[1]` is the authoritative exact value vector. - CandidateTopK { - inputs: [QueryNodeId; 2], - k: u64, - grouping: logical::Grouping, - completeness: CandidateCompleteness, - }, - /// An exact subtree evaluated outside ASAP. Its results enter the query DAG - /// like any other node output and may depend on summary-produced inputs. - ExternalExact { - request: ExternalExactRequest, - inputs: Vec, - }, - ExactFallback { - reason: String, - }, -} - -impl QueryPlanNode { - pub fn inputs(&self) -> &[QueryNodeId] { - match self { - Self::Scalar { .. } | Self::ReadMaterialization { .. } | Self::ExactFallback { .. } => { - &[] - } - Self::Binary { inputs, .. } | Self::RelationalJoin { inputs, .. } => inputs, - Self::ReduceSum { input, .. } - | Self::Relational { input, .. } - | Self::SummaryEstimate { input, .. } - | Self::ExactReadout { input, .. } => std::slice::from_ref(input), - Self::SummaryMerge { inputs } - | Self::Logical { inputs, .. } - | Self::ExternalExact { inputs, .. } => inputs, - Self::CandidateTopK { inputs, .. } => inputs, - } - } +pub fn compile_bound_mapped( + query_id: String, + canonical_query: String, + root: &Rc, + instant: InstantExecution, + fallback: FallbackPolicy, + mut bind: F, + mut lowered: G, +) -> Result +where + F: FnMut( + &Rc, + &SummaryFamilyType, + ) -> Result, + G: FnMut(&Rc, QueryNodeId), +{ + let mut compiler = DagCompiler { + next_id: 0, + nodes: BTreeMap::new(), + seen: BTreeMap::new(), + bind: &mut bind, + logical_source: None, + preserve_relational: false, + lowered: Some(&mut lowered), + }; + let root = compiler.lower(root)?; + Ok(QueryPlanEntry { + language: QueryLanguage::PromQl, + query_id, + canonical_query, + fixed_evaluation: None, + root, + nodes: compiler.nodes, + instant, + fallback, + }) } -pub use planner_types::post_asap::CandidateCompleteness; - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum ExactReadout { - Sum, - Count, - Increase, - Rate, - Max, +pub fn compile_bound_relational( + query_id: String, + canonical_query: String, + root: &Rc, + fixed_evaluation: FixedEvaluationRange, + instant: InstantExecution, + fallback: FallbackPolicy, + bind: F, +) -> Result +where + F: FnMut( + &Rc, + &SummaryFamilyType, + ) -> Result, +{ + compile_bound_relational_mapped( + query_id, + canonical_query, + root, + fixed_evaluation, + instant, + fallback, + bind, + |_, _| {}, + ) } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum QueryReadout { - FrequencyL2, - FrequencyEntropy, - Quantile { - q: f64, - }, - PointCount { - key: planner_types::pre_asap::ColumnRef, - value: Option, - }, - Cardinality, - TopK { - k: usize, - }, +/// Preserve Planner-to-runtime node identities for installed SQL DAGs. +pub fn compile_bound_relational_mapped( + query_id: String, + canonical_query: String, + root: &Rc, + fixed_evaluation: FixedEvaluationRange, + instant: InstantExecution, + fallback: FallbackPolicy, + mut bind: F, + mut lowered: G, +) -> Result +where + F: FnMut( + &Rc, + &SummaryFamilyType, + ) -> Result, + G: FnMut(&Rc, QueryNodeId), +{ + let mut compiler = DagCompiler { + next_id: 0, + nodes: BTreeMap::new(), + seen: BTreeMap::new(), + bind: &mut bind, + logical_source: None, + preserve_relational: true, + lowered: Some(&mut lowered), + }; + let root = compiler.lower(root)?; + Ok(QueryPlanEntry { + language: QueryLanguage::ClickHouseSql, + query_id, + canonical_query, + fixed_evaluation: Some(fixed_evaluation), + root, + nodes: compiler.nodes, + instant, + fallback, + }) } -impl From for QueryReadout { - fn from(query: SketchQuery) -> Self { - match query { - SketchQuery::FrequencyL2 => Self::FrequencyL2, - SketchQuery::FrequencyEntropy => Self::FrequencyEntropy, - SketchQuery::Quantile { q } => Self::Quantile { q }, - SketchQuery::PointCount { key, value } => Self::PointCount { key, value }, - SketchQuery::Cardinality => Self::Cardinality, - SketchQuery::TopK { k } => Self::TopK { k }, - } - } +/// Compile selected summary nodes and verified native residuals into one DAG. +/// This is a distinct physical alternative; native execution remains available. +pub fn compile_bound_composable( + query_id: String, + canonical_query: String, + root: &Rc, + instant: InstantExecution, + fallback: FallbackPolicy, + bind: F, +) -> Result +where + F: FnMut( + &Rc, + &SummaryFamilyType, + ) -> Result, +{ + compile_bound_composable_mapped( + query_id, + canonical_query, + root, + instant, + fallback, + bind, + |_, _| {}, + ) } -impl From for SketchQuery { - fn from(query: QueryReadout) -> Self { - match query { - QueryReadout::FrequencyL2 => Self::FrequencyL2, - QueryReadout::FrequencyEntropy => Self::FrequencyEntropy, - QueryReadout::Quantile { q } => Self::Quantile { q }, - QueryReadout::PointCount { key, value } => Self::PointCount { key, value }, - QueryReadout::Cardinality => Self::Cardinality, - QueryReadout::TopK { k } => Self::TopK { k }, - } - } +/// Compile a composable query while exposing the stable mapping from +/// Planner semantic nodes to installed query nodes. The control-plane +/// physical compiler uses this to persist backend placement without +/// relying on pointer values or reconstructing query shape later. +pub fn compile_bound_composable_mapped( + query_id: String, + canonical_query: String, + root: &Rc, + instant: InstantExecution, + fallback: FallbackPolicy, + mut bind: F, + mut lowered: G, +) -> Result +where + F: FnMut( + &Rc, + &SummaryFamilyType, + ) -> Result, + G: FnMut(&Rc, QueryNodeId), +{ + let mut compiler = DagCompiler { + next_id: 0, + nodes: BTreeMap::new(), + seen: BTreeMap::new(), + bind: &mut bind, + logical_source: Some(canonical_query.clone()), + preserve_relational: false, + lowered: Some(&mut lowered), + }; + let root = compiler.lower(root)?; + let mut entry = QueryPlanEntry { + language: QueryLanguage::PromQl, + query_id, + canonical_query, + fixed_evaluation: None, + root, + nodes: compiler.nodes, + instant, + fallback, + }; + logical::finalize_residuals(&mut entry)?; + Ok(entry) } struct DagCompiler<'a, F> { @@ -1625,24 +987,6 @@ fn physical_grouping( Ok(PhysicalGrouping::Reduce(names)) } -#[derive(Debug, Error)] -pub enum QueryPlanError { - #[error("invalid PromQL query identity: {0}")] - InvalidPromql(String), - #[error("query is absent from the active QueryPlan: {0}")] - QueryNotPlanned(String), - #[error("post-ASAP DAG cannot be represented by the query executor: {0}")] - UnsupportedNode(String), - #[error("invalid QueryPlan: {0}")] - Invalid(String), -} - -pub fn canonical_promql(query: &str) -> Result { - promql_parser::parser::parse(query.trim()) - .map(|expr| expr.to_string()) - .map_err(|error| QueryPlanError::InvalidPromql(error.to_string())) -} - #[cfg(test)] mod tests { use super::*; diff --git a/control_plane/src/query_plan/logical.rs b/control_plane/src/query_plan/logical.rs index 05d474715..6b20a5a84 100644 --- a/control_plane/src/query_plan/logical.rs +++ b/control_plane/src/query_plan/logical.rs @@ -6,62 +6,13 @@ use promql_parser::{ label::MatchOp, parser::{self, Expr, LabelModifier, Offset, VectorSelector}, }; -use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum LogicalOperator { - /// A maximal exact scalar/vector subtree evaluated by Prometheus. - ExactSubquery { - query: String, - }, - /// Prometheus exact subtree whose selectors are restricted at runtime by - /// the candidate vector produced by its single input. - CandidateExactSubquery { - query: String, - item_label: String, - }, - Scan { - metric: Option, - matchers: Vec, - range_ms: Option, - offset_ms: i64, - }, - UnaryNegate, - VectorToScalar, - Aggregate { - operation: Aggregation, - grouping: Grouping, - }, - /// PromQL `topk(k, vector)` selection over values produced by the child. - /// This is distinct from a frequency-sketch TopK readout: any exact or - /// summary-backed instant-vector child may feed this query-time operator. - TopKSelection { - k: u64, - grouping: Grouping, - }, - Binary { - operation: BinaryOperation, - return_bool: bool, - }, - Temporal { - operation: TemporalOperation, - }, - Sort { - descending: bool, - }, - HistogramQuantile, - Subquery { - range_ms: u64, - step_ms: u64, - offset_ms: i64, - }, -} +pub use asap_types::query_plan::logical::*; /// Stable identity of a Planner-authorized materializable DAG leaf. This is a /// workload-selection key, not another physical materialization definition. -#[derive(Debug, Clone, Serialize, PartialEq)] +#[derive(Debug, Clone, serde::Serialize, PartialEq)] struct MaterializationCandidateIdentity { metric: String, matchers: Vec, @@ -69,63 +20,6 @@ struct MaterializationCandidateIdentity { offset_ms: i64, operation: TemporalOperation, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct Grouping { - pub labels: Vec, - pub without: bool, -} -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct LabelMatcher { - pub name: String, - pub value: String, - pub operation: LabelMatch, -} -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum LabelMatch { - Equal, - NotEqual, - Regex, - NotRegex, -} -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum Aggregation { - Sum, - Max, - Min, - Avg, - Count, -} -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum BinaryOperation { - Add, - Sub, - Mul, - Div, - Mod, - Pow, - Equal, - NotEqual, - Less, - LessEqual, - Greater, - GreaterEqual, -} -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum TemporalOperation { - Rate, - Increase, - Avg, - Max, - Min, - Sum, - Count, -} fn invalid(message: impl Into) -> QueryPlanError { QueryPlanError::Invalid(message.into()) @@ -143,46 +37,6 @@ fn offset(value: &Option) -> Result { } } -impl LogicalOperator { - pub fn validate(&self, inputs: usize) -> Result<(), QueryPlanError> { - let expected = match self { - Self::Scan { .. } | Self::ExactSubquery { .. } => 0, - Self::CandidateExactSubquery { .. } => 1, - Self::Binary { .. } | Self::HistogramQuantile => 2, - _ => 1, - }; - if inputs != expected { - return Err(invalid("logical operator input arity mismatch")); - } - if matches!( - self, - Self::Scan { - range_ms: Some(0), - .. - } - ) { - return Err(invalid("zero range")); - } - if let Self::ExactSubquery { query } | Self::CandidateExactSubquery { query, .. } = self { - let parsed = parser::parse(query).map_err(|e| invalid(e.to_string()))?; - if matches!(parsed, Expr::MatrixSelector(_) | Expr::Subquery(_)) { - return Err(invalid( - "exact subtree boundary must return scalar or instant vector", - )); - } - } - if let Self::Subquery { - range_ms, step_ms, .. - } = self - { - if *range_ms == 0 || *step_ms == 0 || range_ms / step_ms > 100_000 { - return Err(invalid("invalid or excessive subquery grid")); - } - } - Ok(()) - } -} - struct Lower { nodes: BTreeMap, seen: BTreeMap, @@ -409,53 +263,32 @@ impl Lower { } } -impl QueryPlanEntry { - /// Lower a Planner-authorized native residual into typed backend operations. - /// Callers retain a separate external-native alternative for cost comparison. - pub fn compile_logical( - query_id: String, - canonical_query: String, - instant: InstantExecution, - fallback: FallbackPolicy, - ) -> Result { - let expr = parser::parse(&canonical_query).map_err(|e| invalid(e.to_string()))?; - let mut lower = Lower { - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - }; - let root = lower.lower(&expr)?; - let entry = Self { - language: super::QueryLanguage::PromQl, - query_id, - canonical_query, - fixed_evaluation: None, - root, - nodes: lower.nodes, - instant, - fallback, - }; - entry.validate(&Default::default())?; - Ok(entry) - } - /// Promote only a wholly native entry; never discard selected summary bindings. - pub fn lower_native_residual(&self) -> Result { - if self.nodes.len() != 1 - || !matches!( - self.nodes.get(&self.root), - Some(QueryPlanNode::ExactFallback { .. }) - ) - { - return Err(invalid( - "logical residual promotion requires a whole native root", - )); - } - Self::compile_logical( - self.query_id.clone(), - self.canonical_query.clone(), - self.instant, - self.fallback, - ) - } +/// Lower a Planner-authorized native residual into typed backend operations. +/// Callers retain a separate external-native alternative for cost comparison. +pub fn compile_logical( + query_id: String, + canonical_query: String, + instant: InstantExecution, + fallback: FallbackPolicy, +) -> Result { + let expr = parser::parse(&canonical_query).map_err(|e| invalid(e.to_string()))?; + let mut lower = Lower { + nodes: BTreeMap::new(), + seen: BTreeMap::new(), + }; + let root = lower.lower(&expr)?; + let entry = QueryPlanEntry { + language: super::QueryLanguage::PromQl, + query_id, + canonical_query, + fixed_evaluation: None, + root, + nodes: lower.nodes, + instant, + fallback, + }; + entry.validate(&Default::default())?; + Ok(entry) } /// Match residuals by semantic IR equality, not display text or source names. @@ -555,7 +388,7 @@ mod tests { serde_json::from_str(include_str!("../../tests/fixtures/o11y_queries.json")).unwrap(); for row in corpus["queries"].as_array().unwrap() { let query = row["query"].as_str().unwrap(); - let entry = QueryPlanEntry::compile_logical( + let entry = crate::query_plan::logical::compile_logical( row["id"].as_str().unwrap().into(), query.into(), instant(), @@ -587,7 +420,7 @@ mod tests { #[test] fn repeated_subexpressions_share_node_identity() { // Serialized edges must retain CSE rather than duplicating raw work. - let entry = QueryPlanEntry::compile_logical( + let entry = crate::query_plan::logical::compile_logical( "q".into(), "sum(up) / sum(up)".into(), instant(), @@ -633,7 +466,7 @@ mod tests { 3, ), ] { - let entry = QueryPlanEntry::compile_logical( + let entry = crate::query_plan::logical::compile_logical( "topk".into(), query.into(), instant(), @@ -652,7 +485,7 @@ mod tests { #[test] fn topk_keeps_unsupported_child_as_exact_leaf() { - let entry = QueryPlanEntry::compile_logical( + let entry = crate::query_plan::logical::compile_logical( "topk-subquery".into(), "topk(3, label_replace(memory_bytes, \"dst\", \"$1\", \"src\", \"(.*)\"))".into(), instant(), @@ -681,7 +514,7 @@ mod tests { ("topk by (cluster) (2, m)", vec!["cluster"], false), ("topk without (pod) (2, m)", vec!["pod"], true), ] { - let entry = QueryPlanEntry::compile_logical( + let entry = crate::query_plan::logical::compile_logical( "topk-group".into(), query.into(), instant(), @@ -799,7 +632,7 @@ mod hybrid_tests { .unwrap(); let selected = crate::planner_selection::select_summary_default(&canonical).unwrap(); let entry = - QueryPlanEntry::compile_bound_composable( + crate::query_plan::compile_bound_composable( "hybrid".into(), query.into(), &selected, diff --git a/control_plane/tests/o11y_exact_fallback.rs b/control_plane/tests/o11y_exact_fallback.rs index 6f0b12d80..e43508a03 100644 --- a/control_plane/tests/o11y_exact_fallback.rs +++ b/control_plane/tests/o11y_exact_fallback.rs @@ -26,7 +26,7 @@ fn o11y_non_summary_roots_preserve_exact_query_semantics() { }; assert_eq!(original.as_ref(), &expr, "query semantics changed: {query}"); assert_eq!(node.schema.fields.len(), expr.output_schema().unwrap().columns.len()); - let executable = control_plane::query_plan::QueryPlanEntry::compile_bound( + let executable = control_plane::query_plan::compile_bound( "fixture".into(), query.into(), &node, control_plane::query_plan::InstantExecution { lookback_ms: 300_000, full_history: false, cumulative_readout: false, diff --git a/control_plane/tests/offline_evidence.rs b/control_plane/tests/offline_evidence.rs index 769bdd6bd..b36bd3fdb 100644 --- a/control_plane/tests/offline_evidence.rs +++ b/control_plane/tests/offline_evidence.rs @@ -346,9 +346,7 @@ fn incompatible_evidence_preserves_deployment_behavior() { /// fallback even though the warm tier now supports exact additive binaries. #[test] fn binary_summary_has_explicit_warm_tier_fallback() { - use control_plane::query_plan::{ - FallbackPolicy, InstantExecution, QueryPlanEntry, QueryPlanNode, - }; + use control_plane::query_plan::{FallbackPolicy, InstantExecution, QueryPlanNode}; use planner_types::{post_asap::BinaryOperator, pre_asap::BinaryOpKind}; let child = bound(&model()); let root = std::rc::Rc::new(SummaryNode { @@ -363,7 +361,7 @@ fn binary_summary_has_explicit_warm_tier_fallback() { schema: child.schema.clone(), guarantee: None, }); - let plan = QueryPlanEntry::compile_bound( + let plan = control_plane::query_plan::compile_bound( "test".into(), "left / right".into(), &root, diff --git a/crates/asap_types/Cargo.toml b/crates/asap_types/Cargo.toml index 9a51159b1..e5f35c4cc 100644 --- a/crates/asap_types/Cargo.toml +++ b/crates/asap_types/Cargo.toml @@ -4,6 +4,7 @@ version.workspace = true edition.workspace = true [dependencies] +promql-parser.workspace = true tracing.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/crates/asap_types/src/lib.rs b/crates/asap_types/src/lib.rs index ccf828726..73fe5aa8e 100644 --- a/crates/asap_types/src/lib.rs +++ b/crates/asap_types/src/lib.rs @@ -31,3 +31,5 @@ pub use routing_index::RoutingIndex; pub use storage_backend::*; pub mod precompute_plan; + +pub mod query_plan; diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs new file mode 100644 index 000000000..945608afe --- /dev/null +++ b/crates/asap_types/src/query_plan.rs @@ -0,0 +1,677 @@ +//! Shared installed QueryPlan contract and activation validation. +//! +//! ASAPPlanner owns semantic post-ASAP IR. Physical compilation binds every +//! maintained-summary leaf to one materialization and lowers edges to stable +//! node IDs. Serving executes this graph without reconstructing Planner IR or +//! searching for compatible materializations. + +pub mod logical; + +use std::collections::{BTreeMap, BTreeSet}; + +use planner_types::post_asap::SketchQuery; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +pub use crate::QueryLanguage; +use crate::{sds::SummaryDefinitionId, PolicyFingerprint}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct QueryPlan { + pub plan_id: u64, + pub plan_version: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub clickhouse_context: Option, + pub entries: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ClickHousePlanningContext { + pub tables: std::collections::HashMap, + pub accuracy: planner_types::types::AccuracyTarget, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct FixedEvaluationRange { + pub start_ms: u64, + pub end_ms: u64, + pub cumulative: bool, +} + +impl QueryPlan { + pub fn empty() -> Self { + Self { + plan_id: 0, + plan_version: 0, + clickhouse_context: None, + entries: BTreeMap::new(), + } + } + + pub fn lookup(&self, promql: &str) -> Result<&QueryPlanEntry, QueryPlanError> { + let identity = canonical_promql(promql)?; + self.lookup_canonical(QueryLanguage::PromQl, &identity) + } + + pub fn lookup_canonical( + &self, + language: QueryLanguage, + identity: &str, + ) -> Result<&QueryPlanEntry, QueryPlanError> { + let key = Self::catalog_key(language, identity); + self.entries + .get(&key) + .filter(|entry| entry.language == language) + .ok_or_else(|| QueryPlanError::QueryNotPlanned(identity.into())) + } + + pub fn catalog_key(language: QueryLanguage, identity: &str) -> String { + match language { + QueryLanguage::PromQl => identity.to_owned(), + QueryLanguage::MetricsQl => format!("metricsql:{identity}"), + QueryLanguage::ClickHouseSql => format!("clickhouse:{identity}"), + } + } + + pub fn lookup_clickhouse( + &self, + canonical_sql: &str, + ) -> Result<&QueryPlanEntry, QueryPlanError> { + self.lookup_canonical(QueryLanguage::ClickHouseSql, canonical_sql) + } + + /// Validate semantic bindings against the authoritative snapshot before use. + pub fn validate_against_catalog( + &self, + catalog: &crate::summary_catalog::SummaryCatalog, + ) -> Result<(), QueryPlanError> { + catalog + .validate() + .map_err(|error| QueryPlanError::Invalid(error.to_string()))?; + if self.plan_id != catalog.plan_id || self.plan_version != catalog.plan_version { + return Err(QueryPlanError::Invalid( + "QueryPlan and SummaryCatalog have different plan identity/version".into(), + )); + } + let available = catalog + .materializations + .keys() + .copied() + .map(Into::into) + .collect(); + self.validate(&available)?; + for entry in self.entries.values() { + for binding in entry.materialization_bindings() { + let identity = catalog + .materializations + .get(&binding.materialization) + .ok_or_else(|| { + QueryPlanError::Invalid( + "query binding references absent catalog materialization".into(), + ) + })?; + let _data = &catalog.data_descriptors[&identity.data_descriptor_id]; + if binding.window_ms == 0 { + return Err(QueryPlanError::Invalid( + "zero physical pane duration".into(), + )); + } + if binding.pane_origin_ms != identity.pane_origin_ms { + return Err(QueryPlanError::Invalid( + "query pane origin differs from catalog definition".into(), + )); + } + } + for node in entry.nodes.values() { + let QueryPlanNode::ExactReadout { input, readout } = node else { + continue; + }; + if !matches!(readout, ExactReadout::Increase | ExactReadout::Rate) { + continue; + } + let Some(QueryPlanNode::ReadMaterialization { binding }) = entry.nodes.get(input) + else { + return Err(QueryPlanError::Invalid( + "counter readout must directly consume one catalog materialization".into(), + )); + }; + let identity = &catalog.materializations[&binding.materialization]; + let descriptor = &catalog.summary_descriptors[&identity.summary_descriptor_id]; + if !matches!( + descriptor.fidelity, + crate::sds::FidelityGuarantee::ExactCounter { + full_pane_coverage_required: true, + .. + } + ) { + return Err(QueryPlanError::Invalid( + "rate/increase binding does not reference an exact counter SDS".into(), + )); + } + } + } + Ok(()) + } + + pub fn validate(&self, available: &BTreeSet) -> Result<(), QueryPlanError> { + if self.plan_id != 0 && self.plan_version == 0 { + return Err(QueryPlanError::Invalid( + "non-bootstrap QueryPlan has zero plan_version".into(), + )); + } + for (identity, entry) in &self.entries { + let expected = Self::catalog_key(entry.language, &entry.canonical_query); + if identity != &expected { + return Err(QueryPlanError::Invalid(format!( + "query map key `{identity}` differs from entry identity `{}`", + entry.canonical_query + ))); + } + match entry.language { + QueryLanguage::PromQl | QueryLanguage::MetricsQl + if entry.fixed_evaluation.is_some() => + { + return Err(QueryPlanError::Invalid( + "PromQL query entry carries a ClickHouse fixed evaluation range".into(), + )); + } + QueryLanguage::ClickHouseSql => { + if self.clickhouse_context.is_none() { + return Err(QueryPlanError::Invalid( + "ClickHouse query entry has no planning context".into(), + )); + } + let Some(range) = entry.fixed_evaluation else { + return Err(QueryPlanError::Invalid( + "ClickHouse query entry has no fixed evaluation range".into(), + )); + }; + if range.end_ms <= range.start_ms { + return Err(QueryPlanError::Invalid( + "ClickHouse query entry has an empty evaluation range".into(), + )); + } + } + QueryLanguage::PromQl | QueryLanguage::MetricsQl => {} + } + entry.validate(available)?; + } + Ok(()) + } +} + +pub use crate::executable_plan::QueryNodeId; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct QueryPlanEntry { + #[serde(default)] + pub language: QueryLanguage, + pub query_id: String, + #[serde(alias = "canonical_promql")] + pub canonical_query: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fixed_evaluation: Option, + pub root: QueryNodeId, + pub nodes: BTreeMap, + pub instant: InstantExecution, + pub fallback: FallbackPolicy, +} + +fn topological_order( + root: QueryNodeId, + nodes: &BTreeMap, +) -> Result, QueryPlanError> { + fn visit( + id: QueryNodeId, + nodes: &BTreeMap, + visiting: &mut BTreeSet, + visited: &mut BTreeSet, + out: &mut Vec, + ) -> Result<(), QueryPlanError> { + if visited.contains(&id) { + return Ok(()); + } + if !visiting.insert(id) { + return Err(QueryPlanError::Invalid(format!( + "cycle detected at query node {}", + id.0 + ))); + } + let node = nodes + .get(&id) + .ok_or_else(|| QueryPlanError::Invalid(format!("missing query node {}", id.0)))?; + for input in node.inputs() { + visit(*input, nodes, visiting, visited, out)?; + } + visiting.remove(&id); + visited.insert(id); + out.push(id); + Ok(()) + } + let mut out = Vec::with_capacity(nodes.len()); + visit( + root, + nodes, + &mut BTreeSet::new(), + &mut BTreeSet::new(), + &mut out, + )?; + Ok(out) +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct InstantExecution { + pub lookback_ms: u64, + pub full_history: bool, + pub cumulative_readout: bool, +} + +impl QueryPlanEntry { + /// Replace an explicit planner fallback cut with a typed external-exact leaf. + /// The control plane chooses the cut; serving only executes the published DAG. + pub fn bind_external_exact_leaf( + &mut self, + node_id: QueryNodeId, + request: ExternalExactRequest, + ) -> Result<(), QueryPlanError> { + if request.language != self.language { + return Err(QueryPlanError::Invalid( + "external exact language differs from its query plan".into(), + )); + } + if !request.input_contracts.is_empty() { + return Err(QueryPlanError::Invalid( + "leaf binding cannot declare DAG input contracts".into(), + )); + } + match self.nodes.get(&node_id) { + Some(QueryPlanNode::ExactFallback { .. }) => {} + Some(_) => { + return Err(QueryPlanError::Invalid( + "external exact binding must replace a planner fallback cut".into(), + )) + } + None => { + return Err(QueryPlanError::Invalid( + "external exact cut node is absent".into(), + )) + } + } + self.nodes.insert( + node_id, + QueryPlanNode::ExternalExact { + request, + inputs: Vec::new(), + }, + ); + Ok(()) + } + + /// Materializations this executable DAG reads, in stable node order. + /// Serving uses this set for readiness accounting; it never performs a + /// catalog candidate search to reconstruct dependencies. + pub fn materialization_bindings(&self) -> Vec<&MaterializationBinding> { + self.nodes + .values() + .filter_map(|node| match node { + QueryPlanNode::ReadMaterialization { binding } => Some(binding), + _ => None, + }) + .collect() + } + + pub fn topological_order(&self) -> Result, QueryPlanError> { + topological_order(self.root, &self.nodes) + } + + pub fn topological_order_from( + &self, + root: QueryNodeId, + ) -> Result, QueryPlanError> { + topological_order(root, &self.nodes) + } + + /// Validate references, bindings, reachability, and cycles before activation. + pub fn validate(&self, available: &BTreeSet) -> Result<(), QueryPlanError> { + if !self.nodes.contains_key(&self.root) { + return Err(QueryPlanError::Invalid(format!( + "query `{}` has missing root {}", + self.query_id, self.root.0 + ))); + } + for (id, node) in &self.nodes { + if let QueryPlanNode::Logical { operator, inputs } = node { + operator.validate(inputs.len())?; + } + if matches!(node, QueryPlanNode::Scalar { value } if !value.is_finite()) { + return Err(QueryPlanError::Invalid("non-finite scalar constant".into())); + } + if let QueryPlanNode::ExternalExact { request, inputs } = node { + if request.expression.trim().is_empty() { + return Err(QueryPlanError::Invalid( + "external exact expression must not be empty".into(), + )); + } + if request.input_contracts.len() != inputs.len() { + return Err(QueryPlanError::Invalid( + "external exact input contracts must match DAG inputs".into(), + )); + } + if request.input_contracts.iter().any(|contract| { + matches!(contract, ExternalExactInput::CandidateMembership { item_label } if item_label.is_empty()) + }) { + return Err(QueryPlanError::Invalid( + "external exact candidate item label must not be empty".into(), + )); + } + } + if let QueryPlanNode::CandidateTopK { + k, completeness, .. + } = node + { + if *k == 0 { + return Err(QueryPlanError::Invalid( + "CandidateTopK requires k > 0".into(), + )); + } + if matches!( + completeness, + CandidateCompleteness::Certified { guarantee } + if guarantee.metric + != planner_types::post_asap::ErrorMetric::TopKMembership + || guarantee.bound.evaluate().is_none() + || guarantee.failure_probability.evaluate().is_none() + ) { + return Err(QueryPlanError::Invalid( + "invalid CandidateTopK completeness certificate".into(), + )); + } + } + for input in node.inputs() { + if !self.nodes.contains_key(input) { + return Err(QueryPlanError::Invalid(format!( + "query `{}` node {} references missing input {}", + self.query_id, id.0, input.0 + ))); + } + } + if let QueryPlanNode::ReadMaterialization { binding } = node { + if binding.readout_lookback_ms == Some(0) { + return Err(QueryPlanError::Invalid( + "zero semantic readout lookback".into(), + )); + } + if !available.contains(&binding.materialization.fingerprint()) { + return Err(QueryPlanError::Invalid(format!( + "query `{}` node {} references absent materialization {}", + self.query_id, + id.0, + binding.materialization.as_u64() + ))); + } + } + } + let order = self.topological_order()?; + if order.len() != self.nodes.len() { + return Err(QueryPlanError::Invalid(format!( + "query `{}` contains unreachable nodes", + self.query_id + ))); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum FallbackPolicy { + ExactBackend, + Reject, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct MaterializationBinding { + pub materialization: SummaryDefinitionId, + /// Query operator grouping applied while folding those SIDs. + pub output_grouping: PhysicalGrouping, + /// Labels whose values form an item identity inside a keyed sketch. + #[serde(default, alias = "itemLabels", skip_serializing_if = "Vec::is_empty")] + pub item_labels: Vec, + pub window_ms: u64, + /// Unix millisecond timestamp on the materialized pane-boundary grid. + /// Legacy plans deserialize this as unknown and fall back at read time. + #[serde( + default, + alias = "paneOriginMs", + skip_serializing_if = "Option::is_none" + )] + pub pane_origin_ms: Option, + /// Semantic query lookback, independent of the physical pane duration. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub readout_lookback_ms: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "mode", content = "keys", rename_all = "snake_case")] +pub enum PhysicalGrouping { + PerEntity, + Reduce(Vec), +} + +/// Result shape promised by an external exact engine. The backend uses this +/// contract to type-check downstream DAG nodes without depending on an +/// engine-specific response envelope. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum ExternalExactOutput { + Scalar, + InstantVector, + RangeVector, + Relation { schema: serde_json::Value }, +} + +/// How an ordinary DAG input constrains an external exact evaluation. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum ExternalExactInput { + CandidateMembership { item_label: String }, +} + +/// Language-neutral request contract for an exact subtree. Evaluation time is +/// inherited from the containing QueryPlanEntry, avoiding a second time-range +/// envelope that could drift from the installed query plan. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct ExternalExactRequest { + pub language: QueryLanguage, + pub expression: String, + pub output: ExternalExactOutput, + /// Engine parameters forwarded without embedding transport details in the DAG. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub parameters: BTreeMap, + /// Optional parameter names populated from the query entry's evaluation range. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub start_parameter: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub end_parameter: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub input_contracts: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "op", rename_all = "snake_case", deny_unknown_fields)] +pub enum QueryPlanNode { + RelationalJoin { + inputs: [QueryNodeId; 2], + join_kind: planner_types::pre_asap::JoinKind, + pred: serde_json::Value, + left_schema: planner_types::post_asap::SummarySchema, + right_schema: planner_types::post_asap::SummarySchema, + output_schema: planner_types::post_asap::SummarySchema, + }, + Relational { + input: QueryNodeId, + /// Serialized planner-owned operation. Keeping the wire form here makes + /// the published catalog Send + Sync even though the planner AST uses Rc. + operation: serde_json::Value, + input_schema: planner_types::post_asap::SummarySchema, + output_schema: planner_types::post_asap::SummarySchema, + }, + Logical { + operator: logical::LogicalOperator, + inputs: Vec, + }, + Scalar { + value: f64, + }, + Binary { + inputs: [QueryNodeId; 2], + operator: planner_types::pre_asap::ArithmeticOpKind, + }, + ReduceSum { + input: QueryNodeId, + grouping: PhysicalGrouping, + }, + ReadMaterialization { + binding: MaterializationBinding, + }, + SummaryEstimate { + input: QueryNodeId, + query: QueryReadout, + }, + ExactReadout { + input: QueryNodeId, + readout: ExactReadout, + }, + SummaryMerge { + inputs: Vec, + }, + /// Use an approximate heap only as a membership sidecar, then rerank the + /// matching exact counter readouts. `inputs[0]` is candidate membership; + /// `inputs[1]` is the authoritative exact value vector. + CandidateTopK { + inputs: [QueryNodeId; 2], + k: u64, + grouping: logical::Grouping, + completeness: CandidateCompleteness, + }, + /// An exact subtree evaluated outside ASAP. Its results enter the query DAG + /// like any other node output and may depend on summary-produced inputs. + ExternalExact { + request: ExternalExactRequest, + inputs: Vec, + }, + ExactFallback { + reason: String, + }, +} + +impl QueryPlanNode { + pub fn inputs(&self) -> &[QueryNodeId] { + match self { + Self::Scalar { .. } | Self::ReadMaterialization { .. } | Self::ExactFallback { .. } => { + &[] + } + Self::Binary { inputs, .. } | Self::RelationalJoin { inputs, .. } => inputs, + Self::ReduceSum { input, .. } + | Self::Relational { input, .. } + | Self::SummaryEstimate { input, .. } + | Self::ExactReadout { input, .. } => std::slice::from_ref(input), + Self::SummaryMerge { inputs } + | Self::Logical { inputs, .. } + | Self::ExternalExact { inputs, .. } => inputs, + Self::CandidateTopK { inputs, .. } => inputs, + } + } +} + +pub use planner_types::post_asap::CandidateCompleteness; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ExactReadout { + Sum, + Count, + Increase, + Rate, + Max, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum QueryReadout { + FrequencyL2, + FrequencyEntropy, + Quantile { + q: f64, + }, + PointCount { + key: planner_types::pre_asap::ColumnRef, + value: Option, + }, + Cardinality, + TopK { + k: usize, + }, +} + +impl From for QueryReadout { + fn from(query: SketchQuery) -> Self { + match query { + SketchQuery::FrequencyL2 => Self::FrequencyL2, + SketchQuery::FrequencyEntropy => Self::FrequencyEntropy, + SketchQuery::Quantile { q } => Self::Quantile { q }, + SketchQuery::PointCount { key, value } => Self::PointCount { key, value }, + SketchQuery::Cardinality => Self::Cardinality, + SketchQuery::TopK { k } => Self::TopK { k }, + } + } +} + +impl From for SketchQuery { + fn from(query: QueryReadout) -> Self { + match query { + QueryReadout::FrequencyL2 => Self::FrequencyL2, + QueryReadout::FrequencyEntropy => Self::FrequencyEntropy, + QueryReadout::Quantile { q } => Self::Quantile { q }, + QueryReadout::PointCount { key, value } => Self::PointCount { key, value }, + QueryReadout::Cardinality => Self::Cardinality, + QueryReadout::TopK { k } => Self::TopK { k }, + } + } +} + +#[derive(Debug, Error)] +pub enum QueryPlanError { + #[error("invalid PromQL query identity: {0}")] + InvalidPromql(String), + #[error("query is absent from the active QueryPlan: {0}")] + QueryNotPlanned(String), + #[error("post-ASAP DAG cannot be represented by the query executor: {0}")] + UnsupportedNode(String), + #[error("invalid QueryPlan: {0}")] + Invalid(String), +} + +pub fn canonical_promql(query: &str) -> Result { + promql_parser::parser::parse(query.trim()) + .map(|expr| expr.to_string()) + .map_err(|error| QueryPlanError::InvalidPromql(error.to_string())) +} + +#[cfg(test)] +mod contract_tests { + // Installed plans cross producer/query threads without Planner Rc state. + #[test] + fn installed_query_contract_is_send_sync() { + fn assert_send_sync() {} + assert_send_sync::(); + assert_send_sync::(); + } +} diff --git a/crates/asap_types/src/query_plan/logical.rs b/crates/asap_types/src/query_plan/logical.rs new file mode 100644 index 000000000..5c3f6e138 --- /dev/null +++ b/crates/asap_types/src/query_plan/logical.rs @@ -0,0 +1,155 @@ +//! Typed installed residual operators; no Planner selection or AST lowering. +use super::QueryPlanError; +use promql_parser::parser::{self, Expr}; +use serde::{Deserialize, Serialize}; +fn invalid(message: impl Into) -> QueryPlanError { + QueryPlanError::Invalid(message.into()) +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum LogicalOperator { + /// A maximal exact scalar/vector subtree evaluated by Prometheus. + ExactSubquery { + query: String, + }, + /// Prometheus exact subtree whose selectors are restricted at runtime by + /// the candidate vector produced by its single input. + CandidateExactSubquery { + query: String, + item_label: String, + }, + Scan { + metric: Option, + matchers: Vec, + range_ms: Option, + offset_ms: i64, + }, + UnaryNegate, + VectorToScalar, + Aggregate { + operation: Aggregation, + grouping: Grouping, + }, + /// PromQL `topk(k, vector)` selection over values produced by the child. + /// This is distinct from a frequency-sketch TopK readout: any exact or + /// summary-backed instant-vector child may feed this query-time operator. + TopKSelection { + k: u64, + grouping: Grouping, + }, + Binary { + operation: BinaryOperation, + return_bool: bool, + }, + Temporal { + operation: TemporalOperation, + }, + Sort { + descending: bool, + }, + HistogramQuantile, + Subquery { + range_ms: u64, + step_ms: u64, + offset_ms: i64, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct Grouping { + pub labels: Vec, + pub without: bool, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct LabelMatcher { + pub name: String, + pub value: String, + pub operation: LabelMatch, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum LabelMatch { + Equal, + NotEqual, + Regex, + NotRegex, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum Aggregation { + Sum, + Max, + Min, + Avg, + Count, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum BinaryOperation { + Add, + Sub, + Mul, + Div, + Mod, + Pow, + Equal, + NotEqual, + Less, + LessEqual, + Greater, + GreaterEqual, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum TemporalOperation { + Rate, + Increase, + Avg, + Max, + Min, + Sum, + Count, +} + +impl LogicalOperator { + pub fn validate(&self, inputs: usize) -> Result<(), QueryPlanError> { + let expected = match self { + Self::Scan { .. } | Self::ExactSubquery { .. } => 0, + Self::CandidateExactSubquery { .. } => 1, + Self::Binary { .. } | Self::HistogramQuantile => 2, + _ => 1, + }; + if inputs != expected { + return Err(invalid("logical operator input arity mismatch")); + } + if matches!( + self, + Self::Scan { + range_ms: Some(0), + .. + } + ) { + return Err(invalid("zero range")); + } + if let Self::ExactSubquery { query } | Self::CandidateExactSubquery { query, .. } = self { + let parsed = parser::parse(query).map_err(|e| invalid(e.to_string()))?; + if matches!(parsed, Expr::MatrixSelector(_) | Expr::Subquery(_)) { + return Err(invalid( + "exact subtree boundary must return scalar or instant vector", + )); + } + } + if let Self::Subquery { + range_ms, step_ms, .. + } = self + { + if *range_ms == 0 || *step_ms == 0 || range_ms / step_ms > 100_000 { + return Err(invalid("invalid or excessive subquery grid")); + } + } + Ok(()) + } +} diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index 5004547db..618fda6ee 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -817,7 +817,7 @@ mod tests { rules: Vec::new(), }, runtime_config: Arc::new(streaming), - query_plan: Arc::new(control_plane::query_plan::QueryPlan::empty()), + query_plan: Arc::new(asap_types::query_plan::QueryPlan::empty()), storage_routing: Arc::new(BackendStorageRouting::empty()), }; HotReloadStreamingConfig::from_active(HotReloadActivePhysicalPlan::new(active)) diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 087b3d586..8454890aa 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -2726,7 +2726,7 @@ mod tests { rules: Vec::new(), }, runtime_config: streaming_config.clone(), - query_plan: Arc::new(control_plane::query_plan::QueryPlan { + query_plan: Arc::new(asap_types::query_plan::QueryPlan { plan_id: 7, plan_version: 1, clickhouse_context: None, @@ -6208,13 +6208,13 @@ async fn handle_post_physical_plan( .query_plan .entries .values() - .filter(|entry| entry.language == control_plane::query_plan::QueryLanguage::MetricsQl) + .filter(|entry| entry.language == asap_types::query_plan::QueryLanguage::MetricsQl) .count(); let clickhouse_plan_count = active .query_plan .entries .values() - .filter(|entry| entry.language == control_plane::query_plan::QueryLanguage::ClickHouseSql) + .filter(|entry| entry.language == asap_types::query_plan::QueryLanguage::ClickHouseSql) .count(); let plan_version = active.plan_version(); let now = unix_time_ms(); @@ -6290,7 +6290,7 @@ async fn handle_activate_physical_plan( .query_plan .entries .values() - .filter(|entry| entry.language == control_plane::query_plan::QueryLanguage::ClickHouseSql) + .filter(|entry| entry.language == asap_types::query_plan::QueryLanguage::ClickHouseSql) .count(); if old.plan_id() != 0 { let draining_id = old.plan_id(); @@ -7171,7 +7171,7 @@ mod catalog_install_tests { let before = handle.snapshot(); let mut candidate = request(); candidate.query_plan.clickhouse_context = - Some(control_plane::query_plan::ClickHousePlanningContext { + Some(asap_types::query_plan::ClickHousePlanningContext { tables: Default::default(), accuracy: planner_types::types::AccuracyTarget::Exact, }); @@ -7180,8 +7180,8 @@ mod catalog_install_tests { .entries .pop_first() .expect("fixture has a query entry"); - entry.language = control_plane::query_plan::QueryLanguage::ClickHouseSql; - entry.fixed_evaluation = Some(control_plane::query_plan::FixedEvaluationRange { + entry.language = asap_types::query_plan::QueryLanguage::ClickHouseSql; + entry.fixed_evaluation = Some(asap_types::query_plan::FixedEvaluationRange { start_ms: 0, end_ms: 1_000, cumulative: true, @@ -7191,7 +7191,7 @@ mod catalog_install_tests { .nodes .values_mut() .find_map(|node| match node { - control_plane::query_plan::QueryPlanNode::ReadMaterialization { binding } => { + asap_types::query_plan::QueryPlanNode::ReadMaterialization { binding } => { Some(binding) } _ => None, @@ -7264,7 +7264,7 @@ mod catalog_install_tests { .values_mut() .flat_map(|entry| entry.nodes.values_mut()) .find_map(|node| match node { - control_plane::query_plan::QueryPlanNode::ReadMaterialization { binding } => { + asap_types::query_plan::QueryPlanNode::ReadMaterialization { binding } => { Some(binding) } _ => None, @@ -7279,22 +7279,20 @@ mod catalog_install_tests { let mut request = request(); let key = request.query_plan.entries.keys().next().unwrap().clone(); let mut entry = request.query_plan.entries.remove(&key).unwrap(); - entry.language = control_plane::query_plan::QueryLanguage::MetricsQl; + entry.language = asap_types::query_plan::QueryLanguage::MetricsQl; let binding = entry .nodes .values_mut() .find_map(|node| match node { - control_plane::query_plan::QueryPlanNode::ReadMaterialization { binding } => { + asap_types::query_plan::QueryPlanNode::ReadMaterialization { binding } => { Some(binding) } _ => None, }) .expect("demo has maintained summaries"); binding.pane_origin_ms = Some(1); - let key = control_plane::query_plan::QueryPlan::catalog_key( - entry.language, - &entry.canonical_query, - ); + let key = + asap_types::query_plan::QueryPlan::catalog_key(entry.language, &entry.canonical_query); request.query_plan.entries.insert(key, entry); let error = install(request).unwrap_err(); assert!(error.contains("pane origin"), "{error}"); diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index c751ddf9f..9d2960b07 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -779,7 +779,7 @@ async fn main() -> Result<()> { precompute_plan: initial_precompute_plan, transmission_plan: initial_transmission_plan, runtime_config: streaming_config.clone(), - query_plan: Arc::new(control_plane::query_plan::QueryPlan::empty()), + query_plan: Arc::new(asap_types::query_plan::QueryPlan::empty()), storage_routing: Arc::new( data_plane::storage_engines::types::BackendStorageRouting::empty(), ), diff --git a/data_plane/src/precompute_engine/maintenance_runtime.rs b/data_plane/src/precompute_engine/maintenance_runtime.rs index 7a284b480..a802dc582 100644 --- a/data_plane/src/precompute_engine/maintenance_runtime.rs +++ b/data_plane/src/precompute_engine/maintenance_runtime.rs @@ -618,7 +618,7 @@ mod tests { let binding = BackendExecutableBinding { nodes: BTreeMap::new(), query_sink: PostAsapNodeId(2), - query_plan_sink: control_plane::query_plan::QueryNodeId(2), + query_plan_sink: asap_types::query_plan::QueryNodeId(2), precompute_sinks: vec![PostAsapNodeId(1)], }; let adapter = OperatorAdapter { @@ -771,12 +771,12 @@ mod tests { ( PostAsapNodeId(2), BackendNodeBinding::Query { - query_node: control_plane::query_plan::QueryNodeId(9), + query_node: asap_types::query_plan::QueryNodeId(9), }, ), ]), query_sink: PostAsapNodeId(2), - query_plan_sink: control_plane::query_plan::QueryNodeId(9), + query_plan_sink: asap_types::query_plan::QueryNodeId(9), precompute_sinks: vec![PostAsapNodeId(1)], }; bundle.precompute_plan.executable_dags = BTreeMap::from([( @@ -900,12 +900,12 @@ mod tests { .chain([( PostAsapNodeId(4), BackendNodeBinding::Query { - query_node: control_plane::query_plan::QueryNodeId(9), + query_node: asap_types::query_plan::QueryNodeId(9), }, )]) .collect(), query_sink: PostAsapNodeId(4), - query_plan_sink: control_plane::query_plan::QueryNodeId(9), + query_plan_sink: asap_types::query_plan::QueryNodeId(9), precompute_sinks: vec![PostAsapNodeId(3)], }; let source = sum(2.0); @@ -973,12 +973,12 @@ mod tests { ( PostAsapNodeId(2), BackendNodeBinding::Query { - query_node: control_plane::query_plan::QueryNodeId(9), + query_node: asap_types::query_plan::QueryNodeId(9), }, ), ]), query_sink: PostAsapNodeId(2), - query_plan_sink: control_plane::query_plan::QueryNodeId(9), + query_plan_sink: asap_types::query_plan::QueryNodeId(9), precompute_sinks: vec![PostAsapNodeId(1)], }; let adapter = OperatorAdapter { diff --git a/data_plane/src/precompute_engine/subdag_scheduler.rs b/data_plane/src/precompute_engine/subdag_scheduler.rs index 6f632f44c..6a52b88df 100644 --- a/data_plane/src/precompute_engine/subdag_scheduler.rs +++ b/data_plane/src/precompute_engine/subdag_scheduler.rs @@ -185,12 +185,12 @@ mod tests { .chain([( PostAsapNodeId(4), BackendNodeBinding::Query { - query_node: control_plane::query_plan::QueryNodeId(9), + query_node: asap_types::query_plan::QueryNodeId(9), }, )]) .collect(), query_sink: PostAsapNodeId(4), - query_plan_sink: control_plane::query_plan::QueryNodeId(9), + query_plan_sink: asap_types::query_plan::QueryNodeId(9), precompute_sinks: vec![PostAsapNodeId(3)], } } @@ -376,7 +376,7 @@ mod tests { ( PostAsapNodeId(0), BackendNodeBinding::Query { - query_node: control_plane::query_plan::QueryNodeId(1), + query_node: asap_types::query_plan::QueryNodeId(1), }, ), ( @@ -389,7 +389,7 @@ mod tests { .into_iter() .collect(), query_sink: PostAsapNodeId(0), - query_plan_sink: control_plane::query_plan::QueryNodeId(1), + query_plan_sink: asap_types::query_plan::QueryNodeId(1), precompute_sinks: vec![PostAsapNodeId(1)], }; assert!(matches!( diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs index a4a023a4f..4973f5778 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs @@ -64,14 +64,14 @@ impl CatalogClickHouseAccelerator { async fn prepare_external_exact( &self, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, start_ms: u64, end_ms: u64, request_context: &ClickHouseQueryRequest, ) -> Result { let mut prepared = PreparedExternalLeaves::new(); let leaves = entry.nodes.iter().filter_map(|(id, node)| match node { - control_plane::query_plan::QueryPlanNode::ExternalExact { request, inputs } + asap_types::query_plan::QueryPlanNode::ExternalExact { request, inputs } if request.language == asap_types::QueryLanguage::ClickHouseSql && inputs.is_empty() => { @@ -85,7 +85,7 @@ impl CatalogClickHouseAccelerator { .as_ref() .ok_or_else(|| "ClickHouse exact subtree endpoint unavailable".to_owned())?; let schema = match &bound.output { - control_plane::query_plan::ExternalExactOutput::Relation { schema } => { + asap_types::query_plan::ExternalExactOutput::Relation { schema } => { serde_json::from_value(schema.clone()).map_err(|error| error.to_string())? } _ => return Err("ClickHouse exact subtree must produce a relation".into()), @@ -272,14 +272,14 @@ mod tests { precompute_engine::operators::SumAccumulator, storage_engines::sketch_db::index::{AggKind, Capability, SketchInstanceMetadata}, }; - use asap_types::summary_catalog::SummaryCatalog; - use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; - use axum::http::Method; - use control_plane::query_plan::{ + use asap_types::query_plan::{ ClickHousePlanningContext, ExactReadout, ExternalExactOutput, ExternalExactRequest, FallbackPolicy, FixedEvaluationRange, InstantExecution, MaterializationBinding, PhysicalGrouping, QueryLanguage, QueryNodeId, QueryPlan, QueryPlanEntry, QueryPlanNode, }; + use asap_types::summary_catalog::SummaryCatalog; + use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + use axum::http::Method; struct FixedExactSubtree; diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs index 8215af484..cf40ef43d 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs @@ -9,8 +9,8 @@ use crate::{ }, storage_engines::sketch_db::index::SketchStore, }; +use asap_types::query_plan::{QueryNodeId, QueryPlanEntry, QueryPlanNode}; use asap_types::summary_catalog::SummaryCatalog; -use control_plane::query_plan::{QueryNodeId, QueryPlanEntry, QueryPlanNode}; use planner_types::post_asap::ValueOperation; use std::collections::{BTreeMap, BTreeSet}; @@ -65,8 +65,7 @@ fn execute_relation_subtree( if request.language != asap_types::QueryLanguage::ClickHouseSql { return Err("ClickHouse DAG contains an external leaf for another language".into()); } - let control_plane::query_plan::ExternalExactOutput::Relation { schema } = - &request.output + let asap_types::query_plan::ExternalExactOutput::Relation { schema } = &request.output else { return Err("ClickHouse external leaf must declare relation output".into()); }; @@ -249,7 +248,7 @@ pub fn execute_sql_dag_with_external( Some(QueryPlanNode::Relational { output_schema, .. }) | Some(QueryPlanNode::RelationalJoin { output_schema, .. }) => output_schema.clone(), Some(QueryPlanNode::ExternalExact { request, .. }) => { - let control_plane::query_plan::ExternalExactOutput::Relation { schema } = + let asap_types::query_plan::ExternalExactOutput::Relation { schema } = &request.output else { return ClickHouseDagOutcome::Fallback(ClickHouseDagFallback::UnsupportedPlan( diff --git a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs index 4ae79947a..6bb5877e2 100644 --- a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs +++ b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs @@ -3,10 +3,10 @@ //! query execution checks only reachable IDs and their requested capabilities. use std::collections::{BTreeMap, BTreeSet}; +use asap_types::query_plan::{ExactReadout, QueryPlanEntry, QueryPlanNode, QueryReadout}; use asap_types::sds::{SummaryDefinitionId, SummaryDescriptor, SummaryOperator}; use asap_types::summary_catalog::SummaryCatalog; use asap_types::AggregationType; -use control_plane::query_plan::{ExactReadout, QueryPlanEntry, QueryPlanNode, QueryReadout}; use crate::query_engines::EngineError; diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 766947b8c..6e909c092 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -10,7 +10,7 @@ struct QueryReadinessRequirement { } fn readiness_requirement( - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, ) -> QueryReadinessRequirement { let bindings = entry.materialization_bindings(); let mut materializations = bindings @@ -128,10 +128,7 @@ impl ASAPQueryEngine { })?; let planned = physical .query_plan - .lookup_canonical( - control_plane::query_plan::QueryLanguage::MetricsQl, - identity, - ) + .lookup_canonical(asap_types::query_plan::QueryLanguage::MetricsQl, identity) .map_err(|error| { crate::query_engines::EngineError::capability_miss("query_plan", error.to_string()) })?; @@ -160,10 +157,7 @@ impl ASAPQueryEngine { })?; let planned = physical .query_plan - .lookup_canonical( - control_plane::query_plan::QueryLanguage::MetricsQl, - identity, - ) + .lookup_canonical(asap_types::query_plan::QueryLanguage::MetricsQl, identity) .map_err(|error| { crate::query_engines::EngineError::capability_miss("query_plan", error.to_string()) })?; @@ -201,7 +195,7 @@ impl ASAPQueryEngine { async fn prepare_logical( &self, physical: &crate::storage_engines::types::ActivePhysicalPlan, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, times: &[u64], ) -> Result { super::catalog_resolver::validate_entry( @@ -266,7 +260,7 @@ impl ASAPQueryEngine { fn execute_logical_entry( &self, physical: &crate::storage_engines::types::ActivePhysicalPlan, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, leaves: &super::logical_dag::PreparedLeaves, at: u64, ) -> Result< @@ -379,7 +373,7 @@ impl ASAPQueryEngine { async fn execute_logical_range( &self, physical: &crate::storage_engines::types::ActivePhysicalPlan, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, start: u64, end: u64, step: u64, @@ -636,10 +630,7 @@ impl ASAPQueryEngine { if let Some(physical) = self.physical_plan_snapshot() { if let Ok(entry) = physical.query_plan.lookup(query) { if entry.nodes.values().any(|node| { - matches!( - node, - control_plane::query_plan::QueryPlanNode::Logical { .. } - ) + matches!(node, asap_types::query_plan::QueryPlanNode::Logical { .. }) }) { return self .execute_logical_range(&physical, entry, start_ms, end_ms, step_ms) @@ -3591,7 +3582,7 @@ mod range_stitch_tests { #[tokio::test] async fn active_metricsql_entry_reaches_the_shared_dag_executor() { - use control_plane::query_plan::{ + use asap_types::query_plan::{ FallbackPolicy, InstantExecution, QueryLanguage, QueryNodeId, QueryPlanEntry, QueryPlanNode, }; @@ -3603,7 +3594,7 @@ mod range_stitch_tests { let mut plan = snapshot.compile().unwrap(); let identity = control_plane::query_plan::canonical_promql("1 + 2").unwrap(); plan.query_plan.entries.insert( - control_plane::query_plan::QueryPlan::catalog_key(QueryLanguage::MetricsQl, &identity), + asap_types::query_plan::QueryPlan::catalog_key(QueryLanguage::MetricsQl, &identity), QueryPlanEntry { language: QueryLanguage::MetricsQl, query_id: "vm-scalar".into(), diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index 70221144b..aa1537b3e 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -1,7 +1,7 @@ //! Fetch installed exact cuts from Prometheus before composing them with ASAP state. use super::logical_dag::{PreparedLeaf, PreparedLeaves, Value}; use crate::query_engines::EngineError; -use control_plane::query_plan::{ +use asap_types::query_plan::{ logical::LogicalOperator, ExternalExactInput, ExternalExactRequest, QueryLanguage, QueryNodeId, QueryPlanEntry, QueryPlanNode, }; @@ -441,10 +441,10 @@ pub(super) async fn prepare_external( #[cfg(test)] mod tests { use super::*; - use control_plane::query_plan::{FallbackPolicy, InstantExecution}; + use asap_types::query_plan::{FallbackPolicy, InstantExecution}; fn entry(nodes: BTreeMap) -> QueryPlanEntry { QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "remote-cut".into(), canonical_query: "a / b".into(), fixed_evaluation: None, @@ -491,7 +491,7 @@ mod tests { request: ExternalExactRequest { language: QueryLanguage::MetricsQl, expression: "sum(rate(m[5m]))".into(), - output: control_plane::query_plan::ExternalExactOutput::InstantVector, + output: asap_types::query_plan::ExternalExactOutput::InstantVector, parameters: BTreeMap::new(), start_parameter: None, end_parameter: None, @@ -526,7 +526,7 @@ mod tests { request: ExternalExactRequest { language: QueryLanguage::PromQl, expression: query.into(), - output: control_plane::query_plan::ExternalExactOutput::InstantVector, + output: asap_types::query_plan::ExternalExactOutput::InstantVector, parameters: BTreeMap::new(), start_parameter: None, end_parameter: None, @@ -630,7 +630,7 @@ mod tests { #[tokio::test] async fn candidate_exact_is_discovered_and_prepared_behind_candidate_topk_root() { - use control_plane::query_plan::{logical::Grouping, CandidateCompleteness}; + use asap_types::query_plan::{logical::Grouping, CandidateCompleteness}; let mut entry = candidate_entry("sum by (job) (rate(m[5m]))"); entry.nodes.insert( QueryNodeId(2), @@ -751,7 +751,7 @@ mod tests { #[tokio::test] async fn exact_leaf_calls_prometheus_and_combines_with_prepared_summary() { // A successful exact branch remains an intermediate, not a whole-root fallback. - use control_plane::query_plan::logical::BinaryOperation; + use asap_types::query_plan::logical::BinaryOperation; use std::sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -866,7 +866,7 @@ mod tests { index::{Capability, SketchInstanceMetadata}, }; use crate::storage_engines::types::{KeyByLabelValues, Measurement}; - use control_plane::query_plan::{ + use asap_types::query_plan::{ logical::BinaryOperation, ExactReadout, MaterializationBinding, PhysicalGrouping, }; use std::sync::{ diff --git a/data_plane/src/query_engines/asap_query_engine/live_serve.rs b/data_plane/src/query_engines/asap_query_engine/live_serve.rs index aac4fca19..ccfb7333a 100644 --- a/data_plane/src/query_engines/asap_query_engine/live_serve.rs +++ b/data_plane/src/query_engines/asap_query_engine/live_serve.rs @@ -160,7 +160,7 @@ pub fn serve_instant_from_summary_executor( pub fn serve_from_query_plan( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, t0_ms: u64, t1_ms: u64, is_cumulative: bool, @@ -181,7 +181,7 @@ pub fn serve_from_query_plan( /// timestamps never leak into the public range response. pub fn serve_range_steps_from_query_plan( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, start_ms: u64, end_ms: u64, step_ms: u64, @@ -263,7 +263,7 @@ fn coverage_covers_closed_windows( pub fn serve_instant_from_query_plan( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, now_ms: u64, ) -> Result<(ASAPTierResult, u64), LoweringSkip> { if !summary_executor_live_enabled() { @@ -447,27 +447,27 @@ mod tests { Box::new(crate::precompute_engine::operators::SumAccumulator::with_sum(value)), ); } - let entry = control_plane::query_plan::QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + let entry = asap_types::query_plan::QueryPlanEntry { + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "q-sum".into(), canonical_query: "sum_over_time(bytes[1s])".into(), fixed_evaluation: None, - root: control_plane::query_plan::QueryNodeId(0), + root: asap_types::query_plan::QueryNodeId(0), nodes: BTreeMap::from([ ( - control_plane::query_plan::QueryNodeId(0), - control_plane::query_plan::QueryPlanNode::ExactReadout { - input: control_plane::query_plan::QueryNodeId(1), - readout: control_plane::query_plan::ExactReadout::Sum, + asap_types::query_plan::QueryNodeId(0), + asap_types::query_plan::QueryPlanNode::ExactReadout { + input: asap_types::query_plan::QueryNodeId(1), + readout: asap_types::query_plan::ExactReadout::Sum, }, ), ( - control_plane::query_plan::QueryNodeId(1), - control_plane::query_plan::QueryPlanNode::ReadMaterialization { - binding: control_plane::query_plan::MaterializationBinding { + asap_types::query_plan::QueryNodeId(1), + asap_types::query_plan::QueryPlanNode::ReadMaterialization { + binding: asap_types::query_plan::MaterializationBinding { item_labels: Vec::new(), materialization: policy.into(), - output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, + output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 1_000, pane_origin_ms: Some(0), readout_lookback_ms: Some(1_000), @@ -475,12 +475,12 @@ mod tests { }, ), ]), - instant: control_plane::query_plan::InstantExecution { + instant: asap_types::query_plan::InstantExecution { lookback_ms: 1_000, full_history: false, cumulative_readout: true, }, - fallback: control_plane::query_plan::FallbackPolicy::ExactBackend, + fallback: asap_types::query_plan::FallbackPolicy::ExactBackend, }; let result = serve_range_steps_from_query_plan(&idx, &entry, 1_000, 3_000, 1_000) diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs index b07ecf64a..dbb5eee54 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs @@ -4,12 +4,10 @@ use crate::query_engines::{ EngineError, }; use crate::storage_engines::types::KeyByLabelValues; -use control_plane::query_plan::logical::{ +use asap_types::query_plan::logical::{ Aggregation, BinaryOperation, Grouping, LogicalOperator, TemporalOperation, }; -use control_plane::query_plan::{ - CandidateCompleteness, QueryNodeId, QueryPlanEntry, QueryPlanNode, -}; +use asap_types::query_plan::{CandidateCompleteness, QueryNodeId, QueryPlanEntry, QueryPlanNode}; use std::collections::{BTreeMap, BTreeSet}; type Labels = BTreeMap; @@ -713,7 +711,7 @@ fn bucket_quantile(q: f64, mut b: Vec<(f64, f64)>) -> f64 { #[cfg(test)] mod topk_tests { use super::*; - use control_plane::query_plan::{FallbackPolicy, InstantExecution}; + use asap_types::query_plan::{FallbackPolicy, InstantExecution}; fn labels(items: &[(&str, &str)]) -> Labels { items @@ -789,7 +787,7 @@ mod topk_tests { #[test] fn installed_topk_combines_with_prometheus_exact_child() { - let mut entry = QueryPlanEntry::compile_logical( + let mut entry = control_plane::query_plan::logical::compile_logical( "hybrid-topk".into(), "topk(2, m)".into(), InstantExecution { @@ -856,7 +854,7 @@ mod topk_tests { let summary = QueryNodeId(0); let root = QueryNodeId(1); let entry = QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "summary-rate-topk".into(), canonical_query: "topk(2, rate(requests_total[5m]))".into(), fixed_evaluation: None, @@ -866,7 +864,7 @@ mod topk_tests { summary, QueryPlanNode::ExactReadout { input: QueryNodeId(99), - readout: control_plane::query_plan::ExactReadout::Rate, + readout: asap_types::query_plan::ExactReadout::Rate, }, ), ( @@ -985,7 +983,7 @@ mod topk_tests { let value_id = QueryNodeId(1); let root = QueryNodeId(2); let entry = QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "candidate-topk".into(), canonical_query: "topk(1, rate(requests_total[5m]))".into(), fixed_evaluation: None, diff --git a/data_plane/src/query_engines/asap_query_engine/physical_dag.rs b/data_plane/src/query_engines/asap_query_engine/physical_dag.rs index a917ff23a..ea6d9f63b 100644 --- a/data_plane/src/query_engines/asap_query_engine/physical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/physical_dag.rs @@ -6,7 +6,7 @@ use std::collections::BTreeMap; -use control_plane::query_plan::{QueryNodeId, QueryPlanEntry, QueryPlanNode}; +use asap_types::query_plan::{QueryNodeId, QueryPlanEntry, QueryPlanNode}; use thiserror::Error; pub trait QueryNodeRuntime { @@ -146,7 +146,7 @@ mod tests { use std::cell::RefCell; use std::collections::BTreeMap; - use control_plane::query_plan::{FallbackPolicy, InstantExecution, QueryReadout}; + use asap_types::query_plan::{FallbackPolicy, InstantExecution, QueryReadout}; use super::*; @@ -204,7 +204,7 @@ mod tests { .into_iter() .collect(); let entry = QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "q".into(), canonical_query: "up".into(), fixed_evaluation: None, @@ -277,7 +277,7 @@ mod tests { .into_iter() .collect(); let entry = QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "q".into(), canonical_query: "up".into(), fixed_evaluation: None, diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index a908595dd..e8a6a5111 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -5,7 +5,7 @@ use std::collections::BTreeMap; use crate::query_engines::asap_query_engine::summary_exec::{execute, ExecOutcome}; -use control_plane::query_plan::{QueryNodeId, QueryPlanNode}; +use asap_types::query_plan::{QueryNodeId, QueryPlanNode}; use control_plane::types_v2::AccuracyTarget; use crate::query_engines::asap_query_engine::physical_dag::{self, QueryNodeRuntime}; @@ -82,7 +82,7 @@ pub fn execute_post_asap_readout( /// Installed QueryPlan materialization resolution occurs before this legacy test helper. pub fn execute_query_plan_readout( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, t0_ms: u64, t1_ms: u64, is_cumulative: bool, @@ -92,8 +92,8 @@ pub fn execute_query_plan_readout( pub fn execute_query_plan_from_readout( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, - root: control_plane::query_plan::QueryNodeId, + entry: &asap_types::query_plan::QueryPlanEntry, + root: asap_types::query_plan::QueryNodeId, t0_ms: u64, t1_ms: u64, is_cumulative: bool, @@ -103,7 +103,7 @@ pub fn execute_query_plan_from_readout( pub fn execute_query_plan_instant( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, now_ms: u64, ) -> Result<(PostAsapReadoutOutcome, u64), LoweringSkip> { let t0_ms = if entry.instant.full_history { @@ -476,11 +476,11 @@ fn intersect_coverage(left: Option<(u64, u64)>, right: Option<(u64, u64)>) -> Op } fn reduce_sum_values( - grouping: &control_plane::query_plan::PhysicalGrouping, + grouping: &asap_types::query_plan::PhysicalGrouping, values: &[(BTreeMap, SummaryValue)], coverage: Option<(u64, u64)>, ) -> Result { - let control_plane::query_plan::PhysicalGrouping::Reduce(keys) = grouping else { + let asap_types::query_plan::PhysicalGrouping::Reduce(keys) = grouping else { return Ok(PhysicalQueryOutput::Value(values.to_vec(), coverage)); }; let mut groups = BTreeMap::new(); @@ -519,7 +519,7 @@ fn reduce_sum_values( fn execute_physical_query_plan( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, t0_ms: u64, t1_ms: u64, is_cumulative: bool, @@ -529,8 +529,8 @@ fn execute_physical_query_plan( fn execute_physical_query_payload( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, - root: control_plane::query_plan::QueryNodeId, + entry: &asap_types::query_plan::QueryPlanEntry, + root: asap_types::query_plan::QueryNodeId, t0_ms: u64, t1_ms: u64, is_cumulative: bool, @@ -784,7 +784,7 @@ mod tests { }) .collect::>(); let PhysicalQueryOutput::Value(result, _) = reduce_sum_values( - &control_plane::query_plan::PhysicalGrouping::Reduce(vec!["service".into()]), + &asap_types::query_plan::PhysicalGrouping::Reduce(vec!["service".into()]), &values, Some((1000, 2000)), ) @@ -899,21 +899,21 @@ mod tests { let node = plan_promql_to_post_asap(&idx, "count(unique_users)", accuracy()) .expect("compile-stage fixture"); let canonical = control_plane::query_plan::canonical_promql("count(unique_users)").unwrap(); - let entry = control_plane::query_plan::QueryPlanEntry::compile_bound( + let entry = control_plane::query_plan::compile_bound( "q-cardinality".into(), canonical, &node, - control_plane::query_plan::InstantExecution { + asap_types::query_plan::InstantExecution { lookback_ms: 60_000, full_history: false, cumulative_readout: true, }, - control_plane::query_plan::FallbackPolicy::ExactBackend, + asap_types::query_plan::FallbackPolicy::ExactBackend, |_node, _family| { - Ok(control_plane::query_plan::MaterializationBinding { + Ok(asap_types::query_plan::MaterializationBinding { item_labels: Vec::new(), materialization: asap_types::PolicyFingerprint(123).into(), - output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, + output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 60_000, pane_origin_ms: Some(2_000), readout_lookback_ms: Some(60_000), @@ -1049,27 +1049,27 @@ mod tests { ); } - let entry = control_plane::query_plan::QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + let entry = asap_types::query_plan::QueryPlanEntry { + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "q-rate".into(), canonical_query: "rate(requests_total[1m])".into(), fixed_evaluation: None, - root: control_plane::query_plan::QueryNodeId(0), + root: asap_types::query_plan::QueryNodeId(0), nodes: BTreeMap::from([ ( - control_plane::query_plan::QueryNodeId(0), + asap_types::query_plan::QueryNodeId(0), QueryPlanNode::ExactReadout { - input: control_plane::query_plan::QueryNodeId(1), - readout: control_plane::query_plan::ExactReadout::Sum, + input: asap_types::query_plan::QueryNodeId(1), + readout: asap_types::query_plan::ExactReadout::Sum, }, ), ( - control_plane::query_plan::QueryNodeId(1), + asap_types::query_plan::QueryNodeId(1), QueryPlanNode::ReadMaterialization { - binding: control_plane::query_plan::MaterializationBinding { + binding: asap_types::query_plan::MaterializationBinding { item_labels: Vec::new(), materialization: policy.into(), - output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, + output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 10_000, pane_origin_ms: Some(0), readout_lookback_ms: Some(60_000), @@ -1077,12 +1077,12 @@ mod tests { }, ), ]), - instant: control_plane::query_plan::InstantExecution { + instant: asap_types::query_plan::InstantExecution { lookback_ms: 60_000, full_history: false, cumulative_readout: true, }, - fallback: control_plane::query_plan::FallbackPolicy::ExactBackend, + fallback: asap_types::query_plan::FallbackPolicy::ExactBackend, }; for (now, expected) in [(60_000, 21.0), (70_000, 27.0), (80_000, 33.0)] { let (outcome, _) = execute_query_plan_instant(&idx, &entry, now).unwrap(); @@ -1145,27 +1145,27 @@ mod tests { accumulator.update(Measurement::new(13.0), 50_000); idx.append_precompute(7, BTreeMap::new(), (0, 60_000), Box::new(accumulator)); - let entry = control_plane::query_plan::QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + let entry = asap_types::query_plan::QueryPlanEntry { + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "q-rate".into(), canonical_query: "rate(requests_total[1m])".into(), fixed_evaluation: None, - root: control_plane::query_plan::QueryNodeId(0), + root: asap_types::query_plan::QueryNodeId(0), nodes: BTreeMap::from([ ( - control_plane::query_plan::QueryNodeId(0), + asap_types::query_plan::QueryNodeId(0), QueryPlanNode::ExactReadout { - input: control_plane::query_plan::QueryNodeId(1), - readout: control_plane::query_plan::ExactReadout::Rate, + input: asap_types::query_plan::QueryNodeId(1), + readout: asap_types::query_plan::ExactReadout::Rate, }, ), ( - control_plane::query_plan::QueryNodeId(1), + asap_types::query_plan::QueryNodeId(1), QueryPlanNode::ReadMaterialization { - binding: control_plane::query_plan::MaterializationBinding { + binding: asap_types::query_plan::MaterializationBinding { item_labels: Vec::new(), materialization: policy.into(), - output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, + output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 60_000, pane_origin_ms: Some(0), readout_lookback_ms: Some(60_000), @@ -1173,12 +1173,12 @@ mod tests { }, ), ]), - instant: control_plane::query_plan::InstantExecution { + instant: asap_types::query_plan::InstantExecution { lookback_ms: 60_000, full_history: false, cumulative_readout: true, }, - fallback: control_plane::query_plan::FallbackPolicy::ExactBackend, + fallback: asap_types::query_plan::FallbackPolicy::ExactBackend, }; let outcome = execute_query_plan_readout(&idx, &entry, 0, 60_000, true) .expect("execute exact rate DAG"); diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index 8192069a9..0e8745ebb 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -228,7 +228,7 @@ impl GroupState { /// in QueryPlan so serving never infers semantics from PromQL text. pub fn exact_value_for( &self, - readout: control_plane::query_plan::ExactReadout, + readout: asap_types::query_plan::ExactReadout, key: &Option, range_start_ms: u64, range_end_ms: u64, @@ -237,23 +237,23 @@ impl GroupState { return None; }; let stat = match (readout, agg_type) { - (control_plane::query_plan::ExactReadout::Count, AggregationType::Sum) => { + (asap_types::query_plan::ExactReadout::Count, AggregationType::Sum) => { asap_types::Statistic::Count } ( - control_plane::query_plan::ExactReadout::Sum, + asap_types::query_plan::ExactReadout::Sum, AggregationType::Sum | AggregationType::MultipleSum, ) => asap_types::Statistic::Sum, ( - control_plane::query_plan::ExactReadout::Increase, + asap_types::query_plan::ExactReadout::Increase, AggregationType::Increase | AggregationType::MultipleIncrease, ) => asap_types::Statistic::Increase, ( - control_plane::query_plan::ExactReadout::Rate, + asap_types::query_plan::ExactReadout::Rate, AggregationType::Increase | AggregationType::MultipleIncrease, ) => asap_types::Statistic::Rate, ( - control_plane::query_plan::ExactReadout::Max, + asap_types::query_plan::ExactReadout::Max, AggregationType::MinMax | AggregationType::MultipleMinMax, ) => asap_types::Statistic::Max, _ => return None, @@ -284,7 +284,7 @@ impl GroupState { if matches!( agg_type, AggregationType::MinMax | AggregationType::MultipleMinMax - ) && readout == control_plane::query_plan::ExactReadout::Max + ) && readout == asap_types::query_plan::ExactReadout::Max { return entries .iter() @@ -312,7 +312,7 @@ impl GroupState { ("range_end_ms".to_string(), range_end_ms.to_string()), ]); let merged = merged?; - if readout == control_plane::query_plan::ExactReadout::Count { + if readout == asap_types::query_plan::ExactReadout::Count { return merged.aux_stats().count.map(|count| count as f64); } merged.query_statistic(stat, key, &query_kwargs).ok() @@ -406,7 +406,7 @@ impl SummaryValue { } fn validate_binding_phase( - binding: &control_plane::query_plan::MaterializationBinding, + binding: &asap_types::query_plan::MaterializationBinding, evaluation_ms: u64, ) -> Result<(), SummaryExecutorError> { if i64::try_from(binding.window_ms).is_err() { @@ -435,9 +435,9 @@ impl QueryExecutionContext<'_> { /// are integrity checks and never broaden the candidate set. pub fn read_bound_materialization( &self, - binding: &control_plane::query_plan::MaterializationBinding, + binding: &asap_types::query_plan::MaterializationBinding, ) -> Result, GroupState)>, SummaryExecutorError> { - use control_plane::query_plan::PhysicalGrouping; + use asap_types::query_plan::PhysicalGrouping; validate_binding_phase(binding, self.t1_ms)?; @@ -1381,10 +1381,10 @@ mod tests { #[test] fn pane_only_reads_require_the_planned_evaluation_phase() { - let binding = control_plane::query_plan::MaterializationBinding { + let binding = asap_types::query_plan::MaterializationBinding { item_labels: Vec::new(), materialization: asap_types::PolicyFingerprint(7).into(), - output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, + output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 60_000, pane_origin_ms: Some(7_000), readout_lookback_ms: Some(60_000), @@ -1392,7 +1392,7 @@ mod tests { validate_binding_phase(&binding, 67_000).unwrap(); assert!(validate_binding_phase(&binding, 68_000).is_err()); - let legacy = control_plane::query_plan::MaterializationBinding { + let legacy = asap_types::query_plan::MaterializationBinding { item_labels: Vec::new(), pane_origin_ms: None, ..binding @@ -1790,7 +1790,7 @@ mod tests { use crate::precompute_engine::operators::univmon_accumulator::UnivMonAccumulator; use crate::storage_engines::sketch_db::index::SketchEncoding; use crate::storage_engines::types::SerializableToSink; - use control_plane::query_plan::{MaterializationBinding, PhysicalGrouping}; + use asap_types::query_plan::{MaterializationBinding, PhysicalGrouping}; let index = SketchStore::new(); let fp = asap_types::PolicyFingerprint(701); let mut meta = kll_meta(1, "m", &["job"]); diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index c007f8f5b..9cba552f9 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -94,7 +94,7 @@ pub struct ActivePhysicalPlan { pub precompute_plan: asap_types::precompute_plan::PrecomputePlan, pub transmission_plan: control_plane::physical::compiler::TransmissionPlan, pub runtime_config: Arc, - pub query_plan: Arc, + pub query_plan: Arc, pub storage_routing: Arc, } @@ -707,7 +707,7 @@ mod tests { rules: Vec::new(), }, runtime_config: Arc::new(StreamingConfig::new(HashMap::new())), - query_plan: Arc::new(control_plane::query_plan::QueryPlan { + query_plan: Arc::new(asap_types::query_plan::QueryPlan { plan_id, plan_version, clickhouse_context: None, diff --git a/data_plane/tests/clickhouse_differential_e2e.rs b/data_plane/tests/clickhouse_differential_e2e.rs index 9089fe47f..fdecd1b06 100644 --- a/data_plane/tests/clickhouse_differential_e2e.rs +++ b/data_plane/tests/clickhouse_differential_e2e.rs @@ -203,11 +203,11 @@ async fn compiled_publication_executes_mixed_dag_in_data_plane_process() { let entry = publication.query_plan.entries.values().next().unwrap(); assert!(entry.nodes.values().any(|node| matches!( node, - control_plane::query_plan::QueryPlanNode::ExternalExact { .. } + asap_types::query_plan::QueryPlanNode::ExternalExact { .. } ))); assert!(entry.nodes.values().any(|node| matches!( node, - control_plane::query_plan::QueryPlanNode::ReadMaterialization { .. } + asap_types::query_plan::QueryPlanNode::ReadMaterialization { .. } ))); let install = publication.install_request(None, Vec::new()).unwrap(); diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index a6d9366fd..0bf5ff300 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -90,8 +90,7 @@ async fn post_full_config(client: &reqwest::Client, stack: &FullStack, json: &Js { entry.instant.lookback_ms = 1000; for node in entry.nodes.values_mut() { - if let control_plane::query_plan::QueryPlanNode::ReadMaterialization { binding } = node - { + if let asap_types::query_plan::QueryPlanNode::ReadMaterialization { binding } = node { binding.readout_lookback_ms = Some(1000); } } diff --git a/data_plane/tests/support/physical_fixture.rs b/data_plane/tests/support/physical_fixture.rs index 30de3d05d..00ae0e862 100644 --- a/data_plane/tests/support/physical_fixture.rs +++ b/data_plane/tests/support/physical_fixture.rs @@ -104,7 +104,7 @@ pub fn artifact(config: &StreamingConfig) -> PhysicalPlanInstallRequest { query_plan.entries.insert( canonical.clone(), QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: canonical.clone(), canonical_query: canonical, fixed_evaluation: None, diff --git a/docs/design_docs/asapplanner-integration.md b/docs/design_docs/asapplanner-integration.md index f1f1d082b..c6cc83be6 100644 --- a/docs/design_docs/asapplanner-integration.md +++ b/docs/design_docs/asapplanner-integration.md @@ -106,7 +106,8 @@ Evidence: [selection adapter](../../control_plane/src/planner_selection.rs), [physical compiler](../../control_plane/src/physical/compiler.rs), [legacy workload adapter](../../control_plane/src/physical/workload_planner.rs), -[QueryPlan](../../control_plane/src/query_plan.rs), and +[shared QueryPlan](../../crates/asap_types/src/query_plan.rs), +[query lowering](../../control_plane/src/query_plan.rs), and [bound serving executor](../../data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs). The selection adapter explicitly commits a ranked Planner candidate downstream. Consequently, the figure does not imply that the Planner library deploys or diff --git a/docs/design_docs/summary-catalog-sds-architecture.md b/docs/design_docs/summary-catalog-sds-architecture.md index baf28e179..8b22f2bd1 100644 --- a/docs/design_docs/summary-catalog-sds-architecture.md +++ b/docs/design_docs/summary-catalog-sds-architecture.md @@ -158,8 +158,12 @@ bindings against QueryPlan; precompute execution consumes the shared contract. `PrecomputePlan`, its envelope, ingest, producer, state schema, and catalog consistency checks live in `asap_types::precompute_plan`. The compiler chooses materializations and placement; data-plane installation uses the shared -contract. `QueryPlan` definitions still reside in the control-plane -crate while their remaining compilation methods are separated from wire types. +contract. `asap_types::query_plan` owns QueryPlan, materialization bindings, +logical operator DTOs, and activation validation. The control plane reexports +those types for existing callers and owns the `compile_bound*` and +`logical::compile_logical` functions; Planner traversal and AST lowering do not +move into the shared contract. Data-plane engines import the shared types +directly. No wrapper plan or second wire definition is introduced. The implemented ownership split is: From cf5f32640a6b0890ca442a986b6ae301abab7f7f Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 20:38:52 -0600 Subject: [PATCH 2/4] refactor(query): use shared identity contract in MetricsQL adapter --- data_plane/src/drivers/ingest/prometheus_remote_write.rs | 4 ++-- data_plane/src/drivers/query/adapters/victoriametrics_http.rs | 2 +- data_plane/src/query_engines/asap_query_engine/engine.rs | 2 +- .../src/query_engines/asap_query_engine/post_asap_readout.rs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index f4cce5683..92b75fb4b 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -1464,9 +1464,9 @@ mod tests { .keys() .next() .unwrap(); - let binding = control_plane::query_plan::MaterializationBinding { + let binding = asap_types::query_plan::MaterializationBinding { materialization: asap_types::PolicyFingerprint(policy).into(), - output_grouping: control_plane::query_plan::PhysicalGrouping::Reduce( + output_grouping: asap_types::query_plan::PhysicalGrouping::Reduce( vec!["job".into()], ), item_labels: vec![], diff --git a/data_plane/src/drivers/query/adapters/victoriametrics_http.rs b/data_plane/src/drivers/query/adapters/victoriametrics_http.rs index a75bbd264..e23cf4a9b 100644 --- a/data_plane/src/drivers/query/adapters/victoriametrics_http.rs +++ b/data_plane/src/drivers/query/adapters/victoriametrics_http.rs @@ -104,7 +104,7 @@ impl HttpProtocolAdapter for VictoriaMetricsHttpAdapter { "VictoriaMetrics HTTP / MetricsQL" } fn canonical_plan_identity(&self, query: &str) -> Result, AdapterError> { - control_plane::query_plan::canonical_promql(query) + asap_types::query_plan::canonical_promql(query) .map(Some) .map_err(|error| AdapterError::ParseError(format!("promql-compatible subset: {error}"))) } diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 5e90a82ff..ccd3a2464 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -3623,7 +3623,7 @@ mod range_stitch_tests { )) .unwrap(); let mut plan = snapshot.compile().unwrap(); - let identity = control_plane::query_plan::canonical_promql("1 + 2").unwrap(); + let identity = asap_types::query_plan::canonical_promql("1 + 2").unwrap(); plan.query_plan.entries.insert( asap_types::query_plan::QueryPlan::catalog_key(QueryLanguage::MetricsQl, &identity), QueryPlanEntry { diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index bd55b073a..eaa5a2ec3 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -907,7 +907,7 @@ mod tests { register_hll(&idx, 2, "worker", &["b", "c"]); let node = plan_promql_to_post_asap(&idx, "count(unique_users)", accuracy()) .expect("compile-stage fixture"); - let canonical = control_plane::query_plan::canonical_promql("count(unique_users)").unwrap(); + let canonical = asap_types::query_plan::canonical_promql("count(unique_users)").unwrap(); let entry = control_plane::query_plan::compile_bound( "q-cardinality".into(), canonical, From 3b93279c4b3591aa5b063326ba483ebc71833730 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 20:53:54 -0600 Subject: [PATCH 3/4] style: format shared query identity fixture --- data_plane/src/drivers/ingest/prometheus_remote_write.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index 92b75fb4b..67faa2554 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -1466,9 +1466,7 @@ mod tests { .unwrap(); let binding = asap_types::query_plan::MaterializationBinding { materialization: asap_types::PolicyFingerprint(policy).into(), - output_grouping: asap_types::query_plan::PhysicalGrouping::Reduce( - vec!["job".into()], - ), + output_grouping: asap_types::query_plan::PhysicalGrouping::Reduce(vec!["job".into()]), item_labels: vec![], window_ms: 60_000, pane_origin_ms: Some(0), From e4bd62a8ea3fac1f0887dedcceef9e8f0933bc8f Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 21:05:36 -0600 Subject: [PATCH 4/4] refactor(plans): share producer and publication contracts --- .../examples/audit_clickhouse_corpus.rs | 10 +- control_plane/src/clickhouse.rs | 13 +- control_plane/src/emit/backend_push.rs | 2 +- control_plane/src/physical/compiler.rs | 1158 +++-------------- control_plane/src/physical/publication.rs | 127 +- crates/asap_types/src/lib.rs | 2 + crates/asap_types/src/plan_publication.rs | 126 ++ crates/asap_types/src/producer_plan.rs | 879 +++++++++++++ .../examples/audit_clickhouse_fallback.rs | 12 +- data_plane/src/drivers/ingest/otel.rs | 13 +- .../drivers/ingest/prometheus_remote_write.rs | 9 +- data_plane/src/drivers/query/servers/http.rs | 7 +- data_plane/src/main.rs | 6 +- .../src/precompute_engine/frame_lineage.rs | 2 +- .../accelerator.rs | 2 +- .../storage_engines/sketch_db/index/mod.rs | 13 +- .../types/hot_reload_config.rs | 17 +- ...e2e_controller_plans_and_backend_serves.rs | 4 +- data_plane/tests/support/physical_fixture.rs | 8 +- .../summary-catalog-sds-architecture.md | 7 + 20 files changed, 1245 insertions(+), 1172 deletions(-) create mode 100644 crates/asap_types/src/plan_publication.rs create mode 100644 crates/asap_types/src/producer_plan.rs diff --git a/control_plane/examples/audit_clickhouse_corpus.rs b/control_plane/examples/audit_clickhouse_corpus.rs index b5187a70f..b9f1c13e7 100644 --- a/control_plane/examples/audit_clickhouse_corpus.rs +++ b/control_plane/examples/audit_clickhouse_corpus.rs @@ -2,7 +2,7 @@ use asap_frontend_sql::SqlCatalog; use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; use control_plane::clickhouse::{ClickHouseSqlWorkload, ClickHouseSqlWorkloadEntry}; use control_plane::physical::compiler::{ - PlanEnvelope, PrecomputePlan, TransmissionPlan, BACKEND_COMPAT, PLANNER_REVISION, + PlanEnvelope, PrecomputePlan, BACKEND_COMPAT, PLANNER_REVISION, }; use planner_types::pre_asap::{Column, DataType, Schema}; use planner_types::types::AccuracyTarget; @@ -54,8 +54,12 @@ fn publication_inputs(schema: &Schema, sql: String) -> ClickHouseSqlWorkload { let mut precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![materialization.clone()]) .unwrap(); - let mut transmission_plan = - TransmissionPlan::build(envelope, &precompute_plan, &Default::default()).unwrap(); + let mut transmission_plan = control_plane::physical::compiler::compile_transmission_plan( + envelope, + &precompute_plan, + &Default::default(), + ) + .unwrap(); let sds = asap_types::summary_catalog::SummaryCatalog::from_materializations( 27, 1, diff --git a/control_plane/src/clickhouse.rs b/control_plane/src/clickhouse.rs index 99f70a59a..3ebcdbdea 100644 --- a/control_plane/src/clickhouse.rs +++ b/control_plane/src/clickhouse.rs @@ -183,7 +183,7 @@ pub async fn compile_automatic_clickhouse_workload( .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?, ); precompute.executable_dags = installed_dags; - let mut transmission = TransmissionPlan::build( + let mut transmission = crate::physical::compiler::compile_transmission_plan( request.envelope.clone(), &precompute, &std::collections::BTreeMap::new(), @@ -813,9 +813,12 @@ mod tests { let mut precompute = PrecomputePlan::build_backend_local(envelope.clone(), vec![config]).unwrap(); precompute.summary_catalog = Some(sds.reference().unwrap()); - let mut transmission = - TransmissionPlan::build(envelope, &precompute, &std::collections::BTreeMap::new()) - .unwrap(); + let mut transmission = crate::physical::compiler::compile_transmission_plan( + envelope, + &precompute, + &std::collections::BTreeMap::new(), + ) + .unwrap(); transmission.summary_catalog = Some(sds.reference().unwrap()); let timestamped = |time_name: &str, value_name: &str| { Schema::with_time_index( @@ -974,7 +977,7 @@ mod tests { request.precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![config]).unwrap(); request.precompute_plan.summary_catalog = Some(request.sds.reference().unwrap()); - request.transmission_plan = TransmissionPlan::build( + request.transmission_plan = crate::physical::compiler::compile_transmission_plan( envelope, &request.precompute_plan, &std::collections::BTreeMap::new(), diff --git a/control_plane/src/emit/backend_push.rs b/control_plane/src/emit/backend_push.rs index d3f4861de..9be93c761 100644 --- a/control_plane/src/emit/backend_push.rs +++ b/control_plane/src/emit/backend_push.rs @@ -260,7 +260,7 @@ async fn push_documents_coupled( warn!(%error, "failed to bind compatibility PrecomputePlan to SummaryCatalog"); return (false, false, 0); } - let transmission_plan = match crate::physical::compiler::TransmissionPlan::build( + let transmission_plan = match crate::physical::compiler::compile_transmission_plan( precompute_plan.envelope.clone(), &precompute_plan, &Default::default(), diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 9d40737cd..a3b92e298 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -258,47 +258,136 @@ pub use asap_types::precompute_plan::{ StateWindowContract, TimestampUnit, }; -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct CollectorMaterialization { - pub query_id: String, - pub materialization: asap_types::sds::SummaryDefinitionId, - pub metric: String, - pub algorithm: String, - pub parameters: Value, - pub group_by: Vec, - pub window_secs: u64, - pub abstract_window_framework: SummaryWindowFramework, - pub window_implementation_id: String, - pub slide_secs: u64, - #[serde( - default, - alias = "paneOriginMs", - skip_serializing_if = "Option::is_none" - )] - pub pane_origin_ms: Option, - pub window_layout: asap_types::WindowMaterializationLayout, - pub evidence_source: Option, - pub lifecycle: CollectorLifecycle, +pub use asap_types::producer_plan::{ + AdaptiveF64Bounds, AdaptiveU64Bounds, CollectorLifecycle, CollectorMaterialization, + CollectorPlan, DeltaPolicy, FrameIdentityContract, GosPolicy, GosThresholdMode, + RuntimeAdaptationEvidence, RuntimeAdaptationPolicy, RuntimeRulePolicy, SamplingEstimator, + SamplingPolicy, SequenceScope, SummaryFrameIdentity, SummaryFrameKind, TransmissionMode, + TransmissionPlan, TransmissionPlanError, TransmissionRule, +}; + +/// Build the fixed physical knob from the controller's canonical +/// epsilon-floor allocator. Degenerate budgets/rates disable sampling. +pub fn sampling_policy_from_accuracy_budget( + epsilon_sampling: f64, + updates_per_window: f64, + estimator: SamplingEstimator, +) -> SamplingPolicy { + if !epsilon_sampling.is_finite() + || epsilon_sampling <= 0.0 + || !updates_per_window.is_finite() + || updates_per_window <= 0.0 + { + return SamplingPolicy::Disabled; + } + let probability = crate::epsilon_alloc::derive_sample_p(epsilon_sampling, updates_per_window); + if probability >= 1.0 { + SamplingPolicy::Disabled + } else { + SamplingPolicy::Fixed { + probability, + estimator, + } + } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct CollectorLifecycle { - pub kind: String, - pub maintenance_mode: String, - pub evaluation_schedule: String, - pub output_representation: String, +/// Allocate the deterministic staleness share with the same linear-peel +/// composition used by `epsilon_alloc`. `None` means the selected sketch +/// already consumes the budget or communication has no allocated weight. +pub fn gos_policy_from_accuracy_budget( + epsilon_total: f64, + epsilon_sketch: f64, + sites: u32, + edge_cpu_weight: f64, + communication_weight: f64, + threshold_mode: GosThresholdMode, +) -> Option { + if !epsilon_total.is_finite() + || !epsilon_sketch.is_finite() + || !(0.0..=1.0).contains(&epsilon_total) + || epsilon_sketch < 0.0 + || !edge_cpu_weight.is_finite() + || edge_cpu_weight < 0.0 + || !communication_weight.is_finite() + || communication_weight <= 0.0 + { + return None; + } + let (_, epsilon_staleness) = crate::epsilon_alloc::split_budget( + epsilon_total, + epsilon_sketch, + edge_cpu_weight, + communication_weight, + ); + (epsilon_staleness.is_finite() && epsilon_staleness > 0.0).then_some(GosPolicy { + epsilon_staleness, + sites: sites.max(1), + threshold_mode, + }) } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct CollectorPlan { - /// Absent only in legacy artifacts; catalog-aware validation requires it. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub summary_catalog: Option, - pub collector_id: String, - pub envelope: PlanEnvelope, - pub materializations: Vec, - pub transmission_rules: Vec, +pub fn compile_transmission_plan( + envelope: PlanEnvelope, + precompute: &PrecomputePlan, + runtime_policies: &BTreeMap, +) -> Result { + if envelope != precompute.envelope { + return Err(TransmissionPlanError::EnvelopeMismatch); + } + let schemas: BTreeMap<_, _> = precompute + .schemas + .iter() + .map(|schema| (schema.materialization, schema)) + .collect(); + let rules = precompute + .producers + .iter() + .map(|producer| { + let schema = schemas + .get(&producer.materialization) + .expect("validated PrecomputePlan schema binding"); + let materialization = precompute + .materializations + .iter() + .find(|m| m.policy_fingerprint() == producer.materialization.fingerprint()) + .expect("validated PrecomputePlan materialization binding"); + let runtime_policy = runtime_policies + .get(&producer.materialization.fingerprint()) + .cloned() + .unwrap_or_default(); + let mode = if runtime_policy.delta.is_some() { + TransmissionMode::Delta + } else { + TransmissionMode::Full + }; + let emit_every_ms = materialization.window_size.saturating_mul(1_000); + TransmissionRule { + materialization: producer.materialization, + producer_id: producer.producer_id.clone(), + schema_id: producer.schema_id.clone(), + mode, + encoding: schema.encodings[0].clone(), + emit_every_ms, + full_checkpoint_every_ms: (mode == TransmissionMode::Delta) + .then(|| emit_every_ms.saturating_mul(10)), + destination_ref: "asapquery-backend".into(), + runtime_policy, + } + }) + .collect(); + let plan = TransmissionPlan { + summary_catalog: precompute.summary_catalog.clone(), + envelope, + frame_identity: FrameIdentityContract { + identity_version: 1, + sequence_scope: SequenceScope::MaterializationSeriesProducerEpoch, + require_checkpoint_for_full: true, + require_base_checkpoint_for_delta: true, + }, + rules, + }; + plan.validate(precompute)?; + Ok(plan) } /// Complete physical projection of one post-ASAP planning decision. @@ -327,905 +416,6 @@ pub struct MaterializationLifecycleEstimate { pub lifecycle_cost: f64, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -#[serde(rename_all = "snake_case")] -pub enum TransmissionMode { - Full, - Delta, -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum SequenceScope { - MaterializationSeriesProducerEpoch, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct FrameIdentityContract { - pub identity_version: u32, - pub sequence_scope: SequenceScope, - pub require_checkpoint_for_full: bool, - pub require_base_checkpoint_for_delta: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct TransmissionRule { - pub materialization: asap_types::sds::SummaryDefinitionId, - pub producer_id: String, - pub schema_id: String, - pub mode: TransmissionMode, - pub encoding: StateEncoding, - pub emit_every_ms: u64, - pub full_checkpoint_every_ms: Option, - pub destination_ref: String, - /// Plan-owned runtime knobs. These values are part of the immutable plan - /// generation; live feedback may only change them by publishing a - /// successor generation accepted by [`TransmissionPlan::authorize_successor`]. - #[serde(default)] - pub runtime_policy: RuntimeRulePolicy, -} - -/// How the collector admits updates before sketch maintenance. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(tag = "mode", rename_all = "snake_case", deny_unknown_fields)] -pub enum SamplingPolicy { - Disabled, - Fixed { - /// Probability in `(0, 1]`; `1` is valid but should normally be - /// represented by `Disabled`. - probability: f64, - estimator: SamplingEstimator, - }, -} - -impl Default for SamplingPolicy { - fn default() -> Self { - Self::Disabled - } -} - -impl SamplingPolicy { - /// Build the fixed physical knob from the controller's canonical - /// epsilon-floor allocator. Degenerate budgets/rates disable sampling. - pub fn from_accuracy_budget( - epsilon_sampling: f64, - updates_per_window: f64, - estimator: SamplingEstimator, - ) -> Self { - if !epsilon_sampling.is_finite() - || epsilon_sampling <= 0.0 - || !updates_per_window.is_finite() - || updates_per_window <= 0.0 - { - return Self::Disabled; - } - let probability = - crate::epsilon_alloc::derive_sample_p(epsilon_sampling, updates_per_window); - if probability >= 1.0 { - Self::Disabled - } else { - Self::Fixed { - probability, - estimator, - } - } - } -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum SamplingEstimator { - /// Hash-threshold element sampling, used by cardinality summaries. - HashThreshold, - /// Geometric admission/Nitro-style update sampling, used by frequency - /// summaries. The sketch readout carries the corresponding correction. - GeometricAdmission, -} - -/// Norm-adaptive Group-of-Sketches delta gating. GOS is meaningful only for -/// CountSketch families and only when the transmission rule is in delta mode. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct GosPolicy { - pub epsilon_staleness: f64, - pub sites: u32, - pub threshold_mode: GosThresholdMode, -} - -impl GosPolicy { - /// Allocate the deterministic staleness share with the same linear-peel - /// composition used by `epsilon_alloc`. `None` means the selected sketch - /// already consumes the budget or communication has no allocated weight. - pub fn from_accuracy_budget( - epsilon_total: f64, - epsilon_sketch: f64, - sites: u32, - edge_cpu_weight: f64, - communication_weight: f64, - threshold_mode: GosThresholdMode, - ) -> Option { - if !epsilon_total.is_finite() - || !epsilon_sketch.is_finite() - || !(0.0..=1.0).contains(&epsilon_total) - || epsilon_sketch < 0.0 - || !edge_cpu_weight.is_finite() - || edge_cpu_weight < 0.0 - || !communication_weight.is_finite() - || communication_weight <= 0.0 - { - return None; - } - let (_, epsilon_staleness) = crate::epsilon_alloc::split_budget( - epsilon_total, - epsilon_sketch, - edge_cpu_weight, - communication_weight, - ); - (epsilon_staleness.is_finite() && epsilon_staleness > 0.0).then_some(Self { - epsilon_staleness, - sites: sites.max(1), - threshold_mode, - }) - } -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum GosThresholdMode { - Isotropic, - Anisotropic, -} - -/// Sparse-delta semantics within a delta transmission rule. An absolute -/// threshold of zero sends every changed cell. When `gos` is present it -/// replaces the fixed threshold with the GOS norm-adaptive threshold. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct DeltaPolicy { - pub absolute_threshold: f64, - pub gos: Option, -} - -/// Inclusive bounds for one floating-point adaptation knob. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct AdaptiveF64Bounds { - pub min: f64, - pub max: f64, - pub max_step: f64, -} - -/// Inclusive bounds for one integer adaptation knob. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct AdaptiveU64Bounds { - pub min: u64, - pub max: u64, - pub max_step: u64, -} - -/// Guardrails for telemetry-driven runtime adaptation. This is an -/// authorization contract, not an instruction to mutate the active plan. -/// Every accepted change becomes a staged successor PhysicalPlan. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct RuntimeAdaptationPolicy { - pub enabled: bool, - pub not_before_unix_ms: u64, - pub max_evidence_age_ms: u64, - pub min_evidence_samples: u64, - pub sample_probability: Option, - pub emit_every_ms: Option, - pub delta_threshold: Option, - pub gos_epsilon_staleness: Option, -} - -impl Default for RuntimeAdaptationPolicy { - fn default() -> Self { - Self { - enabled: false, - not_before_unix_ms: 0, - max_evidence_age_ms: 0, - min_evidence_samples: 0, - sample_probability: None, - emit_every_ms: None, - delta_threshold: None, - gos_epsilon_staleness: None, - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] -#[serde(deny_unknown_fields)] -pub struct RuntimeRulePolicy { - #[serde(default)] - pub sampling: SamplingPolicy, - pub delta: Option, - #[serde(default)] - pub adaptation: RuntimeAdaptationPolicy, -} - -/// Identity and sufficiency information for evidence authorizing one rule's -/// successor knobs. Raw measurements remain in the runtime-samples store; the -/// authorization boundary needs only their exact provenance and sample count. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct RuntimeAdaptationEvidence { - pub plan_id: u64, - pub plan_version: u64, - pub materialization: asap_types::sds::SummaryDefinitionId, - pub producer_id: String, - pub schema_id: String, - pub producer_version: String, - pub observed_at_unix_ms: u64, - pub sample_count: u64, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct TransmissionPlan { - /// Absent only in legacy artifacts; catalog-aware validation requires it. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub summary_catalog: Option, - pub envelope: PlanEnvelope, - pub frame_identity: FrameIdentityContract, - pub rules: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum SummaryFrameKind { - Full, - Delta, -} - -/// Identity attached to every summary record. Window bounds come from the -/// data point; the remaining fields are carried as reserved `asap.frame.*` -/// attributes until the modified-OTLP schema gains a dedicated message. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct SummaryFrameIdentity { - pub identity_version: u32, - pub plan_id: u64, - pub plan_version: u64, - pub backend_compat: String, - pub materialization: asap_types::sds::SummaryDefinitionId, - /// Canonical producer-side identity for one concrete retained-label group. - pub series_identity: String, - pub schema_id: String, - pub producer_id: String, - pub producer_epoch: String, - pub window_start_unix_nano: u64, - pub window_end_unix_nano: u64, - pub sequence: u64, - pub kind: SummaryFrameKind, - pub encoding: StateEncoding, - pub checkpoint_id: Option, - pub base_checkpoint_id: Option, -} - -#[derive(Debug, Error, PartialEq, Eq)] -pub enum TransmissionPlanError { - #[error("summary catalog mismatch: {0}")] - Catalog(String), - #[error("TransmissionPlan envelope differs from PrecomputePlan")] - EnvelopeMismatch, - #[error("transmission rules do not exactly match precompute producer bindings")] - ProducerSetMismatch, - #[error("invalid transmission rule for producer {0}")] - InvalidRule(String), - #[error("frame identity is invalid: {0}")] - InvalidFrame(String), - #[error("frame has no matching transmission rule")] - UnknownFrame, - #[error("runtime policy for producer {producer_id} is invalid: {reason}")] - InvalidRuntimePolicy { producer_id: String, reason: String }, - #[error("runtime adaptation successor is invalid: {0}")] - InvalidSuccessor(String), - #[error("runtime adaptation evidence for producer {0} is missing or invalid")] - InvalidAdaptationEvidence(String), - #[error("runtime adaptation for producer {producer_id} exceeds guardrails: {knob}")] - AdaptationOutOfBounds { - producer_id: String, - knob: &'static str, - }, -} - -fn validate_catalog_projection( - reference: Option<&asap_types::sds::CatalogGeneration>, - envelope: &PlanEnvelope, - materializations: impl IntoIterator, - catalog: &super::summary_catalog::SummaryCatalog, -) -> Result<(), TransmissionPlanError> { - let expected = catalog - .reference() - .map_err(|error| TransmissionPlanError::Catalog(error.to_string()))?; - if reference != Some(&expected) - || envelope.plan_id != catalog.plan_id - || envelope.plan_version != catalog.plan_version - { - return Err(TransmissionPlanError::Catalog( - "missing or different snapshot reference".into(), - )); - } - for id in materializations { - if !catalog.materializations.contains_key(&id) { - return Err(TransmissionPlanError::Catalog(format!( - "unknown materialization {}", - id.as_u64() - ))); - } - } - Ok(()) -} - -impl CollectorPlan { - pub fn validate_against_catalog( - &self, - catalog: &super::summary_catalog::SummaryCatalog, - ) -> Result<(), TransmissionPlanError> { - validate_catalog_projection( - self.summary_catalog.as_ref(), - &self.envelope, - self.materializations - .iter() - .map(|m| m.materialization) - .chain(self.transmission_rules.iter().map(|r| r.materialization)), - catalog, - ) - } -} - -impl TransmissionPlan { - pub fn validate_against_catalog( - &self, - catalog: &super::summary_catalog::SummaryCatalog, - ) -> Result<(), TransmissionPlanError> { - validate_catalog_projection( - self.summary_catalog.as_ref(), - &self.envelope, - self.rules.iter().map(|r| r.materialization), - catalog, - ) - } - - pub fn build( - envelope: PlanEnvelope, - precompute: &PrecomputePlan, - runtime_policies: &BTreeMap, - ) -> Result { - if envelope != precompute.envelope { - return Err(TransmissionPlanError::EnvelopeMismatch); - } - let schemas: BTreeMap<_, _> = precompute - .schemas - .iter() - .map(|schema| (schema.materialization, schema)) - .collect(); - let rules = precompute - .producers - .iter() - .map(|producer| { - let schema = schemas - .get(&producer.materialization) - .expect("validated PrecomputePlan schema binding"); - let materialization = precompute - .materializations - .iter() - .find(|m| m.policy_fingerprint() == producer.materialization.fingerprint()) - .expect("validated PrecomputePlan materialization binding"); - let runtime_policy = runtime_policies - .get(&producer.materialization.fingerprint()) - .cloned() - .unwrap_or_default(); - let mode = if runtime_policy.delta.is_some() { - TransmissionMode::Delta - } else { - TransmissionMode::Full - }; - let emit_every_ms = materialization.window_size.saturating_mul(1_000); - TransmissionRule { - materialization: producer.materialization, - producer_id: producer.producer_id.clone(), - schema_id: producer.schema_id.clone(), - mode, - encoding: schema.encodings[0].clone(), - emit_every_ms, - full_checkpoint_every_ms: (mode == TransmissionMode::Delta) - .then(|| emit_every_ms.saturating_mul(10)), - destination_ref: "asapquery-backend".into(), - runtime_policy, - } - }) - .collect(); - let plan = Self { - summary_catalog: precompute.summary_catalog.clone(), - envelope, - frame_identity: FrameIdentityContract { - identity_version: 1, - sequence_scope: SequenceScope::MaterializationSeriesProducerEpoch, - require_checkpoint_for_full: true, - require_base_checkpoint_for_delta: true, - }, - rules, - }; - plan.validate(precompute)?; - Ok(plan) - } - - pub fn validate(&self, precompute: &PrecomputePlan) -> Result<(), TransmissionPlanError> { - if self.summary_catalog != precompute.summary_catalog { - return Err(TransmissionPlanError::Catalog( - "transmission and precompute plans reference different catalog snapshots".into(), - )); - } - if self.envelope != precompute.envelope { - return Err(TransmissionPlanError::EnvelopeMismatch); - } - let expected: BTreeSet<_> = precompute - .producers - .iter() - .map(|producer| { - ( - producer.materialization, - producer.producer_id.as_str(), - producer.schema_id.as_str(), - ) - }) - .collect(); - let actual: BTreeSet<_> = self - .rules - .iter() - .map(|rule| { - ( - rule.materialization, - rule.producer_id.as_str(), - rule.schema_id.as_str(), - ) - }) - .collect(); - if expected != actual || actual.len() != self.rules.len() { - return Err(TransmissionPlanError::ProducerSetMismatch); - } - for rule in &self.rules { - let valid_checkpoint_cadence = match (rule.mode, rule.full_checkpoint_every_ms) { - (TransmissionMode::Full, None) => true, - (TransmissionMode::Delta, Some(full_every)) => { - rule.emit_every_ms > 0 - && full_every >= rule.emit_every_ms - && full_every % rule.emit_every_ms == 0 - } - _ => false, - }; - if rule.emit_every_ms == 0 - || rule.destination_ref.is_empty() - || !valid_checkpoint_cadence - { - return Err(TransmissionPlanError::InvalidRule(rule.producer_id.clone())); - } - let schema = precompute - .schemas - .iter() - .find(|schema| schema.materialization == rule.materialization) - .expect("producer set validation guarantees a matching schema"); - if !schema.encodings.contains(&rule.encoding) { - return Err(TransmissionPlanError::InvalidRule(rule.producer_id.clone())); - } - validate_runtime_rule_policy(rule, &schema.family)?; - } - Ok(()) - } - - /// Authorize a telemetry-driven successor without mutating this active - /// plan. Semantic identity, codecs, destination, transmission mode and - /// checkpoint cadence remain fixed. Only explicitly bounded runtime knobs - /// may move, and each changed rule needs fresh evidence attributed to the - /// exact active generation. - pub fn authorize_successor( - &self, - successor: &TransmissionPlan, - evidence: &[RuntimeAdaptationEvidence], - now_unix_ms: u64, - ) -> Result<(), TransmissionPlanError> { - if successor.envelope.plan_id != self.envelope.plan_id - || successor.envelope.plan_version != self.envelope.plan_version.saturating_add(1) - || successor.envelope.backend_compat != self.envelope.backend_compat - || successor.envelope.planner_revision != self.envelope.planner_revision - || successor.envelope.capability_snapshot_id != self.envelope.capability_snapshot_id - || successor.envelope.generated_at_unix_ms < self.envelope.generated_at_unix_ms - || successor.envelope.activation_unix_ms < successor.envelope.generated_at_unix_ms - || successor.rules.len() != self.rules.len() - || successor.frame_identity != self.frame_identity - { - return Err(TransmissionPlanError::InvalidSuccessor( - "successor must be the next version of the same semantic/capability generation" - .into(), - )); - } - - for current in &self.rules { - let Some(next) = successor.rules.iter().find(|candidate| { - candidate.materialization == current.materialization - && candidate.producer_id == current.producer_id - && candidate.schema_id == current.schema_id - }) else { - return Err(TransmissionPlanError::InvalidSuccessor(format!( - "missing rule for producer {}", - current.producer_id - ))); - }; - if current.mode != next.mode - || current.encoding != next.encoding - || current.full_checkpoint_every_ms != next.full_checkpoint_every_ms - || current.destination_ref != next.destination_ref - || current.runtime_policy.adaptation != next.runtime_policy.adaptation - || sampling_estimator(¤t.runtime_policy.sampling) - != sampling_estimator(&next.runtime_policy.sampling) - || delta_shape(¤t.runtime_policy.delta) - != delta_shape(&next.runtime_policy.delta) - { - return Err(TransmissionPlanError::InvalidSuccessor(format!( - "rule identity/codec/mode/guardrails drifted for producer {}", - current.producer_id - ))); - } - if current.emit_every_ms == next.emit_every_ms - && current.runtime_policy.sampling == next.runtime_policy.sampling - && current.runtime_policy.delta == next.runtime_policy.delta - { - continue; - } - let policy = ¤t.runtime_policy.adaptation; - if !policy.enabled || now_unix_ms < policy.not_before_unix_ms { - return Err(TransmissionPlanError::AdaptationOutOfBounds { - producer_id: current.producer_id.clone(), - knob: "adaptation_disabled_or_in_cooldown", - }); - } - let has_evidence = evidence.iter().any(|item| { - item.plan_id == self.envelope.plan_id - && item.plan_version == self.envelope.plan_version - && item.materialization == current.materialization - && item.producer_id == current.producer_id - && item.schema_id == current.schema_id - && !item.producer_version.trim().is_empty() - && item.sample_count >= policy.min_evidence_samples - && item.observed_at_unix_ms <= now_unix_ms - && now_unix_ms.saturating_sub(item.observed_at_unix_ms) - <= policy.max_evidence_age_ms - }); - if !has_evidence { - return Err(TransmissionPlanError::InvalidAdaptationEvidence( - current.producer_id.clone(), - )); - } - authorize_f64_change( - sampling_probability(¤t.runtime_policy.sampling), - sampling_probability(&next.runtime_policy.sampling), - policy.sample_probability.as_ref(), - ¤t.producer_id, - "sample_probability", - )?; - authorize_u64_change( - current.emit_every_ms, - next.emit_every_ms, - policy.emit_every_ms.as_ref(), - ¤t.producer_id, - "emit_every_ms", - )?; - authorize_f64_change( - delta_threshold(¤t.runtime_policy.delta), - delta_threshold(&next.runtime_policy.delta), - policy.delta_threshold.as_ref(), - ¤t.producer_id, - "delta_threshold", - )?; - authorize_f64_change( - gos_epsilon(¤t.runtime_policy.delta), - gos_epsilon(&next.runtime_policy.delta), - policy.gos_epsilon_staleness.as_ref(), - ¤t.producer_id, - "gos_epsilon_staleness", - )?; - } - Ok(()) - } - - pub fn validate_frame( - &self, - frame: &SummaryFrameIdentity, - ) -> Result<(), TransmissionPlanError> { - if frame.identity_version != self.frame_identity.identity_version - || frame.plan_id != self.envelope.plan_id - || frame.plan_version != self.envelope.plan_version - || frame.backend_compat != self.envelope.backend_compat - || frame.series_identity.is_empty() - || frame.producer_epoch.is_empty() - || frame.sequence == 0 - || frame.window_start_unix_nano >= frame.window_end_unix_nano - || (frame.kind == SummaryFrameKind::Full - && self.frame_identity.require_checkpoint_for_full - && frame.checkpoint_id.is_none()) - || (frame.kind == SummaryFrameKind::Delta - && self.frame_identity.require_base_checkpoint_for_delta - && frame.base_checkpoint_id.is_none()) - { - return Err(TransmissionPlanError::InvalidFrame( - "identity/lifecycle/window/checkpoint fields do not satisfy the active contract" - .into(), - )); - } - if self.rules.iter().any(|rule| { - rule.materialization == frame.materialization - && rule.producer_id == frame.producer_id - && rule.schema_id == frame.schema_id - // A delta rule necessarily emits periodic full checkpoints; - // a full-only rule must never emit deltas. - && (frame.kind == SummaryFrameKind::Full - || rule.mode == TransmissionMode::Delta) - && rule.encoding == frame.encoding - }) { - Ok(()) - } else { - Err(TransmissionPlanError::UnknownFrame) - } - } -} - -fn validate_runtime_rule_policy( - rule: &TransmissionRule, - family: &StateFamilyContract, -) -> Result<(), TransmissionPlanError> { - let invalid = |reason: &str| TransmissionPlanError::InvalidRuntimePolicy { - producer_id: rule.producer_id.clone(), - reason: reason.into(), - }; - if let SamplingPolicy::Fixed { - probability, - estimator, - } = &rule.runtime_policy.sampling - { - if !probability.is_finite() || !(0.0..=1.0).contains(probability) || *probability == 0.0 { - return Err(invalid("sample probability must be finite and in (0, 1]")); - } - let supported = matches!( - (family, *estimator), - ( - StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::Hll, - .. - }, - SamplingEstimator::HashThreshold, - ) | ( - StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap, - .. - }, - SamplingEstimator::GeometricAdmission, - ) - ); - if !supported { - return Err(invalid( - "sampling estimator is not implemented for the materialization family", - )); - } - } - - if (rule.mode == TransmissionMode::Delta) != rule.runtime_policy.delta.is_some() { - return Err(invalid( - "delta policy must be present exactly when transmission mode is delta", - )); - } - if let Some(delta) = &rule.runtime_policy.delta { - if !delta.absolute_threshold.is_finite() || delta.absolute_threshold < 0.0 { - return Err(invalid("delta threshold must be finite and non-negative")); - } - if !matches!( - family, - StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::DDSketch - | SketchAlgorithm::Hll - | SketchAlgorithm::Cms - | SketchAlgorithm::CmsWithHeap - | SketchAlgorithm::CountSketch - | SketchAlgorithm::CountSketchWithHeap, - .. - } - ) { - return Err(invalid( - "delta transmission is not implemented for the materialization family", - )); - } - if let Some(gos) = &delta.gos { - if !matches!( - family, - StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap, - .. - } - ) || !gos.epsilon_staleness.is_finite() - || !(0.0..=1.0).contains(&gos.epsilon_staleness) - || gos.epsilon_staleness == 0.0 - || gos.sites == 0 - { - return Err(invalid( - "GOS requires a CountSketch family, epsilon in (0, 1], and at least one site", - )); - } - } - } - - let adaptation = &rule.runtime_policy.adaptation; - if adaptation.enabled - && (adaptation.max_evidence_age_ms == 0 || adaptation.min_evidence_samples == 0) - { - return Err(invalid( - "enabled adaptation requires non-zero evidence age and sample-count requirements", - )); - } - validate_f64_bounds(adaptation.sample_probability.as_ref(), 0.0, 1.0) - .map_err(|reason| invalid(reason))?; - validate_f64_bounds(adaptation.delta_threshold.as_ref(), 0.0, f64::MAX) - .map_err(|reason| invalid(reason))?; - validate_f64_bounds(adaptation.gos_epsilon_staleness.as_ref(), 0.0, 1.0) - .map_err(|reason| invalid(reason))?; - if let Some(bounds) = &adaptation.emit_every_ms { - if bounds.min == 0 - || bounds.min > bounds.max - || bounds.max_step == 0 - || !(bounds.min..=bounds.max).contains(&rule.emit_every_ms) - { - return Err(invalid("emit interval guardrails are invalid")); - } - } - if let Some(bounds) = &adaptation.sample_probability { - let current = sampling_probability(&rule.runtime_policy.sampling); - if current < bounds.min || current > bounds.max { - return Err(invalid( - "current sampling probability is outside guardrails", - )); - } - } - if let Some(bounds) = &adaptation.delta_threshold { - let Some(current) = rule - .runtime_policy - .delta - .as_ref() - .map(|policy| policy.absolute_threshold) - else { - return Err(invalid("delta guardrails require an active delta policy")); - }; - if current < bounds.min || current > bounds.max { - return Err(invalid("current delta threshold is outside guardrails")); - } - } - if let Some(bounds) = &adaptation.gos_epsilon_staleness { - let Some(current) = rule - .runtime_policy - .delta - .as_ref() - .and_then(|policy| policy.gos.as_ref()) - .map(|gos| gos.epsilon_staleness) - else { - return Err(invalid("GOS guardrails require an active GOS policy")); - }; - if current < bounds.min || current > bounds.max { - return Err(invalid("current GOS epsilon is outside guardrails")); - } - } - Ok(()) -} - -fn validate_f64_bounds( - bounds: Option<&AdaptiveF64Bounds>, - domain_min: f64, - domain_max: f64, -) -> Result<(), &'static str> { - let Some(bounds) = bounds else { - return Ok(()); - }; - if !bounds.min.is_finite() - || !bounds.max.is_finite() - || !bounds.max_step.is_finite() - || bounds.min < domain_min - || bounds.max > domain_max - || bounds.min > bounds.max - || bounds.max_step <= 0.0 - { - Err("floating-point adaptation guardrails are invalid") - } else { - Ok(()) - } -} - -fn sampling_probability(policy: &SamplingPolicy) -> f64 { - match policy { - SamplingPolicy::Disabled => 1.0, - SamplingPolicy::Fixed { probability, .. } => *probability, - } -} - -fn sampling_estimator(policy: &SamplingPolicy) -> Option { - match policy { - SamplingPolicy::Disabled => None, - SamplingPolicy::Fixed { estimator, .. } => Some(*estimator), - } -} - -fn delta_threshold(policy: &Option) -> f64 { - policy - .as_ref() - .map(|policy| policy.absolute_threshold) - .unwrap_or(0.0) -} - -fn gos_epsilon(policy: &Option) -> f64 { - policy - .as_ref() - .and_then(|policy| policy.gos.as_ref()) - .map(|gos| gos.epsilon_staleness) - .unwrap_or(0.0) -} - -fn delta_shape(policy: &Option) -> Option<(Option<(u32, GosThresholdMode)>,)> { - policy.as_ref().map(|policy| { - (policy - .gos - .as_ref() - .map(|gos| (gos.sites, gos.threshold_mode)),) - }) -} - -fn authorize_f64_change( - current: f64, - next: f64, - bounds: Option<&AdaptiveF64Bounds>, - producer_id: &str, - knob: &'static str, -) -> Result<(), TransmissionPlanError> { - if current == next { - return Ok(()); - } - let allowed = bounds.is_some_and(|bounds| { - next.is_finite() - && (bounds.min..=bounds.max).contains(&next) - && (next - current).abs() <= bounds.max_step - }); - if allowed { - Ok(()) - } else { - Err(TransmissionPlanError::AdaptationOutOfBounds { - producer_id: producer_id.into(), - knob, - }) - } -} - -fn authorize_u64_change( - current: u64, - next: u64, - bounds: Option<&AdaptiveU64Bounds>, - producer_id: &str, - knob: &'static str, -) -> Result<(), TransmissionPlanError> { - if current == next { - return Ok(()); - } - let allowed = bounds.is_some_and(|bounds| { - (bounds.min..=bounds.max).contains(&next) && current.abs_diff(next) <= bounds.max_step - }); - if allowed { - Ok(()) - } else { - Err(TransmissionPlanError::AdaptationOutOfBounds { - producer_id: producer_id.into(), - knob, - }) - } -} - #[derive(Debug, Error)] pub enum CompileError { #[error("invalid backend-local workload snapshot: {0}")] @@ -2089,12 +1279,15 @@ impl PhysicalCompiler { query_id: "precompute-plan".into(), reason: error.to_string(), })?; - let mut transmission_plan = - TransmissionPlan::build(envelope.clone(), &precompute_plan, &runtime_policies) - .map_err(|error| CompileError::Query { - query_id: "transmission-plan".into(), - reason: error.to_string(), - })?; + let mut transmission_plan = crate::physical::compiler::compile_transmission_plan( + envelope.clone(), + &precompute_plan, + &runtime_policies, + ) + .map_err(|error| CompileError::Query { + query_id: "transmission-plan".into(), + reason: error.to_string(), + })?; let mut collector_plans = producer_ids .into_iter() .map(|collector_id| CollectorPlan { @@ -5222,13 +4415,21 @@ mod tests { collector.validate_against_catalog(catalog).unwrap(); collector.envelope.plan_version += 1; assert!(collector.validate_against_catalog(catalog).is_err()); - assert!(validate_catalog_projection( - Some(&catalog.reference().unwrap()), - &bundle.envelope, - [asap_types::PolicyFingerprint(u64::MAX).into()], - catalog - ) - .is_err()); + let mut unknown_materialization = bundle.transmission_plan.clone(); + unknown_materialization.rules.push(TransmissionRule { + materialization: asap_types::PolicyFingerprint(u64::MAX).into(), + producer_id: "foreign".into(), + schema_id: "foreign".into(), + mode: TransmissionMode::Full, + encoding: StateEncoding::ExactAccumulatorV1, + emit_every_ms: 60_000, + full_checkpoint_every_ms: None, + destination_ref: "backend".into(), + runtime_policy: Default::default(), + }); + assert!(unknown_materialization + .validate_against_catalog(catalog) + .is_err()); let encoded = serde_json::to_value(&collector).unwrap(); assert!(encoded["summary_catalog"] .get("summary_descriptors") @@ -5542,10 +4743,14 @@ mod tests { assert_eq!(plan.ingest.timestamp_unit, TimestampUnit::UnixMilliseconds); assert!(plan.producers.is_empty()); plan.validate().expect("valid backend-local projection"); - TransmissionPlan::build(bundle.envelope, &plan, &BTreeMap::new()) - .expect("empty backend-local transmission contract") - .validate(&plan) - .expect("valid empty transmission plan"); + crate::physical::compiler::compile_transmission_plan( + bundle.envelope, + &plan, + &BTreeMap::new(), + ) + .expect("empty backend-local transmission contract") + .validate(&plan) + .expect("valid empty transmission plan"); } #[test] @@ -6005,6 +5210,9 @@ mod tests { ), full_selected ); + // Publication validates stored extent independently of slide cadence. + // In particular the full-window candidate stores 60s at a 10s slide. + bundle.publication().unwrap().validate().unwrap(); assert_eq!(bundle.precompute_plan.materializations[0].window_size, 60); assert_eq!( bundle.precompute_plan.materializations[0].slide_interval, @@ -6213,7 +5421,7 @@ mod tests { #[test] fn runtime_policy_uses_canonical_sampling_and_gos_allocators() { - let sampling = SamplingPolicy::from_accuracy_budget( + let sampling = sampling_policy_from_accuracy_budget( 0.05, 1_000.0, SamplingEstimator::GeometricAdmission, @@ -6224,64 +5432,26 @@ mod tests { assert!((probability - 1.0 / 3.5).abs() < 1e-9); let gos = - GosPolicy::from_accuracy_budget(0.1, 0.03, 4, 1.0, 1.0, GosThresholdMode::Isotropic) + gos_policy_from_accuracy_budget(0.1, 0.03, 4, 1.0, 1.0, GosThresholdMode::Isotropic) .expect("positive communication allocation"); assert!((gos.epsilon_staleness - 0.035).abs() < 1e-12); assert_eq!(gos.sites, 4); } #[test] - fn runtime_policy_is_family_and_mode_checked() { + fn runtime_policy_encoding_is_checked() { let bundle = PhysicalCompiler .compile( request("q", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) .expect("compile"); - let mut rule = bundle.transmission_plan.rules[0].clone(); let mut invalid_encoding = bundle.transmission_plan.clone(); invalid_encoding.rules[0].encoding = StateEncoding::ExactAccumulatorV1; assert!(matches!( invalid_encoding.validate(&bundle.precompute_plan), Err(TransmissionPlanError::InvalidRule(_)) )); - rule.runtime_policy.sampling = SamplingPolicy::Fixed { - probability: 0.5, - estimator: SamplingEstimator::HashThreshold, - }; - let hll = StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::Hll, - parameters: SketchParams::Hll { precision: 14 }, - }; - let count_sketch = StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::CountSketch, - parameters: SketchParams::CountSketch { - width: 128, - depth: 4, - }, - }; - validate_runtime_rule_policy(&rule, &hll).expect("HLL supports hash-threshold sampling"); - assert!(matches!( - validate_runtime_rule_policy(&rule, &count_sketch), - Err(TransmissionPlanError::InvalidRuntimePolicy { .. }) - )); - - rule.runtime_policy.sampling = SamplingPolicy::Disabled; - rule.mode = TransmissionMode::Delta; - rule.full_checkpoint_every_ms = Some(300_000); - rule.runtime_policy.delta = Some(DeltaPolicy { - absolute_threshold: 0.0, - gos: Some(GosPolicy { - epsilon_staleness: 0.02, - sites: 2, - threshold_mode: GosThresholdMode::Isotropic, - }), - }); - validate_runtime_rule_policy(&rule, &count_sketch).expect("CountSketch supports delta GOS"); - assert!(matches!( - validate_runtime_rule_policy(&rule, &hll), - Err(TransmissionPlanError::InvalidRuntimePolicy { .. }) - )); } #[test] diff --git a/control_plane/src/physical/publication.rs b/control_plane/src/physical/publication.rs index 42e28d086..63081fa5b 100644 --- a/control_plane/src/physical/publication.rs +++ b/control_plane/src/physical/publication.rs @@ -1,128 +1,7 @@ -//! Canonical publication document for one catalog generation. -use super::compiler::{CollectorPlan, PhysicalPlan, PrecomputePlan, TransmissionPlan}; -use super::summary_catalog::SummaryCatalog; -use crate::query_plan::QueryPlan; -use serde::{Deserialize, Serialize}; +//! Control-plane construction of the shared catalog publication contract. +use super::compiler::PhysicalPlan; +pub use asap_types::plan_publication::{PhysicalPlanInstallRequest, PhysicalPlanPublication}; -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PhysicalPlanPublication { - pub summary_catalog: SummaryCatalog, - pub precompute_plan: PrecomputePlan, - pub collector_plans: Vec, - pub transmission_plan: TransmissionPlan, - pub query_plan: QueryPlan, -} - -/// Complete typed envelope accepted by the data-plane install endpoint. -/// -/// Runtime-only routing and adaptation evidence decorate the compiler-owned -/// publication without changing its catalog generation. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PhysicalPlanInstallRequest { - pub summary_catalog: SummaryCatalog, - #[serde(default)] - pub collector_plans: Vec, - pub precompute_plan: PrecomputePlan, - pub transmission_plan: TransmissionPlan, - pub query_plan: QueryPlan, - pub storage_routing: Option, - #[serde(default)] - pub adaptation_evidence: Vec, -} - -impl PhysicalPlanPublication { - /// Validate every plan against the shared catalog snapshot. - pub fn validate(&self) -> Result<(), String> { - let catalog = &self.summary_catalog; - self.precompute_plan - .validate_against_catalog(catalog) - .map_err(|e| e.to_string())?; - self.transmission_plan - .validate(&self.precompute_plan) - .map_err(|e| e.to_string())?; - self.transmission_plan - .validate_against_catalog(catalog) - .map_err(|e| e.to_string())?; - self.query_plan - .validate_against_catalog(catalog) - .map_err(|e| e.to_string())?; - let materializations = self - .precompute_plan - .materializations - .iter() - .map(|config| (config.policy_fingerprint(), config)) - .collect::>(); - for entry in self.query_plan.entries.values() { - for binding in entry.materialization_bindings() { - let config = materializations - .get(&binding.materialization.fingerprint()) - .copied() - .ok_or("query binding has no precompute materialization")?; - if config.slide_interval.checked_mul(1000) != Some(binding.window_ms) { - return Err("query pane differs from precompute emission interval".into()); - } - if config.pane_origin_ms != binding.pane_origin_ms { - return Err("query pane origin differs from precompute definition".into()); - } - } - } - let mut collectors = std::collections::BTreeSet::new(); - for collector in &self.collector_plans { - if collector.envelope != self.precompute_plan.envelope - || !collectors.insert(&collector.collector_id) - { - return Err("collector envelope mismatch or duplicate collector".into()); - } - collector - .validate_against_catalog(catalog) - .map_err(|e| e.to_string())?; - for rule in &collector.transmission_rules { - if !self.transmission_plan.rules.contains(rule) { - return Err( - "collector transmission rule absent from published transmission plan" - .into(), - ); - } - } - } - for producer in &self.precompute_plan.producers { - let collector = self - .collector_plans - .iter() - .find(|c| c.collector_id == producer.collector_id) - .ok_or("precompute producer has no published CollectorPlan")?; - if !collector - .materializations - .iter() - .any(|m| m.materialization == producer.materialization) - { - return Err( - "collector does not produce referenced precompute materialization".into(), - ); - } - } - Ok(()) - } - - pub fn install_request( - &self, - storage_routing: Option, - adaptation_evidence: Vec, - ) -> Result { - self.validate()?; - Ok(PhysicalPlanInstallRequest { - summary_catalog: self.summary_catalog.clone(), - collector_plans: self.collector_plans.clone(), - precompute_plan: self.precompute_plan.clone(), - transmission_plan: self.transmission_plan.clone(), - query_plan: self.query_plan.clone(), - storage_routing, - adaptation_evidence, - }) - } -} impl PhysicalPlan { pub fn publication(&self) -> Result { let artifact = PhysicalPlanPublication { diff --git a/crates/asap_types/src/lib.rs b/crates/asap_types/src/lib.rs index 73fe5aa8e..c525f6d61 100644 --- a/crates/asap_types/src/lib.rs +++ b/crates/asap_types/src/lib.rs @@ -6,8 +6,10 @@ pub mod enums; pub mod executable_plan; pub mod key_by_label_names; pub mod monitor_spec; +pub mod plan_publication; pub mod policy_fingerprint; pub mod policy_registry; +pub mod producer_plan; pub mod query_requirements; pub mod routing_index; pub mod sds; diff --git a/crates/asap_types/src/plan_publication.rs b/crates/asap_types/src/plan_publication.rs new file mode 100644 index 000000000..35ce812f5 --- /dev/null +++ b/crates/asap_types/src/plan_publication.rs @@ -0,0 +1,126 @@ +//! Canonical publication document for one catalog generation. +use crate::precompute_plan::PrecomputePlan; +use crate::producer_plan::{CollectorPlan, TransmissionPlan}; +use crate::query_plan::QueryPlan; +use crate::summary_catalog::SummaryCatalog; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PhysicalPlanPublication { + pub summary_catalog: SummaryCatalog, + pub precompute_plan: PrecomputePlan, + pub collector_plans: Vec, + pub transmission_plan: TransmissionPlan, + pub query_plan: QueryPlan, +} + +/// Complete typed envelope accepted by the data-plane install endpoint. +/// +/// Runtime-only routing and adaptation evidence decorate the shared +/// publication without changing its catalog generation. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PhysicalPlanInstallRequest { + pub summary_catalog: SummaryCatalog, + #[serde(default)] + pub collector_plans: Vec, + pub precompute_plan: PrecomputePlan, + pub transmission_plan: TransmissionPlan, + pub query_plan: QueryPlan, + pub storage_routing: Option, + #[serde(default)] + pub adaptation_evidence: Vec, +} + +impl PhysicalPlanPublication { + /// Validate every plan against the shared catalog snapshot. + pub fn validate(&self) -> Result<(), String> { + let catalog = &self.summary_catalog; + self.precompute_plan + .validate_against_catalog(catalog) + .map_err(|e| e.to_string())?; + self.transmission_plan + .validate(&self.precompute_plan) + .map_err(|e| e.to_string())?; + self.transmission_plan + .validate_against_catalog(catalog) + .map_err(|e| e.to_string())?; + self.query_plan + .validate_against_catalog(catalog) + .map_err(|e| e.to_string())?; + let materializations = self + .precompute_plan + .materializations + .iter() + .map(|config| (config.policy_fingerprint(), config)) + .collect::>(); + for entry in self.query_plan.entries.values() { + for binding in entry.materialization_bindings() { + let config = materializations + .get(&binding.materialization.fingerprint()) + .copied() + .ok_or("query binding has no precompute materialization")?; + if config.stored_window_ms() != binding.window_ms { + return Err("query pane differs from precompute stored window".into()); + } + if config.pane_origin_ms != binding.pane_origin_ms { + return Err("query pane origin differs from precompute definition".into()); + } + } + } + let mut collectors = std::collections::BTreeSet::new(); + for collector in &self.collector_plans { + if collector.envelope != self.precompute_plan.envelope + || !collectors.insert(&collector.collector_id) + { + return Err("collector envelope mismatch or duplicate collector".into()); + } + collector + .validate_against_catalog(catalog) + .map_err(|e| e.to_string())?; + for rule in &collector.transmission_rules { + if !self.transmission_plan.rules.contains(rule) { + return Err( + "collector transmission rule absent from published transmission plan" + .into(), + ); + } + } + } + for producer in &self.precompute_plan.producers { + let collector = self + .collector_plans + .iter() + .find(|c| c.collector_id == producer.collector_id) + .ok_or("precompute producer has no published CollectorPlan")?; + if !collector + .materializations + .iter() + .any(|m| m.materialization == producer.materialization) + { + return Err( + "collector does not produce referenced precompute materialization".into(), + ); + } + } + Ok(()) + } + + pub fn install_request( + &self, + storage_routing: Option, + adaptation_evidence: Vec, + ) -> Result { + self.validate()?; + Ok(PhysicalPlanInstallRequest { + summary_catalog: self.summary_catalog.clone(), + collector_plans: self.collector_plans.clone(), + precompute_plan: self.precompute_plan.clone(), + transmission_plan: self.transmission_plan.clone(), + query_plan: self.query_plan.clone(), + storage_routing, + adaptation_evidence, + }) + } +} diff --git a/crates/asap_types/src/producer_plan.rs b/crates/asap_types/src/producer_plan.rs new file mode 100644 index 000000000..81d4b36e6 --- /dev/null +++ b/crates/asap_types/src/producer_plan.rs @@ -0,0 +1,879 @@ +//! Installed collector and transmission contracts shared across producers and consumers. +//! Deployment choices and accuracy-budget allocation remain in the control plane. +use crate::precompute_plan::{PlanEnvelope, PrecomputePlan, StateEncoding, StateFamilyContract}; +use planner_types::post_asap::{SketchAlgorithm, SummaryWindowFramework}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::BTreeSet; +use thiserror::Error; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct CollectorMaterialization { + pub query_id: String, + pub materialization: crate::sds::SummaryDefinitionId, + pub metric: String, + pub algorithm: String, + pub parameters: Value, + pub group_by: Vec, + pub window_secs: u64, + pub abstract_window_framework: SummaryWindowFramework, + pub window_implementation_id: String, + pub slide_secs: u64, + #[serde( + default, + alias = "paneOriginMs", + skip_serializing_if = "Option::is_none" + )] + pub pane_origin_ms: Option, + pub window_layout: crate::WindowMaterializationLayout, + pub evidence_source: Option, + pub lifecycle: CollectorLifecycle, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct CollectorLifecycle { + pub kind: String, + pub maintenance_mode: String, + pub evaluation_schedule: String, + pub output_representation: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct CollectorPlan { + /// Absent only in legacy artifacts; catalog-aware validation requires it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub summary_catalog: Option, + pub collector_id: String, + pub envelope: PlanEnvelope, + pub materializations: Vec, + pub transmission_rules: Vec, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum TransmissionMode { + Full, + Delta, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum SequenceScope { + MaterializationSeriesProducerEpoch, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct FrameIdentityContract { + pub identity_version: u32, + pub sequence_scope: SequenceScope, + pub require_checkpoint_for_full: bool, + pub require_base_checkpoint_for_delta: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TransmissionRule { + pub materialization: crate::sds::SummaryDefinitionId, + pub producer_id: String, + pub schema_id: String, + pub mode: TransmissionMode, + pub encoding: StateEncoding, + pub emit_every_ms: u64, + pub full_checkpoint_every_ms: Option, + pub destination_ref: String, + /// Plan-owned runtime knobs. These values are part of the immutable plan + /// generation; live feedback may only change them by publishing a + /// successor generation accepted by [`TransmissionPlan::authorize_successor`]. + #[serde(default)] + pub runtime_policy: RuntimeRulePolicy, +} + +/// How the collector admits updates before sketch maintenance. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "mode", rename_all = "snake_case", deny_unknown_fields)] +pub enum SamplingPolicy { + Disabled, + Fixed { + /// Probability in `(0, 1]`; `1` is valid but should normally be + /// represented by `Disabled`. + probability: f64, + estimator: SamplingEstimator, + }, +} + +impl Default for SamplingPolicy { + fn default() -> Self { + Self::Disabled + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum SamplingEstimator { + /// Hash-threshold element sampling, used by cardinality summaries. + HashThreshold, + /// Geometric admission/Nitro-style update sampling, used by frequency + /// summaries. The sketch readout carries the corresponding correction. + GeometricAdmission, +} + +/// Norm-adaptive Group-of-Sketches delta gating. GOS is meaningful only for +/// CountSketch families and only when the transmission rule is in delta mode. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct GosPolicy { + pub epsilon_staleness: f64, + pub sites: u32, + pub threshold_mode: GosThresholdMode, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum GosThresholdMode { + Isotropic, + Anisotropic, +} + +/// Sparse-delta semantics within a delta transmission rule. An absolute +/// threshold of zero sends every changed cell. When `gos` is present it +/// replaces the fixed threshold with the GOS norm-adaptive threshold. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct DeltaPolicy { + pub absolute_threshold: f64, + pub gos: Option, +} + +/// Inclusive bounds for one floating-point adaptation knob. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct AdaptiveF64Bounds { + pub min: f64, + pub max: f64, + pub max_step: f64, +} + +/// Inclusive bounds for one integer adaptation knob. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdaptiveU64Bounds { + pub min: u64, + pub max: u64, + pub max_step: u64, +} + +/// Guardrails for telemetry-driven runtime adaptation. This is an +/// authorization contract, not an instruction to mutate the active plan. +/// Every accepted change becomes a staged successor PhysicalPlan. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct RuntimeAdaptationPolicy { + pub enabled: bool, + pub not_before_unix_ms: u64, + pub max_evidence_age_ms: u64, + pub min_evidence_samples: u64, + pub sample_probability: Option, + pub emit_every_ms: Option, + pub delta_threshold: Option, + pub gos_epsilon_staleness: Option, +} + +impl Default for RuntimeAdaptationPolicy { + fn default() -> Self { + Self { + enabled: false, + not_before_unix_ms: 0, + max_evidence_age_ms: 0, + min_evidence_samples: 0, + sample_probability: None, + emit_every_ms: None, + delta_threshold: None, + gos_epsilon_staleness: None, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] +#[serde(deny_unknown_fields)] +pub struct RuntimeRulePolicy { + #[serde(default)] + pub sampling: SamplingPolicy, + pub delta: Option, + #[serde(default)] + pub adaptation: RuntimeAdaptationPolicy, +} + +/// Identity and sufficiency information for evidence authorizing one rule's +/// successor knobs. Raw measurements remain in the runtime-samples store; the +/// authorization boundary needs only their exact provenance and sample count. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct RuntimeAdaptationEvidence { + pub plan_id: u64, + pub plan_version: u64, + pub materialization: crate::sds::SummaryDefinitionId, + pub producer_id: String, + pub schema_id: String, + pub producer_version: String, + pub observed_at_unix_ms: u64, + pub sample_count: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TransmissionPlan { + /// Absent only in legacy artifacts; catalog-aware validation requires it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub summary_catalog: Option, + pub envelope: PlanEnvelope, + pub frame_identity: FrameIdentityContract, + pub rules: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum SummaryFrameKind { + Full, + Delta, +} + +/// Identity attached to every summary record. Window bounds come from the +/// data point; the remaining fields are carried as reserved `asap.frame.*` +/// attributes until the modified-OTLP schema gains a dedicated message. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct SummaryFrameIdentity { + pub identity_version: u32, + pub plan_id: u64, + pub plan_version: u64, + pub backend_compat: String, + pub materialization: crate::sds::SummaryDefinitionId, + /// Canonical producer-side identity for one concrete retained-label group. + pub series_identity: String, + pub schema_id: String, + pub producer_id: String, + pub producer_epoch: String, + pub window_start_unix_nano: u64, + pub window_end_unix_nano: u64, + pub sequence: u64, + pub kind: SummaryFrameKind, + pub encoding: StateEncoding, + pub checkpoint_id: Option, + pub base_checkpoint_id: Option, +} + +#[derive(Debug, Error, PartialEq, Eq)] +pub enum TransmissionPlanError { + #[error("summary catalog mismatch: {0}")] + Catalog(String), + #[error("TransmissionPlan envelope differs from PrecomputePlan")] + EnvelopeMismatch, + #[error("transmission rules do not exactly match precompute producer bindings")] + ProducerSetMismatch, + #[error("invalid transmission rule for producer {0}")] + InvalidRule(String), + #[error("frame identity is invalid: {0}")] + InvalidFrame(String), + #[error("frame has no matching transmission rule")] + UnknownFrame, + #[error("runtime policy for producer {producer_id} is invalid: {reason}")] + InvalidRuntimePolicy { producer_id: String, reason: String }, + #[error("runtime adaptation successor is invalid: {0}")] + InvalidSuccessor(String), + #[error("runtime adaptation evidence for producer {0} is missing or invalid")] + InvalidAdaptationEvidence(String), + #[error("runtime adaptation for producer {producer_id} exceeds guardrails: {knob}")] + AdaptationOutOfBounds { + producer_id: String, + knob: &'static str, + }, +} + +fn validate_catalog_projection( + reference: Option<&crate::sds::CatalogGeneration>, + envelope: &PlanEnvelope, + materializations: impl IntoIterator, + catalog: &crate::summary_catalog::SummaryCatalog, +) -> Result<(), TransmissionPlanError> { + let expected = catalog + .reference() + .map_err(|error| TransmissionPlanError::Catalog(error.to_string()))?; + if reference != Some(&expected) + || envelope.plan_id != catalog.plan_id + || envelope.plan_version != catalog.plan_version + { + return Err(TransmissionPlanError::Catalog( + "missing or different snapshot reference".into(), + )); + } + for id in materializations { + if !catalog.materializations.contains_key(&id) { + return Err(TransmissionPlanError::Catalog(format!( + "unknown materialization {}", + id.as_u64() + ))); + } + } + Ok(()) +} + +impl CollectorPlan { + pub fn validate_against_catalog( + &self, + catalog: &crate::summary_catalog::SummaryCatalog, + ) -> Result<(), TransmissionPlanError> { + validate_catalog_projection( + self.summary_catalog.as_ref(), + &self.envelope, + self.materializations + .iter() + .map(|m| m.materialization) + .chain(self.transmission_rules.iter().map(|r| r.materialization)), + catalog, + ) + } +} + +impl TransmissionPlan { + pub fn validate_against_catalog( + &self, + catalog: &crate::summary_catalog::SummaryCatalog, + ) -> Result<(), TransmissionPlanError> { + validate_catalog_projection( + self.summary_catalog.as_ref(), + &self.envelope, + self.rules.iter().map(|r| r.materialization), + catalog, + ) + } + + pub fn validate(&self, precompute: &PrecomputePlan) -> Result<(), TransmissionPlanError> { + if self.summary_catalog != precompute.summary_catalog { + return Err(TransmissionPlanError::Catalog( + "transmission and precompute plans reference different catalog snapshots".into(), + )); + } + if self.envelope != precompute.envelope { + return Err(TransmissionPlanError::EnvelopeMismatch); + } + let expected: BTreeSet<_> = precompute + .producers + .iter() + .map(|producer| { + ( + producer.materialization, + producer.producer_id.as_str(), + producer.schema_id.as_str(), + ) + }) + .collect(); + let actual: BTreeSet<_> = self + .rules + .iter() + .map(|rule| { + ( + rule.materialization, + rule.producer_id.as_str(), + rule.schema_id.as_str(), + ) + }) + .collect(); + if expected != actual || actual.len() != self.rules.len() { + return Err(TransmissionPlanError::ProducerSetMismatch); + } + for rule in &self.rules { + let valid_checkpoint_cadence = match (rule.mode, rule.full_checkpoint_every_ms) { + (TransmissionMode::Full, None) => true, + (TransmissionMode::Delta, Some(full_every)) => { + rule.emit_every_ms > 0 + && full_every >= rule.emit_every_ms + && full_every % rule.emit_every_ms == 0 + } + _ => false, + }; + if rule.emit_every_ms == 0 + || rule.destination_ref.is_empty() + || !valid_checkpoint_cadence + { + return Err(TransmissionPlanError::InvalidRule(rule.producer_id.clone())); + } + let schema = precompute + .schemas + .iter() + .find(|schema| schema.materialization == rule.materialization) + .expect("producer set validation guarantees a matching schema"); + if !schema.encodings.contains(&rule.encoding) { + return Err(TransmissionPlanError::InvalidRule(rule.producer_id.clone())); + } + validate_runtime_rule_policy(rule, &schema.family)?; + } + Ok(()) + } + + /// Authorize a telemetry-driven successor without mutating this active + /// plan. Semantic identity, codecs, destination, transmission mode and + /// checkpoint cadence remain fixed. Only explicitly bounded runtime knobs + /// may move, and each changed rule needs fresh evidence attributed to the + /// exact active generation. + pub fn authorize_successor( + &self, + successor: &TransmissionPlan, + evidence: &[RuntimeAdaptationEvidence], + now_unix_ms: u64, + ) -> Result<(), TransmissionPlanError> { + if successor.envelope.plan_id != self.envelope.plan_id + || successor.envelope.plan_version != self.envelope.plan_version.saturating_add(1) + || successor.envelope.backend_compat != self.envelope.backend_compat + || successor.envelope.planner_revision != self.envelope.planner_revision + || successor.envelope.capability_snapshot_id != self.envelope.capability_snapshot_id + || successor.envelope.generated_at_unix_ms < self.envelope.generated_at_unix_ms + || successor.envelope.activation_unix_ms < successor.envelope.generated_at_unix_ms + || successor.rules.len() != self.rules.len() + || successor.frame_identity != self.frame_identity + { + return Err(TransmissionPlanError::InvalidSuccessor( + "successor must be the next version of the same semantic/capability generation" + .into(), + )); + } + + for current in &self.rules { + let Some(next) = successor.rules.iter().find(|candidate| { + candidate.materialization == current.materialization + && candidate.producer_id == current.producer_id + && candidate.schema_id == current.schema_id + }) else { + return Err(TransmissionPlanError::InvalidSuccessor(format!( + "missing rule for producer {}", + current.producer_id + ))); + }; + if current.mode != next.mode + || current.encoding != next.encoding + || current.full_checkpoint_every_ms != next.full_checkpoint_every_ms + || current.destination_ref != next.destination_ref + || current.runtime_policy.adaptation != next.runtime_policy.adaptation + || sampling_estimator(¤t.runtime_policy.sampling) + != sampling_estimator(&next.runtime_policy.sampling) + || delta_shape(¤t.runtime_policy.delta) + != delta_shape(&next.runtime_policy.delta) + { + return Err(TransmissionPlanError::InvalidSuccessor(format!( + "rule identity/codec/mode/guardrails drifted for producer {}", + current.producer_id + ))); + } + if current.emit_every_ms == next.emit_every_ms + && current.runtime_policy.sampling == next.runtime_policy.sampling + && current.runtime_policy.delta == next.runtime_policy.delta + { + continue; + } + let policy = ¤t.runtime_policy.adaptation; + if !policy.enabled || now_unix_ms < policy.not_before_unix_ms { + return Err(TransmissionPlanError::AdaptationOutOfBounds { + producer_id: current.producer_id.clone(), + knob: "adaptation_disabled_or_in_cooldown", + }); + } + let has_evidence = evidence.iter().any(|item| { + item.plan_id == self.envelope.plan_id + && item.plan_version == self.envelope.plan_version + && item.materialization == current.materialization + && item.producer_id == current.producer_id + && item.schema_id == current.schema_id + && !item.producer_version.trim().is_empty() + && item.sample_count >= policy.min_evidence_samples + && item.observed_at_unix_ms <= now_unix_ms + && now_unix_ms.saturating_sub(item.observed_at_unix_ms) + <= policy.max_evidence_age_ms + }); + if !has_evidence { + return Err(TransmissionPlanError::InvalidAdaptationEvidence( + current.producer_id.clone(), + )); + } + authorize_f64_change( + sampling_probability(¤t.runtime_policy.sampling), + sampling_probability(&next.runtime_policy.sampling), + policy.sample_probability.as_ref(), + ¤t.producer_id, + "sample_probability", + )?; + authorize_u64_change( + current.emit_every_ms, + next.emit_every_ms, + policy.emit_every_ms.as_ref(), + ¤t.producer_id, + "emit_every_ms", + )?; + authorize_f64_change( + delta_threshold(¤t.runtime_policy.delta), + delta_threshold(&next.runtime_policy.delta), + policy.delta_threshold.as_ref(), + ¤t.producer_id, + "delta_threshold", + )?; + authorize_f64_change( + gos_epsilon(¤t.runtime_policy.delta), + gos_epsilon(&next.runtime_policy.delta), + policy.gos_epsilon_staleness.as_ref(), + ¤t.producer_id, + "gos_epsilon_staleness", + )?; + } + Ok(()) + } + + pub fn validate_frame( + &self, + frame: &SummaryFrameIdentity, + ) -> Result<(), TransmissionPlanError> { + if frame.identity_version != self.frame_identity.identity_version + || frame.plan_id != self.envelope.plan_id + || frame.plan_version != self.envelope.plan_version + || frame.backend_compat != self.envelope.backend_compat + || frame.series_identity.is_empty() + || frame.producer_epoch.is_empty() + || frame.sequence == 0 + || frame.window_start_unix_nano >= frame.window_end_unix_nano + || (frame.kind == SummaryFrameKind::Full + && self.frame_identity.require_checkpoint_for_full + && frame.checkpoint_id.is_none()) + || (frame.kind == SummaryFrameKind::Delta + && self.frame_identity.require_base_checkpoint_for_delta + && frame.base_checkpoint_id.is_none()) + { + return Err(TransmissionPlanError::InvalidFrame( + "identity/lifecycle/window/checkpoint fields do not satisfy the active contract" + .into(), + )); + } + if self.rules.iter().any(|rule| { + rule.materialization == frame.materialization + && rule.producer_id == frame.producer_id + && rule.schema_id == frame.schema_id + // A delta rule necessarily emits periodic full checkpoints; + // a full-only rule must never emit deltas. + && (frame.kind == SummaryFrameKind::Full + || rule.mode == TransmissionMode::Delta) + && rule.encoding == frame.encoding + }) { + Ok(()) + } else { + Err(TransmissionPlanError::UnknownFrame) + } + } +} + +fn validate_runtime_rule_policy( + rule: &TransmissionRule, + family: &StateFamilyContract, +) -> Result<(), TransmissionPlanError> { + let invalid = |reason: &str| TransmissionPlanError::InvalidRuntimePolicy { + producer_id: rule.producer_id.clone(), + reason: reason.into(), + }; + if let SamplingPolicy::Fixed { + probability, + estimator, + } = &rule.runtime_policy.sampling + { + if !probability.is_finite() || !(0.0..=1.0).contains(probability) || *probability == 0.0 { + return Err(invalid("sample probability must be finite and in (0, 1]")); + } + let supported = matches!( + (family, *estimator), + ( + StateFamilyContract::Sketch { + algorithm: SketchAlgorithm::Hll, + .. + }, + SamplingEstimator::HashThreshold, + ) | ( + StateFamilyContract::Sketch { + algorithm: SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap, + .. + }, + SamplingEstimator::GeometricAdmission, + ) + ); + if !supported { + return Err(invalid( + "sampling estimator is not implemented for the materialization family", + )); + } + } + + if (rule.mode == TransmissionMode::Delta) != rule.runtime_policy.delta.is_some() { + return Err(invalid( + "delta policy must be present exactly when transmission mode is delta", + )); + } + if let Some(delta) = &rule.runtime_policy.delta { + if !delta.absolute_threshold.is_finite() || delta.absolute_threshold < 0.0 { + return Err(invalid("delta threshold must be finite and non-negative")); + } + if !matches!( + family, + StateFamilyContract::Sketch { + algorithm: SketchAlgorithm::DDSketch + | SketchAlgorithm::Hll + | SketchAlgorithm::Cms + | SketchAlgorithm::CmsWithHeap + | SketchAlgorithm::CountSketch + | SketchAlgorithm::CountSketchWithHeap, + .. + } + ) { + return Err(invalid( + "delta transmission is not implemented for the materialization family", + )); + } + if let Some(gos) = &delta.gos { + if !matches!( + family, + StateFamilyContract::Sketch { + algorithm: SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap, + .. + } + ) || !gos.epsilon_staleness.is_finite() + || !(0.0..=1.0).contains(&gos.epsilon_staleness) + || gos.epsilon_staleness == 0.0 + || gos.sites == 0 + { + return Err(invalid( + "GOS requires a CountSketch family, epsilon in (0, 1], and at least one site", + )); + } + } + } + + let adaptation = &rule.runtime_policy.adaptation; + if adaptation.enabled + && (adaptation.max_evidence_age_ms == 0 || adaptation.min_evidence_samples == 0) + { + return Err(invalid( + "enabled adaptation requires non-zero evidence age and sample-count requirements", + )); + } + validate_f64_bounds(adaptation.sample_probability.as_ref(), 0.0, 1.0) + .map_err(|reason| invalid(reason))?; + validate_f64_bounds(adaptation.delta_threshold.as_ref(), 0.0, f64::MAX) + .map_err(|reason| invalid(reason))?; + validate_f64_bounds(adaptation.gos_epsilon_staleness.as_ref(), 0.0, 1.0) + .map_err(|reason| invalid(reason))?; + if let Some(bounds) = &adaptation.emit_every_ms { + if bounds.min == 0 + || bounds.min > bounds.max + || bounds.max_step == 0 + || !(bounds.min..=bounds.max).contains(&rule.emit_every_ms) + { + return Err(invalid("emit interval guardrails are invalid")); + } + } + if let Some(bounds) = &adaptation.sample_probability { + let current = sampling_probability(&rule.runtime_policy.sampling); + if current < bounds.min || current > bounds.max { + return Err(invalid( + "current sampling probability is outside guardrails", + )); + } + } + if let Some(bounds) = &adaptation.delta_threshold { + let Some(current) = rule + .runtime_policy + .delta + .as_ref() + .map(|policy| policy.absolute_threshold) + else { + return Err(invalid("delta guardrails require an active delta policy")); + }; + if current < bounds.min || current > bounds.max { + return Err(invalid("current delta threshold is outside guardrails")); + } + } + if let Some(bounds) = &adaptation.gos_epsilon_staleness { + let Some(current) = rule + .runtime_policy + .delta + .as_ref() + .and_then(|policy| policy.gos.as_ref()) + .map(|gos| gos.epsilon_staleness) + else { + return Err(invalid("GOS guardrails require an active GOS policy")); + }; + if current < bounds.min || current > bounds.max { + return Err(invalid("current GOS epsilon is outside guardrails")); + } + } + Ok(()) +} + +fn validate_f64_bounds( + bounds: Option<&AdaptiveF64Bounds>, + domain_min: f64, + domain_max: f64, +) -> Result<(), &'static str> { + let Some(bounds) = bounds else { + return Ok(()); + }; + if !bounds.min.is_finite() + || !bounds.max.is_finite() + || !bounds.max_step.is_finite() + || bounds.min < domain_min + || bounds.max > domain_max + || bounds.min > bounds.max + || bounds.max_step <= 0.0 + { + Err("floating-point adaptation guardrails are invalid") + } else { + Ok(()) + } +} + +fn sampling_probability(policy: &SamplingPolicy) -> f64 { + match policy { + SamplingPolicy::Disabled => 1.0, + SamplingPolicy::Fixed { probability, .. } => *probability, + } +} + +fn sampling_estimator(policy: &SamplingPolicy) -> Option { + match policy { + SamplingPolicy::Disabled => None, + SamplingPolicy::Fixed { estimator, .. } => Some(*estimator), + } +} + +fn delta_threshold(policy: &Option) -> f64 { + policy + .as_ref() + .map(|policy| policy.absolute_threshold) + .unwrap_or(0.0) +} + +fn gos_epsilon(policy: &Option) -> f64 { + policy + .as_ref() + .and_then(|policy| policy.gos.as_ref()) + .map(|gos| gos.epsilon_staleness) + .unwrap_or(0.0) +} + +fn delta_shape(policy: &Option) -> Option<(Option<(u32, GosThresholdMode)>,)> { + policy.as_ref().map(|policy| { + (policy + .gos + .as_ref() + .map(|gos| (gos.sites, gos.threshold_mode)),) + }) +} + +fn authorize_f64_change( + current: f64, + next: f64, + bounds: Option<&AdaptiveF64Bounds>, + producer_id: &str, + knob: &'static str, +) -> Result<(), TransmissionPlanError> { + if current == next { + return Ok(()); + } + let allowed = bounds.is_some_and(|bounds| { + next.is_finite() + && (bounds.min..=bounds.max).contains(&next) + && (next - current).abs() <= bounds.max_step + }); + if allowed { + Ok(()) + } else { + Err(TransmissionPlanError::AdaptationOutOfBounds { + producer_id: producer_id.into(), + knob, + }) + } +} + +fn authorize_u64_change( + current: u64, + next: u64, + bounds: Option<&AdaptiveU64Bounds>, + producer_id: &str, + knob: &'static str, +) -> Result<(), TransmissionPlanError> { + if current == next { + return Ok(()); + } + let allowed = bounds.is_some_and(|bounds| { + (bounds.min..=bounds.max).contains(&next) && current.abs_diff(next) <= bounds.max_step + }); + if allowed { + Ok(()) + } else { + Err(TransmissionPlanError::AdaptationOutOfBounds { + producer_id: producer_id.into(), + knob, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use planner_types::post_asap::SketchParams; + + #[test] + fn runtime_policy_is_family_and_mode_checked() { + let mut rule = TransmissionRule { + materialization: crate::PolicyFingerprint(1).into(), + producer_id: "test".into(), + schema_id: "test".into(), + mode: TransmissionMode::Full, + encoding: StateEncoding::SketchlibProtobufV1, + emit_every_ms: 60_000, + full_checkpoint_every_ms: None, + destination_ref: "backend".into(), + runtime_policy: Default::default(), + }; + rule.runtime_policy.sampling = SamplingPolicy::Fixed { + probability: 0.5, + estimator: SamplingEstimator::HashThreshold, + }; + let hll = StateFamilyContract::Sketch { + algorithm: SketchAlgorithm::Hll, + parameters: SketchParams::Hll { precision: 14 }, + }; + let count_sketch = StateFamilyContract::Sketch { + algorithm: SketchAlgorithm::CountSketch, + parameters: SketchParams::CountSketch { + width: 128, + depth: 4, + }, + }; + validate_runtime_rule_policy(&rule, &hll).expect("HLL supports hash-threshold sampling"); + assert!(matches!( + validate_runtime_rule_policy(&rule, &count_sketch), + Err(TransmissionPlanError::InvalidRuntimePolicy { .. }) + )); + + rule.runtime_policy.sampling = SamplingPolicy::Disabled; + rule.mode = TransmissionMode::Delta; + rule.full_checkpoint_every_ms = Some(300_000); + rule.runtime_policy.delta = Some(DeltaPolicy { + absolute_threshold: 0.0, + gos: Some(GosPolicy { + epsilon_staleness: 0.02, + sites: 2, + threshold_mode: GosThresholdMode::Isotropic, + }), + }); + validate_runtime_rule_policy(&rule, &count_sketch).expect("CountSketch supports delta GOS"); + assert!(matches!( + validate_runtime_rule_policy(&rule, &hll), + Err(TransmissionPlanError::InvalidRuntimePolicy { .. }) + )); + } +} diff --git a/data_plane/examples/audit_clickhouse_fallback.rs b/data_plane/examples/audit_clickhouse_fallback.rs index 2ada8ebaf..a5bdd1ac0 100644 --- a/data_plane/examples/audit_clickhouse_fallback.rs +++ b/data_plane/examples/audit_clickhouse_fallback.rs @@ -3,9 +3,7 @@ use axum::{ http::{HeaderMap, Method}, }; use control_plane::{ - physical::compiler::{ - PlanEnvelope, PrecomputePlan, TransmissionPlan, BACKEND_COMPAT, PLANNER_REVISION, - }, + physical::compiler::{PlanEnvelope, PrecomputePlan, BACKEND_COMPAT, PLANNER_REVISION}, query_plan::{ClickHousePlanningContext, QueryPlan}, }; use data_plane::{ @@ -76,8 +74,12 @@ async fn main() { let mut precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![]).unwrap(); precompute_plan.summary_catalog = Some(reference.clone()); - let mut transmission_plan = - TransmissionPlan::build(envelope, &precompute_plan, &BTreeMap::new()).unwrap(); + let mut transmission_plan = control_plane::physical::compiler::compile_transmission_plan( + envelope, + &precompute_plan, + &BTreeMap::new(), + ) + .unwrap(); transmission_plan.summary_catalog = Some(reference); let query_plan = QueryPlan { plan_id: 27, diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index 16d7efcd7..d455d6010 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -1630,7 +1630,7 @@ async fn route_modified_otlp_sketches_to_precompute( crate::precompute_engine::frame_lineage::FrameLineageDecision::Apply, ) => { if frame.kind - == control_plane::physical::compiler::SummaryFrameKind::Full + == asap_types::producer_plan::SummaryFrameKind::Full { ingest_state .sketch_index @@ -2157,7 +2157,7 @@ fn preflight_summary_frames( mut dp: ModifiedOtlpSketchDp, ingest_state: &IngestState, active: &crate::storage_engines::types::ActivePhysicalPlan, - ) -> Result { + ) -> Result { let canonical_name = canonical_sketch_metric_name(metric_name, dp.algorithm.clone()); let frame = take_summary_frame_identity(&mut dp.attrs, dp.start_time_unix_nano, dp.time_unix_nano)?; @@ -2200,7 +2200,7 @@ fn preflight_summary_frames( // A malformed full snapshot must not be discovered after an earlier // frame in the request has already reached SketchStore. - if frame.kind == control_plane::physical::compiler::SummaryFrameKind::Full { + if frame.kind == asap_types::producer_plan::SummaryFrameKind::Full { decode_modified_otlp_sketch_bytes(dp.algorithm.clone(), dp.encoding, &dp.sketch) .map_err(|error| format!("invalid full frame for {metric_name}: {error}"))?; } else { @@ -2358,10 +2358,9 @@ fn take_summary_frame_identity( attrs: &mut HashMap, window_start_unix_nano: u64, window_end_unix_nano: u64, -) -> Result { - use control_plane::physical::compiler::{ - StateEncoding, SummaryFrameIdentity, SummaryFrameKind, - }; +) -> Result { + use asap_types::producer_plan::{SummaryFrameIdentity, SummaryFrameKind}; + use control_plane::physical::compiler::StateEncoding; fn required(attrs: &mut HashMap, key: &str) -> Result { attrs diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index 92b75fb4b..c27904ec9 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -867,9 +867,10 @@ mod tests { } fn physical_config(streaming: StreamingConfig) -> HotReloadStreamingConfig { + use asap_types::producer_plan::{FrameIdentityContract, SequenceScope, TransmissionPlan}; use control_plane::physical::compiler::{ - FrameIdentityContract, IngestContract, IngestProtocol, PlanEnvelope, PrecomputePlan, - SequenceScope, TimestampUnit, TransmissionPlan, PLANNER_REVISION, + IngestContract, IngestProtocol, PlanEnvelope, PrecomputePlan, TimestampUnit, + PLANNER_REVISION, }; let envelope = PlanEnvelope { plan_id: 7, @@ -1466,9 +1467,7 @@ mod tests { .unwrap(); let binding = asap_types::query_plan::MaterializationBinding { materialization: asap_types::PolicyFingerprint(policy).into(), - output_grouping: asap_types::query_plan::PhysicalGrouping::Reduce( - vec!["job".into()], - ), + output_grouping: asap_types::query_plan::PhysicalGrouping::Reduce(vec!["job".into()]), item_labels: vec![], window_ms: 60_000, pane_origin_ms: Some(0), diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 5af5d3f96..261f68a7e 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -2679,9 +2679,10 @@ mod tests { } async fn setup_remote_write_test_server() -> (u16, PrometheusRemoteWriteReceiver) { + use asap_types::producer_plan::{FrameIdentityContract, SequenceScope, TransmissionPlan}; use control_plane::physical::compiler::{ - FrameIdentityContract, IngestContract, IngestProtocol, PlanEnvelope, PrecomputePlan, - SequenceScope, TimestampUnit, TransmissionPlan, PLANNER_REVISION, + IngestContract, IngestProtocol, PlanEnvelope, PrecomputePlan, TimestampUnit, + PLANNER_REVISION, }; let streaming_config = Arc::new(StreamingConfig::default()); let envelope = PlanEnvelope { @@ -6045,7 +6046,7 @@ async fn handle_post_streaming_config( (StatusCode::OK, axum::Json(body)).into_response() } -pub use control_plane::physical::publication::PhysicalPlanInstallRequest; +pub use asap_types::plan_publication::PhysicalPlanInstallRequest; /// Decode and cross-validate every backend view before it can become visible. /// Used by both startup artifact loading and the staged HTTP install path. diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 23409c3b1..97f6386d8 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -768,13 +768,13 @@ async fn main() -> Result<()> { .collect(), executable_dags: Default::default(), }; - let initial_transmission_plan = control_plane::physical::compiler::TransmissionPlan { + let initial_transmission_plan = asap_types::producer_plan::TransmissionPlan { summary_catalog: None, envelope: initial_precompute_plan.envelope.clone(), - frame_identity: control_plane::physical::compiler::FrameIdentityContract { + frame_identity: asap_types::producer_plan::FrameIdentityContract { identity_version: 1, sequence_scope: - control_plane::physical::compiler::SequenceScope::MaterializationSeriesProducerEpoch, + asap_types::producer_plan::SequenceScope::MaterializationSeriesProducerEpoch, require_checkpoint_for_full: true, require_base_checkpoint_for_delta: true, }, diff --git a/data_plane/src/precompute_engine/frame_lineage.rs b/data_plane/src/precompute_engine/frame_lineage.rs index 6598a1384..7f5e0a847 100644 --- a/data_plane/src/precompute_engine/frame_lineage.rs +++ b/data_plane/src/precompute_engine/frame_lineage.rs @@ -5,7 +5,7 @@ //! the wire contract: deltas are only safe to apply after an observed full //! checkpoint, in sequence, within the exact producer/window lineage. -use control_plane::physical::compiler::{SummaryFrameIdentity, SummaryFrameKind}; +use asap_types::producer_plan::{SummaryFrameIdentity, SummaryFrameKind}; use dashmap::mapref::entry::Entry; use thiserror::Error; diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs index 2617de089..49a516759 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs @@ -665,7 +665,7 @@ mod tests { ) .unwrap(); precompute.summary_catalog = Some(sds.reference().unwrap()); - let mut transmission = control_plane::physical::compiler::TransmissionPlan::build( + let mut transmission = control_plane::physical::compiler::compile_transmission_plan( envelope.clone(), &precompute, &BTreeMap::new(), diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index 8a6074f5c..54e9ad02d 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -537,8 +537,8 @@ struct IncompleteSummaryLineage { window_end_unix_ms: u64, } -impl From<&control_plane::physical::compiler::SummaryFrameIdentity> for IncompleteSummaryLineage { - fn from(frame: &control_plane::physical::compiler::SummaryFrameIdentity) -> Self { +impl From<&asap_types::producer_plan::SummaryFrameIdentity> for IncompleteSummaryLineage { + fn from(frame: &asap_types::producer_plan::SummaryFrameIdentity) -> Self { Self { plan_id: frame.plan_id, plan_version: frame.plan_version, @@ -2477,7 +2477,7 @@ impl SketchStore { pub fn mark_summary_lineage_incomplete( &self, sid: u64, - frame: &control_plane::physical::compiler::SummaryFrameIdentity, + frame: &asap_types::producer_plan::SummaryFrameIdentity, ) { self.incomplete_summary_lineages .entry(sid) @@ -2489,7 +2489,7 @@ impl SketchStore { pub fn clear_summary_lineage_incomplete( &self, sid: u64, - frame: &control_plane::physical::compiler::SummaryFrameIdentity, + frame: &asap_types::producer_plan::SummaryFrameIdentity, ) { let key = IncompleteSummaryLineage::from(frame); if let Some(mut lineages) = self.incomplete_summary_lineages.get_mut(&sid) { @@ -3326,9 +3326,8 @@ mod tests { #[test] fn incomplete_delta_window_fails_closed_until_matching_full_checkpoint() { - use control_plane::physical::compiler::{ - StateEncoding, SummaryFrameIdentity, SummaryFrameKind, - }; + use asap_types::producer_plan::{SummaryFrameIdentity, SummaryFrameKind}; + use control_plane::physical::compiler::StateEncoding; let idx = SketchStore::new(); idx.register(meta(12)); diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index 9cba552f9..88786343c 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -92,7 +92,7 @@ pub struct ActivePhysicalPlan { /// Present for authoritative installations; legacy bootstrap has no catalog. pub summary_catalog: Option>, pub precompute_plan: asap_types::precompute_plan::PrecomputePlan, - pub transmission_plan: control_plane::physical::compiler::TransmissionPlan, + pub transmission_plan: asap_types::producer_plan::TransmissionPlan, pub runtime_config: Arc, pub query_plan: Arc, pub storage_routing: Arc, @@ -672,11 +672,9 @@ mod tests { summary_catalog: None, envelope: envelope.clone(), ingest: asap_types::precompute_plan::IngestContract { - protocol: - asap_types::precompute_plan::IngestProtocol::ModifiedOtlpMetricsV1, + protocol: asap_types::precompute_plan::IngestProtocol::ModifiedOtlpMetricsV1, endpoint_path: "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/v1/metrics".into(), - timestamp_unit: - asap_types::precompute_plan::TimestampUnit::UnixNanoseconds, + timestamp_unit: asap_types::precompute_plan::TimestampUnit::UnixNanoseconds, require_plan_identity: true, require_summary_definition_identity: true, require_registered_producer: true, @@ -684,9 +682,9 @@ mod tests { schemas: Vec::new(), producers: Vec::new(), executable_dags: Default::default(), - materializations: Vec::new(), + materializations: Vec::new(), }, - transmission_plan: control_plane::physical::compiler::TransmissionPlan { + transmission_plan: asap_types::producer_plan::TransmissionPlan { summary_catalog: None, envelope: asap_types::precompute_plan::PlanEnvelope { plan_id, @@ -698,9 +696,10 @@ mod tests { planner_revision: control_plane::physical::compiler::PLANNER_REVISION.into(), capability_snapshot_id: "test".into(), }, - frame_identity: control_plane::physical::compiler::FrameIdentityContract { + frame_identity: asap_types::producer_plan::FrameIdentityContract { identity_version: 1, - sequence_scope: control_plane::physical::compiler::SequenceScope::MaterializationSeriesProducerEpoch, + sequence_scope: + asap_types::producer_plan::SequenceScope::MaterializationSeriesProducerEpoch, require_checkpoint_for_full: true, require_base_checkpoint_for_delta: true, }, diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index 0bf5ff300..cc1b733fa 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -74,9 +74,9 @@ async fn post_full_config(client: &reqwest::Client, stack: &FullStack, json: &Js .any(|c| c.metric == "http_requests_total_latency_ms") { for rule in &mut artifact.transmission_plan.rules { - rule.mode = control_plane::physical::compiler::TransmissionMode::Delta; + rule.mode = asap_types::producer_plan::TransmissionMode::Delta; rule.full_checkpoint_every_ms = Some(rule.emit_every_ms); - rule.runtime_policy.delta = Some(control_plane::physical::compiler::DeltaPolicy { + rule.runtime_policy.delta = Some(asap_types::producer_plan::DeltaPolicy { absolute_threshold: 0.0, gos: None, }); diff --git a/data_plane/tests/support/physical_fixture.rs b/data_plane/tests/support/physical_fixture.rs index 00ae0e862..984fc0d16 100644 --- a/data_plane/tests/support/physical_fixture.rs +++ b/data_plane/tests/support/physical_fixture.rs @@ -37,8 +37,12 @@ pub fn artifact(config: &StreamingConfig) -> PhysicalPlanInstallRequest { let mut precompute = PrecomputePlan::build(envelope.clone(), configs, &["fixture".into()]).unwrap(); precompute.summary_catalog = Some(catalog.reference().unwrap()); - let mut transmission = - TransmissionPlan::build(envelope, &precompute, &BTreeMap::new()).unwrap(); + let mut transmission = control_plane::physical::compiler::compile_transmission_plan( + envelope, + &precompute, + &BTreeMap::new(), + ) + .unwrap(); transmission.summary_catalog = Some(catalog.reference().unwrap()); let mut query_plan = QueryPlan { plan_id: 1, diff --git a/docs/design_docs/summary-catalog-sds-architecture.md b/docs/design_docs/summary-catalog-sds-architecture.md index 110e0c25c..7378f6464 100644 --- a/docs/design_docs/summary-catalog-sds-architecture.md +++ b/docs/design_docs/summary-catalog-sds-architecture.md @@ -165,6 +165,13 @@ those types for existing callers and owns the `compile_bound*` and move into the shared contract. Data-plane engines import the shared types directly. No wrapper plan or second wire definition is introduced. +`asap_types::producer_plan` owns the installed collector and transmission +contracts, frame identities, runtime policy bounds and their validation. The +control plane allocates sampling/GOS budgets and constructs transmission rules +through `sampling_policy_from_accuracy_budget`, `gos_policy_from_accuracy_budget` +and `compile_transmission_plan`. Producers and the data plane import the shared +contracts directly; compilation is not a runtime dependency of those contracts. + The implemented ownership split is: 1. Move the SDS catalog contract into `asap_types`.