From 9a0b5edbb0e6f24e2f33e357b98e47282c72f085 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 13:29:50 -0600 Subject: [PATCH 01/30] Bind QueryPlan materializations to shared catalog identities --- control_plane/src/physical/compiler.rs | 11 +- control_plane/src/query_plan.rs | 172 +++++++++++++++++- control_plane/src/query_plan/logical.rs | 3 +- .../query_engines/asap_query_engine/engine.rs | 4 +- .../asap_query_engine/live_serve.rs | 2 +- .../asap_query_engine/post_asap_readout.rs | 6 +- .../asap_query_engine/summary_executor.rs | 8 +- 7 files changed, 187 insertions(+), 19 deletions(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index d68e9092..1d5b418a 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -2448,7 +2448,7 @@ impl PhysicalCompiler { } Ok(MaterializationBinding { readout_lookback_ms: source_window.map(|seconds| seconds.saturating_mul(1_000)), - materialization: fingerprint, + materialization: fingerprint.into(), metric: planned_metric, sid_grouping: materialization.group_by.clone(), output_grouping: PhysicalGrouping::Reduce(materialization.group_by.clone()), @@ -2503,7 +2503,7 @@ impl PhysicalCompiler { .entries .values() .flat_map(QueryPlanEntry::materialization_bindings) - .filter(|binding| binding.materialization == fingerprint) + .filter(|binding| binding.materialization.fingerprint() == fingerprint) .filter_map(|binding| binding.readout_lookback_ms) .max(); if let Some(lookback_ms) = max_lookback_ms { @@ -2545,6 +2545,7 @@ impl PhysicalCompiler { reason: error.to_string(), })?; } + query_plan.validate_against_catalog(&summary_catalog)?; Ok(PhysicalPlan { envelope, summary_catalog, @@ -3819,7 +3820,7 @@ mod tests { assert_eq!(bindings.len(), 1); for collector in &bundle.collector_plans { assert_eq!(collector.materializations.len(), 1); - assert!(bindings.contains(&collector.materializations[0].materialization)); + assert!(bindings.contains(&collector.materializations[0].materialization.into())); } } } @@ -4230,7 +4231,7 @@ mod tests { .entries .values() .flat_map(|entry| entry.materialization_bindings()) - .map(|binding| binding.materialization) + .map(|binding| binding.materialization.fingerprint()) .collect::>(); assert_eq!(bindings.len(), 1); query_plan.validate(&bindings).unwrap(); @@ -4323,7 +4324,7 @@ mod tests { .summary_catalog .materializations .keys() - .copied() + .map(|id| id.fingerprint()) .collect::>(), bundle .backend_plan diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index fa3a745b..573ca984 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -15,7 +15,7 @@ use planner_types::pre_asap::Reduction; use serde::{Deserialize, Serialize}; use thiserror::Error; -use asap_types::PolicyFingerprint; +use asap_types::{sds::MaterializationId, PolicyFingerprint}; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] @@ -41,6 +41,53 @@ impl QueryPlan { .ok_or(QueryPlanError::QueryNotPlanned(identity)) } + /// Validate semantic bindings against the authoritative snapshot before use. + pub fn validate_against_catalog( + &self, + catalog: &crate::physical::summary_catalog::SummaryCatalog, + ) -> Result<(), QueryPlanError> { + catalog + .validate() + .map_err(|error| QueryPlanError::Invalid(error.to_string()))?; + if self.plan_id != catalog.plan_id || self.plan_version != catalog.plan_version { + return Err(QueryPlanError::Invalid( + "QueryPlan and SummaryCatalog have different plan identity/version".into(), + )); + } + let available = catalog + .materializations + .keys() + .copied() + .map(Into::into) + .collect(); + self.validate(&available)?; + for entry in self.entries.values() { + for binding in entry.materialization_bindings() { + let identity = catalog + .materializations + .get(&binding.materialization) + .ok_or_else(|| { + QueryPlanError::Invalid( + "query binding references absent catalog materialization".into(), + ) + })?; + let data = &catalog.data_descriptors[&identity.data_descriptor_id]; + let grouping: BTreeSet<_> = binding.sid_grouping.iter().cloned().collect(); + if binding.metric != data.metric_name || grouping != data.group_by_keys { + return Err(QueryPlanError::Invalid( + "query binding source/grouping differs from catalog data descriptor".into(), + )); + } + if binding.window_ms == 0 { + return Err(QueryPlanError::Invalid( + "zero physical pane duration".into(), + )); + } + } + } + Ok(()) + } + pub fn validate(&self, available: &BTreeSet) -> Result<(), QueryPlanError> { if self.plan_id != 0 && self.plan_version == 0 { return Err(QueryPlanError::Invalid( @@ -196,10 +243,12 @@ impl QueryPlanEntry { "zero semantic readout lookback".into(), )); } - if !available.contains(&binding.materialization) { + if !available.contains(&binding.materialization.fingerprint()) { return Err(QueryPlanError::Invalid(format!( "query `{}` node {} references absent materialization {}", - self.query_id, id.0, binding.materialization.0 + self.query_id, + id.0, + binding.materialization.as_u64() ))); } } @@ -265,7 +314,7 @@ pub enum FallbackPolicy { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct MaterializationBinding { - pub materialization: PolicyFingerprint, + pub materialization: MaterializationId, pub metric: String, /// Exact label-key layout of the stored materialization. pub sid_grouping: Vec, @@ -840,3 +889,118 @@ mod tests { .contains("cycle")); } } + +#[cfg(test)] +mod catalog_binding_tests { + use super::*; + use crate::physical::summary_catalog::SummaryCatalog; + use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + + fn fixture() -> (QueryPlan, SummaryCatalog) { + let config = PrecomputeMaterialization::new( + AggregationType::Sum, + String::new(), + Default::default(), + KeyByLabelNames::new(vec!["job".into()]), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + 10, + 10, + WindowKind::Tumbling, + String::new(), + "m".into(), + None, + None, + None, + ); + let catalog = SummaryCatalog::from_materializations(7, 2, &[config.clone()]).unwrap(); + let entry = QueryPlanEntry { + query_id: "q".into(), + canonical_promql: "sum_over_time(m[1m])".into(), + root: QueryNodeId(1), + nodes: BTreeMap::from([( + QueryNodeId(1), + QueryPlanNode::ReadMaterialization { + binding: MaterializationBinding { + materialization: config.policy_fingerprint().into(), + metric: "m".into(), + sid_grouping: vec!["job".into()], + output_grouping: PhysicalGrouping::PerEntity, + window_ms: 10_000, + readout_lookback_ms: Some(60_000), + }, + }, + )]), + instant: InstantExecution { + lookback_ms: 60_000, + full_history: false, + cumulative_readout: true, + }, + fallback: FallbackPolicy::ExactBackend, + }; + ( + QueryPlan { + plan_id: 7, + plan_version: 2, + entries: BTreeMap::from([(entry.canonical_promql.clone(), entry)]), + }, + catalog, + ) + } + fn binding(plan: &mut QueryPlan) -> &mut MaterializationBinding { + let QueryPlanNode::ReadMaterialization { binding } = plan + .entries + .values_mut() + .next() + .unwrap() + .nodes + .values_mut() + .next() + .unwrap() + else { + panic!("fixture") + }; + binding + } + + // One pane ID is compatible with a longer semantic readout window. + #[test] + fn catalog_binding_round_trip_preserves_pane_and_readout_windows() { + let (plan, catalog) = fixture(); + let mut decoded: QueryPlan = + serde_json::from_slice(&serde_json::to_vec(&plan).unwrap()).unwrap(); + decoded.validate_against_catalog(&catalog).unwrap(); + assert_eq!(binding(&mut decoded).window_ms, 10_000); + assert_eq!(binding(&mut decoded).readout_lookback_ms, Some(60_000)); + } + + // A valid fingerprint alone cannot attest a different source or grouping. + #[test] + fn catalog_binding_rejects_source_grouping_and_identity_drift() { + let (plan, catalog) = fixture(); + let mut broken = plan.clone(); + binding(&mut broken).metric = "other".into(); + assert!(broken.validate_against_catalog(&catalog).is_err()); + let mut broken = plan.clone(); + binding(&mut broken).sid_grouping.clear(); + assert!(broken.validate_against_catalog(&catalog).is_err()); + let mut broken = plan.clone(); + binding(&mut broken).materialization = PolicyFingerprint(123).into(); + assert!(broken.validate_against_catalog(&catalog).is_err()); + let mut broken = plan.clone(); + broken.plan_version += 1; + assert!(broken.validate_against_catalog(&catalog).is_err()); + let mut broken = plan; + binding(&mut broken).window_ms = 0; + assert!(broken.validate_against_catalog(&catalog).is_err()); + } + + // Catalog descriptor corruption must fail even if the materialization exists. + #[test] + fn catalog_binding_rejects_broken_descriptor_reference() { + let (plan, mut catalog) = fixture(); + catalog.summary_descriptors.clear(); + assert!(plan.validate_against_catalog(&catalog).is_err()); + } +} diff --git a/control_plane/src/query_plan/logical.rs b/control_plane/src/query_plan/logical.rs index 31473cf2..76c0bc2b 100644 --- a/control_plane/src/query_plan/logical.rs +++ b/control_plane/src/query_plan/logical.rs @@ -671,7 +671,8 @@ mod hybrid_tests { Ok(MaterializationBinding { materialization: asap_types::PolicyFingerprint( if spatial_filter.is_empty() { 7 } else { 8 }, - ), + ) + .into(), metric: "m".into(), sid_grouping: vec![], output_grouping: PhysicalGrouping::PerEntity, diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 68445571..460b6d83 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -16,7 +16,7 @@ fn readiness_requirement( let bindings = entry.materialization_bindings(); let mut materializations = bindings .iter() - .map(|binding| binding.materialization) + .map(|binding| binding.materialization.fingerprint()) .collect::>(); materializations.sort_unstable(); materializations.dedup(); @@ -250,7 +250,7 @@ impl ASAPQueryEngine { physical .backend_plan .materializations - .get(&binding.materialization) + .get(&binding.materialization.fingerprint()) .map(|materialization| &materialization.family), Some(planner_types::post_asap::SummaryFamilyType::ExactAggregate( .. diff --git a/data_plane/src/query_engines/asap_query_engine/live_serve.rs b/data_plane/src/query_engines/asap_query_engine/live_serve.rs index fef202bb..119a4ee6 100644 --- a/data_plane/src/query_engines/asap_query_engine/live_serve.rs +++ b/data_plane/src/query_engines/asap_query_engine/live_serve.rs @@ -485,7 +485,7 @@ mod tests { control_plane::query_plan::QueryNodeId(1), control_plane::query_plan::QueryPlanNode::ReadMaterialization { binding: control_plane::query_plan::MaterializationBinding { - materialization: policy, + materialization: policy.into(), metric: "bytes".into(), sid_grouping: vec![], output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 162ce11e..2158b2c7 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -790,7 +790,7 @@ mod tests { control_plane::query_plan::FallbackPolicy::ExactBackend, |_node, _family| { Ok(control_plane::query_plan::MaterializationBinding { - materialization: asap_types::PolicyFingerprint(123), + materialization: asap_types::PolicyFingerprint(123).into(), metric: "unique_users".into(), sid_grouping: vec!["service".into()], output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, @@ -959,7 +959,7 @@ mod tests { control_plane::query_plan::QueryNodeId(1), QueryPlanNode::ReadMaterialization { binding: control_plane::query_plan::MaterializationBinding { - materialization: policy, + materialization: policy.into(), metric: "requests_total".into(), sid_grouping: vec![], output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, @@ -1053,7 +1053,7 @@ mod tests { control_plane::query_plan::QueryNodeId(1), QueryPlanNode::ReadMaterialization { binding: control_plane::query_plan::MaterializationBinding { - materialization: policy, + materialization: policy.into(), metric: "requests_total".into(), sid_grouping: vec![], output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index 5db19a50..57c9fc4f 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -454,7 +454,9 @@ impl QueryExecutionContext<'_> { Ok(()) }; let required_keys: BTreeSet<_> = binding.sid_grouping.iter().cloned().collect(); - let mut sids = self.index.sids_for_policy(binding.materialization); + let mut sids = self + .index + .sids_for_policy(binding.materialization.fingerprint()); sids.sort_unstable(); sids.dedup(); let mut matched_metadata = 0usize; @@ -464,7 +466,7 @@ impl QueryExecutionContext<'_> { let candidate = self .index .with_instance(sid, |meta| { - if meta.policy_fp != binding.materialization + if meta.policy_fp != binding.materialization.fingerprint() || meta.metric_name != binding.metric || meta.group_by_keys != required_keys { @@ -567,7 +569,7 @@ impl QueryExecutionContext<'_> { if by_group.is_empty() { tracing::debug!( metric = %binding.metric, - materialization = %binding.materialization, + materialization = %binding.materialization.as_u64(), ?sids, matched_metadata, t0_ms = self.t0_ms, From 018937572c79aad36493f20ac6d52aea6ef9799c Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 13:44:36 -0600 Subject: [PATCH 02/30] Fix catalog materialization identity assertion --- control_plane/src/physical/compiler.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 1d5b418a..8f8f792c 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -4324,7 +4324,7 @@ mod tests { .summary_catalog .materializations .keys() - .map(|id| id.fingerprint()) + .cloned() .collect::>(), bundle .backend_plan From db3f7cdbae0d7f3a1eb4d6a7cb50c86bfa43727f Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 13:48:45 -0600 Subject: [PATCH 03/30] Publish catalog-authoritative physical plan documents from control plane --- .../examples/calibration_candidates.rs | 2 + .../examples/compile_workload_artifact.rs | 2 + control_plane/src/backend_client.rs | 64 +++++++++++++ control_plane/src/main.rs | 13 +-- control_plane/src/physical/compiler.rs | 20 ++++ control_plane/src/physical/mod.rs | 2 + control_plane/src/physical/publication.rs | 96 +++++++++++++++++++ 7 files changed, 191 insertions(+), 8 deletions(-) create mode 100644 control_plane/src/physical/publication.rs diff --git a/control_plane/examples/calibration_candidates.rs b/control_plane/examples/calibration_candidates.rs index 44ddaaeb..2731b1cd 100644 --- a/control_plane/examples/calibration_candidates.rs +++ b/control_plane/examples/calibration_candidates.rs @@ -126,6 +126,8 @@ fn main() -> Result<(), Box> { "manifest": manifest, "lifecycle_estimates": plan.lifecycle_estimates, "install_request": { + "summary_catalog": plan.summary_catalog, + "collector_plans": plan.collector_plans, "precompute_plan": plan.precompute_plan, "transmission_plan": plan.transmission_plan, "backend_plan": plan.backend_plan.encode_to_vec(), diff --git a/control_plane/examples/compile_workload_artifact.rs b/control_plane/examples/compile_workload_artifact.rs index 0cd9115d..23263654 100644 --- a/control_plane/examples/compile_workload_artifact.rs +++ b/control_plane/examples/compile_workload_artifact.rs @@ -27,6 +27,8 @@ fn main() -> Result<(), Box> { "cost_comparison": comparison, "lifecycle_estimates": plan.lifecycle_estimates, "install_request": { + "summary_catalog": plan.summary_catalog, + "collector_plans": plan.collector_plans, "precompute_plan": plan.precompute_plan, "transmission_plan": plan.transmission_plan, "backend_plan": plan.backend_plan.encode_to_vec(), diff --git a/control_plane/src/backend_client.rs b/control_plane/src/backend_client.rs index de50a861..24984eec 100644 --- a/control_plane/src/backend_client.rs +++ b/control_plane/src/backend_client.rs @@ -360,6 +360,33 @@ impl BackendClient { } } + /// Publish the canonical catalog generation. The optional legacy bytes are + /// an explicit compatibility projection, never used to validate this document. + pub async fn post_catalog_plan_typed( + &self, + publication: &crate::physical::publication::PhysicalPlanPublication, + compatibility_backend_plan: Option>, + storage_routing: Option, + adaptation_evidence: &[crate::physical::compiler::RuntimeAdaptationEvidence], + ) -> std::result::Result<(), BackendPostError> { + publication.validate().map_err(|error| BackendPostError::Permanent(anyhow::anyhow!(error)))?; + let mut body = serde_json::to_value(publication) + .map_err(|error| BackendPostError::Permanent(error.into()))?; + let fields = body.as_object_mut().expect("publication serializes as object"); + if let Some(bytes) = compatibility_backend_plan { + fields.insert("backend_plan".into(), serde_json::json!(bytes)); + } + fields.insert("storage_routing".into(), serde_json::json!(storage_routing)); + fields.insert("adaptation_evidence".into(), serde_json::json!(adaptation_evidence)); + let response = self.http.post(derive_physical_plan_url(&self.endpoint)).json(&body) + .send().await.map_err(classify_reqwest_error)?; + let status = response.status(); + if status.is_success() { Ok(()) } else { + let body = response.text().await.unwrap_or_default(); + Err(classify_http_status(status, body, "catalog physical-plan POST")) + } + } + /// Publish the backend-facing portions of one PhysicalPlan in a single /// request, preventing independently retried documents from mixing /// generations at the backend. @@ -372,11 +399,24 @@ impl BackendClient { storage_routing: Option, adaptation_evidence: &[crate::physical::compiler::RuntimeAdaptationEvidence], ) -> std::result::Result<(), BackendPostError> { + // Compatibility replanner has no Planner-selected query/collector DAG. + // Still publish the actual catalog and bind every provided projection. + let catalog = crate::physical::summary_catalog::SummaryCatalog::from_materializations( + precompute_plan.envelope.plan_id, precompute_plan.envelope.plan_version, + &precompute_plan.materializations, + ).map_err(|error| BackendPostError::Permanent(error.into()))?; + let mut precompute_plan = precompute_plan.clone(); + precompute_plan.bind_catalog(&catalog).map_err(|error| BackendPostError::Permanent(error.into()))?; + let mut transmission_plan = transmission_plan.clone(); + transmission_plan.summary_catalog = Some(catalog.reference().map_err(|error| BackendPostError::Permanent(error.into()))?); + query_plan.validate_against_catalog(&catalog).map_err(|error| BackendPostError::Permanent(error.into()))?; let url = derive_physical_plan_url(&self.endpoint); let response = self .http .post(&url) .json(&serde_json::json!({ + "summary_catalog": catalog, + "collector_plans": [], "precompute_plan": precompute_plan, "transmission_plan": transmission_plan, "backend_plan": backend_plan, @@ -671,6 +711,30 @@ mod tests { assert_eq!(derive_backend_plan_url("http://x/foo"), "http://x/foo"); } + #[tokio::test] + async fn catalog_publication_posts_canonical_document_without_legacy_bytes() { + let snapshot: crate::physical::compiler::BackendLocalPlanningSnapshot = serde_json::from_str(include_str!("../../docs/examples/asapquery-planning-snapshot.json")).unwrap(); + let publication = snapshot.compile().unwrap().publication().unwrap(); + let hits: StdArc>> = StdArc::new(Mutex::new(Vec::new())); + let route_hits = hits.clone(); + let app = Router::new().route("/api/v1/physical-plan", post(move |axum::Json(body): axum::Json| { + let hits = route_hits.clone(); + async move { hits.lock().unwrap().push(body); axum::http::StatusCode::OK } + })); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap(); }); + let client = BackendClient::new(format!("http://{addr}/api/v1/streaming-config")); + client.post_catalog_plan_typed(&publication, None, None, &[]).await.unwrap(); + let bodies = hits.lock().unwrap(); + assert_eq!(bodies.len(), 1); + for field in ["summary_catalog", "precompute_plan", "collector_plans", "transmission_plan", "query_plan"] { + assert!(bodies[0].get(field).is_some(), "missing {field}"); + } + assert!(bodies[0].get("backend_plan").is_none()); + server.abort(); + } + #[tokio::test] async fn backend_plan_post_round_trips_bytes_via_url_rewrite() { let hits: StdArc>>> = StdArc::new(Mutex::new(Vec::new())); diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index 7a5ff20c..aae4ced4 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -669,15 +669,12 @@ async fn handle_compile_and_publish_physical_plan( ) .into_response(); } + let publication = match bundle.publication() { + Ok(publication) => publication, + Err(error) => return (StatusCode::INTERNAL_SERVER_ERROR, format!("invalid catalog publication: {error}")).into_response(), + }; if let Err(error) = backend - .post_physical_plan_typed( - &bundle.precompute_plan, - &bundle.transmission_plan, - bundle.backend_plan.encode_to_vec(), - &bundle.query_plan, - None, - &adaptation_evidence, - ) + .post_catalog_plan_typed(&publication, Some(bundle.backend_plan.encode_to_vec()), None, &adaptation_evidence) .await { return ( diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 8f8f792c..3a1a0dc0 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -4304,6 +4304,26 @@ mod tests { assert!(legacy.validate_against_catalog(catalog).is_err()); } + #[test] + fn publication_is_catalog_authoritative_and_round_trips() { + let mut bundle = PhysicalCompiler.compile(request("publication", "quantile_over_time(0.99, m[1m])"), environment(10_000)).unwrap(); + // Compatibility bytes cannot silently determine publication identity. + bundle.backend_plan.plan_id += 1; + let publication = bundle.publication().unwrap(); + let json = serde_json::to_value(&publication).unwrap(); + assert!(json.get("backend_plan").is_none()); + let mut decoded: super::super::publication::PhysicalPlanPublication = serde_json::from_value(json).unwrap(); + decoded.validate().unwrap(); + decoded.collector_plans.clear(); + assert!(decoded.validate().is_err()); + let mut decoded = publication.clone(); + decoded.summary_catalog.plan_version += 1; + assert!(decoded.validate().is_err()); + let mut decoded = publication; + decoded.collector_plans.push(decoded.collector_plans[0].clone()); + assert!(decoded.validate().is_err()); + } + #[test] fn compiles_one_decision_into_matching_collector_and_backend_views() { let bundle = PhysicalCompiler diff --git a/control_plane/src/physical/mod.rs b/control_plane/src/physical/mod.rs index b347524b..12e331f4 100644 --- a/control_plane/src/physical/mod.rs +++ b/control_plane/src/physical/mod.rs @@ -43,3 +43,5 @@ pub mod workload_planner; // former `algebra` module relied on. pub use allocator::SketchAllocator; pub use plan::{CostEstimate, ExecutionMode, PipelineStage, PlanNode, PlanSummary}; + +pub mod publication; diff --git a/control_plane/src/physical/publication.rs b/control_plane/src/physical/publication.rs new file mode 100644 index 00000000..b19825c2 --- /dev/null +++ b/control_plane/src/physical/publication.rs @@ -0,0 +1,96 @@ +//! Canonical publication document. Legacy BackendPlan is not an authority in +//! this contract; adapters may attach it for old installation endpoints. +use super::compiler::{CollectorPlan, PhysicalPlan, PrecomputePlan, TransmissionPlan}; +use super::summary_catalog::SummaryCatalog; +use crate::query_plan::QueryPlan; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PhysicalPlanPublication { + pub summary_catalog: SummaryCatalog, + pub precompute_plan: PrecomputePlan, + pub collector_plans: Vec, + pub transmission_plan: TransmissionPlan, + pub query_plan: QueryPlan, +} +impl PhysicalPlanPublication { + /// Validate the complete generation without decoding a BackendPlan. + pub fn validate(&self) -> Result<(), String> { + let catalog = &self.summary_catalog; + self.precompute_plan + .validate_against_catalog(catalog) + .map_err(|e| e.to_string())?; + self.transmission_plan + .validate(&self.precompute_plan) + .map_err(|e| e.to_string())?; + self.transmission_plan + .validate_against_catalog(catalog) + .map_err(|e| e.to_string())?; + self.query_plan + .validate_against_catalog(catalog) + .map_err(|e| e.to_string())?; + for entry in self.query_plan.entries.values() { + for binding in entry.materialization_bindings() { + let config = self + .precompute_plan + .materializations + .iter() + .find(|m| m.policy_fingerprint() == binding.materialization.fingerprint()) + .ok_or("query binding has no precompute materialization")?; + if config.slide_interval.checked_mul(1000) != Some(binding.window_ms) { + return Err("query pane differs from precompute emission interval".into()); + } + } + } + let mut collectors = std::collections::BTreeSet::new(); + for collector in &self.collector_plans { + if collector.envelope != self.precompute_plan.envelope + || !collectors.insert(&collector.collector_id) + { + return Err("collector envelope mismatch or duplicate collector".into()); + } + collector + .validate_against_catalog(catalog) + .map_err(|e| e.to_string())?; + for rule in &collector.transmission_rules { + if !self.transmission_plan.rules.contains(rule) { + return Err( + "collector transmission rule absent from published transmission plan" + .into(), + ); + } + } + } + for producer in &self.precompute_plan.producers { + let collector = self + .collector_plans + .iter() + .find(|c| c.collector_id == producer.collector_id) + .ok_or("precompute producer has no published CollectorPlan")?; + if !collector + .materializations + .iter() + .any(|m| m.materialization == producer.materialization) + { + return Err( + "collector does not produce referenced precompute materialization".into(), + ); + } + } + Ok(()) + } +} +impl PhysicalPlan { + pub fn publication(&self) -> Result { + let artifact = PhysicalPlanPublication { + summary_catalog: self.summary_catalog.clone(), + precompute_plan: self.precompute_plan.clone(), + collector_plans: self.collector_plans.clone(), + transmission_plan: self.transmission_plan.clone(), + query_plan: self.query_plan.clone(), + }; + artifact.validate()?; + Ok(artifact) + } +} From 76c7b8cad8eeb61bb656b4fd90fb3ead18add778 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 13:48:02 -0600 Subject: [PATCH 04/30] Install and retain authoritative SummaryCatalog snapshots --- .../drivers/ingest/prometheus_remote_write.rs | 1 + data_plane/src/drivers/query/servers/http.rs | 135 ++++++++++++++++++ data_plane/src/main.rs | 4 + .../types/hot_reload_config.rs | 3 + .../asapquery_compatibility_process_e2e.rs | 2 + 5 files changed, 145 insertions(+) diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index 961b9585..d3fcfa2c 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -773,6 +773,7 @@ mod tests { capability_snapshot_id: "test".into(), }; let active = ActivePhysicalPlan { + summary_catalog: None, precompute_plan: PrecomputePlan { summary_catalog: None, materialization_contracts: Default::default(), diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index acea438a..c3f7726c 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -2522,6 +2522,7 @@ mod tests { }; let active = crate::storage_engines::types::HotReloadActivePhysicalPlan::new( crate::storage_engines::types::ActivePhysicalPlan { + summary_catalog: None, precompute_plan: PrecomputePlan { summary_catalog: None, materialization_contracts: Default::default(), @@ -6030,6 +6031,9 @@ async fn handle_post_backend_plan( #[derive(serde::Serialize, serde::Deserialize)] #[serde(deny_unknown_fields)] pub struct PhysicalPlanInstallRequest { + pub summary_catalog: control_plane::physical::summary_catalog::SummaryCatalog, + #[serde(default)] + pub collector_plans: Vec, pub precompute_plan: control_plane::physical::compiler::PrecomputePlan, pub transmission_plan: control_plane::physical::compiler::TransmissionPlan, pub backend_plan: Vec, @@ -6046,6 +6050,39 @@ pub fn build_active_physical_plan( default_routing: Arc, ) -> Result { use std::collections::BTreeSet; + request + .precompute_plan + .validate_against_catalog(&request.summary_catalog) + .map_err(|error| format!("PrecomputePlan catalog validation error: {error}"))?; + request + .transmission_plan + .validate_against_catalog(&request.summary_catalog) + .map_err(|error| format!("TransmissionPlan catalog validation error: {error}"))?; + request + .query_plan + .validate_against_catalog(&request.summary_catalog) + .map_err(|error| format!("QueryPlan catalog validation error: {error}"))?; + for collector in &request.collector_plans { + collector + .validate_against_catalog(&request.summary_catalog) + .map_err(|error| format!("CollectorPlan catalog validation error: {error}"))?; + } + for entry in request.query_plan.entries.values() { + for binding in entry.materialization_bindings() { + let materialization = request + .precompute_plan + .materializations + .iter() + .find(|config| config.policy_fingerprint() == binding.materialization.fingerprint()) + .ok_or_else(|| "query binding has no precompute definition".to_string())?; + if binding.window_ms != materialization.slide_interval.saturating_mul(1_000) { + return Err( + "query physical pane duration differs from installed precompute definition" + .into(), + ); + } + } + } let runtime_materializations = request .precompute_plan .runtime_materializations() @@ -6099,6 +6136,7 @@ pub fn build_active_physical_plan( None => default_routing, }; Ok(crate::storage_engines::types::ActivePhysicalPlan { + summary_catalog: Some(Arc::new(request.summary_catalog)), precompute_plan: request.precompute_plan, transmission_plan: request.transmission_plan, runtime_config: Arc::new(runtime_config), @@ -7027,3 +7065,100 @@ mod logical_provenance_tests { assert_eq!(extract_logical_provenance(&mut value), Some(Err(()))); } } + +#[cfg(test)] +mod catalog_install_tests { + use super::{build_active_physical_plan, PhysicalPlanInstallRequest}; + use std::sync::Arc; + + fn request() -> PhysicalPlanInstallRequest { + let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + serde_json::from_str(include_str!( + "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let plan = snapshot.compile().unwrap(); + PhysicalPlanInstallRequest { + summary_catalog: plan.summary_catalog, + collector_plans: plan.collector_plans, + precompute_plan: plan.precompute_plan, + transmission_plan: plan.transmission_plan, + backend_plan: plan.backend_plan.encode_to_vec(), + query_plan: plan.query_plan, + storage_routing: None, + adaptation_evidence: vec![], + } + } + fn install( + request: PhysicalPlanInstallRequest, + ) -> Result { + build_active_physical_plan( + request, + Arc::new(crate::storage_engines::types::BackendStorageRouting::empty()), + ) + } + + // Installing transports the exact supplied snapshot, rather than rebuilding it. + #[test] + fn catalog_install_preserves_authoritative_snapshot() { + let request = request(); + let expected = request.summary_catalog.clone(); + let encoded = serde_json::to_vec(&request).unwrap(); + let active = install(serde_json::from_slice(&encoded).unwrap()).unwrap(); + assert_eq!(active.summary_catalog.as_deref(), Some(&expected)); + active + .query_plan + .validate_against_catalog(active.summary_catalog.as_deref().unwrap()) + .unwrap(); + } + + // Catalog-aware artifacts cannot silently downgrade to a legacy installation. + #[test] + fn catalog_install_requires_snapshot_and_matching_references() { + let mut encoded = serde_json::to_value(request()).unwrap(); + encoded.as_object_mut().unwrap().remove("summary_catalog"); + assert!(serde_json::from_value::(encoded).is_err()); + let mut request = request(); + request.transmission_plan.summary_catalog = None; + assert!(install(request) + .unwrap_err() + .contains("TransmissionPlan catalog")); + } + + // A same-version snapshot replacement fails before the active generation changes. + #[test] + fn catalog_install_rejects_drift_without_replacing_active_snapshot() { + let active = crate::storage_engines::types::HotReloadActivePhysicalPlan::new( + install(request()).unwrap(), + ); + let before = active.snapshot(); + let mut changed = request(); + changed.summary_catalog.plan_version += 1; + assert!(install(changed).is_err()); + assert!(Arc::ptr_eq(&before, &active.snapshot())); + let mut changed = request(); + changed.summary_catalog.summary_descriptors.clear(); + assert!(install(changed).is_err()); + assert!(Arc::ptr_eq(&before, &active.snapshot())); + } + + // Physical pane width cannot be replaced by the semantic lookback at install. + #[test] + fn catalog_install_rejects_query_pane_drift() { + let mut request = request(); + let binding = request + .query_plan + .entries + .values_mut() + .flat_map(|entry| entry.nodes.values_mut()) + .find_map(|node| match node { + control_plane::query_plan::QueryPlanNode::ReadMaterialization { binding } => { + Some(binding) + } + _ => None, + }) + .expect("demo has maintained summaries"); + binding.window_ms += 1; + assert!(install(request).unwrap_err().contains("pane duration")); + } +} diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index f50c05a3..0b0f1bb9 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -492,6 +492,8 @@ async fn main() -> Result<()> { } Some( data_plane::drivers::query::servers::http::PhysicalPlanInstallRequest { + summary_catalog: plan.summary_catalog, + collector_plans: plan.collector_plans, precompute_plan: plan.precompute_plan, transmission_plan: plan.transmission_plan, backend_plan: plan.backend_plan.encode_to_vec(), @@ -716,6 +718,7 @@ async fn main() -> Result<()> { }; let initial_active_plan = startup_physical_plan.unwrap_or_else(|| { data_plane::storage_engines::types::ActivePhysicalPlan { + summary_catalog: None, precompute_plan: initial_precompute_plan, transmission_plan: initial_transmission_plan, runtime_config: streaming_config.clone(), @@ -1109,6 +1112,7 @@ async fn main() -> Result<()> { if active_physical_plan.snapshot().backend_plan.plan_id == 0 { let current = active_physical_plan.snapshot(); active_physical_plan.swap(data_plane::storage_engines::types::ActivePhysicalPlan { + summary_catalog: current.summary_catalog.clone(), precompute_plan: current.precompute_plan.clone(), transmission_plan: current.transmission_plan.clone(), runtime_config: current.runtime_config.clone(), diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index 0f53ff2a..aec7bd38 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -86,6 +86,8 @@ use crate::storage_engines::types::StreamingConfig; /// subsystem must project its view from the same `Arc`. #[derive(Debug, Clone)] pub struct ActivePhysicalPlan { + /// Present for authoritative installations; legacy bootstrap has no catalog. + pub summary_catalog: Option>, pub precompute_plan: control_plane::physical::compiler::PrecomputePlan, pub transmission_plan: control_plane::physical::compiler::TransmissionPlan, pub runtime_config: Arc, @@ -821,6 +823,7 @@ mod tests { capability_snapshot_id: "test".into(), }; ActivePhysicalPlan { + summary_catalog: None, precompute_plan: control_plane::physical::compiler::PrecomputePlan { summary_catalog: None, materialization_contracts: Default::default(), diff --git a/data_plane/tests/asapquery_compatibility_process_e2e.rs b/data_plane/tests/asapquery_compatibility_process_e2e.rs index d23b8f2c..b2e73a54 100644 --- a/data_plane/tests/asapquery_compatibility_process_e2e.rs +++ b/data_plane/tests/asapquery_compatibility_process_e2e.rs @@ -208,6 +208,8 @@ async fn registered_temporal_topk(algorithm: planner_types::post_asap::SketchAlg "count" ); let artifact = data_plane::drivers::query::servers::http::PhysicalPlanInstallRequest { + summary_catalog: plan.summary_catalog, + collector_plans: plan.collector_plans, precompute_plan: plan.precompute_plan, transmission_plan: plan.transmission_plan, backend_plan: plan.backend_plan.encode_to_vec(), From 8730429fe409a19468c46b4010d64835968c16bd Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 13:50:24 -0600 Subject: [PATCH 05/30] Resolve installed query capabilities through catalog replica --- .../asap_query_engine/catalog_resolver.rs | 233 ++++++++++++++++++ .../query_engines/asap_query_engine/engine.rs | 31 ++- .../query_engines/asap_query_engine/mod.rs | 1 + 3 files changed, 252 insertions(+), 13 deletions(-) create mode 100644 data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs diff --git a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs new file mode 100644 index 00000000..6e856d94 --- /dev/null +++ b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs @@ -0,0 +1,233 @@ +//! Read-only resolution against the installed catalog replica. Descriptor +//! integrity and binding source/grouping are checked once during installation; +//! query execution checks only reachable IDs and their requested capabilities. +use std::collections::{BTreeMap, BTreeSet}; + +use asap_types::sds::{DataDescriptor, MaterializationId, SummaryDescriptor, SummaryOperator}; +use asap_types::AggregationType; +use control_plane::physical::summary_catalog::{MaterializationIdentity, SummaryCatalog}; +use control_plane::query_plan::{ExactReadout, QueryPlanEntry, QueryPlanNode, QueryReadout}; + +use crate::query_engines::EngineError; + +pub(crate) struct ResolvedMaterialization<'a> { + pub identity: &'a MaterializationIdentity, + pub summary: &'a SummaryDescriptor, + pub data: &'a DataDescriptor, +} + +fn miss(reason: impl Into) -> EngineError { + EngineError::capability_miss("summary_catalog", reason) +} + +/// Borrow descriptors; never reconstruct them from bindings or store payloads. +pub(crate) fn resolve( + catalog: &SummaryCatalog, + id: MaterializationId, +) -> Result, EngineError> { + let identity = catalog + .materializations + .get(&id) + .ok_or_else(|| miss(format!("unknown materialization {}", id.fingerprint().0)))?; + let summary = catalog + .summary_descriptors + .get(&identity.summary_descriptor_id) + .ok_or_else(|| miss("missing summary descriptor"))?; + let data = catalog + .data_descriptors + .get(&identity.data_descriptor_id) + .ok_or_else(|| miss("missing data descriptor"))?; + Ok(ResolvedMaterialization { + identity, + summary, + data, + }) +} + +impl ResolvedMaterialization<'_> { + pub fn is_exact(&self) -> bool { + matches!( + &self.summary.operator, + SummaryOperator::Configured { + aggregation_type: AggregationType::Sum + | AggregationType::MultipleSum + | AggregationType::Increase + | AggregationType::MultipleIncrease + | AggregationType::MinMax + | AggregationType::MultipleMinMax, + .. + } + ) + } + + fn supports(&self, node: &QueryPlanNode) -> bool { + let SummaryOperator::Configured { + aggregation_type, + aggregation_sub_type, + .. + } = &self.summary.operator + else { + // Partial legacy descriptors cannot attest a configured capability. + return false; + }; + use AggregationType::*; + match node { + QueryPlanNode::ExactReadout { readout, .. } => match readout { + ExactReadout::Sum => matches!(aggregation_type, Sum | MultipleSum), + ExactReadout::Count => *aggregation_type == Sum, + ExactReadout::Increase | ExactReadout::Rate => { + matches!(aggregation_type, Increase | MultipleIncrease) + } + ExactReadout::Max => { + matches!(aggregation_type, MinMax | MultipleMinMax) + && aggregation_sub_type.eq_ignore_ascii_case("max") + } + }, + QueryPlanNode::SummaryEstimate { query, .. } => match query { + QueryReadout::Quantile { q } => { + q.is_finite() + && (0.0..=1.0).contains(q) + && matches!(aggregation_type, DatasketchesKLL | HydraKLL | DDSketch) + } + QueryReadout::Cardinality => *aggregation_type == HLL, + QueryReadout::PointCount { .. } => matches!( + aggregation_type, + CountMinSketch | CountMinSketchWithHeap | CountSketch | CountSketchWithHeap + ), + QueryReadout::TopK { .. } => matches!( + aggregation_type, + CountMinSketchWithHeap | CountSketchWithHeap + ), + }, + _ => false, + } + } +} + +/// Capability matching follows installed state edges, including merges; it +/// never searches the catalog for a replacement materialization. +pub(crate) fn validate_entry( + catalog: Option<&SummaryCatalog>, + entry: &QueryPlanEntry, + plan_id: u64, + plan_version: u64, +) -> Result<(), EngineError> { + if entry.materialization_bindings().is_empty() { + return Ok(()); + } + let catalog = catalog.ok_or_else(|| miss("installed catalog replica unavailable"))?; + if (catalog.plan_id, catalog.plan_version) != (plan_id, plan_version) { + return Err(miss("catalog replica belongs to another plan generation")); + } + let mut states = BTreeMap::new(); + let mut resolved = BTreeMap::new(); + for id in entry.topological_order().map_err(|e| miss(e.to_string()))? { + let node = &entry.nodes[&id]; + let state_ids = match node { + QueryPlanNode::ReadMaterialization { binding } => { + if !resolved.contains_key(&binding.materialization) { + resolved.insert( + binding.materialization, + resolve(catalog, binding.materialization)?, + ); + } + BTreeSet::from([binding.materialization]) + } + QueryPlanNode::SummaryMerge { inputs } => { + let mut ids = BTreeSet::new(); + for input in inputs { + let children: &BTreeSet = states + .get(input) + .ok_or_else(|| miss("summary merge has no state input"))?; + if children.is_empty() { + return Err(miss("summary merge has value input")); + } + ids.extend(children); + } + ids + } + QueryPlanNode::SummaryEstimate { input, .. } + | QueryPlanNode::ExactReadout { input, .. } => { + let ids: &BTreeSet = states + .get(input) + .ok_or_else(|| miss("readout has no state input"))?; + if ids.is_empty() || ids.iter().any(|id| !resolved[id].supports(node)) { + return Err(miss("catalog descriptor cannot satisfy installed readout")); + } + BTreeSet::new() + } + _ => BTreeSet::new(), + }; + states.insert(id, state_ids); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use control_plane::physical::compiler::BackendLocalPlanningSnapshot; + + fn fixture() -> control_plane::physical::compiler::PhysicalPlan { + let mut value: serde_json::Value = serde_json::from_str(include_str!( + "../../../../docs/examples/asapquery-planning-snapshot.json" + )) + .unwrap(); + value["query_workload"]["repeating_queries"][0]["query"] = + "sum(sum_over_time(m[1m]))".into(); + let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(value).unwrap(); + snapshot.compile().unwrap() + } + + // A readout cannot relabel a valid sum materialization as a rate capability. + #[test] + fn rejects_wrong_readout_and_missing_or_stale_replica() { + let bundle = fixture(); + let catalog = &bundle.summary_catalog; + let mut entry = bundle.query_plan.entries.values().next().unwrap().clone(); + validate_entry(Some(catalog), &entry, catalog.plan_id, catalog.plan_version).unwrap(); + assert!(validate_entry(None, &entry, catalog.plan_id, catalog.plan_version).is_err()); + assert!(validate_entry( + Some(catalog), + &entry, + catalog.plan_id, + catalog.plan_version + 1 + ) + .is_err()); + let readout = entry + .nodes + .values_mut() + .find(|n| matches!(n, QueryPlanNode::ExactReadout { .. })) + .unwrap(); + if let QueryPlanNode::ExactReadout { readout, .. } = readout { + *readout = ExactReadout::Rate; + } + assert!( + validate_entry(Some(catalog), &entry, catalog.plan_id, catalog.plan_version).is_err() + ); + } + + // Resolution returns borrowed authoritative definitions, and fails closed on missing IDs. + #[test] + fn resolves_without_descriptor_copies() { + let bundle = fixture(); + let catalog = &bundle.summary_catalog; + let id = *catalog.materializations.keys().next().unwrap(); + let result = resolve(catalog, id).unwrap(); + assert!(std::ptr::eq( + result.identity, + &catalog.materializations[&id] + )); + assert!(std::ptr::eq( + result.summary, + &catalog.summary_descriptors[&result.identity.summary_descriptor_id] + )); + assert!(std::ptr::eq( + result.data, + &catalog.data_descriptors[&result.identity.data_descriptor_id] + )); + let mut broken = catalog.clone(); + broken.data_descriptors.clear(); + assert!(resolve(&broken, id).is_err()); + } +} diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 460b6d83..720c3a7b 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -166,10 +166,16 @@ impl ASAPQueryEngine { } async fn prepare_logical( &self, - _physical: &crate::storage_engines::types::ActivePhysicalPlan, + physical: &crate::storage_engines::types::ActivePhysicalPlan, entry: &control_plane::query_plan::QueryPlanEntry, times: &[u64], ) -> Result { + super::catalog_resolver::validate_entry( + physical.summary_catalog.as_deref(), + entry, + physical.query_plan.plan_id, + physical.query_plan.plan_version, + )?; super::exact_subqueries::prepare( entry, times, @@ -245,18 +251,13 @@ impl ASAPQueryEngine { // the range boundary; Prometheus evaluates the samples that exist. // The physical plan's retention bound guarantees stored panes were // not evicted, so requiring a sample at t0 would reject valid data. - let exact_accumulator_bindings = bindings.iter().all(|binding| { - matches!( - physical - .backend_plan - .materializations - .get(&binding.materialization.fingerprint()) - .map(|materialization| &materialization.family), - Some(planner_types::post_asap::SummaryFamilyType::ExactAggregate( - .. - )) - ) - }); + let exact_accumulator_bindings = + physical.summary_catalog.as_deref().is_some_and(|catalog| { + bindings.iter().all(|binding| { + super::catalog_resolver::resolve(catalog, binding.materialization) + .is_ok_and(|resolved| resolved.is_exact()) + }) + }); let sparse_exact_coverage = exact_accumulator_bindings && result .coverage @@ -596,6 +597,10 @@ impl ASAPQueryEngine { let planned = match physical_plan.as_ref() { Some(physical_plan) => match physical_plan.query_plan.lookup(query) { Ok(query_entry) => { + super::catalog_resolver::validate_entry( + physical_plan.summary_catalog.as_deref(), query_entry, + physical_plan.query_plan.plan_id, physical_plan.query_plan.plan_version, + )?; readiness = Some(( physical_plan.backend_plan.plan_id, physical_plan.backend_plan.plan_version, diff --git a/data_plane/src/query_engines/asap_query_engine/mod.rs b/data_plane/src/query_engines/asap_query_engine/mod.rs index d4df4e98..dc328d81 100644 --- a/data_plane/src/query_engines/asap_query_engine/mod.rs +++ b/data_plane/src/query_engines/asap_query_engine/mod.rs @@ -9,6 +9,7 @@ //! the JSONL deprecation means the archive tier or a hard 404 — //! the JSONL leg has been deleted). +mod catalog_resolver; pub mod engine; mod exact_subqueries; pub mod live_serve; From 821da0a6067fb96b4fcf5de9c49ae38e2a2fc41f Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 14:09:59 -0600 Subject: [PATCH 06/30] Test real hybrid error-ratio execution --- .../asap_query_engine/exact_subqueries.rs | 179 ++++++++++++++++++ 1 file changed, 179 insertions(+) diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index b3997755..1cf17ad0 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -333,6 +333,185 @@ mod tests { ); server.abort(); } + + #[tokio::test] + async fn five_minute_error_ratio_combines_prometheus_cut_with_summary_store() { + use crate::precompute_engine::operators::IncreaseAccumulator; + use crate::query_engines::query_result::{InstantVectorElement, QueryResult}; + use crate::storage_engines::sketch_db::{ + data::AggKind, + index::{Capability, SketchInstanceMetadata}, + }; + use crate::storage_engines::types::{KeyByLabelValues, Measurement}; + use control_plane::query_plan::{ + logical::BinaryOperation, ExactReadout, MaterializationBinding, PhysicalGrouping, + }; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + + const AT: u64 = 300_000; + const MATERIALIZATION: asap_types::PolicyFingerprint = + asap_types::PolicyFingerprint(9001); + let store = crate::storage_engines::sketch_db::index::SketchStore::new(); + store.register(SketchInstanceMetadata { + sid: 41, + metric_name: "http_requests_total".into(), + group_by_keys: std::collections::BTreeSet::from(["job".into()]), + capability: Some(Capability::ExactAgg(asap_types::AggregationType::Increase)), + agg_kind: AggKind::ExactAgg { + agg_type: asap_types::AggregationType::Increase, + parameters_canonical: String::new(), + spatial_filter_canonical: String::new(), + }, + accuracy: None, + first_seen_unix_ms: 0, + retired_at_ms: None, + expires_at_ms: None, + policy_fp: MATERIALIZATION, + }); + let mut denominator = IncreaseAccumulator::new( + Measurement::new(100.0), + 0, + Measurement::new(100.0), + 0, + ); + denominator.update(Measurement::new(400.0), AT as i64); + store.append_precompute( + 41, + BTreeMap::from([("job".into(), "user-service".into())]), + (0, AT), + Box::new(denominator), + ); + + let exact_query = + "sum by (job) (rate(http_requests_total{status=~\"5..\",job=\"user-service\"}[5m]))"; + let entry = entry(BTreeMap::from([ + ( + QueryNodeId(0), + QueryPlanNode::Logical { + operator: LogicalOperator::Binary { + operation: BinaryOperation::Div, + return_bool: false, + }, + inputs: vec![QueryNodeId(1), QueryNodeId(2)], + }, + ), + ( + QueryNodeId(1), + QueryPlanNode::Logical { + operator: LogicalOperator::ExactSubquery { + query: exact_query.into(), + }, + inputs: vec![], + }, + ), + ( + QueryNodeId(2), + QueryPlanNode::ExactReadout { + input: QueryNodeId(3), + readout: ExactReadout::Rate, + }, + ), + ( + QueryNodeId(3), + QueryPlanNode::ReadMaterialization { + binding: MaterializationBinding { + materialization: MATERIALIZATION.into(), + metric: "http_requests_total".into(), + sid_grouping: vec!["job".into()], + output_grouping: PhysicalGrouping::Reduce(vec!["job".into()]), + window_ms: AT, + readout_lookback_ms: Some(AT), + }, + }, + ), + ])); + + let calls = Arc::new(AtomicUsize::new(0)); + let observed_calls = calls.clone(); + let app = axum::Router::new().route( + "/api/v1/query", + axum::routing::get( + move |axum::extract::Query(params): axum::extract::Query< + BTreeMap, + >| { + let observed_calls = observed_calls.clone(); + async move { + observed_calls.fetch_add(1, Ordering::SeqCst); + assert_eq!(params["query"], exact_query); + assert_eq!(params["time"], "300.000"); + axum::Json(serde_json::json!({ + "status": "success", + "data": {"resultType": "vector", "result": [{ + "metric": {"job": "user-service"}, + "value": [300, "0.1"] + }]} + })) + } + }, + ), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + + let prepared = prepare( + &entry, + &[AT], + Some(&format!("http://{address}")), + &reqwest::Client::new(), + ) + .await + .unwrap(); + let (result, stats) = super::super::logical_dag::execute_installed( + &entry, + &prepared, + AT, + |root, at| { + assert_eq!(root, QueryNodeId(2)); + let mut summary = entry.clone(); + summary.root = root; + summary.nodes.retain(|id, _| matches!(id.0, 2 | 3)); + let (outcome, _) = super::super::post_asap_readout::execute_query_plan_instant( + &store, &summary, at, + ) + .map_err(|error| miss(format!("summary readout failed: {error:?}")))?; + let rows = outcome + .series + .into_iter() + .map(|(labels, samples)| { + let (keys, values): (Vec<_>, Vec<_>) = labels.into_iter().unzip(); + Ok(InstantVectorElement::new( + KeyByLabelValues::new_with_labels(values), + samples + .last() + .ok_or_else(|| miss("summary returned no point"))? + .1, + ) + .with_label_keys_override(keys)) + }) + .collect::, EngineError>>()?; + Ok(QueryResult::vector(rows, at)) + }, + ) + .unwrap(); + let QueryResult::Vector(result) = result else { + panic!("instant vector expected") + }; + assert_eq!(result.timestamp, AT); + assert_eq!(result.values.len(), 1); + assert_eq!(result.values[0].label_keys_override.as_deref(), Some(&["job".into()][..])); + assert_eq!(result.values[0].labels.labels, vec!["user-service"]); + assert!((result.values[0].value - 0.1).abs() < 1e-12); + assert_eq!(stats.raw_scan_evaluations, 0); + assert_eq!(stats.remote_evaluations, 1); + assert_eq!(stats.remote_rpcs, 1); + assert_eq!(stats.summary_readout_evaluations, 1); + assert_eq!(calls.load(Ordering::SeqCst), 1); + server.abort(); + } #[test] fn deployed_raw_scan_is_rejected_before_execution() { let entry = entry(BTreeMap::from([( From 69d4babb3876236e7cd5c0ed780c37e3726558f6 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 14:16:34 -0600 Subject: [PATCH 07/30] Add exact counter SDS contract and readout --- control_plane/src/physical/compiler.rs | 6 ++ .../src/physical/runtime_capability.rs | 66 +++------------- control_plane/src/query_plan.rs | 77 +++++++++++++++++++ crates/asap_types/src/sds.rs | 50 +++++++++++- .../operators/increase_accumulator.rs | 41 ++++++++++ .../asap_query_engine/post_asap_readout.rs | 8 ++ .../asap_query_engine/summary_executor.rs | 18 +++++ 7 files changed, 207 insertions(+), 59 deletions(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 3a1a0dc0..5b0f17ce 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -320,6 +320,7 @@ pub enum StateEncoding { SketchlibProtobufV1, SketchCoreMsgpackV1, ExactAccumulatorV1, + ExactCounterAccumulatorV2, } /// Serializable physical state identity derived from Planner's canonical @@ -2688,6 +2689,11 @@ pub(super) fn state_schema_id(fingerprint: asap_types::PolicyFingerprint) -> Str pub(super) fn state_encodings(family: &SummaryFamilyType) -> Vec { match family { + SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Increase + | planner_types::post_asap::ExactKind::Rate, + _, + ) => vec![StateEncoding::ExactCounterAccumulatorV2], SummaryFamilyType::ExactAggregate(..) => vec![StateEncoding::ExactAccumulatorV1], SummaryFamilyType::Sketch(kind, _) if matches!( diff --git a/control_plane/src/physical/runtime_capability.rs b/control_plane/src/physical/runtime_capability.rs index ee46c529..67f6caf6 100644 --- a/control_plane/src/physical/runtime_capability.rs +++ b/control_plane/src/physical/runtime_capability.rs @@ -358,14 +358,11 @@ impl Capability { // serving multi-pop) is NOT allowed — the single-pop // policy has lost the key dimension and can't recover it. // - // Cross-family ExactAgg combos are mostly non-satisfiable - // (Sum vs MinMax, etc. are different operations) — except - // Sum-family serving a required Increase/Rate capability, - // which IS sound; see `sum_satisfies_increase`. + // Exact counter summaries are a distinct state contract. A sum of + // cumulative sample values cannot reconstruct reset correction or + // Prometheus boundary extrapolation. (Capability::ExactAgg(req), Capability::ExactAgg(have)) => { - req == have - || multi_pop_satisfies_single(*req, *have) - || sum_satisfies_increase(*req, *have) + req == have || multi_pop_satisfies_single(*req, *have) } _ => false, } @@ -402,47 +399,6 @@ fn multi_pop_satisfies_single(required: AggregationType, available: AggregationT ) } -/// True when an available Sum-family accumulator can answer a required -/// Increase/Rate capability. -/// -/// `rate()`/`increase()` PromQL both lower to a required -/// `Capability::ExactAgg(AggregationType::Increase)` (`capability_for`'s -/// `AggIntent::Rate | AggIntent::Increase` case, matching -/// `asap_aware_mapping::boundary::implementation_for`'s `SummaryKind::Increase | -/// SummaryKind::Rate` — ASAPController models Rate as its own summary -/// family). But this workspace's data plane has no storage kind distinct -/// from Sum for it: `evaluate_exact_agg_rate` (`sketch_reducer.rs`) -/// already reduces `Sum`/`MultipleSum`/`Increase`/`MultipleIncrease` -/// identically — all read as `Statistic::Sum` per window, then divided -/// by the coverage-aware elapsed range — so a Sum-registered sid's raw -/// per-window deltas answer a rate query exactly as well as an -/// Increase-registered one's. This is what lets counter metrics ingested -/// as plain `Sum` (not every ingest path distinguishes Increase from -/// Sum at registration time) still answer `rate(...)`/`increase(...)`. -/// -/// Respects the same single/multi-population direction as -/// [`multi_pop_satisfies_single`]: a multi-pop available (`MultipleSum`) -/// can serve a single- or multi-pop required capability; a single-pop -/// available (`Sum`) can only serve a single-pop required one — it has -/// already lost the per-key breakdown a multi-pop required capability -/// would need. -/// -/// Deliberately does NOT apply to a required `Capability::ExactAgg(Sum)` -/// (plain `sum_over_time`) — reconstructing Σ-of-cumulative-counter- -/// samples from per-window deltas is unsound (issue #301); that shape is -/// refused explicitly at the query-dispatch layer, not routed here. -fn sum_satisfies_increase(required: AggregationType, available: AggregationType) -> bool { - matches!( - (required, available), - (AggregationType::Increase, AggregationType::Sum) - | (AggregationType::Increase, AggregationType::MultipleSum) - | ( - AggregationType::MultipleIncrease, - AggregationType::MultipleSum - ) - ) -} - // ── AggIntent → Capability bridge ──────────────────────────────────────────── /// Map a semantic [`AggIntent`] to the ASAP-tier [`Capability`] that can @@ -914,17 +870,15 @@ mod tests { } #[test] - fn is_satisfied_by_sum_family_answers_required_increase() { - // A Sum-registered sid's raw per-window deltas answer a - // rate()/increase() query exactly as well as an Increase- - // registered one's -- evaluate_exact_agg_rate (sketch_reducer.rs) - // reduces both identically. See sum_satisfies_increase's doc. + fn sum_family_cannot_impersonate_exact_counter_state() { let required = Capability::ExactAgg(AggregationType::Increase); - assert!(required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); - assert!(required.is_satisfied_by(&Capability::ExactAgg(AggregationType::MultipleSum))); + assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); + assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::MultipleSum))); let required_multi = Capability::ExactAgg(AggregationType::MultipleIncrease); - assert!(required_multi.is_satisfied_by(&Capability::ExactAgg(AggregationType::MultipleSum))); + assert!( + !required_multi.is_satisfied_by(&Capability::ExactAgg(AggregationType::MultipleSum)) + ); } #[test] diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index 573ca984..1ccb032d 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -84,6 +84,33 @@ impl QueryPlan { )); } } + for node in entry.nodes.values() { + let QueryPlanNode::ExactReadout { input, readout } = node else { + continue; + }; + if !matches!(readout, ExactReadout::Increase | ExactReadout::Rate) { + continue; + } + let Some(QueryPlanNode::ReadMaterialization { binding }) = entry.nodes.get(input) + else { + return Err(QueryPlanError::Invalid( + "counter readout must directly consume one catalog materialization".into(), + )); + }; + let identity = &catalog.materializations[&binding.materialization]; + let descriptor = &catalog.summary_descriptors[&identity.summary_descriptor_id]; + if !matches!( + descriptor.fidelity, + asap_types::sds::FidelityGuarantee::ExactCounter { + full_pane_coverage_required: true, + .. + } + ) { + return Err(QueryPlanError::Invalid( + "rate/increase binding does not reference an exact counter SDS".into(), + )); + } + } } Ok(()) } @@ -1003,4 +1030,54 @@ mod catalog_binding_tests { catalog.summary_descriptors.clear(); assert!(plan.validate_against_catalog(&catalog).is_err()); } + + #[test] + fn counter_readout_requires_counter_sds_fidelity() { + fn as_rate_plan(mut plan: QueryPlan) -> QueryPlan { + let entry = plan.entries.values_mut().next().unwrap(); + let read = entry.root; + let root = QueryNodeId(2); + entry.root = root; + entry.nodes.insert( + root, + QueryPlanNode::ExactReadout { + input: read, + readout: ExactReadout::Rate, + }, + ); + plan + } + + let (sum_plan, sum_catalog) = fixture(); + assert!(as_rate_plan(sum_plan) + .validate_against_catalog(&sum_catalog) + .unwrap_err() + .to_string() + .contains("exact counter SDS")); + + let counter = PrecomputeMaterialization::new( + AggregationType::Increase, + String::new(), + Default::default(), + KeyByLabelNames::new(vec!["job".into()]), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + 10, + 10, + WindowKind::Tumbling, + String::new(), + "m".into(), + None, + None, + None, + ); + let counter_catalog = + SummaryCatalog::from_materializations(7, 2, &[counter.clone()]).unwrap(); + let (mut counter_plan, _) = fixture(); + binding(&mut counter_plan).materialization = counter.policy_fingerprint().into(); + as_rate_plan(counter_plan) + .validate_against_catalog(&counter_catalog) + .unwrap(); + } } diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index 8ac512ce..da4c80cd 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -81,6 +81,14 @@ pub enum SummaryOperator { #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum FidelityGuarantee { Exact, + /// Exact PromQL counter readout from fixed-size pane summaries. Each pane + /// stores only `(first value/time, last value/time, reset-corrected delta, + /// sample count)`. A query is eligible only when the selected panes fully + /// cover its range; partial boundary panes must fall back to Prometheus. + ExactCounter { + model: String, + full_pane_coverage_required: bool, + }, KllRankError { k: u32, model: String, @@ -125,6 +133,14 @@ impl FidelityGuarantee { }; } match config.accumulator_spec().map(|s| s.family) { + Ok(SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Increase + | planner_types::post_asap::ExactKind::Rate, + _, + )) => Self::ExactCounter { + model: "prometheus.extrapolated-rate.v1".into(), + full_pane_coverage_required: true, + }, Ok(SummaryFamilyType::ExactAggregate(..)) => Self::Exact, Ok(SummaryFamilyType::Sketch(kind, _)) => match kind.params() { SketchParams::Kll { k } => Self::KllRankError { @@ -167,6 +183,10 @@ impl FidelityGuarantee { fn validate(&self) -> Result<(), SdsError> { let valid = match self { Self::Exact => true, + Self::ExactCounter { + model, + full_pane_coverage_required, + } => !model.is_empty() && *full_pane_coverage_required, Self::KllRankError { k, model } => *k > 0 && !model.is_empty(), Self::DdSketchRelativeError { alpha } => { alpha.is_finite() && *alpha > 0.0 && *alpha < 1.0 @@ -265,6 +285,11 @@ impl SummaryDescriptor { /// they cannot attest heap, Hydra, or aggregation-subtype semantics they lost. pub fn from_config(config: &PrecomputeMaterialization) -> Result { let fidelity = FidelityGuarantee::from_config(config); + let state_schema_version = if matches!(fidelity, FidelityGuarantee::ExactCounter { .. }) { + 2 + } else { + 1 + }; Self::new( SummaryOperator::Configured { aggregation_type: config.aggregation_type, @@ -276,7 +301,7 @@ impl SummaryDescriptor { .collect(), }, fidelity, - 1, + state_schema_version, ) } } @@ -513,6 +538,23 @@ mod tests { ); } #[test] + fn increase_config_declares_prometheus_counter_fidelity() { + let yaml: serde_yaml::Value = serde_yaml::from_str( + "aggregationType: Increase\naggregationSubType: ''\nmetric: requests_total\nlabels:\n grouping: []\n rollup: []\n aggregated: []\nparameters: {}\nwindowSize: 60\nwindowType: tumbling\nspatialFilter: ''\n", + ) + .unwrap(); + let config = + PrecomputeMaterialization::from_yaml_data(&yaml, None, crate::QueryLanguage::promql) + .unwrap(); + assert!(matches!( + SummaryDescriptor::from_config(&config).unwrap().fidelity, + FidelityGuarantee::ExactCounter { + ref model, + full_pane_coverage_required: true + } if model == "prometheus.extrapolated-rate.v1" + )); + } + #[test] fn canonical_nested_parameters_and_model_versions_are_identity() { let a = SummaryOperator::Configured { aggregation_type: AggregationType::Sum, @@ -559,8 +601,10 @@ mod tests { assert_eq!(id.fingerprint(), fingerprint); assert_eq!(id.as_u64(), 42); assert_eq!(crate::PolicyFingerprint::from(id), fingerprint); - assert_eq!(serde_json::to_value(id).unwrap(), serde_json::to_value(fingerprint).unwrap()); + assert_eq!( + serde_json::to_value(id).unwrap(), + serde_json::to_value(fingerprint).unwrap() + ); assert_eq!(serde_json::from_str::("42").unwrap(), id); } - } diff --git a/data_plane/src/precompute_engine/operators/increase_accumulator.rs b/data_plane/src/precompute_engine/operators/increase_accumulator.rs index fa44df88..a20d05ef 100644 --- a/data_plane/src/precompute_engine/operators/increase_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/increase_accumulator.rs @@ -629,6 +629,47 @@ mod tests { assert!((rate - 0.575).abs() < 1e-12); } + #[test] + fn pane_merge_preserves_resets_and_prometheus_extrapolation() { + let mut left = IncreaseAccumulator::new( + Measurement::new(10.0), + 10_000, + Measurement::new(10.0), + 10_000, + ); + left.update(Measurement::new(20.0), 20_000); + let mut right = + IncreaseAccumulator::new(Measurement::new(3.0), 30_000, Measurement::new(3.0), 30_000); + right.update(Measurement::new(13.0), 50_000); + let merged = IncreaseAccumulator::merge_accumulators(vec![right, left]).unwrap(); + assert_eq!(merged.total_increase, 23.0); + assert_eq!(merged.sample_count, 4); + let kwargs = HashMap::from([ + ("range_start_ms".into(), "0".into()), + ("range_end_ms".into(), "60000".into()), + ]); + assert_eq!( + crate::SingleSubpopulationAggregate::query(&merged, Statistic::Increase, Some(&kwargs)) + .unwrap(), + 34.5 + ); + } + + #[test] + fn counter_sds_state_is_constant_size_per_pane() { + let mut acc = IncreaseAccumulator::new(Measurement::new(0.0), 0, Measurement::new(0.0), 0); + let initial = acc.serialize_to_bytes().len(); + for second in 1..=86_400 { + acc.update(Measurement::new(second as f64), second * 1_000); + } + assert_eq!(acc.serialize_to_bytes().len(), initial); + assert_eq!(acc.sample_count, 86_401); + assert_eq!( + acc.approx_memory_bytes(), + std::mem::size_of::() + ); + } + #[test] fn test_increase_accumulator_sum_is_latest_cumulative_value() { // Instant `sum ()` semantics: the per-series summand is diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 2158b2c7..ef900b0a 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -1074,5 +1074,13 @@ mod tests { .expect("execute exact rate DAG"); let value = outcome.series[0].1[0].1; assert!((value - 0.575).abs() < 1e-12, "reset-aware rate={value}"); + assert!( + execute_query_plan_readout(&idx, &entry, 1, 60_000, true).is_err(), + "a partial leading counter pane needs Prometheus boundary samples" + ); + assert!( + execute_query_plan_readout(&idx, &entry, 0, 59_999, true).is_err(), + "a partial trailing counter pane needs Prometheus boundary samples" + ); } } diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index 57c9fc4f..c9cbc921 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -506,6 +506,24 @@ impl QueryExecutionContext<'_> { }); } Candidate::ExactAgg(agg_type) => { + if matches!( + agg_type, + AggregationType::Increase | AggregationType::MultipleIncrease + ) { + // Counter pane statistics are sufficient for Prometheus + // extrapolatedRate only when no query boundary cuts a + // pane. A partial pane would require its first/last + // in-range raw sample, which this SDS intentionally + // does not retain. Fail closed to the exact subtree. + let coverage = self + .index + .exact_agg_coverage_bounds(sid, self.t0_ms, self.t1_ms); + if coverage != Some((self.t0_ms, self.t1_ms)) { + return Err(SummaryExecutorError::Unsupported( + "counter SDS requires full-pane query coverage", + )); + } + } if matches!( agg_type, AggregationType::MinMax | AggregationType::MultipleMinMax From ec4ea91376054b3d31a1cbe8a4374757d495b87d Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 14:48:12 -0600 Subject: [PATCH 08/30] Format SummaryCatalog stack --- control_plane/src/backend_client.rs | 89 ++++++--- control_plane/src/main.rs | 15 +- control_plane/src/physical/compiler.rs | 14 +- crates/asap_types/src/sds.rs | 169 +++++++++++++++++- .../asap_query_engine/exact_subqueries.rs | 28 ++- .../types/hot_reload_config.rs | 110 ++++++------ 6 files changed, 329 insertions(+), 96 deletions(-) diff --git a/control_plane/src/backend_client.rs b/control_plane/src/backend_client.rs index 24984eec..f98bb79f 100644 --- a/control_plane/src/backend_client.rs +++ b/control_plane/src/backend_client.rs @@ -369,21 +369,39 @@ impl BackendClient { storage_routing: Option, adaptation_evidence: &[crate::physical::compiler::RuntimeAdaptationEvidence], ) -> std::result::Result<(), BackendPostError> { - publication.validate().map_err(|error| BackendPostError::Permanent(anyhow::anyhow!(error)))?; + publication + .validate() + .map_err(|error| BackendPostError::Permanent(anyhow::anyhow!(error)))?; let mut body = serde_json::to_value(publication) .map_err(|error| BackendPostError::Permanent(error.into()))?; - let fields = body.as_object_mut().expect("publication serializes as object"); + let fields = body + .as_object_mut() + .expect("publication serializes as object"); if let Some(bytes) = compatibility_backend_plan { fields.insert("backend_plan".into(), serde_json::json!(bytes)); } fields.insert("storage_routing".into(), serde_json::json!(storage_routing)); - fields.insert("adaptation_evidence".into(), serde_json::json!(adaptation_evidence)); - let response = self.http.post(derive_physical_plan_url(&self.endpoint)).json(&body) - .send().await.map_err(classify_reqwest_error)?; + fields.insert( + "adaptation_evidence".into(), + serde_json::json!(adaptation_evidence), + ); + let response = self + .http + .post(derive_physical_plan_url(&self.endpoint)) + .json(&body) + .send() + .await + .map_err(classify_reqwest_error)?; let status = response.status(); - if status.is_success() { Ok(()) } else { + if status.is_success() { + Ok(()) + } else { let body = response.text().await.unwrap_or_default(); - Err(classify_http_status(status, body, "catalog physical-plan POST")) + Err(classify_http_status( + status, + body, + "catalog physical-plan POST", + )) } } @@ -402,14 +420,24 @@ impl BackendClient { // Compatibility replanner has no Planner-selected query/collector DAG. // Still publish the actual catalog and bind every provided projection. let catalog = crate::physical::summary_catalog::SummaryCatalog::from_materializations( - precompute_plan.envelope.plan_id, precompute_plan.envelope.plan_version, + precompute_plan.envelope.plan_id, + precompute_plan.envelope.plan_version, &precompute_plan.materializations, - ).map_err(|error| BackendPostError::Permanent(error.into()))?; + ) + .map_err(|error| BackendPostError::Permanent(error.into()))?; let mut precompute_plan = precompute_plan.clone(); - precompute_plan.bind_catalog(&catalog).map_err(|error| BackendPostError::Permanent(error.into()))?; + precompute_plan + .bind_catalog(&catalog) + .map_err(|error| BackendPostError::Permanent(error.into()))?; let mut transmission_plan = transmission_plan.clone(); - transmission_plan.summary_catalog = Some(catalog.reference().map_err(|error| BackendPostError::Permanent(error.into()))?); - query_plan.validate_against_catalog(&catalog).map_err(|error| BackendPostError::Permanent(error.into()))?; + transmission_plan.summary_catalog = Some( + catalog + .reference() + .map_err(|error| BackendPostError::Permanent(error.into()))?, + ); + query_plan + .validate_against_catalog(&catalog) + .map_err(|error| BackendPostError::Permanent(error.into()))?; let url = derive_physical_plan_url(&self.endpoint); let response = self .http @@ -713,22 +741,43 @@ mod tests { #[tokio::test] async fn catalog_publication_posts_canonical_document_without_legacy_bytes() { - let snapshot: crate::physical::compiler::BackendLocalPlanningSnapshot = serde_json::from_str(include_str!("../../docs/examples/asapquery-planning-snapshot.json")).unwrap(); + let snapshot: crate::physical::compiler::BackendLocalPlanningSnapshot = + serde_json::from_str(include_str!( + "../../docs/examples/asapquery-planning-snapshot.json" + )) + .unwrap(); let publication = snapshot.compile().unwrap().publication().unwrap(); let hits: StdArc>> = StdArc::new(Mutex::new(Vec::new())); let route_hits = hits.clone(); - let app = Router::new().route("/api/v1/physical-plan", post(move |axum::Json(body): axum::Json| { - let hits = route_hits.clone(); - async move { hits.lock().unwrap().push(body); axum::http::StatusCode::OK } - })); + let app = Router::new().route( + "/api/v1/physical-plan", + post(move |axum::Json(body): axum::Json| { + let hits = route_hits.clone(); + async move { + hits.lock().unwrap().push(body); + axum::http::StatusCode::OK + } + }), + ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap(); }); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); let client = BackendClient::new(format!("http://{addr}/api/v1/streaming-config")); - client.post_catalog_plan_typed(&publication, None, None, &[]).await.unwrap(); + client + .post_catalog_plan_typed(&publication, None, None, &[]) + .await + .unwrap(); let bodies = hits.lock().unwrap(); assert_eq!(bodies.len(), 1); - for field in ["summary_catalog", "precompute_plan", "collector_plans", "transmission_plan", "query_plan"] { + for field in [ + "summary_catalog", + "precompute_plan", + "collector_plans", + "transmission_plan", + "query_plan", + ] { assert!(bodies[0].get(field).is_some(), "missing {field}"); } assert!(bodies[0].get("backend_plan").is_none()); diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index aae4ced4..bc7aa7aa 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -671,10 +671,21 @@ async fn handle_compile_and_publish_physical_plan( } let publication = match bundle.publication() { Ok(publication) => publication, - Err(error) => return (StatusCode::INTERNAL_SERVER_ERROR, format!("invalid catalog publication: {error}")).into_response(), + Err(error) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("invalid catalog publication: {error}"), + ) + .into_response() + } }; if let Err(error) = backend - .post_catalog_plan_typed(&publication, Some(bundle.backend_plan.encode_to_vec()), None, &adaptation_evidence) + .post_catalog_plan_typed( + &publication, + Some(bundle.backend_plan.encode_to_vec()), + None, + &adaptation_evidence, + ) .await { return ( diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 5b0f17ce..e04c45e1 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -4312,13 +4312,19 @@ mod tests { #[test] fn publication_is_catalog_authoritative_and_round_trips() { - let mut bundle = PhysicalCompiler.compile(request("publication", "quantile_over_time(0.99, m[1m])"), environment(10_000)).unwrap(); + let mut bundle = PhysicalCompiler + .compile( + request("publication", "quantile_over_time(0.99, m[1m])"), + environment(10_000), + ) + .unwrap(); // Compatibility bytes cannot silently determine publication identity. bundle.backend_plan.plan_id += 1; let publication = bundle.publication().unwrap(); let json = serde_json::to_value(&publication).unwrap(); assert!(json.get("backend_plan").is_none()); - let mut decoded: super::super::publication::PhysicalPlanPublication = serde_json::from_value(json).unwrap(); + let mut decoded: super::super::publication::PhysicalPlanPublication = + serde_json::from_value(json).unwrap(); decoded.validate().unwrap(); decoded.collector_plans.clear(); assert!(decoded.validate().is_err()); @@ -4326,7 +4332,9 @@ mod tests { decoded.summary_catalog.plan_version += 1; assert!(decoded.validate().is_err()); let mut decoded = publication; - decoded.collector_plans.push(decoded.collector_plans[0].clone()); + decoded + .collector_plans + .push(decoded.collector_plans[0].clone()); assert!(decoded.validate().is_err()); } diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index da4c80cd..1a29fcef 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -1,5 +1,5 @@ -//! Shared SDS descriptor contracts. Instances, state bytes, and registry lifetimes -//! remain backend-owned. Canonical IDs describe content, never SID or policy IDs. +//! Shared SDS metadata contracts. Summary payload bytes remain storage-engine +//! owned; catalogs and inventories contain identities and state references only. use crate::{AggregationType, PrecomputeMaterialization}; use planner_types::post_asap::{SketchAlgorithm, SketchParams, SummaryFamilyType}; use serde::{Deserialize, Serialize}; @@ -54,6 +54,171 @@ impl From for crate::PolicyFingerprint { descriptor_id!(SummaryDescriptorId); descriptor_id!(DataDescriptorId); +descriptor_id!(SummaryInstanceId); + +impl SummaryInstanceId { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + if value.trim().is_empty() { + return Err(SdsError("summary instance ID must not be empty".into())); + } + Ok(Self(value)) + } +} + +/// Immutable identity of the desired catalog generation used to create state. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CatalogGeneration { + pub schema_version: u32, + pub plan_id: u64, + pub plan_version: u64, + pub snapshot_digest: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HalfOpenTimeRange { + pub start_ms: i64, + pub end_ms: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum InstanceCompleteness { + Complete, + Partial, + Unknown, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SummaryInstanceStatus { + Building, + Ready, + Retiring, + Failed, + MissingPayload, +} + +/// Logical producer and physical state location. Placement changes do not +/// change descriptor or materialization identity. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SummaryPlacement { + pub producer_id: String, + pub storage_node_id: String, +} + +/// Opaque storage-engine locator. `key` identifies payload stored elsewhere; +/// encoded summary state must never be placed in this metadata contract. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SummaryStateReference { + pub store: String, + pub key: String, + pub state_schema_version: u32, + pub generation: u64, + pub sequence: u64, + pub checksum: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EphemeralLease { + pub lease_id: String, + pub owner_id: String, + pub issued_at_ms: i64, + pub expires_at_ms: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum InstanceLifecycle { + Persistent, + Ephemeral { lease: EphemeralLease }, +} + +/// Observed metadata for one concrete SDS instance. Payload remains in the +/// storage engine and is reached only through `state_reference`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SummaryInstance { + pub instance_id: SummaryInstanceId, + pub materialization_id: MaterializationId, + pub summary_descriptor_id: SummaryDescriptorId, + pub data_descriptor_id: DataDescriptorId, + pub time_range: HalfOpenTimeRange, + pub group_values: BTreeMap, + pub catalog_generation: CatalogGeneration, + pub placement: SummaryPlacement, + pub state_reference: SummaryStateReference, + pub status: SummaryInstanceStatus, + pub completeness: InstanceCompleteness, + pub lifecycle: InstanceLifecycle, + pub observed_at_ms: i64, +} + +impl SummaryInstance { + pub fn validate(&self) -> Result<(), SdsError> { + if self.time_range.start_ms >= self.time_range.end_ms { + return Err(SdsError("summary instance time range must be non-empty".into())); + } + if self.catalog_generation.schema_version == 0 + || self.catalog_generation.snapshot_digest.is_empty() + { + return Err(SdsError("summary instance has invalid catalog generation".into())); + } + if self.placement.producer_id.is_empty() || self.placement.storage_node_id.is_empty() { + return Err(SdsError("summary instance placement must be resolved".into())); + } + if self.state_reference.store.is_empty() + || self.state_reference.key.is_empty() + || self.state_reference.state_schema_version == 0 + { + return Err(SdsError("summary instance has invalid state reference".into())); + } + if let InstanceLifecycle::Ephemeral { lease } = &self.lifecycle { + if lease.lease_id.is_empty() + || lease.owner_id.is_empty() + || lease.issued_at_ms >= lease.expires_at_ms + { + return Err(SdsError("summary instance has invalid ephemeral lease".into())); + } + } + Ok(()) + } +} + +/// Data-plane report of what is actually stored. This is observed state and is +/// deliberately separate from the control-plane desired SummaryCatalog. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ObservedSummaryInventory { + pub reporter_id: String, + pub inventory_version: u64, + pub observed_at_ms: i64, + pub instances: BTreeMap, +} + +impl ObservedSummaryInventory { + pub fn validate(&self) -> Result<(), SdsError> { + if self.reporter_id.is_empty() { + return Err(SdsError("inventory reporter ID must not be empty".into())); + } + for (id, instance) in &self.instances { + instance.validate()?; + if id != &instance.instance_id { + return Err(SdsError("inventory key differs from instance ID".into())); + } + if instance.observed_at_ms > self.observed_at_ms { + return Err(SdsError("instance observation is newer than inventory".into())); + } + } + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum SummaryOperator { diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index 1cf17ad0..ba60cf9b 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -352,8 +352,7 @@ mod tests { }; const AT: u64 = 300_000; - const MATERIALIZATION: asap_types::PolicyFingerprint = - asap_types::PolicyFingerprint(9001); + const MATERIALIZATION: asap_types::PolicyFingerprint = asap_types::PolicyFingerprint(9001); let store = crate::storage_engines::sketch_db::index::SketchStore::new(); store.register(SketchInstanceMetadata { sid: 41, @@ -371,12 +370,8 @@ mod tests { expires_at_ms: None, policy_fp: MATERIALIZATION, }); - let mut denominator = IncreaseAccumulator::new( - Measurement::new(100.0), - 0, - Measurement::new(100.0), - 0, - ); + let mut denominator = + IncreaseAccumulator::new(Measurement::new(100.0), 0, Measurement::new(100.0), 0); denominator.update(Measurement::new(400.0), AT as i64); store.append_precompute( 41, @@ -465,11 +460,8 @@ mod tests { ) .await .unwrap(); - let (result, stats) = super::super::logical_dag::execute_installed( - &entry, - &prepared, - AT, - |root, at| { + let (result, stats) = + super::super::logical_dag::execute_installed(&entry, &prepared, AT, |root, at| { assert_eq!(root, QueryNodeId(2)); let mut summary = entry.clone(); summary.root = root; @@ -494,15 +486,17 @@ mod tests { }) .collect::, EngineError>>()?; Ok(QueryResult::vector(rows, at)) - }, - ) - .unwrap(); + }) + .unwrap(); let QueryResult::Vector(result) = result else { panic!("instant vector expected") }; assert_eq!(result.timestamp, AT); assert_eq!(result.values.len(), 1); - assert_eq!(result.values[0].label_keys_override.as_deref(), Some(&["job".into()][..])); + assert_eq!( + result.values[0].label_keys_override.as_deref(), + Some(&["job".into()][..]) + ); assert_eq!(result.values[0].labels.labels, vec!["user-service"]); assert!((result.values[0].value - 0.1).abs() < 1e-12); assert_eq!(stats.raw_scan_evaluations, 0); diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index aec7bd38..2d5f2f81 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -86,16 +86,43 @@ use crate::storage_engines::types::StreamingConfig; /// subsystem must project its view from the same `Arc`. #[derive(Debug, Clone)] pub struct ActivePhysicalPlan { + /// Authoritative generation and lifecycle identity shared by every plan + /// projection in this immutable snapshot. + pub envelope: control_plane::physical::compiler::PlanEnvelope, /// Present for authoritative installations; legacy bootstrap has no catalog. pub summary_catalog: Option>, pub precompute_plan: control_plane::physical::compiler::PrecomputePlan, pub transmission_plan: control_plane::physical::compiler::TransmissionPlan, pub runtime_config: Arc, - pub backend_plan: Arc, pub query_plan: Arc, pub storage_routing: Arc, } +impl ActivePhysicalPlan { + pub fn plan_id(&self) -> u64 { + self.envelope.plan_id + } + pub fn plan_version(&self) -> u64 { + self.envelope.plan_version + } + pub fn activation_unix_ms(&self) -> u64 { + self.envelope.activation_unix_ms + } + pub fn expiry_unix_ms(&self) -> Option { + self.envelope.expiry_unix_ms + } + + fn materialization_fingerprints( + &self, + ) -> impl Iterator + '_ { + self.precompute_plan + .materialization_contracts + .keys() + .copied() + .map(Into::into) + } +} + #[derive(Clone)] pub struct HotReloadActivePhysicalPlan { inner: Arc>, @@ -128,13 +155,10 @@ struct MaterializationReadinessState { impl MaterializationReadinessState { fn for_plan(plan: &ActivePhysicalPlan) -> Self { - let plan_id = plan.backend_plan.plan_id; - let plan_version = plan.backend_plan.plan_version; + let plan_id = plan.plan_id(); + let plan_version = plan.plan_version(); let statuses = plan - .backend_plan - .materializations - .keys() - .copied() + .materialization_fingerprints() .map(|materialization| { ( materialization, @@ -253,12 +277,9 @@ impl PhysicalPlanLifecycle { pub fn new(active: HotReloadActivePhysicalPlan) -> Self { let snapshot = active.snapshot(); let mut statuses = BTreeMap::new(); - if snapshot.backend_plan.plan_id != 0 { + if snapshot.plan_id() != 0 { statuses.insert( - ( - snapshot.backend_plan.plan_id, - snapshot.backend_plan.plan_version, - ), + (snapshot.plan_id(), snapshot.plan_version()), status_for(&snapshot, PhysicalPlanPhase::Active), ); } @@ -276,18 +297,18 @@ impl PhysicalPlanLifecycle { plan: ActivePhysicalPlan, now: u64, ) -> Result<(), PhysicalPlanLifecycleError> { - let key = (plan.backend_plan.plan_id, plan.backend_plan.plan_version); - if let Some(expiry) = plan.backend_plan.expiry_unix_ms { + let key = (plan.plan_id(), plan.plan_version()); + if let Some(expiry) = plan.expiry_unix_ms() { if expiry <= now { return Err(PhysicalPlanLifecycleError::Expired { expiry, now }); } } let active = self.active.snapshot(); - if active.backend_plan.plan_id != 0 && key.1 <= active.backend_plan.plan_version { + if active.plan_id() != 0 && key.1 <= active.plan_version() { return Err(PhysicalPlanLifecycleError::StaleVersion { plan_id: key.0, incoming: key.1, - active: active.backend_plan.plan_version, + active: active.plan_version(), }); } let mut state = self @@ -350,34 +371,29 @@ impl PhysicalPlanLifecycle { plan_id, plan_version, })?; - if now < plan.backend_plan.activation_unix_ms { + if now < plan.activation_unix_ms() { return Err(PhysicalPlanLifecycleError::ActivationNotReached { - activation: plan.backend_plan.activation_unix_ms, + activation: plan.activation_unix_ms(), now, }); } - if let Some(expiry) = plan.backend_plan.expiry_unix_ms { + if let Some(expiry) = plan.expiry_unix_ms() { if now >= expiry { return Err(PhysicalPlanLifecycleError::Expired { expiry, now }); } } let current = self.active.snapshot(); - if current.backend_plan.plan_id != 0 - && plan.backend_plan.plan_version <= current.backend_plan.plan_version - { + if current.plan_id() != 0 && plan.plan_version() <= current.plan_version() { return Err(PhysicalPlanLifecycleError::StaleVersion { plan_id, incoming: plan_version, - active: current.backend_plan.plan_version, + active: current.plan_version(), }); } let plan = state.staged.remove(&key).expect("staged plan disappeared"); let old = self.active.swap(plan.clone()); - if old.backend_plan.plan_id != 0 { - if let Some(status) = state - .statuses - .get_mut(&(old.backend_plan.plan_id, old.backend_plan.plan_version)) - { + if old.plan_id() != 0 { + if let Some(status) = state.statuses.get_mut(&(old.plan_id(), old.plan_version())) { status.phase = PhysicalPlanPhase::Draining; } } @@ -416,11 +432,11 @@ impl PhysicalPlanLifecycle { fn status_for(plan: &ActivePhysicalPlan, phase: PhysicalPlanPhase) -> PhysicalPlanStatus { PhysicalPlanStatus { - plan_id: plan.backend_plan.plan_id, - plan_version: plan.backend_plan.plan_version, + plan_id: plan.plan_id(), + plan_version: plan.plan_version(), phase, - activation_unix_ms: plan.backend_plan.activation_unix_ms, - expiry_unix_ms: plan.backend_plan.expiry_unix_ms, + activation_unix_ms: plan.activation_unix_ms(), + expiry_unix_ms: plan.expiry_unix_ms(), } } @@ -524,7 +540,7 @@ impl std::fmt::Debug for HotReloadActivePhysicalPlan { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let snapshot = self.snapshot(); f.debug_struct("HotReloadActivePhysicalPlan") - .field("plan_id", &snapshot.backend_plan.plan_id) + .field("plan_id", &snapshot.plan_id()) .field("query_count", &snapshot.query_plan.entries.len()) .field( "materializations", @@ -547,7 +563,6 @@ impl std::fmt::Debug for HotReloadActivePhysicalPlan { pub struct HotReloadBackendPlan { inner: Arc>, install_lock: Arc>, - active: Option, } impl HotReloadBackendPlan { @@ -555,7 +570,6 @@ impl HotReloadBackendPlan { Self { inner: Arc::new(ArcSwap::new(Arc::new(initial))), install_lock: Arc::new(std::sync::Mutex::new(())), - active: None, } } @@ -563,24 +577,16 @@ impl HotReloadBackendPlan { Self { inner: Arc::new(ArcSwap::new(initial)), install_lock: Arc::new(std::sync::Mutex::new(())), - active: None, } } pub fn from_active(active: HotReloadActivePhysicalPlan) -> Self { - let initial = active.snapshot().backend_plan.clone(); - Self { - inner: Arc::new(ArcSwap::new(initial)), - install_lock: Arc::new(std::sync::Mutex::new(())), - active: Some(active), - } + let _ = active; + Self::default() } pub fn snapshot(&self) -> Arc { - self.active - .as_ref() - .map(|a| a.snapshot().backend_plan.clone()) - .unwrap_or_else(|| self.inner.load_full()) + self.inner.load_full() } pub fn swap( @@ -991,14 +997,14 @@ mod tests { .stage(physical_plan(7, 2, 200, Some(500)), 150) .unwrap(); - assert_eq!(active.snapshot().backend_plan.plan_version, 1); + assert_eq!(active.snapshot().plan_version(), 1); assert!(matches!( lifecycle.activate(7, 2, 199), Err(PhysicalPlanLifecycleError::ActivationNotReached { .. }) )); let old = lifecycle.activate(7, 2, 200).unwrap(); - assert_eq!(old.backend_plan.plan_version, 1); - assert_eq!(active.snapshot().backend_plan.plan_version, 2); + assert_eq!(old.plan_version(), 1); + assert_eq!(active.snapshot().plan_version(), 2); let statuses = lifecycle.statuses(); assert_eq!(statuses.len(), 2); @@ -1023,8 +1029,8 @@ mod tests { .unwrap(); lifecycle.activate(7, 2, 300).unwrap(); assert!(lifecycle.discard_staged(7, 2).is_err()); - assert_eq!(active.snapshot().backend_plan.plan_version, 2); - assert_eq!(held_reader.backend_plan.plan_version, 1); + assert_eq!(active.snapshot().plan_version(), 2); + assert_eq!(held_reader.plan_version(), 1); } #[test] @@ -1092,6 +1098,6 @@ mod tests { .. }) )); - assert_eq!(active.snapshot().backend_plan.plan_version, 3); + assert_eq!(active.snapshot().plan_version(), 3); } } From cd1bae07837908630fd7dc8c28a3dfcaaa0820d7 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 14:50:08 -0600 Subject: [PATCH 09/30] Remove BackendPlan from active runtime snapshots --- data_plane/src/drivers/ingest/otel.rs | 2 +- .../drivers/ingest/prometheus_remote_write.rs | 2 +- data_plane/src/drivers/query/servers/http.rs | 31 +++++++------------ data_plane/src/main.rs | 19 +++++------- .../query_engines/asap_query_engine/engine.rs | 14 ++++----- .../types/hot_reload_config.rs | 12 ++----- 6 files changed, 29 insertions(+), 51 deletions(-) diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index a5fdc728..4a568a47 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -871,7 +871,7 @@ async fn route_modified_otlp_sketches_to_precompute( .as_ref() .map(|plan| plan.runtime_config.clone()) .unwrap_or_else(|| ingest_state.config_snapshot()); - let active_physical_plan = physical_plan_snapshot.filter(|plan| plan.backend_plan.plan_id != 0); + let active_physical_plan = physical_plan_snapshot.filter(|plan| plan.plan_id() != 0); let lineage_batch_guard = active_physical_plan .as_ref() .map(|_| ingest_state.observability.frame_lineage.lock_batch()); diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index d3fcfa2c..e6184d62 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -773,6 +773,7 @@ mod tests { capability_snapshot_id: "test".into(), }; let active = ActivePhysicalPlan { + envelope: envelope.clone(), summary_catalog: None, precompute_plan: PrecomputePlan { summary_catalog: None, @@ -802,7 +803,6 @@ mod tests { rules: Vec::new(), }, runtime_config: Arc::new(streaming), - backend_plan: Arc::new(control_plane::backend_plan::BackendPlan::default()), query_plan: Arc::new(control_plane::query_plan::QueryPlan::empty()), storage_routing: Arc::new(BackendStorageRouting::empty()), }; diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index c3f7726c..56e20e63 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -2522,6 +2522,7 @@ mod tests { }; let active = crate::storage_engines::types::HotReloadActivePhysicalPlan::new( crate::storage_engines::types::ActivePhysicalPlan { + envelope: envelope.clone(), summary_catalog: None, precompute_plan: PrecomputePlan { summary_catalog: None, @@ -2551,13 +2552,6 @@ mod tests { rules: Vec::new(), }, runtime_config: streaming_config.clone(), - backend_plan: Arc::new(control_plane::backend_plan::BackendPlan { - plan_id: 7, - plan_version: 1, - activation_unix_ms: 1, - backend_compat: control_plane::backend_plan::BACKEND_COMPAT.into(), - ..Default::default() - }), query_plan: Arc::new(control_plane::query_plan::QueryPlan { plan_id: 7, plan_version: 1, @@ -5776,12 +5770,9 @@ async fn handle_health(State(state): State) -> axum::response::Respons return (StatusCode::SERVICE_UNAVAILABLE, "no active PhysicalPlan").into_response(); }; let now = unix_time_ms(); - let lifecycle_ready = active.backend_plan.plan_id != 0 - && active.backend_plan.activation_unix_ms <= now - && active - .backend_plan - .expiry_unix_ms - .is_none_or(|expiry| now < expiry); + let lifecycle_ready = active.plan_id() != 0 + && active.activation_unix_ms() <= now + && active.expiry_unix_ms().is_none_or(|expiry| now < expiry); let ingest_ready = matches!( active.precompute_plan.ingest.protocol, control_plane::physical::compiler::IngestProtocol::PrometheusRemoteWriteV1 @@ -6136,11 +6127,11 @@ pub fn build_active_physical_plan( None => default_routing, }; Ok(crate::storage_engines::types::ActivePhysicalPlan { + envelope: envelope.clone(), summary_catalog: Some(Arc::new(request.summary_catalog)), precompute_plan: request.precompute_plan, transmission_plan: request.transmission_plan, runtime_config: Arc::new(runtime_config), - backend_plan: Arc::new(backend_plan), query_plan: Arc::new(request.query_plan), storage_routing, }) @@ -6202,7 +6193,7 @@ async fn handle_post_physical_plan( } }; if state.remote_write.is_some() - && (active.backend_plan.plan_id == 0 + && (active.plan_id() == 0 || !matches!( active.precompute_plan.ingest.protocol, control_plane::physical::compiler::IngestProtocol::PrometheusRemoteWriteV1 @@ -6218,9 +6209,9 @@ async fn handle_post_physical_plan( ) .into_response(); } - let plan_id = active.backend_plan.plan_id; + let plan_id = active.plan_id(); let materialization_count = active.precompute_plan.materializations.len(); - let plan_version = active.backend_plan.plan_version; + let plan_version = active.plan_version(); let now = unix_time_ms(); if let Err(error) = lifecycle.stage(active, now) { return ( @@ -6276,9 +6267,9 @@ async fn handle_activate_physical_plan( .into_response() } }; - if old.backend_plan.plan_id != 0 { - let draining_id = old.backend_plan.plan_id; - let draining_version = old.backend_plan.plan_version; + if old.plan_id() != 0 { + let draining_id = old.plan_id(); + let draining_version = old.plan_version(); let lifecycle = lifecycle.clone(); tokio::spawn(async move { loop { diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 0b0f1bb9..d2fe5851 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -520,7 +520,7 @@ async fn main() -> Result<()> { ) .map_err(|error| format!("invalid startup PhysicalPlan: {error}"))?; if args.profile == RuntimeProfile::Asapquery - && (active.backend_plan.plan_id == 0 + && (active.plan_id() == 0 || !matches!( active.precompute_plan.ingest.protocol, control_plane::physical::compiler::IngestProtocol::PrometheusRemoteWriteV1 @@ -530,18 +530,14 @@ async fn main() -> Result<()> { return Err("the asapquery profile requires a non-bootstrap physical plan declaring prometheus_remote_write_v1 at /api/v1/write".into()); } let now = unix_time_ms(); - if active.backend_plan.activation_unix_ms > now { + if active.activation_unix_ms() > now { return Err(format!( "physical plan activation {} is later than startup time {now}", - active.backend_plan.activation_unix_ms + active.activation_unix_ms() ) .into()); } - if active - .backend_plan - .expiry_unix_ms - .is_some_and(|expiry| expiry <= now) - { + if active.expiry_unix_ms().is_some_and(|expiry| expiry <= now) { return Err("physical plan artifact is expired".into()); } Some(active) @@ -674,7 +670,6 @@ async fn main() -> Result<()> { // engine (serving-time cutover, Phase 4) and the HTTP server (the // push target) so a POST is observable by the next query, same // sharing contract as `hot_reload_config`. - let initial_backend_plan = control_plane::backend_plan::BackendPlan::default(); let initial_precompute_plan = control_plane::physical::compiler::PrecomputePlan { summary_catalog: None, materialization_contracts: Default::default(), @@ -718,11 +713,11 @@ async fn main() -> Result<()> { }; let initial_active_plan = startup_physical_plan.unwrap_or_else(|| { data_plane::storage_engines::types::ActivePhysicalPlan { + envelope: initial_precompute_plan.envelope.clone(), summary_catalog: None, precompute_plan: initial_precompute_plan, transmission_plan: initial_transmission_plan, runtime_config: streaming_config.clone(), - backend_plan: Arc::new(initial_backend_plan), query_plan: Arc::new(control_plane::query_plan::QueryPlan::empty()), storage_routing: Arc::new( data_plane::storage_engines::types::BackendStorageRouting::empty(), @@ -1109,14 +1104,14 @@ async fn main() -> Result<()> { ); data_plane::storage_engines::types::BackendStorageRouting::empty() }; - if active_physical_plan.snapshot().backend_plan.plan_id == 0 { + if active_physical_plan.snapshot().plan_id() == 0 { let current = active_physical_plan.snapshot(); active_physical_plan.swap(data_plane::storage_engines::types::ActivePhysicalPlan { + envelope: current.envelope.clone(), summary_catalog: current.summary_catalog.clone(), precompute_plan: current.precompute_plan.clone(), transmission_plan: current.transmission_plan.clone(), runtime_config: current.runtime_config.clone(), - backend_plan: current.backend_plan.clone(), query_plan: current.query_plan.clone(), storage_routing: Arc::new(bootstrap_routing), }); diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 720c3a7b..4ef15e79 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -244,8 +244,8 @@ impl ASAPQueryEngine { "readiness registry unavailable", ) })?; - let plan_id = physical.backend_plan.plan_id; - let version = physical.backend_plan.plan_version; + let plan_id = physical.plan_id(); + let version = physical.plan_version(); // Exact range accumulators preserve their actual first/last sample // timestamps. Sparse counter series may legitimately begin after // the range boundary; Prometheus evaluates the samples that exist. @@ -387,7 +387,7 @@ impl ASAPQueryEngine { self.active_physical_plan .as_ref() .map(|handle| handle.snapshot()) - .filter(|plan| plan.backend_plan.plan_id != 0) + .filter(|plan| plan.plan_id() != 0) } /// Phase-5 hybrid-stitch builder — attach an archive engine the @@ -602,8 +602,8 @@ impl ASAPQueryEngine { physical_plan.query_plan.plan_id, physical_plan.query_plan.plan_version, )?; readiness = Some(( - physical_plan.backend_plan.plan_id, - physical_plan.backend_plan.plan_version, + physical_plan.plan_id(), + physical_plan.plan_version(), readiness_requirement(query_entry), )); crate::query_engines::asap_query_engine::live_serve::serve_range_steps_from_query_plan( @@ -961,8 +961,8 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu Some(physical_plan) => match physical_plan.query_plan.lookup(query) { Ok(query_entry) => { readiness = Some(( - physical_plan.backend_plan.plan_id, - physical_plan.backend_plan.plan_version, + physical_plan.plan_id(), + physical_plan.plan_version(), readiness_requirement(query_entry), )); crate::query_engines::asap_query_engine::live_serve::serve_instant_from_query_plan( diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index 2d5f2f81..af56422d 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -829,11 +829,12 @@ mod tests { capability_snapshot_id: "test".into(), }; ActivePhysicalPlan { + envelope: envelope.clone(), summary_catalog: None, precompute_plan: control_plane::physical::compiler::PrecomputePlan { summary_catalog: None, materialization_contracts: Default::default(), - envelope, + envelope: envelope.clone(), ingest: control_plane::physical::compiler::IngestContract { protocol: control_plane::physical::compiler::IngestProtocol::ModifiedOtlpMetricsV1, @@ -869,15 +870,6 @@ mod tests { rules: Vec::new(), }, runtime_config: Arc::new(StreamingConfig::new(HashMap::new())), - backend_plan: Arc::new(control_plane::backend_plan::BackendPlan { - plan_id, - plan_version, - generated_at_unix_ms: activation_unix_ms, - activation_unix_ms, - expiry_unix_ms, - backend_compat: "asap-query-backend.v1".into(), - ..Default::default() - }), query_plan: Arc::new(control_plane::query_plan::QueryPlan { plan_id, plan_version, From 37458e7d52f6f5b61ed0fe6b8956a9eae7833405 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 14:52:18 -0600 Subject: [PATCH 10/30] Add SDS instance inventory reconciliation --- control_plane/src/physical/mod.rs | 1 + .../src/physical/precompute_contract.rs | 15 +- .../src/physical/summary_reconcile.rs | 326 ++++++++++++++++++ crates/asap_types/src/sds.rs | 104 +++++- .../summary-catalog-sds-architecture.md | 38 +- 5 files changed, 462 insertions(+), 22 deletions(-) create mode 100644 control_plane/src/physical/summary_reconcile.rs diff --git a/control_plane/src/physical/mod.rs b/control_plane/src/physical/mod.rs index 12e331f4..d1d2404c 100644 --- a/control_plane/src/physical/mod.rs +++ b/control_plane/src/physical/mod.rs @@ -34,6 +34,7 @@ pub mod runtime_capability; pub mod sketch_catalog; pub mod stage_split; pub mod summary_catalog; +pub mod summary_reconcile; pub mod topology; pub mod window_fusion; pub mod workload_cost; diff --git a/control_plane/src/physical/precompute_contract.rs b/control_plane/src/physical/precompute_contract.rs index 302999cd..a3d36f20 100644 --- a/control_plane/src/physical/precompute_contract.rs +++ b/control_plane/src/physical/precompute_contract.rs @@ -1,7 +1,7 @@ //! Self-contained precompute execution contracts over a SummaryCatalog snapshot. //! BackendPlan remains a compatibility projection, not a validation authority. use super::compiler::*; -use super::summary_catalog::{MaterializationIdentity, SummaryCatalog}; +use super::summary_catalog::SummaryCatalog; use asap_types::sds::{MaterializationId, SummaryDescriptor}; use planner_types::pre_asap::{ColumnRef, Source}; use serde::{Deserialize, Serialize}; @@ -22,7 +22,6 @@ pub enum PrecomputePlacement { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct PrecomputeMaterializationContract { - pub descriptors: MaterializationIdentity, pub schema_id: String, pub update: PrecomputeUpdate, pub placement: PrecomputePlacement, @@ -41,11 +40,9 @@ impl PrecomputePlan { let mut contracts = BTreeMap::new(); for config in &self.materializations { let id = MaterializationId::from(config.policy_fingerprint()); - let descriptors = catalog - .materializations - .get(&id) - .ok_or_else(|| invalid(format!("missing catalog materialization {}", id.as_u64())))? - .clone(); + catalog.materializations.get(&id).ok_or_else(|| { + invalid(format!("missing catalog materialization {}", id.as_u64())) + })?; let schema = self .schemas .iter() @@ -55,7 +52,6 @@ impl PrecomputePlan { contracts.insert( id, PrecomputeMaterializationContract { - descriptors, schema_id: schema.schema_id.clone(), update, placement, @@ -151,9 +147,6 @@ impl PrecomputePlan { let id = MaterializationId::from(config.policy_fingerprint()); let binding = &catalog.materializations[&id]; let contract = &self.materialization_contracts[&id]; - if &contract.descriptors != binding { - return Err(invalid("materialization descriptor reference drift")); - } let expected = SummaryDescriptor::from_config(config).map_err(|e| invalid(e.to_string()))?; if binding.summary_descriptor_id != expected.id { diff --git a/control_plane/src/physical/summary_reconcile.rs b/control_plane/src/physical/summary_reconcile.rs new file mode 100644 index 00000000..16c3da5c --- /dev/null +++ b/control_plane/src/physical/summary_reconcile.rs @@ -0,0 +1,326 @@ +//! Reconcile the control-plane desired SummaryCatalog with data-plane observed +//! instance metadata. Summary payloads remain owned by SummaryStore. + +use std::collections::BTreeSet; + +use asap_types::sds::{ + CatalogGeneration, InstanceLifecycle, MaterializationId, ObservedSummaryInventory, + SummaryInstanceId, SummaryInstanceStatus, +}; +use serde::{Deserialize, Serialize}; + +use super::summary_catalog::{SummaryCatalog, SummaryCatalogError}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "action", rename_all = "snake_case", deny_unknown_fields)] +pub enum SummaryReconcileAction { + Create { + materialization_id: MaterializationId, + }, + Update { + instance_id: SummaryInstanceId, + materialization_id: MaterializationId, + }, + Recover { + instance_id: SummaryInstanceId, + materialization_id: MaterializationId, + }, + Retire { + instance_id: SummaryInstanceId, + }, + GarbageCollect { + instance_id: SummaryInstanceId, + }, + PromoteEphemeral { + instance_id: SummaryInstanceId, + materialization_id: MaterializationId, + }, + ExpireEphemeral { + instance_id: SummaryInstanceId, + }, +} + +#[derive(Debug, thiserror::Error)] +pub enum SummaryReconcileError { + #[error("invalid desired catalog: {0}")] + Catalog(#[from] SummaryCatalogError), + #[error("invalid observed inventory: {0}")] + Inventory(String), +} + +pub fn reconcile_summary_inventory( + desired: &SummaryCatalog, + observed: &ObservedSummaryInventory, + now_ms: i64, + retiring_grace_ms: i64, +) -> Result, SummaryReconcileError> { + desired.validate()?; + observed + .validate() + .map_err(|error| SummaryReconcileError::Inventory(error.to_string()))?; + let desired_generation = catalog_generation(desired)?; + let mut represented = BTreeSet::new(); + let mut actions = Vec::new(); + + for instance in observed.instances.values() { + let desired_identity = desired.materializations.get(&instance.materialization_id); + match &instance.lifecycle { + InstanceLifecycle::Ephemeral { lease } if lease.expires_at_ms <= now_ms => { + actions.push(SummaryReconcileAction::ExpireEphemeral { + instance_id: instance.instance_id.clone(), + }); + continue; + } + InstanceLifecycle::Ephemeral { .. } => { + if desired_identity.is_some_and(|identity| { + identity.summary_descriptor_id == instance.summary_descriptor_id + && identity.data_descriptor_id == instance.data_descriptor_id + }) { + represented.insert(instance.materialization_id); + actions.push(SummaryReconcileAction::PromoteEphemeral { + instance_id: instance.instance_id.clone(), + materialization_id: instance.materialization_id, + }); + } + continue; + } + InstanceLifecycle::Persistent => {} + } + + let Some(identity) = desired_identity else { + if instance.status == SummaryInstanceStatus::Retiring + && now_ms.saturating_sub(instance.observed_at_ms) >= retiring_grace_ms.max(0) + { + actions.push(SummaryReconcileAction::GarbageCollect { + instance_id: instance.instance_id.clone(), + }); + } else if instance.status != SummaryInstanceStatus::Retiring { + actions.push(SummaryReconcileAction::Retire { + instance_id: instance.instance_id.clone(), + }); + } + continue; + }; + represented.insert(instance.materialization_id); + + if matches!( + instance.status, + SummaryInstanceStatus::Failed | SummaryInstanceStatus::MissingPayload + ) { + actions.push(SummaryReconcileAction::Recover { + instance_id: instance.instance_id.clone(), + materialization_id: instance.materialization_id, + }); + } else if identity.summary_descriptor_id != instance.summary_descriptor_id + || identity.data_descriptor_id != instance.data_descriptor_id + || instance.catalog_generation != desired_generation + || instance.status == SummaryInstanceStatus::Retiring + { + actions.push(SummaryReconcileAction::Update { + instance_id: instance.instance_id.clone(), + materialization_id: instance.materialization_id, + }); + } + } + + for id in desired.materializations.keys() { + if !represented.contains(id) { + actions.push(SummaryReconcileAction::Create { + materialization_id: *id, + }); + } + } + Ok(actions) +} + +fn catalog_generation(catalog: &SummaryCatalog) -> Result { + let reference = catalog.reference()?; + Ok(CatalogGeneration { + schema_version: reference.schema_version, + plan_id: reference.plan_id, + plan_version: reference.plan_version, + snapshot_digest: reference.snapshot_sha256, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use asap_types::sds::{ + EphemeralLease, HalfOpenTimeRange, InstanceCompleteness, SummaryPlacement, + SummaryStateReference, + }; + use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + use std::collections::BTreeMap; + + fn config(metric: &str) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( + AggregationType::Sum, + String::new(), + Default::default(), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + 60, + 60, + WindowKind::Tumbling, + String::new(), + metric.into(), + None, + None, + None, + ) + } + + fn instance( + catalog: &SummaryCatalog, + id: &str, + lifecycle: InstanceLifecycle, + status: SummaryInstanceStatus, + observed_at_ms: i64, + ) -> asap_types::sds::SummaryInstance { + let (materialization_id, identity) = catalog.materializations.iter().next().unwrap(); + asap_types::sds::SummaryInstance { + instance_id: SummaryInstanceId::new(id).unwrap(), + materialization_id: *materialization_id, + summary_descriptor_id: identity.summary_descriptor_id.clone(), + data_descriptor_id: identity.data_descriptor_id.clone(), + time_range: HalfOpenTimeRange { + start_ms: 0, + end_ms: 60_000, + }, + group_values: BTreeMap::new(), + catalog_generation: catalog_generation(catalog).unwrap(), + placement: SummaryPlacement { + producer_id: "precompute-1".into(), + storage_node_id: "store-1".into(), + }, + state_reference: SummaryStateReference { + store: "summary-store".into(), + key: id.into(), + state_schema_version: 1, + generation: 1, + sequence: 1, + checksum: None, + }, + status, + completeness: InstanceCompleteness::Complete, + lifecycle, + observed_at_ms, + } + } + + fn inventory(instance: asap_types::sds::SummaryInstance, now: i64) -> ObservedSummaryInventory { + ObservedSummaryInventory { + reporter_id: "store-1".into(), + inventory_version: 1, + observed_at_ms: now, + instances: BTreeMap::from([(instance.instance_id.clone(), instance)]), + } + } + + #[test] + fn creates_missing_and_updates_stale_generation() { + let catalog = SummaryCatalog::from_materializations(1, 2, &[config("a")]).unwrap(); + let empty = ObservedSummaryInventory { + reporter_id: "store-1".into(), + inventory_version: 1, + observed_at_ms: 100, + instances: BTreeMap::new(), + }; + assert!(matches!( + reconcile_summary_inventory(&catalog, &empty, 100, 10).unwrap()[0], + SummaryReconcileAction::Create { .. } + )); + let mut value = instance( + &catalog, + "i", + InstanceLifecycle::Persistent, + SummaryInstanceStatus::Ready, + 100, + ); + value.catalog_generation.plan_version -= 1; + assert!(matches!( + reconcile_summary_inventory(&catalog, &inventory(value, 100), 100, 10).unwrap()[0], + SummaryReconcileAction::Update { .. } + )); + } + + #[test] + fn recovers_missing_payload_and_retires_then_collects_removed_state() { + let catalog = SummaryCatalog::from_materializations(1, 2, &[config("a")]).unwrap(); + let missing = instance( + &catalog, + "missing", + InstanceLifecycle::Persistent, + SummaryInstanceStatus::MissingPayload, + 90, + ); + assert!(matches!( + reconcile_summary_inventory(&catalog, &inventory(missing, 100), 100, 10).unwrap()[0], + SummaryReconcileAction::Recover { .. } + )); + let empty = SummaryCatalog::from_materializations(1, 3, &[]).unwrap(); + let ready = instance( + &catalog, + "old", + InstanceLifecycle::Persistent, + SummaryInstanceStatus::Ready, + 90, + ); + assert!(matches!( + reconcile_summary_inventory(&empty, &inventory(ready, 100), 100, 10).unwrap()[0], + SummaryReconcileAction::Retire { .. } + )); + let retiring = instance( + &catalog, + "old", + InstanceLifecycle::Persistent, + SummaryInstanceStatus::Retiring, + 80, + ); + assert!(matches!( + reconcile_summary_inventory(&empty, &inventory(retiring, 100), 100, 10).unwrap()[0], + SummaryReconcileAction::GarbageCollect { .. } + )); + } + + #[test] + fn promotes_matching_ephemeral_and_expires_lease_first() { + let catalog = SummaryCatalog::from_materializations(1, 2, &[config("a")]).unwrap(); + let lease = |expires_at_ms| InstanceLifecycle::Ephemeral { + lease: EphemeralLease { + lease_id: "lease-1".into(), + owner_id: "fast-path".into(), + issued_at_ms: 10, + expires_at_ms, + }, + }; + let live = instance( + &catalog, + "live", + lease(200), + SummaryInstanceStatus::Ready, + 100, + ); + assert!(matches!( + reconcile_summary_inventory(&catalog, &inventory(live, 100), 100, 10).unwrap()[0], + SummaryReconcileAction::PromoteEphemeral { .. } + )); + let expired = instance( + &catalog, + "expired", + lease(100), + SummaryInstanceStatus::Ready, + 100, + ); + let actions = + reconcile_summary_inventory(&catalog, &inventory(expired, 100), 100, 10).unwrap(); + assert!(matches!( + actions[0], + SummaryReconcileAction::ExpireEphemeral { .. } + )); + assert!(matches!(actions[1], SummaryReconcileAction::Create { .. })); + } +} diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index 1a29fcef..dfd1a8b6 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -162,28 +162,38 @@ pub struct SummaryInstance { impl SummaryInstance { pub fn validate(&self) -> Result<(), SdsError> { if self.time_range.start_ms >= self.time_range.end_ms { - return Err(SdsError("summary instance time range must be non-empty".into())); + return Err(SdsError( + "summary instance time range must be non-empty".into(), + )); } if self.catalog_generation.schema_version == 0 || self.catalog_generation.snapshot_digest.is_empty() { - return Err(SdsError("summary instance has invalid catalog generation".into())); + return Err(SdsError( + "summary instance has invalid catalog generation".into(), + )); } if self.placement.producer_id.is_empty() || self.placement.storage_node_id.is_empty() { - return Err(SdsError("summary instance placement must be resolved".into())); + return Err(SdsError( + "summary instance placement must be resolved".into(), + )); } if self.state_reference.store.is_empty() || self.state_reference.key.is_empty() || self.state_reference.state_schema_version == 0 { - return Err(SdsError("summary instance has invalid state reference".into())); + return Err(SdsError( + "summary instance has invalid state reference".into(), + )); } if let InstanceLifecycle::Ephemeral { lease } = &self.lifecycle { if lease.lease_id.is_empty() || lease.owner_id.is_empty() || lease.issued_at_ms >= lease.expires_at_ms { - return Err(SdsError("summary instance has invalid ephemeral lease".into())); + return Err(SdsError( + "summary instance has invalid ephemeral lease".into(), + )); } } Ok(()) @@ -212,7 +222,9 @@ impl ObservedSummaryInventory { return Err(SdsError("inventory key differs from instance ID".into())); } if instance.observed_at_ms > self.observed_at_ms { - return Err(SdsError("instance observation is newer than inventory".into())); + return Err(SdsError( + "instance observation is newer than inventory".into(), + )); } } Ok(()) @@ -563,6 +575,86 @@ fn data_descriptor_id( #[cfg(test)] mod tests { use super::*; + + fn observed_instance(lifecycle: InstanceLifecycle) -> SummaryInstance { + SummaryInstance { + instance_id: SummaryInstanceId::new("instance-1").unwrap(), + materialization_id: MaterializationId(crate::PolicyFingerprint(7)), + summary_descriptor_id: descriptor( + 200, + FidelityGuarantee::Unknown { + reason: "test".into(), + }, + 1, + ) + .id, + data_descriptor_id: DataDescriptor::new("cpu", "", []).id, + time_range: HalfOpenTimeRange { + start_ms: 0, + end_ms: 10, + }, + group_values: BTreeMap::new(), + catalog_generation: CatalogGeneration { + schema_version: 1, + plan_id: 1, + plan_version: 2, + snapshot_digest: "abc".into(), + }, + placement: SummaryPlacement { + producer_id: "producer".into(), + storage_node_id: "store".into(), + }, + state_reference: SummaryStateReference { + store: "summary-store".into(), + key: "state/1".into(), + state_schema_version: 1, + generation: 1, + sequence: 3, + checksum: None, + }, + status: SummaryInstanceStatus::Ready, + completeness: InstanceCompleteness::Complete, + lifecycle, + observed_at_ms: 10, + } + } + + #[test] + fn instance_inventory_has_metadata_reference_without_payload() { + let instance = observed_instance(InstanceLifecycle::Persistent); + instance.validate().unwrap(); + let encoded = serde_json::to_value(&instance).unwrap(); + assert!(encoded.get("state_reference").is_some()); + assert!(encoded.get("state").is_none()); + assert!(encoded.get("payload").is_none()); + let inventory = ObservedSummaryInventory { + reporter_id: "store".into(), + inventory_version: 4, + observed_at_ms: 10, + instances: BTreeMap::from([(instance.instance_id.clone(), instance)]), + }; + inventory.validate().unwrap(); + } + + #[test] + fn invalid_range_and_lease_are_rejected() { + let mut instance = observed_instance(InstanceLifecycle::Ephemeral { + lease: EphemeralLease { + lease_id: "lease".into(), + owner_id: "fast-path".into(), + issued_at_ms: 10, + expires_at_ms: 20, + }, + }); + instance.validate().unwrap(); + instance.time_range.end_ms = instance.time_range.start_ms; + assert!(instance.validate().is_err()); + instance.time_range.end_ms = 10; + if let InstanceLifecycle::Ephemeral { lease } = &mut instance.lifecycle { + lease.expires_at_ms = lease.issued_at_ms; + } + assert!(instance.validate().is_err()); + } fn descriptor(k: u32, fidelity: FidelityGuarantee, version: u32) -> SummaryDescriptor { SummaryDescriptor::new( SummaryOperator::Sketch { diff --git a/docs/design_docs/summary-catalog-sds-architecture.md b/docs/design_docs/summary-catalog-sds-architecture.md index 519e0266..eafee1a6 100644 --- a/docs/design_docs/summary-catalog-sds-architecture.md +++ b/docs/design_docs/summary-catalog-sds-architecture.md @@ -58,16 +58,26 @@ struct DataDescriptor { struct SummaryInstance { id: SummaryInstanceId, + materialization_id: MaterializationId, summary_descriptor_id: SummaryDescriptorId, data_descriptor_id: DataDescriptorId, interval: HalfOpenInterval, group_values: BTreeMap, completeness: Completeness, - lineage: Lineage, - state: AggPayload, + catalog_generation: CatalogGeneration, + placement: SummaryPlacement, + state_reference: SummaryStateReference, + status: SummaryInstanceStatus, + lifecycle: Persistent | Ephemeral(EphemeralLease), } ``` +The instance contract contains no payload bytes. `SummaryStateReference` is an +opaque storage-engine locator with state-schema version, generation, sequence +and optional checksum. `ObservedSummaryInventory` is a versioned data-plane +report keyed by `SummaryInstanceId`; it is observed state and never part of the +desired catalog snapshot. + ## Authoritative SummaryCatalog and execution plans The control-plane `SummaryCatalog` is the metadata authority. It stores immutable @@ -75,6 +85,21 @@ Summary and Data Descriptors plus stable materialization identities. It does not store pane payloads, watermarks, completeness, or observed availability; those are data-plane instance/runtime metadata. +The control plane reconciles two explicitly separate views: + +- **Desired SummaryCatalog:** persistent materializations selected through + workload feedback and Planner decisions. +- **Observed Summary Inventory:** instances actually building or stored, + including placement, time coverage, state reference, status and generation. + +Reconciliation creates missing desired materializations, updates instances from +old catalog generations, recovers failed or missing payloads, and retires then +garbage-collects materializations removed from desired state. A data-plane fast +path may create only an ephemeral instance with a finite lease and must report +it immediately. A matching desired materialization promotes it; otherwise it +expires and is collected. The data plane cannot promote an ephemeral instance +or create persistent desired state by itself. + ```text ASAPPlanner post-ASAP DAG | @@ -142,9 +167,12 @@ compatibility DTO while older sidecars are read. The implemented `SummaryDescriptor` currently contains one `SummaryOperator`, one derived `FidelityGuarantee`, and a numeric state-schema version. The implemented `DataDescriptor` contains metric name, canonical population filter, -and grouping keys. Observation semantics, structured state schemas, and a -standalone `SummaryInstance` API remain target-model work; pane state and -completeness/lineage tracking currently live in existing `SketchStore` tables. +grouping keys and versioned observation semantics. The shared contract now also +defines `SummaryInstance`, `ObservedSummaryInventory`, placement, completeness, +state references, catalog generation and ephemeral leases. The control-plane +reconciler emits create, update, recover, retire, garbage-collect, promote and +expire actions. Summary payloads and the application of those actions remain in +the SummaryStore runtime. The durable `sid_metadata.json` format is versioned independently. Version 2 contains `summary_descriptors`, `data_descriptors`, and `bindings` tables. A From b9496aac78acbe9bfc215eb684cd2821275270f2 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 14:53:16 -0600 Subject: [PATCH 11/30] Remove BackendPlan capability projection from query runtime --- control_plane/src/physical/compiler.rs | 7 - .../src/physical/precompute_contract.rs | 8 +- .../query_engines/asap_query_engine/engine.rs | 3 +- .../asap_query_engine/live_serve.rs | 56 +-- .../asap_query_engine/post_asap_planner.rs | 371 +----------------- .../asap_query_engine/post_asap_readout.rs | 58 +-- 6 files changed, 45 insertions(+), 458 deletions(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index e04c45e1..b4c5cfd6 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -4273,13 +4273,6 @@ mod tests { bad.schemas[0].window.size_ms += 1; reject(bad); let mut bad = original.clone(); - bad.materialization_contracts - .values_mut() - .next() - .unwrap() - .activation_unix_ms += 1; - reject(bad); - let mut bad = original.clone(); bad.materialization_contracts .values_mut() .next() diff --git a/control_plane/src/physical/precompute_contract.rs b/control_plane/src/physical/precompute_contract.rs index a3d36f20..ac82589a 100644 --- a/control_plane/src/physical/precompute_contract.rs +++ b/control_plane/src/physical/precompute_contract.rs @@ -25,8 +25,6 @@ pub struct PrecomputeMaterializationContract { pub schema_id: String, pub update: PrecomputeUpdate, pub placement: PrecomputePlacement, - pub activation_unix_ms: u64, - pub expiry_unix_ms: Option, pub retained_windows: Option, } fn invalid(reason: impl Into) -> PrecomputePlanError { @@ -55,8 +53,6 @@ impl PrecomputePlan { schema_id: schema.schema_id.clone(), update, placement, - activation_unix_ms: self.envelope.activation_unix_ms, - expiry_unix_ms: self.envelope.expiry_unix_ms, retained_windows: config.num_aggregates_to_retain, }, ); @@ -230,9 +226,7 @@ impl PrecomputePlan { "update/placement does not match ingress and producer bindings", )); } - if contract.activation_unix_ms != self.envelope.activation_unix_ms - || contract.expiry_unix_ms != self.envelope.expiry_unix_ms - || contract.retained_windows != config.num_aggregates_to_retain + if contract.retained_windows != config.num_aggregates_to_retain || contract.retained_windows == Some(0) { return Err(invalid("materialization lifecycle/retention mismatch")); diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 4ef15e79..6725c1e0 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -624,7 +624,6 @@ impl ASAPQueryEngine { end_ms, false, control_plane::types_v2::AccuracyTarget::Epsilon(0.01), - None, ), #[cfg(not(test))] None => Err(crate::query_engines::asap_query_engine::post_asap_planner::LoweringSkip::QueryNotPlanned( @@ -973,7 +972,7 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu }, #[cfg(test)] None => crate::query_engines::asap_query_engine::live_serve::serve_instant_from_summary_executor( - idx, query, now_ms, None, + idx, query, now_ms, ), #[cfg(not(test))] None => Err(crate::query_engines::asap_query_engine::post_asap_planner::LoweringSkip::QueryNotPlanned( diff --git a/data_plane/src/query_engines/asap_query_engine/live_serve.rs b/data_plane/src/query_engines/asap_query_engine/live_serve.rs index 119a4ee6..6aebcd77 100644 --- a/data_plane/src/query_engines/asap_query_engine/live_serve.rs +++ b/data_plane/src/query_engines/asap_query_engine/live_serve.rs @@ -104,29 +104,20 @@ pub fn try_serve_from_summary_executor( t0_ms: u64, t1_ms: u64, is_cumulative: bool, - backend_plan: Option<&control_plane::backend_plan::BackendPlan>, ) -> Option { if !summary_executor_live_enabled() { return None; } - serve_from_summary_executor( - index, - query, - t0_ms, - t1_ms, - is_cumulative, - live_accuracy(), - backend_plan, - ) - .map_err(|skip| { - tracing::debug!( - query, - ?skip, - "live: query not servable from SummaryExecutor, falling back" - ); - }) - .ok() + serve_from_summary_executor(index, query, t0_ms, t1_ms, is_cumulative, live_accuracy()) + .map_err(|skip| { + tracing::debug!( + query, + ?skip, + "live: query not servable from SummaryExecutor, falling back" + ); + }) + .ok() } pub fn serve_from_summary_executor( @@ -136,20 +127,11 @@ pub fn serve_from_summary_executor( t1_ms: u64, is_cumulative: bool, accuracy: AccuracyTarget, - backend_plan: Option<&control_plane::backend_plan::BackendPlan>, ) -> Result { if !summary_executor_live_enabled() { return Err(LoweringSkip::Disabled); } - let outcome = execute_post_asap_readout( - index, - query, - t0_ms, - t1_ms, - is_cumulative, - accuracy, - backend_plan, - )?; + let outcome = execute_post_asap_readout(index, query, t0_ms, t1_ms, is_cumulative, accuracy)?; tracing::debug!(query, "live: served from SummaryExecutor"); Ok(ASAPTierResult { @@ -162,13 +144,11 @@ pub fn serve_instant_from_summary_executor( index: &SketchStore, query: &str, now_ms: u64, - backend_plan: Option<&control_plane::backend_plan::BackendPlan>, ) -> Result<(ASAPTierResult, u64), LoweringSkip> { if !summary_executor_live_enabled() { return Err(LoweringSkip::Disabled); } - let (outcome, t0_ms) = - execute_post_asap_instant(index, query, now_ms, live_accuracy(), backend_plan)?; + let (outcome, t0_ms) = execute_post_asap_instant(index, query, now_ms, live_accuracy())?; Ok(( ASAPTierResult { series: outcome.series, @@ -405,7 +385,6 @@ mod tests { 1_000, 2_000, true, - None, ); assert!(result.is_some(), "unset flag must default to serving"); } @@ -434,7 +413,6 @@ mod tests { 1_000, 2_000, true, - None, ); let result = result.expect("unambiguous single-series quantile must serve"); assert_eq!(result.series.len(), 1); @@ -525,7 +503,7 @@ mod tests { register_hll(&idx, 1, "svc-a", &["a", "b", "c"]); register_hll(&idx, 2, "svc-b", &["d", "e", "f"]); let result = - try_serve_from_summary_executor(&idx, "count(unique_users)", 1_000, 2_000, true, None); + try_serve_from_summary_executor(&idx, "count(unique_users)", 1_000, 2_000, true); let result = result .expect("global-merge shape is no longer ambiguous -- it must be served, not declined"); assert_eq!( @@ -545,14 +523,8 @@ mod tests { #[test] fn flag_on_unservable_query_falls_back() { let idx = SketchStore::new(); - let result = try_serve_from_summary_executor( - &idx, - "rate(http_requests_total[5m])", - 0, - 1000, - true, - None, - ); + let result = + try_serve_from_summary_executor(&idx, "rate(http_requests_total[5m])", 0, 1000, true); assert!(result.is_none()); } } diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_planner.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_planner.rs index fb9fc4d3..b6d54df9 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_planner.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_planner.rs @@ -224,155 +224,6 @@ fn observed_family_for_metric( None } -/// Look up what family/params `plan` says is materialized for `metric` — -/// the `BackendPlan`-sourced sibling of [`observed_family_for_metric`]. -/// Unlike that function, no reconstruction is needed: -/// `Materialization.family` already carries the canonical `SketchKind` this -/// needs, straight off the wire the control plane pushed. Returns the first -/// matching materialization found (mirrors -/// `observed_family_for_metric`'s "first sketch-typed one found" -/// semantics); `None` when the plan has no materialization for this -/// metric. -fn observed_families_for_metric_from_plan( - plan: &control_plane::backend_plan::BackendPlan, - metric: &str, -) -> Vec<(SketchAlgorithm, SketchParams)> { - let mut warm_fingerprints: Vec<_> = plan - .routing - .iter() - .filter(|route| { - route.storage_backend == control_plane::backend_plan::StorageBackend::SketchStore - }) - .map(|route| route.materialization) - .collect(); - warm_fingerprints.sort_by_key(|fp| fp.0); - warm_fingerprints.dedup(); - warm_fingerprints.into_iter().filter_map(|fingerprint| { - let m = plan.materializations.get(&fingerprint)?; - if !matches!(&m.source, planner_types::pre_asap::Source::TimeSeries { metric: mm } if mm == metric) - { - return None; - } - match &m.family { - planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) => { - Some((kind.algorithm().clone(), kind.params().clone())) - } - _ => None, - } - }).collect() -} - -pub fn resolve_materializations_for_post_asap( - plan: &control_plane::backend_plan::BackendPlan, - node: &SummaryNode, - query: &str, - accuracy: AccuracyTarget, -) -> Result, LoweringSkip> { - let parsed = control_plane::query_parser::parse_query(query, accuracy) - .map_err(|e| LoweringSkip::ParseFailed(e.to_string()))?; - let spatial_filter = if parsed.label_filters.is_empty() { - String::new() - } else { - let rendered = parsed - .label_filters - .iter() - .map(|(key, value)| format!("{key}=\"{value}\"")) - .collect::>() - .join(","); - asap_types::utils::normalize_spatial_filter(&rendered) - }; - let required_groups: std::collections::BTreeSet<_> = - parsed.group_by_labels.into_iter().collect(); - let query_window_ms = parsed.time_window.as_millis() as u64; - - fn visit( - plan: &control_plane::backend_plan::BackendPlan, - node: &SummaryNode, - spatial_filter: &str, - required_groups: &std::collections::BTreeSet, - query_window_ms: u64, - resolved: &mut std::collections::BTreeSet, - ) -> Result<(), LoweringSkip> { - match &node.expr { - SummaryExpr::SummaryAgg { child, family, .. } => { - let metric = super::summary_executor::find_metric_in_query_expr_from_summary(child) - .ok_or_else(|| { - LoweringSkip::NoWarmRoute("summary has no time-series source".into()) - })?; - if !matches!( - family, - planner_types::post_asap::SummaryFamilyType::ExactAggregate(..) - | planner_types::post_asap::SummaryFamilyType::Sketch(..) - ) { - return Err(LoweringSkip::NoWarmRoute(format!( - "unsupported maintained family for metric `{metric}`" - ))); - } - let matches: Vec<_> = plan.routing.iter().filter_map(|route| { - (route.storage_backend == control_plane::backend_plan::StorageBackend::SketchStore - && plan.materializations.get(&route.materialization).is_some_and(|m| { - matches!(&m.source, planner_types::pre_asap::Source::TimeSeries { metric: mm } if mm == &metric) - && &m.family == family - && m.spatial_filter == spatial_filter - && required_groups.iter().all(|key| m.group_by.contains(key)) - && m.window.size_ms <= query_window_ms - })) - .then_some(route.materialization) - }).collect(); - if matches.is_empty() { - return Err(LoweringSkip::NoWarmRoute(format!( - "no warm BackendPlan route for metric `{metric}` and family `{family:?}`" - ))); - } - resolved.extend(matches); - visit( - plan, - child, - spatial_filter, - required_groups, - query_window_ms, - resolved, - ) - } - SummaryExpr::SummaryEstimate { summary_input, .. } => visit( - plan, - summary_input, - spatial_filter, - required_groups, - query_window_ms, - resolved, - ), - SummaryExpr::SummaryMerge { children } => { - for child in children { - visit( - plan, - child, - spatial_filter, - required_groups, - query_window_ms, - resolved, - )?; - } - Ok(()) - } - SummaryExpr::KeepPreAsap(_) => Ok(()), - _ => Err(LoweringSkip::NoWarmRoute( - "post-ASAP operator is not executable by the warm runtime".into(), - )), - } - } - let mut resolved = std::collections::BTreeSet::new(); - visit( - plan, - node, - &spatial_filter, - &required_groups, - query_window_ms, - &mut resolved, - )?; - Ok(resolved) -} - fn query_expr_contains_time_range(qe: &planner_types::pre_asap::QueryExpr) -> bool { use planner_types::pre_asap::QueryExpr; match qe { @@ -646,7 +497,6 @@ pub fn plan_promql_to_post_asap( index: &SketchStore, query: &str, accuracy: AccuracyTarget, - backend_plan: Option<&control_plane::backend_plan::BackendPlan>, ) -> Result, LoweringSkip> { let qe = control_plane::query_parser::parse_query_expr_canonical(query, accuracy.clone()) .map_err(|e| LoweringSkip::ParseFailed(e.to_string()))?; @@ -667,19 +517,12 @@ pub fn plan_promql_to_post_asap( // `CostModel` entirely) -- `ObservedFamilyCostModel` then falls back // further to the accuracy-driven default. let metric = find_metric_in_query_expr(&qe); - let mut observed = metric + let mut observed = Vec::new(); + if let Some(family) = metric .as_deref() - .and_then(|metric| { - backend_plan.map(|plan| observed_families_for_metric_from_plan(plan, metric)) - }) - .unwrap_or_default(); - if observed.is_empty() { - if let Some(family) = metric - .as_deref() - .and_then(|metric| observed_family_for_metric(index, metric)) - { - observed.push(family); - } + .and_then(|metric| observed_family_for_metric(index, metric)) + { + observed.push(family); } // No installed family means the planner may use its accuracy-driven // default. With a BackendPlan, try every family materialized for the @@ -726,20 +569,6 @@ pub fn plan_promql_to_post_asap( last_skip = skip; continue; } - if let Some(plan) = backend_plan { - match resolve_materializations_for_post_asap( - plan, - &node, - query, - accuracy.clone(), - ) { - Ok(_) => return Ok(node), - Err(skip) => { - last_skip = skip; - continue; - } - } - } return Ok(node); } } @@ -769,8 +598,7 @@ mod tests { #[test] fn rate_query_is_skipped_before_binding() { let idx = empty_index(); - let result = - plan_promql_to_post_asap(&idx, "rate(http_requests_total[5m])", accuracy(), None); + let result = plan_promql_to_post_asap(&idx, "rate(http_requests_total[5m])", accuracy()); assert!( matches!(result, Err(LoweringSkip::RateShape)), "expected RateShape, got {result:?}" @@ -780,8 +608,7 @@ mod tests { #[test] fn irate_query_is_skipped_before_binding() { let idx = empty_index(); - let result = - plan_promql_to_post_asap(&idx, "irate(http_requests_total[5m])", accuracy(), None); + let result = plan_promql_to_post_asap(&idx, "irate(http_requests_total[5m])", accuracy()); assert!( matches!(result, Err(LoweringSkip::RateShape)), "expected RateShape, got {result:?}" @@ -791,7 +618,7 @@ mod tests { #[test] fn unparseable_query_is_skipped() { let idx = empty_index(); - let result = plan_promql_to_post_asap(&idx, "this is not promql (((", accuracy(), None); + let result = plan_promql_to_post_asap(&idx, "this is not promql (((", accuracy()); assert!( matches!(result, Err(LoweringSkip::ParseFailed(_))), "expected ParseFailed, got {result:?}" @@ -811,7 +638,7 @@ mod tests { // expression stays one opaque `Logical` blob, which this module // surfaces as `NotRealized`. let idx = empty_index(); - let result = plan_promql_to_post_asap(&idx, "http_requests_total", accuracy(), None); + let result = plan_promql_to_post_asap(&idx, "http_requests_total", accuracy()); assert!( matches!(result, Err(LoweringSkip::NotRealized)), "expected NotRealized, got {result:?}" @@ -832,13 +659,9 @@ mod tests { // falls back to the accuracy-driven default -- same outcome as // before this module started consulting the `SketchStore`. let idx = empty_index(); - let node = plan_promql_to_post_asap( - &idx, - "count_over_time(http_requests_total[5m])", - accuracy(), - None, - ) - .expect("Frequency intent must realize via bind_query_expr/ControlPlaneCostModel"); + let node = + plan_promql_to_post_asap(&idx, "count_over_time(http_requests_total[5m])", accuracy()) + .expect("Frequency intent must realize via bind_query_expr/ControlPlaneCostModel"); assert!( !matches!(node.expr, SummaryExpr::KeepPreAsap(_)), "expected a real SummaryAgg/SummaryEstimate binding, got Logical (the gap \ @@ -850,13 +673,9 @@ mod tests { #[test] fn execution_hints_come_from_post_asap_dag() { let idx = empty_index(); - let ranged = plan_promql_to_post_asap( - &idx, - "quantile_over_time(0.99, latency[5m])", - accuracy(), - None, - ) - .expect("quantile plan"); + let ranged = + plan_promql_to_post_asap(&idx, "quantile_over_time(0.99, latency[5m])", accuracy()) + .expect("quantile plan"); assert_eq!( execution_hints(&ranged), PostAsapExecutionHints { @@ -867,7 +686,7 @@ mod tests { ); let plain_sum = - plan_promql_to_post_asap(&idx, "sum(requests)", accuracy(), None).expect("sum plan"); + plan_promql_to_post_asap(&idx, "sum(requests)", accuracy()).expect("sum plan"); assert_eq!( execution_hints(&plain_sum), PostAsapExecutionHints { @@ -889,168 +708,10 @@ mod tests { &idx, "topk(5, sum by (host) (rate(http_requests_total[5m])))", accuracy(), - None, ); assert!( matches!(result, Err(LoweringSkip::NotRealized) | Err(LoweringSkip::RateShape)), "expected NotRealized or RateShape (both are valid skips for this shape), got {result:?}" ); } - - // ── BackendPlan-sourced family lookup (design-backend-plan-wire-format.md §5) ──── - - mod backend_plan_cutover { - use super::*; - use crate::storage_engines::sketch_db::index::{ - AccuracyBound, Capability, SketchAlgorithm, SketchInstanceMetadata, - }; - use asap_types::enums::WindowKind; - use control_plane::backend_plan::{ - BackendPlan, Materialization, RoutingEntry, StorageBackend, WindowSpec, - }; - use planner_types::pre_asap::{ColumnRef, Source}; - use std::collections::HashMap; - - fn register_kll(idx: &SketchStore, metric: &str) { - let cfg = SketchConfig::Kll { k: 269 }; - idx.register(SketchInstanceMetadata { - sid: 1, - metric_name: metric.to_string(), - group_by_keys: Default::default(), - capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::Kll))), - agg_kind: AggKind::Sketch { - algorithm: SketchAlgorithm::Kll, - config: cfg.clone(), - spatial_filter_canonical: String::new(), - }, - accuracy: Some(AccuracyBound::from_config(&cfg)), - first_seen_unix_ms: 0, - retired_at_ms: None, - expires_at_ms: None, - policy_fp: asap_types::PolicyFingerprint::UNSET, - }); - } - - fn plan_with_ddsketch_materialization(metric: &str) -> BackendPlan { - let fingerprint = asap_types::PolicyFingerprint(42); - let mut materializations = HashMap::new(); - materializations.insert( - fingerprint, - Materialization { - fingerprint, - source: Source::TimeSeries { - metric: metric.to_string(), - }, - window: WindowSpec { - kind: WindowKind::Tumbling, - size_ms: 60_000, - slide_ms: None, - }, - group_by: Vec::new(), - rollup: Vec::new(), - spatial_filter: String::new(), - // `Materialization.family` is Planner's canonical - // `SummaryFamilyType`; its sketch branch carries a - // validated `SketchKind` (category + algorithm + params). - family: planner_types::post_asap::SummaryFamilyType::Sketch( - planner_types::post_asap::SketchKind::new( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { alpha: 0.01 }, - ), - planner_types::post_asap::GroupingStrategy::PerSubpopulationInstance, - ), - col: ColumnRef::SampleValue, - retention: None, - lifecycle: None, - }, - ); - BackendPlan { - plan_id: 1, - generated_at_unix_ms: 0, - plan_version: 1, - activation_unix_ms: 1, - expiry_unix_ms: None, - backend_compat: "asap-query-backend.v1".into(), - materializations, - routing: vec![RoutingEntry { - satisfies: Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch)), - materialization: fingerprint, - storage_backend: StorageBackend::SketchStore, - }], - monitors: Vec::new(), - } - } - - /// Extract the bound `(SketchAlgorithm, SketchParams)` from the - /// `SummaryEstimate { summary_input: SummaryNode { expr: SummaryAgg { - /// summary, params, .. }, .. }, .. }` shape a bare - /// `quantile_over_time` query lowers to (confirmed by inspecting - /// the tree directly). - fn bound_family(node: &SummaryNode) -> (SketchAlgorithm, SketchParams) { - match &node.expr { - SummaryExpr::SummaryEstimate { summary_input, .. } => match &summary_input.expr { - SummaryExpr::SummaryAgg { - family: planner_types::post_asap::SummaryFamilyType::Sketch(kind, _), - .. - } => (kind.algorithm().clone(), kind.params().clone()), - other => panic!("expected a Sketch SummaryAgg, got {other:?}"), - }, - other => panic!("expected SummaryEstimate, got {other:?}"), - } - } - - #[test] - fn without_a_plan_sketchstore_reconstruction_wins() { - // Baseline: no `BackendPlan` -- `observed_family_for_metric`'s - // SketchStore reconstruction is the only source. - let idx = SketchStore::new(); - register_kll(&idx, "m"); - let node = - plan_promql_to_post_asap(&idx, "quantile_over_time(0.99, m[1m])", accuracy(), None) - .expect("should lower"); - assert_eq!(bound_family(&node).0, SketchAlgorithm::Kll); - } - - #[test] - fn a_plan_materialization_wins_over_sketchstore_reconstruction() { - // `SketchStore` has Kll registered for `m` (what - // reconstruction alone would find), but the installed - // `BackendPlan` says DDSketch for - // the SAME metric. The plan must win -- serving time reads - // planning's real (plan-sourced) decision, not whatever - // `SketchStore` metadata happens to reconstruct to. - let idx = SketchStore::new(); - register_kll(&idx, "m"); - let plan = plan_with_ddsketch_materialization("m"); - let node = plan_promql_to_post_asap( - &idx, - "quantile_over_time(0.99, m[1m])", - accuracy(), - Some(&plan), - ) - .expect("should lower"); - assert_eq!( - bound_family(&node).0, - SketchAlgorithm::DDSketch, - "BackendPlan's materialization must take priority over SketchStore reconstruction" - ); - } - - #[test] - fn installed_plan_without_metric_route_fails_closed() { - // Once a BackendPlan is installed it is authoritative. Store - // contents that are absent from the plan must not make a query - // warm-eligible, even when a matching legacy SID still exists. - let idx = SketchStore::new(); - register_kll(&idx, "m"); - let plan = plan_with_ddsketch_materialization("some_other_metric"); - let result = plan_promql_to_post_asap( - &idx, - "quantile_over_time(0.99, m[1m])", - accuracy(), - Some(&plan), - ); - assert!(matches!(result, Err(LoweringSkip::NoWarmRoute(_)))); - } - } } diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index ef900b0a..ff97bc00 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -10,7 +10,7 @@ use control_plane::types_v2::AccuracyTarget; use crate::query_engines::asap_query_engine::physical_dag::{self, QueryNodeRuntime}; use crate::query_engines::asap_query_engine::post_asap_planner::{ - execution_hints, plan_promql_to_post_asap, resolve_materializations_for_post_asap, LoweringSkip, + execution_hints, plan_promql_to_post_asap, LoweringSkip, }; use crate::query_engines::asap_query_engine::summary_executor::{ GroupState, QueryExecutionContext, SummaryExecutorError, SummaryValue, @@ -72,19 +72,9 @@ pub fn execute_post_asap_readout( t1_ms: u64, is_cumulative: bool, accuracy: AccuracyTarget, - backend_plan: Option<&control_plane::backend_plan::BackendPlan>, ) -> Result { - let node = plan_promql_to_post_asap(index, query, accuracy.clone(), backend_plan)?; - execute_planned_post_asap( - index, - &node, - query, - accuracy, - t0_ms, - t1_ms, - is_cumulative, - backend_plan, - ) + let node = plan_promql_to_post_asap(index, query, accuracy.clone())?; + execute_planned_post_asap(index, &node, query, accuracy, t0_ms, t1_ms, is_cumulative) } /// Execute an already-bound QueryPlan entry. This is the production serving @@ -455,10 +445,9 @@ pub fn execute_post_asap_instant( query: &str, now_ms: u64, accuracy: AccuracyTarget, - backend_plan: Option<&control_plane::backend_plan::BackendPlan>, ) -> Result<(PostAsapReadoutOutcome, u64), LoweringSkip> { const DEFAULT_LOOKBACK_MS: u64 = 5 * 60 * 1000; - let node = plan_promql_to_post_asap(index, query, accuracy.clone(), backend_plan)?; + let node = plan_promql_to_post_asap(index, query, accuracy.clone())?; let hints = execution_hints(&node); let t0_ms = if hints.full_history { 0 @@ -473,7 +462,6 @@ pub fn execute_post_asap_instant( t0_ms, now_ms, hints.cumulative_readout, - backend_plan, )?; Ok((outcome, t0_ms)) } @@ -486,14 +474,9 @@ fn execute_planned_post_asap( t0_ms: u64, t1_ms: u64, is_cumulative: bool, - backend_plan: Option<&control_plane::backend_plan::BackendPlan>, ) -> Result { - let allowed_materializations = match backend_plan { - Some(plan) => Some(resolve_materializations_for_post_asap( - plan, node, query, accuracy, - )?), - None => None, - }; + let _ = (query, accuracy); + let allowed_materializations = None; let ctx = QueryExecutionContext { index, t0_ms, @@ -775,7 +758,7 @@ mod tests { let idx = SketchStore::new(); register_hll(&idx, 1, "api", &["a", "b"]); register_hll(&idx, 2, "worker", &["b", "c"]); - let node = plan_promql_to_post_asap(&idx, "count(unique_users)", accuracy(), None) + let node = plan_promql_to_post_asap(&idx, "count(unique_users)", accuracy()) .expect("compile-stage fixture"); let canonical = control_plane::query_plan::canonical_promql("count(unique_users)").unwrap(); let entry = control_plane::query_plan::QueryPlanEntry::compile_bound( @@ -815,7 +798,6 @@ mod tests { 2_000, true, accuracy(), - None, ) .expect("should execute"); assert_eq!(outcome.series.len(), 1); @@ -839,16 +821,9 @@ mod tests { let idx = SketchStore::new(); register_hll(&idx, 1, "svc-a", &["a", "b", "c"]); register_hll(&idx, 2, "svc-b", &["d", "e", "f"]); - let outcome = execute_post_asap_readout( - &idx, - "count(unique_users)", - 1_000, - 2_000, - true, - accuracy(), - None, - ) - .expect("should execute"); + let outcome = + execute_post_asap_readout(&idx, "count(unique_users)", 1_000, 2_000, true, accuracy()) + .expect("should execute"); assert_eq!( outcome.series.len(), 1, @@ -892,16 +867,9 @@ mod tests { (1_000, 2_000), Box::new(crate::precompute_engine::operators::SumAccumulator::with_sum(42.0)), ); - let outcome = execute_post_asap_readout( - &idx, - "sum(bytes_total)", - 1_000, - 2_000, - true, - accuracy(), - None, - ) - .expect("should execute"); + let outcome = + execute_post_asap_readout(&idx, "sum(bytes_total)", 1_000, 2_000, true, accuracy()) + .expect("should execute"); // Window-end-only coverage: a single window (1_000, 2_000) is // keyed by its end (2_000) alone, so both bounds equal 2_000 -- // same semantics as `SummaryValue::coverage()`, reconfirmed for From 594eacf6346d6764d3a2b8be26b0e68958ac6d81 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 14:54:41 -0600 Subject: [PATCH 12/30] Version observed SDS inventory metadata --- control_plane/src/physical/summary_reconcile.rs | 4 ++++ crates/asap_types/src/sds.rs | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/control_plane/src/physical/summary_reconcile.rs b/control_plane/src/physical/summary_reconcile.rs index 16c3da5c..b3c05997 100644 --- a/control_plane/src/physical/summary_reconcile.rs +++ b/control_plane/src/physical/summary_reconcile.rs @@ -114,6 +114,8 @@ pub fn reconcile_summary_inventory( } else if identity.summary_descriptor_id != instance.summary_descriptor_id || identity.data_descriptor_id != instance.data_descriptor_id || instance.catalog_generation != desired_generation + || desired.summary_descriptors[&identity.summary_descriptor_id].state_schema_version + != instance.state_reference.state_schema_version || instance.status == SummaryInstanceStatus::Retiring { actions.push(SummaryReconcileAction::Update { @@ -213,6 +215,7 @@ mod tests { fn inventory(instance: asap_types::sds::SummaryInstance, now: i64) -> ObservedSummaryInventory { ObservedSummaryInventory { + schema_version: 1, reporter_id: "store-1".into(), inventory_version: 1, observed_at_ms: now, @@ -224,6 +227,7 @@ mod tests { fn creates_missing_and_updates_stale_generation() { let catalog = SummaryCatalog::from_materializations(1, 2, &[config("a")]).unwrap(); let empty = ObservedSummaryInventory { + schema_version: 1, reporter_id: "store-1".into(), inventory_version: 1, observed_at_ms: 100, diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index dfd1a8b6..7e383168 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -205,6 +205,7 @@ impl SummaryInstance { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ObservedSummaryInventory { + pub schema_version: u32, pub reporter_id: String, pub inventory_version: u64, pub observed_at_ms: i64, @@ -213,6 +214,9 @@ pub struct ObservedSummaryInventory { impl ObservedSummaryInventory { pub fn validate(&self) -> Result<(), SdsError> { + if self.schema_version != 1 { + return Err(SdsError("unsupported observed inventory schema".into())); + } if self.reporter_id.is_empty() { return Err(SdsError("inventory reporter ID must not be empty".into())); } @@ -628,6 +632,7 @@ mod tests { assert!(encoded.get("state").is_none()); assert!(encoded.get("payload").is_none()); let inventory = ObservedSummaryInventory { + schema_version: 1, reporter_id: "store".into(), inventory_version: 4, observed_at_ms: 10, From 723b320064e60b4e9b276ddc1075b72cc11002f7 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 14:55:36 -0600 Subject: [PATCH 13/30] Delete legacy BackendPlan hot reload endpoint --- data_plane/src/drivers/query/servers/http.rs | 253 ------------------ data_plane/src/main.rs | 17 +- .../types/hot_reload_config.rs | 181 ------------- ...e2e_controller_plans_and_backend_serves.rs | 163 +---------- 4 files changed, 3 insertions(+), 611 deletions(-) diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 56e20e63..25cb92b0 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -149,12 +149,6 @@ pub struct HttpServer { /// Hot-reloadable `StreamingConfig` source. `None` when hot-reload /// is not wired up by the caller (unit tests, legacy binaries). hot_reload_config: Option, - /// Hot-reloadable `BackendPlan` handle for `GET/POST - /// /api/v1/backend-plan` (see `control_plane/docs/design-backend-plan-wire-format.md`). - /// `None` when not wired up (unit tests, legacy binaries) — the - /// endpoints return `503`. Shared with `ASAPQueryEngine` so its - /// serving-time lookup sees the same installed plan. - hot_reload_backend_plan: Option, /// Per-metric storage-backend routing table consulted by the HTTP /// instant-query handler at request time. When `Some(..)` and the /// query parses, the handler extracts the metric name from the @@ -217,8 +211,6 @@ struct AppState { adapter: Arc, fallback: Option>, hot_reload_config: Option, - /// See [`HttpServer::hot_reload_backend_plan`]. - hot_reload_backend_plan: Option, /// See [`HttpServer::backend_storage_routing`]. backend_storage_routing: Option, /// Backfill registry (sketch DB §10). See `HttpServer::backfill`. @@ -250,7 +242,6 @@ impl HttpServer { query_router, sketch_index, hot_reload_config: None, - hot_reload_backend_plan: None, backend_storage_routing: None, backfill: None, data_retention_ms: None, @@ -311,20 +302,6 @@ impl HttpServer { self } - /// Attach a `HotReloadBackendPlan` handle so the - /// `GET/POST /api/v1/backend-plan` endpoints can install and read - /// the control plane's typed `BackendPlan` push. Additive alongside - /// [`Self::with_hot_reload_config`] — without this handle the - /// endpoints return `503 Service Unavailable`, same contract as the - /// legacy streaming-config handle. - pub fn with_hot_reload_backend_plan( - mut self, - handle: crate::storage_engines::types::HotReloadBackendPlan, - ) -> Self { - self.hot_reload_backend_plan = Some(handle); - self - } - pub fn with_active_physical_plan( mut self, handle: crate::storage_engines::types::HotReloadActivePhysicalPlan, @@ -442,7 +419,6 @@ impl HttpServer { adapter: adapter.clone(), fallback: self.config.adapter_config.fallback.clone(), hot_reload_config: self.hot_reload_config.clone(), - hot_reload_backend_plan: self.hot_reload_backend_plan.clone(), backend_storage_routing: self.backend_storage_routing.clone(), backfill: self.backfill.clone(), data_retention_ms: self.data_retention_ms, @@ -475,13 +451,6 @@ impl HttpServer { "/api/v1/streaming-config", get(handle_get_streaming_config).post(handle_post_streaming_config), ) - // BackendPlan wire format (design-backend-plan-wire-format.md): - // sibling of streaming-config above, read by ASAPQueryEngine's - // serving-time lookup. POST body is raw protobuf bytes. - .route( - "/api/v1/backend-plan", - get(handle_get_backend_plan).post(handle_post_backend_plan), - ) .route("/api/v1/physical-plan", post(handle_post_physical_plan)) .route( "/api/v1/physical-plan/discard", @@ -555,7 +524,6 @@ impl HttpServer { adapter: adapter.clone(), fallback: self.config.adapter_config.fallback.clone(), hot_reload_config: self.hot_reload_config.clone(), - hot_reload_backend_plan: self.hot_reload_backend_plan.clone(), backend_storage_routing: self.backend_storage_routing.clone(), backfill: self.backfill.clone(), data_retention_ms: self.data_retention_ms, @@ -586,13 +554,6 @@ impl HttpServer { "/api/v1/streaming-config", get(handle_get_streaming_config).post(handle_post_streaming_config), ) - // BackendPlan wire format (design-backend-plan-wire-format.md): - // sibling of streaming-config above, read by ASAPQueryEngine's - // serving-time lookup. POST body is raw protobuf bytes. - .route( - "/api/v1/backend-plan", - get(handle_get_backend_plan).post(handle_post_backend_plan), - ) .route("/api/v1/physical-plan", post(handle_post_physical_plan)) .route( "/api/v1/physical-plan/discard", @@ -2682,32 +2643,6 @@ mod tests { .expect("Failed to start test server") } - async fn setup_test_server_with_backend_plan( - hot_reload: Option, - ) -> u16 { - let adapter_config = - AdapterConfig::prometheus_promql("http://127.0.0.1:9999".to_string(), false); - let config = HttpServerConfig { - port: 0, - handle_http_requests: true, - adapter_config, - }; - let streaming_config = Arc::new(StreamingConfig::default()); - let query_engine = Arc::new(ASAPQueryEngine::new(streaming_config.clone(), 15000)); - let mut server = HttpServer::new( - config, - query_engine, - Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), - ); - if let Some(handle) = hot_reload { - server = server.with_hot_reload_backend_plan(handle); - } - server - .start_test_server() - .await - .expect("Failed to start test server") - } - #[tokio::test] async fn test_get_endpoint_plus_symbol_decoding() { // Enable debug logging for this test @@ -2920,120 +2855,6 @@ aggregations: assert_eq!(body["status"], "error"); } - // ── BackendPlan hot-reload (design-backend-plan-wire-format.md) ───── - - /// POST an encoded `BackendPlan` and verify the active state via GET - /// reflects the swap, and that the underlying hot-reload handle - /// (cloned into the server at setup) sees it too — mirroring - /// `test_streaming_config_hot_reload_round_trip`. - #[tokio::test] - async fn test_backend_plan_hot_reload_round_trip() { - use control_plane::backend_plan::BackendPlan; - - let hot_reload = - crate::storage_engines::types::HotReloadBackendPlan::new(BackendPlan::default()); - let server_port = setup_test_server_with_backend_plan(Some(hot_reload.clone())).await; - let client = Client::new(); - - let initial = client - .get(format!( - "http://127.0.0.1:{server_port}/api/v1/backend-plan" - )) - .send() - .await - .expect("GET failed"); - assert!(initial.status().is_success()); - let initial_body: serde_json::Value = initial.json().await.unwrap(); - assert_eq!(initial_body["materialization_count"], 0); - - let new_plan = BackendPlan { - plan_id: 7, - generated_at_unix_ms: 123, - plan_version: 1, - activation_unix_ms: 123, - backend_compat: "asap-query-backend.v1".into(), - ..Default::default() - }; - let bytes = new_plan.encode_to_vec(); - - let post_resp = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/backend-plan" - )) - .header("content-type", "application/x-protobuf") - .body(bytes) - .send() - .await - .expect("POST failed"); - let post_status = post_resp.status(); - let post_body: serde_json::Value = post_resp.json().await.unwrap(); - assert!( - post_status.is_success(), - "POST returned {post_status}: {post_body}" - ); - assert_eq!(post_body["status"], "success"); - assert_eq!(post_body["plan_id"], 7); - - let after = client - .get(format!( - "http://127.0.0.1:{server_port}/api/v1/backend-plan" - )) - .send() - .await - .expect("GET after swap failed"); - let after_body: serde_json::Value = after.json().await.unwrap(); - assert_eq!(after_body["plan_id"], 7); - assert_eq!(after_body["generated_at_unix_ms"], 123); - - assert_eq!(hot_reload.snapshot().plan_id, 7); - } - - #[tokio::test] - async fn test_backend_plan_hot_reload_missing_handle_503() { - let server_port = setup_test_server_with_backend_plan(None).await; - let client = Client::new(); - - let get_resp = client - .get(format!( - "http://127.0.0.1:{server_port}/api/v1/backend-plan" - )) - .send() - .await - .unwrap(); - assert_eq!(get_resp.status(), reqwest::StatusCode::SERVICE_UNAVAILABLE); - - let post_resp = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/backend-plan" - )) - .body("anything") - .send() - .await - .unwrap(); - assert_eq!(post_resp.status(), reqwest::StatusCode::SERVICE_UNAVAILABLE); - } - - #[tokio::test] - async fn test_backend_plan_hot_reload_rejects_bad_bytes() { - use control_plane::backend_plan::BackendPlan; - let hot_reload = - crate::storage_engines::types::HotReloadBackendPlan::new(BackendPlan::default()); - let server_port = setup_test_server_with_backend_plan(Some(hot_reload)).await; - let client = Client::new(); - - let resp = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/backend-plan" - )) - .body(vec![0xFFu8, 0xFF, 0xFF]) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), reqwest::StatusCode::BAD_REQUEST); - let body: serde_json::Value = resp.json().await.unwrap(); - assert_eq!(body["status"], "error"); - } - /// Set up a test server wired with a hot-reload handle and a /// shared `SketchStore` (the sid catalog the new sid-level /// reconcile reads + writes). Returns `(port, sketch_index)` so @@ -5945,80 +5766,6 @@ async fn handle_post_streaming_config( (StatusCode::OK, axum::Json(body)).into_response() } -// ── BackendPlan hot-reload (design-backend-plan-wire-format.md) ───────── -// -// `GET /api/v1/backend-plan` — return the currently installed plan as -// JSON (debug / verification). -// `POST /api/v1/backend-plan` — accept a protobuf body, decode, and -// atomically swap via ArcSwap. -// -// Sits alongside `/api/v1/streaming-config`, not in place of it — the -// swap here does NOT touch the sid catalog / SketchStore reconciliation; -// that lifecycle management stays on the streaming-config path. - -async fn handle_get_backend_plan(State(state): State) -> axum::response::Response { - use axum::http::StatusCode; - use axum::response::IntoResponse; - - let Some(handle) = state.hot_reload_backend_plan else { - let body = serde_json::json!({ - "status": "error", - "error": "hot-reload backend-plan handle not attached; backend was built without HttpServer::with_hot_reload_backend_plan"}); - return (StatusCode::SERVICE_UNAVAILABLE, axum::Json(body)).into_response(); - }; - let snap = handle.snapshot(); - let body = serde_json::json!({ - "status": "success", - "plan_id": snap.plan_id, - "generated_at_unix_ms": snap.generated_at_unix_ms, - "materialization_count": snap.materializations.len(), - "routing_count": snap.routing.len(), - "monitor_count": snap.monitors.len()}); - (StatusCode::OK, axum::Json(body)).into_response() -} - -async fn handle_post_backend_plan( - State(state): State, - body: axum::body::Bytes, -) -> axum::response::Response { - use axum::http::StatusCode; - use axum::response::IntoResponse; - - let Some(handle) = state.hot_reload_backend_plan else { - let body = serde_json::json!({ - "status": "error", - "error": "hot-reload backend-plan handle not attached; backend was built without HttpServer::with_hot_reload_backend_plan"}); - return (StatusCode::SERVICE_UNAVAILABLE, axum::Json(body)).into_response(); - }; - - let new_plan = match control_plane::backend_plan::BackendPlan::decode(&body) { - Ok(p) => p, - Err(e) => { - let body = serde_json::json!({ - "status": "error", - "error": format!("BackendPlan decode error: {e}")}); - return (StatusCode::BAD_REQUEST, axum::Json(body)).into_response(); - } - }; - - let materialization_count = new_plan.materializations.len(); - let routing_count = new_plan.routing.len(); - let plan_id = new_plan.plan_id; - if let Err(error) = handle.install(new_plan) { - let body = serde_json::json!({ - "status": "error", - "error": format!("BackendPlan validation error: {error}")}); - return (StatusCode::UNPROCESSABLE_ENTITY, axum::Json(body)).into_response(); - } - - let body = serde_json::json!({ - "status": "success", - "plan_id": plan_id, - "materialization_count": materialization_count, - "routing_count": routing_count}); - (StatusCode::OK, axum::Json(body)).into_response() -} - #[derive(serde::Serialize, serde::Deserialize)] #[serde(deny_unknown_fields)] pub struct PhysicalPlanInstallRequest { diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index d2fe5851..1d1848fa 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -662,14 +662,7 @@ async fn main() -> Result<()> { None }; - // BackendPlan wire format (design-backend-plan-wire-format.md): - // install an empty hot-reload handle so `GET/POST - // /api/v1/backend-plan` don't 503 before the control plane's first - // push lands — same "install empty, let the first push fill it in" - // pattern as `bootstrap_routing` below. Shared with both the query - // engine (serving-time cutover, Phase 4) and the HTTP server (the - // push target) so a POST is observable by the next query, same - // sharing contract as `hot_reload_config`. + // Bootstrap projections share one immutable physical-plan envelope. let initial_precompute_plan = control_plane::physical::compiler::PrecomputePlan { summary_catalog: None, materialization_contracts: Default::default(), @@ -730,10 +723,6 @@ async fn main() -> Result<()> { data_plane::storage_engines::types::HotReloadStreamingConfig::from_active( active_physical_plan.clone(), ); - let hot_reload_backend_plan = - data_plane::storage_engines::types::HotReloadBackendPlan::from_active( - active_physical_plan.clone(), - ); // Setup query engine. ASAPQueryEngine shares the same // HotReloadStreamingConfig handle as the HTTP server, so a POST @@ -1041,9 +1030,7 @@ async fn main() -> Result<()> { // Legacy partial-document endpoints remain available to distributed // deployments. The compatibility profile deliberately exposes only // the atomic PhysicalPlan stage/activate lifecycle. - server = server - .with_hot_reload_config(hot_reload_config.clone()) - .with_hot_reload_backend_plan(hot_reload_backend_plan.clone()); + server = server.with_hot_reload_config(hot_reload_config.clone()); } if args.enable_remote_write || args.profile == RuntimeProfile::Asapquery { diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index af56422d..35171ec6 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -550,187 +550,6 @@ impl std::fmt::Debug for HotReloadActivePhysicalPlan { } } -/// Hot-reloadable `BackendPlan` state — same `ArcSwap` shape as -/// [`HotReloadStreamingConfig`], applied to -/// `control_plane::backend_plan::BackendPlan` (see -/// `control_plane/docs/design-backend-plan-wire-format.md`). Lives -/// alongside [`HotReloadStreamingConfig`], not in place of it: -/// `POST /api/v1/backend-plan` installs the latest plan here for -/// `ASAPQueryEngine`'s serving-time lookup to read, while -/// `POST /api/v1/streaming-config` still drives sid-catalog lifecycle -/// (registration/retirement) on its own path. -#[derive(Clone)] -pub struct HotReloadBackendPlan { - inner: Arc>, - install_lock: Arc>, -} - -impl HotReloadBackendPlan { - pub fn new(initial: control_plane::backend_plan::BackendPlan) -> Self { - Self { - inner: Arc::new(ArcSwap::new(Arc::new(initial))), - install_lock: Arc::new(std::sync::Mutex::new(())), - } - } - - pub fn from_arc(initial: Arc) -> Self { - Self { - inner: Arc::new(ArcSwap::new(initial)), - install_lock: Arc::new(std::sync::Mutex::new(())), - } - } - - pub fn from_active(active: HotReloadActivePhysicalPlan) -> Self { - let _ = active; - Self::default() - } - - pub fn snapshot(&self) -> Arc { - self.inner.load_full() - } - - pub fn swap( - &self, - new: control_plane::backend_plan::BackendPlan, - ) -> Arc { - self.inner.swap(Arc::new(new)) - } - - /// Validate then atomically install a plan. A rejected plan never becomes - /// observable and the previous snapshot remains active. - pub fn install( - &self, - new: control_plane::backend_plan::BackendPlan, - ) -> Result< - Arc, - control_plane::backend_plan::ValidationError, - > { - let _guard = self - .install_lock - .lock() - .expect("backend-plan install lock poisoned"); - new.validate()?; - let active = self.snapshot(); - if new.plan_id == active.plan_id && new.plan_id != 0 { - if new.plan_version < active.plan_version { - return Err( - control_plane::backend_plan::ValidationError::StalePlanVersion { - plan_id: new.plan_id, - incoming: new.plan_version, - active: active.plan_version, - }, - ); - } - if new.plan_version == active.plan_version { - if new == *active { - return Ok(active); - } - return Err( - control_plane::backend_plan::ValidationError::ReusedPlanVersion { - plan_id: new.plan_id, - plan_version: new.plan_version, - }, - ); - } - } - if new.generated_at_unix_ms < active.generated_at_unix_ms { - return Err( - control_plane::backend_plan::ValidationError::StaleGeneration { - incoming: new.generated_at_unix_ms, - active: active.generated_at_unix_ms, - }, - ); - } - Ok(self.swap(new)) - } -} - -impl Default for HotReloadBackendPlan { - fn default() -> Self { - Self::new(control_plane::backend_plan::BackendPlan::default()) - } -} - -impl std::fmt::Debug for HotReloadBackendPlan { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let snap = self.snapshot(); - f.debug_struct("HotReloadBackendPlan") - .field("plan_id", &snap.plan_id) - .field("materializations", &snap.materializations.len()) - .field("routing", &snap.routing.len()) - .finish() - } -} - -#[cfg(test)] -mod hot_reload_backend_plan_tests { - use super::*; - use control_plane::backend_plan::BackendPlan; - - fn plan(plan_id: u64) -> BackendPlan { - BackendPlan { - plan_id, - plan_version: 1, - activation_unix_ms: 1, - backend_compat: "asap-query-backend.v1".into(), - ..Default::default() - } - } - - #[test] - fn snapshot_reflects_initial_plan() { - let hr = HotReloadBackendPlan::new(plan(1)); - assert_eq!(hr.snapshot().plan_id, 1); - } - - #[test] - fn swap_replaces_plan_atomically() { - let hr = HotReloadBackendPlan::new(plan(1)); - let old = hr.swap(plan(2)); - assert_eq!(old.plan_id, 1, "swap returns the pre-swap snapshot"); - assert_eq!(hr.snapshot().plan_id, 2); - } - - #[test] - fn clones_share_underlying_swap() { - let hr = HotReloadBackendPlan::new(plan(1)); - let hr_clone = hr.clone(); - hr.swap(plan(2)); - assert_eq!(hr_clone.snapshot().plan_id, 2); - } - - #[test] - fn invalid_plan_is_rejected_without_replacing_snapshot() { - let hr = HotReloadBackendPlan::new(plan(1)); - let mut invalid = plan(2); - invalid - .routing - .push(control_plane::backend_plan::RoutingEntry { - satisfies: control_plane::physical::runtime_capability::Capability::ExactAgg( - asap_types::AggregationType::Sum, - ), - materialization: asap_types::PolicyFingerprint(99), - storage_backend: control_plane::backend_plan::StorageBackend::SketchStore, - }); - assert!(hr.install(invalid).is_err()); - assert_eq!(hr.snapshot().plan_id, 1); - } - - #[test] - fn older_generation_is_rejected_without_replacing_snapshot() { - let mut current = plan(2); - current.generated_at_unix_ms = 200; - let hr = HotReloadBackendPlan::new(current); - let mut stale = plan(3); - stale.generated_at_unix_ms = 199; - assert!(matches!( - hr.install(stale), - Err(control_plane::backend_plan::ValidationError::StaleGeneration { .. }) - )); - assert_eq!(hr.snapshot().plan_id, 2); - } -} - /// Thin wrapper around `ArcSwap` with ergonomic /// snapshot + swap helpers. Cloneable; clones share the same /// underlying `ArcSwap` so all holders see the same swaps. diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index 56dcc934..b6bb21b2 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -247,31 +247,6 @@ async fn get_streaming_config(client: &reqwest::Client, port: u16) -> JsonValue resp.json().await.expect("parse GET response as JSON") } -/// POST an encoded `BackendPlan` to `/api/v1/backend-plan` on the -/// in-process backend, using the control plane's real -/// `BackendClient::post_backend_plan_typed` — the exact code path a live -/// control plane process uses, not a hand-rolled request. -async fn post_backend_plan(port: u16, plan: &control_plane::backend_plan::BackendPlan) { - let endpoint = format!("http://127.0.0.1:{port}/api/v1/streaming-config"); - let client = control_plane::backend_client::BackendClient::new(endpoint); - client - .post_backend_plan_typed(plan.encode_to_vec()) - .await - .expect("POST /api/v1/backend-plan via BackendClient must succeed"); -} - -/// GET `/api/v1/backend-plan` and return the active-plan snapshot as a -/// `serde_json::Value`. Verifies the active state after a POST. -async fn get_backend_plan(client: &reqwest::Client, port: u16) -> JsonValue { - let resp = client - .get(format!("http://127.0.0.1:{port}/api/v1/backend-plan")) - .send() - .await - .expect("GET /api/v1/backend-plan"); - assert!(resp.status().is_success(), "GET returned {}", resp.status()); - resp.json().await.expect("parse GET response as JSON") -} - /// Suppress the `WorkloadCharacteristics` unused warning — kept around /// in case future tests need to pass per-workload resource caps. #[allow(dead_code)] @@ -290,7 +265,6 @@ fn _wc_anchor() -> WorkloadCharacteristics { struct FullStack { backend_port: u16, otlp_http_port: u16, - hot_reload_backend_plan: data_plane::storage_engines::types::HotReloadBackendPlan, } async fn start_full_stack(otlp_http_port: u16, otlp_grpc_port: u16) -> FullStack { @@ -355,9 +329,6 @@ async fn start_full_stack(otlp_http_port: u16, otlp_grpc_port: u16) -> FullStack }); // HTTP query server sharing the same SketchStore + hot-reload handle. - let hot_reload_backend_plan = data_plane::storage_engines::types::HotReloadBackendPlan::new( - control_plane::backend_plan::BackendPlan::default(), - ); let adapter_config = AdapterConfig::prometheus_promql("http://127.0.0.1:9999".to_string(), false); let http_config = HttpServerConfig { @@ -376,8 +347,7 @@ async fn start_full_stack(otlp_http_port: u16, otlp_grpc_port: u16) -> FullStack .with_sketch_index(sketch_index.clone()), ); let server = HttpServer::new(http_config, query_engine, sketch_index) - .with_hot_reload_config(hot_reload.clone()) - .with_hot_reload_backend_plan(hot_reload_backend_plan.clone()); + .with_hot_reload_config(hot_reload.clone()); let backend_port = server .start_test_server() .await @@ -389,7 +359,6 @@ async fn start_full_stack(otlp_http_port: u16, otlp_grpc_port: u16) -> FullStack FullStack { backend_port, otlp_http_port, - hot_reload_backend_plan, } } @@ -1079,136 +1048,6 @@ async fn controller_plan_to_query_full_roundtrip_ddsketch() { ); } -// ── Test — BackendPlan wire format: real push, real install, real serve ──── -// -// Same fixture as `controller_plan_to_query_full_roundtrip_ddsketch`, but -// this time the controller ALSO builds a real `BackendPlan` (via -// `control_plane::backend_plan::from_stage_config`, the exact function a -// live control plane process calls) and pushes it through -// `control_plane::backend_client::BackendClient::post_backend_plan_typed` -// — the exact HTTP client code a live control plane process uses, not a -// hand-rolled request. This proves the whole wire is real end to end: -// control-plane-side construction → protobuf encode → HTTP POST → -// backend-side decode → `ArcSwap` install → a live PromQL query answered -// correctly with the plan installed. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn controller_really_pushes_backend_plan_and_query_really_serves() { - let stack = start_full_stack(19_597, 19_598).await; - let client = reqwest::Client::new(); - - // ── 1. Controller plans the workload, POSTs the legacy streaming-config - // (still required: it's what drives sid registration on ingest) - // AND a real BackendPlan built from the SAME BackendStageConfig. ── - let workload = build_workload( - "http_latency_ms", - vec![AggType::Quantile], - 0.01, - Duration::from_secs(1), - vec!["service".to_string()], - vec![0.99], - ); - let backend_cfg = plan_backend_stage_config(&workload); - let streaming_config_json = - control_plane::emit::emit_backend_streaming_config_json(&backend_cfg, &[]) - .expect("emit_backend_streaming_config_json must succeed"); - post_streaming_config(&client, stack.backend_port, &streaming_config_json).await; - - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("system time before UNIX epoch") - .as_millis() as u64; - let plan = control_plane::backend_plan::from_stage_config(&backend_cfg, &[], 1, now_ms) - .expect("backend_plan::from_stage_config must succeed"); - assert_eq!( - plan.materializations.len(), - 1, - "the DDSketch quantile workload must produce exactly one materialization" - ); - post_backend_plan(stack.backend_port, &plan).await; - - // ── 2. Confirm the backend really installed it (not just accepted the - // POST) — read it back via GET, the same handle - // `post_asap_planner.rs`'s serving-time lookup reads from. ── - let installed = get_backend_plan(&client, stack.backend_port).await; - assert_eq!(installed["status"], "success"); - assert_eq!(installed["plan_id"], 1); - assert_eq!( - installed["materialization_count"], - 1, - "GET /api/v1/backend-plan must reflect the just-installed plan, not a stale/empty one:\n{}", - serde_json::to_string_pretty(&installed).unwrap_or_default() - ); - assert_eq!( - stack.hot_reload_backend_plan.snapshot().plan_id, - 1, - "the ASAPQueryEngine-side handle (shared with the HTTP server, per main.rs's wiring) \ - must observe the same installed plan the GET endpoint just reported" - ); - - // ── 3. Ingest a real DDSketch via OTLP (same fixture as Test 3). ── - let alpha = 0.01; - let store_counts = vec![5u64, 10, 15, 20]; - let dd_state = build_dd_sketch_state(alpha, store_counts, -1); - let sketch_bytes = dd_state.encode_to_vec(); - - let now_ns = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("system time before UNIX epoch") - .as_nanos() as u64; - let sketch_t_ns = now_ns.saturating_sub(3_000_000_000); - let watermark_t_ns = now_ns.saturating_sub(1_000_000_000); - - let req = build_dd_sketch_export( - "http_latency_ms", - &[("service", "e2e-test")], - sketch_t_ns, - sketch_bytes, - alpha, - ); - post_otlp_http(&client, stack.otlp_http_port, req).await; - - let watermark_state = build_dd_sketch_state(alpha, Vec::new(), 0); - let watermark_req = build_dd_sketch_export( - "http_latency_ms", - &[("service", "e2e-test")], - watermark_t_ns, - watermark_state.encode_to_vec(), - alpha, - ); - post_otlp_http(&client, stack.otlp_http_port, watermark_req).await; - - tokio::time::sleep(Duration::from_millis(800)).await; - - // ── 4. Query via PromQL — this must actually succeed AND return a - // real, sane numeric quantile value, with the BackendPlan - // installed (not just "some code path returned success"). ── - let query_url = format!("http://127.0.0.1:{}/api/v1/query", stack.backend_port); - let response: JsonValue = client - .get(&query_url) - .query(&[("query", "quantile_over_time(0.99, http_latency_ms[10s])")]) - .send() - .await - .expect("PromQL query failed to send") - .json() - .await - .expect("PromQL response was not JSON"); - - assert_eq!( - response["status"].as_str().unwrap_or("(missing)"), - "success", - "query must succeed with a real BackendPlan installed. Response:\n{}", - serde_json::to_string_pretty(&response).unwrap_or_default() - ); - let value = response["data"]["result"] - .as_array() - .and_then(|r| extract_first_scalar(&JsonValue::Array(r.clone()))) - .expect("expected a scalar quantile result"); - assert!( - value.is_finite() && value > 0.0, - "expected a real, finite p99 value from the DDSketch fixture, got {value}" - ); -} - // ── Test 4 — full roundtrip with KLL ──────────────────────────────────────── // // Same shape as Test 3 but the workload pins KLL via From 524405642870249881e3fcfa332577de5450dfe2 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 14:56:51 -0600 Subject: [PATCH 14/30] Remove stale BackendPlan runtime terminology --- data_plane/src/drivers/query/servers/http.rs | 2 +- .../query_engines/asap_query_engine/engine.rs | 8 +++--- .../asap_query_engine/post_asap_planner.rs | 27 ++++++------------- .../asap_query_engine/post_asap_readout.rs | 2 +- .../asap_query_engine/summary_executor.rs | 6 ++--- .../src/storage_engines/sketch_db/accuracy.rs | 2 +- 6 files changed, 18 insertions(+), 29 deletions(-) diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 25cb92b0..98c1f81d 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -191,7 +191,7 @@ pub struct HttpServer { /// archive and observes the 60–90 s flush gap). probe_cache: Option>, /// Serializes multi-document physical-plan publication so two control - /// plane generations cannot interleave their config and BackendPlan. + /// plane generations cannot interleave their plan projections and catalog. physical_plan_lock: Arc>, active_physical_plan: Option, physical_plan_lifecycle: Option, diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 6725c1e0..54f67584 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -117,7 +117,7 @@ pub struct ASAPQueryEngine { archive_engine: Option>, /// Generation-consistent physical snapshot used by the production query - /// path. QueryPlan and BackendPlan must never be sampled separately. + /// path. The QueryPlan and SummaryCatalog must come from the same snapshot. active_physical_plan: Option, exact_subquery_endpoint: Option, exact_subquery_client: reqwest::Client, @@ -683,7 +683,7 @@ impl ASAPQueryEngine { crate::query_engines::EngineError::capability_miss( crate::storage_engines::types::StorageBackend::SketchStore.data_source_id(), format!( - "post-ASAP/BackendPlan resolver could not serve `{query}` over \ + "installed QueryPlan resolver could not serve `{query}` over \ [{start_ms}, {end_ms}]: {reason:?} — failing over to archive" ), ) @@ -950,7 +950,7 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu } } // One authoritative warm path: ASAPPlanner post-ASAP DAG → - // BackendPlan/materialization resolver → SID lookup → DAG executor. + // SummaryCatalog/materialization resolver → SID lookup → DAG executor. // A typed resolver/executor error becomes CapabilityMiss, which lets // EngineRouter continue to the archive backend. if let Some(idx) = self.sketch_index.as_ref() { @@ -1032,7 +1032,7 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu crate::query_engines::EngineError::capability_miss( crate::storage_engines::types::StorageBackend::SketchStore.data_source_id(), format!( - "post-ASAP/BackendPlan resolver could not serve `{query}`: \n {reason:?} — failing over to archive" + "installed QueryPlan resolver could not serve `{query}`: \n {reason:?} — failing over to archive" ), ) })?; diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_planner.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_planner.rs index b6d54df9..ce9871ca 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_planner.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_planner.rs @@ -110,9 +110,9 @@ pub enum LoweringSkip { /// outcome, just detected one step earlier so the caller can skip /// without even constructing a `QueryExecutionContext`. NotRealized, - /// ASAPPlanner produced a maintained-summary plan, but the installed - /// BackendPlan has no warm route to a materialization with the exact - /// source, family and parameters required by that post-ASAP tree. + /// ASAPPlanner produced a maintained-summary plan, but no installed + /// QueryPlan binding has a ready materialization with the required + /// source, family and parameters. NoWarmRoute(String), /// The tree lowered successfully, but `crate::query_engines::asap_query_engine::summary_exec::execute()` /// itself returned `Err` (`NoCandidates`, `MergeKindParamsMismatch`, @@ -505,17 +505,9 @@ pub fn plan_promql_to_post_asap( } let source_has_filter = query_expr_has_filter(&qe); - // Serving time must reproduce the REAL planning decision, not - // independently re-derive one -- see this module's docs. Prefer - // reading it straight off an installed `BackendPlan`'s - // materializations when one covers this metric -- - // `Materialization.family` already carries the canonical `SketchKind`, no - // `AggregationConfig` reconstruction required (design-backend-plan-wire-format.md - // §5). Otherwise fall back to the `SketchStore`-reconstruction path - // (`observed_family_for_metric`), which is `None` when this metric - // has nothing registered (or only an `ExactAgg` sid, which bypasses - // `CostModel` entirely) -- `ObservedFamilyCostModel` then falls back - // further to the accuracy-driven default. + // This dynamic lowering path is retained for isolated executor tests. + // Production serving executes the installed QueryPlan and resolves its + // MaterializationId bindings through SummaryCatalog. let metric = find_metric_in_query_expr(&qe); let mut observed = Vec::new(); if let Some(family) = metric @@ -524,11 +516,8 @@ pub fn plan_promql_to_post_asap( { observed.push(family); } - // No installed family means the planner may use its accuracy-driven - // default. With a BackendPlan, try every family materialized for the - // metric: a fingerprint does not uniquely identify query semantics, - // and choosing the first family could incorrectly fall back while a - // later materialization is an exact match. + // No observed family means the test-only planner may use its + // accuracy-driven default. let candidates: Vec<_> = if observed.is_empty() { vec![None] } else { diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index ff97bc00..9221c6e2 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -79,7 +79,7 @@ pub fn execute_post_asap_readout( /// Execute an already-bound QueryPlan entry. This is the production serving /// path: no PromQL lowering, planner cost model, observed-family lookup, or -/// BackendPlan materialization search occurs here. +/// Installed QueryPlan materialization resolution occurs before this legacy test helper. pub fn execute_query_plan_readout( index: &SketchStore, entry: &control_plane::query_plan::QueryPlanEntry, diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index c9cbc921..8e6fa0e7 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -98,8 +98,8 @@ pub struct QueryExecutionContext<'a> { /// `readout_cumulative`); `false` for a per-window matrix (one merged /// answer per window, via `readout_per_window`). pub is_cumulative: bool, - /// Materializations authorized by the active BackendPlan's warm routes. - /// `None` is the explicit legacy/no-plan mode; `Some` fails closed and + /// Materializations authorized by the installed QueryPlan bindings. + /// `None` is the explicit dynamic-test mode; `Some` fails closed and /// excludes stale or unrelated SIDs even when their metric/family match. pub allowed_materializations: Option>, } @@ -1709,7 +1709,7 @@ mod tests { } #[test] - fn backend_plan_materialization_filter_excludes_stale_sid() { + fn query_plan_materialization_filter_excludes_stale_sid() { let idx = SketchStore::new(); let mut stale = kll_meta(1, "latency_ms", &[]); stale.policy_fp = asap_types::PolicyFingerprint(10); diff --git a/data_plane/src/storage_engines/sketch_db/accuracy.rs b/data_plane/src/storage_engines/sketch_db/accuracy.rs index 10e0f3e9..951d763b 100644 --- a/data_plane/src/storage_engines/sketch_db/accuracy.rs +++ b/data_plane/src/storage_engines/sketch_db/accuracy.rs @@ -1,7 +1,7 @@ //! `AccuracyProfile` — derived error / confidence bound for each //! `AggregationConfig`. //! -//! Implements backend accuracy metadata consumed under BackendPlan. Logical +//! Implements backend accuracy metadata consumed through SummaryCatalog and QueryPlan. Logical //! guarantees are owned by ASAPPlanner and family bounds by summary libraries. //! Given the `aggregation_type` + `parameters` pinned on an //! `AggSchema`, the registry can expose the theoretical accuracy From 5dfac847988cae430672545649ed9a04af533f9c Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 15:15:20 -0600 Subject: [PATCH 15/30] Execute PromQL topk as a value selection node --- control_plane/src/query_plan/logical.rs | 219 ++++++++++++++++-- .../asap_query_engine/logical_dag.rs | 198 ++++++++++++++++ 2 files changed, 404 insertions(+), 13 deletions(-) diff --git a/control_plane/src/query_plan/logical.rs b/control_plane/src/query_plan/logical.rs index 76c0bc2b..2018e994 100644 --- a/control_plane/src/query_plan/logical.rs +++ b/control_plane/src/query_plan/logical.rs @@ -23,10 +23,18 @@ pub enum LogicalOperator { offset_ms: i64, }, UnaryNegate, + VectorToScalar, Aggregate { operation: Aggregation, grouping: Grouping, }, + /// PromQL `topk(k, vector)` selection over values produced by the child. + /// This is distinct from a frequency-sketch TopK readout: any exact or + /// summary-backed instant-vector child may feed this query-time operator. + TopKSelection { + k: u64, + grouping: Grouping, + }, Binary { operation: BinaryOperation, return_bool: bool, @@ -244,9 +252,13 @@ impl Lower { self.operation( LogicalOperator::Subquery { range_ms: millis(s.range)?, + // Prometheus uses its configured default evaluation + // interval when `[range:]` omits the resolution. The + // backend-local deployment uses the Prometheus default + // of one minute; unsupported subquery operands are + // externalized as one exact subtree before execution. step_ms: millis( - s.step - .ok_or_else(|| invalid("explicit subquery step required"))?, + s.step.unwrap_or_else(|| std::time::Duration::from_secs(60)), )?, offset_ms: offset(&s.offset)?, }, @@ -254,17 +266,6 @@ impl Lower { ) } Expr::Aggregate(a) => { - if a.param.is_some() { - return Err(invalid("parameterized aggregate unsupported")); - } - let operation = match a.op.to_string().as_str() { - "sum" => Aggregation::Sum, - "max" => Aggregation::Max, - "min" => Aggregation::Min, - "avg" => Aggregation::Avg, - "count" => Aggregation::Count, - other => return Err(invalid(format!("unsupported logical aggregate {other}"))), - }; let grouping = match &a.modifier { None => Grouping { labels: vec![], @@ -279,6 +280,54 @@ impl Lower { without: true, }, }; + if a.op.to_string() == "topk" { + let Some(Expr::NumberLiteral(parameter)) = a.param.as_deref() else { + return Err(invalid("topk requires a literal scalar parameter")); + }; + if !parameter.val.is_finite() { + return Err(invalid("topk requires a finite scalar parameter")); + } + // Prometheus converts the scalar parameter to int64 before + // selection. Values below one produce an empty vector. + let k = parameter.val as i64; + // Keep the selection node local even when its operand has + // unsupported syntax (for example a subquery with an + // implicit resolution). Prometheus evaluates that maximal + // instant-vector child; the backend still performs topk. + let nodes_before = self.nodes.clone(); + let seen_before = self.seen.clone(); + let input = match self.lower(&a.expr) { + Ok(input) => input, + Err(_) => { + self.nodes = nodes_before; + self.seen = seen_before; + self.operation( + LogicalOperator::ExactSubquery { + query: a.expr.to_string(), + }, + vec![], + )? + } + }; + return self.operation( + LogicalOperator::TopKSelection { + k: u64::try_from(k).unwrap_or(0), + grouping, + }, + vec![input], + ); + } + if a.param.is_some() { + return Err(invalid("unsupported parameterized aggregate")); + } + let operation = match a.op.to_string().as_str() { + "sum" => Aggregation::Sum, + "max" => Aggregation::Max, + "min" => Aggregation::Min, + "avg" => Aggregation::Avg, + "count" => Aggregation::Count, + other => return Err(invalid(format!("unsupported logical aggregate {other}"))), + }; let input = self.lower(&a.expr)?; self.operation( LogicalOperator::Aggregate { @@ -290,6 +339,7 @@ impl Lower { } Expr::Call(c) => { let operator = match c.func.name { + "scalar" => LogicalOperator::VectorToScalar, "histogram_quantile" => LogicalOperator::HistogramQuantile, "sort" => LogicalOperator::Sort { descending: false }, "sort_desc" => LogicalOperator::Sort { descending: true }, @@ -552,6 +602,92 @@ mod tests { .validate(1) .is_err()); } + + #[test] + fn real_topk_queries_lower_to_value_selection() { + for (query, k) in [ + ( + "topk(2, sum by (job) (rate(backend_process_cpu_seconds_total[1h])))", + 2, + ), + ( + "topk(2, sum by (job) (backend_process_resident_memory_bytes))", + 2, + ), + ("topk(2, max_over_time(backend_retry_backlog_depth[6h]))", 2), + ( + "topk(1, sum by (job) (increase(backend_http_5xx_total[6h])) / sum by (job) (increase(backend_http_requests_total[6h])))", + 1, + ), + ( + "topk(3, avg_over_time((sum by (job) (backend_process_resident_memory_bytes))[6h:]))", + 3, + ), + ] { + let entry = QueryPlanEntry::compile_logical( + "topk".into(), + query.into(), + instant(), + FallbackPolicy::Reject, + ) + .unwrap_or_else(|error| panic!("{query}: {error}")); + assert!(matches!( + entry.nodes[&entry.root], + QueryPlanNode::Logical { + operator: LogicalOperator::TopKSelection { k: actual, .. }, + .. + } if actual == k + )); + } + } + + #[test] + fn topk_keeps_unsupported_child_as_exact_leaf() { + let entry = QueryPlanEntry::compile_logical( + "topk-subquery".into(), + "topk(3, label_replace(memory_bytes, \"dst\", \"$1\", \"src\", \"(.*)\"))".into(), + instant(), + FallbackPolicy::Reject, + ) + .unwrap(); + assert!(matches!( + entry.nodes[&entry.root], + QueryPlanNode::Logical { + operator: LogicalOperator::TopKSelection { k: 3, .. }, + .. + } + )); + assert!(entry.nodes.values().any(|node| matches!( + node, + QueryPlanNode::Logical { + operator: LogicalOperator::ExactSubquery { .. }, + .. + } + ))); + } + + #[test] + fn topk_preserves_by_and_without_partitioning() { + for (query, labels, without) in [ + ("topk by (cluster) (2, m)", vec!["cluster"], false), + ("topk without (pod) (2, m)", vec!["pod"], true), + ] { + let entry = QueryPlanEntry::compile_logical( + "topk-group".into(), + query.into(), + instant(), + FallbackPolicy::Reject, + ) + .unwrap(); + assert!(matches!( + &entry.nodes[&entry.root], + QueryPlanNode::Logical { + operator: LogicalOperator::TopKSelection { grouping, .. }, + .. + } if grouping.labels == labels && grouping.without == without + )); + } + } } /// Prove a physical-native substitute represents exactly the selected summary leaf. @@ -749,6 +885,48 @@ mod planner_workload_tests { } } + fn compile_one(query: &str) -> crate::physical::compiler::PhysicalPlan { + let mut fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../../docs/examples/asapquery-planning-snapshot.json" + )) + .unwrap(); + let mut entry = fixture["query_workload"]["repeating_queries"][0].clone(); + entry["query"] = query.into(); + entry["requirements"]["accuracy"] = serde_json::json!({"explicit":"Exact"}); + let window = lookback(&parser::parse(query).unwrap()); + entry["time_selection"]["lookback"] = (if window == 0 { 300_000 } else { window }).into(); + fixture["query_workload"]["repeating_queries"] = vec![entry].into(); + let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(fixture).unwrap(); + let (request, environment) = snapshot.planning_request().unwrap(); + PhysicalCompiler + .compile(request, environment) + .unwrap_or_else(|error| panic!("{query}: {error}")) + } + + #[test] + fn evaluation_topk_queries_retain_a_local_selection_root() { + for query in [ + "topk(2, sum by (job) (rate(backend_process_cpu_seconds_total[1h])))", + "topk(2, sum by (job) (backend_process_resident_memory_bytes))", + "topk(1, sum by (job) (increase(backend_http_5xx_total[6h])) / sum by (job) (increase(backend_http_requests_total[6h])))", + "topk(2, max_over_time(backend_retry_backlog_depth[6h]))", + "topk(1, sum by (job) (increase(backend_http_5xx_total[24h])) / scalar(sum(increase(backend_http_5xx_total[24h]))))", + "topk(1, sum by (job) (rate(backend_process_cpu_seconds_total[6h])))", + "topk(3, avg_over_time((sum by (job) (backend_process_resident_memory_bytes))[6h:]))", + ] { + let plan = compile_one(query); + let entry = plan.query_plan.entries.values().next().unwrap(); + assert!(matches!( + entry.nodes[&entry.root], + QueryPlanNode::Logical { + operator: LogicalOperator::TopKSelection { .. }, + .. + } + ), "{query}: {:?}", entry.nodes[&entry.root]); + assert!(!entry.nodes.values().any(|node| matches!(node, QueryPlanNode::ExactFallback { .. })), "{query}"); + } + } + #[test] fn whole_o11y_planner_candidate_lowers_every_original_query() { // AST support alone is insufficient: the actual selected Planner forest must bind too. @@ -1178,6 +1356,12 @@ pub fn externalize_residuals(entry: &mut QueryPlanEntry) -> Result<(), QueryPlan let mut indexed = !matches!( node, QueryPlanNode::Logical { .. } | QueryPlanNode::Scalar { .. } + ) || matches!( + node, + QueryPlanNode::Logical { + operator: LogicalOperator::TopKSelection { .. }, + .. + } ); let mut exact = false; if let QueryPlanNode::Logical { operator, .. } = node { @@ -1195,6 +1379,15 @@ pub fn externalize_residuals(entry: &mut QueryPlanEntry) -> Result<(), QueryPlan } let mut pending = vec![entry.root]; while let Some(id) = pending.pop() { + if matches!( + entry.nodes.get(&id), + Some(QueryPlanNode::Logical { + operator: LogicalOperator::ExactSubquery { .. }, + .. + }) + ) { + continue; + } let (indexed, exact) = flags(id, &entry.nodes); if !indexed && exact { if let Ok(shape) = expression_shape(id, &entry.nodes) { diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs index ec1f8728..9ec0c3b2 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs @@ -231,6 +231,14 @@ impl Result> Evaluator<' )), _ => Err(miss("cannot negate range vector")), }, + LogicalOperator::VectorToScalar => { + let values = vector(self.eval(input(0)?, at)?)?; + Ok(Value::Scalar(if values.len() == 1 { + values[0].1 + } else { + f64::NAN + })) + } LogicalOperator::Aggregate { operation, grouping, @@ -238,6 +246,10 @@ impl Result> Evaluator<' let values = vector(self.eval(input(0)?, at)?)?; Ok(Value::Vector(aggregate(operation, &grouping, values))) } + LogicalOperator::TopKSelection { k, grouping } => { + let values = vector(self.eval(input(0)?, at)?)?; + Ok(Value::Vector(topk_selection(k, &grouping, values))) + } LogicalOperator::Binary { operation, return_bool, @@ -398,6 +410,50 @@ fn aggregate(operation: Aggregation, grouping: &Grouping, values: Vector) -> Vec .collect() } +fn grouping_key(labels: &Labels, grouping: &Grouping) -> Labels { + labels + .iter() + .filter(|(key, _)| { + if grouping.without { + key.as_str() != "__name__" && !grouping.labels.contains(key) + } else { + grouping.labels.contains(key) + } + }) + .map(|(key, value)| (key.clone(), value.clone())) + .collect() +} + +/// Select by the child sample value while retaining every selected series' +/// labels. NaN ranks below every numeric value, matching Prometheus' TOPK heap. +/// Stable sorting also leaves equal-valued series in the child's order. +fn topk_selection(k: u64, grouping: &Grouping, values: Vector) -> Vector { + if k == 0 { + return Vec::new(); + } + let mut groups: BTreeMap = BTreeMap::new(); + for (labels, value) in values { + groups + .entry(grouping_key(&labels, grouping)) + .or_default() + .push((labels, value)); + } + let limit = usize::try_from(k).unwrap_or(usize::MAX); + groups + .into_values() + .flat_map(|mut group| { + group.sort_by(|a, b| match (a.1.is_nan(), b.1.is_nan()) { + (true, true) => std::cmp::Ordering::Equal, + (true, false) => std::cmp::Ordering::Greater, + (false, true) => std::cmp::Ordering::Less, + (false, false) => b.1.total_cmp(&a.1), + }); + group.truncate(limit); + group + }) + .collect() +} + fn binary( operation: BinaryOperation, boolean: bool, @@ -589,3 +645,145 @@ fn bucket_quantile(q: f64, mut b: Vec<(f64, f64)>) -> f64 { let (end, upper) = buckets[idx]; start + (end - start) * (rank - base) / (upper - base) } + +#[cfg(test)] +mod topk_tests { + use super::*; + use control_plane::query_plan::{FallbackPolicy, InstantExecution}; + + fn labels(items: &[(&str, &str)]) -> Labels { + items + .iter() + .map(|(key, value)| ((*key).into(), (*value).into())) + .collect() + } + + #[test] + fn topk_selects_by_sample_value_and_preserves_series_labels() { + let values = vec![ + ( + labels(&[("__name__", "cpu"), ("job", "api"), ("pod", "a")]), + 4.0, + ), + ( + labels(&[("__name__", "cpu"), ("job", "api"), ("pod", "b")]), + 9.0, + ), + ( + labels(&[("__name__", "cpu"), ("job", "db"), ("pod", "c")]), + 7.0, + ), + ( + labels(&[("__name__", "cpu"), ("job", "db"), ("pod", "d")]), + 2.0, + ), + ]; + let selected = topk_selection( + 1, + &Grouping { + labels: vec!["job".into()], + without: false, + }, + values, + ); + assert_eq!(selected.len(), 2); + assert_eq!(selected[0].0["pod"], "b"); + assert_eq!(selected[0].1, 9.0); + assert_eq!(selected[1].0["pod"], "c"); + assert_eq!(selected[1].1, 7.0); + assert!(selected + .iter() + .all(|(labels, _)| labels.contains_key("__name__"))); + } + + #[test] + fn topk_ranks_nan_below_numbers_and_keeps_exact_child_values() { + let selected = topk_selection( + 2, + &Grouping { + labels: vec![], + without: false, + }, + vec![ + (labels(&[("series", "nan")]), f64::NAN), + (labels(&[("series", "low")]), -1.0), + (labels(&[("series", "high")]), 3.0), + ], + ); + assert_eq!( + selected + .iter() + .map(|row| row.0["series"].as_str()) + .collect::>(), + vec!["high", "low"] + ); + assert_eq!( + selected.iter().map(|row| row.1).collect::>(), + vec![3.0, -1.0] + ); + } + + #[test] + fn installed_topk_combines_with_prometheus_exact_child() { + let mut entry = QueryPlanEntry::compile_logical( + "hybrid-topk".into(), + "topk(2, m)".into(), + InstantExecution { + lookback_ms: 300_000, + full_history: false, + cumulative_readout: false, + }, + FallbackPolicy::ExactBackend, + ) + .unwrap(); + control_plane::query_plan::logical::finalize_residuals(&mut entry).unwrap(); + let leaf = entry + .nodes + .iter() + .find_map(|(id, node)| { + matches!( + node, + QueryPlanNode::Logical { + operator: LogicalOperator::ExactSubquery { .. }, + .. + } + ) + .then_some(*id) + }) + .unwrap(); + let at = 1_000_u64; + let leaves = [( + (leaf, at as i64), + PreparedLeaf { + value: Value::Vector(vec![ + (labels(&[("pod", "a")]), 1.0), + (labels(&[("pod", "b")]), 8.0), + (labels(&[("pod", "c")]), 5.0), + ]), + remote: true, + remote_evaluations: 1, + remote_rpcs: 1, + }, + )] + .into_iter() + .collect(); + let (result, stats) = execute_installed(&entry, &leaves, at, |_, _| { + panic!("summary callback must not run for an exact-child topk") + }) + .unwrap(); + let QueryResult::Vector(result) = result else { + panic!("instant vector expected") + }; + assert_eq!( + result + .values + .iter() + .map(|point| point.value) + .collect::>(), + vec![8.0, 5.0] + ); + assert_eq!(stats.remote_branch_evaluations, 1); + assert_eq!(stats.remote_rpcs, 1); + assert_eq!(stats.raw_scan_evaluations, 0); + } +} From c678c5f79a47756c426ff4c76f852a4d0a14e650 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 15:25:06 -0600 Subject: [PATCH 16/30] Remove duplicate catalog plan projections --- control_plane/src/physical/compiler.rs | 80 ++++++-------- .../src/physical/precompute_contract.rs | 103 +----------------- control_plane/src/physical/publication.rs | 14 ++- control_plane/src/query_plan.rs | 21 +--- control_plane/src/query_plan/logical.rs | 2 - .../drivers/ingest/prometheus_remote_write.rs | 1 - data_plane/src/drivers/query/servers/http.rs | 1 - data_plane/src/main.rs | 1 - .../asap_query_engine/exact_subqueries.rs | 2 - .../asap_query_engine/live_serve.rs | 2 - .../asap_query_engine/post_asap_readout.rs | 6 - .../asap_query_engine/summary_executor.rs | 7 +- .../types/hot_reload_config.rs | 8 +- 13 files changed, 51 insertions(+), 197 deletions(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index b4c5cfd6..e1678706 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -275,11 +275,6 @@ pub struct CollectorPlan { pub struct PrecomputePlan { #[serde(default, skip_serializing_if = "Option::is_none")] pub summary_catalog: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub materialization_contracts: BTreeMap< - asap_types::sds::MaterializationId, - super::precompute_contract::PrecomputeMaterializationContract, - >, pub envelope: PlanEnvelope, pub ingest: IngestContract, pub schemas: Vec, @@ -481,7 +476,6 @@ impl PrecomputePlan { .collect(); let plan = Self { summary_catalog: None, - materialization_contracts: BTreeMap::new(), envelope, ingest: IngestContract { protocol: IngestProtocol::ModifiedOtlpMetricsV1, @@ -620,7 +614,7 @@ impl PrecomputePlan { return Err(PrecomputePlanError::MissingProducer(missing.0)); } } - self.validate_catalog_contract() + Ok(()) } pub fn validate_against_backend( @@ -1096,18 +1090,8 @@ impl TransmissionPlan { } }) .collect(); - let catalog = super::summary_catalog::SummaryCatalog::from_materializations( - envelope.plan_id, - envelope.plan_version, - &precompute.materializations, - ) - .map_err(|error| TransmissionPlanError::Catalog(error.to_string()))?; let plan = Self { - summary_catalog: Some( - catalog - .reference() - .map_err(|error| TransmissionPlanError::Catalog(error.to_string()))?, - ), + summary_catalog: precompute.summary_catalog.clone(), envelope, frame_identity: FrameIdentityContract { identity_version: 1, @@ -1122,14 +1106,10 @@ impl TransmissionPlan { } pub fn validate(&self, precompute: &PrecomputePlan) -> Result<(), TransmissionPlanError> { - if self.summary_catalog.is_some() { - let catalog = super::summary_catalog::SummaryCatalog::from_materializations( - precompute.envelope.plan_id, - precompute.envelope.plan_version, - &precompute.materializations, - ) - .map_err(|error| TransmissionPlanError::Catalog(error.to_string()))?; - self.validate_against_catalog(&catalog)?; + if self.summary_catalog != precompute.summary_catalog { + return Err(TransmissionPlanError::Catalog( + "transmission and precompute plans reference different catalog snapshots".into(), + )); } if self.envelope != precompute.envelope { return Err(TransmissionPlanError::EnvelopeMismatch); @@ -2375,13 +2355,13 @@ impl PhysicalCompiler { query_id: "precompute-plan".into(), reason: error.to_string(), })?; - let transmission_plan = + let mut transmission_plan = TransmissionPlan::build(envelope.clone(), &precompute_plan, &runtime_policies) .map_err(|error| CompileError::Query { query_id: "transmission-plan".into(), reason: error.to_string(), })?; - let collector_plans = producer_ids + let mut collector_plans = producer_ids .into_iter() .map(|collector_id| CollectorPlan { summary_catalog: transmission_plan.summary_catalog.clone(), @@ -2402,7 +2382,7 @@ impl PhysicalCompiler { for query in &request.queries { let canonical = canonical_promql(&query.query_string)?; let binding = |node: &SummaryNode, node_family: &SummaryFamilyType| -> Result { - let planned_metric = summary_agg_metric(node).ok_or_else(|| { + summary_agg_metric(node).ok_or_else(|| { crate::query_plan::QueryPlanError::Invalid( "materialized node has no unique time-series source".into(), ) @@ -2450,8 +2430,6 @@ impl PhysicalCompiler { Ok(MaterializationBinding { readout_lookback_ms: source_window.map(|seconds| seconds.saturating_mul(1_000)), materialization: fingerprint.into(), - metric: planned_metric, - sid_grouping: materialization.group_by.clone(), output_grouping: PhysicalGrouping::Reduce(materialization.group_by.clone()), window_ms: pane_ms, }) @@ -2532,6 +2510,10 @@ impl PhysicalCompiler { query_id: "precompute-catalog".into(), reason: error.to_string(), })?; + transmission_plan.summary_catalog = precompute_plan.summary_catalog.clone(); + for collector in &mut collector_plans { + collector.summary_catalog = precompute_plan.summary_catalog.clone(); + } transmission_plan .validate_against_catalog(&summary_catalog) .map_err(|error| CompileError::Query { @@ -4063,8 +4045,10 @@ mod tests { let actual = bindings .iter() .map(|binding| { + let identity = &plan.summary_catalog.materializations[&binding.materialization]; + let data = &plan.summary_catalog.data_descriptors[&identity.data_descriptor_id]; ( - binding.metric.as_str(), + data.metric_name.as_str(), binding.window_ms, binding.readout_lookback_ms, ) @@ -4094,7 +4078,12 @@ mod tests { let query = plan.query_plan.entries.values().next().unwrap(); let bindings = query.materialization_bindings(); assert_eq!(bindings.len(), 1); - assert_eq!((&*bindings[0].metric, bindings[0].window_ms), ("a", 60_000)); + let identity = &plan.summary_catalog.materializations[&bindings[0].materialization]; + let data = &plan.summary_catalog.data_descriptors[&identity.data_descriptor_id]; + assert_eq!( + (data.metric_name.as_str(), bindings[0].window_ms), + ("a", 60_000) + ); assert!(!query .nodes .values() @@ -4273,18 +4262,7 @@ mod tests { bad.schemas[0].window.size_ms += 1; reject(bad); let mut bad = original.clone(); - bad.materialization_contracts - .values_mut() - .next() - .unwrap() - .retained_windows = Some(0); - reject(bad); - let mut bad = original.clone(); - bad.materialization_contracts - .values_mut() - .next() - .unwrap() - .update = super::super::precompute_contract::PrecomputeUpdate::RawSamples; + bad.materializations[0].num_aggregates_to_retain = Some(0); reject(bad); let mut bad = original.clone(); bad.summary_catalog @@ -4298,7 +4276,6 @@ mod tests { assert!(original.validate_against_catalog(&corrupt_catalog).is_err()); let mut legacy = original.clone(); legacy.summary_catalog = None; - legacy.materialization_contracts.clear(); legacy.validate().unwrap(); assert!(legacy.validate_against_catalog(catalog).is_err()); } @@ -4812,9 +4789,14 @@ mod tests { .nodes .values() .filter_map(|node| match node { - crate::query_plan::QueryPlanNode::ReadMaterialization { binding } => { - Some(binding.metric.as_str()) - } + crate::query_plan::QueryPlanNode::ReadMaterialization { binding } => Some( + bundle.summary_catalog.data_descriptors[&bundle + .summary_catalog + .materializations[&binding.materialization] + .data_descriptor_id] + .metric_name + .as_str(), + ), _ => None, }) .collect::>(); diff --git a/control_plane/src/physical/precompute_contract.rs b/control_plane/src/physical/precompute_contract.rs index ac82589a..4503f8bf 100644 --- a/control_plane/src/physical/precompute_contract.rs +++ b/control_plane/src/physical/precompute_contract.rs @@ -1,32 +1,9 @@ -//! Self-contained precompute execution contracts over a SummaryCatalog snapshot. -//! BackendPlan remains a compatibility projection, not a validation authority. +//! Catalog consistency checks for the precompute execution plan. use super::compiler::*; use super::summary_catalog::SummaryCatalog; use asap_types::sds::{MaterializationId, SummaryDescriptor}; use planner_types::pre_asap::{ColumnRef, Source}; -use serde::{Deserialize, Serialize}; -use std::collections::{BTreeMap, BTreeSet}; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum PrecomputeUpdate { - RawSamples, - SummaryFrames, -} -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum PrecomputePlacement { - BackendLocal, - Collectors { producer_ids: BTreeSet }, -} -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct PrecomputeMaterializationContract { - pub schema_id: String, - pub update: PrecomputeUpdate, - pub placement: PrecomputePlacement, - pub retained_windows: Option, -} +use std::collections::BTreeSet; fn invalid(reason: impl Into) -> PrecomputePlanError { PrecomputePlanError::CatalogContract(reason.into()) } @@ -35,55 +12,19 @@ impl PrecomputePlan { /// Bind references after retention decisions are finalized. The plan keeps /// only the immutable snapshot reference; descriptors are installed once. pub fn bind_catalog(&mut self, catalog: &SummaryCatalog) -> Result<(), PrecomputePlanError> { - let mut contracts = BTreeMap::new(); for config in &self.materializations { let id = MaterializationId::from(config.policy_fingerprint()); catalog.materializations.get(&id).ok_or_else(|| { invalid(format!("missing catalog materialization {}", id.as_u64())) })?; - let schema = self - .schemas - .iter() - .find(|s| s.materialization == id.fingerprint()) - .ok_or(PrecomputePlanError::SchemaSetMismatch)?; - let (update, placement) = self.expected_placement(id); - contracts.insert( - id, - PrecomputeMaterializationContract { - schema_id: schema.schema_id.clone(), - update, - placement, - retained_windows: config.num_aggregates_to_retain, - }, - ); } self.summary_catalog = Some( catalog .reference() .map_err(|error| invalid(error.to_string()))?, ); - self.materialization_contracts = contracts; self.validate_against_catalog(catalog) } - fn expected_placement(&self, id: MaterializationId) -> (PrecomputeUpdate, PrecomputePlacement) { - match self.ingest.protocol { - IngestProtocol::PrometheusRemoteWriteV1 => ( - PrecomputeUpdate::RawSamples, - PrecomputePlacement::BackendLocal, - ), - IngestProtocol::ModifiedOtlpMetricsV1 => ( - PrecomputeUpdate::SummaryFrames, - PrecomputePlacement::Collectors { - producer_ids: self - .producers - .iter() - .filter(|p| p.materialization == id.fingerprint()) - .map(|p| p.producer_id.clone()) - .collect(), - }, - ), - } - } /// Strict validation resolves this plan's reference against the one catalog /// snapshot carried by the enclosing physical-plan installation. pub fn validate_against_catalog( @@ -93,20 +34,6 @@ impl PrecomputePlan { self.validate()?; self.validate_catalog_contents(catalog) } - pub(super) fn validate_catalog_contract(&self) -> Result<(), PrecomputePlanError> { - match ( - &self.summary_catalog, - self.materialization_contracts.is_empty(), - ) { - (None, true) | (Some(_), false) => Ok(()), - (None, false) => Err(invalid("descriptor references without catalog")), - (Some(_), true) if self.materializations.is_empty() => Ok(()), - (Some(_), true) => Err(invalid( - "catalog reference without materialization contracts", - )), - } - } - fn validate_catalog_contents( &self, catalog: &SummaryCatalog, @@ -121,28 +48,17 @@ impl PrecomputePlan { { return Err(PrecomputePlanError::PlanIdentityMismatch); } - if self.envelope.activation_unix_ms < self.envelope.generated_at_unix_ms - || self - .envelope - .expiry_unix_ms - .is_some_and(|expiry| expiry <= self.envelope.activation_unix_ms) - { - return Err(invalid("invalid plan activation/expiry lifecycle")); - } let ids: BTreeSet<_> = self .materializations .iter() .map(|m| MaterializationId::from(m.policy_fingerprint())) .collect(); - if ids != catalog.materializations.keys().copied().collect() - || ids != self.materialization_contracts.keys().copied().collect() - { + if ids != catalog.materializations.keys().copied().collect() { return Err(invalid("catalog/reference/materialization sets differ")); } for config in &self.materializations { let id = MaterializationId::from(config.policy_fingerprint()); let binding = &catalog.materializations[&id]; - let contract = &self.materialization_contracts[&id]; let expected = SummaryDescriptor::from_config(config).map_err(|e| invalid(e.to_string()))?; if binding.summary_descriptor_id != expected.id { @@ -203,8 +119,7 @@ impl PrecomputePlan { return Err(invalid("session lifecycle is not supported")) } }; - if schema.schema_id != contract.schema_id - || schema.schema_id != state_schema_id(id.fingerprint()) + if schema.schema_id != state_schema_id(id.fingerprint()) || schema.family != expected_family || schema.source != source || schema.value_column != col @@ -220,15 +135,7 @@ impl PrecomputePlan { if schema.encodings != state_encodings(&family) { return Err(invalid("encoding does not match state family")); } - let (update, placement) = self.expected_placement(id); - if contract.update != update || contract.placement != placement { - return Err(invalid( - "update/placement does not match ingress and producer bindings", - )); - } - if contract.retained_windows != config.num_aggregates_to_retain - || contract.retained_windows == Some(0) - { + if config.num_aggregates_to_retain == Some(0) { return Err(invalid("materialization lifecycle/retention mismatch")); } } diff --git a/control_plane/src/physical/publication.rs b/control_plane/src/physical/publication.rs index b19825c2..9740cde9 100644 --- a/control_plane/src/physical/publication.rs +++ b/control_plane/src/physical/publication.rs @@ -30,13 +30,17 @@ impl PhysicalPlanPublication { self.query_plan .validate_against_catalog(catalog) .map_err(|e| e.to_string())?; + let materializations = self + .precompute_plan + .materializations + .iter() + .map(|config| (config.policy_fingerprint(), config)) + .collect::>(); for entry in self.query_plan.entries.values() { for binding in entry.materialization_bindings() { - let config = self - .precompute_plan - .materializations - .iter() - .find(|m| m.policy_fingerprint() == binding.materialization.fingerprint()) + let config = materializations + .get(&binding.materialization.fingerprint()) + .copied() .ok_or("query binding has no precompute materialization")?; if config.slide_interval.checked_mul(1000) != Some(binding.window_ms) { return Err("query pane differs from precompute emission interval".into()); diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index 1ccb032d..cab06fc4 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -71,13 +71,7 @@ impl QueryPlan { "query binding references absent catalog materialization".into(), ) })?; - let data = &catalog.data_descriptors[&identity.data_descriptor_id]; - let grouping: BTreeSet<_> = binding.sid_grouping.iter().cloned().collect(); - if binding.metric != data.metric_name || grouping != data.group_by_keys { - return Err(QueryPlanError::Invalid( - "query binding source/grouping differs from catalog data descriptor".into(), - )); - } + let _data = &catalog.data_descriptors[&identity.data_descriptor_id]; if binding.window_ms == 0 { return Err(QueryPlanError::Invalid( "zero physical pane duration".into(), @@ -342,9 +336,6 @@ pub enum FallbackPolicy { #[serde(deny_unknown_fields)] pub struct MaterializationBinding { pub materialization: MaterializationId, - pub metric: String, - /// Exact label-key layout of the stored materialization. - pub sid_grouping: Vec, /// Query operator grouping applied while folding those SIDs. pub output_grouping: PhysicalGrouping, pub window_ms: u64, @@ -951,8 +942,6 @@ mod catalog_binding_tests { QueryPlanNode::ReadMaterialization { binding: MaterializationBinding { materialization: config.policy_fingerprint().into(), - metric: "m".into(), - sid_grouping: vec!["job".into()], output_grouping: PhysicalGrouping::PerEntity, window_ms: 10_000, readout_lookback_ms: Some(60_000), @@ -1002,17 +991,11 @@ mod catalog_binding_tests { assert_eq!(binding(&mut decoded).readout_lookback_ms, Some(60_000)); } - // A valid fingerprint alone cannot attest a different source or grouping. + // The catalog owns source and grouping; the binding owns only its stable ID. #[test] fn catalog_binding_rejects_source_grouping_and_identity_drift() { let (plan, catalog) = fixture(); let mut broken = plan.clone(); - binding(&mut broken).metric = "other".into(); - assert!(broken.validate_against_catalog(&catalog).is_err()); - let mut broken = plan.clone(); - binding(&mut broken).sid_grouping.clear(); - assert!(broken.validate_against_catalog(&catalog).is_err()); - let mut broken = plan.clone(); binding(&mut broken).materialization = PolicyFingerprint(123).into(); assert!(broken.validate_against_catalog(&catalog).is_err()); let mut broken = plan.clone(); diff --git a/control_plane/src/query_plan/logical.rs b/control_plane/src/query_plan/logical.rs index 2018e994..f83d5bf5 100644 --- a/control_plane/src/query_plan/logical.rs +++ b/control_plane/src/query_plan/logical.rs @@ -809,8 +809,6 @@ mod hybrid_tests { if spatial_filter.is_empty() { 7 } else { 8 }, ) .into(), - metric: "m".into(), - sid_grouping: vec![], output_grouping: PhysicalGrouping::PerEntity, window_ms: 300_000, readout_lookback_ms: Some(300_000), diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index e6184d62..9a3e4f9e 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -777,7 +777,6 @@ mod tests { summary_catalog: None, precompute_plan: PrecomputePlan { summary_catalog: None, - materialization_contracts: Default::default(), envelope: envelope.clone(), ingest: IngestContract { protocol: IngestProtocol::PrometheusRemoteWriteV1, diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 98c1f81d..a1e9820e 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -2487,7 +2487,6 @@ mod tests { summary_catalog: None, precompute_plan: PrecomputePlan { summary_catalog: None, - materialization_contracts: Default::default(), envelope: envelope.clone(), ingest: IngestContract { protocol: IngestProtocol::PrometheusRemoteWriteV1, diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 1d1848fa..87d57839 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -665,7 +665,6 @@ async fn main() -> Result<()> { // Bootstrap projections share one immutable physical-plan envelope. let initial_precompute_plan = control_plane::physical::compiler::PrecomputePlan { summary_catalog: None, - materialization_contracts: Default::default(), envelope: control_plane::physical::compiler::PlanEnvelope { plan_id: 0, plan_version: 0, diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index ba60cf9b..0ab1d34a 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -414,8 +414,6 @@ mod tests { QueryPlanNode::ReadMaterialization { binding: MaterializationBinding { materialization: MATERIALIZATION.into(), - metric: "http_requests_total".into(), - sid_grouping: vec!["job".into()], output_grouping: PhysicalGrouping::Reduce(vec!["job".into()]), window_ms: AT, readout_lookback_ms: Some(AT), diff --git a/data_plane/src/query_engines/asap_query_engine/live_serve.rs b/data_plane/src/query_engines/asap_query_engine/live_serve.rs index 6aebcd77..41032d3b 100644 --- a/data_plane/src/query_engines/asap_query_engine/live_serve.rs +++ b/data_plane/src/query_engines/asap_query_engine/live_serve.rs @@ -464,8 +464,6 @@ mod tests { control_plane::query_plan::QueryPlanNode::ReadMaterialization { binding: control_plane::query_plan::MaterializationBinding { materialization: policy.into(), - metric: "bytes".into(), - sid_grouping: vec![], output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, window_ms: 1_000, readout_lookback_ms: Some(1_000), diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 9221c6e2..45d51bc2 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -774,8 +774,6 @@ mod tests { |_node, _family| { Ok(control_plane::query_plan::MaterializationBinding { materialization: asap_types::PolicyFingerprint(123).into(), - metric: "unique_users".into(), - sid_grouping: vec!["service".into()], output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, window_ms: 60_000, readout_lookback_ms: Some(60_000), @@ -928,8 +926,6 @@ mod tests { QueryPlanNode::ReadMaterialization { binding: control_plane::query_plan::MaterializationBinding { materialization: policy.into(), - metric: "requests_total".into(), - sid_grouping: vec![], output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, window_ms: 10_000, readout_lookback_ms: Some(60_000), @@ -1022,8 +1018,6 @@ mod tests { QueryPlanNode::ReadMaterialization { binding: control_plane::query_plan::MaterializationBinding { materialization: policy.into(), - metric: "requests_total".into(), - sid_grouping: vec![], output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, window_ms: 60_000, readout_lookback_ms: Some(60_000), diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index 8e6fa0e7..033b9d66 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -453,7 +453,6 @@ impl QueryExecutionContext<'_> { } Ok(()) }; - let required_keys: BTreeSet<_> = binding.sid_grouping.iter().cloned().collect(); let mut sids = self .index .sids_for_policy(binding.materialization.fingerprint()); @@ -466,10 +465,7 @@ impl QueryExecutionContext<'_> { let candidate = self .index .with_instance(sid, |meta| { - if meta.policy_fp != binding.materialization.fingerprint() - || meta.metric_name != binding.metric - || meta.group_by_keys != required_keys - { + if meta.policy_fp != binding.materialization.fingerprint() { return None; } match &meta.agg_kind { @@ -586,7 +582,6 @@ impl QueryExecutionContext<'_> { } if by_group.is_empty() { tracing::debug!( - metric = %binding.metric, materialization = %binding.materialization.as_u64(), ?sids, matched_metadata, diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index 35171ec6..3de1ee0b 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -116,10 +116,9 @@ impl ActivePhysicalPlan { &self, ) -> impl Iterator + '_ { self.precompute_plan - .materialization_contracts - .keys() - .copied() - .map(Into::into) + .materializations + .iter() + .map(asap_types::PrecomputeMaterialization::policy_fingerprint) } } @@ -652,7 +651,6 @@ mod tests { summary_catalog: None, precompute_plan: control_plane::physical::compiler::PrecomputePlan { summary_catalog: None, - materialization_contracts: Default::default(), envelope: envelope.clone(), ingest: control_plane::physical::compiler::IngestContract { protocol: From b7682bf1c138d543741f7dbb656a97abe11b4670 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 15:25:37 -0600 Subject: [PATCH 17/30] Move SummaryCatalog contract into shared types --- Cargo.lock | 3 +- control_plane/Cargo.toml | 1 - control_plane/src/physical/summary_catalog.rs | 350 +--------------- crates/asap_types/Cargo.toml | 2 + crates/asap_types/src/lib.rs | 1 + crates/asap_types/src/sds.rs | 147 ++++++- crates/asap_types/src/summary_catalog.rs | 376 ++++++++++++++++++ .../asap_query_engine/catalog_resolver.rs | 49 ++- 8 files changed, 567 insertions(+), 362 deletions(-) create mode 100644 crates/asap_types/src/summary_catalog.rs diff --git a/Cargo.lock b/Cargo.lock index 24c229e7..e861dd5a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -418,6 +418,8 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "sha2", + "thiserror 1.0.69", "tracing", "xxhash-rust", ] @@ -801,7 +803,6 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "sha2", "thiserror 1.0.69", "tokio", "tokio-stream", diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index 00c8fe2c..b96327c7 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -17,7 +17,6 @@ axum = { version = "0.7", features = ["ws"] } futures-util = "0.3" serde = { version = "1", features = ["derive"] } serde_json = "1" -sha2 = "0.10" serde_yaml = "0.9" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } anyhow = "1" diff --git a/control_plane/src/physical/summary_catalog.rs b/control_plane/src/physical/summary_catalog.rs index caea096b..4b2d61cf 100644 --- a/control_plane/src/physical/summary_catalog.rs +++ b/control_plane/src/physical/summary_catalog.rs @@ -1,348 +1,6 @@ -//! Authoritative descriptor snapshot for a compiled physical plan. +//! Control-plane compatibility surface for the shared catalog wire contract. //! -//! Execution plans keep their compatibility fields during migration. This -//! catalog owns semantic definitions, not producer placement or pane state. +//! The authoritative catalog is constructed and published by the control +//! plane, while its immutable contract is shared with every consumer. -use std::collections::BTreeMap; - -use asap_types::sds::{ - DataDescriptor, DataDescriptorId, MaterializationId, SummaryDescriptor, SummaryDescriptorId, -}; -use asap_types::PolicyFingerprint; -use serde::{Deserialize, Serialize}; - -pub const SUMMARY_CATALOG_SCHEMA_VERSION: u32 = 1; - -/// Identifies one immutable catalog snapshot without duplicating descriptors. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct SummaryCatalogReference { - pub schema_version: u32, - pub plan_id: u64, - pub plan_version: u64, - pub snapshot_sha256: String, -} - -/// Stable materialization identity binds operator and population descriptors. -/// Concrete intervals, groups and completeness belong to runtime instances. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct MaterializationIdentity { - pub summary_descriptor_id: SummaryDescriptorId, - pub data_descriptor_id: DataDescriptorId, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct SummaryCatalog { - pub schema_version: u32, - pub plan_id: u64, - pub plan_version: u64, - pub summary_descriptors: BTreeMap, - pub data_descriptors: BTreeMap, - pub materializations: BTreeMap, -} - -#[derive(Debug, thiserror::Error)] -pub enum SummaryCatalogError { - #[error("unsupported summary catalog schema version {0}")] - SchemaVersion(u32), - #[error("invalid summary catalog descriptor: {0}")] - Descriptor(String), - #[error("materialization {0} has conflicting descriptor bindings")] - ConflictingMaterialization(u64), - #[error("materialization {0} references a missing descriptor")] - MissingDescriptor(u64), -} - -impl SummaryCatalog { - pub fn reference(&self) -> Result { - use sha2::{Digest, Sha256}; - self.validate()?; - // BTreeMap tables and typed descriptor fields serialize deterministically. - let bytes = serde_json::to_vec(self) - .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; - Ok(SummaryCatalogReference { - schema_version: self.schema_version, - plan_id: self.plan_id, - plan_version: self.plan_version, - snapshot_sha256: format!("{:x}", Sha256::digest(bytes)), - }) - } - - pub fn from_materializations( - plan_id: u64, - plan_version: u64, - materializations: &[asap_types::PrecomputeMaterialization], - ) -> Result { - let entries = materializations - .iter() - .map(|config| { - let summary = SummaryDescriptor::from_config(config) - .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; - let data = DataDescriptor::new( - config.metric.clone(), - asap_types::utils::normalize_spatial_filter(&config.spatial_filter), - config.grouping_labels.labels.clone(), - ); - Ok((config.policy_fingerprint(), summary, data)) - }) - .collect::, SummaryCatalogError>>()?; - Self::build(plan_id, plan_version, entries) - } - - pub fn build( - plan_id: u64, - plan_version: u64, - entries: impl IntoIterator, - ) -> Result { - let mut catalog = Self { - schema_version: SUMMARY_CATALOG_SCHEMA_VERSION, - plan_id, - plan_version, - summary_descriptors: BTreeMap::new(), - data_descriptors: BTreeMap::new(), - materializations: BTreeMap::new(), - }; - for (fingerprint, summary, data) in entries { - let materialization = MaterializationId::from(fingerprint); - summary - .validate() - .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; - data.validate() - .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; - let binding = MaterializationIdentity { - summary_descriptor_id: summary.id().clone(), - data_descriptor_id: data.id().clone(), - }; - if catalog - .materializations - .get(&materialization) - .is_some_and(|old| old != &binding) - { - return Err(SummaryCatalogError::ConflictingMaterialization( - materialization.as_u64(), - )); - } - catalog - .summary_descriptors - .insert(binding.summary_descriptor_id.clone(), summary); - catalog - .data_descriptors - .insert(binding.data_descriptor_id.clone(), data); - catalog.materializations.insert(materialization, binding); - } - catalog.validate()?; - Ok(catalog) - } - - pub fn validate(&self) -> Result<(), SummaryCatalogError> { - if self.schema_version != SUMMARY_CATALOG_SCHEMA_VERSION { - return Err(SummaryCatalogError::SchemaVersion(self.schema_version)); - } - for (key, descriptor) in &self.summary_descriptors { - descriptor - .validate() - .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; - if key != descriptor.id() { - return Err(SummaryCatalogError::Descriptor( - "summary table key differs from content identity".into(), - )); - } - } - for (key, descriptor) in &self.data_descriptors { - descriptor - .validate() - .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; - if key != descriptor.id() { - return Err(SummaryCatalogError::Descriptor( - "data table key differs from content identity".into(), - )); - } - } - for (id, binding) in &self.materializations { - if !self - .summary_descriptors - .contains_key(&binding.summary_descriptor_id) - || !self - .data_descriptors - .contains_key(&binding.data_descriptor_id) - { - return Err(SummaryCatalogError::MissingDescriptor(id.as_u64())); - } - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; - - fn config(metric: &str, filter: &str, window: u64) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - Default::default(), - KeyByLabelNames::new(vec!["job".into()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - window, - window, - WindowKind::Tumbling, - filter.into(), - metric.into(), - None, - None, - None, - ) - } - - // Content changes invalidate references even when plan/version are reused. - #[test] - fn snapshot_reference_is_deterministic_and_content_sensitive() { - let a = config("requests", "", 60); - let b = config("errors", "", 60); - let left = SummaryCatalog::from_materializations(1, 2, &[a.clone(), b.clone()]).unwrap(); - let reordered = SummaryCatalog::from_materializations(1, 2, &[b, a.clone()]).unwrap(); - assert_eq!(left.reference().unwrap(), reordered.reference().unwrap()); - let changed = SummaryCatalog::from_materializations(1, 2, &[a]).unwrap(); - assert_ne!(left.reference().unwrap(), changed.reference().unwrap()); - } - - // Panes share definitions but retain distinct physical materialization IDs. - #[test] - fn shares_descriptors_across_windows_and_deduplicates_materializations() { - let one = config("requests", "", 60); - let two = config("requests", "", 120); - let catalog = - SummaryCatalog::from_materializations(7, 2, &[one.clone(), two, one]).unwrap(); - assert_eq!(catalog.summary_descriptors.len(), 1); - assert_eq!(catalog.data_descriptors.len(), 1); - assert_eq!(catalog.materializations.len(), 2); - assert_eq!((catalog.plan_id, catalog.plan_version), (7, 2)); - } - - // Source/population changes never alias, while the operator can be reused. - #[test] - fn separates_population_and_operator_identity() { - let catalog = SummaryCatalog::from_materializations( - 1, - 1, - &[ - config("requests", "", 60), - config("errors", "", 60), - config("requests", "job=user", 60), - ], - ) - .unwrap(); - assert_eq!(catalog.summary_descriptors.len(), 1); - assert_eq!(catalog.data_descriptors.len(), 3); - } - - // Operator changes share population metadata without sharing state identity. - #[test] - fn changing_operator_parameters_creates_a_new_summary_descriptor() { - let mut a = config("requests", "", 60); - a.aggregation_type = AggregationType::DatasketchesKLL; - a.parameters.insert("K".into(), serde_json::json!(100)); - let mut b = a.clone(); - b.parameters.insert("K".into(), serde_json::json!(200)); - let catalog = SummaryCatalog::from_materializations(1, 1, &[a, b]).unwrap(); - assert_eq!(catalog.summary_descriptors.len(), 2); - assert_eq!(catalog.data_descriptors.len(), 1); - assert_eq!(catalog.materializations.len(), 2); - } - - // Construction order cannot affect the published snapshot bytes. - #[test] - fn snapshot_is_deterministic_and_round_trips() { - let a = config("requests", "", 60); - let b = config("errors", "", 60); - let forward = SummaryCatalog::from_materializations(7, 2, &[a.clone(), b.clone()]).unwrap(); - let backward = SummaryCatalog::from_materializations(7, 2, &[b, a]).unwrap(); - let bytes = serde_json::to_vec(&forward).unwrap(); - assert_eq!(bytes, serde_json::to_vec(&backward).unwrap()); - let decoded: SummaryCatalog = serde_json::from_slice(&bytes).unwrap(); - decoded.validate().unwrap(); - assert_eq!(decoded, forward); - } - - // The same materialization cannot silently rebind to another population. - #[test] - fn conflicting_materialization_is_rejected() { - let first = config("requests", "", 60); - let second = config("errors", "", 60); - let catalog = - SummaryCatalog::from_materializations(1, 1, &[first.clone(), second]).unwrap(); - let summary = catalog.summary_descriptors.values().next().unwrap().clone(); - let data = catalog - .data_descriptors - .values() - .cloned() - .collect::>(); - let error = SummaryCatalog::build( - 1, - 1, - [ - (first.policy_fingerprint(), summary.clone(), data[0].clone()), - (first.policy_fingerprint(), summary, data[1].clone()), - ], - ) - .unwrap_err(); - assert!(matches!( - error, - SummaryCatalogError::ConflictingMaterialization(_) - )); - } - - // Imported catalogs must resolve every foreign key and content identity. - #[test] - fn rejects_dangling_refs_tampered_keys_and_unknown_schema() { - let catalog = - SummaryCatalog::from_materializations(1, 1, &[config("requests", "", 60)]).unwrap(); - let mut broken = catalog.clone(); - broken.data_descriptors.clear(); - assert!(matches!( - broken.validate(), - Err(SummaryCatalogError::MissingDescriptor(_)) - )); - let mut broken = catalog.clone(); - let (_, descriptor) = broken.summary_descriptors.pop_first().unwrap(); - broken.summary_descriptors.insert( - serde_json::from_value(serde_json::json!("forged")).unwrap(), - descriptor, - ); - assert!(matches!( - broken.validate(), - Err(SummaryCatalogError::Descriptor(_)) - )); - let mut broken = catalog.clone(); - broken - .summary_descriptors - .values_mut() - .next() - .unwrap() - .state_schema_version += 1; - assert!(matches!( - broken.validate(), - Err(SummaryCatalogError::Descriptor(_)) - )); - let mut broken = catalog; - broken.schema_version = SUMMARY_CATALOG_SCHEMA_VERSION + 1; - assert!(matches!( - broken.validate(), - Err(SummaryCatalogError::SchemaVersion(_)) - )); - } - - // Native exact-only plans have a valid empty catalog, not dummy state. - #[test] - fn empty_catalog_is_valid() { - let catalog = SummaryCatalog::from_materializations(1, 1, &[]).unwrap(); - assert!(catalog.materializations.is_empty()); - catalog.validate().unwrap(); - } -} +pub use asap_types::summary_catalog::*; diff --git a/crates/asap_types/Cargo.toml b/crates/asap_types/Cargo.toml index b1d27c1c..b67cdde3 100644 --- a/crates/asap_types/Cargo.toml +++ b/crates/asap_types/Cargo.toml @@ -9,8 +9,10 @@ serde.workspace = true serde_json.workspace = true serde_yaml.workspace = true anyhow.workspace = true +thiserror.workspace = true clap.workspace = true xxhash-rust = { version = "0.8", features = ["xxh64"] } +sha2 = "0.10" # First cross-crate ASAPPlanner dependency for data_plane (transitively, # via this crate): WindowType -> planner_types::pre_asap::query_expr::WindowKind diff --git a/crates/asap_types/src/lib.rs b/crates/asap_types/src/lib.rs index f205af63..62f4812e 100644 --- a/crates/asap_types/src/lib.rs +++ b/crates/asap_types/src/lib.rs @@ -10,6 +10,7 @@ pub mod query_requirements; pub mod routing_index; pub mod sds; pub mod storage_backend; +pub mod summary_catalog; pub mod traits; pub mod utils; diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index 7e383168..fd466afe 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -436,6 +436,11 @@ impl SummaryDescriptor { return Err(SdsError("state schema version must be positive".into())); } fidelity.validate()?; + if !fidelity.is_compatible_with(&operator) { + return Err(SdsError( + "summary operator and fidelity guarantee are incompatible".into(), + )); + } let content = json!({"operator":operator,"fidelity":fidelity,"state_schema_version":state_schema_version}); let id = SummaryDescriptorId(format!("summary:v2:{}", canonical(&content))); Ok(Self { @@ -487,6 +492,80 @@ impl SummaryDescriptor { } } +impl FidelityGuarantee { + /// Reject a descriptor that advertises an error model belonging to a + /// different state family. This is part of the shared wire contract, so a + /// consumer must not need to trust the process that produced the catalog. + pub fn is_compatible_with(&self, operator: &SummaryOperator) -> bool { + use AggregationType as A; + use FidelityGuarantee::*; + use SketchAlgorithm as S; + + let configured = |aggregation_type| match (aggregation_type, self) { + (A::Sum | A::MultipleSum | A::MinMax | A::MultipleMinMax, Exact) => true, + (A::Increase | A::MultipleIncrease, ExactCounter { .. }) => true, + (A::DatasketchesKLL | A::HydraKLL, KllRankError { .. }) => true, + (A::DDSketch, DdSketchRelativeError { .. }) => true, + (A::HLL, HllCardinalityError { .. }) => true, + (A::CountMinSketch | A::CountMinSketchWithHeap, CmsFrequencyError { .. }) => true, + (A::CountSketch | A::CountSketchWithHeap, CountSketchFrequencyError { .. }) => true, + (A::SingleSubpopulation | A::MultipleSubpopulation, Unknown { .. }) => true, + _ => false, + }; + + match operator { + SummaryOperator::LegacyPartial { .. } => matches!(self, Unknown { .. }), + SummaryOperator::ExactAgg { agg_type, .. } => configured(*agg_type), + SummaryOperator::Configured { + aggregation_type, + parameters, + .. + } => configured(*aggregation_type) && self.matches_parameters(parameters), + SummaryOperator::Sketch { + algorithm, + parameters, + } => { + matches!( + (algorithm, self), + (S::Kll, KllRankError { .. }) + | (S::DDSketch, DdSketchRelativeError { .. }) + | (S::Hll, HllCardinalityError { .. }) + | (S::Cms | S::CmsWithHeap, CmsFrequencyError { .. }) + | ( + S::CountSketch | S::CountSketchWithHeap, + CountSketchFrequencyError { .. } + ) + | (S::Kmv | S::Theta, Unknown { .. }) + ) && self.matches_parameters(parameters) + } + } + } + + fn matches_parameters(&self, parameters: &BTreeMap) -> bool { + let u32_parameter = |names: &[&str], expected: u32| { + names + .iter() + .find_map(|name| parameters.get(*name)) + .is_none_or(|value| value.as_u64() == Some(u64::from(expected))) + }; + match self { + Self::KllRankError { k, .. } => u32_parameter(&["k", "K"], *k), + Self::HllCardinalityError { precision, .. } => { + u32_parameter(&["precision", "p"], *precision) + } + Self::CmsFrequencyError { width, depth, .. } + | Self::CountSketchFrequencyError { width, depth, .. } => { + u32_parameter(&["width"], *width) && u32_parameter(&["depth"], *depth) + } + Self::DdSketchRelativeError { alpha } => parameters + .get("alpha") + .or_else(|| parameters.get("relative_accuracy")) + .is_none_or(|value| value.as_f64() == Some(*alpha)), + _ => true, + } + } +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct DataDescriptor { @@ -586,8 +665,9 @@ mod tests { materialization_id: MaterializationId(crate::PolicyFingerprint(7)), summary_descriptor_id: descriptor( 200, - FidelityGuarantee::Unknown { - reason: "test".into(), + FidelityGuarantee::KllRankError { + k: 200, + model: "rank.v1".into(), }, 1, ) @@ -675,8 +755,9 @@ mod tests { fn identity_includes_configuration_fidelity_and_state_schema() { let base = descriptor( 200, - FidelityGuarantee::Unknown { - reason: "not supplied".into(), + FidelityGuarantee::KllRankError { + k: 200, + model: "rank.v1".into(), }, 1, ); @@ -684,20 +765,30 @@ mod tests { base.id, descriptor( 201, - FidelityGuarantee::Unknown { - reason: "not supplied".into() + FidelityGuarantee::KllRankError { + k: 201, + model: "rank.v1".into() }, 1 ) .id ); - assert_ne!(base.id, descriptor(200, FidelityGuarantee::Exact, 1).id); + assert!(SummaryDescriptor::new( + SummaryOperator::Sketch { + algorithm: SketchAlgorithm::Kll, + parameters: BTreeMap::from([("k".into(), json!(200))]), + }, + FidelityGuarantee::Exact, + 1, + ) + .is_err()); assert_ne!( base.id, descriptor( 200, - FidelityGuarantee::Unknown { - reason: "not supplied".into() + FidelityGuarantee::KllRankError { + k: 200, + model: "rank.v1".into() }, 2 ) @@ -736,8 +827,9 @@ mod tests { fn wire_roundtrip_and_tampered_id_validation() { let original = descriptor( 200, - FidelityGuarantee::Unknown { - reason: "not supplied".into(), + FidelityGuarantee::KllRankError { + k: 200, + model: "rank.v1".into(), }, 1, ); @@ -762,6 +854,31 @@ mod tests { 1 ) .is_err()); + assert!(SummaryDescriptor::new( + SummaryOperator::Configured { + aggregation_type: AggregationType::Sum, + aggregation_sub_type: String::new(), + parameters: BTreeMap::new(), + }, + FidelityGuarantee::KllRankError { + k: 200, + model: "rank.v1".into(), + }, + 1, + ) + .is_err()); + assert!(SummaryDescriptor::new( + SummaryOperator::Sketch { + algorithm: SketchAlgorithm::Kll, + parameters: BTreeMap::from([("k".into(), json!(100))]), + }, + FidelityGuarantee::KllRankError { + k: 200, + model: "rank.v1".into(), + }, + 1, + ) + .is_err()); } #[test] fn configured_identity_preserves_heap_hydra_and_subtype_and_excludes_population() { @@ -836,8 +953,12 @@ mod tests { .unwrap() .id ); + let kll = SummaryOperator::Sketch { + algorithm: SketchAlgorithm::Kll, + parameters: BTreeMap::from([("k".into(), json!(200))]), + }; let first = SummaryDescriptor::new( - a.clone(), + kll.clone(), FidelityGuarantee::KllRankError { k: 200, model: "rank.v1".into(), @@ -846,7 +967,7 @@ mod tests { ) .unwrap(); let second = SummaryDescriptor::new( - a, + kll, FidelityGuarantee::KllRankError { k: 200, model: "rank.v2".into(), diff --git a/crates/asap_types/src/summary_catalog.rs b/crates/asap_types/src/summary_catalog.rs new file mode 100644 index 00000000..1e206224 --- /dev/null +++ b/crates/asap_types/src/summary_catalog.rs @@ -0,0 +1,376 @@ +//! Authoritative descriptor snapshot for a compiled physical plan. +//! +//! Execution plans keep their compatibility fields during migration. This +//! catalog owns semantic definitions, not producer placement or pane state. + +use std::collections::BTreeMap; + +use crate::sds::{ + DataDescriptor, DataDescriptorId, MaterializationId, SummaryDescriptor, SummaryDescriptorId, +}; +use crate::PolicyFingerprint; +use serde::{Deserialize, Serialize}; + +pub const SUMMARY_CATALOG_SCHEMA_VERSION: u32 = 1; + +/// Identifies one immutable catalog snapshot without duplicating descriptors. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct SummaryCatalogReference { + pub schema_version: u32, + pub plan_id: u64, + pub plan_version: u64, + pub snapshot_sha256: String, +} + +/// Stable materialization identity binds operator and population descriptors. +/// Concrete intervals, groups and completeness belong to runtime instances. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct MaterializationIdentity { + pub summary_descriptor_id: SummaryDescriptorId, + pub data_descriptor_id: DataDescriptorId, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct SummaryCatalog { + pub schema_version: u32, + pub plan_id: u64, + pub plan_version: u64, + pub summary_descriptors: BTreeMap, + pub data_descriptors: BTreeMap, + pub materializations: BTreeMap, +} + +#[derive(Debug, thiserror::Error)] +pub enum SummaryCatalogError { + #[error("unsupported summary catalog schema version {0}")] + SchemaVersion(u32), + #[error("invalid summary catalog descriptor: {0}")] + Descriptor(String), + #[error("materialization {0} has conflicting descriptor bindings")] + ConflictingMaterialization(u64), + #[error("materialization {0} references a missing descriptor")] + MissingDescriptor(u64), + #[error("catalog reference does not identify the supplied snapshot")] + ReferenceMismatch, +} + +impl SummaryCatalogReference { + /// Validate an untrusted wire reference against the installed immutable + /// snapshot and its enclosing plan generation. + pub fn validate_snapshot( + &self, + catalog: &SummaryCatalog, + plan_id: u64, + plan_version: u64, + ) -> Result<(), SummaryCatalogError> { + let expected = catalog.reference()?; + if self != &expected || (plan_id, plan_version) != (catalog.plan_id, catalog.plan_version) { + return Err(SummaryCatalogError::ReferenceMismatch); + } + Ok(()) + } +} + +impl SummaryCatalog { + pub fn reference(&self) -> Result { + use sha2::{Digest, Sha256}; + self.validate()?; + // BTreeMap tables and typed descriptor fields serialize deterministically. + let bytes = serde_json::to_vec(self) + .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; + Ok(SummaryCatalogReference { + schema_version: self.schema_version, + plan_id: self.plan_id, + plan_version: self.plan_version, + snapshot_sha256: format!("{:x}", Sha256::digest(bytes)), + }) + } + + pub fn from_materializations( + plan_id: u64, + plan_version: u64, + materializations: &[crate::PrecomputeMaterialization], + ) -> Result { + let entries = materializations + .iter() + .map(|config| { + let summary = SummaryDescriptor::from_config(config) + .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; + let data = DataDescriptor::new( + config.metric.clone(), + crate::utils::normalize_spatial_filter(&config.spatial_filter), + config.grouping_labels.labels.clone(), + ); + Ok((config.policy_fingerprint(), summary, data)) + }) + .collect::, SummaryCatalogError>>()?; + Self::build(plan_id, plan_version, entries) + } + + pub fn build( + plan_id: u64, + plan_version: u64, + entries: impl IntoIterator, + ) -> Result { + let mut catalog = Self { + schema_version: SUMMARY_CATALOG_SCHEMA_VERSION, + plan_id, + plan_version, + summary_descriptors: BTreeMap::new(), + data_descriptors: BTreeMap::new(), + materializations: BTreeMap::new(), + }; + for (fingerprint, summary, data) in entries { + let materialization = MaterializationId::from(fingerprint); + summary + .validate() + .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; + data.validate() + .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; + let binding = MaterializationIdentity { + summary_descriptor_id: summary.id().clone(), + data_descriptor_id: data.id().clone(), + }; + if catalog + .materializations + .get(&materialization) + .is_some_and(|old| old != &binding) + { + return Err(SummaryCatalogError::ConflictingMaterialization( + materialization.as_u64(), + )); + } + catalog + .summary_descriptors + .insert(binding.summary_descriptor_id.clone(), summary); + catalog + .data_descriptors + .insert(binding.data_descriptor_id.clone(), data); + catalog.materializations.insert(materialization, binding); + } + catalog.validate()?; + Ok(catalog) + } + + pub fn validate(&self) -> Result<(), SummaryCatalogError> { + if self.schema_version != SUMMARY_CATALOG_SCHEMA_VERSION { + return Err(SummaryCatalogError::SchemaVersion(self.schema_version)); + } + for (key, descriptor) in &self.summary_descriptors { + descriptor + .validate() + .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; + if key != descriptor.id() { + return Err(SummaryCatalogError::Descriptor( + "summary table key differs from content identity".into(), + )); + } + } + for (key, descriptor) in &self.data_descriptors { + descriptor + .validate() + .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; + if key != descriptor.id() { + return Err(SummaryCatalogError::Descriptor( + "data table key differs from content identity".into(), + )); + } + } + for (id, binding) in &self.materializations { + if !self + .summary_descriptors + .contains_key(&binding.summary_descriptor_id) + || !self + .data_descriptors + .contains_key(&binding.data_descriptor_id) + { + return Err(SummaryCatalogError::MissingDescriptor(id.as_u64())); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + + fn config(metric: &str, filter: &str, window: u64) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( + AggregationType::Sum, + String::new(), + Default::default(), + KeyByLabelNames::new(vec!["job".into()]), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + window, + window, + WindowKind::Tumbling, + filter.into(), + metric.into(), + None, + None, + None, + ) + } + + // Content changes invalidate references even when plan/version are reused. + #[test] + fn snapshot_reference_is_deterministic_and_content_sensitive() { + let a = config("requests", "", 60); + let b = config("errors", "", 60); + let left = SummaryCatalog::from_materializations(1, 2, &[a.clone(), b.clone()]).unwrap(); + let reordered = SummaryCatalog::from_materializations(1, 2, &[b, a.clone()]).unwrap(); + assert_eq!(left.reference().unwrap(), reordered.reference().unwrap()); + let changed = SummaryCatalog::from_materializations(1, 2, &[a]).unwrap(); + assert_ne!(left.reference().unwrap(), changed.reference().unwrap()); + left.reference() + .unwrap() + .validate_snapshot(&left, 1, 2) + .unwrap(); + assert!(left + .reference() + .unwrap() + .validate_snapshot(&left, 1, 3) + .is_err()); + } + + // Panes share definitions but retain distinct physical materialization IDs. + #[test] + fn shares_descriptors_across_windows_and_deduplicates_materializations() { + let one = config("requests", "", 60); + let two = config("requests", "", 120); + let catalog = + SummaryCatalog::from_materializations(7, 2, &[one.clone(), two, one]).unwrap(); + assert_eq!(catalog.summary_descriptors.len(), 1); + assert_eq!(catalog.data_descriptors.len(), 1); + assert_eq!(catalog.materializations.len(), 2); + assert_eq!((catalog.plan_id, catalog.plan_version), (7, 2)); + } + + // Source/population changes never alias, while the operator can be reused. + #[test] + fn separates_population_and_operator_identity() { + let catalog = SummaryCatalog::from_materializations( + 1, + 1, + &[ + config("requests", "", 60), + config("errors", "", 60), + config("requests", "job=user", 60), + ], + ) + .unwrap(); + assert_eq!(catalog.summary_descriptors.len(), 1); + assert_eq!(catalog.data_descriptors.len(), 3); + } + + // Operator changes share population metadata without sharing state identity. + #[test] + fn changing_operator_parameters_creates_a_new_summary_descriptor() { + let mut a = config("requests", "", 60); + a.aggregation_type = AggregationType::DatasketchesKLL; + a.parameters.insert("K".into(), serde_json::json!(100)); + let mut b = a.clone(); + b.parameters.insert("K".into(), serde_json::json!(200)); + let catalog = SummaryCatalog::from_materializations(1, 1, &[a, b]).unwrap(); + assert_eq!(catalog.summary_descriptors.len(), 2); + assert_eq!(catalog.data_descriptors.len(), 1); + assert_eq!(catalog.materializations.len(), 2); + } + + // Construction order cannot affect the published snapshot bytes. + #[test] + fn snapshot_is_deterministic_and_round_trips() { + let a = config("requests", "", 60); + let b = config("errors", "", 60); + let forward = SummaryCatalog::from_materializations(7, 2, &[a.clone(), b.clone()]).unwrap(); + let backward = SummaryCatalog::from_materializations(7, 2, &[b, a]).unwrap(); + let bytes = serde_json::to_vec(&forward).unwrap(); + assert_eq!(bytes, serde_json::to_vec(&backward).unwrap()); + let decoded: SummaryCatalog = serde_json::from_slice(&bytes).unwrap(); + decoded.validate().unwrap(); + assert_eq!(decoded, forward); + } + + // The same materialization cannot silently rebind to another population. + #[test] + fn conflicting_materialization_is_rejected() { + let first = config("requests", "", 60); + let second = config("errors", "", 60); + let catalog = + SummaryCatalog::from_materializations(1, 1, &[first.clone(), second]).unwrap(); + let summary = catalog.summary_descriptors.values().next().unwrap().clone(); + let data = catalog + .data_descriptors + .values() + .cloned() + .collect::>(); + let error = SummaryCatalog::build( + 1, + 1, + [ + (first.policy_fingerprint(), summary.clone(), data[0].clone()), + (first.policy_fingerprint(), summary, data[1].clone()), + ], + ) + .unwrap_err(); + assert!(matches!( + error, + SummaryCatalogError::ConflictingMaterialization(_) + )); + } + + // Imported catalogs must resolve every foreign key and content identity. + #[test] + fn rejects_dangling_refs_tampered_keys_and_unknown_schema() { + let catalog = + SummaryCatalog::from_materializations(1, 1, &[config("requests", "", 60)]).unwrap(); + let mut broken = catalog.clone(); + broken.data_descriptors.clear(); + assert!(matches!( + broken.validate(), + Err(SummaryCatalogError::MissingDescriptor(_)) + )); + let mut broken = catalog.clone(); + let (_, descriptor) = broken.summary_descriptors.pop_first().unwrap(); + broken.summary_descriptors.insert( + serde_json::from_value(serde_json::json!("forged")).unwrap(), + descriptor, + ); + assert!(matches!( + broken.validate(), + Err(SummaryCatalogError::Descriptor(_)) + )); + let mut broken = catalog.clone(); + broken + .summary_descriptors + .values_mut() + .next() + .unwrap() + .state_schema_version += 1; + assert!(matches!( + broken.validate(), + Err(SummaryCatalogError::Descriptor(_)) + )); + let mut broken = catalog; + broken.schema_version = SUMMARY_CATALOG_SCHEMA_VERSION + 1; + assert!(matches!( + broken.validate(), + Err(SummaryCatalogError::SchemaVersion(_)) + )); + } + + // Native exact-only plans have a valid empty catalog, not dummy state. + #[test] + fn empty_catalog_is_valid() { + let catalog = SummaryCatalog::from_materializations(1, 1, &[]).unwrap(); + assert!(catalog.materializations.is_empty()); + catalog.validate().unwrap(); + } +} diff --git a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs index 6e856d94..bcbf71f2 100644 --- a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs +++ b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs @@ -4,8 +4,8 @@ use std::collections::{BTreeMap, BTreeSet}; use asap_types::sds::{DataDescriptor, MaterializationId, SummaryDescriptor, SummaryOperator}; +use asap_types::summary_catalog::{MaterializationIdentity, SummaryCatalog}; use asap_types::AggregationType; -use control_plane::physical::summary_catalog::{MaterializationIdentity, SummaryCatalog}; use control_plane::query_plan::{ExactReadout, QueryPlanEntry, QueryPlanNode, QueryReadout}; use crate::query_engines::EngineError; @@ -37,6 +37,14 @@ pub(crate) fn resolve( .data_descriptors .get(&identity.data_descriptor_id) .ok_or_else(|| miss("missing data descriptor"))?; + // Installation validates the whole snapshot. Keep resolution fail-closed as + // defense in depth for catalogs restored from disk or supplied by a future + // transport implementation. + summary + .validate() + .map_err(|error| miss(format!("invalid summary descriptor: {error}")))?; + data.validate() + .map_err(|error| miss(format!("invalid data descriptor: {error}")))?; Ok(ResolvedMaterialization { identity, summary, @@ -166,6 +174,8 @@ pub(crate) fn validate_entry( #[cfg(test)] mod tests { use super::*; + use asap_types::summary_catalog::SummaryCatalog; + use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; use control_plane::physical::compiler::BackendLocalPlanningSnapshot; fn fixture() -> control_plane::physical::compiler::PhysicalPlan { @@ -179,6 +189,27 @@ mod tests { snapshot.compile().unwrap() } + fn catalog_fixture() -> SummaryCatalog { + let config = PrecomputeMaterialization::new( + AggregationType::Sum, + String::new(), + Default::default(), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + 60, + 60, + WindowKind::Tumbling, + String::new(), + "m".into(), + None, + None, + None, + ); + SummaryCatalog::from_materializations(1, 1, &[config]).unwrap() + } + // A readout cannot relabel a valid sum materialization as a rate capability. #[test] fn rejects_wrong_readout_and_missing_or_stale_replica() { @@ -230,4 +261,20 @@ mod tests { broken.data_descriptors.clear(); assert!(resolve(&broken, id).is_err()); } + + #[test] + fn rejects_operator_fidelity_mismatch_during_resolution() { + let mut catalog = catalog_fixture(); + let id = *catalog.materializations.keys().next().unwrap(); + let descriptor_id = catalog.materializations[&id].summary_descriptor_id.clone(); + catalog + .summary_descriptors + .get_mut(&descriptor_id) + .unwrap() + .fidelity = asap_types::sds::FidelityGuarantee::KllRankError { + k: 200, + model: "rank.v1".into(), + }; + assert!(resolve(&catalog, id).is_err()); + } } From c451949992e0cccb0b34db87ba7205422d533e8f Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 14:49:36 -0600 Subject: [PATCH 18/30] Stop publishing legacy BackendPlan bytes --- control_plane/src/backend_client.rs | 127 +------------------------ control_plane/src/emit/backend_push.rs | 13 +-- control_plane/src/main.rs | 13 +-- 3 files changed, 8 insertions(+), 145 deletions(-) diff --git a/control_plane/src/backend_client.rs b/control_plane/src/backend_client.rs index f98bb79f..83ea80ac 100644 --- a/control_plane/src/backend_client.rs +++ b/control_plane/src/backend_client.rs @@ -325,47 +325,10 @@ impl BackendClient { } } - /// POST an encoded `BackendPlan` (protobuf bytes) to the backend's - /// `POST /api/v1/backend-plan` endpoint (see - /// `control_plane/docs/design-backend-plan-wire-format.md`), sent - /// alongside the streaming-config/storage-routing push, not in place - /// of it (see `emit::backend_push`'s call site). Same - /// transient/permanent classification as the other typed POST - /// methods. - pub async fn post_backend_plan_typed( - &self, - bytes: Vec, - ) -> std::result::Result<(), BackendPostError> { - let url = derive_backend_plan_url(&self.endpoint); - debug!( - endpoint = %url, - plan_bytes = bytes.len(), - "posting BackendPlan to ASAPQuery-backend (typed)" - ); - let resp = self - .http - .post(&url) - .header("content-type", "application/x-protobuf") - .body(bytes) - .send() - .await - .map_err(classify_reqwest_error)?; - - let status = resp.status(); - if status.is_success() { - Ok(()) - } else { - let body = resp.text().await.unwrap_or_default(); - Err(classify_http_status(status, body, "BackendPlan POST")) - } - } - - /// Publish the canonical catalog generation. The optional legacy bytes are - /// an explicit compatibility projection, never used to validate this document. + /// Publish one authoritative catalog generation and all plans that reference it. pub async fn post_catalog_plan_typed( &self, publication: &crate::physical::publication::PhysicalPlanPublication, - compatibility_backend_plan: Option>, storage_routing: Option, adaptation_evidence: &[crate::physical::compiler::RuntimeAdaptationEvidence], ) -> std::result::Result<(), BackendPostError> { @@ -377,9 +340,6 @@ impl BackendClient { let fields = body .as_object_mut() .expect("publication serializes as object"); - if let Some(bytes) = compatibility_backend_plan { - fields.insert("backend_plan".into(), serde_json::json!(bytes)); - } fields.insert("storage_routing".into(), serde_json::json!(storage_routing)); fields.insert( "adaptation_evidence".into(), @@ -412,7 +372,6 @@ impl BackendClient { &self, precompute_plan: &crate::physical::compiler::PrecomputePlan, transmission_plan: &crate::physical::compiler::TransmissionPlan, - backend_plan: Vec, query_plan: &crate::query_plan::QueryPlan, storage_routing: Option, adaptation_evidence: &[crate::physical::compiler::RuntimeAdaptationEvidence], @@ -447,7 +406,6 @@ impl BackendClient { "collector_plans": [], "precompute_plan": precompute_plan, "transmission_plan": transmission_plan, - "backend_plan": backend_plan, "query_plan": query_plan, "storage_routing": storage_routing, "adaptation_evidence": adaptation_evidence, @@ -550,21 +508,6 @@ fn derive_storage_routing_url(endpoint: &str) -> String { endpoint.to_string() } -/// Map a streaming-config endpoint URL to the sibling `backend-plan` -/// endpoint, same rewrite convention as [`derive_storage_routing_url`]. -fn derive_backend_plan_url(endpoint: &str) -> String { - const STREAMING_PATH_DASH: &str = "/api/v1/streaming-config"; - const STREAMING_PATH_UNDERSCORE: &str = "/api/v1/streaming_config"; - const PLAN_PATH: &str = "/api/v1/backend-plan"; - if let Some(stripped) = endpoint.strip_suffix(STREAMING_PATH_DASH) { - return format!("{stripped}{PLAN_PATH}"); - } - if let Some(stripped) = endpoint.strip_suffix(STREAMING_PATH_UNDERSCORE) { - return format!("{stripped}{PLAN_PATH}"); - } - endpoint.to_string() -} - /// Fire-and-forget convenience helper used by the replanner. Logs /// errors at WARN and never propagates them — the replanner should /// never fail an entire replan because the backend was temporarily @@ -718,27 +661,6 @@ mod tests { assert_eq!(derive_storage_routing_url("http://x/foo"), "http://x/foo"); } - #[test] - fn backend_plan_url_rewrites_streaming_path() { - assert_eq!( - derive_backend_plan_url("http://backend:8088/api/v1/streaming-config"), - "http://backend:8088/api/v1/backend-plan" - ); - assert_eq!( - derive_backend_plan_url("http://backend:8088/api/v1/streaming_config"), - "http://backend:8088/api/v1/backend-plan" - ); - } - - #[test] - fn backend_plan_url_preserves_unknown_paths_for_tests() { - assert_eq!( - derive_backend_plan_url("http://127.0.0.1:1/api/v1/backend-plan"), - "http://127.0.0.1:1/api/v1/backend-plan" - ); - assert_eq!(derive_backend_plan_url("http://x/foo"), "http://x/foo"); - } - #[tokio::test] async fn catalog_publication_posts_canonical_document_without_legacy_bytes() { let snapshot: crate::physical::compiler::BackendLocalPlanningSnapshot = @@ -766,7 +688,7 @@ mod tests { }); let client = BackendClient::new(format!("http://{addr}/api/v1/streaming-config")); client - .post_catalog_plan_typed(&publication, None, None, &[]) + .post_catalog_plan_typed(&publication, None, &[]) .await .unwrap(); let bodies = hits.lock().unwrap(); @@ -784,51 +706,6 @@ mod tests { server.abort(); } - #[tokio::test] - async fn backend_plan_post_round_trips_bytes_via_url_rewrite() { - let hits: StdArc>>> = StdArc::new(Mutex::new(Vec::new())); - let hits_for_route = hits.clone(); - let app = Router::new().route( - "/api/v1/backend-plan", - post(move |body: axum::body::Bytes| { - let hits = hits_for_route.clone(); - async move { - hits.lock().unwrap().push(body.to_vec()); - axum::http::StatusCode::OK - } - }), - ); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve(listener, app).await.unwrap(); - }); - tokio::time::sleep(Duration::from_millis(50)).await; - - let client = BackendClient::new(format!("http://{addr}/api/v1/streaming-config")); - let bytes = vec![1u8, 2, 3, 4]; - client - .post_backend_plan_typed(bytes.clone()) - .await - .expect("backend-plan post ok"); - - let received = hits.lock().unwrap(); - assert_eq!(received.len(), 1); - assert_eq!(received[0], bytes); - } - - #[tokio::test] - async fn backend_plan_post_404_is_transient() { - let sink = SharedSink(StdArc::new(Mutex::new(Vec::new()))); - let url = start_mock_backend(sink.clone(), axum::http::StatusCode::NOT_FOUND).await; - let client = BackendClient::new(url); - let err = client - .post_backend_plan_typed(vec![1, 2, 3]) - .await - .expect_err("404 should surface as Err"); - assert!(err.is_transient(), "404 must classify as transient: {err}"); - } - /// Phase α: full happy path. A mock backend hosts the storage /// routing endpoint; the client POSTs the control-plane-emitted JSON /// and the body round-trips verbatim. Mirrors `json_post_round_trips_body`. diff --git a/control_plane/src/emit/backend_push.rs b/control_plane/src/emit/backend_push.rs index cdf49b93..ebf80ee5 100644 --- a/control_plane/src/emit/backend_push.rs +++ b/control_plane/src/emit/backend_push.rs @@ -230,7 +230,6 @@ async fn push_documents_coupled( client: &Arc, precompute_plan: &PrecomputePlan, routing_body: String, - plan_bytes: Vec, ) -> (bool, bool, u32) { let start = Instant::now(); let routing: serde_json::Value = match serde_json::from_str(&routing_body) { @@ -244,12 +243,8 @@ async fn push_documents_coupled( // may still install producer/storage state, but publishes an empty // QueryPlan so every serving request fails closed to the exact tier. let query_plan = crate::query_plan::QueryPlan { - plan_id: crate::backend_plan::BackendPlan::decode(&plan_bytes) - .map(|plan| plan.plan_id) - .unwrap_or_default(), - plan_version: crate::backend_plan::BackendPlan::decode(&plan_bytes) - .map(|plan| plan.plan_version) - .unwrap_or_default(), + plan_id: precompute_plan.envelope.plan_id, + plan_version: precompute_plan.envelope.plan_version, entries: Default::default(), }; let transmission_plan = match crate::physical::compiler::TransmissionPlan::build( @@ -269,7 +264,6 @@ async fn push_documents_coupled( .post_physical_plan_typed( precompute_plan, &transmission_plan, - plan_bytes.clone(), &query_plan, Some(routing.clone()), &[], @@ -467,7 +461,6 @@ async fn push_cumulative_entries( return PushOutcome::EmitFailed; } }; - let plan_bytes = backend_plan.encode_to_vec(); let precompute_envelope = PlanEnvelope { plan_id, plan_version: 1, @@ -561,7 +554,7 @@ async fn push_cumulative_entries( }; let (streaming_ok, routing_ok, attempts) = - push_documents_coupled(client, &precompute_plan, routing_body, plan_bytes).await; + push_documents_coupled(client, &precompute_plan, routing_body).await; let plan_ok = streaming_ok; if streaming_ok && routing_ok && plan_ok { diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index bc7aa7aa..3a1adc17 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -628,10 +628,8 @@ struct CompileAndPublishPhysicalPlanResponse { lifecycle_estimates: Vec, } -/// Compile one Planner IR decision into matching Collector, Precompute, and Backend views -/// and install them in dependency order. Unlike the legacy planning endpoint, -/// this MVP boundary is fail-closed: the Collector plan is never published -/// unless the backend accepted the exact matching BackendPlan first. +/// Compile one Planner IR decision into one catalog-backed physical plan and +/// install it atomically before publishing collector projections. async fn handle_compile_and_publish_physical_plan( State(st): State, Json(request): Json, @@ -680,12 +678,7 @@ async fn handle_compile_and_publish_physical_plan( } }; if let Err(error) = backend - .post_catalog_plan_typed( - &publication, - Some(bundle.backend_plan.encode_to_vec()), - None, - &adaptation_evidence, - ) + .post_catalog_plan_typed(&publication, None, &adaptation_evidence) .await { return ( From 6332f505e9910ef1f8a40f1ee167f75720be2312 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 15:00:42 -0600 Subject: [PATCH 19/30] Remove BackendPlan from control-plane compilation --- control_plane/build.rs | 7 - .../examples/calibration_candidates.rs | 1 - .../examples/compile_workload_artifact.rs | 1 - control_plane/proto/backend_plan.proto | 198 --- control_plane/src/backend_client.rs | 1 - .../src/backend_plan/from_stage_config.rs | 545 -------- control_plane/src/backend_plan/mod.rs | 1240 ----------------- control_plane/src/emit/backend_push.rs | 50 +- control_plane/src/lib.rs | 1 - control_plane/src/main.rs | 4 +- control_plane/src/physical/compiler.rs | 381 +++-- .../src/physical/post_asap/cost_model.rs | 4 +- control_plane/src/physical/post_asap/mod.rs | 2 +- control_plane/src/physical/publication.rs | 5 +- 14 files changed, 195 insertions(+), 2245 deletions(-) delete mode 100644 control_plane/proto/backend_plan.proto delete mode 100644 control_plane/src/backend_plan/from_stage_config.rs delete mode 100644 control_plane/src/backend_plan/mod.rs diff --git a/control_plane/build.rs b/control_plane/build.rs index 4c8b9236..2a86a87b 100644 --- a/control_plane/build.rs +++ b/control_plane/build.rs @@ -1,12 +1,5 @@ fn main() -> Result<(), Box> { prost_build::compile_protos(&["proto/opamp.proto"], &["proto/"])?; - // `BackendPlan` wire contract — see - // `control_plane/docs/design-backend-plan-wire-format.md`. Needs - // `--experimental_allow_proto3_optional` for the `optional` scalar - // fields (`WindowSpec.slide_ms`, `RetentionPolicy.num_aggregates_to_retain`). - prost_build::Config::new() - .protoc_arg("--experimental_allow_proto3_optional") - .compile_protos(&["proto/backend_plan.proto"], &["proto/"])?; // asap.runtime.v1.RuntimeSamples service — receives // PushExporter batches from agents. Must stay in lockstep // with `sketch-bench/sketch-runtime/proto/feedback.proto`. diff --git a/control_plane/examples/calibration_candidates.rs b/control_plane/examples/calibration_candidates.rs index 2731b1cd..c69f4f85 100644 --- a/control_plane/examples/calibration_candidates.rs +++ b/control_plane/examples/calibration_candidates.rs @@ -130,7 +130,6 @@ fn main() -> Result<(), Box> { "collector_plans": plan.collector_plans, "precompute_plan": plan.precompute_plan, "transmission_plan": plan.transmission_plan, - "backend_plan": plan.backend_plan.encode_to_vec(), "query_plan": plan.query_plan, "storage_routing": null, "adaptation_evidence": [] diff --git a/control_plane/examples/compile_workload_artifact.rs b/control_plane/examples/compile_workload_artifact.rs index 23263654..54fd24fb 100644 --- a/control_plane/examples/compile_workload_artifact.rs +++ b/control_plane/examples/compile_workload_artifact.rs @@ -31,7 +31,6 @@ fn main() -> Result<(), Box> { "collector_plans": plan.collector_plans, "precompute_plan": plan.precompute_plan, "transmission_plan": plan.transmission_plan, - "backend_plan": plan.backend_plan.encode_to_vec(), "query_plan": plan.query_plan, "storage_routing": null, "adaptation_evidence": [] diff --git a/control_plane/proto/backend_plan.proto b/control_plane/proto/backend_plan.proto deleted file mode 100644 index 5c56e447..00000000 --- a/control_plane/proto/backend_plan.proto +++ /dev/null @@ -1,198 +0,0 @@ -syntax = "proto3"; - -package control_plane.backend_plan.v1; - -// `BackendPlan` — the typed control-plane -> data-plane wire contract. -// See `control_plane/docs/design-backend-plan-wire-format.md` for the -// full design rationale. This file is the wire schema only; the -// hand-written Rust domain types + conversions live in -// `control_plane/src/backend_plan/mod.rs`. - -// ── Summary family (asap_sketch::SummaryKind / SummaryParams) ─────────────── -// -// `SummaryParams`'s oneof tag already identifies which `SummaryKind` a -// `Materialization` uses -- exact accumulators (Sum/Count/MinMax/Increase/ -// Rate) carry no tuning parameters at all, approximate sketches do. - -message KllParams { uint32 k = 1; } -message CmsParams { uint32 width = 1; uint32 depth = 2; } -message HllParams { uint32 precision = 1; } -message DdSketchParams { double alpha = 1; } -message CmsWithHeapParams { uint32 width = 1; uint32 depth = 2; uint32 heap_size = 3; } -message KmvParams { uint32 k = 1; } -message ThetaParams { uint32 k = 1; } -message CountSketchParams { uint32 width = 1; uint32 depth = 2; } -message CountSketchWithHeapParams { uint32 width = 1; uint32 depth = 2; uint32 heap_size = 3; } - -message SummaryParams { - oneof params { - bool sum = 1; - bool count = 2; - bool min_max = 3; - bool increase = 4; - bool rate = 5; - KllParams kll = 6; - CmsParams cms = 7; - HllParams hll = 8; - DdSketchParams ddsketch = 9; - CmsWithHeapParams cms_with_heap = 10; - KmvParams kmv = 11; - ThetaParams theta = 12; - CountSketchParams count_sketch = 13; - CountSketchWithHeapParams count_sketch_with_heap = 14; - } -} - -// ── Capability (control_plane::physical::post_asap::capability) ────────────────── - -enum SketchKindHandle { - SKETCH_KIND_HANDLE_UNSPECIFIED = 0; - SKETCH_KIND_HANDLE_DDSKETCH = 1; - SKETCH_KIND_HANDLE_KLL = 2; - SKETCH_KIND_HANDLE_HLL = 3; - SKETCH_KIND_HANDLE_COUNT_SKETCH = 4; - SKETCH_KIND_HANDLE_COUNT_MIN = 5; - SKETCH_KIND_HANDLE_CMS_WITH_HEAP = 6; - SKETCH_KIND_HANDLE_COUNT_SKETCH_WITH_HEAP = 7; - SKETCH_KIND_HANDLE_ANY = 8; -} - -// `asap_types::AggregationType` -- the data plane's exact-accumulator-family -// enum `Capability::ExactAgg` names. -enum AggregationType { - AGGREGATION_TYPE_UNSPECIFIED = 0; - AGGREGATION_TYPE_SUM = 1; - AGGREGATION_TYPE_INCREASE = 2; - AGGREGATION_TYPE_MIN_MAX = 3; - AGGREGATION_TYPE_DATASKETCHES_KLL = 4; - AGGREGATION_TYPE_MULTIPLE_SUM = 5; - AGGREGATION_TYPE_MULTIPLE_INCREASE = 6; - AGGREGATION_TYPE_MULTIPLE_MIN_MAX = 7; - AGGREGATION_TYPE_HYDRA_KLL = 8; - AGGREGATION_TYPE_COUNT_MIN_SKETCH = 9; - AGGREGATION_TYPE_COUNT_MIN_SKETCH_WITH_HEAP = 10; - AGGREGATION_TYPE_COUNT_SKETCH = 11; - AGGREGATION_TYPE_COUNT_SKETCH_WITH_HEAP = 12; - AGGREGATION_TYPE_HLL = 13; - AGGREGATION_TYPE_DDSKETCH = 14; - AGGREGATION_TYPE_SINGLE_SUBPOPULATION = 15; - AGGREGATION_TYPE_MULTIPLE_SUBPOPULATION = 16; -} - -message Capability { - oneof capability { - SketchKindHandle quantile_approx = 1; - bool cardinality_approx = 2; - SketchKindHandle frequency_estimate = 3; - SketchKindHandle frequency_topk = 4; - AggregationType exact_agg = 5; - } -} - -// ── L3 IR fragments (asap_ir::intent_algebra) ──────────────────────────────── - -message Source { - oneof source { - string time_series_metric = 1; - string table_ref = 2; - } -} - -message ColumnRef { - oneof column_ref { - string named = 1; - QualifiedColumn qualified = 2; - bool sample_value = 3; - bool wildcard = 4; - } -} - -message QualifiedColumn { - string table = 1; - string name = 2; -} - -enum WindowKind { - WINDOW_KIND_UNSPECIFIED = 0; - WINDOW_KIND_TUMBLING = 1; - WINDOW_KIND_SLIDING = 2; - WINDOW_KIND_SESSION = 3; -} - -message WindowSpec { - WindowKind kind = 1; - uint64 size_ms = 2; - optional uint64 slide_ms = 3; -} - -// ── Deployment-local types (no ASAPController/asap_ir equivalent) ─────────── - -// Mirrors `data_plane::storage_engines::types::StorageBackend`. Kept as its -// own copy rather than a shared dependency -- `control_plane` cannot depend -// on `data_plane` (the dependency runs the other way; see -// `asap_types::MonitorSpec`'s doc for the same constraint on that type). -enum StorageBackend { - STORAGE_BACKEND_UNSPECIFIED = 0; - STORAGE_BACKEND_SKETCH_STORE = 1; - STORAGE_BACKEND_GORILLA_OBJECT_STORE = 2; - STORAGE_BACKEND_DOUBLE_WRITE = 3; - STORAGE_BACKEND_PROMETHEUS_REMOTE = 4; -} - -message RetentionPolicy { - optional uint32 num_aggregates_to_retain = 1; -} - -// Mirrors `asap_types::MonitorSpec`. -message MonitorSpec { - uint64 agg_id = 1; - string functional = 2; - string key = 3; - double tau = 4; - double epsilon = 5; - uint64 window_ms = 6; - uint64 d = 7; - uint64 w = 8; - string mode = 9; -} - -// ── BackendPlan itself ─────────────────────────────────────────────────────── - -message Materialization { - uint64 fingerprint = 1; - Source source = 2; - WindowSpec window = 3; - repeated string group_by = 4; - repeated string rollup = 5; - SummaryParams params = 6; - ColumnRef col = 7; - optional RetentionPolicy retention = 8; - optional SummaryMaintenanceLifecycle lifecycle = 9; - // Canonical label matcher set (sorted `key="value"` entries). - string spatial_filter = 10; -} - -message SummaryMaintenanceLifecycle { - string kind = 1; - string maintenance_mode = 2; - string evaluation_schedule = 3; - string output_representation = 4; -} - -message RoutingEntry { - Capability satisfies = 1; - uint64 materialization = 2; - StorageBackend storage_backend = 3; -} - -message BackendPlan { - uint64 plan_id = 1; - uint64 generated_at_unix_ms = 2; - map materializations = 3; - repeated RoutingEntry routing = 4; - repeated MonitorSpec monitors = 5; - uint64 plan_version = 6; - uint64 activation_unix_ms = 7; - optional uint64 expiry_unix_ms = 8; - string backend_compat = 9; -} diff --git a/control_plane/src/backend_client.rs b/control_plane/src/backend_client.rs index 83ea80ac..2f6a798d 100644 --- a/control_plane/src/backend_client.rs +++ b/control_plane/src/backend_client.rs @@ -702,7 +702,6 @@ mod tests { ] { assert!(bodies[0].get(field).is_some(), "missing {field}"); } - assert!(bodies[0].get("backend_plan").is_none()); server.abort(); } diff --git a/control_plane/src/backend_plan/from_stage_config.rs b/control_plane/src/backend_plan/from_stage_config.rs deleted file mode 100644 index aa7079c4..00000000 --- a/control_plane/src/backend_plan/from_stage_config.rs +++ /dev/null @@ -1,545 +0,0 @@ -//! Build a [`BackendPlan`] from the L5 emitter's [`BackendStageConfig`] — -//! the same input `crate::emit::stage_config::emit_backend_streaming_config_json` -//! consumes to produce the legacy JSON wire format. This is the -//! `BackendPlan`-side sibling of that function; see this crate's design -//! doc (`control_plane/docs/design-backend-plan-wire-format.md`) for why -//! both exist side by side (dual-push, see `backend_client.rs`). -//! -//! **Fingerprint parity is the load-bearing invariant here.** A -//! `Materialization`'s `fingerprint` must equal what `data_plane` -//! independently computes for the equivalent policy today (via -//! `AggregationConfig::from_yaml_data` + `PolicyFingerprint::from_config` -//! on the legacy JSON/YAML `StreamingConfig` push) — `SketchStore` sid -//! registration and any future cross-reference between the two wire -//! formats depend on the two identity spaces staying unified. Rather -//! than re-deriving the field mapping a second time (real drift risk), -//! this module reuses `build_backend_aggregation_json` (the exact same -//! JSON `AggregationConfig::from_yaml_data` parses on the receiving end) -//! and computes the fingerprint from *that*. - -use std::collections::HashMap; - -use anyhow::{Context, Result}; -use asap_types::{MonitorSpec, PolicyFingerprint, PrecomputeMaterialization, QueryLanguage}; - -use crate::emit::monitor::{agg_id_for_metric, MonitorIntent}; -use crate::emit::stage_config::build_backend_aggregation_json; -use crate::physical::colored_dag::emitter::{BackendAggregation, BackendStageConfig}; -use crate::physical::runtime_capability::{Capability, SketchAlgorithm}; -use asap_types::enums::WindowKind; -use planner_types::pre_asap::{ColumnRef, Source}; - -use super::{BackendPlan, Materialization, RoutingEntry, StorageBackend, WindowSpec}; - -/// Build a `BackendPlan` from a planning cycle's `BackendStageConfig` + -/// declared CDM monitors. `plan_id`/`generated_at_unix_ms` are -/// observability-only (see `BackendPlan`'s own doc) — callers typically -/// reuse whatever counter/clock they already thread through the legacy -/// `StreamingConfig` emit path. -pub fn from_stage_config( - cfg: &BackendStageConfig, - monitors: &[MonitorIntent], - plan_id: u64, - generated_at_unix_ms: u64, -) -> Result { - let mut materializations = HashMap::with_capacity(cfg.aggregations.len()); - let mut fingerprint_by_agg_id: HashMap<&str, PolicyFingerprint> = - HashMap::with_capacity(cfg.aggregations.len()); - - for agg in &cfg.aggregations { - let fingerprint = aggregation_config_for_materialization(agg) - .with_context(|| format!("aggregation_id {:?}", agg.aggregation_id))?; - let fingerprint = fingerprint.policy_fingerprint(); - fingerprint_by_agg_id.insert(agg.aggregation_id.as_str(), fingerprint); - - let family = agg.family.clone(); - - materializations.insert( - fingerprint, - Materialization { - fingerprint, - source: Source::TimeSeries { - metric: agg.metric_name.clone(), - }, - window: WindowSpec { - kind: WindowKind::Tumbling, - size_ms: agg.window_secs.saturating_mul(1000), - slide_ms: None, - }, - group_by: agg.grouping.clone(), - rollup: Vec::new(), - spatial_filter: asap_types::utils::normalize_spatial_filter(&agg.spatial_filter), - family, - col: ColumnRef::SampleValue, - retention: None, - lifecycle: None, - }, - ); - } - - // Exact aggregates are already finalized by their accumulator and do not - // have a SketchQuery readout node. Their warm route therefore comes from - // the physical aggregation itself; approximate routes remain readout- - // driven below. - let mut routing = cfg - .aggregations - .iter() - .filter_map(|agg| { - let planner_types::post_asap::SummaryFamilyType::ExactAggregate(kind, _) = &agg.family - else { - return None; - }; - let fingerprint = *fingerprint_by_agg_id.get(agg.aggregation_id.as_str())?; - let agg_type = match kind { - planner_types::post_asap::ExactKind::Sum - | planner_types::post_asap::ExactKind::Count => asap_types::AggregationType::Sum, - planner_types::post_asap::ExactKind::MinMax => asap_types::AggregationType::MinMax, - planner_types::post_asap::ExactKind::Increase - | planner_types::post_asap::ExactKind::Rate => { - asap_types::AggregationType::Increase - } - }; - Some(RoutingEntry { - satisfies: Capability::ExactAgg(agg_type), - materialization: fingerprint, - storage_backend: StorageBackend::SketchStore, - }) - }) - .collect::>(); - routing.reserve(cfg.readouts.len()); - for readout in &cfg.readouts { - let Some(&fingerprint) = fingerprint_by_agg_id.get(readout.aggregation_id.as_str()) else { - // Orphan readout (no matching aggregation in this cycle's - // config) — nothing to route. Same "tolerate, don't error" - // stance the legacy JSON emitter takes toward its own - // readouts list. - continue; - }; - let Some(agg) = cfg - .aggregations - .iter() - .find(|a| a.aggregation_id == readout.aggregation_id) - else { - continue; - }; - let satisfies = capability_for_readout(agg, &readout.op)?; - let route = RoutingEntry { - satisfies, - materialization: fingerprint, - storage_backend: StorageBackend::SketchStore, - }; - if !routing.contains(&route) { - routing.push(route); - } - } - - let plan_monitors = monitors - .iter() - .map(|m| MonitorSpec { - agg_id: agg_id_for_metric(&m.metric), - functional: m.functional.as_str().to_string(), - key: m.key.clone(), - tau: m.tau, - epsilon: m.epsilon, - window_ms: m.window_ms, - d: 0, - w: 0, - mode: String::new(), - }) - .collect(); - - Ok(BackendPlan { - plan_id, - generated_at_unix_ms, - plan_version: 1, - activation_unix_ms: generated_at_unix_ms, - expiry_unix_ms: None, - backend_compat: super::BACKEND_COMPAT.into(), - materializations, - routing, - monitors: plan_monitors, - }) -} - -/// Derive the fingerprint data_plane will independently compute for this -/// aggregation. Round-trips through `build_backend_aggregation_json` + -/// `AggregationConfig::from_yaml_data` — the exact same JSON shape and -/// parser the real `POST /api/v1/streaming-config` handler uses (which -/// parses its body as YAML regardless of declared content-type, since -/// JSON is valid YAML) — rather than re-deriving the field mapping here. -pub fn aggregation_config_for_materialization( - agg: &BackendAggregation, -) -> Result { - let json = build_backend_aggregation_json(agg); - let text = serde_json::to_string(&json).context("serialize synthesized aggregation JSON")?; - let yaml_value: serde_yaml::Value = - serde_yaml::from_str(&text).context("parse synthesized aggregation JSON as YAML")?; - PrecomputeMaterialization::from_yaml_data(&yaml_value, None, QueryLanguage::promql) - .context("build AggregationConfig from synthesized aggregation JSON") -} - -/// Capability this readout satisfies, given the aggregation it reads -/// from. Exact-agg overrides always report `Capability::ExactAgg` -/// (mirrors the wire's `aggregationType` bypass — see -/// `BackendAggregation::agg_type_override`'s doc); otherwise derive from -/// the sketch family + readout op, matching -/// `physical::runtime_capability::Capability`'s variant-per-query-shape -/// design. -fn capability_for_readout( - agg: &BackendAggregation, - op: &planner_types::post_asap::SketchQuery, -) -> Result { - use asap_types::AggregationType; - use planner_types::post_asap::SketchQuery; - - match &agg.family { - planner_types::post_asap::SummaryFamilyType::ExactAggregate(kind, _) => { - let agg_type = match kind { - planner_types::post_asap::ExactKind::Sum => AggregationType::Sum, - // The backend implements exact count with its sum-as-count - // accumulator; `ExactKind::Count` remains the canonical - // planner identity at the domain boundary. - planner_types::post_asap::ExactKind::Count => AggregationType::Sum, - planner_types::post_asap::ExactKind::MinMax => AggregationType::MinMax, - planner_types::post_asap::ExactKind::Increase - | planner_types::post_asap::ExactKind::Rate => AggregationType::Increase, - }; - Ok(Capability::ExactAgg(agg_type)) - } - planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) => { - let handle = sketch_algorithm_handle(kind.algorithm())?; - Ok(match op { - SketchQuery::Quantile { .. } => Capability::QuantileApprox(Some(handle)), - SketchQuery::Cardinality => Capability::CardinalityApprox, - SketchQuery::PointCount { .. } => Capability::FrequencyEstimate(Some(handle)), - SketchQuery::TopK { .. } => Capability::FrequencyTopk(Some(handle)), - }) - } - other => anyhow::bail!("unsupported backend summary family {other:?}"), - } -} - -/// Validate that a Planner `SketchAlgorithm` is implemented by this runtime. -fn sketch_algorithm_handle( - kind: &planner_types::post_asap::SketchAlgorithm, -) -> Result { - use planner_types::post_asap::SketchAlgorithm; - Ok(match kind { - SketchAlgorithm::DDSketch => SketchAlgorithm::DDSketch, - SketchAlgorithm::Kll => SketchAlgorithm::Kll, - SketchAlgorithm::Hll => SketchAlgorithm::Hll, - SketchAlgorithm::CountSketch => SketchAlgorithm::CountSketch, - SketchAlgorithm::Cms => SketchAlgorithm::Cms, - SketchAlgorithm::CmsWithHeap => SketchAlgorithm::CmsWithHeap, - SketchAlgorithm::CountSketchWithHeap => SketchAlgorithm::CountSketchWithHeap, - other => anyhow::bail!("SketchAlgorithm {other:?} is not implemented by this runtime"), - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::physical::colored_dag::emitter::{AggregationInput, BackendReadout}; - use asap_types::{AggregationType, KeyByLabelNames, WindowKind as AsapWindowKind}; - use planner_types::post_asap::{ - GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SketchQuery, SummaryFamilyType, - }; - use std::collections::HashMap as StdHashMap; - - fn agg( - aggregation_id: &str, - metric_name: &str, - algorithm: SketchAlgorithm, - sketch_params: SketchParams, - grouping: Vec, - ) -> BackendAggregation { - BackendAggregation { - aggregation_id: aggregation_id.to_string(), - metric_name: metric_name.to_string(), - family: SummaryFamilyType::Sketch( - SketchKind::new(algorithm, sketch_params), - GroupingStrategy::PerSubpopulationInstance, - ), - window_secs: 60, - spatial_filter: String::new(), - grouping, - item_label: None, - heap_update_mode: None, - aggregation_input: AggregationInput::SketchEnvelope, - } - } - - /// Independently construct the `AggregationConfig` a hand-written - /// (non-JSON-round-trip) reader would build for this fixture, so the - /// parity test doesn't just check the implementation against itself. - fn hand_built_config(agg: &BackendAggregation) -> PrecomputeMaterialization { - let (kind, params) = match &agg.family { - SummaryFamilyType::Sketch(kind, _) => (kind.algorithm(), kind.params()), - other => unreachable!("fixture only uses sketches, got {other:?}"), - }; - let parameters: StdHashMap = match params { - SketchParams::DDSketch { alpha } => { - StdHashMap::from([("alpha".to_string(), serde_json::json!(alpha))]) - } - SketchParams::Hll { precision } => { - StdHashMap::from([("precision".to_string(), serde_json::json!(precision))]) - } - SketchParams::CountSketchWithHeap { width, depth, .. } => StdHashMap::from([ - ("w".to_string(), serde_json::json!(width)), - ("d".to_string(), serde_json::json!(depth)), - ("with_heap".to_string(), serde_json::json!(true)), - ]), - SketchParams::Cms { width, depth } => StdHashMap::from([ - ("w".to_string(), serde_json::json!(width)), - ("d".to_string(), serde_json::json!(depth)), - ]), - other => unreachable!("fixture doesn't exercise {other:?}"), - }; - PrecomputeMaterialization::new( - match kind { - SketchAlgorithm::DDSketch => AggregationType::DDSketch, - SketchAlgorithm::Kll => AggregationType::DatasketchesKLL, - SketchAlgorithm::Hll => AggregationType::HLL, - SketchAlgorithm::Cms => AggregationType::CountMinSketch, - SketchAlgorithm::CmsWithHeap => AggregationType::CountMinSketchWithHeap, - SketchAlgorithm::CountSketch => AggregationType::CountSketch, - SketchAlgorithm::CountSketchWithHeap => AggregationType::CountSketchWithHeap, - _ => unreachable!("fixture only uses sketch-typed kinds"), - }, - String::new(), - parameters, - KeyByLabelNames::new(agg.grouping.clone()), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - agg.window_secs, - agg.window_secs, - AsapWindowKind::Tumbling, - agg.spatial_filter.clone(), - agg.metric_name.clone(), - None, - None, - None, - ) - } - - fn sample_cfg() -> BackendStageConfig { - BackendStageConfig { - aggregations: vec![ - agg( - "agg0", - "http_latency_ms", - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { alpha: 0.01 }, - Vec::new(), - ), - agg( - "agg1", - "http_requests_total", - SketchAlgorithm::Hll, - SketchParams::Hll { precision: 14 }, - vec!["zone".to_string()], - ), - ], - readouts: vec![ - BackendReadout { - aggregation_id: "agg0".into(), - op: SketchQuery::Quantile { q: 0.99 }, - }, - BackendReadout { - aggregation_id: "agg1".into(), - op: SketchQuery::Cardinality, - }, - ], - } - } - - #[test] - fn every_aggregation_produces_exactly_one_materialization() { - let cfg = sample_cfg(); - let plan = from_stage_config(&cfg, &[], 1, 0).expect("build plan"); - assert_eq!(plan.materializations.len(), cfg.aggregations.len()); - } - - #[test] - fn fingerprint_matches_hand_built_aggregation_config() { - let cfg = sample_cfg(); - let plan = from_stage_config(&cfg, &[], 1, 0).expect("build plan"); - - for agg in &cfg.aggregations { - let expected = hand_built_config(agg).policy_fingerprint(); - let m = plan.materializations.get(&expected).unwrap_or_else(|| { - panic!("no materialization for expected fingerprint of {agg:?}") - }); - assert_eq!(m.fingerprint, expected); - } - } - - #[test] - fn sketch_kind_and_params_pass_through_unchanged() { - let cfg = sample_cfg(); - let plan = from_stage_config(&cfg, &[], 1, 0).expect("build plan"); - let ddsketch = plan - .materializations - .values() - .find(|m| matches!(m.source, Source::TimeSeries { ref metric } if metric == "http_latency_ms")) - .expect("ddsketch materialization present"); - assert!(matches!( - &ddsketch.family, - planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) - if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::DDSketch - && kind.params() == &planner_types::post_asap::SketchParams::DDSketch { alpha: 0.01 } - )); - - let hll = plan - .materializations - .values() - .find(|m| matches!(m.source, Source::TimeSeries { ref metric } if metric == "http_requests_total")) - .expect("hll materialization present"); - assert!(matches!( - &hll.family, - planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) - if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::Hll - && kind.params() == &planner_types::post_asap::SketchParams::Hll { precision: 14 } - )); - assert_eq!(hll.group_by, vec!["zone".to_string()]); - } - - #[test] - fn readouts_map_to_routing_entries_pointing_at_the_right_fingerprint() { - let cfg = sample_cfg(); - let plan = from_stage_config(&cfg, &[], 1, 0).expect("build plan"); - assert_eq!(plan.routing.len(), 2); - - let ddsketch_fp = plan - .materializations - .iter() - .find(|(_, m)| matches!(m.source, Source::TimeSeries { ref metric } if metric == "http_latency_ms")) - .map(|(fp, _)| *fp) - .expect("ddsketch fingerprint"); - let quantile_entry = plan - .routing - .iter() - .find(|r| r.materialization == ddsketch_fp) - .expect("routing entry for ddsketch"); - assert_eq!( - quantile_entry.satisfies, - Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch)) - ); - - let hll_fp = plan - .materializations - .iter() - .find(|(_, m)| matches!(m.source, Source::TimeSeries { ref metric } if metric == "http_requests_total")) - .map(|(fp, _)| *fp) - .expect("hll fingerprint"); - let cardinality_entry = plan - .routing - .iter() - .find(|r| r.materialization == hll_fp) - .expect("routing entry for hll"); - assert_eq!(cardinality_entry.satisfies, Capability::CardinalityApprox); - } - - #[test] - fn topk_and_point_count_readouts_map_to_frequency_capabilities() { - let cfg = BackendStageConfig { - aggregations: vec![ - agg( - "agg0", - "endpoint_count", - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 2048, - depth: 5, - heap_size: 10, - }, - Vec::new(), - ), - agg( - "agg1", - "endpoint_hits", - SketchAlgorithm::Cms, - SketchParams::Cms { - width: 4096, - depth: 4, - }, - Vec::new(), - ), - ], - readouts: vec![ - BackendReadout { - aggregation_id: "agg0".into(), - op: SketchQuery::TopK { k: 10 }, - }, - BackendReadout { - aggregation_id: "agg1".into(), - op: SketchQuery::PointCount { - key: ColumnRef::Named("user_42".into()), - value: None, - }, - }, - ], - }; - let plan = from_stage_config(&cfg, &[], 1, 0).expect("build plan"); - let topk_entry = plan.routing.iter().find(|r| { - r.satisfies == Capability::FrequencyTopk(Some(SketchAlgorithm::CountSketchWithHeap)) - }); - assert!( - topk_entry.is_some(), - "expected a FrequencyTopk routing entry: {:?}", - plan.routing - ); - - let freq_entry = plan - .routing - .iter() - .find(|r| r.satisfies == Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms))); - assert!( - freq_entry.is_some(), - "expected a FrequencyEstimate routing entry: {:?}", - plan.routing - ); - } - - #[test] - fn exact_agg_override_reports_exact_agg_capability() { - let a = BackendAggregation { - aggregation_id: "agg0".into(), - metric_name: "http_requests_total".into(), - family: SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Sum, - planner_types::post_asap::ExactParams::Sum, - ), - window_secs: 60, - spatial_filter: String::new(), - grouping: vec!["zone".to_string()], - item_label: None, - heap_update_mode: None, - aggregation_input: AggregationInput::Raw, - }; - let cfg = BackendStageConfig { - aggregations: vec![a], - readouts: vec![BackendReadout { - aggregation_id: "agg0".into(), - op: SketchQuery::Cardinality, // op is irrelevant for override rows - }], - }; - let plan = from_stage_config(&cfg, &[], 1, 0).expect("build plan"); - let (_, m) = plan - .materializations - .iter() - .next() - .expect("one materialization"); - assert!(matches!( - m.family, - planner_types::post_asap::SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Sum, - planner_types::post_asap::ExactParams::Sum - ) - )); - - let entry = &plan.routing[0]; - assert_eq!(entry.satisfies, Capability::ExactAgg(AggregationType::Sum)); - } -} diff --git a/control_plane/src/backend_plan/mod.rs b/control_plane/src/backend_plan/mod.rs deleted file mode 100644 index a834920b..00000000 --- a/control_plane/src/backend_plan/mod.rs +++ /dev/null @@ -1,1240 +0,0 @@ -//! `BackendPlan` — the typed control-plane → data-plane wire contract. -//! -//! See `control_plane/docs/design-backend-plan-wire-format.md` for the -//! full design. This module is the wire-types half only (§3 of that -//! doc): the `BackendPlan`/`Materialization`/`RoutingEntry` domain types, -//! their proto encoding (`proto` submodule, generated from -//! `proto/backend_plan.proto`), and the conversions between them. -//! -//! Deliberately reuses this deployment's existing canonical vocabulary -//! rather than re-encoding it: `planner_types::post_asap::SummaryFamilyType` -//! for the materialization payload (including canonical `ExactKind` and -//! `SketchKind` choices; no backend-owned summary-family enum — -//! see the design doc §3 for why), `asap_ir`/`crate::intent_algebra`'s -//! `Source`/`ColumnRef`/`WindowKind` for the L3 IR fragments, -//! `crate::physical::runtime_capability::Capability` for routing, and -//! `asap_types::{PolicyFingerprint, MonitorSpec}` for the two types -//! already shared with `data_plane` for exactly this cross-crate reason. -//! -//! Not yet wired into `emit/`, `main.rs`'s planning path, or any -//! `data_plane` consumer — see `RoutingIndex` (design doc §4), which is -//! what actually reads this at query time, landing separately. - -pub mod proto { - #![allow(clippy::all)] - include!(concat!( - env!("OUT_DIR"), - "/control_plane.backend_plan.v1.rs" - )); -} - -mod from_stage_config; -pub use from_stage_config::aggregation_config_for_materialization; -pub use from_stage_config::from_stage_config; - -use std::collections::HashMap; - -pub use asap_types::StorageBackend; -use asap_types::{AggregationType, MonitorSpec, PolicyFingerprint}; -use prost::Message as _; -use thiserror::Error; - -pub const BACKEND_COMPAT: &str = "asap-query-backend.v1"; - -use crate::physical::runtime_capability::Capability; -use asap_types::enums::WindowKind; -use planner_types::post_asap::{ - EvaluationSchedule, ExactKind, ExactParams, GroupingStrategy, OutputRepresentation, - SketchAlgorithm, SketchKind, SketchParams, SummaryFamilyType, SummaryMaintenanceLifecycle, - SummaryMaintenanceLifecycleGuarantee, SummaryMaintenanceMode, -}; -use planner_types::pre_asap::{ColumnRef, Source}; - -/// Errors decoding a `BackendPlan` (or one of its parts) from its proto -/// wire form. Encoding (`From<&T> for proto::T`) is always infallible — -/// every domain type is a strict subset of what the wire schema can -/// represent — but decoding a proto message built from *bytes* can -/// always fail (a missing `oneof`, an out-of-range enum value from a -/// future wire version, ...). -#[derive(Debug, Error)] -pub enum DecodeError { - #[error("prost decode failed: {0}")] - Prost(#[from] prost::DecodeError), - #[error("{0} missing its oneof field")] - MissingOneof(&'static str), - #[error("unspecified/unknown enum value {value} for {field}")] - UnknownEnumValue { field: &'static str, value: i32 }, - #[error("unsupported summary-maintenance lifecycle: {0}")] - UnsupportedLifecycle(String), -} - -#[derive(Debug, Error, PartialEq, Eq)] -pub enum ValidationError { - #[error("materialization map key {key} does not match embedded fingerprint {embedded}")] - FingerprintMismatch { key: u64, embedded: u64 }, - #[error("materialization {fingerprint} has a zero-sized window")] - ZeroWindow { fingerprint: u64 }, - #[error("materialization {fingerprint} has a zero slide")] - ZeroSlide { fingerprint: u64 }, - #[error("route references unknown materialization {fingerprint}")] - UnknownMaterialization { fingerprint: u64 }, - #[error("materialization {fingerprint} has mismatched kind/parameters")] - KindParamsMismatch { fingerprint: u64 }, - #[error("route capability is incompatible with materialization {fingerprint}")] - IncompatibleRoute { fingerprint: u64 }, - #[error("stale plan generation: incoming={incoming}, active={active}")] - StaleGeneration { incoming: u64, active: u64 }, - #[error("stale plan version for plan {plan_id}: incoming={incoming}, active={active}")] - StalePlanVersion { - plan_id: u64, - incoming: u64, - active: u64, - }, - #[error("plan {plan_id} version {plan_version} was reused with different content")] - ReusedPlanVersion { plan_id: u64, plan_version: u64 }, - #[error("non-bootstrap plan must have a non-zero plan version")] - ZeroPlanVersion, - #[error("non-bootstrap plan must have an activation time")] - MissingActivation, - #[error("plan expiry {expiry} is not after activation {activation}")] - InvalidExpiry { activation: u64, expiry: u64 }, - #[error("non-bootstrap plan must declare backend compatibility")] - MissingBackendCompat, - #[error("unsupported backend compatibility `{actual}`; expected `{expected}`")] - UnsupportedBackendCompat { - actual: String, - expected: &'static str, - }, -} - -// ── WindowSpec ─────────────────────────────────────────────────────────────── - -/// `kind`/`size`/`slide` triple — mirrors `QueryExpr::Window`'s fields -/// without carrying the rest of that node (a `Materialization` names a -/// window shape, not an L3 subtree). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct WindowSpec { - pub kind: WindowKind, - pub size_ms: u64, - pub slide_ms: Option, -} - -impl From<&WindowSpec> for proto::WindowSpec { - fn from(w: &WindowSpec) -> Self { - proto::WindowSpec { - kind: proto::WindowKind::from(w.kind) as i32, - size_ms: w.size_ms, - slide_ms: w.slide_ms, - } - } -} - -impl TryFrom for WindowSpec { - type Error = DecodeError; - fn try_from(w: proto::WindowSpec) -> Result { - let kind = - proto::WindowKind::try_from(w.kind).map_err(|_| DecodeError::UnknownEnumValue { - field: "WindowSpec.kind", - value: w.kind, - })?; - Ok(WindowSpec { - kind: kind.try_into()?, - size_ms: w.size_ms, - slide_ms: w.slide_ms, - }) - } -} - -impl From for proto::WindowKind { - fn from(k: WindowKind) -> Self { - match k { - WindowKind::Tumbling => proto::WindowKind::Tumbling, - WindowKind::Sliding => proto::WindowKind::Sliding, - WindowKind::Session => proto::WindowKind::Session, - } - } -} - -impl TryFrom for WindowKind { - type Error = DecodeError; - fn try_from(k: proto::WindowKind) -> Result { - match k { - proto::WindowKind::Tumbling => Ok(WindowKind::Tumbling), - proto::WindowKind::Sliding => Ok(WindowKind::Sliding), - proto::WindowKind::Session => Ok(WindowKind::Session), - proto::WindowKind::Unspecified => Err(DecodeError::UnknownEnumValue { - field: "WindowKind", - value: proto::WindowKind::Unspecified as i32, - }), - } - } -} - -// ── Source / ColumnRef ────────────────────────────────────────────────────── - -impl From<&Source> for proto::Source { - fn from(s: &Source) -> Self { - use proto::source::Source as Wire; - let source = match s { - Source::TimeSeries { metric } => Wire::TimeSeriesMetric(metric.clone()), - Source::Table { table_ref } => Wire::TableRef(table_ref.clone()), - }; - proto::Source { - source: Some(source), - } - } -} - -impl TryFrom for Source { - type Error = DecodeError; - fn try_from(s: proto::Source) -> Result { - use proto::source::Source as Wire; - match s.source.ok_or(DecodeError::MissingOneof("Source"))? { - Wire::TimeSeriesMetric(metric) => Ok(Source::TimeSeries { metric }), - Wire::TableRef(table_ref) => Ok(Source::Table { table_ref }), - } - } -} - -impl From<&ColumnRef> for proto::ColumnRef { - fn from(c: &ColumnRef) -> Self { - use proto::column_ref::ColumnRef as Wire; - let column_ref = match c { - ColumnRef::Named(name) => Wire::Named(name.clone()), - ColumnRef::Qualified { table, name } => Wire::Qualified(proto::QualifiedColumn { - table: table.clone(), - name: name.clone(), - }), - ColumnRef::SampleValue => Wire::SampleValue(true), - ColumnRef::Wildcard => Wire::Wildcard(true), - }; - proto::ColumnRef { - column_ref: Some(column_ref), - } - } -} - -impl TryFrom for ColumnRef { - type Error = DecodeError; - fn try_from(c: proto::ColumnRef) -> Result { - use proto::column_ref::ColumnRef as Wire; - match c.column_ref.ok_or(DecodeError::MissingOneof("ColumnRef"))? { - Wire::Named(name) => Ok(ColumnRef::Named(name)), - Wire::Qualified(q) => Ok(ColumnRef::Qualified { - table: q.table, - name: q.name, - }), - Wire::SampleValue(_) => Ok(ColumnRef::SampleValue), - Wire::Wildcard(_) => Ok(ColumnRef::Wildcard), - } - } -} - -// ── SummaryFamilyType wire adapter ────────────────────────────────────────── -// -// The legacy `SummaryParams` protobuf is a single self-describing `oneof`. -// Encoding and decoding keep that wire compatibility at the boundary while -// the domain model carries Planner's canonical `SummaryFamilyType`. - -impl From<&SummaryFamilyType> for proto::SummaryParams { - fn from(family: &SummaryFamilyType) -> Self { - use proto::summary_params::Params as Wire; - let params = match family { - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) => Wire::Sum(true), - SummaryFamilyType::ExactAggregate(ExactKind::Count, ExactParams::Count) => { - Wire::Count(true) - } - SummaryFamilyType::ExactAggregate(ExactKind::MinMax, ExactParams::MinMax) => { - Wire::MinMax(true) - } - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase) => { - Wire::Increase(true) - } - SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) => { - Wire::Rate(true) - } - SummaryFamilyType::Sketch(kind, _) => match kind.params() { - SketchParams::Kll { k } => Wire::Kll(proto::KllParams { k: *k }), - SketchParams::Cms { width, depth } => Wire::Cms(proto::CmsParams { - width: *width, - depth: *depth, - }), - SketchParams::Hll { precision } => Wire::Hll(proto::HllParams { - precision: *precision as u32, - }), - SketchParams::DDSketch { alpha } => { - Wire::Ddsketch(proto::DdSketchParams { alpha: *alpha }) - } - SketchParams::CmsWithHeap { - width, - depth, - heap_size, - } => Wire::CmsWithHeap(proto::CmsWithHeapParams { - width: *width, - depth: *depth, - heap_size: *heap_size, - }), - SketchParams::Kmv { k } => Wire::Kmv(proto::KmvParams { k: *k }), - SketchParams::Theta { k } => Wire::Theta(proto::ThetaParams { k: *k }), - SketchParams::CountSketch { width, depth } => { - Wire::CountSketch(proto::CountSketchParams { - width: *width, - depth: *depth, - }) - } - SketchParams::CountSketchWithHeap { - width, - depth, - heap_size, - } => Wire::CountSketchWithHeap(proto::CountSketchWithHeapParams { - width: *width, - depth: *depth, - heap_size: *heap_size, - }), - }, - other => panic!("BackendPlan cannot encode unsupported summary family {other:?}"), - }; - proto::SummaryParams { - params: Some(params), - } - } -} - -/// Decode the legacy wire `SummaryParams` into Planner's canonical family. -pub fn decode_summary_params(p: proto::SummaryParams) -> Result { - use proto::summary_params::Params as Wire; - let params = p.params.ok_or(DecodeError::MissingOneof("SummaryParams"))?; - let exact = |kind, params| SummaryFamilyType::ExactAggregate(kind, params); - let sketch = |algorithm, params| { - SummaryFamilyType::Sketch( - SketchKind::new(algorithm, params), - GroupingStrategy::PerSubpopulationInstance, - ) - }; - Ok(match params { - Wire::Sum(_) => exact(ExactKind::Sum, ExactParams::Sum), - Wire::Count(_) => exact(ExactKind::Count, ExactParams::Count), - Wire::MinMax(_) => exact(ExactKind::MinMax, ExactParams::MinMax), - Wire::Increase(_) => exact(ExactKind::Increase, ExactParams::Increase), - Wire::Rate(_) => exact(ExactKind::Rate, ExactParams::Rate), - Wire::Kll(k) => sketch(SketchAlgorithm::Kll, SketchParams::Kll { k: k.k }), - Wire::Cms(c) => sketch( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: c.width, - depth: c.depth, - }, - ), - Wire::Hll(h) => sketch( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: h.precision as u8, - }, - ), - Wire::Ddsketch(d) => sketch( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { alpha: d.alpha }, - ), - Wire::CmsWithHeap(c) => sketch( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: c.width, - depth: c.depth, - heap_size: c.heap_size, - }, - ), - Wire::Kmv(k) => sketch(SketchAlgorithm::Kmv, SketchParams::Kmv { k: k.k }), - Wire::Theta(t) => sketch(SketchAlgorithm::Theta, SketchParams::Theta { k: t.k }), - Wire::CountSketch(c) => sketch( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { - width: c.width, - depth: c.depth, - }, - ), - Wire::CountSketchWithHeap(c) => sketch( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: c.width, - depth: c.depth, - heap_size: c.heap_size, - }, - ), - }) -} - -// ── SketchAlgorithm / AggregationType / Capability ───────────────────────── - -impl From> for proto::SketchKindHandle { - fn from(h: Option) -> Self { - match h { - Some(SketchAlgorithm::DDSketch) => Self::Ddsketch, - Some(SketchAlgorithm::Kll) => Self::Kll, - Some(SketchAlgorithm::Hll) => Self::Hll, - Some(SketchAlgorithm::CountSketch) => Self::CountSketch, - Some(SketchAlgorithm::Cms) => Self::CountMin, - Some(SketchAlgorithm::CmsWithHeap) => Self::CmsWithHeap, - Some(SketchAlgorithm::CountSketchWithHeap) => Self::CountSketchWithHeap, - Some(SketchAlgorithm::Kmv | SketchAlgorithm::Theta) | None => Self::Any, - } - } -} - -impl TryFrom for Option { - type Error = DecodeError; - fn try_from(h: proto::SketchKindHandle) -> Result { - match h { - proto::SketchKindHandle::Ddsketch => Ok(Some(SketchAlgorithm::DDSketch)), - proto::SketchKindHandle::Kll => Ok(Some(SketchAlgorithm::Kll)), - proto::SketchKindHandle::Hll => Ok(Some(SketchAlgorithm::Hll)), - proto::SketchKindHandle::CountSketch => Ok(Some(SketchAlgorithm::CountSketch)), - proto::SketchKindHandle::CountMin => Ok(Some(SketchAlgorithm::Cms)), - proto::SketchKindHandle::CmsWithHeap => Ok(Some(SketchAlgorithm::CmsWithHeap)), - proto::SketchKindHandle::CountSketchWithHeap => { - Ok(Some(SketchAlgorithm::CountSketchWithHeap)) - } - proto::SketchKindHandle::Any => Ok(None), - proto::SketchKindHandle::Unspecified => Err(DecodeError::UnknownEnumValue { - field: "SketchKindHandle", - value: proto::SketchKindHandle::Unspecified as i32, - }), - } - } -} - -impl From for proto::AggregationType { - fn from(a: AggregationType) -> Self { - match a { - AggregationType::Sum => proto::AggregationType::Sum, - AggregationType::Increase => proto::AggregationType::Increase, - AggregationType::MinMax => proto::AggregationType::MinMax, - AggregationType::DatasketchesKLL => proto::AggregationType::DatasketchesKll, - AggregationType::MultipleSum => proto::AggregationType::MultipleSum, - AggregationType::MultipleIncrease => proto::AggregationType::MultipleIncrease, - AggregationType::MultipleMinMax => proto::AggregationType::MultipleMinMax, - AggregationType::HydraKLL => proto::AggregationType::HydraKll, - AggregationType::CountMinSketch => proto::AggregationType::CountMinSketch, - AggregationType::CountMinSketchWithHeap => { - proto::AggregationType::CountMinSketchWithHeap - } - AggregationType::CountSketch => proto::AggregationType::CountSketch, - AggregationType::CountSketchWithHeap => proto::AggregationType::CountSketchWithHeap, - AggregationType::HLL => proto::AggregationType::Hll, - AggregationType::DDSketch => proto::AggregationType::Ddsketch, - AggregationType::SingleSubpopulation => proto::AggregationType::SingleSubpopulation, - AggregationType::MultipleSubpopulation => proto::AggregationType::MultipleSubpopulation, - } - } -} - -impl TryFrom for AggregationType { - type Error = DecodeError; - fn try_from(a: proto::AggregationType) -> Result { - match a { - proto::AggregationType::Sum => Ok(AggregationType::Sum), - proto::AggregationType::Increase => Ok(AggregationType::Increase), - proto::AggregationType::MinMax => Ok(AggregationType::MinMax), - proto::AggregationType::DatasketchesKll => Ok(AggregationType::DatasketchesKLL), - proto::AggregationType::MultipleSum => Ok(AggregationType::MultipleSum), - proto::AggregationType::MultipleIncrease => Ok(AggregationType::MultipleIncrease), - proto::AggregationType::MultipleMinMax => Ok(AggregationType::MultipleMinMax), - proto::AggregationType::HydraKll => Ok(AggregationType::HydraKLL), - proto::AggregationType::CountMinSketch => Ok(AggregationType::CountMinSketch), - proto::AggregationType::CountMinSketchWithHeap => { - Ok(AggregationType::CountMinSketchWithHeap) - } - proto::AggregationType::CountSketch => Ok(AggregationType::CountSketch), - proto::AggregationType::CountSketchWithHeap => Ok(AggregationType::CountSketchWithHeap), - proto::AggregationType::Hll => Ok(AggregationType::HLL), - proto::AggregationType::Ddsketch => Ok(AggregationType::DDSketch), - proto::AggregationType::SingleSubpopulation => Ok(AggregationType::SingleSubpopulation), - proto::AggregationType::MultipleSubpopulation => { - Ok(AggregationType::MultipleSubpopulation) - } - proto::AggregationType::Unspecified => Err(DecodeError::UnknownEnumValue { - field: "AggregationType", - value: proto::AggregationType::Unspecified as i32, - }), - } - } -} - -impl From<&Capability> for proto::Capability { - fn from(c: &Capability) -> Self { - use proto::capability::Capability as Wire; - let capability = match c { - Capability::QuantileApprox(h) => { - Wire::QuantileApprox(proto::SketchKindHandle::from(h.clone()) as i32) - } - Capability::CardinalityApprox => Wire::CardinalityApprox(true), - Capability::FrequencyEstimate(h) => { - Wire::FrequencyEstimate(proto::SketchKindHandle::from(h.clone()) as i32) - } - Capability::FrequencyTopk(h) => { - Wire::FrequencyTopk(proto::SketchKindHandle::from(h.clone()) as i32) - } - Capability::ExactAgg(a) => Wire::ExactAgg(proto::AggregationType::from(*a) as i32), - }; - proto::Capability { - capability: Some(capability), - } - } -} - -impl TryFrom for Capability { - type Error = DecodeError; - fn try_from(c: proto::Capability) -> Result { - use proto::capability::Capability as Wire; - let decode_handle = - |v: i32, field: &'static str| -> Result, DecodeError> { - proto::SketchKindHandle::try_from(v) - .map_err(|_| DecodeError::UnknownEnumValue { field, value: v })? - .try_into() - }; - match c - .capability - .ok_or(DecodeError::MissingOneof("Capability"))? - { - Wire::QuantileApprox(v) => Ok(Capability::QuantileApprox(decode_handle( - v, - "Capability.quantile_approx", - )?)), - Wire::CardinalityApprox(_) => Ok(Capability::CardinalityApprox), - Wire::FrequencyEstimate(v) => Ok(Capability::FrequencyEstimate(decode_handle( - v, - "Capability.frequency_estimate", - )?)), - Wire::FrequencyTopk(v) => Ok(Capability::FrequencyTopk(decode_handle( - v, - "Capability.frequency_topk", - )?)), - Wire::ExactAgg(v) => { - let agg = proto::AggregationType::try_from(v).map_err(|_| { - DecodeError::UnknownEnumValue { - field: "Capability.exact_agg", - value: v, - } - })?; - Ok(Capability::ExactAgg(agg.try_into()?)) - } - } - } -} - -// ── StorageBackend (deployment-local; control_plane cannot depend on -// data_plane::storage_engines::types::StorageBackend, which this -// mirrors -- see that type's own doc and asap_types::MonitorSpec's for -// the same constraint) ─────────────────────────────────────────────────── - -impl From for proto::StorageBackend { - fn from(b: StorageBackend) -> Self { - match b { - StorageBackend::SketchStore => proto::StorageBackend::SketchStore, - StorageBackend::GorillaObjectStore => proto::StorageBackend::GorillaObjectStore, - StorageBackend::DoubleWrite => proto::StorageBackend::DoubleWrite, - StorageBackend::PrometheusRemote => proto::StorageBackend::PrometheusRemote, - } - } -} - -impl TryFrom for StorageBackend { - type Error = DecodeError; - fn try_from(b: proto::StorageBackend) -> Result { - match b { - proto::StorageBackend::SketchStore => Ok(StorageBackend::SketchStore), - proto::StorageBackend::GorillaObjectStore => Ok(StorageBackend::GorillaObjectStore), - proto::StorageBackend::DoubleWrite => Ok(StorageBackend::DoubleWrite), - proto::StorageBackend::PrometheusRemote => Ok(StorageBackend::PrometheusRemote), - proto::StorageBackend::Unspecified => Err(DecodeError::UnknownEnumValue { - field: "StorageBackend", - value: proto::StorageBackend::Unspecified as i32, - }), - } - } -} - -// ── RetentionPolicy ────────────────────────────────────────────────────────── - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub struct RetentionPolicy { - pub num_aggregates_to_retain: Option, -} - -impl From<&RetentionPolicy> for proto::RetentionPolicy { - fn from(r: &RetentionPolicy) -> Self { - proto::RetentionPolicy { - num_aggregates_to_retain: r.num_aggregates_to_retain, - } - } -} - -impl From for RetentionPolicy { - fn from(r: proto::RetentionPolicy) -> Self { - RetentionPolicy { - num_aggregates_to_retain: r.num_aggregates_to_retain, - } - } -} - -// ── MonitorSpec ────────────────────────────────────────────────────────────── - -impl From<&MonitorSpec> for proto::MonitorSpec { - fn from(m: &MonitorSpec) -> Self { - proto::MonitorSpec { - agg_id: m.agg_id, - functional: m.functional.clone(), - key: m.key.clone(), - tau: m.tau, - epsilon: m.epsilon, - window_ms: m.window_ms, - d: m.d as u64, - w: m.w as u64, - mode: m.mode.clone(), - } - } -} - -impl From for MonitorSpec { - fn from(m: proto::MonitorSpec) -> Self { - MonitorSpec { - agg_id: m.agg_id, - functional: m.functional, - key: m.key, - tau: m.tau, - epsilon: m.epsilon, - window_ms: m.window_ms, - d: m.d as usize, - w: m.w as usize, - mode: m.mode, - } - } -} - -// ── Materialization ────────────────────────────────────────────────────────── - -/// One materialization `control_plane` has decided `data_plane` should -/// build/maintain. `kind`/`params` together are the exact pair -/// `SummaryExecutor::find_candidates` matches on — see this crate's -/// design doc §3 for why there's no separate exact-vs-approximate arm. -#[derive(Debug, Clone, PartialEq)] -pub struct Materialization { - pub fingerprint: PolicyFingerprint, - pub source: Source, - pub window: WindowSpec, - pub group_by: Vec, - pub rollup: Vec, - pub spatial_filter: String, - pub family: SummaryFamilyType, - pub col: ColumnRef, - pub retention: Option, - pub lifecycle: Option, -} - -impl From<&SummaryMaintenanceLifecycleGuarantee> for proto::SummaryMaintenanceLifecycle { - fn from(value: &SummaryMaintenanceLifecycleGuarantee) -> Self { - Self { - kind: match value.summary_maintenance_lifecycle { - SummaryMaintenanceLifecycle::Ephemeral => "ephemeral", - SummaryMaintenanceLifecycle::Prepared { .. } => "prepared", - SummaryMaintenanceLifecycle::Shared { .. } => "shared", - SummaryMaintenanceLifecycle::ContinuouslyMaintained => "continuously_maintained", - } - .into(), - maintenance_mode: value.summary_maintenance_mode.as_str().into(), - evaluation_schedule: match value.evaluation_schedule { - EvaluationSchedule::OneShot => "one_shot", - EvaluationSchedule::PerUpdate => "per_update", - EvaluationSchedule::OnRead => "on_read", - } - .into(), - output_representation: match value.output_representation { - OutputRepresentation::PlainRows => "plain_rows", - OutputRepresentation::SummaryState => "summary_state", - OutputRepresentation::FinalizedValue => "finalized_value", - } - .into(), - } - } -} - -impl TryFrom for SummaryMaintenanceLifecycleGuarantee { - type Error = DecodeError; - - fn try_from(value: proto::SummaryMaintenanceLifecycle) -> Result { - let summary_maintenance_lifecycle = match value.kind.as_str() { - "ephemeral" => SummaryMaintenanceLifecycle::Ephemeral, - "continuously_maintained" => SummaryMaintenanceLifecycle::ContinuouslyMaintained, - // Prepared/Shared carry timestamps/retention that the v1 wire message cannot - // represent. Reject them instead of manufacturing a lossy Planner value. - other => return Err(DecodeError::UnsupportedLifecycle(other.into())), - }; - let summary_maintenance_mode = match value.maintenance_mode.as_str() { - "direct_build" => SummaryMaintenanceMode::DirectBuild, - "incremental" => SummaryMaintenanceMode::Incremental, - other => return Err(DecodeError::UnsupportedLifecycle(other.into())), - }; - let evaluation_schedule = match value.evaluation_schedule.as_str() { - "one_shot" => EvaluationSchedule::OneShot, - "per_update" => EvaluationSchedule::PerUpdate, - "on_read" => EvaluationSchedule::OnRead, - other => return Err(DecodeError::UnsupportedLifecycle(other.into())), - }; - let output_representation = match value.output_representation.as_str() { - "plain_rows" => OutputRepresentation::PlainRows, - "summary_state" => OutputRepresentation::SummaryState, - "finalized_value" => OutputRepresentation::FinalizedValue, - other => return Err(DecodeError::UnsupportedLifecycle(other.into())), - }; - Ok(Self { - summary_maintenance_lifecycle, - summary_maintenance_mode, - evaluation_schedule, - output_representation, - }) - } -} - -impl From<&Materialization> for proto::Materialization { - fn from(m: &Materialization) -> Self { - proto::Materialization { - fingerprint: m.fingerprint.0, - source: Some((&m.source).into()), - window: Some((&m.window).into()), - group_by: m.group_by.clone(), - rollup: m.rollup.clone(), - spatial_filter: m.spatial_filter.clone(), - params: Some((&m.family).into()), - col: Some((&m.col).into()), - retention: m.retention.as_ref().map(Into::into), - lifecycle: m.lifecycle.as_ref().map(Into::into), - } - } -} - -impl TryFrom for Materialization { - type Error = DecodeError; - fn try_from(m: proto::Materialization) -> Result { - let family = decode_summary_params( - m.params - .ok_or(DecodeError::MissingOneof("Materialization.params"))?, - )?; - let lifecycle = m.lifecycle.map(TryInto::try_into).transpose()?; - Ok(Materialization { - fingerprint: PolicyFingerprint(m.fingerprint), - source: m - .source - .ok_or(DecodeError::MissingOneof("Materialization.source"))? - .try_into()?, - window: m - .window - .ok_or(DecodeError::MissingOneof("Materialization.window"))? - .try_into()?, - group_by: m.group_by, - rollup: m.rollup, - spatial_filter: m.spatial_filter, - family, - col: m - .col - .ok_or(DecodeError::MissingOneof("Materialization.col"))? - .try_into()?, - retention: m.retention.map(Into::into), - lifecycle, - }) - } -} - -// ── RoutingEntry ───────────────────────────────────────────────────────────── - -/// One "this capability, for this metric, is answered by that -/// materialization" fact. Kept as its own table rather than folded into -/// `Materialization` 1:1 — see design doc §3. -#[derive(Debug, Clone, PartialEq)] -pub struct RoutingEntry { - pub satisfies: Capability, - pub materialization: PolicyFingerprint, - pub storage_backend: StorageBackend, -} - -impl From<&RoutingEntry> for proto::RoutingEntry { - fn from(r: &RoutingEntry) -> Self { - proto::RoutingEntry { - satisfies: Some((&r.satisfies).into()), - materialization: r.materialization.0, - storage_backend: proto::StorageBackend::from(r.storage_backend) as i32, - } - } -} - -impl TryFrom for RoutingEntry { - type Error = DecodeError; - fn try_from(r: proto::RoutingEntry) -> Result { - let storage_backend = proto::StorageBackend::try_from(r.storage_backend).map_err(|_| { - DecodeError::UnknownEnumValue { - field: "RoutingEntry.storage_backend", - value: r.storage_backend, - } - })?; - Ok(RoutingEntry { - satisfies: r - .satisfies - .ok_or(DecodeError::MissingOneof("RoutingEntry.satisfies"))? - .try_into()?, - materialization: PolicyFingerprint(r.materialization), - storage_backend: storage_backend.try_into()?, - }) - } -} - -// ── BackendPlan ────────────────────────────────────────────────────────────── - -/// The message `control_plane` pushes to `data_plane`'s backend process. -/// See this module's doc + the design doc for the full rationale. -#[derive(Debug, Clone, PartialEq, Default)] -pub struct BackendPlan { - /// Observability only, not identity (mirrors `plan_streaming_config`'s - /// existing convention for the pre-`BackendPlan` wire format). - pub plan_id: u64, - pub generated_at_unix_ms: u64, - pub plan_version: u64, - pub activation_unix_ms: u64, - pub expiry_unix_ms: Option, - pub backend_compat: String, - pub materializations: HashMap, - pub routing: Vec, - pub monitors: Vec, -} - -impl From<&BackendPlan> for proto::BackendPlan { - fn from(p: &BackendPlan) -> Self { - proto::BackendPlan { - plan_id: p.plan_id, - generated_at_unix_ms: p.generated_at_unix_ms, - plan_version: p.plan_version, - activation_unix_ms: p.activation_unix_ms, - expiry_unix_ms: p.expiry_unix_ms, - backend_compat: p.backend_compat.clone(), - materializations: p - .materializations - .iter() - .map(|(fp, m)| (fp.0, m.into())) - .collect(), - routing: p.routing.iter().map(Into::into).collect(), - monitors: p.monitors.iter().map(Into::into).collect(), - } - } -} - -impl TryFrom for BackendPlan { - type Error = DecodeError; - fn try_from(p: proto::BackendPlan) -> Result { - let materializations = p - .materializations - .into_iter() - .map(|(fp, m)| Ok((PolicyFingerprint(fp), Materialization::try_from(m)?))) - .collect::, DecodeError>>()?; - let routing = p - .routing - .into_iter() - .map(RoutingEntry::try_from) - .collect::, _>>()?; - Ok(BackendPlan { - plan_id: p.plan_id, - generated_at_unix_ms: p.generated_at_unix_ms, - plan_version: p.plan_version, - activation_unix_ms: p.activation_unix_ms, - expiry_unix_ms: p.expiry_unix_ms, - backend_compat: p.backend_compat, - materializations, - routing, - monitors: p.monitors.into_iter().map(Into::into).collect(), - }) - } -} - -impl BackendPlan { - /// Encode to the wire-format bytes `data_plane` consumes. - pub fn encode_to_vec(&self) -> Vec { - proto::BackendPlan::from(self).encode_to_vec() - } - - /// Decode from wire bytes. `Err` covers both a malformed proto - /// stream and a structurally-incomplete message (a missing `oneof`, - /// an enum value this build doesn't recognize). - pub fn decode(bytes: &[u8]) -> Result { - BackendPlan::try_from(proto::BackendPlan::decode(bytes)?) - } - - /// Validate cross-references and invariants required before a decoded - /// plan may become visible to ingest or query readers. - pub fn validate(&self) -> Result<(), ValidationError> { - if self.plan_id != 0 { - if self.plan_version == 0 { - return Err(ValidationError::ZeroPlanVersion); - } - if self.activation_unix_ms == 0 { - return Err(ValidationError::MissingActivation); - } - if self.backend_compat.trim().is_empty() { - return Err(ValidationError::MissingBackendCompat); - } - if self.backend_compat != BACKEND_COMPAT { - return Err(ValidationError::UnsupportedBackendCompat { - actual: self.backend_compat.clone(), - expected: BACKEND_COMPAT, - }); - } - if let Some(expiry) = self.expiry_unix_ms { - if expiry <= self.activation_unix_ms { - return Err(ValidationError::InvalidExpiry { - activation: self.activation_unix_ms, - expiry, - }); - } - } - } - for (key, materialization) in &self.materializations { - if *key != materialization.fingerprint { - return Err(ValidationError::FingerprintMismatch { - key: key.0, - embedded: materialization.fingerprint.0, - }); - } - if materialization.window.size_ms == 0 { - return Err(ValidationError::ZeroWindow { fingerprint: key.0 }); - } - if materialization.window.slide_ms == Some(0) { - return Err(ValidationError::ZeroSlide { fingerprint: key.0 }); - } - if !summary_family_is_valid(&materialization.family) { - return Err(ValidationError::KindParamsMismatch { fingerprint: key.0 }); - } - } - for route in &self.routing { - let Some(materialization) = self.materializations.get(&route.materialization) else { - return Err(ValidationError::UnknownMaterialization { - fingerprint: route.materialization.0, - }); - }; - if !materialization_satisfies(&route.satisfies, materialization) { - return Err(ValidationError::IncompatibleRoute { - fingerprint: route.materialization.0, - }); - } - } - Ok(()) - } -} - -fn summary_family_is_valid(family: &SummaryFamilyType) -> bool { - matches!( - family, - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - | SummaryFamilyType::ExactAggregate(ExactKind::Count, ExactParams::Count) - | SummaryFamilyType::ExactAggregate(ExactKind::MinMax, ExactParams::MinMax) - | SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase) - | SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) - | SummaryFamilyType::Sketch(..) - ) -} - -fn materialization_satisfies(required: &Capability, m: &Materialization) -> bool { - let available = match &m.family { - SummaryFamilyType::Sketch(kind, _) => match kind.algorithm() { - SketchAlgorithm::DDSketch => { - Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch)) - } - SketchAlgorithm::Kll => Capability::QuantileApprox(Some(SketchAlgorithm::Kll)), - SketchAlgorithm::Hll => Capability::CardinalityApprox, - SketchAlgorithm::Cms => Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms)), - SketchAlgorithm::CountSketch => { - Capability::FrequencyEstimate(Some(SketchAlgorithm::CountSketch)) - } - SketchAlgorithm::CmsWithHeap => { - Capability::FrequencyTopk(Some(SketchAlgorithm::CmsWithHeap)) - } - SketchAlgorithm::CountSketchWithHeap => { - Capability::FrequencyTopk(Some(SketchAlgorithm::CountSketchWithHeap)) - } - SketchAlgorithm::Kmv | SketchAlgorithm::Theta => return false, - }, - SummaryFamilyType::ExactAggregate(ExactKind::Sum, _) => { - Capability::ExactAgg(AggregationType::Sum) - } - SummaryFamilyType::ExactAggregate(ExactKind::MinMax, _) => { - Capability::ExactAgg(AggregationType::MinMax) - } - SummaryFamilyType::ExactAggregate(ExactKind::Increase, _) => { - Capability::ExactAgg(AggregationType::Increase) - } - SummaryFamilyType::ExactAggregate(ExactKind::Count | ExactKind::Rate, _) => return false, - _ => return false, - }; - required.is_satisfied_by(&available) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::time::Duration; - - fn sample_materialization(fingerprint: u64, family: SummaryFamilyType) -> Materialization { - Materialization { - fingerprint: PolicyFingerprint(fingerprint), - source: Source::TimeSeries { - metric: "http_requests_total".to_string(), - }, - window: WindowSpec { - kind: WindowKind::Sliding, - size_ms: Duration::from_secs(300).as_millis() as u64, - slide_ms: None, - }, - group_by: vec!["zone".to_string()], - rollup: vec![], - spatial_filter: String::new(), - family, - col: ColumnRef::SampleValue, - retention: Some(RetentionPolicy { - num_aggregates_to_retain: Some(1000), - }), - lifecycle: Some(SummaryMaintenanceLifecycleGuarantee { - summary_maintenance_lifecycle: SummaryMaintenanceLifecycle::ContinuouslyMaintained, - summary_maintenance_mode: SummaryMaintenanceMode::Incremental, - evaluation_schedule: EvaluationSchedule::PerUpdate, - output_representation: OutputRepresentation::SummaryState, - }), - } - } - - fn exact(kind: ExactKind, params: ExactParams) -> SummaryFamilyType { - SummaryFamilyType::ExactAggregate(kind, params) - } - - fn sketch(algorithm: SketchAlgorithm, params: SketchParams) -> SummaryFamilyType { - SummaryFamilyType::Sketch( - SketchKind::new(algorithm, params), - GroupingStrategy::PerSubpopulationInstance, - ) - } - - fn sample_plan() -> BackendPlan { - let mut materializations = HashMap::new(); - // Approximate sketch. - materializations.insert( - PolicyFingerprint(1), - sample_materialization( - 1, - sketch(SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), - ), - ); - // Exact accumulator -- same `SummaryKind`/`SummaryParams` vocabulary, - // no separate wire representation (design doc §3). - materializations.insert( - PolicyFingerprint(2), - sample_materialization(2, exact(ExactKind::Sum, ExactParams::Sum)), - ); - - BackendPlan { - plan_id: 42, - generated_at_unix_ms: 1_735_000_000_000, - plan_version: 1, - activation_unix_ms: 1_735_000_000_000, - expiry_unix_ms: None, - backend_compat: "asap-query-backend.v1".into(), - materializations, - routing: vec![ - RoutingEntry { - satisfies: Capability::QuantileApprox(None), - materialization: PolicyFingerprint(1), - storage_backend: StorageBackend::SketchStore, - }, - RoutingEntry { - satisfies: Capability::ExactAgg(AggregationType::Sum), - materialization: PolicyFingerprint(2), - storage_backend: StorageBackend::SketchStore, - }, - ], - monitors: vec![MonitorSpec { - agg_id: 2, - functional: "sum".to_string(), - key: String::new(), - tau: 100.0, - epsilon: 0.05, - window_ms: 60_000, - d: 0, - w: 0, - mode: String::new(), - }], - } - } - - #[test] - fn ephemeral_lifecycle_round_trips() { - let mut plan = sample_plan(); - plan.materializations - .values_mut() - .next() - .unwrap() - .lifecycle - .as_mut() - .unwrap() - .summary_maintenance_lifecycle = SummaryMaintenanceLifecycle::Ephemeral; - let decoded = BackendPlan::decode(&plan.encode_to_vec()).expect("decode"); - assert_eq!(decoded, plan); - } - - #[test] - fn round_trips_a_plan_with_both_exact_and_approximate_materializations() { - let plan = sample_plan(); - let bytes = plan.encode_to_vec(); - let decoded = BackendPlan::decode(&bytes).expect("decode must succeed"); - assert_eq!(decoded, plan); - } - - #[test] - fn kll_survives_round_trip_with_kind_and_params_agreeing() { - let m = sample_materialization( - 1, - sketch(SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), - ); - let wire = proto::Materialization::from(&m); - let back = Materialization::try_from(wire).expect("decode must succeed"); - assert_eq!(back.family, m.family); - assert_eq!(back, m); - } - - #[test] - fn sum_is_exact_and_carries_no_tuning_parameters() { - let m = sample_materialization(2, exact(ExactKind::Sum, ExactParams::Sum)); - let wire = proto::Materialization::from(&m); - let back = Materialization::try_from(wire).expect("decode must succeed"); - assert!(matches!(back.family, SummaryFamilyType::ExactAggregate(..))); - assert_eq!(back, m); - } - - #[test] - fn every_summary_kind_round_trips() { - let cases = vec![ - exact(ExactKind::Sum, ExactParams::Sum), - exact(ExactKind::Count, ExactParams::Count), - exact(ExactKind::MinMax, ExactParams::MinMax), - exact(ExactKind::Increase, ExactParams::Increase), - exact(ExactKind::Rate, ExactParams::Rate), - sketch(SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), - sketch( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: 64, - depth: 4, - }, - ), - sketch(SketchAlgorithm::Hll, SketchParams::Hll { precision: 14 }), - sketch( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { alpha: 0.01 }, - ), - sketch( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 64, - depth: 4, - heap_size: 10, - }, - ), - sketch(SketchAlgorithm::Kmv, SketchParams::Kmv { k: 1024 }), - sketch(SketchAlgorithm::Theta, SketchParams::Theta { k: 1024 }), - sketch( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { - width: 64, - depth: 4, - }, - ), - sketch( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 64, - depth: 4, - heap_size: 10, - }, - ), - ]; - for family in cases { - let wire = proto::SummaryParams::from(&family); - let decoded = decode_summary_params(wire).expect("decode must succeed"); - assert_eq!(decoded, family); - } - } - - #[test] - fn every_capability_variant_round_trips() { - let cases = [ - Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch)), - Capability::QuantileApprox(None), - Capability::CardinalityApprox, - Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms)), - Capability::FrequencyTopk(Some(SketchAlgorithm::CmsWithHeap)), - Capability::ExactAgg(AggregationType::Sum), - Capability::ExactAgg(AggregationType::MultipleSubpopulation), - ]; - for cap in cases { - let wire = proto::Capability::from(&cap); - let back = Capability::try_from(wire).expect("decode must succeed"); - assert_eq!(back, cap, "round-trip mismatch for {cap:?}"); - } - } - - #[test] - fn every_storage_backend_round_trips() { - for backend in [ - StorageBackend::SketchStore, - StorageBackend::GorillaObjectStore, - StorageBackend::DoubleWrite, - StorageBackend::PrometheusRemote, - ] { - let wire = proto::StorageBackend::from(backend); - let back = StorageBackend::try_from(wire).expect("decode must succeed"); - assert_eq!(back, backend); - } - } - - #[test] - fn source_and_column_ref_round_trip() { - let sources = [ - Source::TimeSeries { - metric: "m".to_string(), - }, - Source::Table { - table_ref: "t".to_string(), - }, - ]; - for s in sources { - let wire = proto::Source::from(&s); - let back = Source::try_from(wire).expect("decode must succeed"); - assert_eq!(back, s); - } - - let columns = [ - ColumnRef::Named("service".to_string()), - ColumnRef::Qualified { - table: "t".to_string(), - name: "c".to_string(), - }, - ColumnRef::SampleValue, - ColumnRef::Wildcard, - ]; - for c in columns { - let wire = proto::ColumnRef::from(&c); - let back = ColumnRef::try_from(wire).expect("decode must succeed"); - assert_eq!(back, c); - } - } - - #[test] - fn missing_oneof_decodes_to_an_error_not_a_panic() { - let empty = proto::SummaryParams { params: None }; - assert!(decode_summary_params(empty).is_err()); - - let empty_cap = proto::Capability { capability: None }; - assert!(Capability::try_from(empty_cap).is_err()); - } -} diff --git a/control_plane/src/emit/backend_push.rs b/control_plane/src/emit/backend_push.rs index ebf80ee5..f6fbd974 100644 --- a/control_plane/src/emit/backend_push.rs +++ b/control_plane/src/emit/backend_push.rs @@ -75,9 +75,8 @@ const RETRY_MAX_ATTEMPTS: u32 = 5; const RETRY_BASE_DELAY: Duration = Duration::from_millis(100); const RETRY_DELAY_CAP: Duration = Duration::from_millis(2700); -/// Monotonic counter for `BackendPlan.plan_id` — observability only, not -/// identity (see `BackendPlan`'s own doc). One process-wide sequence is -/// enough; there's no existing streaming-config version counter to +/// Monotonic plan identity for compatibility-replanner publications. One +/// process-wide sequence is enough; there's no existing streaming-config counter to /// reuse for parity. static PLAN_ID_COUNTER: AtomicU64 = AtomicU64::new(1); @@ -187,10 +186,7 @@ pub type BackendRoutingCache = Mutex, - monitors: &[crate::emit::monitor::MonitorIntent], + _monitors: &[crate::emit::monitor::MonitorIntent], ) -> PushOutcome { // ── Build BOTH cumulative documents up front (P2-3) ─────────────────── // @@ -442,25 +431,8 @@ async fn push_cumulative_entries( .flat_map(|(_, c)| c.readouts.iter().cloned()) .collect(), }; - // BackendPlan (design-backend-plan-wire-format.md): built from the - // SAME `cumulative_be` snapshot as the legacy documents above, so all - // three describe one consistent generation of planning state. Until - // BackendPlan fully replaces the compatibility documents, publication - // succeeds only when all three are accepted. let plan_id = PLAN_ID_COUNTER.fetch_add(1, Ordering::Relaxed); let generated_at_unix_ms = now_unix_ms(); - let backend_plan = match crate::backend_plan::from_stage_config( - &cumulative_be, - monitors, - plan_id, - generated_at_unix_ms, - ) { - Ok(plan) => plan, - Err(e) => { - warn!(error = %e, "backend_plan::from_stage_config failed; refusing partial publication"); - return PushOutcome::EmitFailed; - } - }; let precompute_envelope = PlanEnvelope { plan_id, plan_version: 1, @@ -474,7 +446,7 @@ async fn push_cumulative_entries( let materializations = match cumulative_be .aggregations .iter() - .map(crate::backend_plan::aggregation_config_for_materialization) + .map(crate::physical::compiler::aggregation_config_for_materialization) .collect::>>() { Ok(materializations) => materializations, @@ -486,7 +458,6 @@ async fn push_cumulative_entries( let precompute_plan = match PrecomputePlan::build( precompute_envelope, materializations, - &backend_plan, &["legacy-replanner".into()], ) { Ok(plan) => plan, @@ -906,10 +877,9 @@ mod tests { assert_eq!(mock.routing_hits.load(StdOrdering::SeqCst), 1); } - /// The dual-push also fires a best-effort `POST /api/v1/backend-plan`, - /// alongside — not instead of — the legacy documents. + /// The compatibility replanner publishes one atomic catalog-backed generation. #[tokio::test] - async fn coupled_push_also_fires_backend_plan_push() { + async fn coupled_push_publishes_atomic_physical_plan() { let (url, mock) = start_dual_mock(axum::http::StatusCode::OK, axum::http::StatusCode::OK).await; let client = StdArc::new(BackendClient::new(url)); @@ -927,10 +897,10 @@ mod tests { assert_eq!(mock.plan_hits.load(StdOrdering::SeqCst), 1); } - /// A BackendPlan push failure makes the publication generation + /// A physical-plan push failure makes the publication generation /// explicitly incomplete even if both compatibility documents landed. #[tokio::test] - async fn backend_plan_push_failure_is_reported_as_desync() { + async fn physical_plan_push_failure_is_reported_as_desync() { // A mock without the atomic endpoint: the whole generation fails. let app = Router::new(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); diff --git a/control_plane/src/lib.rs b/control_plane/src/lib.rs index e65c2b19..023b7021 100644 --- a/control_plane/src/lib.rs +++ b/control_plane/src/lib.rs @@ -58,7 +58,6 @@ pub mod accuracy; pub mod backend_client; -pub mod backend_plan; pub mod emit; pub mod epsilon_alloc; pub mod metrics_exposer; diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index 3a1adc17..1b476635 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -768,7 +768,7 @@ fn compile_physical_plan_request( } if request.plan_version == 0 || request.activation_unix_ms == 0 - || request.backend_compat != control_plane::backend_plan::BACKEND_COMPAT + || request.backend_compat != control_plane::physical::compiler::BACKEND_COMPAT || request .expiry_unix_ms .is_some_and(|expiry| expiry <= request.activation_unix_ms) @@ -2137,7 +2137,7 @@ mod api_tests { "collector_ids": ["test"], "capability_snapshot_id": "test", "planner_revision": physical::compiler::PLANNER_REVISION, "max_evidence_age_ms": 60000, "plan_version": 1, - "activation_unix_ms": now, "backend_compat": control_plane::backend_plan::BACKEND_COMPAT + "activation_unix_ms": now, "backend_compat": control_plane::physical::compiler::BACKEND_COMPAT })) .unwrap(); let response = handle_workload_cost_manifests(Json(request)) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index e1678706..4858e0c8 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -2,7 +2,7 @@ //! //! Planner owns semantic alternatives and guarantees. This module owns the //! deployment decision: evidence freshness, target capabilities, windows, the -//! Collector execution projection, and the matching BackendPlan. +//! Collector execution projection, SummaryCatalog, and executable plans. use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::rc::Rc; @@ -16,8 +16,7 @@ use asap_aware_mapping::{ use planner_types::post_asap::{ CompositionOperator, EvaluationSchedule, OutputRepresentation, SketchAlgorithm, SketchParams, SketchQuery, SummaryExpr, SummaryFamilyType, SummaryMaintenanceLifecycle, - SummaryMaintenanceLifecycleGuarantee, SummaryMaintenanceMode, SummaryNode, - SummaryWindowFramework, + SummaryMaintenanceMode, SummaryNode, SummaryWindowFramework, }; use planner_types::pre_asap::QueryExpr; use planner_types::workload::{ @@ -29,11 +28,7 @@ use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use thiserror::Error; -use crate::backend_plan::{self, BackendPlan}; -use crate::emit::monitor::MonitorIntent; -use crate::physical::colored_dag::emitter::{ - AggregationInput, BackendAggregation, BackendReadout, BackendStageConfig, -}; +use crate::physical::colored_dag::emitter::{AggregationInput, BackendAggregation}; use crate::physical::post_asap::cost_model::ControlPlaneCostModel; use crate::query_plan::{ canonical_promql, FallbackPolicy, InstantExecution, MaterializationBinding, PhysicalGrouping, @@ -43,6 +38,7 @@ use crate::types_v2::AccuracyTarget; use planner_types::pre_asap::Source; pub const PLANNER_REVISION: &str = "abb2f20e27091ac0c60715dc42b9b59c75b3447c"; +pub const BACKEND_COMPAT: &str = "asap-query-backend.v1"; #[derive(Debug, Clone)] pub struct PlanningQuery { @@ -404,7 +400,7 @@ pub struct ProducerContract { pub enum PrecomputePlanError { #[error("invalid precompute catalog contract: {0}")] CatalogContract(String), - #[error("PrecomputePlan envelope does not match BackendPlan identity/lifecycle")] + #[error("PrecomputePlan envelope does not match its SummaryCatalog identity/lifecycle")] PlanIdentityMismatch, #[error("unsupported precompute ingest protocol/endpoint/identity contract")] UnsupportedIngestEndpoint, @@ -428,38 +424,50 @@ impl PrecomputePlan { pub fn build( envelope: PlanEnvelope, materializations: Vec, - backend_plan: &BackendPlan, producer_ids: &[String], ) -> Result { - if envelope.plan_id != backend_plan.plan_id - || envelope.plan_version != backend_plan.plan_version - || envelope.generated_at_unix_ms != backend_plan.generated_at_unix_ms - || envelope.activation_unix_ms != backend_plan.activation_unix_ms - || envelope.expiry_unix_ms != backend_plan.expiry_unix_ms - || envelope.backend_compat != backend_plan.backend_compat - { - return Err(PrecomputePlanError::PlanIdentityMismatch); - } - let schemas = backend_plan - .materializations + let schemas = materializations .iter() - .map(|(fingerprint, materialization)| { - let family = StateFamilyContract::try_from(&materialization.family) + .map(|materialization| { + let fingerprint = materialization.policy_fingerprint(); + let accumulator = materialization + .accumulator_spec() + .map_err(|_| PrecomputePlanError::UnsupportedFamily(fingerprint.0))?; + let family = StateFamilyContract::try_from(&accumulator.family) .map_err(|_| PrecomputePlanError::UnsupportedFamily(fingerprint.0))?; + let source = materialization.table_name.as_ref().map_or_else( + || Source::TimeSeries { + metric: materialization.metric.clone(), + }, + |table_ref| Source::Table { + table_ref: table_ref.clone(), + }, + ); + let value_column = materialization + .value_column + .clone() + .map(planner_types::pre_asap::ColumnRef::Named) + .unwrap_or(planner_types::pre_asap::ColumnRef::SampleValue); Ok(StateSchemaContract { - schema_id: state_schema_id(*fingerprint), + schema_id: state_schema_id(fingerprint), schema_version: 1, - materialization: *fingerprint, + materialization: fingerprint, family, - source: materialization.source.clone(), - value_column: materialization.col.clone(), - group_by: materialization.group_by.clone(), + source, + value_column, + group_by: materialization.grouping_labels.labels.clone(), window: StateWindowContract { - kind: materialization.window.kind, - size_ms: materialization.window.size_ms, - slide_ms: materialization.window.slide_ms, + kind: materialization.window_type, + size_ms: materialization.window_size.saturating_mul(1_000), + slide_ms: match materialization.window_type { + asap_types::WindowKind::Tumbling => None, + asap_types::WindowKind::Sliding => { + Some(materialization.slide_interval.saturating_mul(1_000)) + } + asap_types::WindowKind::Session => None, + }, }, - encodings: state_encodings(&materialization.family), + encodings: state_encodings(&accumulator.family), }) }) .collect::, PrecomputePlanError>>()?; @@ -489,7 +497,7 @@ impl PrecomputePlan { producers, materializations, }; - plan.validate_against_backend(backend_plan)?; + plan.validate()?; Ok(plan) } @@ -498,14 +506,8 @@ impl PrecomputePlan { pub fn build_backend_local( envelope: PlanEnvelope, materializations: Vec, - backend_plan: &BackendPlan, ) -> Result { - let mut plan = Self::build( - envelope, - materializations, - backend_plan, - &["backend-local".into()], - )?; + let mut plan = Self::build(envelope, materializations, &["backend-local".into()])?; plan.ingest = IngestContract { protocol: IngestProtocol::PrometheusRemoteWriteV1, endpoint_path: "/api/v1/write".into(), @@ -515,7 +517,7 @@ impl PrecomputePlan { require_registered_producer: false, }; plan.producers.clear(); - plan.validate_against_backend(backend_plan)?; + plan.validate()?; Ok(plan) } @@ -584,6 +586,52 @@ impl PrecomputePlan { .iter() .map(|schema| (schema.materialization, schema.schema_id.as_str())) .collect(); + for schema in &self.schemas { + let materialization = self + .materializations + .iter() + .find(|candidate| candidate.policy_fingerprint() == schema.materialization) + .ok_or(PrecomputePlanError::SchemaSetMismatch)?; + let accumulator = materialization + .accumulator_spec() + .map_err(|_| PrecomputePlanError::UnsupportedFamily(schema.materialization.0))?; + let family = StateFamilyContract::try_from(&accumulator.family) + .map_err(|_| PrecomputePlanError::UnsupportedFamily(schema.materialization.0))?; + let source = materialization.table_name.as_ref().map_or_else( + || Source::TimeSeries { + metric: materialization.metric.clone(), + }, + |table_ref| Source::Table { + table_ref: table_ref.clone(), + }, + ); + let value_column = materialization + .value_column + .clone() + .map(planner_types::pre_asap::ColumnRef::Named) + .unwrap_or(planner_types::pre_asap::ColumnRef::SampleValue); + if schema.schema_id != state_schema_id(schema.materialization) + || schema.family != family + || schema.source != source + || schema.value_column != value_column + || schema.group_by != materialization.grouping_labels.labels + || schema.window.kind != materialization.window_type + || schema.window.size_ms != materialization.window_size.saturating_mul(1_000) + || schema.window.slide_ms + != match materialization.window_type { + asap_types::WindowKind::Tumbling => None, + asap_types::WindowKind::Sliding => { + Some(materialization.slide_interval.saturating_mul(1_000)) + } + asap_types::WindowKind::Session => None, + } + || schema.encodings != state_encodings(&accumulator.family) + { + return Err(PrecomputePlanError::InvalidSchema { + schema_id: schema.schema_id.clone(), + }); + } + } let mut producers = BTreeSet::new(); let mut produced = BTreeSet::new(); for producer in &self.producers { @@ -616,35 +664,6 @@ impl PrecomputePlan { } Ok(()) } - - pub fn validate_against_backend( - &self, - backend_plan: &BackendPlan, - ) -> Result<(), PrecomputePlanError> { - self.validate()?; - for schema in &self.schemas { - let Some(materialization) = backend_plan.materializations.get(&schema.materialization) - else { - return Err(PrecomputePlanError::SchemaSetMismatch); - }; - let expected_family = StateFamilyContract::try_from(&materialization.family) - .map_err(|_| PrecomputePlanError::UnsupportedFamily(schema.materialization.0))?; - if schema.family != expected_family - || schema.schema_id != state_schema_id(schema.materialization) - || schema.source != materialization.source - || schema.value_column != materialization.col - || schema.group_by != materialization.group_by - || schema.window.kind != materialization.window.kind - || schema.window.size_ms != materialization.window.size_ms - || schema.window.slide_ms != materialization.window.slide_ms - { - return Err(PrecomputePlanError::InvalidSchema { - schema_id: schema.schema_id.clone(), - }); - } - } - Ok(()) - } } /// Complete physical projection of one post-ASAP planning decision. @@ -656,7 +675,6 @@ pub struct PhysicalPlan { pub collector_plans: Vec, pub precompute_plan: PrecomputePlan, pub transmission_plan: TransmissionPlan, - pub backend_plan: BackendPlan, pub query_plan: QueryPlan, /// Lifecycle component only, not a complete physical-plan comparison. pub lifecycle_estimates: Vec, @@ -1591,8 +1609,8 @@ pub enum CompileError { InvalidEvidence { query_id: String, reason: String }, #[error("query {query_id}: lifecycle planning failed: {reason}")] Lifecycle { query_id: String, reason: String }, - #[error("failed to construct BackendPlan: {0}")] - BackendPlan(#[from] anyhow::Error), + #[error("failed to construct a physical materialization: {0}")] + Materialization(#[from] anyhow::Error), #[error("failed to construct QueryPlan: {0}")] QueryPlan(#[from] crate::query_plan::QueryPlanError), } @@ -1951,7 +1969,6 @@ impl PhysicalCompiler { request.queries[id].post_asap = root; } let mut aggregations = Vec::with_capacity(request.queries.len()); - let mut readouts = Vec::with_capacity(request.queries.len()); let mut collector_materializations = Vec::with_capacity(request.queries.len()); let mut plan_materializations = Vec::with_capacity(request.queries.len()); let mut shared_materializations = BTreeMap::new(); @@ -1959,7 +1976,7 @@ impl PhysicalCompiler { // Binding is established while compiling the physical // materializations, then consumed by QueryPlan lowering. The key is // the planner DAG node identity across the workload; serving never scans - // BackendPlan candidates to rediscover this decision. + // downstream components to rediscover this decision. let mut node_bindings = HashMap::::new(); let consumers = materialization_consumers( &request.queries, @@ -2105,7 +2122,7 @@ impl PhysicalCompiler { environment.target, ); let precompute_materialization = - backend_plan::aggregation_config_for_materialization(&aggregation)?; + aggregation_config_for_materialization(&aggregation)?; let materialization = precompute_materialization.policy_fingerprint(); let state_consumers = consumers[&materialization] .iter() @@ -2157,8 +2174,7 @@ impl PhysicalCompiler { } else { window_implementation.pane_secs }; - let runtime_materialization = - backend_plan::aggregation_config_for_materialization(&aggregation)?; + let runtime_materialization = aggregation_config_for_materialization(&aggregation)?; let materialization = runtime_materialization.policy_fingerprint(); let consumer_query_ids = state_consumers .iter() @@ -2210,12 +2226,6 @@ impl PhysicalCompiler { } let physical_group_by = aggregation.grouping.clone(); aggregations.push(aggregation); - if let Some(readout) = selected.readout.clone() { - readouts.push(BackendReadout { - aggregation_id, - op: readout, - }); - } let collector_materialization = CollectorMaterialization { query_id: format!("state-{}", materialization.0), materialization, @@ -2279,34 +2289,6 @@ impl PhysicalCompiler { planner_revision: PLANNER_REVISION.into(), capability_snapshot_id: environment.capability_snapshot_id, }; - let backend_stage_config = BackendStageConfig { - aggregations: aggregations.clone(), - readouts, - }; - let mut backend_plan = backend_plan::from_stage_config( - &backend_stage_config, - &Vec::::new(), - plan_id, - environment.observed_at_unix_ms, - )?; - backend_plan.plan_version = envelope.plan_version; - backend_plan.activation_unix_ms = envelope.activation_unix_ms; - backend_plan.expiry_unix_ms = envelope.expiry_unix_ms; - backend_plan.backend_compat = envelope.backend_compat.clone(); - backend_plan - .validate() - .map_err(|error| CompileError::Query { - query_id: "physical-plan-envelope".into(), - reason: error.to_string(), - })?; - for materialization in backend_plan.materializations.values_mut() { - materialization.lifecycle = Some(SummaryMaintenanceLifecycleGuarantee { - summary_maintenance_lifecycle: SummaryMaintenanceLifecycle::ContinuouslyMaintained, - summary_maintenance_mode: SummaryMaintenanceMode::Incremental, - evaluation_schedule: EvaluationSchedule::PerUpdate, - output_representation: OutputRepresentation::SummaryState, - }); - } let producer_ids = match environment.target { PhysicalDeploymentTarget::DistributedCollectors => environment.collector_ids.clone(), PhysicalDeploymentTarget::BackendLocalRemoteWrite => Vec::new(), @@ -2315,40 +2297,18 @@ impl PhysicalCompiler { // summary. PrecomputePlan is keyed by physical identity, not query ID. let mut materializations_by_fingerprint = BTreeMap::new(); for aggregation in &aggregations { - let materialization = - backend_plan::aggregation_config_for_materialization(aggregation)?; + let materialization = aggregation_config_for_materialization(aggregation)?; materializations_by_fingerprint .entry(materialization.policy_fingerprint()) .or_insert(materialization); } - // The compatibility emitter may clamp pane sizes. Publish the actual - // executable configuration in both projections, never the pre-clamp size. - for (id, config) in &materializations_by_fingerprint { - if let Some(state) = backend_plan.materializations.get_mut(id) { - state.window.size_ms = - config - .window_size - .checked_mul(1000) - .ok_or_else(|| CompileError::Query { - query_id: "precompute-window".into(), - reason: "window overflow".into(), - })?; - } - } let materializations = materializations_by_fingerprint.into_values().collect(); let mut precompute_plan = match environment.target { - PhysicalDeploymentTarget::DistributedCollectors => PrecomputePlan::build( - envelope.clone(), - materializations, - &backend_plan, - &producer_ids, - ), + PhysicalDeploymentTarget::DistributedCollectors => { + PrecomputePlan::build(envelope.clone(), materializations, &producer_ids) + } PhysicalDeploymentTarget::BackendLocalRemoteWrite => { - PrecomputePlan::build_backend_local( - envelope.clone(), - materializations, - &backend_plan, - ) + PrecomputePlan::build_backend_local(envelope.clone(), materializations) } } .map_err(|error| CompileError::Query { @@ -2376,8 +2336,11 @@ impl PhysicalCompiler { materializations: collector_materializations.clone(), }) .collect::>(); - let materialization_fingerprints: BTreeSet<_> = - backend_plan.materializations.keys().copied().collect(); + let materialization_fingerprints: BTreeSet<_> = precompute_plan + .materializations + .iter() + .map(asap_types::PrecomputeMaterialization::policy_fingerprint) + .collect(); let mut query_entries = BTreeMap::new(); for query in &request.queries { let canonical = canonical_promql(&query.query_string)?; @@ -2395,12 +2358,13 @@ impl PhysicalCompiler { "post-ASAP node has no compiled physical binding for {node_family:?}" )) })?; - let materialization = backend_plan + let materialization = precompute_plan .materializations - .get(&fingerprint) + .iter() + .find(|candidate| candidate.policy_fingerprint() == fingerprint) .ok_or_else(|| { crate::query_plan::QueryPlanError::Invalid(format!( - "compiled binding {} is absent from BackendPlan", + "compiled binding {} is absent from PrecomputePlan", fingerprint.0 )) })?; @@ -2417,10 +2381,14 @@ impl PhysicalCompiler { })?; let (_, source_window, _) = materialization_leaf_contract(node) .map_err(crate::query_plan::QueryPlanError::Invalid)?; - if materialization.family != physical_materialization_family(node_family) - || materialization.window.size_ms == 0 + let materialization_family = materialization.accumulator_spec() + .map_err(|error| crate::query_plan::QueryPlanError::Invalid(error.to_string()))? + .family; + let window_ms = materialization.window_size.saturating_mul(1_000); + if materialization_family != physical_materialization_family(node_family) + || window_ms == 0 || source_window.unwrap_or(query.window_secs).saturating_mul(1_000) - % materialization.window.size_ms != 0 + % window_ms != 0 { return Err(crate::query_plan::QueryPlanError::Invalid(format!( "compiled binding {} disagrees with post-ASAP/deployment semantics", @@ -2535,7 +2503,6 @@ impl PhysicalCompiler { collector_plans, precompute_plan, transmission_plan, - backend_plan, query_plan, lifecycle_estimates: lifecycle_estimates.into_values().collect(), cost_comparison: None, @@ -2662,11 +2629,7 @@ pub fn select_post_asap( } pub(super) fn state_schema_id(fingerprint: asap_types::PolicyFingerprint) -> String { - format!( - "{}:summary-state:v1:{}", - backend_plan::BACKEND_COMPAT, - fingerprint.0 - ) + format!("{}:summary-state:v1:{}", BACKEND_COMPAT, fingerprint.0) } pub(super) fn state_encodings(family: &SummaryFamilyType) -> Vec { @@ -3102,7 +3065,6 @@ struct SelectedMaterialization { spatial_filter: String, group_by: Option>, family: SummaryFamilyType, - readout: Option, algorithm: String, parameters: Value, } @@ -3138,6 +3100,26 @@ fn physical_aggregation( } } +/// Convert the typed physical aggregation into the runtime's canonical +/// content-addressed materialization contract. This is the one conversion +/// shared by the physical compiler and the compatibility replanner; it does +/// not create a second registry or wire plan. +pub(crate) fn aggregation_config_for_materialization( + aggregation: &BackendAggregation, +) -> anyhow::Result { + use anyhow::Context as _; + let json = crate::emit::stage_config::build_backend_aggregation_json(aggregation); + let text = serde_json::to_string(&json).context("serialize synthesized aggregation JSON")?; + let yaml: serde_yaml::Value = + serde_yaml::from_str(&text).context("parse synthesized aggregation JSON as YAML")?; + asap_types::PrecomputeMaterialization::from_yaml_data( + &yaml, + None, + asap_types::QueryLanguage::promql, + ) + .context("build materialization from physical aggregation") +} + fn materialization_consumers( queries: &[PlanningQuery], target: PhysicalDeploymentTarget, @@ -3163,9 +3145,12 @@ fn materialization_consumers( { continue; } - let config = backend_plan::aggregation_config_for_materialization( - &physical_aggregation(query, &state, query.query_id.clone(), target), - )?; + let config = aggregation_config_for_materialization(&physical_aggregation( + query, + &state, + query.query_id.clone(), + target, + ))?; consumers .entry(config.policy_fingerprint()) .or_default() @@ -3299,7 +3284,6 @@ fn collect_selected_materializations( kind.clone(), planner_types::post_asap::GroupingStrategy::PerSubpopulationInstance, ), - readout: Some(readout.clone()), algorithm: format!("{:?}", kind.algorithm()).to_ascii_lowercase(), parameters, }); @@ -3323,7 +3307,6 @@ fn collect_selected_materializations( spatial_filter, group_by: grouping.clone(), family: SummaryFamilyType::ExactAggregate(kind.clone(), params.clone()), - readout: None, algorithm: format!("{kind:?}").to_ascii_lowercase(), parameters: Value::Object(Default::default()), }); @@ -3796,7 +3779,7 @@ mod tests { .compile(workload, env) .expect("shared compile"); assert_eq!(bundle.query_plan.entries.len(), 2); - assert_eq!(bundle.backend_plan.materializations.len(), 1); + assert_eq!(bundle.summary_catalog.materializations.len(), 1); assert_eq!(bundle.precompute_plan.materializations.len(), 1); assert_eq!(bundle.precompute_plan.schemas.len(), 1); let bindings = bundle @@ -3839,7 +3822,7 @@ mod tests { let bundle = PhysicalCompiler .compile(workload, environment(10_000)) .unwrap(); - assert_eq!(bundle.backend_plan.materializations.len(), 2); + assert_eq!(bundle.summary_catalog.materializations.len(), 2); assert_eq!(bundle.precompute_plan.materializations.len(), 2); for collector in &bundle.collector_plans { assert_eq!(collector.materializations.len(), 2); @@ -4282,17 +4265,14 @@ mod tests { #[test] fn publication_is_catalog_authoritative_and_round_trips() { - let mut bundle = PhysicalCompiler + let bundle = PhysicalCompiler .compile( request("publication", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) .unwrap(); - // Compatibility bytes cannot silently determine publication identity. - bundle.backend_plan.plan_id += 1; let publication = bundle.publication().unwrap(); let json = serde_json::to_value(&publication).unwrap(); - assert!(json.get("backend_plan").is_none()); let mut decoded: super::super::publication::PhysicalPlanPublication = serde_json::from_value(json).unwrap(); decoded.validate().unwrap(); @@ -4331,11 +4311,10 @@ mod tests { .cloned() .collect::>(), bundle - .backend_plan + .precompute_plan .materializations - .keys() - .copied() - .map(asap_types::sds::MaterializationId::from) + .iter() + .map(|config| config.policy_fingerprint().into()) .collect::>() ); assert_eq!(bundle.precompute_plan.envelope, bundle.envelope); @@ -4378,21 +4357,16 @@ mod tests { bundle.transmission_plan.validate_frame(&wrong_version), Err(TransmissionPlanError::InvalidFrame(_)) )); - assert_eq!(bundle.backend_plan.materializations.len(), 1); + assert_eq!(bundle.summary_catalog.materializations.len(), 1); assert_eq!( bundle - .backend_plan - .materializations + .query_plan + .entries .values() - .next() - .unwrap() - .lifecycle - .as_ref() - .unwrap() - .summary_maintenance_lifecycle, - SummaryMaintenanceLifecycle::ContinuouslyMaintained + .flat_map(QueryPlanEntry::materialization_bindings) + .count(), + 1 ); - assert_eq!(bundle.backend_plan.routing.len(), 1); assert_eq!(bundle.query_plan.plan_id, bundle.envelope.plan_id); let entry = bundle .query_plan @@ -4413,10 +4387,10 @@ mod tests { entry .validate( &bundle - .backend_plan + .precompute_plan .materializations - .keys() - .copied() + .iter() + .map(asap_types::PrecomputeMaterialization::policy_fingerprint) .collect(), ) .expect("executable physical DAG"); @@ -4464,7 +4438,6 @@ mod tests { let plan = PrecomputePlan::build_backend_local( bundle.envelope.clone(), bundle.precompute_plan.materializations.clone(), - &bundle.backend_plan, ) .expect("backend-local contract"); @@ -4475,8 +4448,7 @@ mod tests { assert_eq!(plan.ingest.endpoint_path, "/api/v1/write"); assert_eq!(plan.ingest.timestamp_unit, TimestampUnit::UnixMilliseconds); assert!(plan.producers.is_empty()); - plan.validate_against_backend(&bundle.backend_plan) - .expect("valid backend-local projection"); + plan.validate().expect("valid backend-local projection"); TransmissionPlan::build(bundle.envelope, &plan, &BTreeMap::new()) .expect("empty backend-local transmission contract") .validate(&plan) @@ -4562,7 +4534,7 @@ mod tests { .compile() .expect("second deterministic plan"); assert_eq!(first.envelope, second.envelope); - assert_eq!(first.backend_plan, second.backend_plan); + assert_eq!(first.summary_catalog, second.summary_catalog); assert_eq!(first.query_plan, second.query_plan); assert_eq!(first.transmission_plan, second.transmission_plan); assert_eq!( @@ -4610,7 +4582,7 @@ mod tests { deployment.collector_ids.clear(); let compiled = PhysicalCompiler.compile(request(query_id, promql), deployment); let plan = compiled.unwrap_or_else(|error| panic!("{promql} must compile: {error}")); - assert_eq!(plan.backend_plan.materializations.len(), 1, "{promql}"); + assert_eq!(plan.summary_catalog.materializations.len(), 1, "{promql}"); assert_eq!(plan.query_plan.entries.len(), 1, "{promql}"); assert!(plan.collector_plans.is_empty(), "{promql}"); let entry = plan.query_plan.entries.values().next().unwrap(); @@ -4620,9 +4592,9 @@ mod tests { if *readout == expected_readout )); if expected_readout == crate::query_plan::ExactReadout::Rate { - let materialization = plan.backend_plan.materializations.values().next().unwrap(); + let materialization = plan.precompute_plan.materializations.first().unwrap(); assert_eq!( - materialization.family, + materialization.accumulator_spec().unwrap().family, SummaryFamilyType::ExactAggregate( planner_types::post_asap::ExactKind::Increase, planner_types::post_asap::ExactParams::Increase, @@ -4724,7 +4696,7 @@ mod tests { .unwrap(); assert_eq!(bundle.query_plan.entries.len(), 2); - assert_eq!(bundle.backend_plan.materializations.len(), 1); + assert_eq!(bundle.summary_catalog.materializations.len(), 1); assert_eq!(bundle.precompute_plan.materializations.len(), 1); assert_eq!(bundle.precompute_plan.schemas.len(), 1); assert_eq!(bundle.precompute_plan.producers.len(), 2); @@ -4766,9 +4738,17 @@ mod tests { let bundle = PhysicalCompiler .compile(planning_request, environment(10_000)) .expect("compile merged post-ASAP DAG"); - assert_eq!(bundle.backend_plan.materializations.len(), 2); + assert_eq!(bundle.summary_catalog.materializations.len(), 2); assert_eq!(bundle.precompute_plan.materializations.len(), 2); - assert_eq!(bundle.backend_plan.routing.len(), 2); + assert_eq!( + bundle + .query_plan + .entries + .values() + .flat_map(QueryPlanEntry::materialization_bindings) + .count(), + 2 + ); let entry = bundle.query_plan.entries.values().next().unwrap(); assert_eq!( entry @@ -4804,7 +4784,7 @@ mod tests { } #[test] - fn precompute_schema_must_match_backend_semantics() { + fn precompute_schema_must_match_materialization_semantics() { let bundle = PhysicalCompiler .compile( request("q-quantile", "quantile_over_time(0.99, m[1m])"), @@ -4814,7 +4794,7 @@ mod tests { let mut plan = bundle.precompute_plan.clone(); plan.schemas[0].group_by.push("invented".into()); assert!(matches!( - plan.validate_against_backend(&bundle.backend_plan), + plan.validate(), Err(PrecomputePlanError::InvalidSchema { .. }) )); } @@ -4836,13 +4816,8 @@ mod tests { env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; env.collector_ids.clear(); let bundle = PhysicalCompiler.compile(request, env).unwrap(); - let materialization = bundle - .backend_plan - .materializations - .values() - .next() - .unwrap(); - assert_eq!(materialization.window.size_ms, expected_secs * 1000); + let materialization = bundle.precompute_plan.materializations.first().unwrap(); + assert_eq!(materialization.window_size, expected_secs); let entry = bundle .query_plan .lookup("sum(sum_over_time(m[1m]))") @@ -5023,7 +4998,7 @@ mod tests { let bundle = PhysicalCompiler .compile(request, environment(10_000)) .expect("certified TopK compiles"); - assert_eq!(bundle.backend_plan.materializations.len(), 1); + assert_eq!(bundle.summary_catalog.materializations.len(), 1); assert_eq!( bundle.collector_plans[0].materializations[0] .evidence_source diff --git a/control_plane/src/physical/post_asap/cost_model.rs b/control_plane/src/physical/post_asap/cost_model.rs index 67ba378f..94c185be 100644 --- a/control_plane/src/physical/post_asap/cost_model.rs +++ b/control_plane/src/physical/post_asap/cost_model.rs @@ -703,11 +703,11 @@ impl CostModel for ForcedFamilyCostModel { /// /// **Fallback status (design-backend-plan-wire-format.md §5):** /// `post_asap_planner.rs` prefers reading planning's decision directly off an -/// installed `BackendPlan`'s materializations (no reconstruction needed +/// installed SummaryCatalog materializations (no reconstruction needed /// there — `Materialization.kind`/`.params` already ARE the pair /// `observed` needs). This type's caller /// (`observed_family_for_metric`, the `SketchStore`-metadata -/// reconstruction) is the fallback for deploys with no `BackendPlan` +/// reconstruction) is the fallback for deployments without a catalog snapshot /// installed yet, or for metrics a partial/stale plan doesn't cover. pub struct ObservedFamilyCostModel { inner: ControlPlaneCostModel, diff --git a/control_plane/src/physical/post_asap/mod.rs b/control_plane/src/physical/post_asap/mod.rs index fa02f2d2..d446bc26 100644 --- a/control_plane/src/physical/post_asap/mod.rs +++ b/control_plane/src/physical/post_asap/mod.rs @@ -2,7 +2,7 @@ //! //! ASAPPlanner owns summary selection and the post-ASAP `SummaryNode` tree. //! This module adds only backend-specific concerns required to compile that -//! tree into a [`crate::backend_plan::BackendPlan`]: deployment cost input, +//! tree into catalog-backed executable plans: deployment cost input, //! named sharing/placement wrappers, and runtime family matching. It belongs //! under `physical`; it is not another logical or sketch-algebra layer. diff --git a/control_plane/src/physical/publication.rs b/control_plane/src/physical/publication.rs index 9740cde9..bfecf8ce 100644 --- a/control_plane/src/physical/publication.rs +++ b/control_plane/src/physical/publication.rs @@ -1,5 +1,4 @@ -//! Canonical publication document. Legacy BackendPlan is not an authority in -//! this contract; adapters may attach it for old installation endpoints. +//! Canonical publication document for one catalog generation. use super::compiler::{CollectorPlan, PhysicalPlan, PrecomputePlan, TransmissionPlan}; use super::summary_catalog::SummaryCatalog; use crate::query_plan::QueryPlan; @@ -15,7 +14,7 @@ pub struct PhysicalPlanPublication { pub query_plan: QueryPlan, } impl PhysicalPlanPublication { - /// Validate the complete generation without decoding a BackendPlan. + /// Validate every plan against the shared catalog snapshot. pub fn validate(&self) -> Result<(), String> { let catalog = &self.summary_catalog; self.precompute_plan From 6d7dcd75c6c227f566efe1a8a4149278ee31d1ff Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 15:54:35 -0600 Subject: [PATCH 20/30] Complete BackendPlan transport removal --- control_plane/src/physical/compiler.rs | 4 ++- crates/asap_types/src/routing_index.rs | 9 ++--- .../drivers/ingest/prometheus_remote_write.rs | 2 +- data_plane/src/drivers/query/servers/http.rs | 34 ++++--------------- data_plane/src/main.rs | 1 - .../asapquery_compatibility_process_e2e.rs | 6 ++-- data_plane/tests/backend_process_e2e.rs | 6 ++-- 7 files changed, 20 insertions(+), 42 deletions(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 4858e0c8..230e0d3d 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -2398,7 +2398,9 @@ impl PhysicalCompiler { Ok(MaterializationBinding { readout_lookback_ms: source_window.map(|seconds| seconds.saturating_mul(1_000)), materialization: fingerprint.into(), - output_grouping: PhysicalGrouping::Reduce(materialization.group_by.clone()), + output_grouping: PhysicalGrouping::Reduce( + materialization.grouping_labels.labels.clone(), + ), window_ms: pane_ms, }) }; diff --git a/crates/asap_types/src/routing_index.rs b/crates/asap_types/src/routing_index.rs index 4729d4c4..2a21f237 100644 --- a/crates/asap_types/src/routing_index.rs +++ b/crates/asap_types/src/routing_index.rs @@ -1,10 +1,7 @@ //! `RoutingIndex` — a metric-bucketed structural index over a -//! [`PolicyRegistry`], per `control_plane/docs/design-backend-plan-wire-format.md` -//! §4's Tier-1/Tier-2 design (currently sourced from `PolicyRegistry` — the -//! content-addressed view over `StreamingConfig`'s `AggregationConfig`s, -//! which is genuinely "what control_plane planned" today, not a -//! reconstruction from ingest-side-effects — pending that doc's `BackendPlan` -//! wire format actually existing). +//! [`PolicyRegistry`]. It is sourced from the content-addressed view over a +//! `StreamingConfig`'s `AggregationConfig`s, so it represents planned policy +//! rather than a reconstruction from ingest side effects. //! //! **Tier 1** (exact `PolicyFingerprint` → config) is [`PolicyRegistry::get`] //! itself — already O(1), nothing to add here. diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index 9a3e4f9e..a0012383 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -768,7 +768,7 @@ mod tests { generated_at_unix_ms: 1, activation_unix_ms: 1, expiry_unix_ms: None, - backend_compat: control_plane::backend_plan::BACKEND_COMPAT.into(), + backend_compat: control_plane::physical::compiler::BACKEND_COMPAT.into(), planner_revision: PLANNER_REVISION.into(), capability_snapshot_id: "test".into(), }; diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index a1e9820e..5214ddce 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -2477,7 +2477,7 @@ mod tests { generated_at_unix_ms: 0, activation_unix_ms: 1, expiry_unix_ms: None, - backend_compat: control_plane::backend_plan::BACKEND_COMPAT.into(), + backend_compat: control_plane::physical::compiler::BACKEND_COMPAT.into(), planner_revision: PLANNER_REVISION.into(), capability_snapshot_id: "test".into(), }; @@ -5773,7 +5773,6 @@ pub struct PhysicalPlanInstallRequest { pub collector_plans: Vec, pub precompute_plan: control_plane::physical::compiler::PrecomputePlan, pub transmission_plan: control_plane::physical::compiler::TransmissionPlan, - pub backend_plan: Vec, pub query_plan: control_plane::query_plan::QueryPlan, pub storage_routing: Option, #[serde(default)] @@ -5828,39 +5827,21 @@ pub fn build_active_physical_plan( .transmission_plan .validate(&request.precompute_plan) .map_err(|error| format!("TransmissionPlan validation error: {error}"))?; - let backend_plan = control_plane::backend_plan::BackendPlan::decode(&request.backend_plan) - .map_err(|error| format!("BackendPlan decode error: {error}"))?; - backend_plan - .validate() - .map_err(|error| format!("BackendPlan validation error: {error}"))?; - request - .precompute_plan - .validate_against_backend(&backend_plan) - .map_err(|error| format!("PrecomputePlan validation error: {error}"))?; let envelope = &request.precompute_plan.envelope; - if request.query_plan.plan_id != backend_plan.plan_id - || request.query_plan.plan_version != backend_plan.plan_version - || envelope.plan_id != backend_plan.plan_id - || envelope.plan_version != backend_plan.plan_version - || envelope.activation_unix_ms != backend_plan.activation_unix_ms - || envelope.expiry_unix_ms != backend_plan.expiry_unix_ms - || envelope.backend_compat != backend_plan.backend_compat + if request.query_plan.plan_id != envelope.plan_id + || request.query_plan.plan_version != envelope.plan_version || request.transmission_plan.envelope != *envelope { return Err("physical subplans have different plan identity/version".into()); } let runtime_config = crate::storage_engines::types::StreamingConfig::new(runtime_materializations); - let config_fps: BTreeSet = runtime_config.aggregation_configs.keys().copied().collect(); - let plan_fps: BTreeSet = backend_plan - .materializations + let typed_fps: BTreeSet<_> = runtime_config + .aggregation_configs .keys() - .map(|fp| fp.0) + .copied() + .map(asap_types::PolicyFingerprint) .collect(); - if config_fps != plan_fps { - return Err("PrecomputePlan and BackendPlan materialization fingerprints differ".into()); - } - let typed_fps: BTreeSet<_> = backend_plan.materializations.keys().copied().collect(); request .query_plan .validate(&typed_fps) @@ -6820,7 +6801,6 @@ mod catalog_install_tests { collector_plans: plan.collector_plans, precompute_plan: plan.precompute_plan, transmission_plan: plan.transmission_plan, - backend_plan: plan.backend_plan.encode_to_vec(), query_plan: plan.query_plan, storage_routing: None, adaptation_evidence: vec![], diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 87d57839..72e04e7f 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -496,7 +496,6 @@ async fn main() -> Result<()> { collector_plans: plan.collector_plans, precompute_plan: plan.precompute_plan, transmission_plan: plan.transmission_plan, - backend_plan: plan.backend_plan.encode_to_vec(), query_plan: plan.query_plan, storage_routing: None, adaptation_evidence: Vec::new(), diff --git a/data_plane/tests/asapquery_compatibility_process_e2e.rs b/data_plane/tests/asapquery_compatibility_process_e2e.rs index b2e73a54..9218f575 100644 --- a/data_plane/tests/asapquery_compatibility_process_e2e.rs +++ b/data_plane/tests/asapquery_compatibility_process_e2e.rs @@ -212,7 +212,6 @@ async fn registered_temporal_topk(algorithm: planner_types::post_asap::SketchAlg collector_plans: plan.collector_plans, precompute_plan: plan.precompute_plan, transmission_plan: plan.transmission_plan, - backend_plan: plan.backend_plan.encode_to_vec(), query_plan: plan.query_plan, storage_routing: None, adaptation_evidence: vec![], @@ -1295,8 +1294,9 @@ async fn collector_free_profile_serves_complete_matrix_and_falls_back_exactly() legacy.precompute_plan.ingest.require_registered_producer = false; legacy.precompute_plan.producers.clear(); legacy.transmission_plan.rules.clear(); - let artifact = serde_json::json!({"precompute_plan": legacy.precompute_plan, - "transmission_plan": legacy.transmission_plan, "backend_plan": legacy.backend_plan.encode_to_vec(), + let artifact = serde_json::json!({"summary_catalog": legacy.summary_catalog, + "collector_plans": legacy.collector_plans, "precompute_plan": legacy.precompute_plan, + "transmission_plan": legacy.transmission_plan, "query_plan": legacy.query_plan, "storage_routing": null, "adaptation_evidence": []}); let built = data_plane::drivers::query::servers::http::build_active_physical_plan( serde_json::from_value(artifact.clone()).unwrap(), diff --git a/data_plane/tests/backend_process_e2e.rs b/data_plane/tests/backend_process_e2e.rs index f6ed75ef..0aa88cc0 100644 --- a/data_plane/tests/backend_process_e2e.rs +++ b/data_plane/tests/backend_process_e2e.rs @@ -134,7 +134,7 @@ fn ddsketch_export( ("plan_version", plan["envelope"]["plan_version"].to_string()), ( "backend_compat", - control_plane::backend_plan::BACKEND_COMPAT.into(), + control_plane::physical::compiler::BACKEND_COMPAT.into(), ), ("materialization", materialization.to_string()), ( @@ -145,7 +145,7 @@ fn ddsketch_export( "schema_id", format!( "{}:summary-state:v1:{materialization}", - control_plane::backend_plan::BACKEND_COMPAT + control_plane::physical::compiler::BACKEND_COMPAT ), ), ("producer_id", "whole-e2e-collector".into()), @@ -515,7 +515,7 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { "plan_version": 1, "activation_unix_ms": observed_at_ms, "expiry_unix_ms": null, - "backend_compat": control_plane::backend_plan::BACKEND_COMPAT, + "backend_compat": control_plane::physical::compiler::BACKEND_COMPAT, "apply_timeout_ms": 10000 }); let mut second = request["queries"][0].clone(); From 7e377641c9a10b05d069651790050d26d790c678 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 15:58:05 -0600 Subject: [PATCH 21/30] Update canonical publication tests after BackendPlan removal --- control_plane/src/main.rs | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index 1b476635..5b770444 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -3544,14 +3544,12 @@ mod api_tests { .route("/api/v1/plan", axum::routing::post(handle_plan)) .with_state(state.clone()); - // The four evidence-safe sketched contract metrics. Each gets a + // The two quantile-compatible sketched contract metrics. Each gets a // separate POST /api/v1/plan, mirroring the demo's // per-workload plan-emit cycle. let sketched = [ "http_requests_total_latency_ms", // DDSketch "request_size_bytes", // KLL - "unique_users_per_min", // HLL - "endpoint_request_freq", // CountMinSketch ]; for m in &sketched { @@ -3570,8 +3568,15 @@ mod api_tests { ); } - // Drain the mock sink: every plan-emit must have produced - // exactly one body. + // Publication is dispatched asynchronously; wait for every accepted + // plan instead of racing the background HTTP tasks. + for _ in 0..100 { + if sink.lock().unwrap().len() == sketched.len() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + // Drain the mock sink: every plan-emit must have produced exactly one body. let bodies = sink.lock().unwrap().clone(); assert_eq!( bodies.len(), @@ -3694,21 +3699,14 @@ mod api_tests { .route("/api/v1/plan", axum::routing::post(handle_plan)) .with_state(state.clone()); - // The evidence-safe sketched contract metrics — the same + // The quantile-compatible sketched contract metrics — the same // set the sibling `storage_routing_cumulative_push_...` test - // exercises. Each gets a separate `POST /api/v1/plan` with - // the metric-name → classified sketch family from - // `classify_demo_metric`. Pre-fix the metric-only cache + // exercises. Each gets a separate `POST /api/v1/plan`. Pre-fix the metric-only cache // would have collapsed sequential same-metric POSTs onto one // slot; this test uses 5 distinct metrics so the assertion // surfaces the cumulative-merge gap (every metric's row must // survive every other metric's swap). - let sketched = [ - "http_requests_total_latency_ms", - "request_size_bytes", - "unique_users_per_min", - "endpoint_request_freq", - ]; + let sketched = ["http_requests_total_latency_ms", "request_size_bytes"]; for m in &sketched { let app = app.clone(); @@ -3726,8 +3724,15 @@ mod api_tests { ); } - // Drain the mock sink: every plan-emit must have produced - // exactly one streaming-config body. + // Publication is dispatched asynchronously; wait for every accepted + // plan instead of racing the background HTTP tasks. + for _ in 0..100 { + if sink.lock().unwrap().len() == sketched.len() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + // Drain the mock sink: every plan-emit must have produced exactly one body. let bodies = sink.lock().unwrap().clone(); assert_eq!( bodies.len(), From 7276851ce3310f814b6b92b4b27e01bda2af96fd Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 16:15:06 -0600 Subject: [PATCH 22/30] Integrate Planner value operations into query plans --- Cargo.lock | 6 +- control_plane/Cargo.toml | 6 +- .../examples/calibration_candidates.rs | 9 ++ .../examples/offline_planner_replay.rs | 1 + control_plane/src/emit/mod.rs | 1 + control_plane/src/emit/stage_config.rs | 1 + control_plane/src/physical/allocator.rs | 1 + .../src/physical/colored_dag/allocator.rs | 8 + .../src/physical/colored_dag/emitter.rs | 1 + control_plane/src/physical/compiler.rs | 8 +- .../src/physical/post_asap/matcher.rs | 1 + control_plane/src/physical/post_asap/tests.rs | 1 + control_plane/src/planner_selection.rs | 8 +- control_plane/src/query_plan.rs | 137 ++++++++++++++++++ control_plane/src/query_plan/logical.rs | 44 +++++- control_plane/src/query_planning.rs | 3 +- control_plane/src/replan.rs | 3 + crates/asap_types/Cargo.toml | 2 +- crates/asap_types/src/sds.rs | 3 +- data_plane/Cargo.toml | 4 +- .../asap_query_engine/summary_exec.rs | 1 + 21 files changed, 230 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e861dd5a..286f3a07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -343,7 +343,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=abb2f20e27091ac0c60715dc42b9b59c75b3447c#abb2f20e27091ac0c60715dc42b9b59c75b3447c" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=24735a35442c0ada3dd4ba1e2b3ab671b29f31b6#24735a35442c0ada3dd4ba1e2b3ab671b29f31b6" dependencies = [ "asap-types", "serde", @@ -354,7 +354,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=abb2f20e27091ac0c60715dc42b9b59c75b3447c#abb2f20e27091ac0c60715dc42b9b59c75b3447c" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=24735a35442c0ada3dd4ba1e2b3ab671b29f31b6#24735a35442c0ada3dd4ba1e2b3ab671b29f31b6" dependencies = [ "asap-types", "promql-parser 0.10.0", @@ -374,7 +374,7 @@ dependencies = [ [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=abb2f20e27091ac0c60715dc42b9b59c75b3447c#abb2f20e27091ac0c60715dc42b9b59c75b3447c" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=24735a35442c0ada3dd4ba1e2b3ab671b29f31b6#24735a35442c0ada3dd4ba1e2b3ab671b29f31b6" dependencies = [ "serde", "serde_json", diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index b96327c7..3cfa5e7d 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -93,8 +93,8 @@ asap_types.workspace = true # scaffolding, unaware that `data_plane`'s `summary_executor.rs` in *this* # repo is a real one. Vendored locally instead of chased upstream -- see # `data_plane/src/query_engines/asap_query_engine/summary_exec.rs`. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "abb2f20e27091ac0c60715dc42b9b59c75b3447c" } -asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "abb2f20e27091ac0c60715dc42b9b59c75b3447c" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "24735a35442c0ada3dd4ba1e2b3ab671b29f31b6" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "24735a35442c0ada3dd4ba1e2b3ab671b29f31b6" } # L1 adoption (design-target-architecture.md Part B): the PromQL front # end itself, replacing control_plane's own query_parser/promql.rs. @@ -102,7 +102,7 @@ asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = # `planner-types`/`asap-aware-mapping` above -- these three MUST move # together (two revs of the same upstream repo's types in one workspace # resolve to distinct Rust types that won't unify). -asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "abb2f20e27091ac0c60715dc42b9b59c75b3447c" } +asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "24735a35442c0ada3dd4ba1e2b3ab671b29f31b6" } [dev-dependencies] tokio = { version = "1", features = ["full", "test-util"] } diff --git a/control_plane/examples/calibration_candidates.rs b/control_plane/examples/calibration_candidates.rs index c69f4f85..5eb0e78d 100644 --- a/control_plane/examples/calibration_candidates.rs +++ b/control_plane/examples/calibration_candidates.rs @@ -32,6 +32,15 @@ fn planner_forest(queries: &[control_plane::physical::compiler::PlanningQuery]) vec![lhs, rhs], json!({"operator_debug":format!("{operator:?}")}), ), + SummaryExpr::ValueOperation { + child, + operation, + timing, + } => ( + "ValueOperation", + vec![child], + json!({"operation_debug":format!("{operation:?}"),"timing_debug":format!("{timing:?}")}), + ), SummaryExpr::SummaryAgg { child, family, diff --git a/control_plane/examples/offline_planner_replay.rs b/control_plane/examples/offline_planner_replay.rs index d7941ad3..b35f5501 100644 --- a/control_plane/examples/offline_planner_replay.rs +++ b/control_plane/examples/offline_planner_replay.rs @@ -28,6 +28,7 @@ fn inspect( } match &node.expr { SummaryExpr::KeepPreAsap(_) => *raw += 1, + SummaryExpr::ValueOperation { child, .. } => inspect(child, model, seen, states, raw), SummaryExpr::SummaryAgg { family, child, .. } => { if let SummaryFamilyType::Sketch(kind, _) = family { let lookup = model diff --git a/control_plane/src/emit/mod.rs b/control_plane/src/emit/mod.rs index fd6cb80e..0ca0dc69 100644 --- a/control_plane/src/emit/mod.rs +++ b/control_plane/src/emit/mod.rs @@ -319,6 +319,7 @@ fn extract_from_node(node: &Rc) -> Option { SummaryExpr::SummaryAgg { .. } => None, SummaryExpr::SummaryEstimate { summary_input, .. } => extract_from_node(summary_input), SummaryExpr::SummaryMerge { children } => children.iter().find_map(extract_from_node), + SummaryExpr::ValueOperation { child, .. } => extract_from_node(child), // Not surfaced by any `Bind*` path yet (gated on rules that // haven't landed — see `deployment_expr.rs`'s module docs). SummaryExpr::BinaryOp { .. } diff --git a/control_plane/src/emit/stage_config.rs b/control_plane/src/emit/stage_config.rs index 0120b517..cdedfca9 100644 --- a/control_plane/src/emit/stage_config.rs +++ b/control_plane/src/emit/stage_config.rs @@ -2966,6 +2966,7 @@ pub(crate) fn build_backend_aggregation_json(agg: &BackendAggregation) -> JsonVa ExactKind::MinMax => "MinMax", ExactKind::Increase => "Increase", ExactKind::Rate => "Rate", + ExactKind::IRate => "IRate", } .to_string(), json!({}), diff --git a/control_plane/src/physical/allocator.rs b/control_plane/src/physical/allocator.rs index a5055da8..2b070f52 100644 --- a/control_plane/src/physical/allocator.rs +++ b/control_plane/src/physical/allocator.rs @@ -775,6 +775,7 @@ fn canonical_intent_kind_str(intent: &AggIntent) -> &'static str { AggIntent::TopK { .. } => "topk", AggIntent::Cardinality { .. } => "cardinality", AggIntent::Rate => "rate", + AggIntent::IRate => "irate", AggIntent::Increase => "increase", AggIntent::Absent => "absent", AggIntent::AbsentOverTime => "absent_over_time", diff --git a/control_plane/src/physical/colored_dag/allocator.rs b/control_plane/src/physical/colored_dag/allocator.rs index ced8118b..fd149dd9 100644 --- a/control_plane/src/physical/colored_dag/allocator.rs +++ b/control_plane/src/physical/colored_dag/allocator.rs @@ -184,6 +184,14 @@ impl ThreeStageWalker { } StageId::Backend } + SummaryExpr::ValueOperation { child, timing, .. } => { + let (cid, child_stage) = self.visit_l4node(child)?; + self.dag.edges.push((id, cid)); + match timing { + planner_types::post_asap::ExecutionTiming::ReadTime => StageId::Backend, + planner_types::post_asap::ExecutionTiming::MaintenanceTime => child_stage, + } + } // ── SummaryAgg: always edge per design.md §6 batched-queries // table — true for both approximate sketches (the old diff --git a/control_plane/src/physical/colored_dag/emitter.rs b/control_plane/src/physical/colored_dag/emitter.rs index ecaefb29..7306e164 100644 --- a/control_plane/src/physical/colored_dag/emitter.rs +++ b/control_plane/src/physical/colored_dag/emitter.rs @@ -117,6 +117,7 @@ fn classify(expr: &PhysicalExpr) -> NodeKind<'_> { SummaryExpr::SummaryEstimate { query, .. } => NodeKind::SketchEstimate { query }, SummaryExpr::SummaryMerge { .. } => NodeKind::SketchMerge, SummaryExpr::BinaryOp { .. } + | SummaryExpr::ValueOperation { .. } | SummaryExpr::SummaryJoin { .. } | SummaryExpr::SummarySubtract { .. } | SummaryExpr::SummaryDelete { .. } => NodeKind::Other, diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 230e0d3d..3051e5e7 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -37,7 +37,7 @@ use crate::query_plan::{ use crate::types_v2::AccuracyTarget; use planner_types::pre_asap::Source; -pub const PLANNER_REVISION: &str = "abb2f20e27091ac0c60715dc42b9b59c75b3447c"; +pub const PLANNER_REVISION: &str = "24735a35442c0ada3dd4ba1e2b3ab671b29f31b6"; pub const BACKEND_COMPAT: &str = "asap-query-backend.v1"; #[derive(Debug, Clone)] @@ -336,6 +336,7 @@ pub enum ExactStateKind { MinMax, Increase, Rate, + IRate, } impl TryFrom<&SummaryFamilyType> for StateFamilyContract { @@ -351,6 +352,7 @@ impl TryFrom<&SummaryFamilyType> for StateFamilyContract { ExactKind::MinMax => ExactStateKind::MinMax, ExactKind::Increase => ExactStateKind::Increase, ExactKind::Rate => ExactStateKind::Rate, + ExactKind::IRate => ExactStateKind::IRate, }, }, SummaryFamilyType::Sketch(kind, _) => Self::Sketch { @@ -2522,6 +2524,7 @@ fn summary_agg_metric(node: &SummaryNode) -> Option { } } SummaryExpr::SummaryAgg { child, .. } => walk(child, metrics), + SummaryExpr::ValueOperation { child, .. } => walk(child, metrics), SummaryExpr::SummaryEstimate { summary_input, .. } => walk(summary_input, metrics), SummaryExpr::SummaryMerge { children } => { for child in children { @@ -3212,6 +3215,9 @@ fn collect_selected_materializations( None }; match &node.expr { + SummaryExpr::ValueOperation { child, .. } => { + walk(child, readout, composable, grouping.clone(), selected)?; + } SummaryExpr::BinaryOp { lhs, rhs, .. } if composable || crate::query_plan::exact_value_executable(node) => { diff --git a/control_plane/src/physical/post_asap/matcher.rs b/control_plane/src/physical/post_asap/matcher.rs index fadd444e..db69c59c 100644 --- a/control_plane/src/physical/post_asap/matcher.rs +++ b/control_plane/src/physical/post_asap/matcher.rs @@ -187,6 +187,7 @@ mod tests { ExactKind::MinMax => ExactParams::MinMax, ExactKind::Increase => ExactParams::Increase, ExactKind::Rate => ExactParams::Rate, + ExactKind::IRate => ExactParams::IRate, } } diff --git a/control_plane/src/physical/post_asap/tests.rs b/control_plane/src/physical/post_asap/tests.rs index 4a1aaae4..3cd0c879 100644 --- a/control_plane/src/physical/post_asap/tests.rs +++ b/control_plane/src/physical/post_asap/tests.rs @@ -110,6 +110,7 @@ fn node_is_archive(node: &Rc) -> bool { _ => false, }, SummaryExpr::SummaryAgg { child, .. } => node_is_archive(child), + SummaryExpr::ValueOperation { child, .. } => node_is_archive(child), SummaryExpr::SummaryEstimate { summary_input, .. } => node_is_archive(summary_input), SummaryExpr::SummaryMerge { children } => children.iter().any(node_is_archive), SummaryExpr::SummaryJoin { outer, inner, .. } => { diff --git a/control_plane/src/planner_selection.rs b/control_plane/src/planner_selection.rs index c464bfed..d0e78f40 100644 --- a/control_plane/src/planner_selection.rs +++ b/control_plane/src/planner_selection.rs @@ -143,7 +143,9 @@ pub fn select_summary( .ok_or(SelectionError::NoLegalCandidate)?; match candidate.replacement { Replacement::Summary(node) => Ok(node), - Replacement::Rewrite(_) => Err(SelectionError::UnexpectedRewrite), + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => { + Err(SelectionError::UnexpectedRewrite) + } } } @@ -235,7 +237,9 @@ pub fn select_summary_with_evidence( .ok_or(SelectionError::NoLegalCandidate)?; match candidate.replacement { Replacement::Summary(node) => Ok(node), - Replacement::Rewrite(_) => Err(SelectionError::UnexpectedRewrite), + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => { + Err(SelectionError::UnexpectedRewrite) + } } } diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index cab06fc4..4a5bb70a 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -519,6 +519,143 @@ where } let physical = match &node.expr { + SummaryExpr::ValueOperation { + child, + operation: + planner_types::post_asap::ValueOperation::Exact( + planner_types::post_asap::ExactOperation::Aggregate { + reduction, + measures, + having: None, + .. + }, + ), + timing: planner_types::post_asap::ExecutionTiming::ReadTime, + } if measures.len() == 1 => { + use planner_types::pre_asap::AggIntent; + let operation = match &measures[0] { + AggIntent::Sum { .. } => logical::Aggregation::Sum, + AggIntent::Count { .. } => logical::Aggregation::Count, + AggIntent::Min { .. } => logical::Aggregation::Min, + AggIntent::Max { .. } => logical::Aggregation::Max, + AggIntent::Avg { .. } => logical::Aggregation::Avg, + _ => { + return Err(QueryPlanError::Invalid( + "unsupported exact value aggregation".into(), + )) + } + }; + let keys = reduction.group_keys().ok_or_else(|| { + QueryPlanError::Invalid( + "per-entity exact value aggregation has no grouping".into(), + ) + })?; + let labels = keys + .keys() + .iter() + .map(|&column| { + child + .schema + .fields + .get(column) + .map(|field| field.name.clone()) + .ok_or_else(|| { + QueryPlanError::Invalid( + "unresolved exact aggregation column".into(), + ) + }) + }) + .collect::, _>>()?; + QueryPlanNode::Logical { + operator: logical::LogicalOperator::Aggregate { + operation, + grouping: logical::Grouping { + labels, + without: keys.is_without(), + }, + }, + inputs: vec![self.lower(child)?], + } + } + SummaryExpr::ValueOperation { + child: sort, + operation: planner_types::post_asap::ValueOperation::Limit { n, offset: 0 }, + timing: planner_types::post_asap::ExecutionTiming::ReadTime, + } => { + let SummaryExpr::ValueOperation { + child, + operation: planner_types::post_asap::ValueOperation::Sort { keys, partition_by }, + timing: planner_types::post_asap::ExecutionTiming::ReadTime, + } = &sort.expr + else { + return Err(QueryPlanError::Invalid( + "query-time Limit must consume a query-time Sort".into(), + )); + }; + if keys.len() != 1 || keys[0].ascending { + return Err(QueryPlanError::Invalid( + "only descending value-ranked TopK is executable".into(), + )); + } + let planner_types::pre_asap::QueryExpr::Column(sort_column) = &keys[0].expr else { + return Err(QueryPlanError::Invalid( + "TopK sort key must reference the child value column".into(), + )); + }; + if !matches!( + child + .schema + .fields + .get(*sort_column) + .map(|field| &field.dtype), + Some(SummaryFamilyType::Plain( + planner_types::pre_asap::DataType::Float64 + )) | Some(SummaryFamilyType::ExactAggregate(..)) + ) { + return Err(QueryPlanError::Invalid( + "TopK sort key must produce a numeric value".into(), + )); + } + let labels = partition_by + .keys() + .iter() + .map(|&column| { + child + .schema + .fields + .get(column) + .map(|field| field.name.clone()) + .ok_or_else(|| { + QueryPlanError::Invalid("unresolved TopK partition column".into()) + }) + }) + .collect::, _>>()?; + QueryPlanNode::Logical { + operator: logical::LogicalOperator::TopKSelection { + k: u64::try_from(*n).map_err(|_| { + QueryPlanError::Invalid("TopK limit exceeds u64".into()) + })?, + grouping: logical::Grouping { + labels, + without: partition_by.is_without(), + }, + }, + inputs: vec![self.lower(child)?], + } + } + SummaryExpr::ValueOperation { + child, + operation: planner_types::post_asap::ValueOperation::Sort { keys, .. }, + timing: planner_types::post_asap::ExecutionTiming::ReadTime, + } if keys.len() == 1 => QueryPlanNode::Logical { + operator: logical::LogicalOperator::Sort { + descending: !keys[0].ascending, + }, + inputs: vec![self.lower(child)?], + }, + SummaryExpr::ValueOperation { .. } => QueryPlanNode::ExactFallback { + reason: "unsupported post-ASAP value operation".into(), + }, SummaryExpr::BinaryOp { lhs, rhs, operator } if self.logical_source.is_some() => { let operator = logical::binary_operator(operator)?; QueryPlanNode::Logical { diff --git a/control_plane/src/query_plan/logical.rs b/control_plane/src/query_plan/logical.rs index f83d5bf5..c5a48e78 100644 --- a/control_plane/src/query_plan/logical.rs +++ b/control_plane/src/query_plan/logical.rs @@ -895,7 +895,9 @@ mod planner_workload_tests { entry["time_selection"]["lookback"] = (if window == 0 { 300_000 } else { window }).into(); fixture["query_workload"]["repeating_queries"] = vec![entry].into(); let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(fixture).unwrap(); - let (request, environment) = snapshot.planning_request().unwrap(); + let (request, environment) = snapshot + .planning_request() + .unwrap_or_else(|error| panic!("{query}: {error}")); PhysicalCompiler .compile(request, environment) .unwrap_or_else(|error| panic!("{query}: {error}")) @@ -906,11 +908,38 @@ mod planner_workload_tests { for query in [ "topk(2, sum by (job) (rate(backend_process_cpu_seconds_total[1h])))", "topk(2, sum by (job) (backend_process_resident_memory_bytes))", - "topk(1, sum by (job) (increase(backend_http_5xx_total[6h])) / sum by (job) (increase(backend_http_requests_total[6h])))", "topk(2, max_over_time(backend_retry_backlog_depth[6h]))", - "topk(1, sum by (job) (increase(backend_http_5xx_total[24h])) / scalar(sum(increase(backend_http_5xx_total[24h]))))", "topk(1, sum by (job) (rate(backend_process_cpu_seconds_total[6h])))", "topk(3, avg_over_time((sum by (job) (backend_process_resident_memory_bytes))[6h:]))", + ] { + let plan = compile_one(query); + let entry = plan.query_plan.entries.values().next().unwrap(); + assert!( + matches!( + entry.nodes[&entry.root], + QueryPlanNode::Logical { + operator: LogicalOperator::TopKSelection { .. }, + .. + } + ), + "{query}: {:?}", + entry.nodes[&entry.root] + ); + assert!( + !entry + .nodes + .values() + .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. })), + "{query}" + ); + } + } + + #[test] + fn planner_value_topk_preserves_direct_summary_children() { + for query in [ + "topk(2, rate(backend_process_cpu_seconds_total[1h]))", + "topk(2, max_over_time(backend_retry_backlog_depth[6h]))", ] { let plan = compile_one(query); let entry = plan.query_plan.entries.values().next().unwrap(); @@ -920,8 +949,12 @@ mod planner_workload_tests { operator: LogicalOperator::TopKSelection { .. }, .. } - ), "{query}: {:?}", entry.nodes[&entry.root]); - assert!(!entry.nodes.values().any(|node| matches!(node, QueryPlanNode::ExactFallback { .. })), "{query}"); + )); + assert!( + !entry.materialization_bindings().is_empty(), + "{query} must retain its SummaryStore child: {:?}", + entry.nodes + ); } } @@ -1245,6 +1278,7 @@ pub fn materialization_candidate_keys( visit(original, lhs, keys)?; visit(original, rhs, keys)?; } + SummaryExpr::ValueOperation { child, .. } => visit(original, child, keys)?, SummaryExpr::SummaryAgg { child, .. } => visit(original, child, keys)?, SummaryExpr::SummaryEstimate { summary_input, .. } | SummaryExpr::SummaryDelete { summary_input, .. } => { diff --git a/control_plane/src/query_planning.rs b/control_plane/src/query_planning.rs index 568a6d03..ccef785d 100644 --- a/control_plane/src/query_planning.rs +++ b/control_plane/src/query_planning.rs @@ -209,7 +209,8 @@ fn planned_capability( planner_types::post_asap::ExactKind::Sum | planner_types::post_asap::ExactKind::Count => asap_types::AggregationType::Sum, planner_types::post_asap::ExactKind::Increase - | planner_types::post_asap::ExactKind::Rate => { + | planner_types::post_asap::ExactKind::Rate + | planner_types::post_asap::ExactKind::IRate => { asap_types::AggregationType::Increase } planner_types::post_asap::ExactKind::MinMax => asap_types::AggregationType::MinMax, diff --git a/control_plane/src/replan.rs b/control_plane/src/replan.rs index aa2b5d73..e0af9191 100644 --- a/control_plane/src/replan.rs +++ b/control_plane/src/replan.rs @@ -671,6 +671,9 @@ impl Replanner { planner_types::post_asap::ExactKind::Rate => { planner_types::post_asap::ExactParams::Rate } + planner_types::post_asap::ExactKind::IRate => { + planner_types::post_asap::ExactParams::IRate + } }; use crate::physical::colored_dag::emitter::{ AggregationInput, BackendAggregation, BackendStageConfig, diff --git a/crates/asap_types/Cargo.toml b/crates/asap_types/Cargo.toml index b67cdde3..c31532fd 100644 --- a/crates/asap_types/Cargo.toml +++ b/crates/asap_types/Cargo.toml @@ -32,4 +32,4 @@ sha2 = "0.10" # exactly (`control_plane/Cargo.toml`) -- two different revs of the same # git dependency in one workspace resolve to two distinct Rust types that # won't unify. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "abb2f20e27091ac0c60715dc42b9b59c75b3447c" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "24735a35442c0ada3dd4ba1e2b3ab671b29f31b6" } diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index fd466afe..86bb1b67 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -316,7 +316,8 @@ impl FidelityGuarantee { match config.accumulator_spec().map(|s| s.family) { Ok(SummaryFamilyType::ExactAggregate( planner_types::post_asap::ExactKind::Increase - | planner_types::post_asap::ExactKind::Rate, + | planner_types::post_asap::ExactKind::Rate + | planner_types::post_asap::ExactKind::IRate, _, )) => Self::ExactCounter { model: "prometheus.extrapolated-rate.v1".into(), diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index 1677d4e5..27c75216 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -38,7 +38,7 @@ control_plane = { path = "../control_plane" } # reduction: Reduction, .. }`) are `pre_asap` types, in the same crate now # (not a separate `asap-ir` import). Query serving consumes the compiled # QueryPlan; these types are used at physical-plan compilation boundaries. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "abb2f20e27091ac0c60715dc42b9b59c75b3447c" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "24735a35442c0ada3dd4ba1e2b3ab671b29f31b6" } # Shared external (workspace) serde.workspace = true @@ -129,7 +129,7 @@ crc32fast = "1.4" # none of them. [dev-dependencies] -asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "abb2f20e27091ac0c60715dc42b9b59c75b3447c" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "24735a35442c0ada3dd4ba1e2b3ab671b29f31b6" } tempfile = "3.20.0" criterion = { version = "0.5", features = ["html_reports"] } tokio-tungstenite = "0.21" diff --git a/data_plane/src/query_engines/asap_query_engine/summary_exec.rs b/data_plane/src/query_engines/asap_query_engine/summary_exec.rs index 10debfd4..b3ba1e0c 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_exec.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_exec.rs @@ -247,6 +247,7 @@ pub fn execute( SummaryExpr::BinaryOp { .. } => Err(ExecError::NotYetSupported("BinaryOp")), SummaryExpr::SummarySubtract { .. } => Err(ExecError::NotYetSupported("SummarySubtract")), SummaryExpr::SummaryDelete { .. } => Err(ExecError::NotYetSupported("SummaryDelete")), + SummaryExpr::ValueOperation { .. } => Err(ExecError::NotYetSupported("ValueOperation")), } } From 90acebad40399c457a474c2685b976e8c7fed0b3 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 16:31:38 -0600 Subject: [PATCH 23/30] feat(planner): accept measured exact composition costs --- Cargo.lock | 6 +- control_plane/Cargo.toml | 6 +- control_plane/src/main.rs | 13 +- control_plane/src/physical/compiler.rs | 213 +++++++++++++++++- .../src/physical/post_asap/cost_model.rs | 134 ++++++++++- control_plane/src/physical/workload_cost.rs | 9 +- control_plane/src/planner_selection.rs | 3 + crates/asap_types/Cargo.toml | 2 +- data_plane/Cargo.toml | 4 +- 9 files changed, 365 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 286f3a07..16a92aa8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -343,7 +343,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=24735a35442c0ada3dd4ba1e2b3ab671b29f31b6#24735a35442c0ada3dd4ba1e2b3ab671b29f31b6" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=54f581b4be69a0d62379869916e387b899e05a43#54f581b4be69a0d62379869916e387b899e05a43" dependencies = [ "asap-types", "serde", @@ -354,7 +354,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=24735a35442c0ada3dd4ba1e2b3ab671b29f31b6#24735a35442c0ada3dd4ba1e2b3ab671b29f31b6" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=54f581b4be69a0d62379869916e387b899e05a43#54f581b4be69a0d62379869916e387b899e05a43" dependencies = [ "asap-types", "promql-parser 0.10.0", @@ -374,7 +374,7 @@ dependencies = [ [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=24735a35442c0ada3dd4ba1e2b3ab671b29f31b6#24735a35442c0ada3dd4ba1e2b3ab671b29f31b6" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=54f581b4be69a0d62379869916e387b899e05a43#54f581b4be69a0d62379869916e387b899e05a43" dependencies = [ "serde", "serde_json", diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index 3cfa5e7d..0719a487 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -93,8 +93,8 @@ asap_types.workspace = true # scaffolding, unaware that `data_plane`'s `summary_executor.rs` in *this* # repo is a real one. Vendored locally instead of chased upstream -- see # `data_plane/src/query_engines/asap_query_engine/summary_exec.rs`. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "24735a35442c0ada3dd4ba1e2b3ab671b29f31b6" } -asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "24735a35442c0ada3dd4ba1e2b3ab671b29f31b6" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "54f581b4be69a0d62379869916e387b899e05a43" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "54f581b4be69a0d62379869916e387b899e05a43" } # L1 adoption (design-target-architecture.md Part B): the PromQL front # end itself, replacing control_plane's own query_parser/promql.rs. @@ -102,7 +102,7 @@ asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = # `planner-types`/`asap-aware-mapping` above -- these three MUST move # together (two revs of the same upstream repo's types in one workspace # resolve to distinct Rust types that won't unify). -asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "24735a35442c0ada3dd4ba1e2b3ab671b29f31b6" } +asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "54f581b4be69a0d62379869916e387b899e05a43" } [dev-dependencies] tokio = { version = "1", features = ["full", "test-util"] } diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index 5b770444..fd5687a8 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -602,6 +602,9 @@ struct CompileAndPublishPhysicalPlanRequest { #[serde(default)] evidence: HashMap, #[serde(default)] + exact_composition_costs: + HashMap>, + #[serde(default)] runtime_adaptation_evidence: Vec, planner_revision: String, max_evidence_age_ms: u64, @@ -820,9 +823,12 @@ fn compile_physical_plan_request( }); } - if let Err(error) = - physical::compiler::select_workload_roots(&mut queries, canonical_roots, &request.evidence) - { + if let Err(error) = physical::compiler::select_workload_roots( + &mut queries, + canonical_roots, + &request.evidence, + &request.exact_composition_costs, + ) { return Err((StatusCode::UNPROCESSABLE_ENTITY, error.to_string())); } @@ -832,6 +838,7 @@ fn compile_physical_plan_request( hybrid_execution: false, materialization_policy: None, evidence: request.evidence, + exact_composition_costs: request.exact_composition_costs, planner_revision: request.planner_revision, source_sample_interval_ms: None, query_staleness_margin_ms: 0, diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 3051e5e7..8afc0d3e 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -29,7 +29,7 @@ use serde_json::{json, Value}; use thiserror::Error; use crate::physical::colored_dag::emitter::{AggregationInput, BackendAggregation}; -use crate::physical::post_asap::cost_model::ControlPlaneCostModel; +use crate::physical::post_asap::cost_model::{ControlPlaneCostModel, ExactCompositionCostEvidence}; use crate::query_plan::{ canonical_promql, FallbackPolicy, InstantExecution, MaterializationBinding, PhysicalGrouping, QueryPlan, QueryPlanEntry, @@ -37,7 +37,7 @@ use crate::query_plan::{ use crate::types_v2::AccuracyTarget; use planner_types::pre_asap::Source; -pub const PLANNER_REVISION: &str = "24735a35442c0ada3dd4ba1e2b3ab671b29f31b6"; +pub const PLANNER_REVISION: &str = "54f581b4be69a0d62379869916e387b899e05a43"; pub const BACKEND_COMPAT: &str = "asap-query-backend.v1"; #[derive(Debug, Clone)] @@ -130,6 +130,9 @@ pub struct PlanningRequest { pub query_workload: Option, pub queries: Vec, pub evidence: HashMap, + /// Fresh measured costs for Planner exact/summary composition sites, + /// scoped to query IDs just like accuracy evidence. + pub exact_composition_costs: HashMap>, pub planner_revision: String, /// Observed cadence of source samples. Exact temporal panes must divide /// both this cadence and the repeated-query evaluation interval. @@ -207,6 +210,10 @@ pub struct BackendLocalImplementation { /// before workload selection so one query cannot borrow another's evidence. #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub topk_evidence: HashMap, + /// Measured exact/summary composition profiles keyed by registered + /// PromQL. Missing rows keep the corresponding Planner site opaque. + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub exact_composition_costs: HashMap>, } fn u64_is_zero(value: &u64) -> bool { @@ -1805,7 +1812,33 @@ impl BackendLocalPlanningSnapshot { runtime_policy: RuntimeRulePolicy::default(), }); } - select_workload_roots(&mut queries, canonical_roots, &topk_evidence_by_id)?; + let mut exact_costs_by_id = HashMap::new(); + for (index, entry) in workload.entries().enumerate() { + if let Some(rows) = self + .implementation + .exact_composition_costs + .get(&entry.query.0) + { + for row in rows { + row.validate( + self.environment.observed_at_unix_ms, + self.environment.max_evidence_age_ms, + ) + .map_err(|reason| { + CompileError::Snapshot(format!( + "query {index}: invalid exact-composition evidence: {reason}" + )) + })?; + } + exact_costs_by_id.insert(format!("compat-query-{index}"), rows.clone()); + } + } + select_workload_roots( + &mut queries, + canonical_roots, + &topk_evidence_by_id, + &exact_costs_by_id, + )?; // Composable lowering residualizes unsafe leaves individually; retain Planner siblings. Ok(( PlanningRequest { @@ -1814,6 +1847,7 @@ impl BackendLocalPlanningSnapshot { query_workload: Some(workload), queries, evidence: topk_evidence_by_id, + exact_composition_costs: exact_costs_by_id, planner_revision: PLANNER_REVISION.into(), source_sample_interval_ms: self.implementation.source_sample_interval_ms, query_staleness_margin_ms: self.implementation.query_staleness_margin_ms, @@ -2562,6 +2596,7 @@ pub fn select_workload_roots( queries: &mut [PlanningQuery], roots: Vec>, evidence: &HashMap, + exact_costs: &HashMap>, ) -> Result<(), CompileError> { if roots.len() != queries.len() { return Err(CompileError::Snapshot( @@ -2572,9 +2607,9 @@ pub fn select_workload_roots( Vec::new(); for (index, root) in roots.into_iter().enumerate() { let accuracy = &queries[index].accuracy; - let certificate_scope = evidence - .contains_key(&queries[index].query_id) - .then(|| queries[index].query_id.clone()); + let certificate_scope = (evidence.contains_key(&queries[index].query_id) + || exact_costs.contains_key(&queries[index].query_id)) + .then(|| queries[index].query_id.clone()); if let Some((_, _, roots)) = cohorts .iter_mut() .find(|(target, scope, _)| target == accuracy && scope == &certificate_scope) @@ -2585,7 +2620,13 @@ pub fn select_workload_roots( } } for (accuracy, scope, roots) in cohorts { - let model = ControlPlaneCostModel::new(accuracy.clone()); + let model = ControlPlaneCostModel::new(accuracy.clone()).with_exact_composition_costs( + scope + .as_ref() + .and_then(|id| exact_costs.get(id)) + .cloned() + .unwrap_or_default(), + ); let certificate = scope.as_ref().and_then(|id| evidence.get(id)); let selected = crate::planner_selection::select_workload_with_evidence( roots, @@ -3642,6 +3683,7 @@ mod tests { runtime_policy: RuntimeRulePolicy::default(), }], evidence: evidence_by_query, + exact_composition_costs: HashMap::new(), planner_revision: PLANNER_REVISION.into(), source_sample_interval_ms: None, query_staleness_margin_ms: 0, @@ -3652,6 +3694,146 @@ mod tests { request_with_evidence(query_id, promql, None).expect("post-ASAP selection") } + fn measured_exact_composition_rows( + expr: &QueryExpr, + observed_at_unix_ms: u64, + ) -> Vec { + fn visit( + expr: &QueryExpr, + observed_at_unix_ms: u64, + rows: &mut Vec, + ) { + use planner_types::post_asap::ExactOperation; + match expr { + QueryExpr::Aggregate { + reduction, + measures, + output_names, + having, + child, + } => { + rows.push(ExactCompositionCostEvidence { + target: serde_json::to_value(expr).unwrap(), + operation: serde_json::to_value(ExactOperation::Aggregate { + reduction: reduction.clone(), + measures: measures.clone(), + output_names: output_names.clone(), + having: having.clone(), + }) + .unwrap(), + placement: + crate::physical::post_asap::cost_model::ExactOperationPlacement::Read, + expected_input_rows: 100.0, + expected_output_rows: 10.0, + exact_cpu_ns_per_row: 2.0, + summary_maintenance_cpu_ns_per_update: 3.0, + summary_read_cpu_ns: 20.0, + update_rate_per_second: 100.0, + evaluation_rate_per_second: 0.1, + raw_recompute_cpu_ns: 100_000.0, + observed_peak_memory_bytes: 4096, + data_snapshot_id: "topk-planning-fixture".into(), + model_version: "measured-test-v1".into(), + observed_at_unix_ms, + valid_for_ms: 60_000, + }); + visit(child, observed_at_unix_ms, rows); + } + QueryExpr::Sort { child, .. } + | QueryExpr::Limit { child, .. } + | QueryExpr::TimeRange { child, .. } + | QueryExpr::Filter { child, .. } + | QueryExpr::Project { child, .. } + | QueryExpr::PromqlSubquery { child, .. } + | QueryExpr::TimeShift { child, .. } + | QueryExpr::PromqlScalarBridge(child) + | QueryExpr::PromqlVectorFromScalar(child) + | QueryExpr::PromqlScalarFromVector(child) + | QueryExpr::PromqlRelabel { child, .. } + | QueryExpr::PromqlSeriesSample { child, .. } + | QueryExpr::Dedup { child, .. } => visit(child, observed_at_unix_ms, rows), + _ => {} + } + } + let mut rows = Vec::new(); + visit(expr, observed_at_unix_ms, &mut rows); + rows + } + + #[test] + fn sum_rate_uses_summary_child_only_with_measured_composition_costs() { + let promql = "sum by (job) (rate(m[1m]))"; + let mut with_evidence = request("topk-rate", promql); + with_evidence.hybrid_execution = true; + let root = Rc::new( + crate::query_parser::parse_query_expr_canonical( + promql, + with_evidence.queries[0].accuracy.clone(), + ) + .unwrap(), + ); + with_evidence.exact_composition_costs.insert( + "topk-rate".into(), + measured_exact_composition_rows(&root, 9_500), + ); + select_workload_roots( + &mut with_evidence.queries, + vec![root], + &with_evidence.evidence, + &with_evidence.exact_composition_costs, + ) + .unwrap(); + let mut backend = environment(10_000); + backend.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; + backend.collector_ids.clear(); + let plan = PhysicalCompiler.compile(with_evidence, backend).unwrap(); + assert!( + !plan.summary_catalog.materializations.is_empty(), + "measured exact-composition evidence must expose the rate child as a SummaryStore binding" + ); + } + + #[test] + fn sum_rate_without_composition_costs_does_not_invent_a_composition_cost() { + let promql = "sum by (job) (rate(m[1m]))"; + let mut unavailable = request("topk-rate", promql); + unavailable.hybrid_execution = true; + let root = Rc::new( + crate::query_parser::parse_query_expr_canonical( + promql, + unavailable.queries[0].accuracy.clone(), + ) + .unwrap(), + ); + select_workload_roots( + &mut unavailable.queries, + vec![root], + &unavailable.evidence, + &unavailable.exact_composition_costs, + ) + .unwrap(); + let mut backend = environment(10_000); + backend.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; + backend.collector_ids.clear(); + let plan = PhysicalCompiler.compile(unavailable, backend).unwrap(); + // Planner 54f can realize this particular shape directly as an exact + // counter readout plus a query-time reduce; it does not require an + // ExactComposition candidate. The absence of evidence must therefore + // leave that direct legal path intact rather than inventing a composed + // cost or forcing an exact fallback. + assert!(!plan.summary_catalog.materializations.is_empty()); + let entry = plan + .query_plan + .entries + .values() + .find(|entry| entry.query_id == "topk-rate") + .unwrap(); + assert!(entry + .nodes + .values() + .any(|node| matches!(node, crate::query_plan::QueryPlanNode::ExactReadout { .. }))); + } + #[test] fn exact_temporal_pane_uses_observed_source_and_query_cadence() { assert_eq!(exact_temporal_pane_secs(86_400, 60_000, Some(30_000)), 30); @@ -3689,7 +3871,13 @@ mod tests { ) }) .collect(); - select_workload_roots(&mut workload.queries, roots, &workload.evidence).unwrap(); + select_workload_roots( + &mut workload.queries, + roots, + &workload.evidence, + &workload.exact_composition_costs, + ) + .unwrap(); let bundle = PhysicalCompiler .compile(workload, environment(10000)) .unwrap(); @@ -3710,7 +3898,13 @@ mod tests { #[test] fn shared_selection_rejects_incomplete_root_mapping() { let mut workload = request("q", "quantile_over_time(0.9, m[1m])"); - assert!(select_workload_roots(&mut workload.queries, vec![], &workload.evidence).is_err()); + assert!(select_workload_roots( + &mut workload.queries, + vec![], + &workload.evidence, + &workload.exact_composition_costs, + ) + .is_err()); } // Adding another readout adds recurring reads, not another update stream. @@ -4520,6 +4714,7 @@ mod tests { source_sample_interval_ms: None, query_staleness_margin_ms: 0, topk_evidence: HashMap::new(), + exact_composition_costs: HashMap::new(), }, environment, }; diff --git a/control_plane/src/physical/post_asap/cost_model.rs b/control_plane/src/physical/post_asap/cost_model.rs index 94c185be..f93690e4 100644 --- a/control_plane/src/physical/post_asap/cost_model.rs +++ b/control_plane/src/physical/post_asap/cost_model.rs @@ -42,8 +42,10 @@ use asap_aware_mapping::empirical_comparison::{ }; use asap_aware_mapping::empirical_cost::EmpiricalEvidenceProvider; use asap_aware_mapping::{ - CompleteSummaryCandidateEstimate, CostModel, Horizon, Implementation, - SummaryMaintenanceCapabilities, SummaryMaintenanceLifecycleCostInputs, + CompleteSummaryCandidateEstimate, CostModel, CostProvenance, EvaluationRate, + ExactCompositionCostInputs, ExactCompositionCostRequest, Horizon, Implementation, + OperationPlacement, SummaryMaintenanceCapabilities, SummaryMaintenanceLifecycleCostInputs, + ValueOperationCapabilities, }; use planner_types::post_asap::{ SketchAlgorithm, SketchParams, SketchQuery, SummaryWindowFramework, @@ -54,6 +56,80 @@ use crate::physical::deployment_cost::wire::WireCostTable; use crate::planner_selection::FREQUENCY_EXT_KIND; use crate::types_v2::AccuracyTarget; use planner_types::pre_asap::AggIntent; +use serde::{Deserialize, Serialize}; + +/// One measured execution profile for an exact operator composed with a +/// maintained summary. Values use CPU nanoseconds so every term in Planner's +/// recurring-cost formula has the same physical unit. Peak memory is retained +/// as measured resource evidence and reported separately; it is deliberately +/// not converted into CPU cost by an invented weight. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ExactCompositionCostEvidence { + /// Exact pre-ASAP target serialized with the pinned Planner revision. + pub target: serde_json::Value, + /// Exact operation serialized with the pinned Planner revision. + pub operation: serde_json::Value, + pub placement: ExactOperationPlacement, + pub expected_input_rows: f64, + pub expected_output_rows: f64, + pub exact_cpu_ns_per_row: f64, + pub summary_maintenance_cpu_ns_per_update: f64, + pub summary_read_cpu_ns: f64, + pub update_rate_per_second: f64, + pub evaluation_rate_per_second: f64, + pub raw_recompute_cpu_ns: f64, + pub observed_peak_memory_bytes: u64, + pub data_snapshot_id: String, + pub model_version: String, + pub observed_at_unix_ms: u64, + pub valid_for_ms: u64, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ExactOperationPlacement { + Read, + Maintenance, +} + +impl ExactCompositionCostEvidence { + pub fn validate(&self, now_unix_ms: u64, max_age_ms: u64) -> Result<(), String> { + let positive = [ + self.exact_cpu_ns_per_row, + self.summary_maintenance_cpu_ns_per_update, + self.summary_read_cpu_ns, + self.update_rate_per_second, + self.evaluation_rate_per_second, + self.raw_recompute_cpu_ns, + ]; + let rows = [self.expected_input_rows, self.expected_output_rows]; + if positive.iter().any(|v| !v.is_finite() || *v <= 0.0) + || rows.iter().any(|v| !v.is_finite() || *v < 0.0) + || self.observed_peak_memory_bytes == 0 + || self.data_snapshot_id.trim().is_empty() + || self.model_version.trim().is_empty() + || self.valid_for_ms == 0 + || self.observed_at_unix_ms > now_unix_ms + || now_unix_ms - self.observed_at_unix_ms > self.valid_for_ms.min(max_age_ms) + { + return Err( + "missing, non-physical, future, or stale exact-composition evidence".into(), + ); + } + Ok(()) + } + + fn matches(&self, request: &ExactCompositionCostRequest<'_>) -> bool { + let placement = match request.composition.placement { + OperationPlacement::Read => ExactOperationPlacement::Read, + OperationPlacement::Maintenance => ExactOperationPlacement::Maintenance, + }; + self.placement == placement + && serde_json::to_value(request.target).ok().as_ref() == Some(&self.target) + && serde_json::to_value(&request.composition.op).ok().as_ref() == Some(&self.operation) + } +} /// See module docs. pub struct ControlPlaneCostModel { @@ -66,6 +142,7 @@ pub struct ControlPlaneCostModel { window_framework_costs: Vec<(Option, SummaryWindowFramework, Cost)>, offline_evidence: Option, offline_frequency_comparison: Option<(OfflineComparisonEvidence, OfflineComparisonRequest)>, + exact_composition_costs: Vec, } impl ControlPlaneCostModel { @@ -77,9 +154,18 @@ impl ControlPlaneCostModel { window_framework_costs: Vec::new(), offline_evidence: None, offline_frequency_comparison: None, + exact_composition_costs: Vec::new(), } } + pub fn with_exact_composition_costs( + mut self, + costs: Vec, + ) -> Self { + self.exact_composition_costs = costs; + self + } + /// Use offline update CPU evidence for algorithm ordering. Physical costs /// and accuracy guarantees retain their deployment-specific contracts. pub fn with_offline_evidence(mut self, evidence: EmpiricalEvidenceProvider) -> Self { @@ -359,6 +445,50 @@ fn intent_accuracy(intent: &AggIntent) -> AccuracyTarget { } impl CostModel for ControlPlaneCostModel { + fn value_operation_capabilities(&self) -> ValueOperationCapabilities { + ValueOperationCapabilities { + read_time: true, + maintenance_time: false, + } + } + + fn exact_composition_cost_inputs( + &self, + request: &ExactCompositionCostRequest<'_>, + ) -> ExactCompositionCostInputs { + let provenance = || CostProvenance { + model: "ASAPQuery measured exact composition".into(), + version: "unavailable".into(), + }; + let Some(row) = self + .exact_composition_costs + .iter() + .find(|row| row.matches(request)) + else { + return ExactCompositionCostInputs::unknown(provenance()); + }; + ExactCompositionCostInputs { + exact_cost_per_row: Some(row.exact_cpu_ns_per_row), + expected_input_rows: Some(row.expected_input_rows), + expected_output_rows: Some(row.expected_output_rows), + summary_maintenance_cost_per_update: Some(row.summary_maintenance_cpu_ns_per_update), + summary_read_cost: Some(row.summary_read_cpu_ns), + update_rate: Some(row.update_rate_per_second), + evaluation_rate: Some(EvaluationRate( + row.evaluation_rate_per_second * request.effective_consumer_count as f64, + )), + raw_recompute_cost: Some(row.raw_recompute_cpu_ns), + unit: asap_aware_mapping::CostUnit::CostUnitsPerSecond, + provenance: CostProvenance { + model: format!( + "ASAPQuery measured exact composition ({})", + row.data_snapshot_id + ), + version: row.model_version.clone(), + }, + } + } + fn summary_maintenance_lifecycle_cost_inputs( &self, _summary: &planner_types::post_asap::SummaryNode, diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index c0fd2315..f8bd122a 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -927,8 +927,13 @@ mod tests { ) }) .collect(); - super::super::compiler::select_workload_roots(&mut shared.queries, roots, &shared.evidence) - .unwrap(); + super::super::compiler::select_workload_roots( + &mut shared.queries, + roots, + &shared.evidence, + &shared.exact_composition_costs, + ) + .unwrap(); let plan = PhysicalCompiler.compile(shared.clone(), env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); let second = manifest(&plan, &shared.queries).unwrap(); diff --git a/control_plane/src/planner_selection.rs b/control_plane/src/planner_selection.rs index d0e78f40..5ae98b1e 100644 --- a/control_plane/src/planner_selection.rs +++ b/control_plane/src/planner_selection.rs @@ -187,6 +187,9 @@ pub fn select_workload_with_evidence( &asap_aware_mapping::EqualSplitAllocator, evidence, )), + Box::new(asap_aware_mapping::ExactCompositionStrategy::new( + cost_model, + )), Box::new(asap_aware_mapping::SemanticEquivalentRewriteStrategy), ]; let space = asap_aware_mapping::search_workload_with_targets( diff --git a/crates/asap_types/Cargo.toml b/crates/asap_types/Cargo.toml index c31532fd..4fb67347 100644 --- a/crates/asap_types/Cargo.toml +++ b/crates/asap_types/Cargo.toml @@ -32,4 +32,4 @@ sha2 = "0.10" # exactly (`control_plane/Cargo.toml`) -- two different revs of the same # git dependency in one workspace resolve to two distinct Rust types that # won't unify. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "24735a35442c0ada3dd4ba1e2b3ab671b29f31b6" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "54f581b4be69a0d62379869916e387b899e05a43" } diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index 27c75216..27f65b12 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -38,7 +38,7 @@ control_plane = { path = "../control_plane" } # reduction: Reduction, .. }`) are `pre_asap` types, in the same crate now # (not a separate `asap-ir` import). Query serving consumes the compiled # QueryPlan; these types are used at physical-plan compilation boundaries. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "24735a35442c0ada3dd4ba1e2b3ab671b29f31b6" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "54f581b4be69a0d62379869916e387b899e05a43" } # Shared external (workspace) serde.workspace = true @@ -129,7 +129,7 @@ crc32fast = "1.4" # none of them. [dev-dependencies] -asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "24735a35442c0ada3dd4ba1e2b3ab671b29f31b6" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "54f581b4be69a0d62379869916e387b899e05a43" } tempfile = "3.20.0" criterion = { version = "0.5", features = ["html_reports"] } tokio-tungstenite = "0.21" From daf9bac6a6d658025fcda9cccb84e1a2b9ad7351 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 16:42:35 -0600 Subject: [PATCH 24/30] Integrate recursive Planner value operations --- Cargo.lock | 6 +++--- control_plane/Cargo.toml | 6 +++--- control_plane/src/physical/compiler.rs | 4 ++-- control_plane/src/physical/post_asap/tests.rs | 19 ++++++------------- control_plane/src/planner_selection.rs | 10 +++++++++- control_plane/src/query_plan.rs | 13 +++++++++++++ crates/asap_types/Cargo.toml | 2 +- data_plane/Cargo.toml | 4 ++-- 8 files changed, 39 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 16a92aa8..87462f10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -343,7 +343,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=54f581b4be69a0d62379869916e387b899e05a43#54f581b4be69a0d62379869916e387b899e05a43" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=661154b6d6e8b6b76102fb9d8cdcc440b298418f#661154b6d6e8b6b76102fb9d8cdcc440b298418f" dependencies = [ "asap-types", "serde", @@ -354,7 +354,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=54f581b4be69a0d62379869916e387b899e05a43#54f581b4be69a0d62379869916e387b899e05a43" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=661154b6d6e8b6b76102fb9d8cdcc440b298418f#661154b6d6e8b6b76102fb9d8cdcc440b298418f" dependencies = [ "asap-types", "promql-parser 0.10.0", @@ -374,7 +374,7 @@ dependencies = [ [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=54f581b4be69a0d62379869916e387b899e05a43#54f581b4be69a0d62379869916e387b899e05a43" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=661154b6d6e8b6b76102fb9d8cdcc440b298418f#661154b6d6e8b6b76102fb9d8cdcc440b298418f" dependencies = [ "serde", "serde_json", diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index 0719a487..ac662ed4 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -93,8 +93,8 @@ asap_types.workspace = true # scaffolding, unaware that `data_plane`'s `summary_executor.rs` in *this* # repo is a real one. Vendored locally instead of chased upstream -- see # `data_plane/src/query_engines/asap_query_engine/summary_exec.rs`. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "54f581b4be69a0d62379869916e387b899e05a43" } -asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "54f581b4be69a0d62379869916e387b899e05a43" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "661154b6d6e8b6b76102fb9d8cdcc440b298418f" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "661154b6d6e8b6b76102fb9d8cdcc440b298418f" } # L1 adoption (design-target-architecture.md Part B): the PromQL front # end itself, replacing control_plane's own query_parser/promql.rs. @@ -102,7 +102,7 @@ asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = # `planner-types`/`asap-aware-mapping` above -- these three MUST move # together (two revs of the same upstream repo's types in one workspace # resolve to distinct Rust types that won't unify). -asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "54f581b4be69a0d62379869916e387b899e05a43" } +asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "661154b6d6e8b6b76102fb9d8cdcc440b298418f" } [dev-dependencies] tokio = { version = "1", features = ["full", "test-util"] } diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 8afc0d3e..9421b43e 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -37,7 +37,7 @@ use crate::query_plan::{ use crate::types_v2::AccuracyTarget; use planner_types::pre_asap::Source; -pub const PLANNER_REVISION: &str = "54f581b4be69a0d62379869916e387b899e05a43"; +pub const PLANNER_REVISION: &str = "661154b6d6e8b6b76102fb9d8cdcc440b298418f"; pub const BACKEND_COMPAT: &str = "asap-query-backend.v1"; #[derive(Debug, Clone)] @@ -3963,7 +3963,7 @@ mod tests { PhysicalDeploymentTarget::BackendLocalRemoteWrite, ] { let (first, second) = if target == PhysicalDeploymentTarget::BackendLocalRemoteWrite { - ("sum(sum_over_time(m[1m]))", "sum(sum_over_time(m[1m])) * 2") + ("sum(rate(m[1m]))", "sum by (job) (rate(m[1m]))") } else { ( "quantile_over_time(0.90, m[1m])", diff --git a/control_plane/src/physical/post_asap/tests.rs b/control_plane/src/physical/post_asap/tests.rs index 3cd0c879..852608da 100644 --- a/control_plane/src/physical/post_asap/tests.rs +++ b/control_plane/src/physical/post_asap/tests.rs @@ -763,24 +763,17 @@ fn phase_b_e2e_rate_falls_through_to_logical() { ); } -/// `topk.yaml` — `topk(10, sum by (label) (rate(...))`. The legacy -/// planner emits a CountSketch+heap row. Control plane path: the parser -/// recognises `topk` as a special node that lowers to `AggIntent::TopK`. -/// At the time of writing, the control plane's `parse_query` may flatten -/// `topk` differently (no `inside_topk` propagation through TopK + -/// nested aggregate). The test asserts the END-STATE: either a -/// CountSketch summary fired, OR a Logical pass-through (which Phase γ -/// can decide whether to refine). The contract Phase β cares about is -/// that the bound expression is well-formed. +/// The legacy single-expression binder cannot choose a frequency sketch for +/// value-ranked `topk(sum(rate(...)))` without membership evidence. The +/// workload planner handles this query as query-time Sort+Limit over its +/// recursively planned child; its coverage lives in `query_plan::logical`. #[test] -fn phase_b_e2e_topk_well_formed() { +fn phase_b_legacy_topk_requires_membership_evidence() { let query = "topk(10, sum by (instance) (rate(http_requests_total[5m])))"; let accuracy = AccuracyTarget::Epsilon(0.05); let expr = crate::query_parser::parse_query_expr_canonical(query, accuracy.clone()) .expect("TopK parses"); - // Current Planner supports the temporal TopK shape; the old pin rejected it. - let bound = bind_query_expr(&expr, accuracy).expect("temporal TopK binds"); - assert!(matches!(bound, PhysicalExpr::Committed(_))); + assert!(bind_query_expr(&expr, accuracy).is_err()); } /// Archive-only routing through the full L1→L3→L4 pipeline. Asserts the diff --git a/control_plane/src/planner_selection.rs b/control_plane/src/planner_selection.rs index 5ae98b1e..48d95158 100644 --- a/control_plane/src/planner_selection.rs +++ b/control_plane/src/planner_selection.rs @@ -347,6 +347,14 @@ mod workload_tests { let SummaryExpr::BinaryOp { lhs, .. } = &roots[1].1.expr else { panic!("{:?}", roots[1].1) }; - assert!(Rc::ptr_eq(&roots[0].1, lhs)); + let shared = match &lhs.expr { + SummaryExpr::ValueOperation { + child, + operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, + .. + } => child, + _ => lhs, + }; + assert!(Rc::ptr_eq(&roots[0].1, shared)); } } diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index 4a5bb70a..edb5801e 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -496,6 +496,19 @@ where if let Some(id) = self.seen.get(&identity) { return Ok(*id); } + if let SummaryExpr::ValueOperation { + child, + operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, + timing: planner_types::post_asap::ExecutionTiming::ReadTime, + } = &node.expr + { + // SummaryAgg lowering already emits the family-specific ExactReadout. + // Preserve the Planner's explicit state boundary without adding a + // second runtime readout node. + let child_id = self.lower(child)?; + self.seen.insert(identity, child_id); + return Ok(child_id); + } let id = QueryNodeId(self.next_id); self.next_id += 1; self.seen.insert(identity, id); diff --git a/crates/asap_types/Cargo.toml b/crates/asap_types/Cargo.toml index 4fb67347..ab10cf39 100644 --- a/crates/asap_types/Cargo.toml +++ b/crates/asap_types/Cargo.toml @@ -32,4 +32,4 @@ sha2 = "0.10" # exactly (`control_plane/Cargo.toml`) -- two different revs of the same # git dependency in one workspace resolve to two distinct Rust types that # won't unify. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "54f581b4be69a0d62379869916e387b899e05a43" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "661154b6d6e8b6b76102fb9d8cdcc440b298418f" } diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index 27f65b12..ef7ecfb0 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -38,7 +38,7 @@ control_plane = { path = "../control_plane" } # reduction: Reduction, .. }`) are `pre_asap` types, in the same crate now # (not a separate `asap-ir` import). Query serving consumes the compiled # QueryPlan; these types are used at physical-plan compilation boundaries. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "54f581b4be69a0d62379869916e387b899e05a43" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "661154b6d6e8b6b76102fb9d8cdcc440b298418f" } # Shared external (workspace) serde.workspace = true @@ -129,7 +129,7 @@ crc32fast = "1.4" # none of them. [dev-dependencies] -asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "54f581b4be69a0d62379869916e387b899e05a43" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "661154b6d6e8b6b76102fb9d8cdcc440b298418f" } tempfile = "3.20.0" criterion = { version = "0.5", features = ["html_reports"] } tokio-tungstenite = "0.21" From 7e48799251ce5c378617306c5a0945cf506ba734 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 17:00:19 -0600 Subject: [PATCH 25/30] Pin the merged recursive Planner release --- Cargo.lock | 6 +++--- control_plane/Cargo.toml | 12 +++++------- control_plane/src/physical/compiler.rs | 2 +- crates/asap_types/Cargo.toml | 2 +- data_plane/Cargo.toml | 4 ++-- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 87462f10..347672b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -343,7 +343,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=661154b6d6e8b6b76102fb9d8cdcc440b298418f#661154b6d6e8b6b76102fb9d8cdcc440b298418f" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=2270f1f8f98b64e2cc8aa5b7602a570aa0e95f1e#2270f1f8f98b64e2cc8aa5b7602a570aa0e95f1e" dependencies = [ "asap-types", "serde", @@ -354,7 +354,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=661154b6d6e8b6b76102fb9d8cdcc440b298418f#661154b6d6e8b6b76102fb9d8cdcc440b298418f" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=2270f1f8f98b64e2cc8aa5b7602a570aa0e95f1e#2270f1f8f98b64e2cc8aa5b7602a570aa0e95f1e" dependencies = [ "asap-types", "promql-parser 0.10.0", @@ -374,7 +374,7 @@ dependencies = [ [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=661154b6d6e8b6b76102fb9d8cdcc440b298418f#661154b6d6e8b6b76102fb9d8cdcc440b298418f" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=2270f1f8f98b64e2cc8aa5b7602a570aa0e95f1e#2270f1f8f98b64e2cc8aa5b7602a570aa0e95f1e" dependencies = [ "serde", "serde_json", diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index ac662ed4..f5dd89b4 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -49,13 +49,11 @@ tonic = { version = "0.12", features = ["gzip"] } tokio-stream = { version = "0.1", features = ["net"] } asap_types.workspace = true -# Transitional pin (docs/migration-plan-backend-plan.md Phase 1b; see also -# control_plane/docs/design-asapplanner-pin-migration.md for the migration -# this bump landed): locked to a specific commit on ASAPPlanner's main, +# Locked to a specific commit on ASAPPlanner's main (see +# control_plane/docs/design-asapplanner-pin-migration.md): # not a tag -- ASAPPlanner has no tagged releases yet. Re-pin as # ASAPPlanner's IR evolves; move to a tag once one exists. # -# Bumped to cb70086 (main tip, 2026-08-22) from cc18c987. In between, # ASAPPlanner consolidated its crates (ASAPPlanner#198): `asap-ir` and # `asap-sketch` merged into one crate -- Cargo package `asap-types`, with # `pre_asap`/`post_asap` modules replacing the old L3/L4 split -- and @@ -93,8 +91,8 @@ asap_types.workspace = true # scaffolding, unaware that `data_plane`'s `summary_executor.rs` in *this* # repo is a real one. Vendored locally instead of chased upstream -- see # `data_plane/src/query_engines/asap_query_engine/summary_exec.rs`. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "661154b6d6e8b6b76102fb9d8cdcc440b298418f" } -asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "661154b6d6e8b6b76102fb9d8cdcc440b298418f" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "2270f1f8f98b64e2cc8aa5b7602a570aa0e95f1e" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "2270f1f8f98b64e2cc8aa5b7602a570aa0e95f1e" } # L1 adoption (design-target-architecture.md Part B): the PromQL front # end itself, replacing control_plane's own query_parser/promql.rs. @@ -102,7 +100,7 @@ asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = # `planner-types`/`asap-aware-mapping` above -- these three MUST move # together (two revs of the same upstream repo's types in one workspace # resolve to distinct Rust types that won't unify). -asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "661154b6d6e8b6b76102fb9d8cdcc440b298418f" } +asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "2270f1f8f98b64e2cc8aa5b7602a570aa0e95f1e" } [dev-dependencies] tokio = { version = "1", features = ["full", "test-util"] } diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 9421b43e..1fbe118d 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -37,7 +37,7 @@ use crate::query_plan::{ use crate::types_v2::AccuracyTarget; use planner_types::pre_asap::Source; -pub const PLANNER_REVISION: &str = "661154b6d6e8b6b76102fb9d8cdcc440b298418f"; +pub const PLANNER_REVISION: &str = "2270f1f8f98b64e2cc8aa5b7602a570aa0e95f1e"; pub const BACKEND_COMPAT: &str = "asap-query-backend.v1"; #[derive(Debug, Clone)] diff --git a/crates/asap_types/Cargo.toml b/crates/asap_types/Cargo.toml index ab10cf39..3823ab43 100644 --- a/crates/asap_types/Cargo.toml +++ b/crates/asap_types/Cargo.toml @@ -32,4 +32,4 @@ sha2 = "0.10" # exactly (`control_plane/Cargo.toml`) -- two different revs of the same # git dependency in one workspace resolve to two distinct Rust types that # won't unify. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "661154b6d6e8b6b76102fb9d8cdcc440b298418f" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "2270f1f8f98b64e2cc8aa5b7602a570aa0e95f1e" } diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index ef7ecfb0..e41ce23e 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -38,7 +38,7 @@ control_plane = { path = "../control_plane" } # reduction: Reduction, .. }`) are `pre_asap` types, in the same crate now # (not a separate `asap-ir` import). Query serving consumes the compiled # QueryPlan; these types are used at physical-plan compilation boundaries. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "661154b6d6e8b6b76102fb9d8cdcc440b298418f" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "2270f1f8f98b64e2cc8aa5b7602a570aa0e95f1e" } # Shared external (workspace) serde.workspace = true @@ -129,7 +129,7 @@ crc32fast = "1.4" # none of them. [dev-dependencies] -asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "661154b6d6e8b6b76102fb9d8cdcc440b298418f" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "2270f1f8f98b64e2cc8aa5b7602a570aa0e95f1e" } tempfile = "3.20.0" criterion = { version = "0.5", features = ["html_reports"] } tokio-tungstenite = "0.21" From 63d145fd86e7e120ea8f8ce9cb058d50fb89190a Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 17:00:19 -0600 Subject: [PATCH 26/30] Bind SummaryStore instances to the active catalog --- data_plane/src/drivers/query/servers/http.rs | 69 +++++- .../storage_engines/sketch_db/index/mod.rs | 197 +++++++++++++++++- .../src/storage_engines/sketch_db/sds.rs | 151 +++++++++++--- .../summary-catalog-sds-architecture.md | 14 +- 4 files changed, 394 insertions(+), 37 deletions(-) diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 5214ddce..4bb13027 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -464,6 +464,7 @@ impl HttpServer { "/api/v1/physical-plan/status", get(handle_physical_plan_status), ) + .route("/api/v1/summary-inventory", get(handle_summary_inventory)) // Phase α (MVP): control-plane-pushed `BackendStorageRouting` // table. POST replaces the current table atomically; GET // returns a JSON snapshot for operator diagnostics. @@ -567,6 +568,7 @@ impl HttpServer { "/api/v1/physical-plan/status", get(handle_physical_plan_status), ) + .route("/api/v1/summary-inventory", get(handle_summary_inventory)) // Phase α (MVP): control-plane-pushed `BackendStorageRouting` // table. POST replaces the current table atomically; GET // returns a JSON snapshot for operator diagnostics. @@ -5994,6 +5996,21 @@ async fn handle_activate_physical_plan( .into_response() } }; + let activated = active_handle.snapshot(); + if let Some(catalog) = activated.summary_catalog.as_ref() { + if let Err(error) = state + .sketch_index + .install_summary_catalog(Arc::clone(catalog)) + { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(serde_json::json!({ + "status": "error", "error": format!("SummaryCatalog install error: {error}") + })), + ) + .into_response(); + } + } if old.plan_id() != 0 { let draining_id = old.plan_id(); let draining_version = old.plan_version(); @@ -6008,7 +6025,7 @@ async fn handle_activate_physical_plan( } }); } - let snap = active_handle.snapshot().runtime_config.clone(); + let snap = activated.runtime_config.clone(); let retired = crate::storage_engines::sketch_db::lifecycle::reconcile_from_streaming_config( state.sketch_index.as_ref(), snap.as_ref(), @@ -6024,6 +6041,56 @@ async fn handle_activate_physical_plan( .into_response() } +async fn handle_summary_inventory(State(state): State) -> axum::response::Response { + use axum::response::IntoResponse; + let Some(active) = state + .active_physical_plan + .as_ref() + .map(|handle| handle.snapshot()) + else { + return ( + StatusCode::SERVICE_UNAVAILABLE, + axum::Json( + serde_json::json!({"status":"error","error":"physical plan is unavailable"}), + ), + ) + .into_response(); + }; + if active.summary_catalog.is_none() { + return ( + StatusCode::SERVICE_UNAVAILABLE, + axum::Json(serde_json::json!({"status":"error","error":"authoritative SummaryCatalog is unavailable"})), + ) + .into_response(); + } + let producers = active + .precompute_plan + .producers + .iter() + .map(|producer| { + ( + asap_types::sds::MaterializationId::from(producer.materialization), + producer.producer_id.clone(), + ) + }) + .collect(); + let reporter = std::env::var("HOSTNAME").unwrap_or_else(|_| "asapquery-backend".into()); + match state.sketch_index.observed_summary_inventory( + &reporter, + &reporter, + &producers, + active.plan_version(), + unix_time_ms() as i64, + ) { + Ok(inventory) => axum::Json(inventory).into_response(), + Err(error) => ( + StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(serde_json::json!({"status":"error","error":error})), + ) + .into_response(), + } +} + async fn handle_discard_physical_plan( State(state): State, axum::Json(request): axum::Json, diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index 43b7b72c..afd96cb0 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -23,6 +23,11 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::sync::{Arc, RwLock}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use asap_types::sds::{ + CatalogGeneration, HalfOpenTimeRange, InstanceCompleteness, InstanceLifecycle, + MaterializationId, ObservedSummaryInventory, SummaryInstance, SummaryInstanceId, + SummaryInstanceStatus, SummaryPlacement, SummaryStateReference, +}; use asap_types::PolicyFingerprint; use dashmap::DashMap; @@ -685,8 +690,17 @@ impl SketchStore { pub fn register(&self, meta: SketchInstanceMetadata) { let sid = meta.sid; let policy_fp = meta.policy_fp; - let metric_name = meta.metric_name.clone(); - let instance = self.descriptors.bind(meta); + // A non-legacy materialization must resolve through the installed + // authoritative catalog. Unknown identities fail closed and never + // become queryable SummaryStore entries. + let instance = match self.descriptors.bind(meta) { + Ok(instance) => instance, + Err(error) => { + tracing::warn!(sid, %error, "rejecting SummaryStore registration outside the active SummaryCatalog"); + return; + } + }; + let metric_name = instance.data_descriptor.metric_name.clone(); // Fixed lock order: instances → policy_to_sids → metric_to_sids. let mut instances = self.instances.write().unwrap(); let mut policy_idx = self.policy_to_sids.write().unwrap(); @@ -698,6 +712,17 @@ impl SketchStore { metric_idx.entry(metric_name).or_default().insert(sid); } + /// Install one authoritative catalog snapshot for future registrations. + /// Existing SIDs retain their generation's descriptor Arcs while draining. + pub fn install_summary_catalog( + &self, + catalog: Arc, + ) -> Result<(), String> { + self.descriptors + .install_catalog(Arc::clone(&catalog)) + .map_err(|error| error.to_string()) + } + /// Record that `sid` is a per-item (item_label-mode) frequency sketch /// keyed by the data-point attribute `label` (e.g. "service"). The /// query engine consults this to decide whether a keyed selector like @@ -775,6 +800,137 @@ impl SketchStore { ) } + /// Build observed SDS inventory from the actual registered SummaryStore + /// entries and their in-memory pane coverage. Payload bytes remain in the + /// store and are referenced by SID. + pub fn observed_summary_inventory( + &self, + reporter_id: &str, + storage_node_id: &str, + producers: &BTreeMap, + inventory_version: u64, + observed_at_ms: i64, + ) -> Result { + let catalog = self + .descriptors + .authoritative_catalog() + .ok_or_else(|| "no authoritative SummaryCatalog is installed".to_string())?; + let reference = catalog.reference().map_err(|error| error.to_string())?; + let generation = CatalogGeneration { + schema_version: reference.schema_version, + plan_id: reference.plan_id, + plan_version: reference.plan_version, + snapshot_digest: reference.snapshot_sha256, + }; + let instances = self.instances.read().unwrap(); + let mut reported = BTreeMap::new(); + for (sid, binding) in instances.iter() { + let materialization_id = MaterializationId::from(binding.metadata.policy_fp); + if binding.metadata.policy_fp.is_unset() + || !catalog.materializations.contains_key(&materialization_id) + { + continue; + } + let producer_id = producers + .get(&materialization_id) + .map(String::as_str) + .ok_or_else(|| { + format!( + "materialization {} has no producer in the active PrecomputePlan", + materialization_id.as_u64() + ) + })?; + let instance_id = SummaryInstanceId::new(format!( + "summary-instance:v1:{}:{}:{}", + generation.plan_version, + materialization_id.as_u64(), + sid + )) + .map_err(|error| error.to_string())?; + let coverage = self.in_memory_coverage_bounds(*sid); + let (time_range, status) = match coverage { + Some((start, end)) if start < end => ( + HalfOpenTimeRange { + start_ms: start as i64, + end_ms: end as i64, + }, + match binding.metadata.status() { + AggStatus::Active => SummaryInstanceStatus::Ready, + AggStatus::Retired | AggStatus::Expired => SummaryInstanceStatus::Retiring, + }, + ), + _ => ( + HalfOpenTimeRange { + start_ms: binding.metadata.first_seen_unix_ms, + end_ms: binding.metadata.first_seen_unix_ms.saturating_add(1), + }, + SummaryInstanceStatus::MissingPayload, + ), + }; + let instance = SummaryInstance { + instance_id: instance_id.clone(), + materialization_id, + summary_descriptor_id: binding.summary_descriptor.id().clone(), + data_descriptor_id: binding.data_descriptor.id().clone(), + time_range, + group_values: BTreeMap::new(), + catalog_generation: generation.clone(), + placement: SummaryPlacement { + producer_id: producer_id.into(), + storage_node_id: storage_node_id.into(), + }, + state_reference: SummaryStateReference { + store: "summary-store".into(), + key: format!("sid:{sid}"), + state_schema_version: binding.summary_descriptor.state_schema_version, + generation: generation.plan_version, + sequence: coverage.map_or(0, |(_, end)| end), + checksum: None, + }, + status, + completeness: InstanceCompleteness::Unknown, + lifecycle: InstanceLifecycle::Persistent, + observed_at_ms, + }; + instance.validate().map_err(|error| error.to_string())?; + reported.insert(instance_id, instance); + } + let inventory = ObservedSummaryInventory { + schema_version: 1, + reporter_id: reporter_id.into(), + inventory_version, + observed_at_ms, + instances: reported, + }; + inventory.validate().map_err(|error| error.to_string())?; + Ok(inventory) + } + + fn in_memory_coverage_bounds(&self, sid: u64) -> Option<(u64, u64)> { + let store = self.series.get(&sid)?.clone(); + let guard = store.read().ok()?; + let mut min_start = u64::MAX; + let mut max_end = 0; + let mut entries: Vec<(TimestampRange, LabelValuesId, &AggPayload)> = Vec::new(); + guard + .current_epoch + .range_query_into(0, u64::MAX, &mut entries); + for (range, _, _) in &entries { + min_start = min_start.min(range.0); + max_end = max_end.max(range.1); + } + entries.clear(); + for epoch in guard.sealed_epochs.values() { + epoch.range_query_into(0, u64::MAX, &mut entries); + for (range, _, _) in &entries { + min_start = min_start.min(range.0); + max_end = max_end.max(range.1); + } + entries.clear(); + } + (min_start != u64::MAX).then_some((min_start, max_end)) + } + /// Borrow-style metadata accessor (P2-2). Runs `f(&meta)` while /// holding the `instances` read lock and returns its result, WITHOUT /// cloning the (allocation-heavy: `String` + `BTreeSet` + @@ -2755,6 +2911,43 @@ mod tests { assert_eq!(store.descriptor_counts(), (1, 1)); } + #[test] + fn observed_inventory_uses_installed_catalog_and_real_store_entries() { + let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + serde_json::from_str(include_str!( + "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let plan = snapshot.compile().unwrap(); + let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); + let store = SketchStore::new(); + store + .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) + .unwrap(); + store.register(meta_with_policy(41, fingerprint)); + + let producers = BTreeMap::from([( + MaterializationId::from(fingerprint), + "producer-a".to_string(), + )]); + let inventory = store + .observed_summary_inventory("backend-a", "store-a", &producers, 1, 100) + .unwrap(); + inventory.validate().unwrap(); + let instance = inventory.instances.values().next().unwrap(); + assert_eq!(instance.materialization_id.fingerprint(), fingerprint); + assert_eq!(instance.status, SummaryInstanceStatus::MissingPayload); + let catalog_identity = &plan.summary_catalog.materializations[&instance.materialization_id]; + assert_eq!( + instance.summary_descriptor_id, + catalog_identity.summary_descriptor_id + ); + assert_eq!( + instance.data_descriptor_id, + catalog_identity.data_descriptor_id + ); + } + fn sample(b: u8) -> SketchSampleState { SketchSampleState { bytes: vec![b], diff --git a/data_plane/src/storage_engines/sketch_db/sds.rs b/data_plane/src/storage_engines/sketch_db/sds.rs index 5e761cfa..ecf8ef71 100644 --- a/data_plane/src/storage_engines/sketch_db/sds.rs +++ b/data_plane/src/storage_engines/sketch_db/sds.rs @@ -4,7 +4,7 @@ //! Descriptors are content-interned and shared by every materialized instance; //! pane-local state remains in `SketchStore`. -use std::collections::{BTreeSet, HashMap}; +use std::collections::HashMap; use std::sync::{Arc, RwLock, Weak}; use super::data::{AggKind, SketchConfig}; @@ -101,48 +101,87 @@ pub struct SummaryDescriptorRegistry { // registry must not turn retired materializations into a permanent leak. summaries: RwLock>>, data: RwLock>>, + authoritative_catalog: RwLock>>, } impl SummaryDescriptorRegistry { - pub fn bind(&self, metadata: SketchInstanceMetadata) -> SdsBinding { - let summary_id = summary_descriptor_id(&metadata.agg_kind); + pub fn install_catalog( + &self, + catalog: Arc, + ) -> Result<(), asap_types::summary_catalog::SummaryCatalogError> { + catalog.validate()?; + *self.authoritative_catalog.write().unwrap() = Some(catalog); + Ok(()) + } + + pub fn authoritative_catalog( + &self, + ) -> Option> { + self.authoritative_catalog.read().unwrap().clone() + } + + pub fn bind(&self, metadata: SketchInstanceMetadata) -> Result { + let authoritative = self.authoritative_catalog.read().unwrap().clone(); + let configured = if metadata.policy_fp.is_unset() { + None + } else if let Some(catalog) = authoritative.as_ref() { + let materialization = asap_types::sds::MaterializationId::from(metadata.policy_fp); + let identity = catalog + .materializations + .get(&materialization) + .ok_or_else(|| { + format!( + "materialization {} is absent from the installed SummaryCatalog", + materialization.as_u64() + ) + })?; + Some(( + catalog.summary_descriptors[&identity.summary_descriptor_id].clone(), + catalog.data_descriptors[&identity.data_descriptor_id].clone(), + )) + } else { + None + }; + let summary = configured + .as_ref() + .map(|(summary, _)| summary.clone()) + .unwrap_or_else(|| legacy_summary(&metadata.agg_kind)); + let summary_id = summary.id().clone(); let summary_descriptor = { let mut summaries = self.summaries.write().unwrap(); if let Some(existing) = summaries.get(&summary_id).and_then(Weak::upgrade) { existing } else { - let descriptor = Arc::new(legacy_summary(&metadata.agg_kind)); + let descriptor = Arc::new(summary); summaries.insert(summary_id, Arc::downgrade(&descriptor)); descriptor } }; - let filter = metadata.agg_kind.spatial_filter_canonical(); - let data_id = data_descriptor_id( - &metadata.metric_name, - filter, - metadata.group_by_keys.iter().map(String::as_str), - ); + let data_descriptor_value = configured.map(|(_, data)| data).unwrap_or_else(|| { + DataDescriptor::new( + metadata.metric_name.clone(), + metadata.agg_kind.spatial_filter_canonical(), + metadata.group_by_keys.iter().cloned(), + ) + }); + let data_id = data_descriptor_value.id().clone(); let data_descriptor = { - let mut data = self.data.write().unwrap(); - if let Some(existing) = data.get(&data_id).and_then(Weak::upgrade) { + let mut data_registry = self.data.write().unwrap(); + if let Some(existing) = data_registry.get(&data_id).and_then(Weak::upgrade) { existing } else { - let descriptor = Arc::new(DataDescriptor::new( - metadata.metric_name.clone(), - filter, - metadata.group_by_keys.iter().cloned(), - )); - data.insert(descriptor.id.clone(), Arc::downgrade(&descriptor)); + let descriptor = Arc::new(data_descriptor_value); + data_registry.insert(descriptor.id.clone(), Arc::downgrade(&descriptor)); descriptor } }; - SdsBinding { + Ok(SdsBinding { metadata: Arc::new(metadata), summary_descriptor, data_descriptor, - } + }) } pub fn summary_count(&self) -> usize { @@ -219,6 +258,7 @@ pub(crate) fn data_descriptor_id<'a>( #[cfg(test)] mod tests { use super::*; + use std::collections::BTreeSet; fn metadata( sid: u64, @@ -248,8 +288,12 @@ mod tests { #[test] fn equivalent_materializations_share_both_descriptors() { let registry = SummaryDescriptorRegistry::default(); - let a = registry.bind(metadata(1, "cpu", r#"{zone="a"}"#, AggregationType::Sum, 7)); - let b = registry.bind(metadata(2, "cpu", r#"{zone="a"}"#, AggregationType::Sum, 8)); + let a = registry + .bind(metadata(1, "cpu", r#"{zone="a"}"#, AggregationType::Sum, 7)) + .unwrap(); + let b = registry + .bind(metadata(2, "cpu", r#"{zone="a"}"#, AggregationType::Sum, 8)) + .unwrap(); assert!(Arc::ptr_eq(&a.summary_descriptor, &b.summary_descriptor)); assert!(Arc::ptr_eq(&a.data_descriptor, &b.data_descriptor)); @@ -259,8 +303,12 @@ mod tests { #[test] fn population_changes_only_the_data_descriptor() { let registry = SummaryDescriptorRegistry::default(); - let a = registry.bind(metadata(1, "cpu", r#"{zone="a"}"#, AggregationType::Sum, 7)); - let b = registry.bind(metadata(2, "cpu", r#"{zone="b"}"#, AggregationType::Sum, 7)); + let a = registry + .bind(metadata(1, "cpu", r#"{zone="a"}"#, AggregationType::Sum, 7)) + .unwrap(); + let b = registry + .bind(metadata(2, "cpu", r#"{zone="b"}"#, AggregationType::Sum, 7)) + .unwrap(); assert!(Arc::ptr_eq(&a.summary_descriptor, &b.summary_descriptor)); assert!(!Arc::ptr_eq(&a.data_descriptor, &b.data_descriptor)); @@ -270,8 +318,12 @@ mod tests { #[test] fn operator_changes_only_the_summary_descriptor() { let registry = SummaryDescriptorRegistry::default(); - let a = registry.bind(metadata(1, "cpu", "", AggregationType::Sum, 7)); - let b = registry.bind(metadata(2, "cpu", "", AggregationType::MinMax, 7)); + let a = registry + .bind(metadata(1, "cpu", "", AggregationType::Sum, 7)) + .unwrap(); + let b = registry + .bind(metadata(2, "cpu", "", AggregationType::MinMax, 7)) + .unwrap(); assert!(!Arc::ptr_eq(&a.summary_descriptor, &b.summary_descriptor)); assert!(Arc::ptr_eq(&a.data_descriptor, &b.data_descriptor)); @@ -281,11 +333,56 @@ mod tests { #[test] fn registry_does_not_retain_descriptors_after_bindings_are_dropped() { let registry = SummaryDescriptorRegistry::default(); - let binding = registry.bind(metadata(1, "cpu", "", AggregationType::Sum, 7)); + let binding = registry + .bind(metadata(1, "cpu", "", AggregationType::Sum, 7)) + .unwrap(); assert_eq!((registry.summary_count(), registry.data_count()), (1, 1)); drop(binding); registry.prune(); assert_eq!((registry.summary_count(), registry.data_count()), (0, 0)); } + + #[test] + fn configured_materializations_bind_only_to_authoritative_catalog_descriptors() { + use asap_types::summary_catalog::SummaryCatalog; + + let summary = SummaryDescriptor::new( + SummaryOperator::ExactAgg { + agg_type: AggregationType::Sum, + parameters_canonical: "authoritative=true".into(), + }, + FidelityGuarantee::Exact, + 1, + ) + .unwrap(); + let data = DataDescriptor::new("cpu", r#"{zone="a"}"#, ["job".into()]); + let catalog = SummaryCatalog::build( + 7, + 1, + [( + asap_types::PolicyFingerprint(7), + summary.clone(), + data.clone(), + )], + ) + .unwrap(); + let registry = SummaryDescriptorRegistry::default(); + registry.install_catalog(Arc::new(catalog)).unwrap(); + + let binding = registry + .bind(metadata( + 1, + "wrong-local-copy", + "", + AggregationType::MinMax, + 7, + )) + .unwrap(); + assert_eq!(binding.summary_descriptor.as_ref(), &summary); + assert_eq!(binding.data_descriptor.as_ref(), &data); + assert!(registry + .bind(metadata(2, "cpu", "", AggregationType::Sum, 8)) + .is_err()); + } } diff --git a/docs/design_docs/summary-catalog-sds-architecture.md b/docs/design_docs/summary-catalog-sds-architecture.md index eafee1a6..71446947 100644 --- a/docs/design_docs/summary-catalog-sds-architecture.md +++ b/docs/design_docs/summary-catalog-sds-architecture.md @@ -137,20 +137,20 @@ source, filter, grouping, fidelity, or state-schema definitions. | `QueryPlan` | Materialization references, readout, DAG composition and exact Prometheus boundaries | | SummaryStore (`SketchStore` today) | Instance state, concrete intervals/groups, completeness, lineage and rebuildable rollups | -`BackendPlan` is transitional. Its materialization registry moves into -`SummaryCatalog`; update/placement/lifecycle moves into `PrecomputePlan`; query -routing moves into `QueryPlan`; and the common deployment envelope becomes shared -plan metadata. After consumers install the same catalog snapshot and these plan -references are validated, the BackendPlan protobuf and endpoint are removed. +The former `BackendPlan` has been removed. `SummaryCatalog` owns materialization +metadata, `PrecomputePlan` owns update/placement/lifecycle, `QueryPlan` owns +readout and fallback routing, and the common deployment envelope carries their +shared plan identity. Consumers atomically install one catalog snapshot with +the plans that reference it. -The migration order is: +The implemented ownership split is: 1. Move the SDS catalog contract into `asap_types`. 2. Make the control plane own the authoritative `SummaryCatalog`. 3. Make `PrecomputePlan` reference catalog descriptors and own update, placement and lifecycle. 4. Make `QueryPlan::MaterializationBinding` reference catalog/materialization IDs directly. 5. Distribute the same catalog snapshot to Collector and backend. -6. Remove `BackendPlan`, its protobuf and install endpoint, and duplicate validation. +6. `BackendPlan`, its protobuf and install endpoint, and duplicate validation are removed. ## Implemented backend representation From c6033e29d1fe16c3e36736a1ab9160c0bd37978f Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 17:00:19 -0600 Subject: [PATCH 27/30] Remove stale BackendPlan runtime references --- .../src/physical/post_asap/cost_model.rs | 2 +- control_plane/src/types_v2.rs | 3 +- .../multiple_increase_accumulator.rs | 8 - .../asap_query_engine/catalog_resolver.rs | 22 +-- .../asap_query_engine/post_asap_planner.rs | 7 +- .../asap_query_engine/summary_executor.rs | 4 - data_plane/tests/backend_process_e2e.rs | 19 ++- ...e2e_controller_plans_and_backend_serves.rs | 2 +- docs/developer_docs/README.md | 2 +- .../query-engine/backend-plan-runtime.md | 140 ++---------------- 10 files changed, 38 insertions(+), 171 deletions(-) diff --git a/control_plane/src/physical/post_asap/cost_model.rs b/control_plane/src/physical/post_asap/cost_model.rs index f93690e4..8fc3d67b 100644 --- a/control_plane/src/physical/post_asap/cost_model.rs +++ b/control_plane/src/physical/post_asap/cost_model.rs @@ -831,7 +831,7 @@ impl CostModel for ForcedFamilyCostModel { /// won't match anything registered either way, so the outcome /// (`find_candidates` finds nothing) is unchanged. /// -/// **Fallback status (design-backend-plan-wire-format.md §5):** +/// **Legacy metadata fallback:** /// `post_asap_planner.rs` prefers reading planning's decision directly off an /// installed SummaryCatalog materializations (no reconstruction needed /// there — `Materialization.kind`/`.params` already ARE the pair diff --git a/control_plane/src/types_v2.rs b/control_plane/src/types_v2.rs index 0797c118..4ee501e9 100644 --- a/control_plane/src/types_v2.rs +++ b/control_plane/src/types_v2.rs @@ -47,8 +47,7 @@ pub enum QueryLanguage { /// Per-target accuracy SLA, in the typed form `design.md` §6 calls for. /// -/// Phase 1b (docs/migration-plan-backend-plan.md): re-exported from -/// `planner_types::types` (formerly `asap_ir::types` -- ASAPPlanner +/// Re-exported from `planner_types::types` (formerly `asap_ir::types` -- ASAPPlanner /// consolidated `asap-ir` into `asap-types`, see /// control_plane/docs/design-asapplanner-pin-migration.md) rather than /// defined locally -- `AggIntent`'s `accuracy` fields are typed against diff --git a/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs b/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs index 7e2268a7..d0bad06e 100644 --- a/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs @@ -16,14 +16,6 @@ pub struct MultipleIncreaseAccumulator { pub increases: HashMap, } -#[derive(Serialize, Deserialize)] -struct MeasurementData { - starting_measurement: f64, - starting_timestamp: i64, - last_seen_measurement: f64, - last_seen_timestamp: i64, -} - impl MultipleIncreaseAccumulator { pub fn new() -> Self { Self { diff --git a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs index bcbf71f2..4bd786a2 100644 --- a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs +++ b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs @@ -3,17 +3,15 @@ //! query execution checks only reachable IDs and their requested capabilities. use std::collections::{BTreeMap, BTreeSet}; -use asap_types::sds::{DataDescriptor, MaterializationId, SummaryDescriptor, SummaryOperator}; -use asap_types::summary_catalog::{MaterializationIdentity, SummaryCatalog}; +use asap_types::sds::{MaterializationId, SummaryDescriptor, SummaryOperator}; +use asap_types::summary_catalog::SummaryCatalog; use asap_types::AggregationType; use control_plane::query_plan::{ExactReadout, QueryPlanEntry, QueryPlanNode, QueryReadout}; use crate::query_engines::EngineError; pub(crate) struct ResolvedMaterialization<'a> { - pub identity: &'a MaterializationIdentity, pub summary: &'a SummaryDescriptor, - pub data: &'a DataDescriptor, } fn miss(reason: impl Into) -> EngineError { @@ -45,11 +43,7 @@ pub(crate) fn resolve( .map_err(|error| miss(format!("invalid summary descriptor: {error}")))?; data.validate() .map_err(|error| miss(format!("invalid data descriptor: {error}")))?; - Ok(ResolvedMaterialization { - identity, - summary, - data, - }) + Ok(ResolvedMaterialization { summary }) } impl ResolvedMaterialization<'_> { @@ -245,17 +239,9 @@ mod tests { let catalog = &bundle.summary_catalog; let id = *catalog.materializations.keys().next().unwrap(); let result = resolve(catalog, id).unwrap(); - assert!(std::ptr::eq( - result.identity, - &catalog.materializations[&id] - )); assert!(std::ptr::eq( result.summary, - &catalog.summary_descriptors[&result.identity.summary_descriptor_id] - )); - assert!(std::ptr::eq( - result.data, - &catalog.data_descriptors[&result.identity.data_descriptor_id] + &catalog.summary_descriptors[&catalog.materializations[&id].summary_descriptor_id] )); let mut broken = catalog.clone(); broken.data_descriptors.clear(); diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_planner.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_planner.rs index ce9871ca..25127a3f 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_planner.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_planner.rs @@ -255,7 +255,10 @@ fn query_expr_contains_rate(qe: &planner_types::pre_asap::QueryExpr) -> bool { QueryExpr::Aggregate { measures, child, .. } => { - measures.iter().any(|m| matches!(m, AggIntent::Rate)) || query_expr_contains_rate(child) + measures + .iter() + .any(|m| matches!(m, AggIntent::Rate | AggIntent::IRate)) + || query_expr_contains_rate(child) } QueryExpr::Filter { child, .. } | QueryExpr::Project { child, .. } @@ -425,7 +428,7 @@ fn ensure_warm_runtime_support( match &node.expr { SummaryExpr::SummaryAgg { child, family, .. } => { match family { - SummaryFamilyType::ExactAggregate(ExactKind::Rate, _) => { + SummaryFamilyType::ExactAggregate(ExactKind::Rate | ExactKind::IRate, _) => { return Err(LoweringSkip::RateShape) } SummaryFamilyType::ExactAggregate(ExactKind::Sum, _) diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index 033b9d66..a7d61b67 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -1255,10 +1255,6 @@ fn find_metric(node: &SummaryNode) -> Option { } } -pub(crate) fn find_metric_in_query_expr_from_summary(node: &SummaryNode) -> Option { - find_metric(node) -} - /// Walk a canonical `QueryExpr` down to its first `Scan { /// source: Source::TimeSeries { metric }, .. }` to recover the target /// metric name. Shared with `post_asap_planner.rs`'s observed-family lookup diff --git a/data_plane/tests/backend_process_e2e.rs b/data_plane/tests/backend_process_e2e.rs index 0aa88cc0..f4d12652 100644 --- a/data_plane/tests/backend_process_e2e.rs +++ b/data_plane/tests/backend_process_e2e.rs @@ -582,15 +582,20 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { let planned_window_secs = installed_aggregation["window_size"] .as_u64() .expect("controller emitted window size"); - let backend_plan: serde_json::Value = client - .get(format!("{data_base}/api/v1/backend-plan")) + let physical_plan_status: serde_json::Value = client + .get(format!("{data_base}/api/v1/physical-plan/status")) .send() .await - .expect("read installed backend plan") + .expect("read installed physical plan") .json() .await - .expect("decode installed backend plan"); - assert_eq!(backend_plan["materialization_count"], 1); + .expect("decode installed physical plan status"); + assert_eq!( + physical_plan_status["materializations"] + .as_array() + .map(Vec::len), + Some(1) + ); let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -714,7 +719,7 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { "inactive generation frame was accepted" ); let still_active: serde_json::Value = client - .get(format!("{data_base}/api/v1/backend-plan")) + .get(format!("{data_base}/api/v1/physical-plan/status")) .send() .await .unwrap() @@ -722,7 +727,7 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { .await .unwrap(); assert_eq!( - still_active, backend_plan, + still_active, physical_plan_status, "failed rollout changed active plan" ); let still_warm: serde_json::Value = client diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index b6bb21b2..8014e50a 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -108,7 +108,7 @@ fn build_workload( /// Run the controller's planning pipeline end-to-end on a `QueryWorkload` /// and return the `BackendStageConfig` the controller would emit from /// for it — the same object both `emit_backend_streaming_config_json` -/// (legacy JSON) and `backend_plan::from_stage_config` (`BackendPlan`) +/// (legacy JSON) and the catalog-backed physical-plan compiler /// consume. /// /// Mirrors the `handle_plan` flow's `StageConfig::Backend(mut be)` diff --git a/docs/developer_docs/README.md b/docs/developer_docs/README.md index bf6fe260..1f35b72c 100644 --- a/docs/developer_docs/README.md +++ b/docs/developer_docs/README.md @@ -11,7 +11,7 @@ cross-repository interfaces live in ## Query engine - [Routing, readout, and query-engine implementation](query-engine/query-engine.md) -- [BackendPlan runtime](query-engine/backend-plan-runtime.md) +- [Catalog-backed physical-plan runtime](query-engine/backend-plan-runtime.md) - [Extension points](query-engine/extension-points.md) ## Ingest engine diff --git a/docs/developer_docs/query-engine/backend-plan-runtime.md b/docs/developer_docs/query-engine/backend-plan-runtime.md index 9ec7bc9b..d167d97d 100644 --- a/docs/developer_docs/query-engine/backend-plan-runtime.md +++ b/docs/developer_docs/query-engine/backend-plan-runtime.md @@ -1,127 +1,13 @@ -# Developing BackendPlan installation - -> Interface status: target public API. Atomic snapshot storage exists; complete -> staging/lifecycle/cross-runtime activation remains partial. - -## 1. Code architecture - -```text -BackendPlan bytes - | - v -BackendPlanDecoder -> BackendPlanValidator -> BackendPlanRuntime - | - BackendPlanSnapshot - / \ - ingest query -``` - -The decoder owns wire decoding, the validator owns semantic/capability checks, -and the runtime owns staged/active snapshots. Ingest and query components only -consume immutable snapshots; they do not mutate plans. - -## 2. Public interfaces and definitions - -```rust -pub trait BackendPlanDecoder { - type Error; - fn decode(&self, bytes: &[u8]) -> Result; -} - -pub trait BackendPlanValidator { - type Error; - fn validate( - &self, - plan: &BackendPlan, - capabilities: &BackendCapabilities, - ) -> Result; -} -``` - -`ValidatedBackendPlan` must be constructible only through validation. It proves -schema/lifecycle ordering, unique identities, resolved routes, supported -families/parameters/windows/readouts, and compatible result guarantees. - -```rust -pub struct ValidatedBackendPlan { - pub plan: BackendPlan, - pub capability_hash: String, - pub validated_at: Timestamp, -} - -pub struct BackendCapabilities { - pub capability_hash: String, - pub ingest: Vec, - pub readouts: Vec, - pub storage: Vec, -} - -pub struct BackendPlanSnapshot { - pub plan: Arc, - pub status: PlanRuntimeStatus, - pub installed_at: Timestamp, -} - -pub enum PlanRuntimeStatus { - Staged, - Active, - Draining, - Expired, -} -``` - -Capability entry definitions: - -| Type | Definition | -| --- | --- | -| `IngestCapability` | Supported family, algorithm, parameters, encoding version, and full/delta semantics. | -| `ReadoutCapability` | Supported Planner readout/operator and guarantee kinds. | -| `StorageCapability` | Supported materialization representation, merge, window, retention, and durability behavior. | - -```rust -pub trait BackendPlanRuntime: Send + Sync { - type Error; - - fn stage(&self, plan: ValidatedBackendPlan) - -> Result; - - fn activate(&self, plan_id: &str, plan_version: u64) - -> Result; - - fn snapshot(&self) -> BackendPlanSnapshot; - - fn retire(&self, plan_id: &str, plan_version: u64) - -> Result; -} -``` - -Why these interfaces exist: decoding, validation, and activation have different -failure semantics. A decoded plan must never become queryable before validation -and matching collector evidence. - -## 3. Adding and verifying functionality - -### Add a BackendPlan field - -1. Add it to the public versioned wire/domain structure. -2. Define requiredness, identity impact, and compatibility behavior. -3. Validate it in `BackendPlanValidator`. -4. Expose it through immutable `BackendPlanSnapshot` to its consumer. -5. Verify missing/unknown/incompatible values fail before `stage`. - -### Add a runtime lifecycle state - -1. Extend `PlanRuntimeStatus` with allowed transitions. -2. Define whether ingest/query may use the state. -3. Return the effective state through `BackendApplicationReport`. -4. Verify invalid transitions do not change `snapshot()`. - -### Interpret and verify output - -- A `ValidatedBackendPlan` means the plan is deployable by this backend, not - active. -- A `Staged` report means resources/routes are prepared, not queryable. -- An `Active` report must match the requested plan/version and materializations. -- One request must observe one `BackendPlanSnapshot`, including during swap. -- Re-delivery of identical content is idempotent; conflicting content for the - same identity fails. +# Catalog-backed physical-plan installation + +`BackendPlan` and its standalone protobuf and installation endpoint have been +removed. The backend now stages and atomically activates one physical-plan +envelope containing the authoritative `SummaryCatalog` snapshot plus the +`PrecomputePlan`, optional `CollectorPlan`, `TransmissionPlan`, and `QueryPlan` +that reference catalog materialization IDs. + +The current contracts and ownership rules are documented in +[SummaryCatalog and SDS architecture](../../design_docs/summary-catalog-sds-architecture.md). +The HTTP lifecycle is exposed through `/api/v1/physical-plan`, +`/api/v1/physical-plan/activate`, `/api/v1/physical-plan/discard`, and +`/api/v1/physical-plan/status`. From 2e8caeb18a3bcbd5e6ff064cac3dae99284f73f7 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 17:11:31 -0600 Subject: [PATCH 28/30] docs: align architecture with SummaryCatalog plans --- docs/01-getting-started/architecture.md | 8 ++-- docs/03-how-to-guides/manual-e2e-tests.md | 2 +- docs/design_docs/asapplanner-integration.md | 2 +- .../design_docs/asapplanner-migration-plan.md | 6 ++- .../asapquery-compatibility-profile.md | 24 +++++------ .../summary-catalog-sds-architecture.md | 18 +++++--- .../control-plane/physical-compiler.md | 42 +++++++++---------- .../control-plane/plan-publication.md | 10 +++-- .../cross-component-adding-summary-family.md | 5 ++- .../ingest-engine/ingest-engine.md | 9 ++-- .../query-engine/extension-points.md | 2 +- .../query-engine/query-engine.md | 9 ++-- .../summary-store-engine/implementation.md | 3 +- docs/user_guide/asapquery-profile.md | 12 +++--- docs/user_guide/querying-asap.md | 2 +- docs/user_guide/running-and-verifying.md | 2 +- 16 files changed, 87 insertions(+), 69 deletions(-) diff --git a/docs/01-getting-started/architecture.md b/docs/01-getting-started/architecture.md index e82a3b93..5c54d48e 100644 --- a/docs/01-getting-started/architecture.md +++ b/docs/01-getting-started/architecture.md @@ -26,7 +26,7 @@ ASAPQuery control plane +---------------------------+ | | v v -CollectorPlan BackendPlan +CollectorPlan SummaryCatalog + execution plans | | v v ASAPCollector -- summary state --> ASAPQuery data plane @@ -49,7 +49,8 @@ See the [control-plane physical-planning design](https://github.com/ProjectASAP/ ASAPCollector applies CollectorPlan, maintains summaries from observed OTLP metrics, and transmits raw observations, full state, or deltas as directed. The -data plane accepts a payload only when it matches the active BackendPlan and +data plane accepts a payload only when it matches the active SummaryCatalog, +PrecomputePlan, and TransmissionPlan, then stores it under the declared materialization and logical window. The distributed Collector profile does not treat legacy Telegraf, OTAP, Kafka, @@ -61,7 +62,8 @@ profile's primary OTel path. ## Query path A PromQL request enters through a protocol server and adapter. The data plane -uses BackendPlan to locate a compatible readout and checks plan identity, +uses QueryPlan bindings resolved through SummaryCatalog to locate a compatible +readout and checks plan identity, coverage, freshness, and state compatibility before execution. If the selected logical plan requires exact execution, the request is sent to the configured exact backend. diff --git a/docs/03-how-to-guides/manual-e2e-tests.md b/docs/03-how-to-guides/manual-e2e-tests.md index d3f2c804..822bb572 100644 --- a/docs/03-how-to-guides/manual-e2e-tests.md +++ b/docs/03-how-to-guides/manual-e2e-tests.md @@ -25,7 +25,7 @@ change process environment or still use fixed loopback ports. It covers: durably writes its TSDB block, and returns the exact chunk over Thanos StoreAPI, plus its compaction, recovery, and shipping suites; and 6. the final production-process path: typed physical-plan compilation, - BackendPlan/precompute installation, collector capability and applied ACK + catalog and execution-plan installation, collector capability and applied ACK over OpAMP, modified-OTLP ingest, SketchStore policy routing, and PromQL query readout of that same plan. diff --git a/docs/design_docs/asapplanner-integration.md b/docs/design_docs/asapplanner-integration.md index 2f4973b3..42690f8f 100644 --- a/docs/design_docs/asapplanner-integration.md +++ b/docs/design_docs/asapplanner-integration.md @@ -135,7 +135,7 @@ One selected DAG can produce several execution projections: - PrecomputePlan: how the selected state is built and maintained. - TransmissionPlan and optional CollectorPlan: how distributed producers implement and deliver that state. -- BackendPlan: compatible storage, catalog and routing bindings. +- SummaryCatalog: canonical summary/data descriptors and stable materialization identities. - QueryPlan: executable reads, merges, readouts and remaining exact operations. These projections may expand one semantic node into several physical tasks. diff --git a/docs/design_docs/asapplanner-migration-plan.md b/docs/design_docs/asapplanner-migration-plan.md index 0cef67a2..babe652c 100644 --- a/docs/design_docs/asapplanner-migration-plan.md +++ b/docs/design_docs/asapplanner-migration-plan.md @@ -99,7 +99,11 @@ Implementation scope: 4. Reject conflicting contracts before any plan is published. Do not pick whichever query happened to be visited first. -Acceptance: both query roots exist; one BackendPlan state and one PrecomputePlan +> Historical note: this acceptance text predates the SummaryCatalog migration; +> the former BackendPlan state is now represented by a catalog materialization +> and its execution-plan references. + +Acceptance: both query roots exist; one catalog materialization and one PrecomputePlan state exist; each Collector has one producer declaration; both bindings point to that state. A different implementation/layout for the same fingerprint fails compilation. Distinct source/window/parameters must remain distinct. diff --git a/docs/design_docs/asapquery-compatibility-profile.md b/docs/design_docs/asapquery-compatibility-profile.md index c3938e8c..5e912e75 100644 --- a/docs/design_docs/asapquery-compatibility-profile.md +++ b/docs/design_docs/asapquery-compatibility-profile.md @@ -98,15 +98,15 @@ configured QueryWorkload + DataWorkload backend-only physical compiler │ │ │ ▼ ▼ ▼ - PrecomputePlan BackendPlan QueryPlan DAG + PrecomputePlan SummaryCatalog QueryPlan DAG │ │ │ ▼ ▼ ▼ precompute catalog SID-bound executor ``` The plan has no collector projection. One compile produces an atomic -`PhysicalPlan` containing sibling backend-local `PrecomputePlan`, `BackendPlan`, -and `QueryPlan` sections from the same selected Post-ASAP candidate. +`PhysicalPlan` containing one `SummaryCatalog` plus sibling backend-local +`PrecomputePlan` and `QueryPlan` sections from the same selected Post-ASAP candidate. `PrecomputePlan` directly is the runtime precompute contract; there is no second streaming-config semantic model or lossy conversion step. All sections share plan and materialization identities. @@ -139,8 +139,8 @@ For the first MVP, operators provide immutable `QueryWorkload` and `DataWorkload`; 3. enumerates only backend-local implementations for Planner candidates; 4. returns implementation-cost evidence needed for selection; -5. compiles the selected candidate into matching PrecomputePlan, BackendPlan, - and QueryPlan views; and +5. compiles the selected candidate into a matching SummaryCatalog, PrecomputePlan, + and QueryPlan; and 6. stages and atomically activates those views. It does not enumerate SDK or Collector placements in this profile. A Planner @@ -247,7 +247,7 @@ require byte-for-byte reproduction of the incoming HTTP request. Routing has two successful outcomes: 1. execute the active QueryPlan DAG, whose materialization bindings were - resolved from BackendPlan, when compatible summary state has + resolved through SummaryCatalog, when compatible summary state has complete and fresh coverage; or 2. forward a semantically equivalent request to the configured Prometheus endpoint. @@ -274,7 +274,7 @@ start backend -> verify Prometheus fallback health -> load configured QueryWorkload and DataWorkload snapshots -> run Planner candidate search and selection - -> compile one PhysicalPlan (PrecomputePlan + BackendPlan + QueryPlan) + -> compile one PhysicalPlan (SummaryCatalog + PrecomputePlan + QueryPlan) -> atomically install precompute + catalog + inactive routes under one version -> accept Remote Write and forward every query to Prometheus -> enter Materializing state @@ -299,7 +299,7 @@ same atomic cutover and warmup rules. | --- | --- | --- | | Ingest source | Prometheus Remote Write raw samples | Collector materializations over modified OTLP and other explicit profiles | | Summary construction | Backend-local only | Collector or backend placement | -| Physical outputs | PrecomputePlan + BackendPlan + QueryPlan | CollectorPlan/PrecomputePlan/TransmissionPlan/BackendPlan/QueryPlan views as applicable | +| Physical outputs | SummaryCatalog + PrecomputePlan + QueryPlan | SummaryCatalog/CollectorPlan/PrecomputePlan/TransmissionPlan/QueryPlan views as applicable | | Query protocol | Prometheus HTTP / PromQL | Additional protocols may be supported | | Exact fallback | Upstream Prometheus | Prometheus or another compiled storage/query route | | Storage required for MVP | In-process warm summary state | Warm, durable, archive, and remote tiers | @@ -319,7 +319,7 @@ stronger activation and retry contracts. | Area | Implemented contract | Executable evidence | | --- | --- | --- | | Startup/profile | Collector-free startup, excluded-component validation, fallback health gate | `data_plane` profile tests and production-process E2E | -| Physical planning | Canonical workload snapshot to one atomic PrecomputePlan/BackendPlan/QueryPlan bundle | `compatibility_demo_snapshot_compiles_the_complete_query_matrix` | +| Physical planning | Canonical workload snapshot to one atomic SummaryCatalog/PrecomputePlan/QueryPlan bundle | `compatibility_demo_snapshot_compiles_the_complete_query_matrix` | | Remote Write | Strict v1 decoding, stale handling, limits, retry-safe deduplication and backpressure | receiver unit tests plus process E2E replay/corrupt-batch assertions | | Precompute/store | Raw samples use the planned family; first catch-up batches close all complete windows; sketch and exact payloads share canonical SID semantics | worker/store tests and four-family process matrix | | Query execution | QueryPlan-only serving-time lookup, node-level materialization binding, generic DAG traversal, exact fallback | instant/range process matrix and fallback request capture | @@ -354,11 +354,11 @@ overloaded requests cannot leave untracked mutations while returning success. Load the configured query and data workload snapshots, call the pinned ASAPPlanner, enumerate backend-local implementations, and compile one atomic -PhysicalPlan with PrecomputePlan, BackendPlan, and QueryPlan. Do not create or +PhysicalPlan with SummaryCatalog, PrecomputePlan, and QueryPlan. Do not create or wait for CollectorPlan. Acceptance: captured Planner input, selected Post-ASAP candidate, -PrecomputePlan, BackendPlan, and QueryPlan are deterministic golden artifacts +SummaryCatalog, PrecomputePlan, and QueryPlan are deterministic golden artifacts with matching plan/materialization/window/family/parameter identities. ### Phase D: atomic activation and warmup @@ -422,7 +422,7 @@ only proves that the endpoint returns HTTP success does not satisfy this matrix. The first query extension should be a SQL endpoint because ASAPQuery exposed SQL as an additional query surface. It must translate SQL into canonical Planner -query semantics and reuse the same BackendPlan readiness, summary store, and +query semantics and reuse the same catalog-backed QueryPlan readiness, summary store, and exact-fallback rules; it must not introduce a second planner or separately configured materializations. Its supported SQL subset and fallback target need their own versioned compatibility contract and conformance cases. diff --git a/docs/design_docs/summary-catalog-sds-architecture.md b/docs/design_docs/summary-catalog-sds-architecture.md index 71446947..53d5080f 100644 --- a/docs/design_docs/summary-catalog-sds-architecture.md +++ b/docs/design_docs/summary-catalog-sds-architecture.md @@ -125,8 +125,13 @@ or create persistent desired state by itself. pane instances, completeness and lineage ``` -All four execution plans reference catalog IDs instead of copying operator, -source, filter, grouping, fidelity, or state-schema definitions. +All four execution plans carry catalog references and use catalog materialization +IDs for cross-plan identity. During the compatibility migration, producer and +precompute DTOs still repeat fields needed by existing runtimes, including +operator parameters, source/filter/grouping, window, and state schema. Install +validation requires those fields to agree exactly with the catalog; they are not +independent semantic definitions. New interfaces should resolve them from the +catalog, allowing the copied fields to be removed as consumers migrate. | Component | Responsibility | | --- | --- | @@ -184,10 +189,11 @@ An ingest record is never an SDS instance. Raw samples can be transient inputs t the precompute engine, but the backend does not retain them as a second exact query store. Exact residual subtrees run in Prometheus. -The target model requires these invariants. The current implementation enforces -descriptor binding and non-overlapping pane selection; the remaining structured -schema and instance contracts must be completed before claiming full SDS -conformance: +The SDS metadata and inventory types represent the following invariants. The +current runtime enforces descriptor binding and non-overlapping pane selection. +Full runtime conformance still requires applying and durably persisting every +reconciliation action, including recovery, promotion, lease expiry, retirement, +and garbage collection: 1. An instance references exactly one immutable Summary Descriptor and one immutable Data Descriptor. diff --git a/docs/developer_docs/control-plane/physical-compiler.md b/docs/developer_docs/control-plane/physical-compiler.md index cc1e851b..97fdd621 100644 --- a/docs/developer_docs/control-plane/physical-compiler.md +++ b/docs/developer_docs/control-plane/physical-compiler.md @@ -8,8 +8,9 @@ The compiler consumes ASAPPlanner types pinned to the revision exposed as `physical::compiler::PLANNER_REVISION`, selects from Planner's legal candidate space with backend-owned cost and evidence -inputs, and emits one `PhysicalPlan`. The plan contains CollectorPlan, -PrecomputePlan, BackendPlan, and QueryPlan projections compiled from the same decision +inputs, and emits one `PhysicalPlan`. The plan contains one SummaryCatalog plus +CollectorPlan, PrecomputePlan, TransmissionPlan, and QueryPlan projections +compiled from the same decision for every target collector. Legacy `StageAllocator`/`ThreeStageEmitter` paths remain for older publication flows; they are not a second semantic planner. @@ -47,8 +48,8 @@ PhysicalCompiler -------> PhysicalPlan - **Physical compiler** enumerates concrete window/pane/state-layout, placement, transport, and runtime implementations without changing the Planner-owned abstract framework. -- **PhysicalPlan** is the only output passed to publication. Its CollectorPlan, - PrecomputePlan, BackendPlan, and QueryPlan projections are created together and share identities. +- **PhysicalPlan** is the only output passed to publication. Its SummaryCatalog, + CollectorPlan, PrecomputePlan, TransmissionPlan, and QueryPlan are created together and share identities. Logical query parsing, summary alternatives, guarantees, and candidate search remain public ASAPPlanner interfaces. Runtime publication is documented in @@ -128,10 +129,10 @@ pub struct DeploymentEnvironment { pub struct PhysicalPlan { pub envelope: PlanEnvelope, + pub summary_catalog: SummaryCatalog, pub collector_plans: Vec, // complete per-target projections pub precompute_plan: PrecomputePlan, // backend streaming materializations pub transmission_plan: TransmissionPlan, // producer/frame wire contract - pub backend_plan: BackendPlan, pub query_plan: QueryPlan, // node-ID physical serving DAG } ``` @@ -145,7 +146,7 @@ Supporting public types: | `CollectorPlan` | Serializable execution projection consumed by ASAPCollector. | | `PrecomputePlan` | Authoritative materialization, ingest, state-schema, and producer contract consumed directly by the backend runtime. | | `TransmissionPlan` | Exact per-producer mode, encoding, cadence, destination, checkpoint policy, and frame identity contract. | -| `BackendPlan` | Versioned public data-plane materialization/routing contract defined in this repository. | +| `SummaryCatalog` | Authoritative immutable descriptor and materialization-identity snapshot. | | `QueryPlan` | Canonical-query keyed executable DAG with exact materialization bindings and fallback policy. | Current MVP limits are explicit: time-series sources and supported summary @@ -177,7 +178,7 @@ Output definitions: | `collector_plans` | One plan per targeted collector, following ASAPCollector's public CollectorPlan schema. | | `precompute_plan` | Materializations plus `/v1/metrics` ingest semantics, typed state schemas/encodings, and allowed producers; it is installed directly and contains no query-string jobs. | | `transmission_plan` | One rule per producer/materialization/schema binding plus the mandatory frame identity and sequencing scope. | -| `backend_plan` | Matching data-plane materialization and routing contract. | +| `summary_catalog` | Canonical descriptors and stable materialization identities shared by every execution plan. | | `query_plan` | Canonical query identity, explicit fallback policy, node-ID DAG, and exact per-node materialization bindings. | ### QueryPlan execution boundary @@ -254,20 +255,17 @@ The materialization `id` above identifies the maintained summary definition. It is not the SID. During collection and ingest, each concrete canonical label set under that definition resolves to its own SID. -### What the compiler puts in BackendPlan +### Catalog and query-serving output -The matching BackendPlan is the consumer and query-serving projection of the -same decision: +SummaryCatalog contains the canonical descriptor bindings. QueryPlan contains +the consumer and query-serving projection of the same decision: -| BackendPlan section | Required content | Consuming ASAPQuery-backend component | +| Output section | Required content | Consuming ASAPQuery-backend component | | --- | --- | --- | -| envelope | The same plan ID/version, activation/expiry, compatibility identity and Planner revision | Backend plan manager | -| materialization descriptor | Same materialization fingerprint, canonical metric, retained grouping keys, capability, aggregation kind/parameters, accuracy, policy fingerprint and lifecycle | SID resolver and instance-metadata registry | -| producers/ingest | Expected Collector/producer IDs, input payload kind, state schema, full/delta lineage and optional backend-precompute placement | OTLP ingest engine and precompute engine | -| window/coverage | Pane/window compatibility, lateness, expected coverage and merge rules | Ingest validation and summary store | -| storage | Warm/archive/remote route, retention, retirement and expiry | Summary store engine and archive adapter | -| query routes | Query IDs or canonical capability match, materialization reference, readout/operator, grouping/window composition and remaining backend operators | Query classifier, router and readout engine | -| guarantee/fallback | Selected guarantee and explicit exact/archive fallback behavior | Query engine and response metadata | +| SummaryCatalog | Canonical Summary and Data Descriptors plus stable materialization bindings | Catalog resolver and SummaryStore registry | +| PrecomputePlan | Backend ingest/update placement, schemas, producers, windows, retention, and lifecycle | Ingest and precompute engines | +| TransmissionPlan | Producer encoding, cadence, identity, sequencing, delta, and checkpoint rules | Collector transport and ingest validation | +| QueryPlan | Materialization IDs, readout/operator DAG, grouping/window composition, guarantees, and explicit exact fallback | Query router and DAG executor | The backend declaration does not contain a pre-enumerated SID for every label value. It installs immutable metadata for the materialization; the ingest/SID @@ -284,7 +282,7 @@ CollectorPlan -> full/delta encoder + SID dictionary -> exporter -BackendPlan +SummaryCatalog + QueryPlan -> plan installer -> ingest validator / optional backend precompute -> SID resolver + instance metadata @@ -294,8 +292,8 @@ BackendPlan Cross-plan validation proves that every Collector-produced materialization has one compatible backend ingest/storage declaration and that every planned query -route references a declared materialization. A Backend-only precompute has a -BackendPlan producer but no Collector materialization; a raw pass-through has +route references a declared materialization. A backend-only precompute has a +PrecomputePlan producer but no Collector materialization; a raw pass-through has matching raw transmission and ingest/archive declarations. ### Summary frame identity @@ -337,7 +335,7 @@ identity. 2. Teach `PhysicalCompiler::compile` how selected operators can be placed on it. 3. Reject plans requiring an unavailable stage/capability. 4. Verify the output contains one complete CollectorPlan for every producer and - one BackendPlan referencing all produced materializations. + one SummaryCatalog and matching QueryPlan referencing all produced materializations. Interpretation: a successful bundle means both runtime views are complete and cross-consistent; it does not mean they have been activated. diff --git a/docs/developer_docs/control-plane/plan-publication.md b/docs/developer_docs/control-plane/plan-publication.md index a5cda3fe..203f29c1 100644 --- a/docs/developer_docs/control-plane/plan-publication.md +++ b/docs/developer_docs/control-plane/plan-publication.md @@ -2,7 +2,8 @@ This page describes the implemented MVP contract. The control plane compiles ASAPPlanner's selected post-ASAP IR once and projects that decision into one -typed `BackendPlan` plus one target-specific `CollectorPlan` per Collector. +authoritative `SummaryCatalog`, matching Precompute, Transmission, and Query +plans, plus one target-specific `CollectorPlan` per Collector. ## API @@ -45,7 +46,7 @@ validate request and compile one bundle preflight every Collector capability | v -POST one atomic PrecomputePlan + TransmissionPlan + BackendPlan + QueryPlan bundle (stage) +POST one atomic SummaryCatalog + PrecomputePlan + TransmissionPlan + QueryPlan bundle (stage) | v publish target-specific CollectorPlans over OpAMP @@ -89,8 +90,9 @@ another Collector's plan. ## Backend wire contract -The matching BackendPlan protobuf is embedded with the typed PrecomputePlan, -TransmissionPlan, and QueryPlan in `POST /api/v1/physical-plan`. The backend validates all shared +The authoritative SummaryCatalog snapshot is embedded with the typed PrecomputePlan, +TransmissionPlan, CollectorPlans, and QueryPlan in `POST /api/v1/physical-plan`. +The backend validates all shared identities, fingerprints, schemas, parameters, producers, and lifecycle fields before returning `staged`; `POST /api/v1/physical-plan/activate` performs the single immutable-snapshot swap. diff --git a/docs/developer_docs/cross-component-adding-summary-family.md b/docs/developer_docs/cross-component-adding-summary-family.md index 2924a018..0f1e1b68 100644 --- a/docs/developer_docs/cross-component-adding-summary-family.md +++ b/docs/developer_docs/cross-component-adding-summary-family.md @@ -12,7 +12,7 @@ ASAPPlanner public summary/readout types v PhysicalCompiler capability match / \ -CollectorPlan BackendPlan +CollectorPlan SummaryCatalog + QueryPlan | | ASAPCollector SummaryDecoder update + encode -> SummaryStore -> SummaryReader @@ -67,7 +67,8 @@ pub trait SummaryReader { &self, request: &QueryRequest, route: &SummaryRoute, - plan: &BackendPlanSnapshot, + catalog: &SummaryCatalog, + plan: &QueryPlan, ) -> Result; } ``` diff --git a/docs/developer_docs/ingest-engine/ingest-engine.md b/docs/developer_docs/ingest-engine/ingest-engine.md index 8cf87ac5..ba64b903 100644 --- a/docs/developer_docs/ingest-engine/ingest-engine.md +++ b/docs/developer_docs/ingest-engine/ingest-engine.md @@ -1,7 +1,7 @@ # Developing OTLP summary ingestion > Interface status: target public API. OTLP decoding, SID resolution, and -> summary handling exist; complete BackendPlan-gated validation is partial. +> summary handling exist; complete catalog and execution-plan validation is partial. ## 1. Code architecture @@ -87,7 +87,9 @@ pub trait SummaryValidator { fn validate( &self, received: ReceivedSummary, - plan: &BackendPlanSnapshot, + catalog: &SummaryCatalog, + precompute: &PrecomputePlan, + transmission: &TransmissionPlan, series: ResolvedSeries, ) -> Result; } @@ -148,7 +150,8 @@ queryable state, and `IngestResult` gives the MVP harness unambiguous evidence. 1. Extend public `SummaryEnvelope` encoding/version definitions. 2. Implement `SummaryDecoder` without applying state. -3. Add compatibility validation against BackendPlan/capabilities. +3. Add compatibility validation against SummaryCatalog, PrecomputePlan, + TransmissionPlan, and producer capabilities. 4. Implement full/delta application through `SummaryStateApplier`. 5. Verify corrupt bytes return an error and do not change storage. diff --git a/docs/developer_docs/query-engine/extension-points.md b/docs/developer_docs/query-engine/extension-points.md index 98e066dc..4a97486e 100644 --- a/docs/developer_docs/query-engine/extension-points.md +++ b/docs/developer_docs/query-engine/extension-points.md @@ -69,7 +69,7 @@ query language/expression, logical evaluation range, requested accuracy, result labels/timestamps/type, source, guarantee, and coverage. Why these interfaces exist: transport/protocol extensions cannot bypass -BackendPlan routing or directly access summary storage, and fallback backends +catalog-backed QueryPlan routing or directly access summary storage, and fallback backends cannot silently reinterpret a request. ## 3. Adding and verifying functionality diff --git a/docs/developer_docs/query-engine/query-engine.md b/docs/developer_docs/query-engine/query-engine.md index 88b59801..bde6f15d 100644 --- a/docs/developer_docs/query-engine/query-engine.md +++ b/docs/developer_docs/query-engine/query-engine.md @@ -1,14 +1,14 @@ # Developing query routing and summary readout > Interface status: target public API. Summary execution and fallback exist; -> BackendPlan is still replacing legacy routing and local query-shape logic. +> catalog-backed QueryPlan routing is replacing legacy local query-shape logic. ## 1. Code architecture ```text protocol request -> QueryAdapter -> QueryService | - BackendPlanSnapshot + SummaryCatalog + QueryPlan / \ SummaryReader ExactQueryClient \ / @@ -50,7 +50,8 @@ pub trait SummaryReader { &self, request: &QueryRequest, route: &SummaryRoute, - plan: &BackendPlanSnapshot, + catalog: &SummaryCatalog, + plan: &QueryPlan, ) -> Result; } @@ -111,7 +112,7 @@ coverage and guarantee. ### Add a readout/operator 1. Add the logical semantics and guarantee to ASAPPlanner. -2. Extend public backend readout capability and BackendPlan route types. +2. Extend public backend readout capability and QueryPlan node and catalog-binding types. 3. Implement it through `SummaryReader`; do not parse and choose a family again. 4. Return labels/timestamps/result type through `PrometheusResult`. 5. Compare with the exact backend over identical series and logical range. diff --git a/docs/developer_docs/summary-store-engine/implementation.md b/docs/developer_docs/summary-store-engine/implementation.md index 42e4de10..ef0206ce 100644 --- a/docs/developer_docs/summary-store-engine/implementation.md +++ b/docs/developer_docs/summary-store-engine/implementation.md @@ -132,7 +132,8 @@ pub trait SummaryStore: Send + Sync { } pub struct CoverageRequest { - pub plan: BackendPlanSnapshot, + pub catalog: SummaryCatalog, + pub query_plan: QueryPlan, pub materialization_ids: Vec, pub range: EvaluationRange, } diff --git a/docs/user_guide/asapquery-profile.md b/docs/user_guide/asapquery-profile.md index ccc3c472..54d65ad6 100644 --- a/docs/user_guide/asapquery-profile.md +++ b/docs/user_guide/asapquery-profile.md @@ -77,7 +77,7 @@ Receiver evidence is exported from `/metrics` as canonical ASAPPlanner types; `implementation` contains only backend-owned cost/window evidence, and `environment.target` must be `backend_local_remote_write`. Startup validates the workload, calls the pinned -ASAPPlanner, compiles matching PrecomputePlan/BackendPlan/QueryPlan views, and +ASAPPlanner, compiles matching SummaryCatalog/PrecomputePlan/QueryPlan views, and installs the resulting immutable snapshot before accepting traffic. It does not create or wait for a CollectorPlan. @@ -87,16 +87,16 @@ A complete canonical input is checked in at For reproducibility/debugging, `--physical-plan` remains an alternative to `--planning-snapshot`; exactly one is required. `physical-plan.json` is the JSON representation accepted by -`POST /api/v1/physical-plan`: a `PrecomputePlan`, `TransmissionPlan`, encoded -`BackendPlan`, and authoritative `QueryPlan` DAG with one shared plan identity -and version. For this profile, the precompute ingest contract must be +`POST /api/v1/physical-plan`: a `SummaryCatalog`, `PrecomputePlan`, +`TransmissionPlan`, and authoritative `QueryPlan` DAG with one shared plan +identity and version. For this profile, the precompute ingest contract must be `prometheus_remote_write_v1` at `/api/v1/write`; raw writes are rejected if a different plan is active. Startup validates all fingerprints, schemas, query bindings, lifecycle fields, and transmission rules before constructing the single active snapshot. Runtime replacement uses `POST /api/v1/physical-plan` to stage the complete successor and `POST /api/v1/physical-plan/activate` for -the atomic cutover. The partial `/streaming-config` and `/backend-plan` install -handles are not attached in this profile. +the atomic cutover. The old partial `/backend-plan` endpoint has been removed; +the `/streaming-config` compatibility endpoint is not attached in this profile. Query serving uses only the installed `QueryPlan` DAG. A query absent from that DAG is a capability miss and goes to the exact Prometheus fallback; the backend diff --git a/docs/user_guide/querying-asap.md b/docs/user_guide/querying-asap.md index 79b0c0c0..db05eb9c 100644 --- a/docs/user_guide/querying-asap.md +++ b/docs/user_guide/querying-asap.md @@ -41,7 +41,7 @@ sum by (region) (rate(http_requests_total[5m])) These examples do not promise that every deployment accelerates each query. The submitted workload, selected Planner result, available summaries, and -active BackendPlan determine the route. +active SummaryCatalog and QueryPlan together determine the route. ## Accuracy diff --git a/docs/user_guide/running-and-verifying.md b/docs/user_guide/running-and-verifying.md index b4ec0ec6..6cd175c1 100644 --- a/docs/user_guide/running-and-verifying.md +++ b/docs/user_guide/running-and-verifying.md @@ -85,7 +85,7 @@ or exact fallback are available. | Build cannot resolve a path dependency | Verify the three sibling repository names and locations | | Data plane exits immediately | Supply `--streaming-config` and check YAML errors and port conflicts | | OTLP connection is refused | Start with `--enable-otel-ingest` and verify ports 4317/4318 | -| Query returns no compatible summary | Inspect streaming config, BackendPlan, storage routing, metric labels, and window | +| Query returns no compatible summary | Inspect the SummaryCatalog, PrecomputePlan, QueryPlan, storage routing, metric labels, and window | | Unsupported query fails | Configure an exact backend or use a supported planned query | | Result comes from an unexpected tier | Inspect `infos` provenance and the installed storage routing table | | Results are stale | Check collector export, OTLP ingest, active-window state, and source timestamps | From 4aee19a846a8b96dadcabe8822100dde90c8c641 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 17:11:49 -0600 Subject: [PATCH 29/30] Reject unscoped state under an active catalog --- data_plane/src/storage_engines/sketch_db/sds.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/data_plane/src/storage_engines/sketch_db/sds.rs b/data_plane/src/storage_engines/sketch_db/sds.rs index ecf8ef71..bf883ee0 100644 --- a/data_plane/src/storage_engines/sketch_db/sds.rs +++ b/data_plane/src/storage_engines/sketch_db/sds.rs @@ -122,9 +122,12 @@ impl SummaryDescriptorRegistry { pub fn bind(&self, metadata: SketchInstanceMetadata) -> Result { let authoritative = self.authoritative_catalog.read().unwrap().clone(); - let configured = if metadata.policy_fp.is_unset() { - None - } else if let Some(catalog) = authoritative.as_ref() { + let configured = if let Some(catalog) = authoritative.as_ref() { + if metadata.policy_fp.is_unset() { + return Err( + "materialization identity is required by the installed SummaryCatalog".into(), + ); + } let materialization = asap_types::sds::MaterializationId::from(metadata.policy_fp); let identity = catalog .materializations @@ -384,5 +387,8 @@ mod tests { assert!(registry .bind(metadata(2, "cpu", "", AggregationType::Sum, 8)) .is_err()); + assert!(registry + .bind(metadata(3, "cpu", "", AggregationType::Sum, 0)) + .is_err()); } } From 50f75f84a73126db664eb511a34ebd8ddf9bf736 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 17:13:33 -0600 Subject: [PATCH 30/30] docs: replace final stale BackendPlan endpoint --- docs/design_docs/asapplanner-integration.md | 2 +- docs/user_guide/running-and-verifying.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/design_docs/asapplanner-integration.md b/docs/design_docs/asapplanner-integration.md index 42690f8f..3895dc82 100644 --- a/docs/design_docs/asapplanner-integration.md +++ b/docs/design_docs/asapplanner-integration.md @@ -302,6 +302,6 @@ backend's scoped consolidation through steps 1–6. - [Physical compiler](../developer_docs/control-plane/physical-compiler.md) - [Plan publication](../developer_docs/control-plane/plan-publication.md) -- [Backend plan runtime](../developer_docs/query-engine/backend-plan-runtime.md) +- [Catalog-backed physical-plan runtime](../developer_docs/query-engine/backend-plan-runtime.md) - [ASAPQuery compatibility profile](asapquery-compatibility-profile.md) - [Runtime accuracy feedback](../developer_docs/control-plane/runtime-accuracy-feedback.md) diff --git a/docs/user_guide/running-and-verifying.md b/docs/user_guide/running-and-verifying.md index 6cd175c1..12710eaa 100644 --- a/docs/user_guide/running-and-verifying.md +++ b/docs/user_guide/running-and-verifying.md @@ -63,7 +63,8 @@ Inspect installed runtime state when diagnosing plan or routing problems: ```bash curl -fsS http://localhost:8088/api/v1/streaming-config -curl -fsS http://localhost:8088/api/v1/backend-plan +curl -fsS http://localhost:8088/api/v1/physical-plan/status +curl -fsS http://localhost:8088/api/v1/summary-inventory curl -fsS http://localhost:8088/api/v1/storage_routing ```